promptfoo 0.10.0 → 0.12.0

package/src/index.ts

@@ -1,10 +1,11 @@
-import { evaluate as doEvaluate } from './evaluator';
-import { loadApiProviders } from './providers';
 import assertions from './assertions';
 import providers from './providers';
+import telemetry from './telemetry';
+import { evaluate as doEvaluate } from './evaluator';
+import { loadApiProviders } from './providers';
+import { readTests } from './util';
 
 import type { EvaluateOptions, TestSuite, TestSuiteConfig } from './types';
-import { readTests } from './util';
 
 export * from './types';
 
@@ -24,7 +25,9 @@ async function evaluate(testSuite: EvaluateTestSuite, options: EvaluateOptions =
       display: promptContent,
     })),
   };
-  return doEvaluate(constructedTestSuite, options);
+  const ret = await doEvaluate(constructedTestSuite, options);
+  await telemetry.send();
+  return ret;
 }
 
 module.exports = {

package/src/main.ts

@@ -6,6 +6,7 @@ import Table from 'cli-table3';
 import chalk from 'chalk';
 import { Command } from 'commander';
 
+import telemetry from './telemetry';
 import logger, { setLogLevel } from './logger';
 import { loadApiProvider, loadApiProviders } from './providers';
 import { evaluate } from './evaluator';
@@ -17,9 +18,11 @@ import {
   writeLatestResults,
   writeOutput,
 } from './util';
+import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
+import { disableCache } from './cache';
 import { getDirectory } from './esm';
 import { init } from './web/server';
-import { disableCache } from './cache';
+import { checkForUpdates } from './updates';
 
 import type {
   CommandLineOptions,
@@ -28,7 +31,6 @@ import type {
   TestSuite,
   UnifiedConfig,
 } from './types';
-import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
 
 function createDummyFiles(directory: string | null) {
   if (directory) {
@@ -60,6 +62,8 @@ function createDummyFiles(directory: string | null) {
 }
 
 async function main() {
+  await checkForUpdates();
+
   const pwd = process.cwd();
   const potentialPaths = [
     pathJoin(pwd, 'promptfooconfig.js'),
@@ -68,7 +72,7 @@ async function main() {
   ];
   let config: Partial<UnifiedConfig> = {};
   for (const path of potentialPaths) {
-    const maybeConfig = maybeReadConfig(path);
+    const maybeConfig = await maybeReadConfig(path);
     if (maybeConfig) {
       config = maybeConfig;
       break;
@@ -95,15 +99,23 @@ async function main() {
   program
     .command('init [directory]')
     .description('Initialize project with dummy files')
-    .action((directory: string | null) => {
+    .action(async (directory: string | null) => {
       createDummyFiles(directory);
+      telemetry.record('command_used', {
+        name: 'init',
+      });
+      await telemetry.send();
     });
 
   program
     .command('view')
     .description('Start browser ui')
     .option('-p, --port <number>', 'Port number', '15500')
-    .action((cmdObj: { port: number } & Command) => {
+    .action(async (cmdObj: { port: number } & Command) => {
+      telemetry.record('command_used', {
+        name: 'view',
+      });
+      await telemetry.send();
       init(cmdObj.port);
     });
 
@@ -154,8 +166,16 @@ async function main() {
       'This suffix is append to every prompt',
       config.defaultTest?.options?.suffix,
     )
-    .option('--no-write', 'Do not write results to promptfoo directory')
-    .option('--no-cache', 'Do not read or write results to disk cache')
+    .option(
+      '--no-write',
+      'Do not write results to promptfoo directory',
+      config?.commandLineOptions?.write,
+    )
+    .option(
+      '--no-cache',
+      'Do not read or write results to disk cache',
+      config?.commandLineOptions?.cache,
+    )
     .option('--grader', 'Model that will grade outputs', config?.commandLineOptions?.grader)
     .option('--verbose', 'Show debug logs', config?.commandLineOptions?.verbose)
     .option('--view [port]', 'View in browser ui')
@@ -172,7 +192,7 @@ async function main() {
       const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
       const configPath = cmdObj.config;
       if (configPath) {
-        config = readConfig(configPath);
+        config = await readConfig(configPath);
       } else {
         config = {
           prompts: cmdObj.prompts || config.prompts,
@@ -256,8 +276,9 @@ async function main() {
           },
         });
         // Skip first row (header) and add the rest. Color PASS/FAIL
-        for (const row of summary.table.body) {
+        for (const row of summary.table.body.slice(0, 25)) {
           table.push([
+            ...row.vars,
             ...row.outputs.map((col) => {
               const tableCellMaxLength = parseInt(cmdObj.tableCellMaxLength || '', 10);
               if (!isNaN(tableCellMaxLength) && col.length > tableCellMaxLength) {
@@ -275,18 +296,29 @@ async function main() {
               }
               return col;
             }),
-            ...row.vars,
           ]);
         }
 
         logger.info('\n' + table.toString());
+        if (summary.table.body.length > 25) {
+          const rowsLeft = summary.table.body.length - 25;
+          logger.info(`... ${rowsLeft} more row${rowsLeft === 1 ? '' : 's'} not shown ...\n`);
+        }
       }
+
+      const border = '='.repeat(process.stdout.columns - 10);
+      logger.info(border);
       if (cmdObj.view || !cmdObj.write) {
-        logger.info('Evaluation complete');
+        logger.info(`${chalk.green('✔')} Evaluation complete`);
       } else {
-        writeLatestResults(summary);
-        logger.info(`Evaluation complete. To use web viewer, run ${chalk.green('promptfoo view')}`);
+        writeLatestResults(summary, config);
+        logger.info(
+          `${chalk.green('✔')} Evaluation complete. To use web viewer, run ${chalk.green(
+            'promptfoo view',
+          )}`,
+        );
       }
+      logger.info(border);
       logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
       logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
       logger.info(
@@ -294,6 +326,11 @@ async function main() {
       );
       logger.info('Done.');
 
+      telemetry.record('command_used', {
+        name: 'eval',
+      });
+      await telemetry.send();
+
       if (cmdObj.view) {
         init(parseInt(cmdObj.view, 10) || 15500);
       }

package/src/telemetry.ts

@@ -0,0 +1,57 @@
+import packageJson from '../package.json';
+import { fetchWithTimeout } from './util';
+
+type TelemetryEvent = {
+  event: string;
+  packageVersion: string;
+  properties: Record<string, string | number>;
+};
+
+type TelemetryEventTypes = 'eval_ran' | 'assertion_used' | 'command_used';
+
+const TELEMETRY_ENDPOINT = 'https://api.promptfoo.dev/telemetry';
+
+const TELEMETRY_TIMEOUT_MS = 1000;
+
+export class Telemetry {
+  private events: TelemetryEvent[] = [];
+
+  get disabled() {
+    return process.env.PROMPTFOO_DISABLE_TELEMETRY === '1';
+  }
+
+  record(eventName: TelemetryEventTypes, properties: Record<string, string | number>): void {
+    if (!this.disabled) {
+      this.events.push({
+        event: eventName,
+        packageVersion: packageJson.version,
+        properties,
+      });
+    }
+  }
+
+  async send(): Promise<void> {
+    if (!this.disabled && this.events.length > 0) {
+      try {
+        const response = await fetchWithTimeout(
+          TELEMETRY_ENDPOINT,
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+            },
+            body: JSON.stringify(this.events),
+          },
+          TELEMETRY_TIMEOUT_MS,
+        );
+
+        if (response.ok) {
+          this.events = [];
+        }
+      } catch (err) {}
+    }
+  }
+}
+
+const telemetry = new Telemetry();
+export default telemetry;

package/src/types.ts

@@ -113,13 +113,34 @@ export interface GradingResult {
   tokensUsed?: TokenUsage;
 }
 
+type BaseAssertionTypes =
+  | 'equals'
+  | 'contains'
+  | 'icontains'
+  | 'contains-all'
+  | 'contains-any'
+  | 'regex'
+  | 'is-json'
+  | 'contains-json'
+  | 'javascript'
+  | 'similar'
+  | 'llm-rubric'
+  | 'webhook'
+  | 'rouge-n'
+  | 'rouge-s'
+  | 'rouge-l';
+
+type NotPrefixed<T extends string> = `not-${T}`;
+
+export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
+
 // TODO(ian): maybe Assertion should support {type: config} to make the yaml cleaner
 export interface Assertion {
   // Type of assertion
-  type: 'equals' | 'is-json' | 'contains-json' | 'javascript' | 'similar' | 'llm-rubric';
+  type: AssertionType;
 
   // The expected value, if applicable
-  value?: string;
+  value?: string | string[];
 
   // The threshold value, only applicable for similarity (cosine distance)
   threshold?: number;

package/src/updates.ts

@@ -0,0 +1,37 @@
+import chalk from 'chalk';
+import semverGt from 'semver/functions/gt';
+
+import logger from './logger';
+import { fetchWithTimeout } from './util';
+import packageJson from '../package.json';
+
+const VERSION = packageJson.version;
+
+export async function getLatestVersion(packageName: string) {
+  const response = await fetchWithTimeout(`https://registry.npmjs.org/${packageName}`, {}, 1000);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch package information for ${packageName}`);
+  }
+  const data = await response.json();
+  return data['dist-tags'].latest;
+}
+
+export async function checkForUpdates(): Promise<boolean> {
+  const latestVersion = await getLatestVersion('promptfoo');
+  if (semverGt(latestVersion, VERSION)) {
+    const border = '='.repeat(process.stdout.columns - 10);
+    logger.info(
+      `\n${border}
+${chalk.yellow('⚠️')} The current version of promptfoo ${chalk.yellow(
+        VERSION,
+      )} is lower than the latest available version ${chalk.green(latestVersion)}.
+
+Please run ${chalk.green('npx promptfoo@latest')} or ${chalk.green(
+        'npm install -g promptfoo@latest',
+      )} to update.
+${border}\n`,
+    );
+    return true;
+  }
+  return false;
+}

package/src/util.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs';
 import * as path from 'node:path';
 import * as os from 'node:os';
 
+import $RefParser from '@apidevtools/json-schema-ref-parser';
 import fetch from 'node-fetch';
 import yaml from 'js-yaml';
 import nunjucks from 'nunjucks';
@@ -15,7 +16,15 @@ import { getDirectory } from './esm';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
 
-import type { Assertion, CsvRow, EvaluateSummary, UnifiedConfig, TestCase, Prompt } from './types';
+import type {
+  Assertion,
+  CsvRow,
+  EvaluateSummary,
+  UnifiedConfig,
+  TestCase,
+  Prompt,
+  TestSuite,
+} from './types';
 import { assertionFromString } from './assertions';
 
 const PROMPT_DELIMITER = '---';
@@ -28,14 +37,14 @@ function parseJson(json: string): any | undefined {
   }
 }
 
-export function maybeReadConfig(configPath: string): UnifiedConfig | undefined {
+export async function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined> {
   if (!fs.existsSync(configPath)) {
     return undefined;
   }
   return readConfig(configPath);
 }
 
-export function readConfig(configPath: string): UnifiedConfig {
+export async function readConfig(configPath: string): Promise<UnifiedConfig> {
   const ext = path.parse(configPath).ext;
   switch (ext) {
     case '.json':
@@ -45,7 +54,9 @@ export function readConfig(configPath: string): UnifiedConfig {
       return require(configPath) as UnifiedConfig;
     case '.yaml':
     case '.yml':
-      return yaml.load(fs.readFileSync(configPath, 'utf-8')) as UnifiedConfig;
+      let ret = yaml.load(fs.readFileSync(configPath, 'utf-8')) as UnifiedConfig;
+      ret = (await $RefParser.dereference(ret)) as UnifiedConfig;
+      return ret;
     default:
       throw new Error(`Unsupported configuration file format: ${ext}`);
   }
@@ -235,11 +246,22 @@ export function getLatestResultsPath(): string {
   return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
 }
 
-export function writeLatestResults(results: EvaluateSummary) {
+export function writeLatestResults(results: EvaluateSummary, config: Partial<UnifiedConfig>) {
   const latestResultsPath = getLatestResultsPath();
   try {
     fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
-    fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
+    fs.writeFileSync(
+      latestResultsPath,
+      JSON.stringify(
+        {
+          version: 1,
+          config,
+          results,
+        },
+        null,
+        2,
+      ),
+    );
   } catch (err) {
     logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
   }

package/src/web/client/package-lock.json

@@ -13,6 +13,7 @@
         "@mui/icons-material": "^5.11.16",
         "@mui/material": "^5.13.0",
         "@tanstack/react-table": "^8.9.1",
+        "js-yaml": "^4.1.0",
         "react": "^18.2.0",
         "react-dnd": "^16.0.1",
         "react-dnd-html5-backend": "^16.0.1",
@@ -1652,8 +1653,7 @@
     "node_modules/argparse": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
     },
     "node_modules/array-union": {
       "version": "2.1.0",
@@ -2512,7 +2512,6 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
       "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
-      "dev": true,
       "dependencies": {
         "argparse": "^2.0.1"
       },
@@ -4436,8 +4435,7 @@
     "argparse": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz",
-      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==",
-      "dev": true
+      "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="
     },
     "array-union": {
       "version": "2.1.0",
@@ -5102,7 +5100,6 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz",
       "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==",
-      "dev": true,
       "requires": {
         "argparse": "^2.0.1"
       }

package/src/web/client/package.json

@@ -15,6 +15,7 @@
     "@mui/icons-material": "^5.11.16",
     "@mui/material": "^5.13.0",
     "@tanstack/react-table": "^8.9.1",
+    "js-yaml": "^4.1.0",
     "react": "^18.2.0",
     "react-dnd": "^16.0.1",
     "react-dnd-html5-backend": "^16.0.1",

package/src/web/client/src/App.tsx

@@ -11,8 +11,9 @@ import { useStore } from './store.js';
 import './App.css';
 
 function App() {
-  const { table, setTable } = useStore();
+  const { table, setTable, setConfig } = useStore();
   const [loaded, setLoaded] = React.useState<boolean>(false);
+  const loadedFromApi = React.useRef(false);
 
   const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
   const [darkMode, setDarkMode] = React.useState(prefersDarkMode);
@@ -37,24 +38,43 @@ function App() {
   };
 
   React.useEffect(() => {
-    //const socket = SocketIOClient(`http://${window.location.host}`);
+    const fetchEvalData = async (id: string) => {
+      if (loadedFromApi.current) {
+        return;
+      }
+      loadedFromApi.current = true;
+      const response = await fetch(`https://api.promptfoo.dev/eval/${id}`);
+      const body = await response.json();
+      setTable(body.data.results.table);
+      setConfig(body.data.config);
+      setLoaded(true);
+    };
+
     const socket = SocketIOClient(`http://localhost:15500`);
 
-    socket.on('init', (data) => {
-      console.log('Initialized socket connection');
-      setLoaded(true);
-      setTable(data.table);
-    });
+    const pathMatch = window.location.pathname.match(/\/eval\/([\w:-]+)/);
+    if (pathMatch) {
+      const id = pathMatch[1];
+      fetchEvalData(id);
+    } else {
+      socket.on('init', (data) => {
+        console.log('Initialized socket connection', data);
+        setLoaded(true);
+        setTable(data.results.table);
+        setConfig(data.config);
+      });
 
-    socket.on('update', (data) => {
-      console.log('Received data update');
-      setTable(data.table);
-    });
+      socket.on('update', (data) => {
+        console.log('Received data update', data);
+        setTable(data.results.table);
+        setConfig(data.config);
+      });
+    }
 
     return () => {
       socket.disconnect();
     };
-  }, [loaded, setTable]);
+  }, [setTable, setConfig]);
 
   return (
     <ThemeProvider theme={theme}>

package/src/web/client/src/ConfigModal.tsx

@@ -0,0 +1,81 @@
+import React from 'react';
+import Dialog from '@mui/material/Dialog';
+import DialogTitle from '@mui/material/DialogTitle';
+import DialogContent from '@mui/material/DialogContent';
+import DialogActions from '@mui/material/DialogActions';
+import Button from '@mui/material/Button';
+import Typography from '@mui/material/Typography';
+import { useStore } from './store';
+import { IconButton, Box } from '@mui/material';
+import { FileCopy, Check } from '@mui/icons-material';
+
+interface ConfigModalProps {
+  open: boolean;
+  onClose: () => void;
+}
+
+export default function ConfigModal({ open, onClose }: ConfigModalProps) {
+  const { config } = useStore();
+  const textareaRef = React.useRef<HTMLTextAreaElement>(null);
+  const [copied, setCopied] = React.useState(false);
+  const [yamlConfig, setYamlConfig] = React.useState('');
+
+  React.useEffect(() => {
+    if (open) {
+      (async () => {
+        const { default: yaml } = await import('js-yaml');
+        setYamlConfig(yaml.dump(config));
+      })();
+    }
+  }, [open, config]);
+
+  const handleCopyClick = () => {
+    if (textareaRef.current) {
+      textareaRef.current.select();
+      document.execCommand('copy');
+      setCopied(true);
+    }
+  };
+
+  const handleClose = () => {
+    setCopied(false);
+    onClose();
+  };
+
+  return (
+    <Dialog
+      open={open}
+      onClose={handleClose}
+      aria-labelledby="config-dialog-title"
+      maxWidth="md"
+      fullWidth
+    >
+      <DialogTitle id="config-dialog-title">
+        <Box display="flex" justifyContent="space-between" alignItems="center">
+          <Typography variant="h6">Config</Typography>
+          <IconButton onClick={handleCopyClick}>{copied ? <Check /> : <FileCopy />}</IconButton>
+        </Box>
+      </DialogTitle>
+      <DialogContent>
+        <Typography variant="body1" component="div">
+          <textarea
+            ref={textareaRef}
+            readOnly
+            value={yamlConfig}
+            style={{
+              width: '100%',
+              minHeight: '400px',
+              fontFamily: 'monospace',
+              border: '1px solid #ccc',
+            }}
+          />
+        </Typography>
+      </DialogContent>
+      <DialogActions>
+        <Button onClick={handleClose} color="primary">
+          Close
+        </Button>
+      </DialogActions>
+    </Dialog>
+  );
+}

package/src/web/client/src/ResultsTable.css

@@ -40,7 +40,6 @@ td,
 .td {
   position: relative;
   box-shadow: inset 0 0 0 1px var(--border-color);
-  word-break: break-all;
   vertical-align: top;
 
   padding: 1.5rem;
@@ -50,11 +49,11 @@ th.variable,
 .th.variable,
 td.variable,
 .td.variable {
-  background-color: #f8fbff;
+  background-color: var(--variable-background-color);
 }
 
 tr.header {
-  background-color: #fffdf7;
+  background-color: var(--header-background-color);
 }
 
 th,
@@ -62,7 +61,7 @@ th,
   padding: 1rem;
   position: relative;
   text-align: center;
-  font-weight: semi-bold;
+  vertical-align: bottom;
 }
 
 tr .cell {
@@ -72,7 +71,7 @@ tr .cell-rating {
   visibility: hidden;
   position: absolute;
   bottom: 1.25rem;
-  right: -1rem;
+  right: 0;
   line-height: 0;
   font-size: 1.75rem;
 }
@@ -83,7 +82,10 @@ tr:hover .cell-rating {
 
 tr .cell-rating .rating {
   cursor: pointer;
-  margin-right: 1rem;
+}
+
+tr .cell-rating .rating:first-child {
+  margin-right: 0.5rem;
 }
 
 th .smalltext {
@@ -97,6 +99,16 @@ th:hover .smalltext {
   visibility: visible;
 }
 
+th .summary {
+  font-weight: normal;
+  font-size: 0.8rem;
+  padding: 0.25rem;
+}
+
+th .summary.highlight {
+  background-color: var(--success-background-color);
+}
+
 td,
 .td {
 }