promptfoo 0.2.2 → 0.4.0

package/src/main.ts

@@ -9,8 +9,9 @@ import { Command } from 'commander';
 import logger, { setLogLevel } from './logger.js';
 import { loadApiProvider } from './providers.js';
 import { evaluate } from './evaluator.js';
-import { readPrompts, readVars, writeOutput } from './util.js';
+import { readPrompts, readVars, writeLatestResults, writeOutput } from './util.js';
 import { getDirectory } from './esm.js';
+import { init } from './web/server.js';
 
 import type { CommandLineOptions, EvaluateOptions, VarMapping } from './types.js';
 
@@ -37,7 +38,7 @@ These prompts are nunjucks templates, so you can use logic like this:
   prompts: ['prompts.txt'],
   providers: ['openai:gpt-3.5-turbo'],
   vars: 'vars.csv',
-  maxConcurrency: 3,
+  maxConcurrency: 4,
 };`;
   const readme = `To get started, set your OPENAI_API_KEY environment variable. Then run:
 \`\`\`
@@ -96,6 +97,14 @@ async function main() {
       createDummyFiles(directory);
     });
 
+  program
+    .command('view')
+    .description('Start browser ui')
+    .option('-p, --port <number>', 'Port number', '15500')
+    .action((cmdObj: { port: number } & Command) => {
+      init(cmdObj.port);
+    });
+
   program
     .command('eval')
     .description('Evaluate prompts')
@@ -129,7 +138,10 @@ async function main() {
       'Maximum number of concurrent API calls',
       String(defaultConfig.maxConcurrency),
     )
+    .option('--no-write', 'Do not write results to promptfoo directory')
+    .option('--grader', 'Model that will grade outputs', defaultConfig.grader)
     .option('--verbose', 'Show debug logs', defaultConfig.verbose)
+    .option('--view', 'View in browser ui')
     .action(async (cmdObj: CommandLineOptions & Command) => {
       if (cmdObj.verbose) {
         setLogLevel('debug');
@@ -170,6 +182,12 @@ async function main() {
         ...config,
       };
 
+      if (cmdObj.grader) {
+        options.grading = {
+          provider: await loadApiProvider(cmdObj.grader),
+        };
+      }
+
       const summary = await evaluate(options);
 
       if (cmdObj.output) {
@@ -178,31 +196,55 @@ async function main() {
       } else {
         // Output table by default
         const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
-        const head = summary.table[0];
+        const head = summary.table.head;
+        const headLength = head.prompts.length + head.vars.length;
         const table = new Table({
-          head,
-          colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
+          head: [...head.prompts, ...head.vars],
+          colWidths: Array(headLength).fill(Math.floor(maxWidth / headLength)),
           wordWrap: true,
           wrapOnWordBoundary: true,
           style: {
             head: ['blue', 'bold'],
           },
         });
-        // Skip first row (header) and add the rest. Color the first column green if it's a success, red if it's a failure.
-        for (const row of summary.table.slice(1)) {
-          const color = row[0] === 'PASS' ? 'green' : row[0].startsWith('FAIL') ? 'red' : undefined;
-          table.push(row.map((col, i) => (i === 0 && color ? chalk[color](col) : col)));
+        // Skip first row (header) and add the rest. Color PASS/FAIL
+        for (const row of summary.table.body) {
+          table.push([
+            ...row.outputs.map((col) => {
+              if (col.startsWith('[PASS]')) {
+                // color '[PASS]' green
+                return chalk.green.bold(col.slice(0, 6)) + col.slice(6);
+              } else if (col.startsWith('[FAIL]')) {
+                // color everything red up until '---'
+                return col
+                  .split('---')
+                  .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
+                  .join('---');
+              }
+              return col;
+            }),
+            ...row.vars,
+          ]);
         }
 
         logger.info('\n' + table.toString());
       }
-      logger.info('Evaluation complete');
+      if (cmdObj.noWrite) {
+        logger.info('Evaluation complete');
+      } else {
+        writeLatestResults(summary);
+        logger.info(`Evaluation complete. To use web viewer, run ${chalk.green('promptfoo view')}`);
+      }
       logger.info(chalk.green.bold(`Successes: ${summary.stats.successes}`));
       logger.info(chalk.red.bold(`Failures: ${summary.stats.failures}`));
       logger.info(
         `Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`,
       );
       logger.info('Done.');
+
+      if (cmdObj.view) {
+        init(15500);
+      }
     });
 
   program.parse(process.argv);

package/src/prompts.ts

@@ -0,0 +1,20 @@
+export const DEFAULT_GRADING_PROMPT = JSON.stringify([
+  {
+    role: 'system',
+    content: `You are grading content according to a user-specified rubric. If the statement in the rubric is true, then the content passes the test. You respond with a JSON object with this structure: {pass: boolean; reason: string;}.
+
+Examples:
+
+Content: Hello world
+Rubric: Contains a greeting
+{"pass": true, "reason": "the content contains the word 'world'"}
+
+Content: Avast ye swabs, repel the invaders!
+Rubric: Does not speak like a pirate
+{"pass": false, "reason": "'avast ye' is a common pirate term"}`,
+  },
+  {
+    role: 'user',
+    content: 'Content: {{ content }}\nRubric: {{ rubric }}',
+  },
+]);

package/src/providers.ts

@@ -1,12 +1,19 @@
-import fetch from 'node-fetch';
 import path from 'node:path';
 
 import { ApiProvider, ProviderResponse } from './types.js';
+import { fetchWithTimeout } from './util.js';
 import logger from './logger.js';
 
+const DEFAULT_OPENAI_HOST = 'api.openai.com';
+
+const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
+  ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
+  : 60_000;
+
 export class OpenAiGenericProvider implements ApiProvider {
   modelName: string;
   apiKey: string;
+  apiHost: string;
 
   constructor(modelName: string, apiKey?: string) {
     this.modelName = modelName;
@@ -18,6 +25,8 @@ export class OpenAiGenericProvider implements ApiProvider {
       );
     }
     this.apiKey = key;
+
+    this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
   }
 
   id(): string {
@@ -56,18 +65,23 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
       prompt,
       max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
       temperature: process.env.OPENAI_TEMPERATURE || 0,
+      stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
     let response, data;
     try {
-      response = await fetch('https://api.openai.com/v1/completions', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${this.apiKey}`,
+      response = await fetchWithTimeout(
+        `https://${this.apiHost}/v1/completions`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          body: JSON.stringify(body),
         },
-        body: JSON.stringify(body),
-      });
+        REQUEST_TIMEOUT_MS,
+      );
 
       data = (await response.json()) as unknown as any;
     } catch (err) {
@@ -129,14 +143,18 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
 
     let response, data;
     try {
-      response = await fetch('https://api.openai.com/v1/chat/completions', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-          Authorization: `Bearer ${this.apiKey}`,
+      response = await fetchWithTimeout(
+        `https://${this.apiHost}/v1/chat/completions`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          body: JSON.stringify(body),
         },
-        body: JSON.stringify(body),
-      });
+        REQUEST_TIMEOUT_MS,
+      );
       data = (await response.json()) as unknown as any;
     } catch (err) {
       return {

package/src/tableOutput.html

@@ -20,17 +20,14 @@
       th,
       td {
         padding: 5px;
+        min-width: 200px;
       }
-      /* If data-content is exactly "PASS", set font color to green */
-      tr > td[data-content='PASS']:first-child {
+
+      tr > td[data-content^='[PASS]'] {
         color: green;
-        font-weight: bold;
       }
-
-      /* If data-content starts with "FAIL", set font color to red */
-      tr > td[data-content^='FAIL']:first-child {
-        color: red;
-        font-weight: bold;
+      tr > td[data-content^='[FAIL]'] {
+        color: #ad0000;
       }
     </style>
   </head>

package/src/types.ts

@@ -6,6 +6,9 @@ export interface CommandLineOptions {
   config?: string;
   verbose?: boolean;
   maxConcurrency?: number;
+  grader?: string;
+  view?: boolean;
+  noWrite?: boolean;
 }
 
 export interface ApiProvider {
@@ -13,7 +16,7 @@ export interface ApiProvider {
   callApi: (prompt: string) => Promise<ProviderResponse>;
 }
 
-interface TokenUsage {
+export interface TokenUsage {
   total: number;
   prompt: number;
   completion: number;
@@ -31,6 +34,11 @@ export interface CsvRow {
 
 export type VarMapping = Record<string, string>;
 
+export interface GradingConfig {
+  prompt?: string;
+  provider: ApiProvider;
+}
+
 export interface EvaluateOptions {
   providers: ApiProvider[];
   prompts: string[];
@@ -38,6 +46,8 @@ export interface EvaluateOptions {
 
   maxConcurrency?: number;
   showProgressBar?: boolean;
+
+  grading?: GradingConfig;
 }
 
 export interface Prompt {
@@ -53,12 +63,27 @@ export interface EvaluateResult {
   success: boolean;
 }
 
+export interface EvaluateTable {
+  head: {
+    prompts: string[];
+    vars: string[];
+  };
+
+  body: {
+    outputs: string[];
+    vars: string[];
+  }[];
+}
+
+export interface EvaluateStats {
+  successes: number;
+  failures: number;
+  tokenUsage: TokenUsage;
+}
+
 export interface EvaluateSummary {
+  version: number;
   results: EvaluateResult[];
-  table: string[][];
-  stats: {
-    successes: number;
-    failures: number;
-    tokenUsage: TokenUsage;
-  };
+  table: EvaluateTable;
+  stats: EvaluateStats;
 }

package/src/util.ts

@@ -1,5 +1,9 @@
 import * as fs from 'fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
 
+import chalk from 'chalk';
+import fetch from 'node-fetch';
 import yaml from 'js-yaml';
 import nunjucks from 'nunjucks';
 import { parse as parsePath } from 'path';
@@ -7,8 +11,11 @@ import { CsvRow } from './types.js';
 import { parse as parseCsv } from 'csv-parse/sync';
 import { stringify } from 'csv-stringify/sync';
 
+import logger from './logger.js';
 import { getDirectory } from './esm.js';
 
+import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+
 import type { EvaluateSummary } from './types.js';
 
 const PROMPT_DELIMITER = '---';
@@ -48,7 +55,10 @@ export function writeOutput(outputPath: string, summary: EvaluateSummary): void
   const outputExtension = outputPath.split('.').pop()?.toLowerCase();
 
   if (outputExtension === 'csv' || outputExtension === 'txt') {
-    const csvOutput = stringify(summary.table);
+    const csvOutput = stringify([
+      [...summary.table.head.prompts, ...summary.table.head.vars],
+      ...summary.table.body.map((row) => [...row.outputs, ...row.vars]),
+    ]);
     fs.writeFileSync(outputPath, csvOutput);
   } else if (outputExtension === 'json') {
     fs.writeFileSync(outputPath, JSON.stringify(summary, null, 2));
@@ -65,3 +75,52 @@ export function writeOutput(outputPath: string, summary: EvaluateSummary): void
     throw new Error('Unsupported output file format. Use CSV, JSON, or YAML.');
   }
 }
+
+export function fetchWithTimeout(
+  url: RequestInfo,
+  options: RequestInit = {},
+  timeout: number,
+): Promise<Response> {
+  return new Promise(async (resolve, reject) => {
+    const controller = new AbortController();
+    const { signal } = controller;
+    options.signal = signal;
+
+    const timeoutId = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`Request timed out after ${timeout} ms`));
+    }, timeout);
+
+    try {
+      const response = await fetch(url, options);
+      clearTimeout(timeoutId);
+      resolve(response);
+    } catch (error) {
+      if (error instanceof Error && error.name === 'AbortError') {
+        // Fetch request was aborted, no need to reject again
+      } else {
+        clearTimeout(timeoutId);
+        reject(error);
+      }
+    }
+  });
+}
+
+export function getConfigDirectoryPath(): string {
+  return path.join(os.homedir(), '.promptfoo');
+}
+
+export function getLatestResultsPath(): string {
+  return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
+}
+
+export function writeLatestResults(results: EvaluateSummary) {
+  const latestResultsPath = getLatestResultsPath();
+  try {
+    fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
+    fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
+    logger.info(`Wrote latest results to ${latestResultsPath}.`);
+  } catch (err) {
+    logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
+  }
+}

package/src/web/client/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  parser: '@typescript-eslint/parser',
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': 'warn',
+  },
+};

package/src/web/client/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>promptfoo web viewer</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

package/src/web/client/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "promptfoo-client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.11.0",
+    "@emotion/styled": "^11.11.0",
+    "@mui/material": "^5.13.0",
+    "@tanstack/react-table": "^8.9.1",
+    "react": "^18.2.0",
+    "react-dnd": "^16.0.1",
+    "react-dnd-html5-backend": "^16.0.1",
+    "react-dom": "^18.2.0",
+    "socket.io-client": "^4.6.1",
+    "tiny-invariant": "^1.3.1",
+    "zustand": "^4.3.8"
+  },
+  "devDependencies": {
+    "@types/react": "^18.0.28",
+    "@types/react-dom": "^18.0.11",
+    "@typescript-eslint/eslint-plugin": "^5.57.1",
+    "@typescript-eslint/parser": "^5.57.1",
+    "@vitejs/plugin-react-swc": "^3.0.0",
+    "eslint": "^8.38.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.3.4",
+    "typescript": "^5.0.2",
+    "vite": "^4.3.2"
+  }
+}

package/src/web/client/public/logo.svg

@@ -0,0 +1,30 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
+ "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
+ width="700.000000pt" height="700.000000pt" viewBox="0 0 700.000000 700.000000"
+ preserveAspectRatio="xMidYMid meet">
+<metadata>
+Created by potrace 1.14, written by Peter Selinger 2001-2017
+</metadata>
+<g transform="translate(0.000000,700.000000) scale(0.100000,-0.100000)"
+fill="#000000" stroke="none">
+<path d="M597 6384 c-77 -13 -152 -51 -204 -103 -44 -43 -88 -101 -77 -101 3
+0 1 -6 -5 -14 -6 -7 -18 -35 -26 -62 -13 -43 -15 -320 -12 -2194 l2 -2145 33
+-67 c53 -109 147 -187 262 -218 33 -9 301 -12 1046 -12 922 0 1004 -1 1017
+-16 8 -10 37 -49 64 -88 274 -400 435 -626 470 -660 124 -118 298 -141 458
+-61 43 21 135 110 187 181 18 24 49 63 69 87 19 24 127 159 238 301 l203 256
+1033 0 c906 0 1040 2 1086 16 130 37 230 136 269 264 20 63 20 95 20 2187 0
+1909 -2 2127 -16 2176 -9 30 -22 61 -30 70 -8 8 -14 19 -14 23 0 17 -55 78
+-99 110 -25 19 -75 44 -111 57 l-65 22 -2868 1 c-1966 1 -2888 -2 -2930 -10z
+m3951 -954 c57 -26 379 -345 411 -408 30 -59 31 -133 3 -196 -26 -58 -220
+-256 -251 -256 -25 0 -599 573 -607 607 -5 19 0 32 23 56 17 18 57 61 89 96
+74 79 106 102 163 117 58 14 111 9 169 -16z m-351 -712 c156 -156 290 -294
+297 -308 8 -15 11 -33 6 -45 -12 -33 -1398 -1412 -1425 -1419 -22 -6 -58 27
+-315 285 -160 160 -295 299 -300 309 -13 24 -14 23 270 310 866 876 1142 1150
+1161 1150 14 0 116 -94 306 -282z m-1617 -1658 c322 -321 325 -326 270 -381
+-16 -16 -37 -29 -47 -30 -22 -1 -12 1 -198 -44 -410 -99 -479 -115 -504 -115
+-16 0 -40 11 -55 25 -25 23 -27 31 -23 73 2 26 8 61 11 77 19 82 79 327 91
+375 8 30 26 101 39 158 27 115 51 152 97 152 24 0 77 -48 319 -290z"/>
+</g>
+</svg>

package/src/web/client/src/App.tsx

@@ -0,0 +1,43 @@
+import * as React from 'react';
+
+import { io as SocketIOClient } from 'socket.io-client';
+
+import ResultsView from './ResultsView.js';
+import NavBar from './NavBar.js';
+import { useStore } from './store.js';
+
+import './App.css';
+
+function App() {
+  const { table, setTable } = useStore();
+  const [loaded, setLoaded] = React.useState<boolean>(false);
+
+  React.useEffect(() => {
+    const socket = SocketIOClient(`http://${window.location.host}`);
+    //const socket = SocketIOClient(`http://localhost:15500`);
+
+    socket.on('init', (data) => {
+      console.log('Initialized socket connection');
+      setLoaded(true);
+      setTable(data.table);
+    });
+
+    socket.on('update', (data) => {
+      console.log('Received data update');
+      setTable(data.table);
+    });
+
+    return () => {
+      socket.disconnect();
+    };
+  }, [loaded, setTable]);
+
+  return (
+    <>
+      <NavBar />
+      {loaded && table ? <ResultsView /> : <div>Loading...</div>}
+    </>
+  );
+}
+
+export default App;

package/src/web/client/src/Logo.css

@@ -0,0 +1,13 @@
+.logo {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+}
+
+.logo img {
+  width: 30px;
+}
+
+.logo span {
+  margin-bottom: 6px;
+}

package/src/web/client/src/Logo.tsx

@@ -0,0 +1,11 @@
+import Box from '@mui/material/Box';
+
+import './Logo.css';
+
+export default function Logo() {
+  return (
+    <Box className="logo">
+      <img src="/logo.svg" alt="Promptfoo logo" /> <span>promptfoo</span>
+    </Box>
+  );
+}

package/src/web/client/src/NavBar.css

@@ -0,0 +1,3 @@
+nav {
+  margin-bottom: 1rem;
+}

package/src/web/client/src/NavBar.tsx

@@ -0,0 +1,11 @@
+import Logo from './Logo';
+
+import './NavBar.css';
+
+export default function NavBar() {
+  return (
+    <nav>
+      <Logo />
+    </nav>
+  );
+}