promptfoo 0.3.0 → 0.5.0

package/src/providers/openai.ts

@@ -0,0 +1,256 @@
+import { LRUCache } from 'lru-cache';
+
+import logger from '../logger.js';
+import { fetchWithTimeout } from '../util.js';
+import { REQUEST_TIMEOUT_MS } from './shared.js';
+
+import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '../types.js';
+
+const DEFAULT_OPENAI_HOST = 'api.openai.com';
+
+const embeddingsCache = new LRUCache<string, ProviderEmbeddingResponse>({
+  max: 1000,
+});
+
+class OpenAiGenericProvider implements ApiProvider {
+  modelName: string;
+  apiKey: string;
+  apiHost: string;
+
+  constructor(modelName: string, apiKey?: string) {
+    this.modelName = modelName;
+
+    const key = apiKey || process.env.OPENAI_API_KEY;
+    if (!key) {
+      throw new Error(
+        'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
+      );
+    }
+    this.apiKey = key;
+
+    this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
+  }
+
+  id(): string {
+    return `openai:${this.modelName}`;
+  }
+
+  toString(): string {
+    return `[OpenAI Provider ${this.modelName}]`;
+  }
+
+  // @ts-ignore: Prompt is not used in this implementation
+  async callApi(prompt: string): Promise<ProviderResponse> {
+    throw new Error('Not implemented');
+  }
+}
+
+export class OpenAiEmbeddingProvider extends OpenAiGenericProvider {
+  async callEmbeddingApi(text: string): Promise<ProviderEmbeddingResponse> {
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key must be set for similarity comparison');
+    }
+
+    // TODO(ian): Improve cache
+    const cached = embeddingsCache.get(text);
+    if (cached) {
+      return cached;
+    }
+
+    const body = {
+      input: text,
+      model: this.modelName,
+    };
+    let response, data;
+    try {
+      response = await fetchWithTimeout(
+        `https://${this.apiHost}/v1/embeddings`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      );
+      data = (await response.json()) as unknown as any;
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+        tokenUsage: {
+          total: 0,
+          prompt: 0,
+          completion: 0,
+        },
+      };
+    }
+    logger.debug(`\tOpenAI API response (embeddings): ${JSON.stringify(data)}`);
+
+    try {
+      const embedding = data?.data?.[0]?.embedding;
+      if (!embedding) {
+        throw new Error('No embedding returned');
+      }
+      const ret = {
+        embedding,
+        tokenUsage: {
+          total: data.usage.total_tokens,
+          prompt: data.usage.prompt_tokens,
+          completion: data.usage.completion_tokens,
+        },
+      };
+      embeddingsCache.set(text, ret);
+      return ret;
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+        tokenUsage: {
+          total: data?.usage?.total_tokens,
+          prompt: data?.usage?.prompt_tokens,
+          completion: data?.usage?.completion_tokens,
+        },
+      };
+    }
+  }
+}
+
+export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
+
+export class OpenAiCompletionProvider extends OpenAiGenericProvider {
+  static OPENAI_COMPLETION_MODELS = [
+    'text-davinci-003',
+    'text-davinci-002',
+    'text-curie-001',
+    'text-babbage-001',
+    'text-ada-001',
+  ];
+
+  constructor(modelName: string, apiKey?: string) {
+    if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
+      logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
+    }
+    super(modelName, apiKey);
+  }
+
+  async callApi(prompt: string): Promise<ProviderResponse> {
+    const body = {
+      model: this.modelName,
+      prompt,
+      max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
+      temperature: process.env.OPENAI_TEMPERATURE || 0,
+      stop: process.env.OPENAI_STOP ? JSON.parse(process.env.OPENAI_STOP) : undefined,
+    };
+    logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
+    let response, data;
+    try {
+      response = await fetchWithTimeout(
+        `https://${this.apiHost}/v1/completions`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      );
+
+      data = (await response.json()) as unknown as any;
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+    logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      return {
+        output: data.choices[0].text,
+        tokenUsage: {
+          total: data.usage.total_tokens,
+          prompt: data.usage.prompt_tokens,
+          completion: data.usage.completion_tokens,
+        },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}
+
+export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
+  static OPENAI_CHAT_MODELS = [
+    'gpt-4',
+    'gpt-4-0314',
+    'gpt-4-32k',
+    'gpt-4-32k-0314',
+    'gpt-3.5-turbo',
+    'gpt-3.5-turbo-0301',
+  ];
+
+  constructor(modelName: string, apiKey?: string) {
+    if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
+      logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
+    }
+    super(modelName, apiKey);
+  }
+
+  async callApi(prompt: string): Promise<ProviderResponse> {
+    let messages: { role: string; content: string }[];
+    try {
+      // User can specify `messages` payload as JSON, or we'll just put the
+      // string prompt into a `messages` array.
+      messages = JSON.parse(prompt);
+    } catch (err) {
+      messages = [{ role: 'user', content: prompt }];
+    }
+    const body = {
+      model: this.modelName,
+      messages: messages,
+      max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
+      temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
+    };
+    logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
+
+    let response, data;
+    try {
+      response = await fetchWithTimeout(
+        `https://${this.apiHost}/v1/chat/completions`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      );
+      data = (await response.json()) as unknown as any;
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+
+    logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
+    try {
+      return {
+        output: data.choices[0].message.content,
+        tokenUsage: {
+          total: data.usage.total_tokens,
+          prompt: data.usage.prompt_tokens,
+          completion: data.usage.completion_tokens,
+        },
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
+      };
+    }
+  }
+}

package/src/providers/shared.ts

@@ -0,0 +1,3 @@
+export const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
+  ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
+  : 60_000;

package/src/providers.ts

@@ -1,183 +1,9 @@
 import path from 'node:path';
 
-import { ApiProvider, ProviderResponse } from './types.js';
-import { fetchWithTimeout } from './util.js';
-import logger from './logger.js';
+import { ApiProvider } from './types.js';
 
-const DEFAULT_OPENAI_HOST = 'api.openai.com';
-
-const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
-  ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
-  : 10_000;
-
-export class OpenAiGenericProvider implements ApiProvider {
-  modelName: string;
-  apiKey: string;
-  apiHost: string;
-
-  constructor(modelName: string, apiKey?: string) {
-    this.modelName = modelName;
-
-    const key = apiKey || process.env.OPENAI_API_KEY;
-    if (!key) {
-      throw new Error(
-        'OpenAI API key is not set. Set OPENAI_API_KEY environment variable or pass it as an argument to the constructor.',
-      );
-    }
-    this.apiKey = key;
-
-    this.apiHost = process.env.OPENAI_API_HOST || DEFAULT_OPENAI_HOST;
-  }
-
-  id(): string {
-    return `openai:${this.modelName}`;
-  }
-
-  toString(): string {
-    return `[OpenAI Provider ${this.modelName}]`;
-  }
-
-  // @ts-ignore: Prompt is not used in this implementation
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    throw new Error('Not implemented');
-  }
-}
-
-export class OpenAiCompletionProvider extends OpenAiGenericProvider {
-  static OPENAI_COMPLETION_MODELS = [
-    'text-davinci-003',
-    'text-davinci-002',
-    'text-curie-001',
-    'text-babbage-001',
-    'text-ada-001',
-  ];
-
-  constructor(modelName: string, apiKey?: string) {
-    if (!OpenAiCompletionProvider.OPENAI_COMPLETION_MODELS.includes(modelName)) {
-      logger.warn(`Using unknown OpenAI completion model: ${modelName}`);
-    }
-    super(modelName, apiKey);
-  }
-
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    const body = {
-      model: this.modelName,
-      prompt,
-      max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
-      temperature: process.env.OPENAI_TEMPERATURE || 0,
-    };
-    logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
-    let response, data;
-    try {
-      response = await fetchWithTimeout(
-        `https://${this.apiHost}/v1/completions`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.apiKey}`,
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      );
-
-      data = (await response.json()) as unknown as any;
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-    logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
-    try {
-      return {
-        output: data.choices[0].text,
-        tokenUsage: {
-          total: data.usage.total_tokens,
-          prompt: data.usage.prompt_tokens,
-          completion: data.usage.completion_tokens,
-        },
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}
-
-export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
-  static OPENAI_CHAT_MODELS = [
-    'gpt-4',
-    'gpt-4-0314',
-    'gpt-4-32k',
-    'gpt-4-32k-0314',
-    'gpt-3.5-turbo',
-    'gpt-3.5-turbo-0301',
-  ];
-
-  constructor(modelName: string, apiKey?: string) {
-    if (!OpenAiChatCompletionProvider.OPENAI_CHAT_MODELS.includes(modelName)) {
-      logger.warn(`Using unknown OpenAI chat model: ${modelName}`);
-    }
-    super(modelName, apiKey);
-  }
-
-  async callApi(prompt: string): Promise<ProviderResponse> {
-    let messages: { role: string; content: string }[];
-    try {
-      // User can specify `messages` payload as JSON, or we'll just put the
-      // string prompt into a `messages` array.
-      messages = JSON.parse(prompt);
-    } catch (err) {
-      messages = [{ role: 'user', content: prompt }];
-    }
-    const body = {
-      model: this.modelName,
-      messages: messages,
-      max_tokens: process.env.OPENAI_MAX_TOKENS || 1024,
-      temperature: process.env.OPENAI_MAX_TEMPERATURE || 0,
-    };
-    logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
-
-    let response, data;
-    try {
-      response = await fetchWithTimeout(
-        `https://${this.apiHost}/v1/chat/completions`,
-        {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            Authorization: `Bearer ${this.apiKey}`,
-          },
-          body: JSON.stringify(body),
-        },
-        REQUEST_TIMEOUT_MS,
-      );
-      data = (await response.json()) as unknown as any;
-    } catch (err) {
-      return {
-        error: `API call error: ${String(err)}`,
-      };
-    }
-
-    logger.debug(`\tOpenAI API response: ${JSON.stringify(data)}`);
-    try {
-      return {
-        output: data.choices[0].message.content,
-        tokenUsage: {
-          total: data.usage.total_tokens,
-          prompt: data.usage.prompt_tokens,
-          completion: data.usage.completion_tokens,
-        },
-      };
-    } catch (err) {
-      return {
-        error: `API response error: ${String(err)}: ${JSON.stringify(data)}`,
-      };
-    }
-  }
-}
+import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai.js';
+import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai.js';
 
 export async function loadApiProvider(providerPath: string): Promise<ApiProvider> {
   if (providerPath?.startsWith('openai:')) {
@@ -201,6 +27,20 @@ export async function loadApiProvider(providerPath: string): Promise<ApiProvider
     }
   }
 
+  if (providerPath?.startsWith('localai:')) {
+    const options = providerPath.split(':');
+    const modelType = options[1];
+    const modelName = options[2];
+
+    if (modelType === 'chat') {
+      return new LocalAiChatProvider(modelName);
+    } else if (modelType === 'completion') {
+      return new LocalAiCompletionProvider(modelName);
+    } else {
+      return new LocalAiChatProvider(modelType);
+    }
+  }
+
   // Load custom module
   const CustomApiProvider = (await import(path.join(process.cwd(), providerPath))).default;
   return new CustomApiProvider();

package/src/types.ts

@@ -5,8 +5,11 @@ export interface CommandLineOptions {
   vars?: string;
   config?: string;
   verbose?: boolean;
-  maxConcurrency?: number;
+  maxConcurrency?: string;
   grader?: string;
+  view?: boolean;
+  noWrite?: boolean;
+  tableCellMaxLength?: string;
 }
 
 export interface ApiProvider {
@@ -26,6 +29,12 @@ export interface ProviderResponse {
   tokenUsage?: TokenUsage;
 }
 
+export interface ProviderEmbeddingResponse {
+  error?: string;
+  embedding?: number[];
+  tokenUsage?: TokenUsage;
+}
+
 export interface CsvRow {
   [key: string]: string;
 }
@@ -61,6 +70,18 @@ export interface EvaluateResult {
   success: boolean;
 }
 
+export interface EvaluateTable {
+  head: {
+    prompts: string[];
+    vars: string[];
+  };
+
+  body: {
+    outputs: string[];
+    vars: string[];
+  }[];
+}
+
 export interface EvaluateStats {
   successes: number;
   failures: number;
@@ -68,7 +89,8 @@ export interface EvaluateStats {
 }
 
 export interface EvaluateSummary {
+  version: number;
   results: EvaluateResult[];
-  table: string[][];
+  table: EvaluateTable;
   stats: EvaluateStats;
 }

package/src/util.ts

@@ -1,4 +1,6 @@
 import * as fs from 'fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
 
 import fetch from 'node-fetch';
 import yaml from 'js-yaml';
@@ -8,6 +10,7 @@ import { CsvRow } from './types.js';
 import { parse as parseCsv } from 'csv-parse/sync';
 import { stringify } from 'csv-stringify/sync';
 
+import logger from './logger.js';
 import { getDirectory } from './esm.js';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
@@ -51,7 +54,10 @@ export function writeOutput(outputPath: string, summary: EvaluateSummary): void
   const outputExtension = outputPath.split('.').pop()?.toLowerCase();
 
   if (outputExtension === 'csv' || outputExtension === 'txt') {
-    const csvOutput = stringify(summary.table);
+    const csvOutput = stringify([
+      [...summary.table.head.prompts, ...summary.table.head.vars],
+      ...summary.table.body.map((row) => [...row.outputs, ...row.vars]),
+    ]);
     fs.writeFileSync(outputPath, csvOutput);
   } else if (outputExtension === 'json') {
     fs.writeFileSync(outputPath, JSON.stringify(summary, null, 2));
@@ -98,3 +104,32 @@ export function fetchWithTimeout(
     }
   });
 }
+
+export function getConfigDirectoryPath(): string {
+  return path.join(os.homedir(), '.promptfoo');
+}
+
+export function getLatestResultsPath(): string {
+  return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
+}
+
+export function writeLatestResults(results: EvaluateSummary) {
+  const latestResultsPath = getLatestResultsPath();
+  try {
+    fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
+    fs.writeFileSync(latestResultsPath, JSON.stringify(results, null, 2));
+    logger.info(`Wrote latest results to ${latestResultsPath}.`);
+  } catch (err) {
+    logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
+  }
+}
+
+export function cosineSimilarity(vecA: number[], vecB: number[]) {
+  if (vecA.length !== vecB.length) {
+    throw new Error('Vectors must be of equal length');
+  }
+  const dotProduct = vecA.reduce((acc, val, idx) => acc + val * vecB[idx], 0);
+  const vecAMagnitude = Math.sqrt(vecA.reduce((acc, val) => acc + val * val, 0));
+  const vecBMagnitude = Math.sqrt(vecB.reduce((acc, val) => acc + val * val, 0));
+  return dotProduct / (vecAMagnitude * vecBMagnitude);
+}

package/src/web/client/.eslintrc.cjs

@@ -0,0 +1,14 @@
+module.exports = {
+  env: { browser: true, es2020: true },
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'plugin:react-hooks/recommended',
+  ],
+  parser: '@typescript-eslint/parser',
+  parserOptions: { ecmaVersion: 'latest', sourceType: 'module' },
+  plugins: ['react-refresh'],
+  rules: {
+    'react-refresh/only-export-components': 'warn',
+  },
+};

package/src/web/client/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>promptfoo web viewer</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

package/src/web/client/package.json

@@ -0,0 +1,37 @@
+{
+  "name": "promptfoo-client",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "tsc && vite build",
+    "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.11.0",
+    "@emotion/styled": "^11.11.0",
+    "@mui/material": "^5.13.0",
+    "@tanstack/react-table": "^8.9.1",
+    "react": "^18.2.0",
+    "react-dnd": "^16.0.1",
+    "react-dnd-html5-backend": "^16.0.1",
+    "react-dom": "^18.2.0",
+    "socket.io-client": "^4.6.1",
+    "tiny-invariant": "^1.3.1",
+    "zustand": "^4.3.8"
+  },
+  "devDependencies": {
+    "@types/react": "^18.0.28",
+    "@types/react-dom": "^18.0.11",
+    "@typescript-eslint/eslint-plugin": "^5.57.1",
+    "@typescript-eslint/parser": "^5.57.1",
+    "@vitejs/plugin-react-swc": "^3.0.0",
+    "eslint": "^8.38.0",
+    "eslint-plugin-react-hooks": "^4.6.0",
+    "eslint-plugin-react-refresh": "^0.3.4",
+    "typescript": "^5.0.2",
+    "vite": "^4.3.2"
+  }
+}

package/src/web/client/public/logo.svg

@@ -0,0 +1,30 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
+ "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
+ width="700.000000pt" height="700.000000pt" viewBox="0 0 700.000000 700.000000"
+ preserveAspectRatio="xMidYMid meet">
+<metadata>
+Created by potrace 1.14, written by Peter Selinger 2001-2017
+</metadata>
+<g transform="translate(0.000000,700.000000) scale(0.100000,-0.100000)"
+fill="#000000" stroke="none">
+<path d="M597 6384 c-77 -13 -152 -51 -204 -103 -44 -43 -88 -101 -77 -101 3
+0 1 -6 -5 -14 -6 -7 -18 -35 -26 -62 -13 -43 -15 -320 -12 -2194 l2 -2145 33
+-67 c53 -109 147 -187 262 -218 33 -9 301 -12 1046 -12 922 0 1004 -1 1017
+-16 8 -10 37 -49 64 -88 274 -400 435 -626 470 -660 124 -118 298 -141 458
+-61 43 21 135 110 187 181 18 24 49 63 69 87 19 24 127 159 238 301 l203 256
+1033 0 c906 0 1040 2 1086 16 130 37 230 136 269 264 20 63 20 95 20 2187 0
+1909 -2 2127 -16 2176 -9 30 -22 61 -30 70 -8 8 -14 19 -14 23 0 17 -55 78
+-99 110 -25 19 -75 44 -111 57 l-65 22 -2868 1 c-1966 1 -2888 -2 -2930 -10z
+m3951 -954 c57 -26 379 -345 411 -408 30 -59 31 -133 3 -196 -26 -58 -220
+-256 -251 -256 -25 0 -599 573 -607 607 -5 19 0 32 23 56 17 18 57 61 89 96
+74 79 106 102 163 117 58 14 111 9 169 -16z m-351 -712 c156 -156 290 -294
+297 -308 8 -15 11 -33 6 -45 -12 -33 -1398 -1412 -1425 -1419 -22 -6 -58 27
+-315 285 -160 160 -295 299 -300 309 -13 24 -14 23 270 310 866 876 1142 1150
+1161 1150 14 0 116 -94 306 -282z m-1617 -1658 c322 -321 325 -326 270 -381
+-16 -16 -37 -29 -47 -30 -22 -1 -12 1 -198 -44 -410 -99 -479 -115 -504 -115
+-16 0 -40 11 -55 25 -25 23 -27 31 -23 73 2 26 8 61 11 77 19 82 79 327 91
+375 8 30 26 101 39 158 27 115 51 152 97 152 24 0 77 -48 319 -290z"/>
+</g>
+</svg>