promptfoo 0.16.0 → 0.17.1

package/dist/src/web/client/assets/{index-c3faa651.css → index-b82d0138.css}

@@ -1 +1 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}tr .cell-rating{visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr:hover .cell-rating{visibility:visible}tr .cell-rating .rating{cursor:pointer}tr .cell-rating .rating:first-child{margin-right:.5rem}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .score{font-weight:400}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}
+:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}th .action{cursor:pointer;margin-left:.5rem}tr .cell-actions{display:flex;gap:.5rem;visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr:hover .cell-actions{visibility:visible}tr .cell-actions .action{cursor:pointer}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .score{font-weight:400}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}

package/dist/src/web/client/index.html

@@ -5,8 +5,8 @@
     <link rel="icon" type="image/svg+xml" href="favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>promptfoo web viewer</title>
-    <script type="module" crossorigin src="/assets/index-eb1e9052.js"></script>
-    <link rel="stylesheet" href="/assets/index-c3faa651.css">
+    <script type="module" crossorigin src="/assets/index-b6617ee8.js"></script>
+    <link rel="stylesheet" href="/assets/index-b82d0138.css">
   </head>
   <body>
     <div id="root"></div>

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "promptfoo",
-  "description": "Prompt engineering toolkit",
+  "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.16.0",
+  "version": "0.17.1",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",

package/src/assertions.ts

@@ -123,12 +123,12 @@ export async function runAssertion(
   }
 
   if (baseType === 'contains') {
-    invariant(assertion.value, '"contains" assertion type must have a string value');
+    invariant(assertion.value, '"contains" assertion type must have a string or number value');
     invariant(
-      typeof assertion.value === 'string',
-      '"contains" assertion type must have a string value',
+      typeof assertion.value === 'string' || typeof assertion.value === 'number',
+      '"contains" assertion type must have a string or number value',
     );
-    pass = output.includes(assertion.value) !== inverse;
+    pass = output.includes(String(assertion.value)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
@@ -192,12 +192,12 @@ export async function runAssertion(
   }
 
   if (baseType === 'icontains') {
-    invariant(assertion.value, '"icontains" assertion type must have a string value');
+    invariant(assertion.value, '"icontains" assertion type must have a string or number value');
     invariant(
-      typeof assertion.value === 'string',
-      '"icontains" assertion type must have a string value',
+      typeof assertion.value === 'string' || typeof assertion.value === 'number',
+      '"icontains" assertion type must have a string or number value',
     );
-    pass = output.toLowerCase().includes(assertion.value.toLowerCase()) !== inverse;
+    pass = output.toLowerCase().includes(String(assertion.value).toLowerCase()) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,

package/src/cache.ts

@@ -97,3 +97,8 @@ export function disableCache() {
   logger.info('Cache is disabled.');
   enabled = false;
 }
+
+export async function clearCache() {
+  logger.info('Clearing cache...');
+  return getCache().reset();
+}

package/src/evaluator.ts

@@ -293,7 +293,7 @@ class Evaluator {
 
     const table: EvaluateTable = {
       head: {
-        prompts: prompts.map((p) => p.display),
+        prompts,
         vars: Array.from(varNames).sort(),
         // TODO(ian): add assertions to table?
       },
@@ -383,6 +383,7 @@ class Evaluator {
           pass: row.success,
           score: row.score,
           text: resultText,
+          prompt: row.prompt.raw,
         };
       },
     );

package/src/main.ts

@@ -19,7 +19,7 @@ import {
   writeOutput,
 } from './util';
 import { DEFAULT_README, DEFAULT_YAML_CONFIG, DEFAULT_PROMPTS } from './onboarding';
-import { disableCache } from './cache';
+import { disableCache, clearCache } from './cache';
 import { getDirectory } from './esm';
 import { init } from './web/server';
 import { checkForUpdates } from './updates';
@@ -139,6 +139,19 @@ async function main() {
       logger.info(`View results: ${chalk.greenBright.bold(url)}`);
     });
 
+  program
+    .command('cache')
+    .description('Manage cache')
+    .command('clear')
+    .description('Clear cache')
+    .action(async () => {
+      await clearCache();
+      telemetry.record('command_used', {
+        name: 'cache_clear',
+      });
+      await telemetry.send();
+    });
+
   program
     .command('eval')
     .description('Evaluate prompts')

package/src/providers/openai.ts

@@ -204,6 +204,8 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     'gpt-3.5-turbo',
     'gpt-3.5-turbo-0301',
     'gpt-3.5-turbo-0613',
+    'gpt-3.5-turbo-16k',
+    'gpt-3.5-turbo-16k-0613',
   ];
 
   options: OpenAiCompletionOptions;
@@ -216,7 +218,6 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     this.options = context || {};
   }
 
-  // TODO(ian): support passing in `messages` directly
   async callApi(prompt: string, options?: OpenAiCompletionOptions): Promise<ProviderResponse> {
     if (!this.apiKey) {
       throw new Error(
@@ -224,12 +225,20 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
       );
     }
 
-    let messages: { role: string; content: string }[];
+    let messages: { role: string; content: string; name?: string }[];
     try {
-      // User can specify `messages` payload as JSON, or we'll just put the
-      // string prompt into a `messages` array.
-      messages = JSON.parse(prompt);
+      messages = JSON.parse(prompt) as { role: string; content: string }[];
     } catch (err) {
+      const trimmedPrompt = prompt.trim();
+      if (
+        process.env.PROMPTFOO_REQUIRE_JSON_PROMPTS ||
+        trimmedPrompt.startsWith('{') ||
+        trimmedPrompt.startsWith('[')
+      ) {
+        throw new Error(
+          `OpenAI Chat Completion prompt is not a valid JSON string: ${err}\n\n${prompt}`,
+        );
+      }
       messages = [{ role: 'user', content: prompt }];
     }
 
@@ -292,4 +301,4 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
 
 export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
 export const DefaultGradingProvider = new OpenAiChatCompletionProvider('gpt-4-0613');
-export const DefaultSuggestionsProvider = new OpenAiChatCompletionProvider('gpt-4');
+export const DefaultSuggestionsProvider = new OpenAiChatCompletionProvider('gpt-4-0613');

package/src/providers/scriptCompletion.ts

@@ -0,0 +1,23 @@
+import { exec } from 'child_process';
+
+import { ApiProvider, ProviderConfig, ProviderResponse } from '../types';
+
+export class ScriptCompletionProvider implements ApiProvider {
+  constructor(private scriptPath: string, private config?: ProviderConfig) {}
+
+  id() {
+    return 'script';
+  }
+
+  async callApi(prompt: string) {
+    return new Promise((resolve, reject) => {
+      exec(`${this.scriptPath} "${prompt}"`, (error, stdout, stderr) => {
+        if (error) {
+          reject(error);
+        } else {
+          resolve({ output: stdout.trim() });
+        }
+      });
+    }) as Promise<ProviderResponse>;
+  }
+}

package/src/providers.ts

@@ -4,6 +4,7 @@ import { ApiProvider, ProviderConfig, ProviderId, RawProviderConfig } from './ty
 
 import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './providers/openai';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
+import { ScriptCompletionProvider } from './providers/scriptCompletion';
 
 export async function loadApiProviders(
   providerPaths: ProviderId | ProviderId[] | RawProviderConfig[],
@@ -30,7 +31,11 @@ export async function loadApiProvider(
   providerPath: string,
   context: ProviderConfig | undefined = undefined,
 ): Promise<ApiProvider> {
-  if (providerPath?.startsWith('openai:')) {
+  if (providerPath?.startsWith('script:')) {
+    // Load script module
+    const scriptPath = providerPath.split(':')[1];
+    return new ScriptCompletionProvider(scriptPath, context?.config);
+  } else if (providerPath?.startsWith('openai:')) {
     // Load OpenAI module
     const options = providerPath.split(':');
     const modelType = options[1];

package/src/table.ts

@@ -7,7 +7,7 @@ export function generateTable(summary: EvaluateSummary, tableCellMaxLength = 250
   const head = summary.table.head;
   const headLength = head.prompts.length + head.vars.length;
   const table = new Table({
-    head: [...head.prompts, ...head.vars],
+    head: [...head.prompts.map((prompt) => prompt.display), ...head.vars],
     colWidths: Array(headLength).fill(Math.floor(maxWidth / headLength)),
     wordWrap: true,
     wrapOnWordBoundary: false,

package/src/types.ts

@@ -95,11 +95,12 @@ export interface EvaluateTableOutput {
   pass: boolean;
   score: number;
   text: string;
+  prompt: string;
 }
 
 export interface EvaluateTable {
   head: {
-    prompts: string[];
+    prompts: Prompt[];
     vars: string[];
   };
 

package/src/util.ts

@@ -347,7 +347,9 @@ export function testCaseFromCsvRow(row: CsvRow): TestCase {
   const asserts: Assertion[] = [];
   for (const [key, value] of Object.entries(row)) {
     if (key === '__expected') {
-      asserts.push(assertionFromString(value));
+      if (value.trim() !== '') {
+        asserts.push(assertionFromString(value));
+      }
     } else {
       vars[key] = value;
     }

package/src/web/client/package-lock.json

@@ -35,6 +35,15 @@
         "vite": "^4.3.2"
       }
     },
+    "node_modules/@aashutoshrathi/word-wrap": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
+      "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
+      "dev": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/@babel/code-frame": {
       "version": "7.21.4",
       "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.21.4.tgz",
@@ -2685,17 +2694,17 @@
       }
     },
     "node_modules/optionator": {
-      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
-      "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+      "version": "0.9.3",
+      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
+      "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
       "dev": true,
       "dependencies": {
+        "@aashutoshrathi/word-wrap": "^1.2.3",
         "deep-is": "^0.1.3",
         "fast-levenshtein": "^2.0.6",
         "levn": "^0.4.1",
         "prelude-ls": "^1.2.1",
-        "type-check": "^0.4.0",
-        "word-wrap": "^1.2.3"
+        "type-check": "^0.4.0"
       },
       "engines": {
         "node": ">= 0.8.0"
@@ -3083,9 +3092,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
-      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "version": "7.5.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
+      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
       "dev": true,
       "dependencies": {
         "lru-cache": "^6.0.0"
@@ -3391,15 +3400,6 @@
         "node": ">= 8"
       }
     },
-    "node_modules/word-wrap": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
-      "dev": true,
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
@@ -3485,6 +3485,12 @@
     }
   },
   "dependencies": {
+    "@aashutoshrathi/word-wrap": {
+      "version": "1.2.6",
+      "resolved": "https://registry.npmjs.org/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz",
+      "integrity": "sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==",
+      "dev": true
+    },
     "@babel/code-frame": {
       "version": "7.21.4",
       "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.21.4.tgz",
@@ -5231,17 +5237,17 @@
       }
     },
     "optionator": {
-      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz",
-      "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==",
+      "version": "0.9.3",
+      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.3.tgz",
+      "integrity": "sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==",
       "dev": true,
       "requires": {
+        "@aashutoshrathi/word-wrap": "^1.2.3",
         "deep-is": "^0.1.3",
         "fast-levenshtein": "^2.0.6",
         "levn": "^0.4.1",
         "prelude-ls": "^1.2.1",
-        "type-check": "^0.4.0",
-        "word-wrap": "^1.2.3"
+        "type-check": "^0.4.0"
       }
     },
     "p-limit": {
@@ -5483,9 +5489,9 @@
       }
     },
     "semver": {
-      "version": "7.5.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.0.tgz",
-      "integrity": "sha512-+XC0AD/R7Q2mPSRuy2Id0+CGTZ98+8f+KvwirxOKIEyid+XSx6HbC63p+O4IndTHuX5Z+JxQ0TghCkO5Cg/2HA==",
+      "version": "7.5.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.3.tgz",
+      "integrity": "sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ==",
       "dev": true,
       "requires": {
         "lru-cache": "^6.0.0"
@@ -5674,12 +5680,6 @@
         "isexe": "^2.0.0"
       }
     },
-    "word-wrap": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz",
-      "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==",
-      "dev": true
-    },
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",

package/src/web/client/src/EvalOutputPromptDialog.tsx

@@ -0,0 +1,61 @@
+import { useState, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import Dialog from '@mui/material/Dialog';
+import DialogActions from '@mui/material/DialogActions';
+import DialogContent from '@mui/material/DialogContent';
+import DialogTitle from '@mui/material/DialogTitle';
+import TextareaAutosize from '@mui/base/TextareaAutosize';
+import IconButton from '@mui/material/IconButton';
+import ContentCopyIcon from '@mui/icons-material/ContentCopy';
+import CheckIcon from '@mui/icons-material/Check';
+
+interface EvalOutputPromptDialogProps {
+  open: boolean;
+  onClose: () => void;
+  prompt: string;
+  output?: string;
+}
+
+export default function EvalOutputPromptDialog({
+  open,
+  onClose,
+  prompt,
+  output,
+}: EvalOutputPromptDialogProps) {
+  const [copied, setCopied] = useState(false);
+
+  useEffect(() => {
+    setCopied(false);
+  }, [prompt]);
+
+  const copyToClipboard = async (text: string) => {
+    await navigator.clipboard.writeText(text);
+    setCopied(true);
+  };
+
+  return (
+    <Dialog open={open} onClose={onClose} fullWidth maxWidth="lg">
+      <DialogTitle>Prompt</DialogTitle>
+      <DialogContent>
+        <TextareaAutosize readOnly value={prompt} style={{ width: '100%' }} />
+        <IconButton
+          onClick={() => copyToClipboard(prompt)}
+          style={{ position: 'absolute', right: '10px', top: '10px' }}
+        >
+          {copied ? <CheckIcon /> : <ContentCopyIcon />}
+        </IconButton>
+      </DialogContent>
+      {output && (
+        <>
+          <DialogTitle>Output</DialogTitle>
+          <DialogContent>
+            <TextareaAutosize readOnly value={output} style={{ width: '100%' }} />
+          </DialogContent>
+        </>
+      )}
+      <DialogActions>
+        <Button onClick={onClose}>Close</Button>
+      </DialogActions>
+    </Dialog>
+  );
+}

package/src/web/client/src/ResultsTable.css

@@ -64,10 +64,17 @@ th,
   vertical-align: bottom;
 }
 
+th .action {
+  cursor: pointer;
+  margin-left: 0.5rem;
+}
+
 tr .cell {
 }
 
-tr .cell-rating {
+tr .cell-actions {
+  display: flex;
+  gap: 0.5rem;
   visibility: hidden;
   position: absolute;
   bottom: 1.25rem;
@@ -76,18 +83,14 @@ tr .cell-rating {
   font-size: 1.75rem;
 }
 
-tr:hover .cell-rating {
+tr:hover .cell-actions {
   visibility: visible;
 }
 
-tr .cell-rating .rating {
+tr .cell-actions .action {
   cursor: pointer;
 }
 
-tr .cell-rating .rating:first-child {
-  margin-right: 0.5rem;
-}
-
 th .smalltext {
   visibility: hidden;
   font-weight: normal;

package/src/web/client/src/ResultsTable.tsx

@@ -16,6 +16,8 @@ import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
+import EvalOutputPromptDialog from './EvalOutputPromptDialog';
+
 import type { EvalRow, EvalRowOutput, FilterMode } from './types.js';
 
 import './ResultsTable.css';
@@ -62,28 +64,24 @@ function TruncatedText({ text: rawText, maxLength }: TruncatedTextProps) {
 
   const renderTruncatedText = () => {
     if (text.length <= maxLength) {
-      return text;
+      return <span>{text}</span>;
     }
     if (isTruncated) {
       return (
-        <>
-          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
-            {text.substring(0, maxLength)} ...
-          </span>
-        </>
+        <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+          {text.substring(0, maxLength)} ...
+        </span>
       );
     } else {
       return (
-        <>
-          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
-            {text}
-          </span>
-        </>
+        <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+          {text}
+        </span>
       );
     }
   };
 
-  return <div>{renderTruncatedText()}</div>;
+  return renderTruncatedText();
 }
 
 interface PromptOutputProps {
@@ -101,6 +99,13 @@ function EvalOutputCell({
   promptIndex,
   onRating,
 }: PromptOutputProps) {
+  const [openPrompt, setOpen] = React.useState(false);
+  const handlePromptOpen = () => {
+    setOpen(true);
+  };
+  const handlePromptClose = () => {
+    setOpen(false);
+  };
   let text = typeof output.text === 'string' ? output.text : JSON.stringify(output.text);
   let chunks: string[] = [];
   if (!output.pass && text.includes('---')) {
@@ -113,6 +118,7 @@ function EvalOutputCell({
     onRating(rowIndex, promptIndex, isPass);
   };
 
+  // TODO(ian): output.prompt check for backwards compatibility, remove after 0.17.0
   return (
     <>
       <div className="cell">
@@ -128,11 +134,24 @@ function EvalOutputCell({
         )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
-      <div className="cell-rating">
-        <span className="rating" onClick={() => handleClick(true)}>
+      <div className="cell-actions">
+        {output.prompt && (
+          <>
+            <span className="action" onClick={handlePromptOpen}>
+              🔎
+            </span>
+            <EvalOutputPromptDialog
+              open={openPrompt}
+              onClose={handlePromptClose}
+              prompt={output.prompt}
+              output={text}
+            />
+          </>
+        )}
+        <span className="action" onClick={() => handleClick(true)}>
           👍
         </span>
-        <span className="rating" onClick={() => handleClick(false)}>
+        <span className="action" onClick={() => handleClick(false)}>
           👎
         </span>
       </div>
@@ -140,11 +159,35 @@ function EvalOutputCell({
   );
 }
 
-function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smallText: string }) {
+function TableHeader({
+  text,
+  maxLength,
+  smallText,
+  expandedText,
+}: TruncatedTextProps & { smallText: string; expandedText?: string }) {
+  const [openPrompt, setOpen] = React.useState(false);
+  const handlePromptOpen = () => {
+    setOpen(true);
+  };
+  const handlePromptClose = () => {
+    setOpen(false);
+  };
   return (
     <div>
       <TruncatedText text={text} maxLength={maxLength} />
-      <span className="smalltext">{smallText}</span>
+      {expandedText && (
+        <>
+          <span className="action" onClick={handlePromptOpen}>
+            🔎
+          </span>
+          <EvalOutputPromptDialog
+            open={openPrompt}
+            onClose={handlePromptClose}
+            prompt={expandedText}
+          />
+        </>
+      )}
+      <div className="smalltext">{smallText}</div>
     </div>
   );
 }
@@ -233,11 +276,13 @@ export default function ResultsTable({
               numGood[idx] === highestPassingCount && highestPassingCount !== 0;
             const columnId = `Prompt ${idx + 1}`;
             const isChecked = failureFilter[columnId] || false;
+            // TODO(ian): prompt string support for backwards compatibility, remove after 0.17.0
             return (
               <>
                 <TableHeader
                   smallText={`Prompt ${idx + 1}`}
-                  text={prompt}
+                  text={typeof prompt === 'string' ? prompt : prompt.display}
+                  expandedText={typeof prompt === 'string' ? undefined : prompt.raw}
                   maxLength={maxTextLength}
                 />
                 {filterMode === 'failures' && (

package/src/web/client/src/types.ts

@@ -1,5 +1,10 @@
+type Prompt = {
+  display: string;
+  raw: string;
+};
+
 export type EvalHead = {
-  prompts: string[];
+  prompts: Prompt[];
   vars: string[];
 };
 
@@ -7,6 +12,7 @@ export type EvalRowOutput = {
   pass: boolean;
   score: number;
   text: string | object;
+  prompt: string;
 };
 
 export type EvalRow = {