promptfoo 0.14.1 → 0.15.0

package/src/web/client/src/ResultsTable.tsx

@@ -16,17 +16,45 @@ import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
-import type { EvalRow, FilterMode } from './types.js';
+import type { EvalRow, EvalRowOutput, FilterMode } from './types.js';
 
 import './ResultsTable.css';
 
+function formatRowOutput(output: EvalRowOutput | string) {
+  if (typeof output === 'string') {
+    // Backwards compatibility for 0.15.0 breaking change. Remove eventually.
+    const pass = output.startsWith('[PASS]');
+    let text = output;
+    if (output.startsWith('[PASS]')) {
+      text = text.slice('[PASS]'.length);
+    } else if (output.startsWith('[FAIL]')) {
+      text = text.slice('[FAIL]'.length);
+    }
+    return {
+      text,
+      pass,
+      score: pass ? 1 : 0,
+    };
+  }
+  return output;
+}
+
+function scoreToString(score: number) {
+  if (score === 0 || score === 1) {
+    // Don't show boolean scores.
+    return '';
+  }
+  return `(${score.toFixed(2)})`;
+}
+
 interface TruncatedTextProps {
-  text: string;
+  text: string | number;
   maxLength: number;
 }
 
-function TruncatedText({ text, maxLength }: TruncatedTextProps) {
+function TruncatedText({ text: rawText, maxLength }: TruncatedTextProps) {
   const [isTruncated, setIsTruncated] = React.useState<boolean>(true);
+  const text = String(rawText);
 
   const toggleTruncate = () => {
     setIsTruncated(!isTruncated);
@@ -59,29 +87,26 @@ function TruncatedText({ text, maxLength }: TruncatedTextProps) {
 }
 
 interface PromptOutputProps {
-  text: string;
+  output: EvalRowOutput;
   maxTextLength: number;
   rowIndex: number;
   promptIndex: number;
   onRating: (rowIndex: number, promptIndex: number, isPass: boolean) => void;
 }
 
-function PromptOutput({ text, maxTextLength, rowIndex, promptIndex, onRating }: PromptOutputProps) {
-  const isPass = text.startsWith('[PASS] ');
-  const isFail = text.startsWith('[FAIL] ');
+function PromptOutput({
+  output,
+  maxTextLength,
+  rowIndex,
+  promptIndex,
+  onRating,
+}: PromptOutputProps) {
+  let text = String(output.text);
   let chunks: string[] = [];
-  if (isPass) {
-    text = text.substring(7);
-  } else if (isFail) {
-    if (text.includes('---')) {
-      chunks = text.split('---');
-      text = chunks.slice(1).join('---');
-    } else {
-      chunks = ['[FAIL]'];
-      if (text.startsWith('[FAIL] ')) {
-        text = text.substring(7);
-      }
-    }
+  if (!output.pass && text.includes('---')) {
+    // TODO(ian): Plumb through failure message instead of parsing it out.
+    chunks = text.split('---');
+    text = chunks.slice(1).join('---');
   }
 
   const handleClick = (isPass: boolean) => {
@@ -91,8 +116,16 @@ function PromptOutput({ text, maxTextLength, rowIndex, promptIndex, onRating }:
   return (
     <>
       <div className="cell">
-        {isPass && <div className="status pass">[PASS]</div>}
-        {isFail && <div className="status fail">{chunks[0]}</div>}{' '}
+        {output.pass && (
+          <div className="status pass">
+            PASS <span className="score">{scoreToString(output.score)}</span>
+          </div>
+        )}
+        {!output.pass && (
+          <div className="status fail">
+            [FAIL<span className="score">{scoreToString(output.score)}</span>] {chunks[0]}
+          </div>
+        )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
       <div className="cell-rating">
@@ -136,10 +169,9 @@ export default function ResultsTable({
   const { table, setTable } = useStore();
   invariant(table, 'Table should be defined');
   const { head, body } = table;
-  // TODO(ian): Correctly plumb through the results instead of parsing the string.
   const numGood = head.prompts.map((_, idx) =>
     body.reduce((acc, row) => {
-      return acc + (row.outputs[idx].startsWith('[PASS]') ? 1 : 0);
+      return acc + (row.outputs[idx].pass ? 1 : 0);
     }, 0),
   );
 
@@ -147,10 +179,8 @@ export default function ResultsTable({
     const updatedData = [...body];
     const updatedRow = { ...updatedData[rowIndex] };
     const updatedOutputs = [...updatedRow.outputs];
-    const updatedOutput = isPass
-      ? `[PASS] ${updatedOutputs[promptIndex].replace(/^\[(PASS|FAIL)\] /, '')}`
-      : `[FAIL] ${updatedOutputs[promptIndex].replace(/^\[(PASS|FAIL)\] /, '')}`;
-    updatedOutputs[promptIndex] = updatedOutput;
+    updatedOutputs[promptIndex].pass = isPass;
+    updatedOutputs[promptIndex].score = isPass ? 1 : 0;
     updatedRow.outputs = updatedOutputs;
     updatedData[rowIndex] = updatedRow;
     setTable({
@@ -190,7 +220,7 @@ export default function ResultsTable({
       id: 'prompts',
       header: () => <span>Outputs</span>,
       columns: head.prompts.map((prompt, idx) =>
-        columnHelper.accessor((row: EvalRow) => row.outputs[idx], {
+        columnHelper.accessor((row: EvalRow) => formatRowOutput(row.outputs[idx]), {
           id: `Prompt ${idx + 1}`,
           header: () => {
             const pct = ((numGood[idx] / body.length) * 100.0).toFixed(2);
@@ -229,7 +259,7 @@ export default function ResultsTable({
           },
           cell: (info: CellContext<EvalRow, string>) => (
             <PromptOutput
-              text={info.getValue()}
+              output={info.getValue() as unknown as EvalRowOutput}
               maxTextLength={maxTextLength}
               rowIndex={info.row.index}
               promptIndex={idx}
@@ -249,7 +279,7 @@ export default function ResultsTable({
       return body.filter((row) => {
         return row.outputs.some((output, idx) => {
           const columnId = `Prompt ${idx + 1}`;
-          const isFail = output.startsWith('[FAIL] ');
+          const isFail = !output.pass;
           return failureFilter[columnId] && isFail;
         });
       });

package/src/web/client/src/types.ts

@@ -3,8 +3,14 @@ export type EvalHead = {
   vars: string[];
 };
 
+export type EvalRowOutput = {
+  pass: boolean;
+  score: number;
+  text: string;
+};
+
 export type EvalRow = {
-  outputs: string[]; // var inputs
+  outputs: EvalRowOutput[];
   vars: string[]; // model outputs
 };
 

package/src/web/server.ts

@@ -44,7 +44,7 @@ export function init(port = 15500) {
       latestJsonPath,
       debounce((event: string) => {
         if (event === 'change') {
-          socket.emit('update', readLatestJson);
+          socket.emit('update', readLatestJson());
         }
       }, 250),
     );