promptfoo 0.18.1 → 0.18.3

package/src/web/client/src/EvalOutputPromptDialog.tsx

@@ -1,4 +1,5 @@
 import { useState, useEffect } from 'react';
+import Box from '@mui/material/Box';
 import Button from '@mui/material/Button';
 import Dialog from '@mui/material/Dialog';
 import DialogActions from '@mui/material/DialogActions';
@@ -8,12 +9,60 @@ import TextareaAutosize from '@mui/base/TextareaAutosize';
 import IconButton from '@mui/material/IconButton';
 import ContentCopyIcon from '@mui/icons-material/ContentCopy';
 import CheckIcon from '@mui/icons-material/Check';
+import Table from '@mui/material/Table';
+import TableBody from '@mui/material/TableBody';
+import TableCell from '@mui/material/TableCell';
+import TableContainer from '@mui/material/TableContainer';
+import TableHead from '@mui/material/TableHead';
+import TableRow from '@mui/material/TableRow';
+import Typography from '@mui/material/Typography';
+
+import type { GradingResult } from '../../../types';
 
 interface EvalOutputPromptDialogProps {
   open: boolean;
   onClose: () => void;
   prompt: string;
   output?: string;
+  gradingResults?: GradingResult[];
+}
+
+function AssertionResults({ gradingResults }: { gradingResults?: GradingResult[] }) {
+  if (!gradingResults) {
+    return null;
+  }
+
+  return (
+    <Box mt={2}>
+      <Typography variant="subtitle1">Assertions</Typography>
+      <TableContainer>
+        <Table>
+          <TableHead>
+            <TableRow>
+              <TableCell style={{ fontWeight: 'bold' }}>Pass</TableCell>
+              <TableCell style={{ fontWeight: 'bold' }}>Score</TableCell>
+              <TableCell style={{ fontWeight: 'bold' }}>Type</TableCell>
+              <TableCell style={{ fontWeight: 'bold' }}>Value</TableCell>
+              <TableCell style={{ fontWeight: 'bold' }}>Reason</TableCell>
+            </TableRow>
+          </TableHead>
+          <TableBody>
+            {gradingResults.map((result, i) => (
+              <TableRow key={i}>
+                <TableCell>{result.pass ? '✅' : '❌'}</TableCell>
+                <TableCell>{result.score}</TableCell>
+                <TableCell>{result.assertion?.type || ''}</TableCell>
+                <TableCell>
+                  {result.assertion?.value ? String(result.assertion.value) : '-'}
+                </TableCell>
+                <TableCell>{result.reason}</TableCell>
+              </TableRow>
+            ))}
+          </TableBody>
+        </Table>
+      </TableContainer>
+    </Box>
+  );
 }
 
 export default function EvalOutputPromptDialog({
@@ -21,6 +70,7 @@ export default function EvalOutputPromptDialog({
   onClose,
   prompt,
   output,
+  gradingResults,
 }: EvalOutputPromptDialogProps) {
   const [copied, setCopied] = useState(false);
 
@@ -35,28 +85,40 @@ export default function EvalOutputPromptDialog({
 
   return (
     <Dialog open={open} onClose={onClose} fullWidth maxWidth="lg">
-      <DialogTitle>Prompt</DialogTitle>
+      <DialogTitle>Details</DialogTitle>
       <DialogContent>
-        <TextareaAutosize readOnly value={prompt} style={{ width: '100%', padding: '0.75rem' }} />
-        <IconButton
-          onClick={() => copyToClipboard(prompt)}
-          style={{ position: 'absolute', right: '10px', top: '10px' }}
-        >
-          {copied ? <CheckIcon /> : <ContentCopyIcon />}
-        </IconButton>
-      </DialogContent>
-      {output && (
-        <>
-          <DialogTitle>Output</DialogTitle>
-          <DialogContent>
+        <Box mb={2}>
+          <Typography variant="subtitle1" style={{ marginBottom: '1rem' }}>
+            Prompt
+          </Typography>
+          <TextareaAutosize
+            readOnly
+            value={prompt}
+            style={{ width: '100%', padding: '0.75rem' }}
+            maxRows={20}
+          />
+          <IconButton
+            onClick={() => copyToClipboard(prompt)}
+            style={{ position: 'absolute', right: '10px', top: '10px' }}
+          >
+            {copied ? <CheckIcon /> : <ContentCopyIcon />}
+          </IconButton>
+        </Box>
+        {output && (
+          <Box my={2}>
+            <Typography variant="subtitle1" style={{ marginBottom: '1rem', marginTop: '1rem' }}>
+              Output
+            </Typography>
             <TextareaAutosize
               readOnly
+              maxRows={20}
               value={output}
               style={{ width: '100%', padding: '0.75rem' }}
             />
-          </DialogContent>
-        </>
-      )}
+          </Box>
+        )}
+        <AssertionResults gradingResults={gradingResults} />
+      </DialogContent>
       <DialogActions>
         <Button onClick={onClose}>Close</Button>
       </DialogActions>

package/src/web/client/src/ResultsTable.tsx

@@ -155,6 +155,7 @@ function EvalOutputCell({
               open={openPrompt}
               onClose={handlePromptClose}
               prompt={output.prompt}
+              gradingResults={output.gradingResult?.componentResults}
               output={text}
             />
           </>
@@ -223,12 +224,25 @@ export default function ResultsTable({
   const { table, setTable } = useStore();
   invariant(table, 'Table should be defined');
   const { head, body } = table;
-  const numGood = head.prompts.map((_, idx) =>
+  const numGoodTests = head.prompts.map((_, idx) =>
     body.reduce((acc, row) => {
       return acc + (row.outputs[idx].pass ? 1 : 0);
     }, 0),
   );
 
+  const numAsserts = head.prompts.map((_, idx) =>
+    body.reduce((acc, row) => {
+      return acc + (row.outputs[idx].gradingResult?.componentResults?.length || 0);
+    }, 0),
+  );
+
+  const numGoodAsserts = head.prompts.map((_, idx) =>
+    body.reduce((acc, row) => {
+      const componentResults = row.outputs[idx].gradingResult?.componentResults;
+      return acc + (componentResults ? componentResults.filter((r) => r.pass).length : 0);
+    }, 0),
+  );
+
   const handleRating = (rowIndex: number, promptIndex: number, isPass: boolean) => {
     const updatedData = [...body];
     const updatedRow = { ...updatedData[rowIndex] };
@@ -243,10 +257,13 @@ export default function ResultsTable({
     });
   };
 
-  const highestPassingIndex = numGood.reduce((maxIndex, currentPassCount, currentIndex, array) => {
-    return currentPassCount > array[maxIndex] ? currentIndex : maxIndex;
-  }, 0);
-  const highestPassingCount = numGood[highestPassingIndex];
+  const highestPassingIndex = numGoodTests.reduce(
+    (maxIndex, currentPassCount, currentIndex, array) => {
+      return currentPassCount > array[maxIndex] ? currentIndex : maxIndex;
+    },
+    0,
+  );
+  const highestPassingCount = numGoodTests[highestPassingIndex];
   const columnHelper = createColumnHelper<EvalRow>();
   const columns = [
     columnHelper.group({
@@ -282,9 +299,9 @@ export default function ResultsTable({
         columnHelper.accessor((row: EvalRow) => formatRowOutput(row.outputs[idx]), {
           id: `Prompt ${idx + 1}`,
           header: () => {
-            const pct = ((numGood[idx] / body.length) * 100.0).toFixed(2);
+            const pct = ((numGoodTests[idx] / body.length) * 100.0).toFixed(2);
             const isHighestPassing =
-              numGood[idx] === highestPassingCount && highestPassingCount !== 0;
+              numGoodTests[idx] === highestPassingCount && highestPassingCount !== 0;
             const columnId = `Prompt ${idx + 1}`;
             const isChecked = failureFilter[columnId] || false;
             // TODO(ian): prompt string support for backwards compatibility, remove after 0.17.0
@@ -313,12 +330,18 @@ export default function ResultsTable({
                   />
                 )}
                 <div className={`summary ${isHighestPassing ? 'highlight' : ''}`}>
-                  Passing: <strong>{pct}%</strong> ({numGood[idx]} / {body.length})
+                  Passing: <strong>{pct}%</strong> ({numGoodTests[idx]}/{body.length} cases
+                  {numAsserts[idx] ? (
+                    <span>
+                      , {numGoodAsserts[idx]}/{numAsserts[idx]} asserts
+                    </span>
+                  ) : null}
+                  )
                 </div>
               </>
             );
           },
-          cell: (info: CellContext<EvalRow, string>) => (
+          cell: (info: CellContext<EvalRow, EvalRowOutput>) => (
             <EvalOutputCell
               output={info.getValue() as unknown as EvalRowOutput}
               maxTextLength={maxTextLength}

package/src/web/client/src/ResultsView.tsx

@@ -86,7 +86,7 @@ export default function ResultsView({ recentFiles, onRecentFileSelected }: Resul
     setFailureFilter(newFailureFilter);
   };
 
-  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-all');
+  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-word');
   const handleWordBreakChange = (event: React.ChangeEvent<HTMLInputElement>) => {
     setWordBreak(event.target.checked ? 'break-all' : 'break-word');
   };

package/src/web/client/src/types.ts

@@ -1,4 +1,4 @@
-import { TokenUsage } from '../../../types';
+import { EvaluateResult, TokenUsage } from '../../../types';
 
 type Prompt = {
   display: string;
@@ -10,6 +10,7 @@ export type EvalHead = {
   vars: string[];
 };
 
+// TODO(ian): Remove this and replace with EvaluateResult
 export type EvalRowOutput = {
   pass: boolean;
   score: number;
@@ -17,6 +18,7 @@ export type EvalRowOutput = {
   prompt: string;
   latencyMs: number;
   tokenUsage?: Partial<TokenUsage>;
+  gradingResult: EvaluateResult['gradingResult'];
 };
 
 export type EvalRow = {