promptfoo 0.10.0 → 0.11.0

package/src/web/client/src/ResultsTable.tsx

@@ -9,12 +9,14 @@ import {
   getCoreRowModel,
   useReactTable,
 } from '@tanstack/react-table';
+import Checkbox from '@mui/material/Checkbox';
+import FormControlLabel from '@mui/material/FormControlLabel';
 
 import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
-import type { EvalRow } from './types.js';
+import type { EvalRow, FilterMode } from './types.js';
 
 import './ResultsTable.css';
 
@@ -114,12 +116,23 @@ function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smal
   );
 }
 
-interface ResultsViewProps {
+interface ResultsTableProps {
   maxTextLength: number;
   columnVisibility: VisibilityState;
+  wordBreak: 'break-word' | 'break-all';
+  filterMode: FilterMode;
+  failureFilter: { [key: string]: boolean };
+  onFailureFilterToggle: (columnId: string, checked: boolean) => void;
 }
 
-export default function ResultsTable({ maxTextLength, columnVisibility }: ResultsViewProps) {
+export default function ResultsTable({
+  maxTextLength,
+  columnVisibility,
+  wordBreak,
+  filterMode,
+  failureFilter,
+  onFailureFilterToggle,
+}: ResultsTableProps) {
   const { table, setTable } = useStore();
   invariant(table, 'Table should be defined');
   const { head, body } = table;
@@ -146,6 +159,10 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
     });
   };
 
+  const highestPassingIndex = numGood.reduce((maxIndex, currentPassCount, currentIndex, array) => {
+    return currentPassCount > array[maxIndex] ? currentIndex : maxIndex;
+  }, 0);
+  const highestPassingCount = numGood[highestPassingIndex];
   const columnHelper = createColumnHelper<EvalRow>();
   const columns = [
     columnHelper.group({
@@ -164,6 +181,8 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
           cell: (info: CellContext<EvalRow, string>) => (
             <TruncatedText text={info.getValue()} maxLength={maxTextLength} />
           ),
+          // Minimize the size of Variable columns.
+          size: 50,
         }),
       ),
     }),
@@ -173,16 +192,41 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
       columns: head.prompts.map((prompt, idx) =>
         columnHelper.accessor((row: EvalRow) => row.outputs[idx], {
           id: `Prompt ${idx + 1}`,
-          header: () => (
-            <>
-              <TableHeader
-                smallText={`Prompt ${idx + 1}`}
-                text={prompt}
-                maxLength={maxTextLength}
-              />
-              {numGood[idx]} / {body.length} 👍
-            </>
-          ),
+          header: () => {
+            const pct = ((numGood[idx] / body.length) * 100.0).toFixed(2);
+            const isHighestPassing =
+              numGood[idx] === highestPassingCount && highestPassingCount !== 0;
+            const columnId = `Prompt ${idx + 1}`;
+            const isChecked = failureFilter[columnId] || false;
+            return (
+              <>
+                <TableHeader
+                  smallText={`Prompt ${idx + 1}`}
+                  text={prompt}
+                  maxLength={maxTextLength}
+                />
+                {filterMode === 'failures' && (
+                  <FormControlLabel
+                    sx={{
+                      '& .MuiFormControlLabel-label': {
+                        fontSize: '0.75rem',
+                      },
+                    }}
+                    control={
+                      <Checkbox
+                        checked={isChecked}
+                        onChange={(event) => onFailureFilterToggle(columnId, event.target.checked)}
+                      />
+                    }
+                    label="Show failures"
+                  />
+                )}
+                <div className={`summary ${isHighestPassing ? 'highlight' : ''}`}>
+                  Passing: <strong>{pct}%</strong> ({numGood[idx]} / {body.length})
+                </div>
+              </>
+            );
+          },
           cell: (info: CellContext<EvalRow, string>) => (
             <PromptOutput
               text={info.getValue()}
@@ -197,8 +241,24 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
     }),
   ];
 
+  const filteredBody = React.useMemo(() => {
+    if (filterMode === 'failures') {
+      if (Object.values(failureFilter).every((v) => !v)) {
+        return body;
+      }
+      return body.filter((row) => {
+        return row.outputs.some((output, idx) => {
+          const columnId = `Prompt ${idx + 1}`;
+          const isFail = output.startsWith('[FAIL] ');
+          return failureFilter[columnId] && isFail;
+        });
+      });
+    }
+    return body;
+  }, [body, failureFilter, filterMode]);
+
   const reactTable = useReactTable({
-    data: body,
+    data: filteredBody,
     columns,
     columnResizeMode: 'onChange',
     getCoreRowModel: getCoreRowModel(),
@@ -209,32 +269,38 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
   });
 
   return (
-    <table>
+    <table
+      style={{
+        wordBreak,
+      }}
+    >
       <thead>
         {reactTable.getHeaderGroups().map((headerGroup) => (
           <tr key={headerGroup.id} className="header">
-            {headerGroup.headers.map((header) => (
-              <th
-                {...{
-                  key: header.id,
-                  colSpan: header.colSpan,
-                  style: {
-                    width: header.getSize(),
-                  },
-                }}
-              >
-                {header.isPlaceholder
-                  ? null
-                  : flexRender(header.column.columnDef.header, header.getContext())}
-                <div
+            {headerGroup.headers.map((header) => {
+              return (
+                <th
                   {...{
-                    onMouseDown: header.getResizeHandler(),
-                    onTouchStart: header.getResizeHandler(),
-                    className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                    key: header.id,
+                    colSpan: header.colSpan,
+                    style: {
+                      width: header.getSize(),
+                    },
                   }}
-                />
-              </th>
-            ))}
+                >
+                  {header.isPlaceholder
+                    ? null
+                    : flexRender(header.column.columnDef.header, header.getContext())}
+                  <div
+                    {...{
+                      onMouseDown: header.getResizeHandler(),
+                      onTouchStart: header.getResizeHandler(),
+                      className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                    }}
+                  />
+                </th>
+              );
+            })}
           </tr>
         ))}
       </thead>

package/src/web/client/src/ResultsView.tsx

@@ -2,22 +2,32 @@ import * as React from 'react';
 
 import invariant from 'tiny-invariant';
 import Box from '@mui/material/Box';
-import Paper from '@mui/material/Box';
-import Stack from '@mui/material/Stack';
-import Slider from '@mui/material/Slider';
-import Typography from '@mui/material/Typography';
-import OutlinedInput from '@mui/material/OutlinedInput';
-import InputLabel from '@mui/material/InputLabel';
-import MenuItem from '@mui/material/MenuItem';
+import Checkbox from '@mui/material/Checkbox';
 import FormControl from '@mui/material/FormControl';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import InputLabel from '@mui/material/InputLabel';
 import ListItemText from '@mui/material/ListItemText';
+import MenuItem from '@mui/material/MenuItem';
+import OutlinedInput from '@mui/material/OutlinedInput';
+import Paper from '@mui/material/Box';
 import Select, { SelectChangeEvent } from '@mui/material/Select';
-import Checkbox from '@mui/material/Checkbox';
+import Slider from '@mui/material/Slider';
+import Stack from '@mui/material/Stack';
+import Tooltip from '@mui/material/Tooltip';
+import Typography from '@mui/material/Typography';
+import { styled } from '@mui/system';
 
 import ResultsTable from './ResultsTable.js';
 import { useStore } from './store.js';
 
 import type { VisibilityState } from '@tanstack/table-core';
+import { FilterMode } from './types.js';
+
+const ResponsiveStack = styled(Stack)(({ theme }) => ({
+  [theme.breakpoints.down('sm')]: {
+    flexDirection: 'column',
+  },
+}));
 
 export default function ResultsView() {
   const { table } = useStore();
@@ -25,6 +35,29 @@ export default function ResultsView() {
   const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
   const [selectedColumns, setSelectedColumns] = React.useState<string[]>([]);
 
+  const [failureFilter, setFailureFilter] = React.useState<{ [key: string]: boolean }>({});
+  const handleFailureFilterToggle = (columnId: string, checked: boolean) => {
+    setFailureFilter((prevFailureFilter) => ({ ...prevFailureFilter, [columnId]: checked }));
+  };
+
+  const [filterMode, setFilterMode] = React.useState<FilterMode>('all');
+  const handleFilterModeChange = (event: SelectChangeEvent<unknown>) => {
+    const mode = event.target.value as FilterMode;
+    setFilterMode(mode);
+
+    const newFailureFilter: { [key: string]: boolean } = {};
+    head.prompts.forEach((_, idx) => {
+      const columnId = `Prompt ${idx + 1}`;
+      newFailureFilter[columnId] = mode === 'failures';
+    });
+    setFailureFilter(newFailureFilter);
+  };
+
+  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-all');
+  const handleWordBreakChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    setWordBreak(event.target.checked ? 'break-all' : 'break-word');
+  };
+
   invariant(table, 'Table data must be loaded before rendering ResultsView');
   const { head } = table;
 
@@ -71,7 +104,7 @@ export default function ResultsView() {
   return (
     <div>
       <Paper py="md">
-        <Stack direction="row" spacing={2} alignItems="center">
+        <ResponsiveStack direction="row" spacing={8} alignItems="center">
           <Box>
             <FormControl sx={{ m: 1, minWidth: 300 }} size="small">
               <InputLabel id="visible-columns-label">Visible columns</InputLabel>
@@ -93,6 +126,21 @@ export default function ResultsView() {
               </Select>
             </FormControl>
           </Box>
+          <Box>
+            <FormControl sx={{ minWidth: 180 }} size="small">
+              <InputLabel id="failure-filter-mode-label">Filter</InputLabel>
+              <Select
+                labelId="filter-mode-label"
+                id="filter-mode"
+                value={filterMode}
+                onChange={handleFilterModeChange}
+                label="Filter"
+              >
+                <MenuItem value="all">Show all results</MenuItem>
+                <MenuItem value="failures">Show only failures</MenuItem>
+              </Select>
+            </FormControl>
+          </Box>
           <Box>
             <Typography mt={2}>Max text length: {maxTextLength}</Typography>
             <Slider
@@ -102,9 +150,26 @@ export default function ResultsView() {
               onChange={(_, val: number | number[]) => setMaxTextLength(val as number)}
             />
           </Box>
-        </Stack>
+          <Box>
+            <Tooltip title="Forcing line breaks makes it easier to adjust column widths to your liking">
+              <FormControlLabel
+                control={
+                  <Checkbox checked={wordBreak === 'break-all'} onChange={handleWordBreakChange} />
+                }
+                label="Force line breaks"
+              />
+            </Tooltip>
+          </Box>
+        </ResponsiveStack>
       </Paper>
-      <ResultsTable maxTextLength={maxTextLength} columnVisibility={columnVisibility} />
+      <ResultsTable
+        maxTextLength={maxTextLength}
+        columnVisibility={columnVisibility}
+        wordBreak={wordBreak}
+        filterMode={filterMode}
+        failureFilter={failureFilter}
+        onFailureFilterToggle={handleFailureFilterToggle}
+      />
     </div>
   );
 }

package/src/web/client/src/index.css

@@ -15,6 +15,9 @@
   --pass-color: green;
   --fail-color: #ad0000;
   --smalltext-color: gray;
+  --success-background-color: #d1ffd7;
+  --variable-background-color: #f7f7f7;
+  --header-background-color: #fffdf7;
 }
 
 /* Dark mode colors */
@@ -38,6 +41,9 @@
   --pass-color: #4caf50;
   --fail-color: #f44336;
   --smalltext-color: #888888;
+  --success-background-color: #216d2b;
+  --variable-background-color: #333;
+  --header-background-color: #333;
 }
 
 html {

package/src/web/client/src/types.ts

@@ -12,3 +12,5 @@ export type EvalTable = {
   head: EvalHead;
   body: EvalRow[];
 };
+
+export type FilterMode = 'all' | 'failures';

package/dist/web/client/assets/index-9a9ba400.css

@@ -1 +0,0 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);word-break:break-all;vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:#f8fbff}tr.header{background-color:#fffdf7}th,.th{padding:1rem;position:relative;text-align:center;font-weight:semi-bold}tr .cell-rating{visibility:hidden;position:absolute;bottom:1.25rem;right:-1rem;line-height:0;font-size:1.75rem}tr:hover .cell-rating{visibility:visible}tr .cell-rating .rating{cursor:pointer;margin-right:1rem}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}td .status{margin-bottom:.5rem;font-weight:700}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}