promptfoo 0.9.0 → 0.11.0

package/src/web/client/src/ResultsTable.css

@@ -40,18 +40,28 @@ td,
 .td {
   position: relative;
   box-shadow: inset 0 0 0 1px var(--border-color);
-  word-break: break-all;
   vertical-align: top;
 
   padding: 1.5rem;
 }
 
+th.variable,
+.th.variable,
+td.variable,
+.td.variable {
+  background-color: var(--variable-background-color);
+}
+
+tr.header {
+  background-color: var(--header-background-color);
+}
+
 th,
 .th {
   padding: 1rem;
   position: relative;
   text-align: center;
-  font-weight: semi-bold;
+  vertical-align: bottom;
 }
 
 tr .cell {
@@ -61,7 +71,7 @@ tr .cell-rating {
   visibility: hidden;
   position: absolute;
   bottom: 1.25rem;
-  right: -1rem;
+  right: 0;
   line-height: 0;
   font-size: 1.75rem;
 }
@@ -72,7 +82,10 @@ tr:hover .cell-rating {
 
 tr .cell-rating .rating {
   cursor: pointer;
-  margin-right: 1rem;
+}
+
+tr .cell-rating .rating:first-child {
+  margin-right: 0.5rem;
 }
 
 th .smalltext {
@@ -86,6 +99,16 @@ th:hover .smalltext {
   visibility: visible;
 }
 
+th .summary {
+  font-weight: normal;
+  font-size: 0.8rem;
+  padding: 0.25rem;
+}
+
+th .summary.highlight {
+  background-color: var(--success-background-color);
+}
+
 td,
 .td {
 }
@@ -103,6 +126,14 @@ td .fail {
   color: var(--fail-color);
 }
 
+.first-prompt-col {
+  border-left: 2px solid #888;
+}
+
+.first-prompt-row {
+  border-top: 2px solid #888;
+}
+
 .resizer {
   position: absolute;
   right: 0;

package/src/web/client/src/ResultsTable.tsx

@@ -9,12 +9,14 @@ import {
   getCoreRowModel,
   useReactTable,
 } from '@tanstack/react-table';
+import Checkbox from '@mui/material/Checkbox';
+import FormControlLabel from '@mui/material/FormControlLabel';
 
 import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
-import type { EvalRow } from './types.js';
+import type { EvalRow, FilterMode } from './types.js';
 
 import './ResultsTable.css';
 
@@ -114,12 +116,23 @@ function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smal
   );
 }
 
-interface ResultsViewProps {
+interface ResultsTableProps {
   maxTextLength: number;
   columnVisibility: VisibilityState;
+  wordBreak: 'break-word' | 'break-all';
+  filterMode: FilterMode;
+  failureFilter: { [key: string]: boolean };
+  onFailureFilterToggle: (columnId: string, checked: boolean) => void;
 }
 
-export default function ResultsTable({ maxTextLength, columnVisibility }: ResultsViewProps) {
+export default function ResultsTable({
+  maxTextLength,
+  columnVisibility,
+  wordBreak,
+  filterMode,
+  failureFilter,
+  onFailureFilterToggle,
+}: ResultsTableProps) {
   const { table, setTable } = useStore();
   invariant(table, 'Table should be defined');
   const { head, body } = table;
@@ -146,36 +159,12 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
     });
   };
 
+  const highestPassingIndex = numGood.reduce((maxIndex, currentPassCount, currentIndex, array) => {
+    return currentPassCount > array[maxIndex] ? currentIndex : maxIndex;
+  }, 0);
+  const highestPassingCount = numGood[highestPassingIndex];
   const columnHelper = createColumnHelper<EvalRow>();
   const columns = [
-    columnHelper.group({
-      id: 'prompts',
-      header: () => <span>Prompts</span>,
-      columns: head.prompts.map((prompt, idx) =>
-        columnHelper.accessor((row: EvalRow) => row.outputs[idx], {
-          id: `Prompt ${idx + 1}`,
-          header: () => (
-            <>
-              <TableHeader
-                smallText={`Prompt ${idx + 1}`}
-                text={prompt}
-                maxLength={maxTextLength}
-              />
-              {numGood[idx]} / {body.length} 👍
-            </>
-          ),
-          cell: (info: CellContext<EvalRow, string>) => (
-            <PromptOutput
-              text={info.getValue()}
-              maxTextLength={maxTextLength}
-              rowIndex={info.row.index}
-              promptIndex={idx}
-              onRating={handleRating}
-            />
-          ),
-        }),
-      ),
-    }),
     columnHelper.group({
       id: 'vars',
       header: () => <span>Variables</span>,
@@ -192,13 +181,84 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
           cell: (info: CellContext<EvalRow, string>) => (
             <TruncatedText text={info.getValue()} maxLength={maxTextLength} />
           ),
+          // Minimize the size of Variable columns.
+          size: 50,
+        }),
+      ),
+    }),
+    columnHelper.group({
+      id: 'prompts',
+      header: () => <span>Outputs</span>,
+      columns: head.prompts.map((prompt, idx) =>
+        columnHelper.accessor((row: EvalRow) => row.outputs[idx], {
+          id: `Prompt ${idx + 1}`,
+          header: () => {
+            const pct = ((numGood[idx] / body.length) * 100.0).toFixed(2);
+            const isHighestPassing =
+              numGood[idx] === highestPassingCount && highestPassingCount !== 0;
+            const columnId = `Prompt ${idx + 1}`;
+            const isChecked = failureFilter[columnId] || false;
+            return (
+              <>
+                <TableHeader
+                  smallText={`Prompt ${idx + 1}`}
+                  text={prompt}
+                  maxLength={maxTextLength}
+                />
+                {filterMode === 'failures' && (
+                  <FormControlLabel
+                    sx={{
+                      '& .MuiFormControlLabel-label': {
+                        fontSize: '0.75rem',
+                      },
+                    }}
+                    control={
+                      <Checkbox
+                        checked={isChecked}
+                        onChange={(event) => onFailureFilterToggle(columnId, event.target.checked)}
+                      />
+                    }
+                    label="Show failures"
+                  />
+                )}
+                <div className={`summary ${isHighestPassing ? 'highlight' : ''}`}>
+                  Passing: <strong>{pct}%</strong> ({numGood[idx]} / {body.length})
+                </div>
+              </>
+            );
+          },
+          cell: (info: CellContext<EvalRow, string>) => (
+            <PromptOutput
+              text={info.getValue()}
+              maxTextLength={maxTextLength}
+              rowIndex={info.row.index}
+              promptIndex={idx}
+              onRating={handleRating}
+            />
+          ),
         }),
       ),
     }),
   ];
 
+  const filteredBody = React.useMemo(() => {
+    if (filterMode === 'failures') {
+      if (Object.values(failureFilter).every((v) => !v)) {
+        return body;
+      }
+      return body.filter((row) => {
+        return row.outputs.some((output, idx) => {
+          const columnId = `Prompt ${idx + 1}`;
+          const isFail = output.startsWith('[FAIL] ');
+          return failureFilter[columnId] && isFail;
+        });
+      });
+    }
+    return body;
+  }, [body, failureFilter, filterMode]);
+
   const reactTable = useReactTable({
-    data: body,
+    data: filteredBody,
     columns,
     columnResizeMode: 'onChange',
     getCoreRowModel: getCoreRowModel(),
@@ -209,52 +269,72 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
   });
 
   return (
-    <table>
+    <table
+      style={{
+        wordBreak,
+      }}
+    >
       <thead>
         {reactTable.getHeaderGroups().map((headerGroup) => (
-          <tr key={headerGroup.id}>
-            {headerGroup.headers.map((header) => (
-              <th
-                {...{
-                  key: header.id,
-                  colSpan: header.colSpan,
-                  style: {
-                    width: header.getSize(),
-                  },
-                }}
-              >
-                {header.isPlaceholder
-                  ? null
-                  : flexRender(header.column.columnDef.header, header.getContext())}
-                <div
+          <tr key={headerGroup.id} className="header">
+            {headerGroup.headers.map((header) => {
+              return (
+                <th
                   {...{
-                    onMouseDown: header.getResizeHandler(),
-                    onTouchStart: header.getResizeHandler(),
-                    className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                    key: header.id,
+                    colSpan: header.colSpan,
+                    style: {
+                      width: header.getSize(),
+                    },
                   }}
-                />
-              </th>
-            ))}
+                >
+                  {header.isPlaceholder
+                    ? null
+                    : flexRender(header.column.columnDef.header, header.getContext())}
+                  <div
+                    {...{
+                      onMouseDown: header.getResizeHandler(),
+                      onTouchStart: header.getResizeHandler(),
+                      className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                    }}
+                  />
+                </th>
+              );
+            })}
           </tr>
         ))}
       </thead>
       <tbody>
-        {reactTable.getRowModel().rows.map((row) => (
-          <tr key={row.id}>
-            {row.getVisibleCells().map((cell) => (
-              <td
-                {...{
-                  key: cell.id,
-                  style: {
-                    width: cell.column.getSize(),
-                  },
-                }}
-              >
-                {flexRender(cell.column.columnDef.cell, cell.getContext())}
-              </td>
-            ))}
-          </tr>
-        ))}
+        {reactTable.getRowModel().rows.map((row, rowIndex) => {
+          let colBorderDrawn = false;
+          return (
+            <tr key={row.id}>
+              {row.getVisibleCells().map((cell) => {
+                const isVariableCol = cell.column.id.startsWith('Variable');
+                const shouldDrawColBorder = !isVariableCol && !colBorderDrawn;
+                if (shouldDrawColBorder) {
+                  colBorderDrawn = true;
+                }
+                const shouldDrawRowBorder = rowIndex === 0 && !isVariableCol;
+                return (
+                  <td
+                    {...{
+                      key: cell.id,
+                      style: {
+                        width: cell.column.getSize(),
+                      },
+                      className: `${isVariableCol ? 'variable' : ''} ${
+                        shouldDrawRowBorder ? 'first-prompt-row' : ''
+                      } ${shouldDrawColBorder ? 'first-prompt-col' : ''}`,
+                    }}
+                  >
+                    {flexRender(cell.column.columnDef.cell, cell.getContext())}
+                  </td>
+                );
+              })}
+            </tr>
+          );
+        })}
       </tbody>
     </table>
   );

package/src/web/client/src/ResultsView.tsx

@@ -2,22 +2,32 @@ import * as React from 'react';
 
 import invariant from 'tiny-invariant';
 import Box from '@mui/material/Box';
-import Paper from '@mui/material/Box';
-import Stack from '@mui/material/Stack';
-import Slider from '@mui/material/Slider';
-import Typography from '@mui/material/Typography';
-import OutlinedInput from '@mui/material/OutlinedInput';
-import InputLabel from '@mui/material/InputLabel';
-import MenuItem from '@mui/material/MenuItem';
+import Checkbox from '@mui/material/Checkbox';
 import FormControl from '@mui/material/FormControl';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import InputLabel from '@mui/material/InputLabel';
 import ListItemText from '@mui/material/ListItemText';
+import MenuItem from '@mui/material/MenuItem';
+import OutlinedInput from '@mui/material/OutlinedInput';
+import Paper from '@mui/material/Box';
 import Select, { SelectChangeEvent } from '@mui/material/Select';
-import Checkbox from '@mui/material/Checkbox';
+import Slider from '@mui/material/Slider';
+import Stack from '@mui/material/Stack';
+import Tooltip from '@mui/material/Tooltip';
+import Typography from '@mui/material/Typography';
+import { styled } from '@mui/system';
 
 import ResultsTable from './ResultsTable.js';
 import { useStore } from './store.js';
 
 import type { VisibilityState } from '@tanstack/table-core';
+import { FilterMode } from './types.js';
+
+const ResponsiveStack = styled(Stack)(({ theme }) => ({
+  [theme.breakpoints.down('sm')]: {
+    flexDirection: 'column',
+  },
+}));
 
 export default function ResultsView() {
   const { table } = useStore();
@@ -25,6 +35,29 @@ export default function ResultsView() {
   const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
   const [selectedColumns, setSelectedColumns] = React.useState<string[]>([]);
 
+  const [failureFilter, setFailureFilter] = React.useState<{ [key: string]: boolean }>({});
+  const handleFailureFilterToggle = (columnId: string, checked: boolean) => {
+    setFailureFilter((prevFailureFilter) => ({ ...prevFailureFilter, [columnId]: checked }));
+  };
+
+  const [filterMode, setFilterMode] = React.useState<FilterMode>('all');
+  const handleFilterModeChange = (event: SelectChangeEvent<unknown>) => {
+    const mode = event.target.value as FilterMode;
+    setFilterMode(mode);
+
+    const newFailureFilter: { [key: string]: boolean } = {};
+    head.prompts.forEach((_, idx) => {
+      const columnId = `Prompt ${idx + 1}`;
+      newFailureFilter[columnId] = mode === 'failures';
+    });
+    setFailureFilter(newFailureFilter);
+  };
+
+  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-all');
+  const handleWordBreakChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    setWordBreak(event.target.checked ? 'break-all' : 'break-word');
+  };
+
   invariant(table, 'Table data must be loaded before rendering ResultsView');
   const { head } = table;
 
@@ -35,8 +68,8 @@ export default function ResultsView() {
     setSelectedColumns(typeof value === 'string' ? value.split(',') : value);
 
     const allColumns = [
-      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
       ...head.vars.map((_, idx) => `Variable ${idx + 1}`),
+      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
     ];
     const newColumnVisibility: VisibilityState = {};
     allColumns.forEach((col) => {
@@ -48,30 +81,30 @@ export default function ResultsView() {
   };
 
   const columnData = [
-    ...head.prompts.map((_, idx) => ({
-      value: `Prompt ${idx + 1}`,
-      label: `Prompt ${idx + 1}`,
-      group: 'Prompts',
-    })),
     ...head.vars.map((_, idx) => ({
       value: `Variable ${idx + 1}`,
       label: `Variable ${idx + 1}`,
       group: 'Variables',
     })),
+    ...head.prompts.map((_, idx) => ({
+      value: `Prompt ${idx + 1}`,
+      label: `Prompt ${idx + 1}`,
+      group: 'Prompts',
+    })),
   ];
 
   // Set all columns as selected by default
   React.useEffect(() => {
     setSelectedColumns([
-      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
       ...head.vars.map((_, idx) => `Variable ${idx + 1}`),
+      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
     ]);
   }, [head]);
 
   return (
     <div>
       <Paper py="md">
-        <Stack direction="row" spacing={2} alignItems="center">
+        <ResponsiveStack direction="row" spacing={8} alignItems="center">
           <Box>
             <FormControl sx={{ m: 1, minWidth: 300 }} size="small">
               <InputLabel id="visible-columns-label">Visible columns</InputLabel>
@@ -93,6 +126,21 @@ export default function ResultsView() {
               </Select>
             </FormControl>
           </Box>
+          <Box>
+            <FormControl sx={{ minWidth: 180 }} size="small">
+              <InputLabel id="failure-filter-mode-label">Filter</InputLabel>
+              <Select
+                labelId="filter-mode-label"
+                id="filter-mode"
+                value={filterMode}
+                onChange={handleFilterModeChange}
+                label="Filter"
+              >
+                <MenuItem value="all">Show all results</MenuItem>
+                <MenuItem value="failures">Show only failures</MenuItem>
+              </Select>
+            </FormControl>
+          </Box>
           <Box>
             <Typography mt={2}>Max text length: {maxTextLength}</Typography>
             <Slider
@@ -102,9 +150,26 @@ export default function ResultsView() {
               onChange={(_, val: number | number[]) => setMaxTextLength(val as number)}
             />
           </Box>
-        </Stack>
+          <Box>
+            <Tooltip title="Forcing line breaks makes it easier to adjust column widths to your liking">
+              <FormControlLabel
+                control={
+                  <Checkbox checked={wordBreak === 'break-all'} onChange={handleWordBreakChange} />
+                }
+                label="Force line breaks"
+              />
+            </Tooltip>
+          </Box>
+        </ResponsiveStack>
       </Paper>
-      <ResultsTable maxTextLength={maxTextLength} columnVisibility={columnVisibility} />
+      <ResultsTable
+        maxTextLength={maxTextLength}
+        columnVisibility={columnVisibility}
+        wordBreak={wordBreak}
+        filterMode={filterMode}
+        failureFilter={failureFilter}
+        onFailureFilterToggle={handleFailureFilterToggle}
+      />
     </div>
   );
 }

package/src/web/client/src/index.css

@@ -15,6 +15,9 @@
   --pass-color: green;
   --fail-color: #ad0000;
   --smalltext-color: gray;
+  --success-background-color: #d1ffd7;
+  --variable-background-color: #f7f7f7;
+  --header-background-color: #fffdf7;
 }
 
 /* Dark mode colors */
@@ -38,6 +41,9 @@
   --pass-color: #4caf50;
   --fail-color: #f44336;
   --smalltext-color: #888888;
+  --success-background-color: #216d2b;
+  --variable-background-color: #333;
+  --header-background-color: #333;
 }
 
 html {

package/src/web/client/src/types.ts

@@ -12,3 +12,5 @@ export type EvalTable = {
   head: EvalHead;
   body: EvalRow[];
 };
+
+export type FilterMode = 'all' | 'failures';

package/src/web/server.ts

@@ -10,9 +10,9 @@ import opener from 'opener';
 import { Server as SocketIOServer } from 'socket.io';
 
 import promptfoo from '../index.js';
-import logger from '../logger.js';
-import { getDirectory } from '../esm.js';
-import { getLatestResultsPath } from '../util.js';
+import logger from '../logger';
+import { getDirectory } from '../esm';
+import { getLatestResultsPath } from '../util';
 
 import type { Request, Response } from 'express';
 

package/dist/web/client/assets/index-207192fc.css

@@ -1 +0,0 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);word-break:break-all;vertical-align:top;padding:1.5rem}th,.th{padding:1rem;position:relative;text-align:center;font-weight:semi-bold}tr .cell-rating{visibility:hidden;position:absolute;bottom:1.25rem;right:-1rem;line-height:0;font-size:1.75rem}tr:hover .cell-rating{visibility:visible}tr .cell-rating .rating{cursor:pointer;margin-right:1rem}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}td .status{margin-bottom:.5rem;font-weight:700}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}