promptfoo 0.10.0 → 0.12.0

package/src/web/client/src/ResultsTable.tsx

@@ -9,12 +9,14 @@ import {
   getCoreRowModel,
   useReactTable,
 } from '@tanstack/react-table';
+import Checkbox from '@mui/material/Checkbox';
+import FormControlLabel from '@mui/material/FormControlLabel';
 
 import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
-import type { EvalRow } from './types.js';
+import type { EvalRow, FilterMode } from './types.js';
 
 import './ResultsTable.css';
 
@@ -114,12 +116,23 @@ function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smal
   );
 }
 
-interface ResultsViewProps {
+interface ResultsTableProps {
   maxTextLength: number;
   columnVisibility: VisibilityState;
+  wordBreak: 'break-word' | 'break-all';
+  filterMode: FilterMode;
+  failureFilter: { [key: string]: boolean };
+  onFailureFilterToggle: (columnId: string, checked: boolean) => void;
 }
 
-export default function ResultsTable({ maxTextLength, columnVisibility }: ResultsViewProps) {
+export default function ResultsTable({
+  maxTextLength,
+  columnVisibility,
+  wordBreak,
+  filterMode,
+  failureFilter,
+  onFailureFilterToggle,
+}: ResultsTableProps) {
   const { table, setTable } = useStore();
   invariant(table, 'Table should be defined');
   const { head, body } = table;
@@ -146,6 +159,10 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
     });
   };
 
+  const highestPassingIndex = numGood.reduce((maxIndex, currentPassCount, currentIndex, array) => {
+    return currentPassCount > array[maxIndex] ? currentIndex : maxIndex;
+  }, 0);
+  const highestPassingCount = numGood[highestPassingIndex];
   const columnHelper = createColumnHelper<EvalRow>();
   const columns = [
     columnHelper.group({
@@ -164,6 +181,8 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
           cell: (info: CellContext<EvalRow, string>) => (
             <TruncatedText text={info.getValue()} maxLength={maxTextLength} />
           ),
+          // Minimize the size of Variable columns.
+          size: 50,
         }),
       ),
     }),
@@ -173,16 +192,41 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
       columns: head.prompts.map((prompt, idx) =>
         columnHelper.accessor((row: EvalRow) => row.outputs[idx], {
           id: `Prompt ${idx + 1}`,
-          header: () => (
-            <>
-              <TableHeader
-                smallText={`Prompt ${idx + 1}`}
-                text={prompt}
-                maxLength={maxTextLength}
-              />
-              {numGood[idx]} / {body.length} 👍
-            </>
-          ),
+          header: () => {
+            const pct = ((numGood[idx] / body.length) * 100.0).toFixed(2);
+            const isHighestPassing =
+              numGood[idx] === highestPassingCount && highestPassingCount !== 0;
+            const columnId = `Prompt ${idx + 1}`;
+            const isChecked = failureFilter[columnId] || false;
+            return (
+              <>
+                <TableHeader
+                  smallText={`Prompt ${idx + 1}`}
+                  text={prompt}
+                  maxLength={maxTextLength}
+                />
+                {filterMode === 'failures' && (
+                  <FormControlLabel
+                    sx={{
+                      '& .MuiFormControlLabel-label': {
+                        fontSize: '0.75rem',
+                      },
+                    }}
+                    control={
+                      <Checkbox
+                        checked={isChecked}
+                        onChange={(event) => onFailureFilterToggle(columnId, event.target.checked)}
+                      />
+                    }
+                    label="Show failures"
+                  />
+                )}
+                <div className={`summary ${isHighestPassing ? 'highlight' : ''}`}>
+                  Passing: <strong>{pct}%</strong> ({numGood[idx]} / {body.length})
+                </div>
+              </>
+            );
+          },
           cell: (info: CellContext<EvalRow, string>) => (
             <PromptOutput
               text={info.getValue()}
@@ -197,8 +241,24 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
     }),
   ];
 
+  const filteredBody = React.useMemo(() => {
+    if (filterMode === 'failures') {
+      if (Object.values(failureFilter).every((v) => !v)) {
+        return body;
+      }
+      return body.filter((row) => {
+        return row.outputs.some((output, idx) => {
+          const columnId = `Prompt ${idx + 1}`;
+          const isFail = output.startsWith('[FAIL] ');
+          return failureFilter[columnId] && isFail;
+        });
+      });
+    }
+    return body;
+  }, [body, failureFilter, filterMode]);
+
   const reactTable = useReactTable({
-    data: body,
+    data: filteredBody,
     columns,
     columnResizeMode: 'onChange',
     getCoreRowModel: getCoreRowModel(),
@@ -209,32 +269,38 @@ export default function ResultsTable({ maxTextLength, columnVisibility }: Result
   });
 
   return (
-    <table>
+    <table
+      style={{
+        wordBreak,
+      }}
+    >
       <thead>
         {reactTable.getHeaderGroups().map((headerGroup) => (
           <tr key={headerGroup.id} className="header">
-            {headerGroup.headers.map((header) => (
-              <th
-                {...{
-                  key: header.id,
-                  colSpan: header.colSpan,
-                  style: {
-                    width: header.getSize(),
-                  },
-                }}
-              >
-                {header.isPlaceholder
-                  ? null
-                  : flexRender(header.column.columnDef.header, header.getContext())}
-                <div
+            {headerGroup.headers.map((header) => {
+              return (
+                <th
                   {...{
-                    onMouseDown: header.getResizeHandler(),
-                    onTouchStart: header.getResizeHandler(),
-                    className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                    key: header.id,
+                    colSpan: header.colSpan,
+                    style: {
+                      width: header.getSize(),
+                    },
                   }}
-                />
-              </th>
-            ))}
+                >
+                  {header.isPlaceholder
+                    ? null
+                    : flexRender(header.column.columnDef.header, header.getContext())}
+                  <div
+                    {...{
+                      onMouseDown: header.getResizeHandler(),
+                      onTouchStart: header.getResizeHandler(),
+                      className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                    }}
+                  />
+                </th>
+              );
+            })}
           </tr>
         ))}
       </thead>

package/src/web/client/src/ResultsView.tsx

@@ -1,23 +1,41 @@
 import * as React from 'react';
 
 import invariant from 'tiny-invariant';
+import Button from '@mui/material/Button';
 import Box from '@mui/material/Box';
-import Paper from '@mui/material/Box';
-import Stack from '@mui/material/Stack';
-import Slider from '@mui/material/Slider';
-import Typography from '@mui/material/Typography';
-import OutlinedInput from '@mui/material/OutlinedInput';
-import InputLabel from '@mui/material/InputLabel';
-import MenuItem from '@mui/material/MenuItem';
+import Checkbox from '@mui/material/Checkbox';
+import CircularProgress from '@mui/material/CircularProgress';
 import FormControl from '@mui/material/FormControl';
+import FormControlLabel from '@mui/material/FormControlLabel';
+import InputLabel from '@mui/material/InputLabel';
 import ListItemText from '@mui/material/ListItemText';
+import MenuItem from '@mui/material/MenuItem';
+import OutlinedInput from '@mui/material/OutlinedInput';
+import Paper from '@mui/material/Box';
 import Select, { SelectChangeEvent } from '@mui/material/Select';
-import Checkbox from '@mui/material/Checkbox';
+import Slider from '@mui/material/Slider';
+import Stack from '@mui/material/Stack';
+import Tooltip from '@mui/material/Tooltip';
+import Typography from '@mui/material/Typography';
+import ShareIcon from '@mui/icons-material/Share';
+import VisibilityIcon from '@mui/icons-material/Visibility';
+import { styled } from '@mui/system';
 
 import ResultsTable from './ResultsTable.js';
+import ConfigModal from './ConfigModal';
+import ShareModal from './ShareModal';
 import { useStore } from './store.js';
 
 import type { VisibilityState } from '@tanstack/table-core';
+import type { FilterMode } from './types.js';
+
+const ResponsiveStack = styled(Stack)(({ theme }) => ({
+  maxWidth: '100%',
+  flexWrap: 'wrap',
+  [theme.breakpoints.down('sm')]: {
+    flexDirection: 'column',
+  },
+}));
 
 export default function ResultsView() {
   const { table } = useStore();
@@ -25,6 +43,62 @@ export default function ResultsView() {
   const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
   const [selectedColumns, setSelectedColumns] = React.useState<string[]>([]);
 
+  const [failureFilter, setFailureFilter] = React.useState<{ [key: string]: boolean }>({});
+  const handleFailureFilterToggle = (columnId: string, checked: boolean) => {
+    setFailureFilter((prevFailureFilter) => ({ ...prevFailureFilter, [columnId]: checked }));
+  };
+
+  const [filterMode, setFilterMode] = React.useState<FilterMode>('all');
+  const handleFilterModeChange = (event: SelectChangeEvent<unknown>) => {
+    const mode = event.target.value as FilterMode;
+    setFilterMode(mode);
+
+    const newFailureFilter: { [key: string]: boolean } = {};
+    head.prompts.forEach((_, idx) => {
+      const columnId = `Prompt ${idx + 1}`;
+      newFailureFilter[columnId] = mode === 'failures';
+    });
+    setFailureFilter(newFailureFilter);
+  };
+
+  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-all');
+  const handleWordBreakChange = (event: React.ChangeEvent<HTMLInputElement>) => {
+    setWordBreak(event.target.checked ? 'break-all' : 'break-word');
+  };
+
+  const [shareModalOpen, setShareModalOpen] = React.useState(false);
+  const [shareUrl, setShareUrl] = React.useState('');
+  const [shareLoading, setShareLoading] = React.useState(false);
+
+  const handleShareButtonClick = async () => {
+    setShareLoading(true);
+    try {
+      const response = await fetch('https://api.promptfoo.dev/eval', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          data: {
+            version: 1,
+            table,
+          },
+        }),
+      });
+
+      const { id } = await response.json();
+      const shareUrl = `https://app.promptfoo.dev/eval/${id}`;
+      setShareUrl(shareUrl);
+      setShareModalOpen(true);
+    } catch {
+      alert('Sorry, something went wrong.');
+    } finally {
+      setShareLoading(false);
+    }
+  };
+
+  const [configModalOpen, setConfigModalOpen] = React.useState(false);
+
   invariant(table, 'Table data must be loaded before rendering ResultsView');
   const { head } = table;
 
@@ -71,9 +145,9 @@ export default function ResultsView() {
   return (
     <div>
       <Paper py="md">
-        <Stack direction="row" spacing={2} alignItems="center">
+        <ResponsiveStack direction="row" spacing={8} alignItems="center">
           <Box>
-            <FormControl sx={{ m: 1, minWidth: 300 }} size="small">
+            <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
               <InputLabel id="visible-columns-label">Visible columns</InputLabel>
               <Select
                 labelId="visible-columns-label"
@@ -93,6 +167,21 @@ export default function ResultsView() {
               </Select>
             </FormControl>
           </Box>
+          <Box>
+            <FormControl sx={{ minWidth: 180 }} size="small">
+              <InputLabel id="failure-filter-mode-label">Filter</InputLabel>
+              <Select
+                labelId="filter-mode-label"
+                id="filter-mode"
+                value={filterMode}
+                onChange={handleFilterModeChange}
+                label="Filter"
+              >
+                <MenuItem value="all">Show all results</MenuItem>
+                <MenuItem value="failures">Show only failures</MenuItem>
+              </Select>
+            </FormControl>
+          </Box>
           <Box>
             <Typography mt={2}>Max text length: {maxTextLength}</Typography>
             <Slider
@@ -102,9 +191,56 @@ export default function ResultsView() {
               onChange={(_, val: number | number[]) => setMaxTextLength(val as number)}
             />
           </Box>
-        </Stack>
+          <Box>
+            <Tooltip title="Forcing line breaks makes it easier to adjust column widths to your liking">
+              <FormControlLabel
+                control={
+                  <Checkbox checked={wordBreak === 'break-all'} onChange={handleWordBreakChange} />
+                }
+                label="Force line breaks"
+              />
+            </Tooltip>
+          </Box>
+          <Box flexGrow={1} />
+          <Box display="flex" justifyContent="flex-end">
+            <ResponsiveStack direction="row" spacing={2}>
+              <Tooltip title="View config">
+                <Button
+                  color="primary"
+                  onClick={() => setConfigModalOpen(true)}
+                  startIcon={<VisibilityIcon />}
+                >
+                  Config
+                </Button>
+              </Tooltip>
+              <Tooltip title="Generate a unique URL that others can access">
+                <Button
+                  color="primary"
+                  onClick={handleShareButtonClick}
+                  disabled={shareLoading}
+                  startIcon={shareLoading ? <CircularProgress size={16} /> : <ShareIcon />}
+                >
+                  Share
+                </Button>
+              </Tooltip>
+            </ResponsiveStack>
+          </Box>
+        </ResponsiveStack>
       </Paper>
-      <ResultsTable maxTextLength={maxTextLength} columnVisibility={columnVisibility} />
+      <ResultsTable
+        maxTextLength={maxTextLength}
+        columnVisibility={columnVisibility}
+        wordBreak={wordBreak}
+        filterMode={filterMode}
+        failureFilter={failureFilter}
+        onFailureFilterToggle={handleFailureFilterToggle}
+      />
+      <ConfigModal open={configModalOpen} onClose={() => setConfigModalOpen(false)} />
+      <ShareModal
+        open={shareModalOpen}
+        onClose={() => setShareModalOpen(false)}
+        shareUrl={shareUrl}
+      />
     </div>
   );
 }

package/src/web/client/src/ShareModal.tsx

@@ -0,0 +1,70 @@
+import React, { useRef, useState } from 'react';
+import Dialog from '@mui/material/Dialog';
+import DialogTitle from '@mui/material/DialogTitle';
+import DialogContent from '@mui/material/DialogContent';
+import DialogContentText from '@mui/material/DialogContentText';
+import DialogActions from '@mui/material/DialogActions';
+import Button from '@mui/material/Button';
+import TextField from '@mui/material/TextField';
+import IconButton from '@mui/material/IconButton';
+import FileCopyIcon from '@mui/icons-material/FileCopy';
+import CheckIcon from '@mui/icons-material/Check';
+
+interface ShareModalProps {
+  open: boolean;
+  onClose: () => void;
+  shareUrl: string;
+}
+
+const ShareModal: React.FC<ShareModalProps> = ({ open, onClose, shareUrl }) => {
+  const inputRef = useRef<HTMLInputElement>(null);
+  const [copied, setCopied] = useState(false);
+
+  const handleCopyClick = () => {
+    if (inputRef.current) {
+      inputRef.current.select();
+      document.execCommand('copy');
+      setCopied(true);
+    }
+  };
+
+  const handleClose = () => {
+    onClose();
+    setCopied(false);
+  };
+
+  return (
+    <Dialog
+      open={open}
+      onClose={handleClose}
+      PaperProps={{ style: { minWidth: 'min(660px, 100%)' } }}
+    >
+      <DialogTitle>Your eval is ready to share</DialogTitle>
+      <DialogContent>
+        <TextField
+          inputRef={inputRef}
+          value={shareUrl}
+          fullWidth
+          InputProps={{
+            readOnly: true,
+            endAdornment: (
+              <IconButton onClick={handleCopyClick}>
+                {copied ? <CheckIcon /> : <FileCopyIcon />}
+              </IconButton>
+            ),
+          }}
+        />
+        <DialogContentText sx={{ fontSize: '0.75rem' }}>
+          Shared URLs are deleted after 1 week.
+        </DialogContentText>
+      </DialogContent>
+      <DialogActions>
+        <Button onClick={handleClose} color="primary">
+          Close
+        </Button>
+      </DialogActions>
+    </Dialog>
+  );
+};
+
+export default ShareModal;

package/src/web/client/src/index.css

@@ -15,6 +15,9 @@
   --pass-color: green;
   --fail-color: #ad0000;
   --smalltext-color: gray;
+  --success-background-color: #d1ffd7;
+  --variable-background-color: #f7f7f7;
+  --header-background-color: #fffdf7;
 }
 
 /* Dark mode colors */
@@ -38,6 +41,9 @@
   --pass-color: #4caf50;
   --fail-color: #f44336;
   --smalltext-color: #888888;
+  --success-background-color: #216d2b;
+  --variable-background-color: #333;
+  --header-background-color: #333;
 }
 
 html {

package/src/web/client/src/store.ts

@@ -1,13 +1,18 @@
 import create from 'zustand';
 
-import type { EvalTable } from './types.js';
+import type { EvalTable, UnifiedConfig } from './types.js';
 
 interface TableState {
   table: EvalTable | null;
   setTable: (table: EvalTable | null) => void;
+
+  config: Partial<UnifiedConfig> | null;
+  setConfig: (config: Partial<UnifiedConfig> | null) => void;
 }
 
 export const useStore = create<TableState>((set) => ({
   table: null,
   setTable: (table: EvalTable | null) => set(() => ({ table })),
+  config: null,
+  setConfig: (config: Partial<UnifiedConfig> | null) => set(() => ({ config })),
 }));

package/src/web/client/src/types.ts

@@ -12,3 +12,7 @@ export type EvalTable = {
   head: EvalHead;
   body: EvalRow[];
 };
+
+export type FilterMode = 'all' | 'failures';
+
+export type { UnifiedConfig } from '../../../types';

package/src/web/server.ts

@@ -9,13 +9,10 @@ import cors from 'cors';
 import opener from 'opener';
 import { Server as SocketIOServer } from 'socket.io';
 
-import promptfoo from '../index.js';
 import logger from '../logger';
 import { getDirectory } from '../esm';
 import { getLatestResultsPath } from '../util';
 
-import type { Request, Response } from 'express';
-
 export function init(port = 15500) {
   const app = express();
 
@@ -35,20 +32,19 @@ export function init(port = 15500) {
   const latestJsonPath = getLatestResultsPath();
   const readLatestJson = () => {
     const data = fs.readFileSync(latestJsonPath, 'utf8');
-    const jsonData = JSON.parse(data);
-    return jsonData.table;
+    return JSON.parse(data);
   };
 
   io.on('connection', (socket) => {
     // Send the initial table data when a client connects
-    socket.emit('init', { table: readLatestJson() });
+    socket.emit('init', readLatestJson());
 
     // Watch for changes to latest.json and emit the update event
     fs.watch(
       latestJsonPath,
       debounce((event: string) => {
         if (event === 'change') {
-          socket.emit('update', { table: readLatestJson() });
+          socket.emit('update', readLatestJson);
         }
       }, 250),
     );

package/dist/__mocks__/esm.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"esm.d.ts","sourceRoot":"","sources":["../../src/__mocks__/esm.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,WAE3B"}

package/dist/__mocks__/esm.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../src/__mocks__/esm.ts"],"names":[],"mappings":";;;AAAA,SAAgB,YAAY;IAC1B,OAAO,WAAW,CAAC;AACrB,CAAC;AAFD,oCAEC"}

package/dist/assertions.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../src/assertions.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAY,aAAa,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAMjG,wBAAsB,aAAa,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAyBhG;AAED,wBAAsB,YAAY,CAChC,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,aAAa,CAAC,CA0DxB;AAoBD,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,aAAa,CAAC,CA0CxB;AAED,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,CAgDxB;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAmC/D;;;;;AAED,wBAGE"}