promptfoo 0.17.7 → 0.17.9

package/src/util.ts

@@ -29,7 +29,10 @@ import type {
   TestSuite,
 } from './types';
 
-export function readProviderPromptMap(config: Partial<UnifiedConfig>, parsedPrompts: Prompt[]): TestSuite["providerPromptMap"] {
+export function readProviderPromptMap(
+  config: Partial<UnifiedConfig>,
+  parsedPrompts: Prompt[],
+): TestSuite['providerPromptMap'] {
   const ret: Record<string, string[]> = {};
 
   if (!config.providers) {
@@ -221,7 +224,31 @@ export async function fetchCsvFromGoogleSheet(url: string): Promise<string> {
   return csvData;
 }
 
-export async function readVars(varsPath: string, basePath: string = ''): Promise<CsvRow[]> {
+export async function readVarsFiles(
+  pathOrGlobs: string | string[],
+  basePath: string = '',
+): Promise<Record<string, string | string[] | object>> {
+  if (typeof pathOrGlobs === 'string') {
+    pathOrGlobs = [pathOrGlobs];
+  }
+
+  const ret: Record<string, string | string[] | object> = {};
+  for (const pathOrGlob of pathOrGlobs) {
+    const resolvedPath = path.resolve(basePath, pathOrGlob);
+    const paths = globSync(resolvedPath);
+
+    for (const p of paths) {
+      const yamlData = yaml.load(fs.readFileSync(p, 'utf-8'));
+      Object.assign(ret, yamlData);
+    }
+  }
+
+  return ret;
+}
+
+export async function readTestsFile(varsPath: string, basePath: string = ''): Promise<CsvRow[]> {
+  // This function is confusingly named - it reads a CSV, JSON, or YAML file of
+  // TESTS or test equivalents.
   const resolvedVarsPath = path.resolve(basePath, varsPath);
   const fileExtension = parsePath(varsPath).ext.slice(1);
   let rows: CsvRow[] = [];
@@ -243,25 +270,53 @@ export async function readVars(varsPath: string, basePath: string = ''): Promise
 }
 
 export async function readTests(
-  tests: string | TestCase[] | undefined,
+  tests: string | string[] | TestCase[] | undefined,
   basePath: string = '',
 ): Promise<TestCase[]> {
-  if (!tests) {
-    return [];
-  }
+  const ret: TestCase[] = [];
+
+  const loadTestsFromGlob = async (loadTestsGlob: string) => {
+    const resolvedPath = path.resolve(basePath, loadTestsGlob);
+    const testFiles = globSync(resolvedPath);
+    for (const testFile of testFiles) {
+      const testFileContent = yaml.load(fs.readFileSync(testFile, 'utf-8')) as TestCase[];
+      for (const testCase of testFileContent) {
+        if (typeof testCase.vars === 'string' || Array.isArray(testCase.vars)) {
+          const testcaseBasePath = path.dirname(testFile);
+          testCase.vars = await readVarsFiles(testCase.vars, testcaseBasePath);
+        }
+      }
+      ret.push(...testFileContent);
+    }
+  };
 
   if (typeof tests === 'string') {
-    // It's a filepath, load from CSV
-    const vars = await readVars(tests, basePath);
-    return vars.map((row, idx) => {
-      const test = testCaseFromCsvRow(row);
-      test.description = `Row #${idx + 1}`;
-      return test;
-    });
+    if (tests.endsWith('yaml') || tests.endsWith('yml')) {
+      // Load testcase config from yaml
+      await loadTestsFromGlob(tests);
+    } else {
+      // Legacy load CSV
+      const vars = await readTestsFile(tests, basePath);
+      return vars.map((row, idx) => {
+        const test = testCaseFromCsvRow(row);
+        test.description = `Row #${idx + 1}`;
+        return test;
+      });
+    }
+  } else if (Array.isArray(tests)) {
+    for (const maybeTestsGlob of tests) {
+      if (typeof maybeTestsGlob === 'string') {
+        // Assume it's a filepath
+        await loadTestsFromGlob(maybeTestsGlob);
+      } else {
+        // Assume it's a full test case
+        ret.push(maybeTestsGlob);
+      }
+    }
   }
 
   // Some validation of the shape of tests
-  for (const test of tests) {
+  for (const test of ret) {
     if (!test.assert && !test.vars) {
       throw new Error(
         `Test case must have either "assert" or "vars" property. Instead got ${JSON.stringify(
@@ -273,7 +328,7 @@ export async function readTests(
     }
   }
 
-  return tests;
+  return ret;
 }
 
 export function writeOutput(
@@ -313,28 +368,31 @@ export function writeOutput(
   }
 }
 
-export async function fetchWithTimeout(
+export function fetchWithTimeout(
   url: RequestInfo,
   options: RequestInit = {},
   timeout: number,
 ): Promise<Response> {
-  const controller = new AbortController();
-  const { signal } = controller;
-  options.signal = signal;
-
-  const timeoutId = setTimeout(() => {
-    controller.abort();
-    throw new Error(`Request timed out after ${timeout} ms`);
-  }, timeout);
-
-  try {
-    const response = await fetch(url, options);
-    clearTimeout(timeoutId);
-    return response;
-  } catch (error) {
-    clearTimeout(timeoutId);
-    throw error;
-  }
+  return new Promise((resolve, reject) => {
+    const controller = new AbortController();
+    const { signal } = controller;
+    options.signal = signal;
+
+    const timeoutId = setTimeout(() => {
+      controller.abort();
+      reject(new Error(`Request timed out after ${timeout} ms`));
+    }, timeout);
+
+    fetch(url, options)
+      .then((response) => {
+        clearTimeout(timeoutId);
+        resolve(response);
+      })
+      .catch((error) => {
+        clearTimeout(timeoutId);
+        reject(error);
+      });
+  });
 }
 
 export async function fetchWithRetries(
@@ -356,6 +414,8 @@ export async function fetchWithRetries(
   throw new Error(`Request failed after ${retries} retries: ${(lastError as Error).message}`);
 }
 
+const RESULT_HISTORY_LENGTH = 50;
+
 export function getConfigDirectoryPath(): string {
   return path.join(os.homedir(), '.promptfoo');
 }
@@ -365,11 +425,17 @@ export function getLatestResultsPath(): string {
 }
 
 export function writeLatestResults(results: EvaluateSummary, config: Partial<UnifiedConfig>) {
+  const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
+
+  // Replace hyphens with colons (Windows compatibility).
+  const timestamp = new Date().toISOString().replace(/:/g, '-');
+
+  const newResultsPath = path.join(resultsDirectory, `eval-${timestamp}.json`);
   const latestResultsPath = getLatestResultsPath();
   try {
-    fs.mkdirSync(path.dirname(latestResultsPath), { recursive: true });
+    fs.mkdirSync(resultsDirectory, { recursive: true });
     fs.writeFileSync(
-      latestResultsPath,
+      newResultsPath,
       JSON.stringify(
         {
           version: 1,
@@ -380,8 +446,45 @@ export function writeLatestResults(results: EvaluateSummary, config: Partial<Uni
         2,
       ),
     );
+    if (fs.existsSync(latestResultsPath)) {
+      fs.unlinkSync(latestResultsPath);
+    }
+    fs.symlinkSync(newResultsPath, latestResultsPath);
+    cleanupOldResults();
   } catch (err) {
-    logger.error(`Failed to write latest results to ${latestResultsPath}:\n${err}`);
+    logger.error(`Failed to write latest results to ${newResultsPath}:\n${err}`);
+  }
+}
+
+export function listPreviousResults(): string[] {
+  const directory = path.join(getConfigDirectoryPath(), 'output');
+  const files = fs.readdirSync(directory);
+  const resultsFiles = files.filter((file) => file.startsWith('eval-') && file.endsWith('.json'));
+  const sortedFiles = resultsFiles.sort((a, b) => {
+    const statA = fs.statSync(path.join(directory, a));
+    const statB = fs.statSync(path.join(directory, b));
+    return statB.birthtime.getTime() - statA.birthtime.getTime(); // sort in descending order
+  });
+  return sortedFiles;
+}
+
+export function cleanupOldResults(remaining = RESULT_HISTORY_LENGTH) {
+  const sortedFiles = listPreviousResults();
+  for (let i = 0; i < sortedFiles.length - remaining; i++) {
+    fs.unlinkSync(path.join(getConfigDirectoryPath(), 'output', sortedFiles[i]));
+  }
+}
+
+export function readResult(
+  name: string,
+): { results: EvaluateSummary; config: Partial<UnifiedConfig> } | undefined {
+  const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
+  const resultsPath = path.join(resultsDirectory, name);
+  try {
+    const results = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
+    return results;
+  } catch (err) {
+    logger.error(`Failed to read results from ${resultsPath}:\n${err}`);
   }
 }
 
@@ -389,12 +492,7 @@ export function readLatestResults():
   | { results: EvaluateSummary; config: Partial<UnifiedConfig> }
   | undefined {
   const latestResultsPath = getLatestResultsPath();
-  try {
-    const latestResults = JSON.parse(fs.readFileSync(latestResultsPath, 'utf-8'));
-    return latestResults;
-  } catch (err) {
-    logger.error(`Failed to read latest results from ${latestResultsPath}:\n${err}`);
-  }
+  return readResult(latestResultsPath);
 }
 
 export function cosineSimilarity(vecA: number[], vecB: number[]) {

package/src/web/client/src/App.tsx

@@ -14,6 +14,7 @@ function App() {
   const { table, setTable, setConfig } = useStore();
   const [loaded, setLoaded] = React.useState<boolean>(false);
   const loadedFromApi = React.useRef(false);
+  const [recentFiles, setRecentFiles] = React.useState<string[]>([]);
 
   const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
   const [darkMode, setDarkMode] = React.useState(prefersDarkMode);
@@ -43,6 +44,22 @@ function App() {
     }
   }, [prefersDarkMode]);
 
+  const fetchRecentFiles = async () => {
+    if (!window.location.href.includes('localhost')) {
+      return;
+    }
+    const resp = await fetch(`http://localhost:15500/results`);
+    const body = await resp.json();
+    setRecentFiles(body.data);
+  };
+
+  const handleRecentFileSelection = async (file: string) => {
+    const resp = await fetch(`http://localhost:15500/results/${file}`);
+    const body = await resp.json();
+    setTable(body.data.results.table);
+    setConfig(body.data.config);
+  };
+
   React.useEffect(() => {
     const fetchEvalData = async (id: string) => {
       if (loadedFromApi.current) {
@@ -72,12 +89,14 @@ function App() {
         setLoaded(true);
         setTable(data.results.table);
         setConfig(data.config);
+        fetchRecentFiles();
       });
 
       socket.on('update', (data) => {
         console.log('Received data update', data);
         setTable(data.results.table);
         setConfig(data.config);
+        fetchRecentFiles();
       });
     }
 
@@ -89,7 +108,11 @@ function App() {
   return (
     <ThemeProvider theme={theme}>
       <NavBar darkMode={darkMode} onToggleDarkMode={toggleDarkMode} />
-      {loaded && table ? <ResultsView /> : <div>Loading...</div>}
+      {loaded && table ? (
+        <ResultsView recentFiles={recentFiles} onRecentFileSelected={handleRecentFileSelection} />
+      ) : (
+        <div>Loading...</div>
+      )}
     </ThemeProvider>
   );
 }

package/src/web/client/src/ResultsTable.css

@@ -83,7 +83,17 @@ tr .cell-actions {
   font-size: 1.75rem;
 }
 
-tr:hover .cell-actions {
+tr .cell-detail {
+  visibility: hidden;
+  position: absolute;
+  bottom: 0.25rem;
+  margin-top: 1rem;
+  font-size: 0.75rem;
+  color: #888;
+}
+
+tr:hover .cell-actions,
+tr:hover .cell-detail {
   visibility: visible;
 }
 

package/src/web/client/src/ResultsTable.tsx

@@ -135,6 +135,16 @@ function EvalOutputCell({
         )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
+      <div className="cell-detail">
+        {output.tokenUsage?.cached ? (
+          <span>{output.tokenUsage.cached} tokens (cached)</span>
+        ) : (
+          <>
+            {output.tokenUsage?.total && <span>{output.tokenUsage.total} tokens</span>} |{' '}
+            <span>{output.latencyMs} ms</span>
+          </>
+        )}
+      </div>
       <div className="cell-actions">
         {output.prompt && (
           <>

package/src/web/client/src/ResultsView.tsx

@@ -37,7 +37,32 @@ const ResponsiveStack = styled(Stack)(({ theme }) => ({
   },
 }));
 
-export default function ResultsView() {
+function filenameToDate(filename: string) {
+  const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
+
+  // Replace hyphens with colons where necessary (Windows compatibility).
+  const dateParts = dateString.split('T');
+  const timePart = dateParts[1].replace(/-/g, ':');
+  const formattedDateString = `${dateParts[0]}T${timePart}`;
+
+  const date = new Date(formattedDateString);
+  return date.toLocaleDateString('en-US', {
+    year: 'numeric',
+    month: 'long',
+    day: 'numeric',
+    hour: '2-digit',
+    minute: '2-digit',
+    second: '2-digit',
+    timeZoneName: 'short',
+  });
+}
+
+interface ResultsViewProps {
+  recentFiles: string[];
+  onRecentFileSelected: (file: string) => void;
+}
+
+export default function ResultsView({ recentFiles, onRecentFileSelected }: ResultsViewProps) {
   const { table, config } = useStore();
   const [maxTextLength, setMaxTextLength] = React.useState(250);
   const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
@@ -148,10 +173,30 @@ export default function ResultsView() {
   return (
     <div>
       <Paper py="md">
-        <ResponsiveStack direction="row" spacing={8} alignItems="center">
+        <ResponsiveStack direction="row" spacing={4} alignItems="center">
+          <Box>
+            {recentFiles && recentFiles.length > 0 && (
+              <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
+                <InputLabel>View run</InputLabel>
+                <Select
+                  key={recentFiles.join(',')}
+                  className="recent-files"
+                  label="Previous runs"
+                  defaultValue={recentFiles[0]}
+                  onChange={(e: SelectChangeEvent) => onRecentFileSelected(e.target.value)}
+                >
+                  {recentFiles.map((file) => (
+                    <MenuItem key={file} value={file}>
+                      {filenameToDate(file)}
+                    </MenuItem>
+                  ))}
+                </Select>
+              </FormControl>
+            )}
+          </Box>
           <Box>
             <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
-              <InputLabel id="visible-columns-label">Visible columns</InputLabel>
+              <InputLabel id="visible-columns-label">Show columns</InputLabel>
               <Select
                 labelId="visible-columns-label"
                 id="visible-columns"

package/src/web/client/src/types.ts

@@ -1,3 +1,5 @@
+import { TokenUsage } from '../../../types';
+
 type Prompt = {
   display: string;
   raw: string;
@@ -13,6 +15,8 @@ export type EvalRowOutput = {
   score: number;
   text: string | object;
   prompt: string;
+  latencyMs: number;
+  tokenUsage?: Partial<TokenUsage>;
 };
 
 export type EvalRow = {

package/src/web/server.ts

@@ -1,4 +1,4 @@
-import fs from 'fs';
+import fs, { Stats } from 'fs';
 import path from 'node:path';
 import readline from 'node:readline';
 import http from 'node:http';
@@ -11,7 +11,7 @@ import { Server as SocketIOServer } from 'socket.io';
 
 import logger from '../logger';
 import { getDirectory } from '../esm';
-import { getLatestResultsPath } from '../util';
+import { getLatestResultsPath, listPreviousResults, readResult } from '../util';
 
 export function init(port = 15500) {
   const app = express();
@@ -40,14 +40,37 @@ export function init(port = 15500) {
     socket.emit('init', readLatestJson());
 
     // Watch for changes to latest.json and emit the update event
-    fs.watch(
-      latestJsonPath,
-      debounce((event: string) => {
-        if (event === 'change') {
-          socket.emit('update', readLatestJson());
-        }
-      }, 250),
-    );
+    const watcher = debounce((curr: Stats, prev: Stats) => {
+      if (curr.mtime !== prev.mtime) {
+        socket.emit('update', readLatestJson());
+      }
+    }, 250);
+    fs.watchFile(latestJsonPath, watcher);
+
+    // Stop watching the file when the socket connection is closed
+    socket.on('disconnect', () => {
+      fs.unwatchFile(latestJsonPath, watcher);
+    });
+  });
+
+  app.get('/results', (req, res) => {
+    const previousResults = listPreviousResults();
+    res.json({ data: previousResults });
+  });
+
+  app.get('/results/:filename', (req, res) => {
+    const filename = req.params.filename;
+    const safeFilename = path.basename(filename);
+    if (safeFilename !== filename || !listPreviousResults().includes(safeFilename)) {
+      res.status(400).send('Invalid filename');
+      return;
+    }
+    const result = readResult(safeFilename);
+    if (!result) {
+      res.status(404).send('Result not found');
+      return;
+    }
+    res.json({ data: result });
   });
 
   httpServer.listen(port, () => {