promptfoo 0.17.7 → 0.17.9

package/dist/src/web/client/assets/{index-f9b230d1.css → index-d2b6a160.css}

@@ -1 +1 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}th .action{cursor:pointer;margin-left:.5rem}tr .cell-actions{display:flex;gap:.5rem;visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr:hover .cell-actions{visibility:visible}tr .cell-actions .action{cursor:pointer}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .score{font-weight:400}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}
+:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}th .action{cursor:pointer;margin-left:.5rem}tr .cell-actions{display:flex;gap:.5rem;visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr .cell-detail{visibility:hidden;position:absolute;bottom:.25rem;margin-top:1rem;font-size:.75rem;color:#888}tr:hover .cell-actions,tr:hover .cell-detail{visibility:visible}tr .cell-actions .action{cursor:pointer}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .score{font-weight:400}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}

package/dist/src/web/client/index.html

@@ -5,8 +5,8 @@
     <link rel="icon" type="image/svg+xml" href="favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>promptfoo web viewer</title>
-    <script type="module" crossorigin src="/assets/index-13198388.js"></script>
-    <link rel="stylesheet" href="/assets/index-f9b230d1.css">
+    <script type="module" crossorigin src="/assets/index-8388d689.js"></script>
+    <link rel="stylesheet" href="/assets/index-d2b6a160.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/src/web/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../../src/web/server.ts"],"names":[],"mappings":"AAeA,wBAAgB,IAAI,CAAC,IAAI,SAAQ,QA0DhC"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../../../src/web/server.ts"],"names":[],"mappings":"AAeA,wBAAgB,IAAI,CAAC,IAAI,SAAQ,QAiFhC"}

package/dist/src/web/server.js

@@ -37,11 +37,34 @@ function init(port = 15500) {
         // Send the initial table data when a client connects
         socket.emit('init', readLatestJson());
         // Watch for changes to latest.json and emit the update event
-        fs_1.default.watch(latestJsonPath, (0, debounce_1.default)((event) => {
-            if (event === 'change') {
+        const watcher = (0, debounce_1.default)((curr, prev) => {
+            if (curr.mtime !== prev.mtime) {
                 socket.emit('update', readLatestJson());
             }
-        }, 250));
+        }, 250);
+        fs_1.default.watchFile(latestJsonPath, watcher);
+        // Stop watching the file when the socket connection is closed
+        socket.on('disconnect', () => {
+            fs_1.default.unwatchFile(latestJsonPath, watcher);
+        });
+    });
+    app.get('/results', (req, res) => {
+        const previousResults = (0, util_1.listPreviousResults)();
+        res.json({ data: previousResults });
+    });
+    app.get('/results/:filename', (req, res) => {
+        const filename = req.params.filename;
+        const safeFilename = node_path_1.default.basename(filename);
+        if (safeFilename !== filename || !(0, util_1.listPreviousResults)().includes(safeFilename)) {
+            res.status(400).send('Invalid filename');
+            return;
+        }
+        const result = (0, util_1.readResult)(safeFilename);
+        if (!result) {
+            res.status(404).send('Result not found');
+            return;
+        }
+        res.json({ data: result });
     });
     httpServer.listen(port, () => {
         const url = `http://localhost:${port}`;

package/dist/src/web/server.js.map

@@ -1 +1 @@
-{"version":3,"file":"server.js","sourceRoot":"","sources":["../../../src/web/server.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,0DAA6B;AAC7B,kEAAqC;AACrC,0DAA6B;AAE7B,wDAAgC;AAChC,sDAA8B;AAC9B,gDAAwB;AACxB,oDAA4B;AAC5B,yCAAqD;AAErD,uDAA+B;AAC/B,gCAAsC;AACtC,kCAA+C;AAE/C,SAAgB,IAAI,CAAC,IAAI,GAAG,KAAK;IAC/B,MAAM,GAAG,GAAG,IAAA,iBAAO,GAAE,CAAC;IAEtB,MAAM,SAAS,GAAG,mBAAI,CAAC,IAAI,CAAC,IAAA,kBAAY,GAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAE7D,GAAG,CAAC,GAAG,CAAC,IAAA,cAAI,GAAE,CAAC,CAAC;IAChB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnC,MAAM,UAAU,GAAG,mBAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,IAAI,kBAAc,CAAC,UAAU,EAAE;QACxC,IAAI,EAAE;YACJ,MAAM,EAAE,GAAG;SACZ;KACF,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAA,2BAAoB,GAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,GAAG,EAAE;QAC1B,MAAM,IAAI,GAAG,YAAE,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE;QAC7B,qDAAqD;QACrD,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;QAEtC,6DAA6D;QAC7D,YAAE,CAAC,KAAK,CACN,cAAc,EACd,IAAA,kBAAQ,EAAC,CAAC,KAAa,EAAE,EAAE;YACzB,IAAI,KAAK,KAAK,QAAQ,EAAE;gBACtB,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;aACzC;QACH,CAAC,EAAE,GAAG,CAAC,CACR,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QAC3B,MAAM,GAAG,GAAG,oBAAoB,IAAI,EAAE,CAAC;QACvC,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,EAAE,CAAC,QAAQ,CAAC,qDAAqD,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClF,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBACxC,IAAI;oBACF,MAAM,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC;oBAClB,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;iBAC3C;gBAAC,OAAO,GAAG,EAAE;oBACZ,gBAAM,CAAC,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;iBACxD;aACF;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,gBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AA1DD,oBA0DC"}
+{"version":3,"file":"server.js","sourceRoot":"","sources":["../../../src/web/server.ts"],"names":[],"mappings":";;;;;;AAAA,4CAA+B;AAC/B,0DAA6B;AAC7B,kEAAqC;AACrC,0DAA6B;AAE7B,wDAAgC;AAChC,sDAA8B;AAC9B,gDAAwB;AACxB,oDAA4B;AAC5B,yCAAqD;AAErD,uDAA+B;AAC/B,gCAAsC;AACtC,kCAAgF;AAEhF,SAAgB,IAAI,CAAC,IAAI,GAAG,KAAK;IAC/B,MAAM,GAAG,GAAG,IAAA,iBAAO,GAAE,CAAC;IAEtB,MAAM,SAAS,GAAG,mBAAI,CAAC,IAAI,CAAC,IAAA,kBAAY,GAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAE7D,GAAG,CAAC,GAAG,CAAC,IAAA,cAAI,GAAE,CAAC,CAAC;IAChB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnC,MAAM,UAAU,GAAG,mBAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,IAAI,kBAAc,CAAC,UAAU,EAAE;QACxC,IAAI,EAAE;YACJ,MAAM,EAAE,GAAG;SACZ;KACF,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAA,2BAAoB,GAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,GAAG,EAAE;QAC1B,MAAM,IAAI,GAAG,YAAE,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE;QAC7B,qDAAqD;QACrD,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;QAEtC,6DAA6D;QAC7D,MAAM,OAAO,GAAG,IAAA,kBAAQ,EAAC,CAAC,IAAW,EAAE,IAAW,EAAE,EAAE;YACpD,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,EAAE;gBAC7B,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;aACzC;QACH,CAAC,EAAE,GAAG,CAAC,CAAC;QACR,YAAE,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAEtC,8DAA8D;QAC9D,MAAM,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE;YAC3B,YAAE,CAAC,WAAW,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QAC/B,MAAM,eAAe,GAAG,IAAA,0BAAmB,GAAE,CAAC;QAC9C,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,GAAG,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACzC,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QACrC,MAAM,YAAY,GAAG,mBAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,YAAY,KAAK,QAAQ,IAAI,CAAC,IAAA,0BAAmB,GAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE;YAC9E,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACzC,OAAO;SACR;QACD,MAAM,MAAM,GAAG,IAAA,iBAAU,EAAC,YAAY,CAAC,CAAC;QACxC,IAAI,CAAC,MAAM,EAAE;YACX,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACzC,OAAO;SACR;QACD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QAC3B,MAAM,GAAG,GAAG,oBAAoB,IAAI,EAAE,CAAC;QACvC,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,EAAE,CAAC,QAAQ,CAAC,qDAAqD,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClF,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBACxC,IAAI;oBACF,MAAM,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC;oBAClB,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;iBAC3C;gBAAC,OAAO,GAAG,EAAE;oBACZ,gBAAM,CAAC,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;iBACxD;aACF;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,gBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAjFD,oBAiFC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.17.7",
+  "version": "0.17.9",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",

package/src/evaluator.ts

@@ -3,6 +3,7 @@ import readline from 'readline';
 import async from 'async';
 import chalk from 'chalk';
 import nunjucks from 'nunjucks';
+import invariant from 'tiny-invariant';
 
 import logger from './logger';
 import telemetry from './telemetry';
@@ -110,18 +111,40 @@ class Evaluator {
       vars,
     };
 
+    let latencyMs = 0;
     try {
+      const startTime = Date.now();
       const response = await provider.callApi(renderedPrompt);
+      const endTime = Date.now();
+      latencyMs = endTime - startTime;
+
       const ret: EvaluateResult = {
         ...setup,
         response,
         success: false,
         score: 0,
+        latencyMs,
       };
       if (response.error) {
         ret.error = response.error;
       } else if (response.output) {
-        const checkResult = await runAssertions(test, response.output);
+        // Create a copy of response so we can potentially mutate it.
+        let processedResponse = { ...response };
+        if (test.options?.postprocess) {
+          const { postprocess } = test.options;
+          const postprocessFn = new Function(
+            'output',
+            'context',
+            postprocess.includes('\n') ? postprocess : `return ${postprocess}`,
+          );
+          processedResponse.output = postprocessFn(processedResponse.output);
+          if (processedResponse.output == null) {
+            throw new Error('Postprocess function did not return a value');
+          }
+        }
+
+        invariant(processedResponse.output != null, 'Response output should not be null');
+        const checkResult = await runAssertions(test, processedResponse.output);
         if (!checkResult.pass) {
           ret.error = checkResult.reason;
         }
@@ -132,6 +155,7 @@ class Evaluator {
           this.stats.tokenUsage.prompt += checkResult.tokensUsed.prompt;
           this.stats.tokenUsage.completion += checkResult.tokensUsed.completion;
         }
+        ret.response = processedResponse;
       } else {
         ret.success = false;
         ret.score = 0;
@@ -159,6 +183,7 @@ class Evaluator {
         error: String(err) + '\n\n' + (err as Error).stack,
         success: false,
         score: 0,
+        latencyMs,
       };
     }
   }
@@ -232,11 +257,13 @@ class Evaluator {
     // Aggregate all vars across test cases
 
     const tests = (
-      testSuite.tests || [
-        {
-          // Dummy test for cases when we're only comparing raw prompts.
-        },
-      ]
+      testSuite.tests && testSuite.tests.length > 0
+        ? testSuite.tests
+        : [
+            {
+              // Dummy test for cases when we're only comparing raw prompts.
+            },
+          ]
     ).map((test) => {
       const finalTestCase: TestCase = Object.assign({}, testSuite.defaultTest);
       return Object.assign(finalTestCase, test);
@@ -270,6 +297,8 @@ class Evaluator {
         testCase.options?.prefix || testSuite.defaultTest?.options?.prefix || '';
       const appendToPrompt =
         testCase.options?.suffix || testSuite.defaultTest?.options?.suffix || '';
+      testCase.options.postprocess =
+        testCase.options.postprocess || testSuite.defaultTest?.options?.postprocess;
 
       // Finalize test case eval
       const varCombinations = generateVarCombinations(testCase.vars || {});
@@ -404,6 +433,8 @@ class Evaluator {
           score: row.score,
           text: resultText,
           prompt: row.prompt.raw,
+          latencyMs: row.latencyMs,
+          tokenUsage: row.response?.tokenUsage,
         };
       },
     );

package/src/main.ts

@@ -11,6 +11,7 @@ import logger, { getLogLevel, setLogLevel } from './logger';
 import { loadApiProvider, loadApiProviders } from './providers';
 import { evaluate } from './evaluator';
 import {
+  cleanupOldResults,
   maybeReadConfig,
   readConfig,
   readLatestResults,
@@ -181,6 +182,7 @@ async function main() {
     .action(async () => {
       telemetry.maybeShowNotice();
       await clearCache();
+      cleanupOldResults(0);
       telemetry.record('command_used', {
         name: 'cache_clear',
       });
@@ -321,6 +323,7 @@ async function main() {
           suffix: cmdObj.promptSuffix,
           provider: cmdObj.grader,
           // rubricPrompt:
+          // postprocess
         },
         ...config.defaultTest,
       };

package/src/providers/azureopenai.ts

@@ -6,6 +6,10 @@ import type { ApiProvider, ProviderEmbeddingResponse, ProviderResponse } from '.
 
 interface AzureOpenAiCompletionOptions {
   temperature?: number;
+  top_p?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  best_of?: number;
   functions?: {
     name: string;
     description?: string;
@@ -144,6 +148,17 @@ export class AzureOpenAiCompletionProvider extends AzureOpenAiGenericProvider {
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
+      best_of:
+        options?.best_of ?? this.options.best_of ?? parseInt(process.env.OPENAI_BEST_OF || '1'),
       stop,
     };
     logger.debug(`Calling Azure OpenAI API: ${JSON.stringify(body)}`);
@@ -214,6 +229,15 @@ export class AzureOpenAiChatCompletionProvider extends AzureOpenAiGenericProvide
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
       functions: options?.functions || this.options.functions || undefined,
       function_call: options?.function_call || this.options.function_call || undefined,
     };

package/src/providers/openai.ts

@@ -1,4 +1,3 @@
-
 import logger from '../logger';
 import { fetchJsonWithCache } from '../cache';
 import { REQUEST_TIMEOUT_MS, parseChatPrompt } from './shared';
@@ -10,6 +9,10 @@ const DEFAULT_OPENAI_HOST = 'api.openai.com';
 interface OpenAiCompletionOptions {
   temperature?: number;
   max_tokens?: number;
+  top_p?: number;
+  frequency_penalty?: number;
+  presence_penalty?: number;
+  best_of?: number;
   functions?: {
     name: string;
     description?: string;
@@ -148,11 +151,25 @@ export class OpenAiCompletionProvider extends OpenAiGenericProvider {
     const body = {
       model: this.modelName,
       prompt,
-      max_tokens: options?.max_tokens ?? this.options.max_tokens ?? parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      max_tokens:
+        options?.max_tokens ??
+        this.options.max_tokens ??
+        parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
       temperature:
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
+      best_of:
+        options?.best_of ?? this.options.best_of ?? parseInt(process.env.OPENAI_BEST_OF || '1'),
       stop,
     };
     logger.debug(`Calling OpenAI API: ${JSON.stringify(body)}`);
@@ -231,11 +248,23 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
     const body = {
       model: this.modelName,
       messages: messages,
-      max_tokens: options?.max_tokens ?? this.options.max_tokens ?? parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
+      max_tokens:
+        options?.max_tokens ??
+        this.options.max_tokens ??
+        parseInt(process.env.OPENAI_MAX_TOKENS || '1024'),
       temperature:
         options?.temperature ??
         this.options.temperature ??
         parseFloat(process.env.OPENAI_TEMPERATURE || '0'),
+      top_p: options?.top_p ?? this.options.top_p ?? parseFloat(process.env.OPENAI_TOP_P || '1'),
+      presence_penalty:
+        options?.presence_penalty ??
+        this.options.presence_penalty ??
+        parseFloat(process.env.OPENAI_PRESENCE_PENALTY || '0'),
+      frequency_penalty:
+        options?.frequency_penalty ??
+        this.options.frequency_penalty ??
+        parseFloat(process.env.OPENAI_FREQUENCY_PENALTY || '0'),
       functions: options?.functions || this.options.functions || undefined,
       function_call: options?.function_call || this.options.function_call || undefined,
     };

package/src/providers/replicate.ts

@@ -65,9 +65,13 @@ export class ReplicateProvider implements ApiProvider {
       const data = {
         input: {
           prompt,
-          max_length: this.options.max_length || parseInt(process.env.REPLICATE_MAX_LENGTH || '2046', 10),
-          temperature: this.options.temperature || parseFloat(process.env.REPLICATE_TEMPERATURE || '0.01'),
-          repetition_penalty: this.options.repetition_penalty || parseFloat(process.env.REPLICATE_REPETITION_PENALTY || '1.0'),
+          max_length:
+            this.options.max_length || parseInt(process.env.REPLICATE_MAX_LENGTH || '2046', 10),
+          temperature:
+            this.options.temperature || parseFloat(process.env.REPLICATE_TEMPERATURE || '0.01'),
+          repetition_penalty:
+            this.options.repetition_penalty ||
+            parseFloat(process.env.REPLICATE_REPETITION_PENALTY || '1.0'),
         },
       };
       response = await replicate.run(this.modelName as any, data);

package/src/providers/shared.ts

@@ -4,7 +4,9 @@ export const REQUEST_TIMEOUT_MS = process.env.REQUEST_TIMEOUT_MS
   ? parseInt(process.env.REQUEST_TIMEOUT_MS, 10)
   : 300_000;
 
-export function parseChatPrompt(prompt: string): { role: string; content: string; name?: string }[] {
+export function parseChatPrompt(
+  prompt: string,
+): { role: string; content: string; name?: string }[] {
   const trimmedPrompt = prompt.trim();
   if (trimmedPrompt.startsWith('- role:')) {
     try {

package/src/types.ts

@@ -73,6 +73,10 @@ export interface PromptConfig {
   suffix?: string;
 }
 
+export interface OutputConfig {
+  postprocess?: string;
+}
+
 export interface EvaluateOptions {
   maxConcurrency?: number;
   showProgressBar?: boolean;
@@ -92,6 +96,7 @@ export interface EvaluateResult {
   error?: string;
   success: boolean;
   score: number;
+  latencyMs: number;
 }
 
 export interface EvaluateTableOutput {
@@ -99,6 +104,8 @@ export interface EvaluateTableOutput {
   score: number;
   text: string;
   prompt: string;
+  latencyMs: number;
+  tokenUsage?: Partial<TokenUsage>;
 }
 
 export interface EvaluateTable {
@@ -181,11 +188,14 @@ export interface TestCase {
   // Key-value pairs to substitute in the prompt
   vars?: Record<string, string | string[] | object>;
 
+  // Optional filepath or glob pattern to load vars from
+  loadVars?: string | string[];
+
   // Optional list of automatic checks to run on the LLM output
   assert?: Assertion[];
 
   // Additional configuration settings for the prompt
-  options?: PromptConfig & GradingConfig;
+  options?: PromptConfig & OutputConfig & GradingConfig;
 }
 
 // Same as a TestCase, except the `vars` object has been flattened into its final form.
@@ -231,7 +241,7 @@ export interface TestSuiteConfig {
   prompts: string | string[];
 
   // Path to a test file, OR list of LLM prompt variations (aka "test case")
-  tests: string | TestCase[];
+  tests: string | string[] | TestCase[];
 
   // Sets the default properties for each test case. Useful for setting an assertion, on all test cases, for example.
   defaultTest?: Omit<TestCase, 'description'>;