promptfoo 0.14.1 → 0.15.0

package/dist/src/web/client/assets/{index-87905193.css → index-c3faa651.css}

@@ -1 +1 @@
-:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}tr .cell-rating{visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr:hover .cell-rating{visibility:visible}tr .cell-rating .rating{cursor:pointer}tr .cell-rating .rating:first-child{margin-right:.5rem}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}
+:root{font-family:system-ui,Avenir,Helvetica,Arial,sans-serif;font-synthesis:none;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-text-size-adjust:100%;--background-color: #ffffff;--text-color: #404040;--border-color: lightgray;--table-border-color: lightgray;--pass-color: green;--fail-color: #ad0000;--smalltext-color: gray;--success-background-color: #d1ffd7;--variable-background-color: #f7f7f7;--header-background-color: #fffdf7}@media (prefers-color-scheme: dark){:root{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888}}[data-theme=dark]{--background-color: #1a1a1a;--text-color: #f0f0f0;--border-color: #444444;--table-border-color: #444444;--pass-color: #4caf50;--fail-color: #f44336;--smalltext-color: #888888;--success-background-color: #216d2b;--variable-background-color: #333;--header-background-color: #333}html{font-size:calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)))}*{box-sizing:border-box}html{font-family:-apple-system,BlinkMacSystemFont,Segoe UI,Helvetica,Arial,sans-serif,"Apple Color Emoji","Segoe UI Emoji",Segoe UI Symbol;font-size:16px;background-color:var(--background-color);color:var(--text-color)}table,.divTable{border:1px solid var(--table-border-color);border-collapse:collapse;width:100%;margin:1rem 0;box-shadow:0 2px 4px #0000001a}.tr{display:flex}tr,.tr{width:fit-content}tr:hover,.tr:hover{background-color:#0000000d}th,.th,td,.td{position:relative;box-shadow:inset 0 0 0 1px var(--border-color);vertical-align:top;padding:1.5rem}th.variable,.th.variable,td.variable,.td.variable{background-color:var(--variable-background-color)}tr.header{background-color:var(--header-background-color)}th,.th{padding:1rem;position:relative;text-align:center;vertical-align:bottom}tr .cell-rating{visibility:hidden;position:absolute;bottom:1.25rem;right:0;line-height:0;font-size:1.75rem}tr:hover .cell-rating{visibility:visible}tr .cell-rating .rating{cursor:pointer}tr .cell-rating .rating:first-child{margin-right:.5rem}th .smalltext{visibility:hidden;font-weight:400;font-size:.75rem;color:var(--smalltext-color)}th:hover .smalltext{visibility:visible}th .summary{font-weight:400;font-size:.8rem;padding:.25rem}th .summary.highlight{background-color:var(--success-background-color)}td .status{margin-bottom:.5rem;font-weight:700}td .score{font-weight:400}td .pass{color:var(--pass-color)}td .fail{color:var(--fail-color)}.first-prompt-col{border-left:2px solid #888}.first-prompt-row{border-top:2px solid #888}.resizer{position:absolute;right:0;top:0;height:100%;width:5px;cursor:col-resize;user-select:none;touch-action:none;background:var(--text-color);opacity:.5}.resizer.isResizing{background:var(--text-color);opacity:1}@media (hover: hover){.resizer{opacity:0}*:hover>.resizer{opacity:1}}.logo{display:flex;align-items:center;gap:4px}.logo img{width:30px}.logo span{margin-bottom:6px;color:var(--text-color)}[data-theme=dark] .logo img{filter:invert(1)}nav{display:flex;justify-content:space-between;align-items:center;margin-bottom:1rem;color:var(--text-color)}.dark-mode-toggle{background-color:transparent;border:none;color:var(--text-color);cursor:pointer;font-size:16px;padding:8px;transition:color .3s}.dark-mode-toggle:hover{color:var(--pass-color)}body{background-color:var(--background-color);color:var(--text-color)}

package/dist/src/web/client/index.html

@@ -5,8 +5,8 @@
     <link rel="icon" type="image/svg+xml" href="favicon.ico" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>promptfoo web viewer</title>
-    <script type="module" crossorigin src="/assets/index-70e6ca57.js"></script>
-    <link rel="stylesheet" href="/assets/index-87905193.css">
+    <script type="module" crossorigin src="/assets/index-9d27a707.js"></script>
+    <link rel="stylesheet" href="/assets/index-c3faa651.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/src/web/server.js

@@ -39,7 +39,7 @@ function init(port = 15500) {
         // Watch for changes to latest.json and emit the update event
         fs_1.default.watch(latestJsonPath, (0, debounce_1.default)((event) => {
             if (event === 'change') {
-                socket.emit('update', readLatestJson);
+                socket.emit('update', readLatestJson());
             }
         }, 250));
     });

package/dist/src/web/server.js.map

@@ -1 +1 @@
-{"version":3,"file":"server.js","sourceRoot":"","sources":["../../../src/web/server.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,0DAA6B;AAC7B,kEAAqC;AACrC,0DAA6B;AAE7B,wDAAgC;AAChC,sDAA8B;AAC9B,gDAAwB;AACxB,oDAA4B;AAC5B,yCAAqD;AAErD,uDAA+B;AAC/B,gCAAsC;AACtC,kCAA+C;AAE/C,SAAgB,IAAI,CAAC,IAAI,GAAG,KAAK;IAC/B,MAAM,GAAG,GAAG,IAAA,iBAAO,GAAE,CAAC;IAEtB,MAAM,SAAS,GAAG,mBAAI,CAAC,IAAI,CAAC,IAAA,kBAAY,GAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAE7D,GAAG,CAAC,GAAG,CAAC,IAAA,cAAI,GAAE,CAAC,CAAC;IAChB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnC,MAAM,UAAU,GAAG,mBAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,IAAI,kBAAc,CAAC,UAAU,EAAE;QACxC,IAAI,EAAE;YACJ,MAAM,EAAE,GAAG;SACZ;KACF,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAA,2BAAoB,GAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,GAAG,EAAE;QAC1B,MAAM,IAAI,GAAG,YAAE,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE;QAC7B,qDAAqD;QACrD,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;QAEtC,6DAA6D;QAC7D,YAAE,CAAC,KAAK,CACN,cAAc,EACd,IAAA,kBAAQ,EAAC,CAAC,KAAa,EAAE,EAAE;YACzB,IAAI,KAAK,KAAK,QAAQ,EAAE;gBACtB,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;aACvC;QACH,CAAC,EAAE,GAAG,CAAC,CACR,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QAC3B,MAAM,GAAG,GAAG,oBAAoB,IAAI,EAAE,CAAC;QACvC,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,EAAE,CAAC,QAAQ,CAAC,qDAAqD,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClF,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBACxC,IAAI;oBACF,MAAM,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC;oBAClB,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;iBAC3C;gBAAC,OAAO,GAAG,EAAE;oBACZ,gBAAM,CAAC,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;iBACxD;aACF;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,gBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AA1DD,oBA0DC"}
+{"version":3,"file":"server.js","sourceRoot":"","sources":["../../../src/web/server.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,0DAA6B;AAC7B,kEAAqC;AACrC,0DAA6B;AAE7B,wDAAgC;AAChC,sDAA8B;AAC9B,gDAAwB;AACxB,oDAA4B;AAC5B,yCAAqD;AAErD,uDAA+B;AAC/B,gCAAsC;AACtC,kCAA+C;AAE/C,SAAgB,IAAI,CAAC,IAAI,GAAG,KAAK;IAC/B,MAAM,GAAG,GAAG,IAAA,iBAAO,GAAE,CAAC;IAEtB,MAAM,SAAS,GAAG,mBAAI,CAAC,IAAI,CAAC,IAAA,kBAAY,GAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAE7D,GAAG,CAAC,GAAG,CAAC,IAAA,cAAI,GAAE,CAAC,CAAC;IAChB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACxB,GAAG,CAAC,GAAG,CAAC,iBAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAEnC,MAAM,UAAU,GAAG,mBAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,IAAI,kBAAc,CAAC,UAAU,EAAE;QACxC,IAAI,EAAE;YACJ,MAAM,EAAE,GAAG;SACZ;KACF,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAA,2BAAoB,GAAE,CAAC;IAC9C,MAAM,cAAc,GAAG,GAAG,EAAE;QAC1B,MAAM,IAAI,GAAG,YAAE,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,EAAE,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,EAAE;QAC7B,qDAAqD;QACrD,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC;QAEtC,6DAA6D;QAC7D,YAAE,CAAC,KAAK,CACN,cAAc,EACd,IAAA,kBAAQ,EAAC,CAAC,KAAa,EAAE,EAAE;YACzB,IAAI,KAAK,KAAK,QAAQ,EAAE;gBACtB,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAC;aACzC;QACH,CAAC,EAAE,GAAG,CAAC,CACR,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;QAC3B,MAAM,GAAG,GAAG,oBAAoB,IAAI,EAAE,CAAC;QACvC,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;QAE1C,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;YAClC,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,MAAM;SACvB,CAAC,CAAC;QACH,EAAE,CAAC,QAAQ,CAAC,qDAAqD,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClF,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;gBACxC,IAAI;oBACF,MAAM,IAAA,gBAAM,EAAC,GAAG,CAAC,CAAC;oBAClB,gBAAM,CAAC,IAAI,CAAC,uBAAuB,GAAG,EAAE,CAAC,CAAC;iBAC3C;gBAAC,OAAO,GAAG,EAAE;oBACZ,gBAAM,CAAC,KAAK,CAAC,2BAA2B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;iBACxD;aACF;YACD,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,gBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AA1DD,oBA0DC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "Prompt engineering toolkit",
   "author": "Ian Webster",
-  "version": "0.14.1",
+  "version": "0.15.0",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",
@@ -79,7 +79,7 @@
     "nunjucks": "^3.2.4",
     "opener": "^1.5.2",
     "rouge": "^1.0.3",
-    "semver": "^7.5.1",
+    "semver": "^7.5.3",
     "socket.io": "^4.6.1",
     "tiny-invariant": "^1.3.1",
     "winston": "^3.8.2"

package/src/assertions.ts

@@ -32,6 +32,7 @@ function handleRougeScore(
 
   return {
     pass,
+    score: inverted ? 1 - score : score,
     reason: pass
       ? `${baseType.toUpperCase()} score ${score} is greater than or equal to threshold ${
           assertion.threshold || 0.75
@@ -49,24 +50,36 @@ export async function runAssertions(test: AtomicTestCase, output: string): Promi
     completion: 0,
   };
 
-  if (!test.assert) {
-    return { pass: true, reason: 'No assertions', tokensUsed };
+  if (!test.assert || test.assert.length < 1) {
+    return { pass: true, score: 1, reason: 'No assertions', tokensUsed };
   }
 
+  let totalScore = 0;
+  let totalWeight = 0;
   for (const assertion of test.assert) {
-    const result = await runAssertion(assertion, test, output);
-    if (!result.pass) {
-      return result;
-    }
+    const weight = assertion.weight || 1;
+    totalWeight += weight;
 
+    const result = await runAssertion(assertion, test, output);
+    totalScore += result.score * weight;
     if (result.tokensUsed) {
       tokensUsed.total += result.tokensUsed.total;
       tokensUsed.prompt += result.tokensUsed.prompt;
       tokensUsed.completion += result.tokensUsed.completion;
     }
+
+    if (!result.pass) {
+      // Short-circuit assertions
+      return result;
+    }
   }
 
-  return { pass: true, reason: 'All assertions passed', tokensUsed };
+  return {
+    pass: true,
+    score: totalScore / totalWeight,
+    reason: 'All assertions passed',
+    tokensUsed,
+  };
 }
 
 export async function runAssertion(
@@ -75,6 +88,7 @@ export async function runAssertion(
   output: string,
 ): Promise<GradingResult> {
   let pass: boolean = false;
+  let score: number = 0.0;
 
   invariant(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
 
@@ -89,6 +103,7 @@ export async function runAssertion(
     pass = assertion.value === output;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass ? 'Assertion passed' : `Expected output "${assertion.value}"`,
     };
   }
@@ -100,7 +115,11 @@ export async function runAssertion(
     } catch (err) {
       pass = inverse;
     }
-    return { pass, reason: pass ? 'Assertion passed' : 'Expected output to be valid JSON' };
+    return {
+      pass,
+      score: pass ? 1 : 0,
+      reason: pass ? 'Assertion passed' : 'Expected output to be valid JSON',
+    };
   }
 
   if (baseType === 'contains') {
@@ -112,6 +131,7 @@ export async function runAssertion(
     pass = output.includes(assertion.value) !== inverse;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
         : `Expected output to ${inverse ? 'not ' : ''}contain "${assertion.value}"`,
@@ -127,6 +147,7 @@ export async function runAssertion(
     pass = assertion.value.some((value) => output.includes(value)) !== inverse;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
         : `Expected output to ${inverse ? 'not ' : ''}contain one of "${assertion.value.join(
@@ -144,6 +165,7 @@ export async function runAssertion(
     pass = assertion.value.every((value) => output.includes(value)) !== inverse;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
         : `Expected output to ${inverse ? 'not ' : ''}contain all of "${assertion.value.join(
@@ -162,6 +184,7 @@ export async function runAssertion(
     pass = regex.test(output) !== inverse;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
         : `Expected output to ${inverse ? 'not ' : ''}match regex "${assertion.value}"`,
@@ -177,6 +200,7 @@ export async function runAssertion(
     pass = output.toLowerCase().includes(assertion.value.toLowerCase()) !== inverse;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
         : `Expected output to ${inverse ? 'not ' : ''}contain "${assertion.value}"`,
@@ -187,6 +211,7 @@ export async function runAssertion(
     pass = containsJSON(output) !== inverse;
     return {
       pass,
+      score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
         : `Expected output to ${inverse ? 'not ' : ''}contain valid JSON`,
@@ -199,16 +224,27 @@ export async function runAssertion(
       const context = {
         vars: test.vars || {},
       };
-      pass = customFunction(output, context) !== inverse;
+      const result = customFunction(output, context) as any;
+      if (typeof result === 'boolean') {
+        pass = result !== inverse;
+        score = 1.0;
+      } else if (typeof result === 'number') {
+        pass = true;
+        score = result;
+      } else {
+        throw new Error('Custom function must return a boolean or number');
+      }
     } catch (err) {
       return {
         pass: false,
+        score: 0,
         reason: `Custom function threw error: ${(err as Error).message}
 ${assertion.value}`,
       };
     }
     return {
       pass,
+      score,
       reason: pass
         ? 'Assertion passed'
         : `Custom function returned ${inverse ? 'true' : 'false'}
@@ -263,15 +299,25 @@ ${assertion.value}`,
 
       const jsonResponse = await response.json();
       pass = jsonResponse.pass !== inverse;
+      score =
+        typeof jsonResponse.score === 'undefined'
+          ? pass
+            ? 1
+            : 0
+          : inverse
+          ? 1 - jsonResponse.score
+          : jsonResponse.score;
     } catch (err) {
       return {
         pass: false,
+        score: 0,
         reason: `Webhook error: ${(err as Error).message}`,
       };
     }
 
     return {
       pass,
+      score,
       reason: pass ? 'Assertion passed' : `Webhook returned ${inverse ? 'true' : 'false'}`,
     };
   }
@@ -322,6 +368,7 @@ export async function matchesSimilarity(
   if (expectedEmbedding.error || outputEmbedding.error) {
     return {
       pass: false,
+      score: 0,
       reason:
         expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings',
       tokensUsed,
@@ -331,6 +378,7 @@ export async function matchesSimilarity(
   if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
     return {
       pass: false,
+      score: 0,
       reason: 'Embedding not found',
       tokensUsed,
     };
@@ -343,12 +391,14 @@ export async function matchesSimilarity(
   if (pass) {
     return {
       pass: true,
+      score: inverse ? 1 - similarity : similarity,
       reason: inverse ? lessThanReason : greaterThanReason,
       tokensUsed,
     };
   }
   return {
     pass: false,
+    score: inverse ? 1 - similarity : similarity,
     reason: inverse ? greaterThanReason : lessThanReason,
     tokensUsed,
   };
@@ -366,7 +416,7 @@ export async function matchesLlmRubric(
   }
 
   const prompt = nunjucks.renderString(options.rubricPrompt || DEFAULT_GRADING_PROMPT, {
-    content: output,
+    output,
     rubric: expected,
   });
 
@@ -378,6 +428,7 @@ export async function matchesLlmRubric(
   if (resp.error || !resp.output) {
     return {
       pass: false,
+      score: 0,
       reason: resp.error || 'No output',
       tokensUsed: {
         total: resp.tokenUsage?.total || 0,
@@ -388,16 +439,17 @@ export async function matchesLlmRubric(
   }
 
   try {
-    const parsed = JSON.parse(resp.output) as GradingResult;
+    const parsed = JSON.parse(resp.output) as Omit<GradingResult, 'score'>;
     parsed.tokensUsed = {
       total: resp.tokenUsage?.total || 0,
       prompt: resp.tokenUsage?.prompt || 0,
       completion: resp.tokenUsage?.completion || 0,
     };
-    return parsed;
+    return { ...parsed, score: parsed.pass ? 1 : 0 };
   } catch (err) {
     return {
       pass: false,
+      score: 0,
       reason: `Output is not valid JSON: ${resp.output}`,
       tokensUsed: {
         total: resp.tokenUsage?.total || 0,

package/src/evaluator.ts

@@ -109,6 +109,7 @@ class Evaluator {
         ...setup,
         response,
         success: false,
+        score: 0,
       };
       if (response.error) {
         ret.error = response.error;
@@ -118,6 +119,7 @@ class Evaluator {
           ret.error = checkResult.reason;
         }
         ret.success = checkResult.pass;
+        ret.score = checkResult.score;
         if (checkResult.tokensUsed) {
           this.stats.tokenUsage.total += checkResult.tokensUsed.total;
           this.stats.tokenUsage.prompt += checkResult.tokensUsed.prompt;
@@ -125,6 +127,7 @@ class Evaluator {
         }
       } else {
         ret.success = false;
+        ret.score = 0;
         ret.error = 'No output';
       }
 
@@ -148,6 +151,7 @@ class Evaluator {
         ...setup,
         error: String(err) + '\n\n' + (err as Error).stack,
         success: false,
+        score: 0,
       };
     }
   }
@@ -323,11 +327,12 @@ class Evaluator {
         if (progressbar) {
           progressbar.increment({
             provider: options.provider.id(),
-            prompt: options.prompt.raw.slice(0, 10),
+            prompt: options.prompt.raw.slice(0, 10).replace(/\n/g, ' '),
             vars: Object.entries(options.test.vars || {})
               .map(([k, v]) => `${k}=${v}`)
               .join(' ')
-              .slice(0, 10),
+              .slice(0, 10)
+              .replace(/\n/g, ' '),
           });
         }
 
@@ -339,12 +344,12 @@ class Evaluator {
         let resultText: string | undefined;
         if (isTest) {
           if (row.success) {
-            resultText = `[PASS] ${row.response?.output || row.error || ''}`;
+            resultText = `${row.response?.output || row.error || ''}`;
           } else {
-            resultText = `[FAIL] ${row.error}\n---\n${row.response?.output || row.error || ''}`;
+            resultText = `${row.error}\n---\n${row.response?.output || row.error || ''}`;
           }
         } else if (row.error) {
-          resultText = `[FAIL] ${row.error}`;
+          resultText = `${row.error}`;
         } else {
           resultText = row.response?.output || row.error || '';
         }
@@ -358,7 +363,11 @@ class Evaluator {
             vars: table.head.vars.map((varName) => options.test.vars?.[varName] || '').flat(),
           };
         }
-        table.body[rowIndex].outputs[colIndex] = resultText;
+        table.body[rowIndex].outputs[colIndex] = {
+          pass: row.success,
+          score: row.score,
+          text: resultText,
+        };
       },
     );
 
@@ -368,7 +377,7 @@ class Evaluator {
 
     telemetry.record('eval_ran', {});
 
-    return { version: 1, results, stats: this.stats, table };
+    return { version: 2, results, stats: this.stats, table };
   }
 }
 

package/src/main.ts

@@ -220,6 +220,7 @@ async function main() {
         prompts: cmdObj.prompts || config.prompts,
         providers: cmdObj.providers || config.providers,
         tests: cmdObj.tests || cmdObj.vars || config.tests,
+        defaultTest: config.defaultTest,
       };
 
       // Validation

package/src/prompts.ts

@@ -1,21 +1,21 @@
 export const DEFAULT_GRADING_PROMPT = JSON.stringify([
   {
     role: 'system',
-    content: `You are grading content according to a user-specified rubric. If the statement in the rubric is true, then the content passes the test. You respond with a JSON object with this structure: {pass: boolean; reason: string;}.
+    content: `You are grading output according to a user-specified rubric. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {pass: boolean; reason: string;}.
 
 Examples:
 
-Content: Hello world
-Rubric: Contains a greeting
+Output: Hello world
+Rubric: Content contains a greeting
 {"pass": true, "reason": "the content contains the word 'world'"}
 
-Content: Avast ye swabs, repel the invaders!
+Output: Avast ye swabs, repel the invaders!
 Rubric: Does not speak like a pirate
 {"pass": false, "reason": "'avast ye' is a common pirate term"}`,
   },
   {
     role: 'user',
-    content: 'Content: {{ content }}\nRubric: {{ rubric }}',
+    content: 'Output: {{ output }}\nRubric: {{ rubric }}',
   },
 ]);
 
@@ -27,3 +27,13 @@ Substantially revise the prompt, revising its structure and content however nece
 
 Your output is going to be copied directly into the program. It should contain the prompt ONLY`,
 };
+
+export const REFUSAL_PROMPT_SYSTEM_MESSAGE = {
+  role: 'system',
+  content: `In this task, you will be given a string of text. Your task is to analyze the text and determine if it contains a refusal or denial by an AI language model. This could be in the form of phrases like 'As an AI language model, I can't...', 'I'm unable to...', 'Sorry, but I can't...', etc. If such a refusal or denial is present in the text, return 'True'. If no such refusal is found, return 'False'.`,
+};
+
+export const AI_SELF_REFERENCE_PROMPT_SYSTEM_MESSAGE = {
+  role: 'system',
+  content: `Analyze the given text and determine whether it refers to itself as an AI, chatbot, assistant, or any similar entity. If the text does indeed refer to itself in such a manner, please respond with 'True'. Otherwise, respond with 'False'.`,
+};

package/src/providers/openai.ts

@@ -291,5 +291,5 @@ export class OpenAiChatCompletionProvider extends OpenAiGenericProvider {
 }
 
 export const DefaultEmbeddingProvider = new OpenAiEmbeddingProvider('text-embedding-ada-002');
-export const DefaultGradingProvider = new OpenAiChatCompletionProvider('gpt-4');
+export const DefaultGradingProvider = new OpenAiChatCompletionProvider('gpt-4-0613');
 export const DefaultSuggestionsProvider = new OpenAiChatCompletionProvider('gpt-4');

package/src/table.ts

@@ -19,21 +19,23 @@ export function generateTable(summary: EvaluateSummary, tableCellMaxLength = 250
   for (const row of summary.table.body.slice(0, maxRows)) {
     table.push([
       ...row.vars,
-      ...row.outputs.map((col) => {
-        if (col.length > tableCellMaxLength) {
-          col = col.slice(0, tableCellMaxLength) + '...';
+      ...row.outputs.map(({ pass, score, text }) => {
+        if (text.length > tableCellMaxLength) {
+          text = text.slice(0, tableCellMaxLength) + '...';
         }
-        if (col.startsWith('[PASS]')) {
-          // color '[PASS]' green
-          return chalk.green.bold(col.slice(0, 6)) + col.slice(6);
-        } else if (col.startsWith('[FAIL]')) {
+        if (pass) {
+          return chalk.green.bold('[PASS] ') + text;
+        } else if (!pass) {
           // color everything red up until '---'
-          return col
-            .split('---')
-            .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
-            .join('---');
+          return (
+            chalk.red.bold('[FAIL] ') +
+            text
+              .split('---')
+              .map((c, idx) => (idx === 0 ? chalk.red.bold(c) : c))
+              .join('---')
+          );
         }
-        return col;
+        return text;
       }),
     ]);
   }

package/src/types.ts

@@ -88,6 +88,13 @@ export interface EvaluateResult {
   response?: ProviderResponse;
   error?: string;
   success: boolean;
+  score: number;
+}
+
+export interface EvaluateTableOutput {
+  pass: boolean;
+  score: number;
+  text: string;
 }
 
 export interface EvaluateTable {
@@ -97,7 +104,7 @@ export interface EvaluateTable {
   };
 
   body: {
-    outputs: string[];
+    outputs: EvaluateTableOutput[];
     vars: string[];
   }[];
 }
@@ -117,6 +124,7 @@ export interface EvaluateSummary {
 
 export interface GradingResult {
   pass: boolean;
+  score: number;
   reason: string;
   tokensUsed?: TokenUsage;
 }
@@ -153,6 +161,9 @@ export interface Assertion {
   // The threshold value, only applicable for similarity (cosine distance)
   threshold?: number;
 
+  // The weight of this assertion compared to other assertions in the test case. Defaults to 1.
+  weight?: number;
+
   // Some assertions (similarity, llm-rubric) require an LLM provider
   provider?: ApiProvider;
 }

package/src/util.ts

@@ -17,7 +17,15 @@ import { getDirectory } from './esm';
 
 import type { RequestInfo, RequestInit, Response } from 'node-fetch';
 
-import type { Assertion, CsvRow, EvaluateSummary, UnifiedConfig, TestCase, Prompt } from './types';
+import type {
+  Assertion,
+  CsvRow,
+  EvaluateSummary,
+  EvaluateTableOutput,
+  UnifiedConfig,
+  TestCase,
+  Prompt,
+} from './types';
 
 const PROMPT_DELIMITER = '---';
 
@@ -211,10 +219,13 @@ export function writeOutput(
 ): void {
   const outputExtension = outputPath.split('.').pop()?.toLowerCase();
 
+  const outputToSimpleString = (output: EvaluateTableOutput) =>
+    `${output.pass ? '[PASS]' : '[FAIL]'} (${output.score.toFixed(2)}) ${output.text}`;
+
   if (outputExtension === 'csv' || outputExtension === 'txt') {
     const csvOutput = stringify([
       [...results.table.head.prompts, ...results.table.head.vars],
-      ...results.table.body.map((row) => [...row.outputs, ...row.vars]),
+      ...results.table.body.map((row) => [...row.outputs.map(outputToSimpleString), ...row.vars]),
     ]);
     fs.writeFileSync(outputPath, csvOutput);
   } else if (outputExtension === 'json') {
@@ -225,7 +236,7 @@ export function writeOutput(
     const template = fs.readFileSync(`${getDirectory()}/tableOutput.html`, 'utf-8');
     const table = [
       [...results.table.head.prompts, ...results.table.head.vars],
-      ...results.table.body.map((row) => [...row.outputs, ...row.vars]),
+      ...results.table.body.map((row) => [...row.outputs.map(outputToSimpleString), ...row.vars]),
     ];
     const htmlOutput = nunjucks.renderString(template, {
       table,

package/src/web/client/src/ResultsTable.css

@@ -118,6 +118,10 @@ td .status {
   font-weight: bold;
 }
 
+td .score {
+  font-weight: normal;
+}
+
 td .pass {
   color: var(--pass-color);
 }