promptfoo 0.21.3 → 0.22.0

package/dist/src/matchers.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.matchesLlmRubric = exports.matchesSimilarity = exports.getGradingProvider = void 0;
+exports.matchesClosedQa = exports.matchesFactuality = exports.matchesLlmRubric = exports.matchesSimilarity = exports.getGradingProvider = void 0;
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const openai_1 = require("./providers/openai");
 const util_1 = require("./util");
@@ -127,4 +127,146 @@ async function matchesLlmRubric(expected, output, grading) {
     }
 }
 exports.matchesLlmRubric = matchesLlmRubric;
+async function matchesFactuality(input, expected, output, grading) {
+    if (!grading) {
+        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
+    }
+    const prompt = nunjucks.renderString(grading.rubricPrompt || prompts_1.OPENAI_FACTUALITY_PROMPT, {
+        input: input.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+        ideal: expected.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+        completion: output.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+    });
+    let provider = grading.provider;
+    let finalProvider = await getGradingProvider(provider, openai_1.DefaultGradingProvider);
+    const resp = await finalProvider.callApi(prompt);
+    if (resp.error || !resp.output) {
+        return {
+            pass: false,
+            score: 0,
+            reason: resp.error || 'No output',
+            tokensUsed: {
+                total: resp.tokenUsage?.total || 0,
+                prompt: resp.tokenUsage?.prompt || 0,
+                completion: resp.tokenUsage?.completion || 0,
+            },
+        };
+    }
+    try {
+        const option = resp.output.trim().charAt(1); // Extract the option character
+        let reason = '';
+        const scoreLookup = {
+            A: grading.closedQa?.subset ?? 1,
+            B: grading.closedQa?.superset ?? 1,
+            C: grading.closedQa?.agree ?? 1,
+            D: grading.closedQa?.disagree ?? 0,
+            E: grading.closedQa?.differButFactual ?? 1,
+        };
+        // Passing is defined as scores with value >0, and failing as scores with value 0.
+        const passing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] > 0);
+        const failing = Object.keys(scoreLookup).filter((key) => scoreLookup[key] === 0);
+        let pass = passing.includes(option) && !failing.includes(option);
+        const optionReasons = {
+            A: `The submitted answer is a subset of the expert answer and is fully consistent with it.`,
+            B: `The submitted answer is a superset of the expert answer and is fully consistent with it.`,
+            C: `The submitted answer contains all the same details as the expert answer.`,
+            D: `There is a disagreement between the submitted answer and the expert answer.`,
+            E: `The answers differ, but these differences don't matter from the perspective of factuality.`,
+        };
+        if (optionReasons[option]) {
+            reason = optionReasons[option];
+        }
+        else {
+            pass = false;
+            reason = `Invalid option: ${option}`;
+        }
+        let score = pass ? 1 : 0;
+        if (typeof scoreLookup[option] !== 'undefined') {
+            score = scoreLookup[option];
+        }
+        return {
+            pass,
+            score,
+            reason,
+            tokensUsed: {
+                total: resp.tokenUsage?.total || 0,
+                prompt: resp.tokenUsage?.prompt || 0,
+                completion: resp.tokenUsage?.completion || 0,
+            },
+        };
+    }
+    catch (err) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Error parsing output: ${err.message}`,
+            tokensUsed: {
+                total: resp.tokenUsage?.total || 0,
+                prompt: resp.tokenUsage?.prompt || 0,
+                completion: resp.tokenUsage?.completion || 0,
+            },
+        };
+    }
+}
+exports.matchesFactuality = matchesFactuality;
+async function matchesClosedQa(input, expected, output, grading) {
+    if (!grading) {
+        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
+    }
+    const prompt = nunjucks.renderString(grading.rubricPrompt || prompts_1.OPENAI_CLOSED_QA_PROMPT, {
+        input: input.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+        criteria: expected.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+        completion: output.replace(/\n/g, '\\n').replace(/"/g, '\\"'),
+    });
+    let provider = grading.provider;
+    let finalProvider = await getGradingProvider(provider, openai_1.DefaultGradingProvider);
+    const resp = await finalProvider.callApi(prompt);
+    if (resp.error || !resp.output) {
+        return {
+            pass: false,
+            score: 0,
+            reason: resp.error || 'No output',
+            tokensUsed: {
+                total: resp.tokenUsage?.total || 0,
+                prompt: resp.tokenUsage?.prompt || 0,
+                completion: resp.tokenUsage?.completion || 0,
+            },
+        };
+    }
+    try {
+        const pass = resp.output.endsWith('Y');
+        let reason;
+        if (pass) {
+            reason = 'The submission meets the criterion';
+        }
+        else if (resp.output.endsWith('N')) {
+            reason = `The submission does not meet the criterion: ${resp.output}`;
+        }
+        else {
+            reason = 'Model grader received an ambiguous response';
+        }
+        return {
+            pass,
+            score: pass ? 1 : 0,
+            reason,
+            tokensUsed: {
+                total: resp.tokenUsage?.total || 0,
+                prompt: resp.tokenUsage?.prompt || 0,
+                completion: resp.tokenUsage?.completion || 0,
+            },
+        };
+    }
+    catch (err) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Error parsing output: ${err.message}`,
+            tokensUsed: {
+                total: resp.tokenUsage?.total || 0,
+                prompt: resp.tokenUsage?.prompt || 0,
+                completion: resp.tokenUsage?.completion || 0,
+            },
+        };
+    }
+}
+exports.matchesClosedQa = matchesClosedQa;
 //# sourceMappingURL=matchers.js.map

package/dist/src/matchers.js.map

@@ -1 +1 @@
-{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,+CAAsF;AACtF,iCAA6D;AAC7D,2CAA8C;AAC9C,uCAAmD;AAInD,MAAM,QAAQ,GAAG,IAAA,wBAAiB,GAAE,CAAC;AAE9B,KAAK,UAAU,kBAAkB,CACtC,QAAmC,EACnC,eAA4B;IAE5B,IAAI,aAA0B,CAAC;IAC/B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,aAAa,GAAG,MAAM,IAAA,2BAAe,EAAC,QAAQ,CAAC,CAAC;KACjD;SAAM,IACL,OAAO,QAAQ,KAAK,QAAQ;QAC5B,OAAQ,QAAwB,CAAC,OAAO,KAAK,UAAU,EACvD;QACA,qCAAqC;QACrC,aAAa,GAAG,QAAuB,CAAC;KACzC;SAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QACvC,6BAA6B;QAC7B,MAAM,UAAU,GAAG,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;QACnF,IAAA,wBAAS,EAAC,UAAU,EAAE,iDAAiD,CAAC,CAAC;QACzE,4DAA4D;QAC5D,aAAa,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,EAAE,EAAE,OAAO,EAAE,QAA2B,EAAE,CAAC,CAAC;KAC7F;SAAM;QACL,aAAa,GAAG,eAAe,CAAC;KACjC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAvBD,gDAuBC;AAEM,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAc,EACd,SAAiB,EACjB,UAAmB,KAAK,EACxB,OAAuB;IAEvB,IAAI,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACjC,IAAI,aAAa,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,iCAAwB,CAAC,CAAC;IAEjF,IAAA,wBAAS,EACP,aAAa,CAAC,gBAAgB,EAC9B,+DAA+D,CAChE,CAAC;IAEF,MAAM,iBAAiB,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzE,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAErE,MAAM,UAAU,GAAG;QACjB,KAAK,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;QAC5F,MAAM,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;QAC/F,UAAU,EACR,CAAC,iBAAiB,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YAC/C,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;KAChD,CAAC;IAEF,IAAI,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,EAAE;QACpD,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EACJ,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,IAAI,mCAAmC;YACzF,UAAU;SACX,CAAC;KACH;IAED,IAAI,CAAC,iBAAiB,CAAC,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;QAC9D,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB;YAC7B,UAAU;SACX,CAAC;KACH;IAED,MAAM,UAAU,GAAG,IAAA,uBAAgB,EAAC,iBAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;IAC5F,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC;IACzE,MAAM,iBAAiB,GAAG,cAAc,UAAU,8BAA8B,SAAS,EAAE,CAAC;IAC5F,MAAM,cAAc,GAAG,cAAc,UAAU,2BAA2B,SAAS,EAAE,CAAC;IACtF,IAAI,IAAI,EAAE;QACR,OAAO;YACL,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU;YAC5C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB;YACpD,UAAU;SACX,CAAC;KACH;IACD,OAAO;QACL,IAAI,EAAE,KAAK;QACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU;QAC5C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,cAAc;QACpD,UAAU;KACX,CAAC;AACJ,CAAC;AA/DD,8CA+DC;AAEM,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,MAAc,EACd,OAAuB;IAEvB,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,IAAI,gCAAsB,EAAE;QACnF,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QACzD,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;KAC5D,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,aAAa,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,+BAAsB,CAAC,CAAC;IAC/E,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAC9B,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,WAAW;YACjC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAED,IAAI;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAiC,CAAC;QACvE,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;SAC7C,CAAC;QACF,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE;YAClD,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;AACH,CAAC;AApDD,4CAoDC"}
+{"version":3,"file":"matchers.js","sourceRoot":"","sources":["../../src/matchers.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,+CAAsF;AACtF,iCAA6D;AAC7D,2CAA8C;AAC9C,uCAImB;AAInB,MAAM,QAAQ,GAAG,IAAA,wBAAiB,GAAE,CAAC;AAE9B,KAAK,UAAU,kBAAkB,CACtC,QAAmC,EACnC,eAA4B;IAE5B,IAAI,aAA0B,CAAC;IAC/B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,aAAa,GAAG,MAAM,IAAA,2BAAe,EAAC,QAAQ,CAAC,CAAC;KACjD;SAAM,IACL,OAAO,QAAQ,KAAK,QAAQ;QAC5B,OAAQ,QAAwB,CAAC,OAAO,KAAK,UAAU,EACvD;QACA,qCAAqC;QACrC,aAAa,GAAG,QAAuB,CAAC;KACzC;SAAM,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QACvC,6BAA6B;QAC7B,MAAM,UAAU,GAAG,OAAO,QAAQ,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,CAAC;QACnF,IAAA,wBAAS,EAAC,UAAU,EAAE,iDAAiD,CAAC,CAAC;QACzE,4DAA4D;QAC5D,aAAa,GAAG,MAAM,IAAA,2BAAe,EAAC,UAAU,EAAE,EAAE,OAAO,EAAE,QAA2B,EAAE,CAAC,CAAC;KAC7F;SAAM;QACL,aAAa,GAAG,eAAe,CAAC;KACjC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAvBD,gDAuBC;AAEM,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAc,EACd,SAAiB,EACjB,UAAmB,KAAK,EACxB,OAAuB;IAEvB,IAAI,QAAQ,GAAG,OAAO,EAAE,QAAQ,CAAC;IACjC,IAAI,aAAa,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,iCAAwB,CAAC,CAAC;IAEjF,IAAA,wBAAS,EACP,aAAa,CAAC,gBAAgB,EAC9B,+DAA+D,CAChE,CAAC;IAEF,MAAM,iBAAiB,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACzE,MAAM,eAAe,GAAG,MAAM,aAAa,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAErE,MAAM,UAAU,GAAG;QACjB,KAAK,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;QAC5F,MAAM,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;QAC/F,UAAU,EACR,CAAC,iBAAiB,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YAC/C,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;KAChD,CAAC;IAEF,IAAI,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,EAAE;QACpD,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EACJ,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,IAAI,mCAAmC;YACzF,UAAU;SACX,CAAC;KACH;IAED,IAAI,CAAC,iBAAiB,CAAC,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;QAC9D,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,qBAAqB;YAC7B,UAAU;SACX,CAAC;KACH;IAED,MAAM,UAAU,GAAG,IAAA,uBAAgB,EAAC,iBAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;IAC5F,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC;IACzE,MAAM,iBAAiB,GAAG,cAAc,UAAU,8BAA8B,SAAS,EAAE,CAAC;IAC5F,MAAM,cAAc,GAAG,cAAc,UAAU,2BAA2B,SAAS,EAAE,CAAC;IACtF,IAAI,IAAI,EAAE;QACR,OAAO;YACL,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU;YAC5C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB;YACpD,UAAU;SACX,CAAC;KACH;IACD,OAAO;QACL,IAAI,EAAE,KAAK;QACX,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU;QAC5C,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,cAAc;QACpD,UAAU;KACX,CAAC;AACJ,CAAC;AA/DD,8CA+DC;AAEM,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,MAAc,EACd,OAAuB;IAEvB,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,IAAI,gCAAsB,EAAE;QACnF,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QACzD,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;KAC5D,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,aAAa,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,+BAAsB,CAAC,CAAC;IAC/E,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAC9B,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,WAAW;YACjC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAED,IAAI;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAiC,CAAC;QACvE,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;SAC7C,CAAC;QACF,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;KAClD;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE;YAClD,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;AACH,CAAC;AApDD,4CAoDC;AAEM,KAAK,UAAU,iBAAiB,CACrC,KAAa,EACb,QAAgB,EAChB,MAAc,EACd,OAAuB;IAEvB,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,IAAI,kCAAwB,EAAE;QACrF,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QACvD,KAAK,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QAC1D,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;KAC9D,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,aAAa,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,+BAAsB,CAAC,CAAC;IAC/E,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAC9B,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,WAAW;YACjC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAED,IAAI;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,+BAA+B;QAC5E,IAAI,MAAM,GAAG,EAAE,CAAC;QAEhB,MAAM,WAAW,GAA2B;YAC1C,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAC;YAChC,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC;YAClC,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK,IAAI,CAAC;YAC/B,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC;YAClC,CAAC,EAAE,OAAO,CAAC,QAAQ,EAAE,gBAAgB,IAAI,CAAC;SAC3C,CAAC;QAEF,kFAAkF;QAClF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAEjF,IAAI,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACjE,MAAM,aAAa,GAA2B;YAC5C,CAAC,EAAE,wFAAwF;YAC3F,CAAC,EAAE,0FAA0F;YAC7F,CAAC,EAAE,0EAA0E;YAC7E,CAAC,EAAE,6EAA6E;YAChF,CAAC,EAAE,4FAA4F;SAChG,CAAC;QACF,IAAI,aAAa,CAAC,MAAM,CAAC,EAAE;YACzB,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;SAChC;aAAM;YACL,IAAI,GAAG,KAAK,CAAC;YACb,MAAM,GAAG,mBAAmB,MAAM,EAAE,CAAC;SACtC;QAED,IAAI,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,OAAO,WAAW,CAAC,MAAM,CAAC,KAAK,WAAW,EAAE;YAC9C,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;SAC7B;QAED,OAAO;YACL,IAAI;YACJ,KAAK;YACL,MAAM;YACN,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,yBAA0B,GAAa,CAAC,OAAO,EAAE;YACzD,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;AACH,CAAC;AA5FD,8CA4FC;AAEM,KAAK,UAAU,eAAe,CACnC,KAAa,EACb,QAAgB,EAChB,MAAc,EACd,OAAuB;IAEvB,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,IAAI,iCAAuB,EAAE;QACpF,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QACvD,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;QAC7D,UAAU,EAAE,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC;KAC9D,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,aAAa,GAAG,MAAM,kBAAkB,CAAC,QAAQ,EAAE,+BAAsB,CAAC,CAAC;IAC/E,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAC9B,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,WAAW;YACjC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAED,IAAI;QACF,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,MAAM,CAAC;QACX,IAAI,IAAI,EAAE;YACR,MAAM,GAAG,oCAAoC,CAAC;SAC/C;aAAM,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;YACpC,MAAM,GAAG,+CAA+C,IAAI,CAAC,MAAM,EAAE,CAAC;SACvE;aAAM;YACL,MAAM,GAAG,6CAA6C,CAAC;SACxD;QACD,OAAO;YACL,IAAI;YACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnB,MAAM;YACN,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,yBAA0B,GAAa,CAAC,OAAO,EAAE;YACzD,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;AACH,CAAC;AAlED,0CAkEC"}

package/dist/src/prompts.d.ts

@@ -1,4 +1,6 @@
 export declare const DEFAULT_GRADING_PROMPT: string;
+export declare const OPENAI_FACTUALITY_PROMPT: string;
+export declare const OPENAI_CLOSED_QA_PROMPT: string;
 export declare const SUGGEST_PROMPTS_SYSTEM_MESSAGE: {
     role: string;
     content: string;

package/dist/src/prompts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,sBAAsB,QAmBjC,CAAC;AAEH,eAAO,MAAM,8BAA8B;;;CAO1C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;CAGzC,CAAC;AAEF,eAAO,MAAM,uCAAuC;;;CAGnD,CAAC"}
+{"version":3,"file":"prompts.d.ts","sourceRoot":"","sources":["../../src/prompts.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,sBAAsB,QAmBjC,CAAC;AAGH,eAAO,MAAM,wBAAwB,QAsBnC,CAAC;AAEH,eAAO,MAAM,uBAAuB,QAiBlC,CAAC;AAEH,eAAO,MAAM,8BAA8B;;;CAO1C,CAAC;AAEF,eAAO,MAAM,6BAA6B;;;CAGzC,CAAC;AAEF,eAAO,MAAM,uCAAuC;;;CAGnD,CAAC"}

package/dist/src/prompts.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AI_SELF_REFERENCE_PROMPT_SYSTEM_MESSAGE = exports.REFUSAL_PROMPT_SYSTEM_MESSAGE = exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = exports.DEFAULT_GRADING_PROMPT = void 0;
+exports.AI_SELF_REFERENCE_PROMPT_SYSTEM_MESSAGE = exports.REFUSAL_PROMPT_SYSTEM_MESSAGE = exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = exports.OPENAI_CLOSED_QA_PROMPT = exports.OPENAI_FACTUALITY_PROMPT = exports.DEFAULT_GRADING_PROMPT = void 0;
 exports.DEFAULT_GRADING_PROMPT = JSON.stringify([
     {
         role: 'system',
@@ -21,6 +21,48 @@ Rubric: Does not speak like a pirate
         content: 'Output: {{ output }}\nRubric: {{ rubric }}',
     },
 ]);
+// https://github.com/openai/evals/blob/main/evals/registry/modelgraded/fact.yaml
+exports.OPENAI_FACTUALITY_PROMPT = JSON.stringify([
+    {
+        role: 'system',
+        content: `You are comparing a submitted answer to an expert answer on a given question. Here is the data:
+[BEGIN DATA]
+************
+[Question]: {{input}}
+************
+[Expert]: {{ideal}}
+************
+[Submission]: {{completion}}
+************
+[END DATA]
+
+Compare the factual content of the submitted answer with the expert answer. Ignore any differences in style, grammar, or punctuation.
+The submitted answer may either be a subset or superset of the expert answer, or it may conflict with it. Determine which case applies. Answer the question by selecting one of the following options:
+(A) The submitted answer is a subset of the expert answer and is fully consistent with it.
+(B) The submitted answer is a superset of the expert answer and is fully consistent with it.
+(C) The submitted answer contains all the same details as the expert answer.
+(D) There is a disagreement between the submitted answer and the expert answer.
+(E) The answers differ, but these differences don't matter from the perspective of factuality.`,
+    },
+]);
+exports.OPENAI_CLOSED_QA_PROMPT = JSON.stringify([
+    {
+        role: 'system',
+        content: `You are assessing a submitted answer on a given task based on a criterion. Here is the data:
+[BEGIN DATA]
+***
+[Task]: {{input}}
+***
+[Submission]: {{completion}}
+***
+[Criterion]: {{criteria}}
+***
+[END DATA]
+Does the submission meet the criterion? First, write out in a step by step manner your reasoning about the criterion to be sure that your conclusion is correct. Avoid simply stating the correct answers at the outset. Then print only the single character "Y" or "N" (without quotes or punctuation) on its own line corresponding to the correct answer. At the end, repeat just the letter again by itself on a new line.
+
+    Reasoning:`,
+    },
+]);
 exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = {
     role: 'system',
     content: `You're helping a scientist who is tuning a prompt for a large language model.  You will receive messages, and each message is a full prompt.  Generate a candidate variation of the given prompt.  This variation will be tested for quality in order to select a winner.

package/dist/src/prompts.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/prompts.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,GAAG,IAAI,CAAC,SAAS,CAAC;IACnD;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;;;;;;;;;;gEAUmD;KAC7D;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,4CAA4C;KACtD;CACF,CAAC,CAAC;AAEU,QAAA,8BAA8B,GAAG;IAC5C,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;;;;+FAIoF;CAC9F,CAAC;AAEW,QAAA,6BAA6B,GAAG;IAC3C,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,8aAA8a;CACxb,CAAC;AAEW,QAAA,uCAAuC,GAAG;IACrD,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,sTAAsT;CAChU,CAAC"}
+{"version":3,"file":"prompts.js","sourceRoot":"","sources":["../../src/prompts.ts"],"names":[],"mappings":";;;AAAa,QAAA,sBAAsB,GAAG,IAAI,CAAC,SAAS,CAAC;IACnD;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;;;;;;;;;;gEAUmD;KAC7D;IACD;QACE,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,4CAA4C;KACtD;CACF,CAAC,CAAC;AAEH,iFAAiF;AACpE,QAAA,wBAAwB,GAAG,IAAI,CAAC,SAAS,CAAC;IACrD;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;;;;;;;;;;;;;;;;;+FAiBkF;KAC5F;CACF,CAAC,CAAC;AAEU,QAAA,uBAAuB,GAAG,IAAI,CAAC,SAAS,CAAC;IACpD;QACE,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE;;;;;;;;;;;;eAYE;KACZ;CACF,CAAC,CAAC;AAEU,QAAA,8BAA8B,GAAG;IAC5C,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE;;;;+FAIoF;CAC9F,CAAC;AAEW,QAAA,6BAA6B,GAAG;IAC3C,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,8aAA8a;CACxb,CAAC;AAEW,QAAA,uCAAuC,GAAG;IACrD,IAAI,EAAE,QAAQ;IACd,OAAO,EAAE,sTAAsT;CAChU,CAAC"}

package/dist/src/providers/llama.d.ts

@@ -27,7 +27,7 @@ export declare class LlamaProvider implements ApiProvider {
     });
     id(): string;
     toString(): string;
-    callApi(prompt: string, options?: LlamaCompletionOptions): Promise<ProviderResponse>;
+    callApi(prompt: string): Promise<ProviderResponse>;
 }
 export {};
 //# sourceMappingURL=llama.d.ts.map

package/dist/src/providers/llama.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llama.d.ts","sourceRoot":"","sources":["../../../src/providers/llama.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEjE,UAAU,sBAAsB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,qBAAa,aAAc,YAAW,WAAW;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,sBAAsB,CAAC;gBAEpB,SAAS,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,sBAAsB,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAO;IAO7F,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoD3F"}
+{"version":3,"file":"llama.d.ts","sourceRoot":"","sources":["../../../src/providers/llama.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEjE,UAAU,sBAAsB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,qBAAa,aAAc,YAAW,WAAW;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,sBAAsB,CAAC;gBAEpB,SAAS,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,MAAM,CAAC,EAAE,sBAAsB,CAAC;QAAC,EAAE,CAAC,EAAE,MAAM,CAAA;KAAO;IAO7F,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAmDzD"}

package/dist/src/providers/llama.js

@@ -16,27 +16,26 @@ class LlamaProvider {
     toString() {
         return `[Llama Provider ${this.modelName}]`;
     }
-    async callApi(prompt, options) {
-        options = Object.assign({}, this.config, options);
+    async callApi(prompt) {
         const body = {
             prompt,
-            n_predict: options?.n_predict || 512,
-            temperature: options?.temperature,
-            top_k: options?.top_k,
-            top_p: options?.top_p,
-            n_keep: options?.n_keep,
-            stop: options?.stop,
-            repeat_penalty: options?.repeat_penalty,
-            repeat_last_n: options?.repeat_last_n,
-            penalize_nl: options?.penalize_nl,
-            presence_penalty: options?.presence_penalty,
-            frequency_penalty: options?.frequency_penalty,
-            mirostat: options?.mirostat,
-            mirostat_tau: options?.mirostat_tau,
-            mirostat_eta: options?.mirostat_eta,
-            seed: options?.seed,
-            ignore_eos: options?.ignore_eos,
-            logit_bias: options?.logit_bias,
+            n_predict: this.config?.n_predict || 512,
+            temperature: this.config?.temperature,
+            top_k: this.config?.top_k,
+            top_p: this.config?.top_p,
+            n_keep: this.config?.n_keep,
+            stop: this.config?.stop,
+            repeat_penalty: this.config?.repeat_penalty,
+            repeat_last_n: this.config?.repeat_last_n,
+            penalize_nl: this.config?.penalize_nl,
+            presence_penalty: this.config?.presence_penalty,
+            frequency_penalty: this.config?.frequency_penalty,
+            mirostat: this.config?.mirostat,
+            mirostat_tau: this.config?.mirostat_tau,
+            mirostat_eta: this.config?.mirostat_eta,
+            seed: this.config?.seed,
+            ignore_eos: this.config?.ignore_eos,
+            logit_bias: this.config?.logit_bias,
         };
         let response;
         try {

package/dist/src/providers/llama.js.map

@@ -1 +1 @@
-{"version":3,"file":"llama.js","sourceRoot":"","sources":["../../../src/providers/llama.ts"],"names":[],"mappings":";;;AAAA,oCAA0C;AAC1C,qCAA8C;AAwB9C,MAAa,aAAa;IAIxB,YAAY,SAAiB,EAAE,UAA4D,EAAE;QAC3F,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,EAAE;QACA,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED,QAAQ;QACN,OAAO,mBAAmB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAgC;QAC5D,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG;YACX,MAAM;YACN,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG;YACpC,WAAW,EAAE,OAAO,EAAE,WAAW;YACjC,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,MAAM,EAAE,OAAO,EAAE,MAAM;YACvB,IAAI,EAAE,OAAO,EAAE,IAAI;YACnB,cAAc,EAAE,OAAO,EAAE,cAAc;YACvC,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,WAAW,EAAE,OAAO,EAAE,WAAW;YACjC,gBAAgB,EAAE,OAAO,EAAE,gBAAgB;YAC3C,iBAAiB,EAAE,OAAO,EAAE,iBAAiB;YAC7C,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,YAAY,EAAE,OAAO,EAAE,YAAY;YACnC,YAAY,EAAE,OAAO,EAAE,YAAY;YACnC,IAAI,EAAE,OAAO,EAAE,IAAI;YACnB,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;SAChC,CAAC;QAEF,IAAI,QAAQ,CAAC;QACb,IAAI;YACF,QAAQ,GAAG,MAAM,IAAA,sBAAc,EAC7B,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,uBAAuB,aAAa,EACrE;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;aAC9B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;aAC9E,CAAC;SACH;IACH,CAAC;CACF;AAvED,sCAuEC"}
+{"version":3,"file":"llama.js","sourceRoot":"","sources":["../../../src/providers/llama.ts"],"names":[],"mappings":";;;AAAA,oCAA0C;AAC1C,qCAA8C;AAwB9C,MAAa,aAAa;IAIxB,YAAY,SAAiB,EAAE,UAA4D,EAAE;QAC3F,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,EAAE;QACA,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED,QAAQ;QACN,OAAO,mBAAmB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc;QAC1B,MAAM,IAAI,GAAG;YACX,MAAM;YACN,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,IAAI,GAAG;YACxC,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW;YACrC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK;YACzB,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM;YAC3B,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;YACvB,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc;YAC3C,aAAa,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa;YACzC,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,WAAW;YACrC,gBAAgB,EAAE,IAAI,CAAC,MAAM,EAAE,gBAAgB;YAC/C,iBAAiB,EAAE,IAAI,CAAC,MAAM,EAAE,iBAAiB;YACjD,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ;YAC/B,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY;YACvC,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY;YACvC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI;YACvB,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU;YACnC,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU;SACpC,CAAC;QAEF,IAAI,QAAQ,CAAC;QACb,IAAI;YACF,QAAQ,GAAG,MAAM,IAAA,sBAAc,EAC7B,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,uBAAuB,aAAa,EACrE;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;aAC9B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;aAC9E,CAAC;SACH;IACH,CAAC;CACF;AAtED,sCAsEC"}

package/dist/src/providers.js

@@ -169,7 +169,7 @@ async function loadApiProvider(providerPath, context = {}) {
         return new llama_1.LlamaProvider(modelName, providerOptions);
     }
     else if (providerPath.startsWith('ollama:')) {
-        const modelName = providerPath.split(':')[1];
+        const modelName = providerPath.split(':').slice(1).join(':');
         return new ollama_1.OllamaProvider(modelName, providerOptions);
     }
     else if (providerPath?.startsWith('localai:')) {

package/dist/src/providers.js.map

@@ -1 +1 @@
-{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAI4B;AAC5B,qDAAoE;AACpE,qDAA0D;AAC1D,iDAAqF;AACrF,6CAAkD;AAClD,+CAAoD;AACpD,iDAAsD;AACtD,mEAAwE;AACxE,yDAGiC;AAW1B,KAAK,UAAU,gBAAgB,CACpC,aAKoB,EACpB,UAGI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAClC,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;KAClE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACrD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM,IAAI,QAAQ,CAAC,EAAE,EAAE;gBACtB,iCAAiC;gBACjC,OAAO,eAAe,CAAE,QAA4B,CAAC,EAAG,EAAE;oBACxD,OAAO,EAAE,QAAQ;oBACjB,QAAQ;oBACR,GAAG;iBACJ,CAAC,CAAC;aACJ;iBAAM;gBACL,yDAAyD;gBACzD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,cAAc,GAAI,QAA+B,CAAC,EAAE,CAAC,CAAC;gBAC5D,MAAM,OAAO,GAAG,EAAE,GAAG,cAAc,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;gBACnE,OAAO,eAAe,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACjE;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAlDD,4CAkDC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,UAII,EAAE;IAEN,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,eAAe,GAAG;QACtB,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG;KACJ,CAAC;IACF,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CAAC,SAAS,IAAI,eAAe,EAAE,eAAe,CAAC,CAAC;SACxF;aAAM,IAAI,SAAS,KAAK,WAAW,EAAE;YACpC,OAAO,IAAI,gCAAuB,CAAC,SAAS,IAAI,wBAAwB,EAAE;gBACxE,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG;aACJ,CAAC,CAAC;SACJ;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SACvF;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACrE;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACjE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC/E;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC3E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SAC1F;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACpE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAC1D;IAED,IAAI,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACvC,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,OAAO,IAAI,yBAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;KACzD;SAAM,IAAI,YAAY,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxE,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,qBAAa,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACtD;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,uBAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACvD;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAClE;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AA9GD,0CA8GC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAI4B;AAC5B,qDAAoE;AACpE,qDAA0D;AAC1D,iDAAqF;AACrF,6CAAkD;AAClD,+CAAoD;AACpD,iDAAsD;AACtD,mEAAwE;AACxE,yDAGiC;AAW1B,KAAK,UAAU,gBAAgB,CACpC,aAKoB,EACpB,UAGI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAClC,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;KAClE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACrD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM,IAAI,QAAQ,CAAC,EAAE,EAAE;gBACtB,iCAAiC;gBACjC,OAAO,eAAe,CAAE,QAA4B,CAAC,EAAG,EAAE;oBACxD,OAAO,EAAE,QAAQ;oBACjB,QAAQ;oBACR,GAAG;iBACJ,CAAC,CAAC;aACJ;iBAAM;gBACL,yDAAyD;gBACzD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,cAAc,GAAI,QAA+B,CAAC,EAAE,CAAC,CAAC;gBAC5D,MAAM,OAAO,GAAG,EAAE,GAAG,cAAc,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;gBACnE,OAAO,eAAe,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACjE;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAlDD,4CAkDC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,UAII,EAAE;IAEN,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,eAAe,GAAG;QACtB,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG;KACJ,CAAC;IACF,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CAAC,SAAS,IAAI,eAAe,EAAE,eAAe,CAAC,CAAC;SACxF;aAAM,IAAI,SAAS,KAAK,WAAW,EAAE;YACpC,OAAO,IAAI,gCAAuB,CAAC,SAAS,IAAI,wBAAwB,EAAE;gBACxE,EAAE,EAAE,OAAO,CAAC,EAAE;gBACd,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,GAAG;aACJ,CAAC,CAAC;SACJ;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SACvF;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACrE;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACjE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC/E;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC3E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SAC1F;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACpE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAC1D;IAED,IAAI,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACvC,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,OAAO,IAAI,yBAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;KACzD;SAAM,IAAI,YAAY,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxE,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,qBAAa,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACtD;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,IAAI,uBAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACvD;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAClE;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AA9GD,0CA8GC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}

package/dist/src/table.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,wBAAgB,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,kBAAkB,SAAM,EAAE,OAAO,SAAK,eAsC7F"}
+{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,wBAAgB,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,kBAAkB,SAAM,EAAE,OAAO,SAAK,eA2C7F"}

package/dist/src/table.js

@@ -22,7 +22,12 @@ function generateTable(summary, tableCellMaxLength = 250, maxRows = 25) {
     // Skip first row (header) and add the rest. Color PASS/FAIL
     for (const row of summary.table.body.slice(0, maxRows)) {
         table.push([
-            ...row.vars,
+            ...row.vars.map((v) => {
+                if (v.length > tableCellMaxLength) {
+                    v = v.slice(0, tableCellMaxLength) + '...';
+                }
+                return v;
+            }),
             ...row.outputs.map(({ pass, score, text }) => {
                 if (text.length > tableCellMaxLength) {
                     text = text.slice(0, tableCellMaxLength) + '...';

package/dist/src/table.js.map

@@ -1 +1 @@
-{"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,kDAA0B;AAG1B,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACpE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,KAAK;QACzB,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI;YACX,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAClD;gBACD,IAAI,IAAI,EAAE;oBACR,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;iBACtC;qBAAM,IAAI,CAAC,IAAI,EAAE;oBAChB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;iBACH;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;KACJ;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAtCD,sCAsCC"}
+{"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,kDAA0B;AAG1B,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACpE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,KAAK;QACzB,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,IAAI,CAAC,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACjC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAC5C;gBACD,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;YACF,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAClD;gBACD,IAAI,IAAI,EAAE;oBACR,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;iBACtC;qBAAM,IAAI,CAAC,IAAI,EAAE;oBAChB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;iBACH;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;KACJ;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AA3CD,sCA2CC"}

package/dist/src/types.d.ts

@@ -39,7 +39,9 @@ export interface ProviderOptions {
 }
 export interface ApiProvider {
     id: () => string;
-    callApi: (prompt: string) => Promise<ProviderResponse>;
+    callApi: (prompt: string, context?: {
+        vars: Record<string, string | object>;
+    }) => Promise<ProviderResponse>;
     callEmbeddingApi?: (prompt: string) => Promise<ProviderEmbeddingResponse>;
 }
 export interface TokenUsage {
@@ -66,6 +68,13 @@ export type VarMapping = Record<string, string>;
 export interface GradingConfig {
     rubricPrompt?: string;
     provider?: string | ProviderOptions | ApiProvider;
+    closedQa?: {
+        subset?: number;
+        superset?: number;
+        agree?: number;
+        disagree?: number;
+        differButFactual?: number;
+    };
 }
 export interface PromptConfig {
     prefix?: string;
@@ -145,18 +154,19 @@ export interface Assertion {
     provider?: GradingConfig['provider'];
     rubricPrompt?: GradingConfig['rubricPrompt'];
 }
-export interface TestCase {
+export interface TestCase<Vars = Record<string, string | string[] | object>> {
     description?: string;
     vars?: Record<string, string | string[] | object>;
     assert?: Assertion[];
     options?: PromptConfig & OutputConfig & GradingConfig;
+    threshold?: number;
 }
 export interface Scenario {
     description?: string;
     config: Partial<TestCase>[];
     tests: TestCase[];
 }
-export interface AtomicTestCase extends TestCase {
+export interface AtomicTestCase<Vars = Record<string, string | object>> extends TestCase {
     vars?: Record<string, string | object>;
 }
 export interface TestSuite {

package/dist/src/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IAGd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,UAAU,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACvD,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;CACnD;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gBAAgB,CAAC,EAAE,aAAa,EAAE,CAAC;IACnC,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;CAC7B;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,SAAS,GACT,YAAY,GACZ,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,aAAa,CAAC;AAElB,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EACF,MAAM,GACN,MAAM,EAAE,GACR,MAAM,GACN,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;IAGjG,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAErC,YAAY,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CAC9C;AAGD,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;IAGlD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;CACvD;AAED,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IAG5B,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAGD,MAAM,WAAW,cAAe,SAAQ,QAAQ;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAIlB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAG7C,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAGhC,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE7E,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;AAGrE,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EACL,UAAU,GACV,UAAU,EAAE,GACZ,kBAAkB,EAAE,GACpB,eAAe,EAAE,GACjB,gBAAgB,CAAC;IAGrB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAGtC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAGlB,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC;AAGF,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,eAAe,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;KAChC,CAAC;CACH"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IAGd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,UAAU,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CACP,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;KACvC,KACE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/B,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;IAClD,QAAQ,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gBAAgB,CAAC,EAAE,aAAa,EAAE,CAAC;IACnC,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;CAC7B;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,SAAS,GACT,YAAY,GACZ,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,aAAa,CAAC;AAElB,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EACF,MAAM,GACN,MAAM,EAAE,GACR,MAAM,GACN,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;IAGjG,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAErC,YAAY,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CAC9C;AAGD,MAAM,WAAW,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;IAEzE,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;IAGlD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;IAGtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IAG5B,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAGD,MAAM,WAAW,cAAc,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAE,SAAQ,QAAQ;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAIlB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAG7C,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAGhC,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE7E,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;AAGrE,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EACL,UAAU,GACV,UAAU,EAAE,GACZ,kBAAkB,EAAE,GACpB,eAAe,EAAE,GACjB,gBAAgB,CAAC;IAGrB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAGtC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAGlB,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC;AAGF,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,eAAe,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;KAChC,CAAC;CACH"}

package/dist/src/web/nextui/404/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-c173e301236a84c2.js" defer=""></script><script src="/_next/static/chunks/framework-8883d1e9be70c3da.js" defer=""></script><script src="/_next/static/chunks/main-8ea85465d428ecfe.js" defer=""></script><script src="/_next/static/chunks/pages/_app-52924524f99094ab.js" defer=""></script><script src="/_next/static/chunks/pages/_error-c92d5c4bb2b49926.js" defer=""></script><script src="/_next/static/iLk2whM1PMpAQs7TcXt-n/_buildManifest.js" defer=""></script><script src="/_next/static/iLk2whM1PMpAQs7TcXt-n/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"iLk2whM1PMpAQs7TcXt-n","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-c173e301236a84c2.js" defer=""></script><script src="/_next/static/chunks/framework-8883d1e9be70c3da.js" defer=""></script><script src="/_next/static/chunks/main-8ea85465d428ecfe.js" defer=""></script><script src="/_next/static/chunks/pages/_app-52924524f99094ab.js" defer=""></script><script src="/_next/static/chunks/pages/_error-c92d5c4bb2b49926.js" defer=""></script><script src="/_next/static/RdnGIHHt12YXTL4egAcOk/_buildManifest.js" defer=""></script><script src="/_next/static/RdnGIHHt12YXTL4egAcOk/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"RdnGIHHt12YXTL4egAcOk","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

package/dist/src/web/nextui/404.html

@@ -1 +1 @@
-<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-c173e301236a84c2.js" defer=""></script><script src="/_next/static/chunks/framework-8883d1e9be70c3da.js" defer=""></script><script src="/_next/static/chunks/main-8ea85465d428ecfe.js" defer=""></script><script src="/_next/static/chunks/pages/_app-52924524f99094ab.js" defer=""></script><script src="/_next/static/chunks/pages/_error-c92d5c4bb2b49926.js" defer=""></script><script src="/_next/static/iLk2whM1PMpAQs7TcXt-n/_buildManifest.js" defer=""></script><script src="/_next/static/iLk2whM1PMpAQs7TcXt-n/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"iLk2whM1PMpAQs7TcXt-n","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>
+<!DOCTYPE html><html><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width"/><title>404: This page could not be found</title><meta name="next-head-count" content="3"/><noscript data-n-css=""></noscript><script defer="" nomodule="" src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js"></script><script src="/_next/static/chunks/webpack-c173e301236a84c2.js" defer=""></script><script src="/_next/static/chunks/framework-8883d1e9be70c3da.js" defer=""></script><script src="/_next/static/chunks/main-8ea85465d428ecfe.js" defer=""></script><script src="/_next/static/chunks/pages/_app-52924524f99094ab.js" defer=""></script><script src="/_next/static/chunks/pages/_error-c92d5c4bb2b49926.js" defer=""></script><script src="/_next/static/RdnGIHHt12YXTL4egAcOk/_buildManifest.js" defer=""></script><script src="/_next/static/RdnGIHHt12YXTL4egAcOk/_ssgManifest.js" defer=""></script></head><body><div id="__next"><div style="font-family:system-ui,&quot;Segoe UI&quot;,Roboto,Helvetica,Arial,sans-serif,&quot;Apple Color Emoji&quot;,&quot;Segoe UI Emoji&quot;;height:100vh;text-align:center;display:flex;flex-direction:column;align-items:center;justify-content:center"><div style="line-height:48px"><style>body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}</style><h1 class="next-error-h1" style="display:inline-block;margin:0 20px 0 0;padding-right:23px;font-size:24px;font-weight:500;vertical-align:top">404</h1><div style="display:inline-block"><h2 style="font-size:14px;font-weight:400;line-height:28px">This page could not be found<!-- -->.</h2></div></div></div></div><script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"statusCode":404}},"page":"/_error","query":{},"buildId":"RdnGIHHt12YXTL4egAcOk","nextExport":true,"isFallback":false,"gip":true,"scriptLoader":[]}</script></body></html>

package/dist/src/web/nextui/eval/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><link rel="preload" as="font" href="/_next/static/media/0e4fe491bf84089c-s.p.woff2" crossorigin="" type="font/woff2"/><link rel="preload" as="font" href="/_next/static/media/627622453ef56b0d-s.p.woff2" crossorigin="" type="font/woff2"/><link rel="preload" as="font" href="/_next/static/media/934c4b7cb736f2a3-s.p.woff2" crossorigin="" type="font/woff2"/><link rel="stylesheet" href="/_next/static/css/408b930062eabefb.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/fc460b8a7cadb952.css" data-precedence="next"/><link rel="preload" href="/_next/static/chunks/webpack-c173e301236a84c2.js" as="script"/><link rel="preload" href="/_next/static/chunks/fd9d1056-fba4b53a2f01213b.js" as="script"/><link rel="preload" href="/_next/static/chunks/596-297f7ff4a0436e87.js" as="script"/><link rel="preload" href="/_next/static/chunks/main-app-581ccf0003955b21.js" as="script"/><title>promptfoo</title><meta name="description" content="LLM testing and evaluation"/><meta name="viewport" content="width=device-width, initial-scale=1"/><meta property="og:image" content="https://www.promptfoo.dev/img/thumbnail.png"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://www.promptfoo.dev/img/thumbnail.png"/><meta name="next-size-adjust"/><script src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js" noModule=""></script></head><body class="__className_979022"><div><style data-emotion="css jj2ztu">.css-jj2ztu{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;}.css-jj2ztu>:not(style):not(style){margin:0;}.css-jj2ztu>:not(style)~:not(style){margin-left:16px;}</style><div class="MuiStack-root nav css-jj2ztu"><div class="logo MuiBox-root css-0"><img alt="Promptfoo logo" loading="lazy" width="25" height="25" decoding="async" data-nimg="1" style="color:transparent" src="/logo.svg"/> <span>promptfoo</span></div><a href="/setup/">New Eval</a><a href="/eval/">View Evals</a><div class="dark-mode-toggle"><style data-emotion="css vubbuv">.css-vubbuv{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;width:1em;height:1em;display:inline-block;fill:currentColor;-webkit-flex-shrink:0;-ms-flex-negative:0;flex-shrink:0;-webkit-transition:fill 200ms cubic-bezier(0.4, 0, 0.2, 1) 0ms;transition:fill 200ms cubic-bezier(0.4, 0, 0.2, 1) 0ms;font-size:1.5rem;}</style><svg class="MuiSvgIcon-root MuiSvgIcon-fontSizeMedium css-vubbuv" focusable="false" aria-hidden="true" viewBox="0 0 24 24" data-testid="LightModeIcon"><path d="M12 7c-2.76 0-5 2.24-5 5s2.24 5 5 5 5-2.24 5-5-2.24-5-5-5zM2 13h2c.55 0 1-.45 1-1s-.45-1-1-1H2c-.55 0-1 .45-1 1s.45 1 1 1zm18 0h2c.55 0 1-.45 1-1s-.45-1-1-1h-2c-.55 0-1 .45-1 1s.45 1 1 1zM11 2v2c0 .55.45 1 1 1s1-.45 1-1V2c0-.55-.45-1-1-1s-1 .45-1 1zm0 18v2c0 .55.45 1 1 1s1-.45 1-1v-2c0-.55-.45-1-1-1s-1 .45-1 1zM5.99 4.58c-.39-.39-1.03-.39-1.41 0-.39.39-.39 1.03 0 1.41l1.06 1.06c.39.39 1.03.39 1.41 0s.39-1.03 0-1.41L5.99 4.58zm12.37 12.37c-.39-.39-1.03-.39-1.41 0-.39.39-.39 1.03 0 1.41l1.06 1.06c.39.39 1.03.39 1.41 0 .39-.39.39-1.03 0-1.41l-1.06-1.06zm1.06-10.96c.39-.39.39-1.03 0-1.41-.39-.39-1.03-.39-1.41 0l-1.06 1.06c-.39.39-.39 1.03 0 1.41s1.03.39 1.41 0l1.06-1.06zM7.05 18.36c.39-.39.39-1.03 0-1.41-.39-.39-1.03-.39-1.41 0l-1.06 1.06c-.39.39-.39 1.03 0 1.41s1.03.39 1.41 0l1.06-1.06z"></path></svg></div></div><div><div class="loading"><div><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:22px;height:22px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span></div><div>Loading eval data</div></div></div></div><script src="/_next/static/chunks/webpack-c173e301236a84c2.js" async=""></script><script src="/_next/static/chunks/fd9d1056-fba4b53a2f01213b.js" async=""></script><script src="/_next/static/chunks/596-297f7ff4a0436e87.js" async=""></script><script src="/_next/static/chunks/main-app-581ccf0003955b21.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/media/0e4fe491bf84089c-s.p.woff2\",{\"as\":\"font\",\"type\":\"font/woff2\"}]\n2:HL[\"/_next/static/media/627622453ef56b0d-s.p.woff2\",{\"as\":\"font\",\"type\":\"font/woff2\"}]\n3:HL[\"/_next/static/media/934c4b7cb736f2a3-s.p.woff2\",{\"as\":\"font\",\"type\":\"font/woff2\"}]\n4:HL[\"/_next/static/css/408b930062eabefb.css\",{\"as\":\"style\"}]\n0:\"$L5\"\n"])</script><script>self.__next_f.push([1,"6:HL[\"/_next/static/css/fc460b8a7cadb952.css\",{\"as\":\"style\"}]\n"])</script><script>self.__next_f.push([1,"7:I{\"id\":7948,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"default\",\"async\":false}\n9:I{\"id\":6628,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"\",\"async\":false}\na:I{\"id\":898,\"chunks\":[\"706:static/chunks/706-be905c29f0baf0b7.js\",\"396:static/chunks/396-0a51429a01e24cdd.js\",\"613:static/"])</script><script>self.__next_f.push([1,"chunks/613-b8dbbee738eefa17.js\",\"185:static/chunks/app/layout-19c92c911a7607b5.js\"],\"name\":\"PageShell\",\"async\":false}\nb:I{\"id\":7767,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"default\",\"async\":false}\nc:I{\"id\":7920,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"default\",\"async\":false"])</script><script>self.__next_f.push([1,"}\ne:I{\"id\":6891,\"chunks\":[\"674:static/chunks/ca377847-26b462611379a4f7.js\",\"706:static/chunks/706-be905c29f0baf0b7.js\",\"249:static/chunks/249-ea9c0f034888ccff.js\",\"226:static/chunks/226-7bbb6c98a19542fd.js\",\"891:static/chunks/891-edb1eb13e9d8fd36.js\",\"11:static/chunks/app/eval/page-a6b1ff91723b7beb.js\"],\"name\":\"\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"5:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/408b930062eabefb.css\",\"precedence\":\"next\"}]],[\"$\",\"$L7\",null,{\"buildId\":\"iLk2whM1PMpAQs7TcXt-n\",\"assetPrefix\":\"\",\"initialCanonicalUrl\":\"/eval/\",\"initialTree\":[\"\",{\"children\":[\"eval\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialHead\":\"$L8\",\"globalErrorComponent\":\"$9\",\"children\":[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_979022\",\"children\":[\"$\",\"$La\",null,{\"children\":[\"$\",\"$Lb\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"template\":[\"$\",\"$Lc\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":\"$undefined\",\"childProp\":{\"current\":[\"$\",\"$Lb\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"eval\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"template\":[\"$\",\"$Lc\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"childProp\":{\"current\":[\"$Ld\",[\"$\",\"$Le\",null,{}],null],\"segment\":\"__PAGE__\"},\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/fc460b8a7cadb952.css\",\"precedence\":\"next\"}]]}],\"segment\":\"eval\"},\"styles\":[]}]}]}]}],null]}]]\n"])</script><script>self.__next_f.push([1,"8:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"1\",{\"children\":\"promptfoo\"}],[\"$\",\"meta\",\"2\",{\"name\":\"description\",\"content\":\"LLM testing and evaluation\"}],[\"$\",\"meta\",\"3\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:image\",\"content\":\"https://www.promptfoo.dev/img/thumbnail.png\"}],[\"$\",\"meta\",\"5\",{\"name\":\"twitter:card\",\"content\":\"summary\"}],[\"$\",\"meta\",\"6\",{\"name\":\"twitter:image\",\"content\":\"https://www.promptfoo.dev/img/thumbnail.png\"}],[\"$\",\"meta\",\"7\",{\"name\":\"next-size-adjust\"}]]\n"])</script><script>self.__next_f.push([1,"d:null\n"])</script></body></html>
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><link rel="preload" as="font" href="/_next/static/media/0e4fe491bf84089c-s.p.woff2" crossorigin="" type="font/woff2"/><link rel="preload" as="font" href="/_next/static/media/627622453ef56b0d-s.p.woff2" crossorigin="" type="font/woff2"/><link rel="preload" as="font" href="/_next/static/media/934c4b7cb736f2a3-s.p.woff2" crossorigin="" type="font/woff2"/><link rel="stylesheet" href="/_next/static/css/408b930062eabefb.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/fc460b8a7cadb952.css" data-precedence="next"/><link rel="preload" href="/_next/static/chunks/webpack-c173e301236a84c2.js" as="script"/><link rel="preload" href="/_next/static/chunks/fd9d1056-fba4b53a2f01213b.js" as="script"/><link rel="preload" href="/_next/static/chunks/596-297f7ff4a0436e87.js" as="script"/><link rel="preload" href="/_next/static/chunks/main-app-581ccf0003955b21.js" as="script"/><title>promptfoo</title><meta name="description" content="LLM testing and evaluation"/><meta name="viewport" content="width=device-width, initial-scale=1"/><meta property="og:image" content="https://www.promptfoo.dev/img/thumbnail.png"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://www.promptfoo.dev/img/thumbnail.png"/><meta name="next-size-adjust"/><script src="/_next/static/chunks/polyfills-78c92fac7aa8fdd8.js" noModule=""></script></head><body class="__className_979022"><div><style data-emotion="css jj2ztu">.css-jj2ztu{display:-webkit-box;display:-webkit-flex;display:-ms-flexbox;display:flex;-webkit-flex-direction:row;-ms-flex-direction:row;flex-direction:row;}.css-jj2ztu>:not(style):not(style){margin:0;}.css-jj2ztu>:not(style)~:not(style){margin-left:16px;}</style><div class="MuiStack-root nav css-jj2ztu"><div class="logo MuiBox-root css-0"><img alt="Promptfoo logo" loading="lazy" width="25" height="25" decoding="async" data-nimg="1" style="color:transparent" src="/logo.svg"/> <span>promptfoo</span></div><a href="/setup/">New Eval</a><a href="/eval/">View Evals</a><div class="dark-mode-toggle"><style data-emotion="css vubbuv">.css-vubbuv{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;width:1em;height:1em;display:inline-block;fill:currentColor;-webkit-flex-shrink:0;-ms-flex-negative:0;flex-shrink:0;-webkit-transition:fill 200ms cubic-bezier(0.4, 0, 0.2, 1) 0ms;transition:fill 200ms cubic-bezier(0.4, 0, 0.2, 1) 0ms;font-size:1.5rem;}</style><svg class="MuiSvgIcon-root MuiSvgIcon-fontSizeMedium css-vubbuv" focusable="false" aria-hidden="true" viewBox="0 0 24 24" data-testid="LightModeIcon"><path d="M12 7c-2.76 0-5 2.24-5 5s2.24 5 5 5 5-2.24 5-5-2.24-5-5-5zM2 13h2c.55 0 1-.45 1-1s-.45-1-1-1H2c-.55 0-1 .45-1 1s.45 1 1 1zm18 0h2c.55 0 1-.45 1-1s-.45-1-1-1h-2c-.55 0-1 .45-1 1s.45 1 1 1zM11 2v2c0 .55.45 1 1 1s1-.45 1-1V2c0-.55-.45-1-1-1s-1 .45-1 1zm0 18v2c0 .55.45 1 1 1s1-.45 1-1v-2c0-.55-.45-1-1-1s-1 .45-1 1zM5.99 4.58c-.39-.39-1.03-.39-1.41 0-.39.39-.39 1.03 0 1.41l1.06 1.06c.39.39 1.03.39 1.41 0s.39-1.03 0-1.41L5.99 4.58zm12.37 12.37c-.39-.39-1.03-.39-1.41 0-.39.39-.39 1.03 0 1.41l1.06 1.06c.39.39 1.03.39 1.41 0 .39-.39.39-1.03 0-1.41l-1.06-1.06zm1.06-10.96c.39-.39.39-1.03 0-1.41-.39-.39-1.03-.39-1.41 0l-1.06 1.06c-.39.39-.39 1.03 0 1.41s1.03.39 1.41 0l1.06-1.06zM7.05 18.36c.39-.39.39-1.03 0-1.41-.39-.39-1.03-.39-1.41 0l-1.06 1.06c-.39.39-.39 1.03 0 1.41s1.03.39 1.41 0l1.06-1.06z"></path></svg></div></div><div><div class="loading"><div><style data-emotion="css z01bqi animation-61bdi0">.css-z01bqi{display:inline-block;color:#1976d2;-webkit-animation:animation-61bdi0 1.4s linear infinite;animation:animation-61bdi0 1.4s linear infinite;}@-webkit-keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}@keyframes animation-61bdi0{0%{-webkit-transform:rotate(0deg);-moz-transform:rotate(0deg);-ms-transform:rotate(0deg);transform:rotate(0deg);}100%{-webkit-transform:rotate(360deg);-moz-transform:rotate(360deg);-ms-transform:rotate(360deg);transform:rotate(360deg);}}</style><span class="MuiCircularProgress-root MuiCircularProgress-indeterminate MuiCircularProgress-colorPrimary css-z01bqi" style="width:22px;height:22px" role="progressbar"><style data-emotion="css 13o7eu2">.css-13o7eu2{display:block;}</style><svg class="MuiCircularProgress-svg css-13o7eu2" viewBox="22 22 44 44"><style data-emotion="css 14891ef animation-1p2h4ri">.css-14891ef{stroke:currentColor;stroke-dasharray:80px,200px;stroke-dashoffset:0;-webkit-animation:animation-1p2h4ri 1.4s ease-in-out infinite;animation:animation-1p2h4ri 1.4s ease-in-out infinite;}@-webkit-keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}@keyframes animation-1p2h4ri{0%{stroke-dasharray:1px,200px;stroke-dashoffset:0;}50%{stroke-dasharray:100px,200px;stroke-dashoffset:-15px;}100%{stroke-dasharray:100px,200px;stroke-dashoffset:-125px;}}</style><circle class="MuiCircularProgress-circle MuiCircularProgress-circleIndeterminate css-14891ef" cx="44" cy="44" r="20.2" fill="none" stroke-width="3.6"></circle></svg></span></div><div>Loading eval data</div></div></div></div><script src="/_next/static/chunks/webpack-c173e301236a84c2.js" async=""></script><script src="/_next/static/chunks/fd9d1056-fba4b53a2f01213b.js" async=""></script><script src="/_next/static/chunks/596-297f7ff4a0436e87.js" async=""></script><script src="/_next/static/chunks/main-app-581ccf0003955b21.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/media/0e4fe491bf84089c-s.p.woff2\",{\"as\":\"font\",\"type\":\"font/woff2\"}]\n2:HL[\"/_next/static/media/627622453ef56b0d-s.p.woff2\",{\"as\":\"font\",\"type\":\"font/woff2\"}]\n3:HL[\"/_next/static/media/934c4b7cb736f2a3-s.p.woff2\",{\"as\":\"font\",\"type\":\"font/woff2\"}]\n4:HL[\"/_next/static/css/408b930062eabefb.css\",{\"as\":\"style\"}]\n0:\"$L5\"\n"])</script><script>self.__next_f.push([1,"6:HL[\"/_next/static/css/fc460b8a7cadb952.css\",{\"as\":\"style\"}]\n"])</script><script>self.__next_f.push([1,"7:I{\"id\":7948,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"default\",\"async\":false}\n9:I{\"id\":6628,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"\",\"async\":false}\na:I{\"id\":898,\"chunks\":[\"706:static/chunks/706-be905c29f0baf0b7.js\",\"396:static/chunks/396-0a51429a01e24cdd.js\",\"613:static/"])</script><script>self.__next_f.push([1,"chunks/613-b8dbbee738eefa17.js\",\"185:static/chunks/app/layout-19c92c911a7607b5.js\"],\"name\":\"PageShell\",\"async\":false}\nb:I{\"id\":7767,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"default\",\"async\":false}\nc:I{\"id\":7920,\"chunks\":[\"272:static/chunks/webpack-c173e301236a84c2.js\",\"971:static/chunks/fd9d1056-fba4b53a2f01213b.js\",\"596:static/chunks/596-297f7ff4a0436e87.js\"],\"name\":\"default\",\"async\":false"])</script><script>self.__next_f.push([1,"}\ne:I{\"id\":6891,\"chunks\":[\"674:static/chunks/ca377847-26b462611379a4f7.js\",\"706:static/chunks/706-be905c29f0baf0b7.js\",\"249:static/chunks/249-ea9c0f034888ccff.js\",\"226:static/chunks/226-7bbb6c98a19542fd.js\",\"891:static/chunks/891-edb1eb13e9d8fd36.js\",\"11:static/chunks/app/eval/page-a6b1ff91723b7beb.js\"],\"name\":\"\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"5:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/408b930062eabefb.css\",\"precedence\":\"next\"}]],[\"$\",\"$L7\",null,{\"buildId\":\"RdnGIHHt12YXTL4egAcOk\",\"assetPrefix\":\"\",\"initialCanonicalUrl\":\"/eval/\",\"initialTree\":[\"\",{\"children\":[\"eval\",{\"children\":[\"__PAGE__\",{}]}]},\"$undefined\",\"$undefined\",true],\"initialHead\":\"$L8\",\"globalErrorComponent\":\"$9\",\"children\":[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_979022\",\"children\":[\"$\",\"$La\",null,{\"children\":[\"$\",\"$Lb\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"template\":[\"$\",\"$Lc\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":\"$undefined\",\"childProp\":{\"current\":[\"$\",\"$Lb\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"eval\",\"children\"],\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"template\":[\"$\",\"$Lc\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"childProp\":{\"current\":[\"$Ld\",[\"$\",\"$Le\",null,{}],null],\"segment\":\"__PAGE__\"},\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/fc460b8a7cadb952.css\",\"precedence\":\"next\"}]]}],\"segment\":\"eval\"},\"styles\":[]}]}]}]}],null]}]]\n"])</script><script>self.__next_f.push([1,"8:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"1\",{\"children\":\"promptfoo\"}],[\"$\",\"meta\",\"2\",{\"name\":\"description\",\"content\":\"LLM testing and evaluation\"}],[\"$\",\"meta\",\"3\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"4\",{\"property\":\"og:image\",\"content\":\"https://www.promptfoo.dev/img/thumbnail.png\"}],[\"$\",\"meta\",\"5\",{\"name\":\"twitter:card\",\"content\":\"summary\"}],[\"$\",\"meta\",\"6\",{\"name\":\"twitter:image\",\"content\":\"https://www.promptfoo.dev/img/thumbnail.png\"}],[\"$\",\"meta\",\"7\",{\"name\":\"next-size-adjust\"}]]\n"])</script><script>self.__next_f.push([1,"d:null\n"])</script></body></html>