promptfoo 0.6.0 → 0.8.0

package/dist/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAEV,eAAe,EAGf,eAAe,EAGhB,MAAM,YAAY,CAAC;AAiRpB,wBAAgB,QAAQ,CAAC,OAAO,EAAE,eAAe,4BAGhD"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAEV,eAAe,EAGf,eAAe,EAEf,SAAS,EAGV,MAAM,YAAY,CAAC;AA6TpB,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}

package/dist/evaluator.js

@@ -36,7 +36,8 @@ const assertions_js_1 = require("./assertions.js");
 const suggestions_js_1 = require("./suggestions.js");
 const DEFAULT_MAX_CONCURRENCY = 4;
 class Evaluator {
-    constructor(options) {
+    constructor(testSuite, options) {
+        this.testSuite = testSuite;
         this.options = options;
         this.stats = {
             successes: 0,
@@ -45,11 +46,12 @@ class Evaluator {
                 total: 0,
                 prompt: 0,
                 completion: 0,
+                cached: 0,
             },
         };
     }
-    async runEval({ provider, prompt, vars, includeProviderId, }) {
-        vars = vars || {};
+    async runEval({ provider, prompt, test, includeProviderId, }) {
+        const vars = test.vars || {};
         const renderedPrompt = nunjucks_1.default.renderString(prompt, vars);
         // Note that we're using original prompt, not renderedPrompt
         const promptDisplay = includeProviderId ? `[${provider.id()}] ${prompt}` : prompt;
@@ -71,22 +73,28 @@ class Evaluator {
                 ret.error = response.error;
             }
             else if (response.output) {
-                const checkResult = vars.__expected
-                    ? await (0, assertions_js_1.matchesExpectedValue)(vars.__expected, response.output, this.options)
-                    : { pass: true };
+                const checkResult = await (0, assertions_js_1.runAssertions)(test, response.output);
                 if (!checkResult.pass) {
-                    ret.error = checkResult.reason || `Expected: ${vars.__expected}`;
+                    ret.error = checkResult.reason;
                 }
                 ret.success = checkResult.pass;
+                if (checkResult.tokensUsed) {
+                    this.stats.tokenUsage.total += checkResult.tokensUsed.total;
+                    this.stats.tokenUsage.prompt += checkResult.tokensUsed.prompt;
+                    this.stats.tokenUsage.completion += checkResult.tokensUsed.completion;
+                }
             }
             else {
                 ret.success = false;
                 ret.error = 'No output';
             }
             // Update token usage stats
-            this.stats.tokenUsage.total += response.tokenUsage?.total || 0;
-            this.stats.tokenUsage.prompt += response.tokenUsage?.prompt || 0;
-            this.stats.tokenUsage.completion += response.tokenUsage?.completion || 0;
+            if (response.tokenUsage) {
+                this.stats.tokenUsage.total += response.tokenUsage.total || 0;
+                this.stats.tokenUsage.prompt += response.tokenUsage.prompt || 0;
+                this.stats.tokenUsage.completion += response.tokenUsage.completion || 0;
+                this.stats.tokenUsage.cached += response.tokenUsage.cached || 0;
+            }
             if (ret.success) {
                 this.stats.successes++;
             }
@@ -104,11 +112,12 @@ class Evaluator {
         }
     }
     async evaluate() {
-        const options = this.options;
+        const { testSuite, options } = this;
         const prompts = [];
-        if (options.prompt?.generateSuggestions) {
+        if (options.generateSuggestions) {
+            // TODO(ian): Move this into its own command/file
             logger_js_1.default.info(`Generating prompt variations...`);
-            const { prompts: newPrompts, error } = await (0, suggestions_js_1.generatePrompts)(options.prompts[0], 1);
+            const { prompts: newPrompts, error } = await (0, suggestions_js_1.generatePrompts)(testSuite.prompts[0], 1);
             if (error || !newPrompts) {
                 throw new Error(`Failed to generate prompts: ${error}`);
             }
@@ -127,7 +136,7 @@ class Evaluator {
                     rl.question(`${chalk_1.default.blue('Do you want to test this prompt?')} (y/N): `, async (answer) => {
                         rl.close();
                         if (answer.toLowerCase().startsWith('y')) {
-                            options.prompts.push(prompt);
+                            testSuite.prompts.push(prompt);
                             numAdded++;
                         }
                         else {
@@ -142,36 +151,51 @@ class Evaluator {
                 process.exit(1);
             }
         }
-        for (const promptContent of options.prompts) {
-            for (const provider of options.providers) {
-                const display = options.providers.length > 1 ? `[${provider.id()}] ${promptContent}` : promptContent;
+        // Split prompts by provider
+        for (const promptContent of testSuite.prompts) {
+            for (const provider of testSuite.providers) {
+                const display = testSuite.providers.length > 1 ? `[${provider.id()}] ${promptContent}` : promptContent;
                 prompts.push({
                     raw: promptContent,
                     display,
                 });
             }
         }
-        const vars = options.vars && options.vars.length > 0 ? options.vars : [{}];
-        const varsWithSpecialColsRemoved = vars.map((v) => {
-            const ret = { ...v };
-            Object.keys(ret).forEach((key) => {
-                if (key.startsWith('__')) {
-                    delete ret[key];
-                }
-            });
-            return ret;
+        // Aggregate all vars across test cases
+        const tests = (testSuite.tests || [
+            {
+            // Dummy test for cases when we're only comparing raw prompts.
+            },
+        ]).map((test) => {
+            const finalTestCase = Object.assign({}, testSuite.defaultTest);
+            return Object.assign(finalTestCase, test);
         });
-        const isTest = vars[0].__expected;
+        const varNames = new Set();
+        const varsWithSpecialColsRemoved = [];
+        for (const testCase of tests) {
+            if (testCase.vars) {
+                const varWithSpecialColsRemoved = {};
+                for (const varName of Object.keys(testCase.vars)) {
+                    varNames.add(varName);
+                    varWithSpecialColsRemoved[varName] = testCase.vars[varName];
+                }
+                varsWithSpecialColsRemoved.push(varWithSpecialColsRemoved);
+            }
+        }
+        // Set up table...
+        const isTest = tests.some((t) => !!t.assert);
         const table = {
             head: {
                 prompts: prompts.map((p) => p.display),
-                vars: Object.keys(varsWithSpecialColsRemoved[0]),
+                vars: Array.from(varNames).sort(),
+                // TODO(ian): add assertions to table?
             },
             body: [],
         };
+        // And progress bar...
         let progressbar;
         if (options.showProgressBar) {
-            const totalNumRuns = options.prompts.length * options.providers.length * (options.vars?.length || 1);
+            const totalNumRuns = testSuite.prompts.length * testSuite.providers.length * (tests.length || 1);
             const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
             progressbar = new cliProgress.SingleBar({
                 format: 'Eval: [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {provider} "{prompt}" {vars}',
@@ -182,19 +206,27 @@ class Evaluator {
                 vars: '',
             });
         }
+        // Set up eval cases
         const runEvalOptions = [];
         let rowIndex = 0;
-        for (const row of vars) {
+        for (const testCase of tests) {
             let colIndex = 0;
-            const prependToPrompt = row.__prefix || options.prompt?.prefix || '';
-            const appendToPrompt = row.__suffix || options.prompt?.suffix || '';
-            for (const promptContent of options.prompts) {
-                for (const provider of options.providers) {
+            // Handle default properties
+            testCase.vars = Object.assign({}, testSuite.defaultTest?.vars, testCase.vars);
+            testCase.assert = [...(testSuite.defaultTest?.assert || []), ...(testCase.assert || [])];
+            testCase.options = testCase.options || {};
+            testCase.options.provider =
+                testCase.options.provider || testSuite.defaultTest?.options?.provider;
+            const prependToPrompt = testCase.options?.prefix || testSuite.defaultTest?.options?.prefix || '';
+            const appendToPrompt = testCase.options?.suffix || testSuite.defaultTest?.options?.suffix || '';
+            // Finalize test case eval
+            for (const promptContent of testSuite.prompts) {
+                for (const provider of testSuite.providers) {
                     runEvalOptions.push({
                         provider,
                         prompt: prependToPrompt + promptContent + appendToPrompt,
-                        vars: row,
-                        includeProviderId: options.providers.length > 1,
+                        test: testCase,
+                        includeProviderId: testSuite.providers.length > 1,
                         rowIndex,
                         colIndex,
                     });
@@ -203,6 +235,7 @@ class Evaluator {
             }
             rowIndex++;
         }
+        // Actually run the eval
         const results = [];
         await async_1.default.forEachOfLimit(runEvalOptions, options.maxConcurrency || DEFAULT_MAX_CONCURRENCY, async (options, index) => {
             const row = await this.runEval(options);
@@ -211,7 +244,7 @@ class Evaluator {
                 progressbar.increment({
                     provider: options.provider.id(),
                     prompt: options.prompt.slice(0, 10),
-                    vars: Object.entries(options.vars || {})
+                    vars: Object.entries(options.test.vars || {})
                         .map(([k, v]) => `${k}=${v}`)
                         .join(' ')
                         .slice(0, 10),
@@ -242,7 +275,7 @@ class Evaluator {
             if (!table.body[rowIndex]) {
                 table.body[rowIndex] = {
                     outputs: [],
-                    vars: Object.values(options.vars || {}),
+                    vars: table.head.vars.map((varName) => options.test.vars?.[varName] || ''),
                 };
             }
             table.body[rowIndex].outputs[colIndex] = resultText;
@@ -253,8 +286,8 @@ class Evaluator {
         return { version: 1, results, stats: this.stats, table };
     }
 }
-function evaluate(options) {
-    const ev = new Evaluator(options);
+function evaluate(testSuite, options) {
+    const ev = new Evaluator(testSuite, options);
     return ev.evaluate();
 }
 exports.evaluate = evaluate;

package/dist/evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,4DAAiC;AACjC,mDAAuD;AAYvD,qDAAmD;AAYnD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,SAAS;IAIb,YAAY,OAAwB;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;aACd;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,kBAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3D,4DAA4D;QAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAElF,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU;oBACjC,CAAC,CAAC,MAAM,IAAA,oCAAoB,EAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;oBAC5E,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,IAAI,aAAa,IAAI,CAAC,UAAU,EAAE,CAAC;iBAClE;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;aAChC;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YAEzE,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,MAAM,EAAE,mBAAmB,EAAE;YACvC,mBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,gCAAe,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;aACzD;YAED,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gBAC/B,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBACxE,mBAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzB,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAExE,wCAAwC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,CAAC,CAAC;oBACH,EAAE,CAAC,QAAQ,CACT,GAAG,eAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,UAAU,EAC3D,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;4BACxC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;4BAC7B,QAAQ,EAAE,CAAC;yBACZ;6BAAM;4BACL,mBAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;yBACtC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC,CACF,CAAC;gBACJ,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE;gBAChB,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,MAAM,OAAO,GACX,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,aAAa;oBAClB,OAAO;iBACR,CAAC,CAAC;aACJ;SACF;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;oBACxB,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;iBACjB;YACH,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAClC,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;aACjD;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;YACrE,MAAM,cAAc,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;YAEpE,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;gBAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;oBACxC,cAAc,CAAC,IAAI,CAAC;wBAClB,QAAQ;wBACR,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,cAAc;wBACxD,IAAI,EAAE,GAAG;wBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;wBAC/C,QAAQ;wBACR,QAAQ;qBACT,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC;SACZ;QAED,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,eAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;iBACxC,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,SAAgB,QAAQ,CAAC,OAAwB;IAC/C,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAHD,4BAGC"}
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,4DAAiC;AACjC,mDAAgD;AAchD,qDAAmD;AAcnD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,SAAS;IAKb,YAAY,SAAoB,EAAE,OAAwB;QACxD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;aACV;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,kBAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3D,4DAA4D;QAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAElF,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,MAAM,IAAA,6BAAa,EAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC;iBAChC;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,WAAW,CAAC,UAAU,EAAE;oBAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC;oBAC5D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC9D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC;iBACvE;aACF;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACvB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC9D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBAChE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;gBACxE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;aACjE;YAED,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,mBAAmB,EAAE;YAC/B,iDAAiD;YACjD,mBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,gCAAe,EAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACtF,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;aACzD;YAED,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gBAC/B,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBACxE,mBAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzB,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAExE,wCAAwC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,CAAC,CAAC;oBACH,EAAE,CAAC,QAAQ,CACT,GAAG,eAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,UAAU,EAC3D,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;4BACxC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;4BAC/B,QAAQ,EAAE,CAAC;yBACZ;6BAAM;4BACL,mBAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;yBACtC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC,CACF,CAAC;gBACJ,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE;gBAChB,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,4BAA4B;QAC5B,KAAK,MAAM,aAAa,IAAI,SAAS,CAAC,OAAO,EAAE;YAC7C,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE;gBAC1C,MAAM,OAAO,GACX,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;gBACzF,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,aAAa;oBAClB,OAAO;iBACR,CAAC,CAAC;aACJ;SACF;QAED,uCAAuC;QAEvC,MAAM,KAAK,GAAG,CACZ,SAAS,CAAC,KAAK,IAAI;YACjB;YACE,8DAA8D;aAC/D;SACF,CACF,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACb,MAAM,aAAa,GAAa,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;YACzE,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAgB,IAAI,GAAG,EAAE,CAAC;QACxC,MAAM,0BAA0B,GAA6B,EAAE,CAAC;QAChE,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE;YAC5B,IAAI,QAAQ,CAAC,IAAI,EAAE;gBACjB,MAAM,yBAAyB,GAA2B,EAAE,CAAC;gBAC7D,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAChD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtB,yBAAyB,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC7D;gBACD,0BAA0B,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;aAC5D;SACF;QAED,kBAAkB;QAClB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;gBACjC,sCAAsC;aACvC;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,sBAAsB;QACtB,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,MAAM,YAAY,GAChB,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;YAC9E,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,oBAAoB;QACpB,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE;YAC5B,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,4BAA4B;YAC5B,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9E,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACzF,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;YAC1C,QAAQ,CAAC,OAAO,CAAC,QAAQ;gBACvB,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC;YACxE,MAAM,eAAe,GACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;YAC3E,MAAM,cAAc,GAClB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;YAE3E,0BAA0B;YAC1B,KAAK,MAAM,aAAa,IAAI,SAAS,CAAC,OAAO,EAAE;gBAC7C,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE;oBAC1C,cAAc,CAAC,IAAI,CAAC;wBAClB,QAAQ;wBACR,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,cAAc;wBACxD,IAAI,EAAE,QAAQ;wBACd,iBAAiB,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;wBACjD,QAAQ;wBACR,QAAQ;qBACT,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC;SACZ;QAED,wBAAwB;QACxB,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,eAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;yBAC1C,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;iBAC3E,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,SAAgB,QAAQ,CAAC,SAAoB,EAAE,OAAwB;IACrE,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC7C,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAHD,4BAGC"}

package/dist/index.d.ts

@@ -1,7 +1,9 @@
-import { loadApiProvider } from './providers.js';
-import type { ApiProvider, EvaluateOptions, EvaluateSummary } from './types.js';
+import type { EvaluateOptions, TestSuiteConfig } from './types.js';
 export * from './types.js';
-declare function evaluate(providers: (string | ApiProvider)[] | (string | ApiProvider), options: Omit<EvaluateOptions, 'providers'>): Promise<EvaluateSummary>;
+interface EvaluateTestSuite extends TestSuiteConfig {
+    prompts: string[];
+}
+declare function evaluate(testSuite: EvaluateTestSuite, options?: EvaluateOptions): Promise<import("./types.js").EvaluateSummary>;
 declare const _default: {
     evaluate: typeof evaluate;
     assertions: {
@@ -13,7 +15,7 @@ declare const _default: {
         OpenAiChatCompletionProvider: typeof import("./providers/openai.js").OpenAiChatCompletionProvider;
         LocalAiCompletionProvider: typeof import("./providers/localai.js").LocalAiCompletionProvider;
         LocalAiChatProvider: typeof import("./providers/localai.js").LocalAiChatProvider;
-        loadApiProvider: typeof loadApiProvider;
+        loadApiProvider: typeof import("./providers.js").loadApiProvider;
     };
 };
 export default _default;

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAIjD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhF,cAAc,YAAY,CAAC;AAE3B,iBAAe,QAAQ,CACrB,SAAS,EAAE,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC,EAC5D,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,WAAW,CAAC,GAC1C,OAAO,CAAC,eAAe,CAAC,CAsB1B;;;;;;;;;;;;;;;AAQD,wBAIE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAa,eAAe,EAAE,MAAM,YAAY,CAAC;AAG9E,cAAc,YAAY,CAAC;AAE3B,UAAU,iBAAkB,SAAQ,eAAe;IACjD,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,iBAAe,QAAQ,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,GAAE,eAAoB,iDAQlF;;;;;;;;;;;;;;;AAQD,wBAIE"}

package/dist/index.js

@@ -21,29 +21,16 @@ const evaluator_js_1 = require("./evaluator.js");
 const providers_js_1 = require("./providers.js");
 const assertions_js_1 = __importDefault(require("./assertions.js"));
 const providers_js_2 = __importDefault(require("./providers.js"));
+const util_js_1 = require("./util.js");
 __exportStar(require("./types.js"), exports);
-async function evaluate(providers, options) {
-    let apiProviders = [];
-    const addProvider = async (provider) => {
-        if (typeof provider === 'string') {
-            apiProviders.push(await (0, providers_js_1.loadApiProvider)(provider));
-        }
-        else {
-            apiProviders.push(provider);
-        }
+async function evaluate(testSuite, options = {}) {
+    const constructedTestSuite = {
+        ...testSuite,
+        prompts: testSuite.prompts,
+        providers: await (0, providers_js_1.loadApiProviders)(testSuite.providers),
+        tests: (0, util_js_1.readTests)(testSuite.tests),
     };
-    if (Array.isArray(providers)) {
-        for (const provider of providers) {
-            await addProvider(provider);
-        }
-    }
-    else {
-        await addProvider(providers);
-    }
-    return (0, evaluator_js_1.evaluate)({
-        ...options,
-        providers: apiProviders,
-    });
+    return (0, evaluator_js_1.evaluate)(constructedTestSuite, options);
 }
 module.exports = {
     evaluate,

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,iDAAwD;AACxD,iDAAiD;AACjD,oEAAyC;AACzC,kEAAuC;AAIvC,6CAA2B;AAE3B,KAAK,UAAU,QAAQ,CACrB,SAA4D,EAC5D,OAA2C;IAE3C,IAAI,YAAY,GAAkB,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,KAAK,EAAE,QAA8B,EAAE,EAAE;QAC3D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;YAChC,YAAY,CAAC,IAAI,CAAC,MAAM,IAAA,8BAAe,EAAC,QAAQ,CAAC,CAAC,CAAC;SACpD;aAAM;YACL,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC7B;IACH,CAAC,CAAC;IAEF,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;SAC7B;KACF;SAAM;QACL,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;KAC9B;IAED,OAAO,IAAA,uBAAU,EAAC;QAChB,GAAG,OAAO;QACV,SAAS,EAAE,YAAY;KACxB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,OAAO,GAAG;IACf,QAAQ;IACR,UAAU,EAAV,uBAAU;IACV,SAAS,EAAT,sBAAS;CACV,CAAC;AAEF,kBAAe;IACb,QAAQ;IACR,UAAU,EAAV,uBAAU;IACV,SAAS,EAAT,sBAAS;CACV,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,iDAAwD;AACxD,iDAAkD;AAClD,oEAAyC;AACzC,kEAAuC;AAGvC,uCAAsC;AAEtC,6CAA2B;AAM3B,KAAK,UAAU,QAAQ,CAAC,SAA4B,EAAE,UAA2B,EAAE;IACjF,MAAM,oBAAoB,GAAc;QACtC,GAAG,SAAS;QACZ,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,SAAS,EAAE,MAAM,IAAA,+BAAgB,EAAC,SAAS,CAAC,SAAS,CAAC;QACtD,KAAK,EAAE,IAAA,mBAAS,EAAC,SAAS,CAAC,KAAK,CAAC;KAClC,CAAC;IACF,OAAO,IAAA,uBAAU,EAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,CAAC,OAAO,GAAG;IACf,QAAQ;IACR,UAAU,EAAV,uBAAU;IACV,SAAS,EAAT,sBAAS;CACV,CAAC;AAEF,kBAAe;IACb,QAAQ;IACR,UAAU,EAAV,uBAAU;IACV,SAAS,EAAT,sBAAS;CACV,CAAC"}

package/dist/main.js

@@ -38,6 +38,8 @@ const evaluator_js_1 = require("./evaluator.js");
 const util_js_1 = require("./util.js");
 const esm_js_1 = require("./esm.js");
 const server_js_1 = require("./web/server.js");
+const cache_js_1 = require("./cache.js");
+const onboarding_js_1 = require("./onboarding.js");
 function createDummyFiles(directory) {
     if (directory) {
         // Make the directory if it doesn't exist
@@ -45,30 +47,6 @@ function createDummyFiles(directory) {
             (0, fs_1.mkdirSync)(directory);
         }
     }
-    const dummyPrompts = `Your first prompt goes here
----
-Next prompt goes here. You can substitute variables like this: {{var1}} {{var2}} {{var3}}
----
-This is the next prompt.
-
-These prompts are nunjucks templates, so you can use logic like this:
-{% if var1 %}
-  {{ var1 }}
-{% endif %}`;
-    const dummyVars = 'var1,var2,var3\nvalue1,value2,value3\nanother value1,another value2,another value3';
-    const dummyConfig = `module.exports = {
-  prompts: ['prompts.txt'],
-  providers: ['openai:gpt-3.5-turbo'],
-  vars: 'vars.csv',
-  maxConcurrency: 4,
-};`;
-    const readme = `To get started, set your OPENAI_API_KEY environment variable. Then run:
-\`\`\`
-promptfoo eval
-\`\`\`
-
-You'll probably want to change a few of the prompts in prompts.txt and the variables in vars.csv before letting it rip.
-`;
     if (directory) {
         if (!(0, fs_1.existsSync)(directory)) {
             logger_js_1.default.info(`Creating directory ${directory} ...`);
@@ -78,10 +56,9 @@ You'll probably want to change a few of the prompts in prompts.txt and the varia
     else {
         directory = '.';
     }
-    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'prompts.txt'), dummyPrompts);
-    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'vars.csv'), dummyVars);
-    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'promptfooconfig.js'), dummyConfig);
-    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'README.md'), readme);
+    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'prompts.txt'), onboarding_js_1.DEFAULT_PROMPTS);
+    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'promptfooconfig.yaml'), onboarding_js_1.DEFAULT_YAML_CONFIG);
+    (0, fs_1.writeFileSync)((0, path_1.join)(process.cwd(), directory, 'README.md'), onboarding_js_1.DEFAULT_README);
     if (directory === '.') {
         logger_js_1.default.info('Wrote prompts.txt, vars.csv, and promptfooconfig.js. Open README.md to get started!');
     }
@@ -91,15 +68,25 @@ You'll probably want to change a few of the prompts in prompts.txt and the varia
     }
 }
 async function main() {
-    let defaultConfig = {};
-    if ((0, fs_1.existsSync)('promptfooconfig.js')) {
-        // @ts-ignore
-        defaultConfig = (await Promise.resolve(`${(0, path_1.join)(process.cwd(), './promptfooconfig.js')}`).then(s => __importStar(require(s)))).default;
-        logger_js_1.default.info('Loaded default config from promptfooconfig.js');
+    const pwd = process.cwd();
+    const potentialPaths = [
+        (0, path_1.join)(pwd, 'promptfooconfig.js'),
+        (0, path_1.join)(pwd, 'promptfooconfig.json'),
+        (0, path_1.join)(pwd, 'promptfooconfig.yaml'),
+    ];
+    let config = {};
+    for (const path of potentialPaths) {
+        const maybeConfig = (0, util_js_1.maybeReadConfig)(path);
+        if (maybeConfig) {
+            config = maybeConfig;
+            break;
+        }
     }
-    if ((0, fs_1.existsSync)('promptfooconfig.json')) {
-        defaultConfig = JSON.parse((0, fs_1.readFileSync)('promptfooconfig.json', 'utf-8'));
-        logger_js_1.default.info('Loaded default config from promptfooconfig.json');
+    let evaluateOptions = {};
+    if (config.evaluateOptions) {
+        evaluateOptions.generateSuggestions = config.evaluateOptions.generateSuggestions;
+        evaluateOptions.maxConcurrency = config.evaluateOptions.maxConcurrency;
+        evaluateOptions.showProgressBar = config.evaluateOptions.showProgressBar;
     }
     const program = new commander_1.Command();
     program.option('--version', 'Print version', () => {
@@ -123,67 +110,92 @@ async function main() {
     program
         .command('eval')
         .description('Evaluate prompts')
-        .requiredOption('-p, --prompts <paths...>', 'Paths to prompt files (.txt)', defaultConfig.prompts)
-        .requiredOption('-r, --providers <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module', defaultConfig.providers)
-        .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)', defaultConfig.output)
-        .option('-v, --vars <path>', 'Path to file with prompt variables (csv, json, yaml)', defaultConfig.vars)
-        .option('-c, --config <path>', 'Path to configuration file. Automatically loads promptfooconfig.js', defaultConfig.config)
-        .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', String(defaultConfig.maxConcurrency))
+        .requiredOption('-p, --prompts <paths...>', 'Paths to prompt files (.txt)', config.prompts)
+        .requiredOption('-r, --providers <name or path...>', 'One of: openai:chat, openai:completion, openai:<model name>, or path to custom API caller module', config?.providers)
+        .option('-c, --config <path>', 'Path to configuration file. Automatically loads promptfooconfig.js/json/yaml')
+        .option(
+    // TODO(ian): Remove `vars` for v1
+    '-v, --vars, -t, --tests <path>', 'Path to CSV with test cases', config?.commandLineOptions?.vars)
+        .option('-o, --output <path>', 'Path to output file (csv, json, yaml, html)', config.outputPath)
+        .option('-j, --max-concurrency <number>', 'Maximum number of concurrent API calls', config.evaluateOptions?.maxConcurrency
+        ? String(config.evaluateOptions.maxConcurrency)
+        : undefined)
         .option('--table-cell-max-length <number>', 'Truncate console table cells to this length', '250')
         .option('--suggest-prompts <number>', 'Generate N new prompts and append them to the prompt list')
-        .option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', defaultConfig.promptPrefix)
-        .option('--prompt-suffix <path>', 'This suffix is append to every prompt', defaultConfig.promptSuffix)
+        .option('--prompt-prefix <path>', 'This prefix is prepended to every prompt', config.defaultTest?.options?.prefix)
+        .option('--prompt-suffix <path>', 'This suffix is append to every prompt', config.defaultTest?.options?.suffix)
         .option('--no-write', 'Do not write results to promptfoo directory')
-        .option('--grader', 'Model that will grade outputs', defaultConfig.grader)
-        .option('--verbose', 'Show debug logs', defaultConfig.verbose)
+        .option('--no-cache', 'Do not read or write results to disk cache')
+        .option('--grader', 'Model that will grade outputs', config?.commandLineOptions?.grader)
+        .option('--verbose', 'Show debug logs', config?.commandLineOptions?.verbose)
         .option('--view [port]', 'View in browser ui')
         .action(async (cmdObj) => {
+        // Misc settings
         if (cmdObj.verbose) {
             (0, logger_js_1.setLogLevel)('debug');
         }
+        if (!cmdObj.cache) {
+            (0, cache_js_1.disableCache)();
+        }
+        // Config parsing
+        const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
         const configPath = cmdObj.config;
-        let config = {};
         if (configPath) {
-            const ext = (0, path_1.parse)(configPath).ext;
-            switch (ext) {
-                case '.json':
-                    const content = (0, fs_1.readFileSync)(configPath, 'utf-8');
-                    config = JSON.parse(content);
-                    break;
-                case '.js':
-                    config = require(configPath);
-                    break;
-                default:
-                    throw new Error(`Unsupported configuration file format: ${ext}`);
-            }
+            config = (0, util_js_1.readConfig)(configPath);
         }
-        let vars = [];
-        if (cmdObj.vars) {
-            vars = (0, util_js_1.readVars)(cmdObj.vars);
+        else {
+            config = {
+                prompts: cmdObj.prompts || config.prompts,
+                providers: cmdObj.providers || config.providers,
+                tests: cmdObj.vars || config.tests,
+            };
         }
-        const providers = await Promise.all(cmdObj.providers.map(async (p) => await (0, providers_js_1.loadApiProvider)(p)));
-        const maxConcurrency = parseInt(cmdObj.maxConcurrency || '', 10);
-        const options = {
-            prompts: (0, util_js_1.readPrompts)(cmdObj.prompts),
-            vars,
-            providers,
-            showProgressBar: true,
-            maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
-            prompt: {
+        // Validation
+        if (!config.prompts || config.prompts.length === 0) {
+            logger_js_1.default.error(chalk_1.default.red('You must provide at least 1 prompt file'));
+            process.exit(1);
+        }
+        if (!config.providers || config.providers.length === 0) {
+            logger_js_1.default.error(chalk_1.default.red('You must specify at least 1 provider (for example, openai:gpt-3.5-turbo)'));
+            process.exit(1);
+        }
+        // Parse prompts, providers, and tests
+        const parsedPrompts = (0, util_js_1.readPrompts)(config.prompts);
+        const parsedProviders = await (0, providers_js_1.loadApiProviders)(config.providers);
+        const parsedTests = (0, util_js_1.readTests)(config.tests);
+        if (parsedPrompts.length === 0) {
+            logger_js_1.default.error(chalk_1.default.red('No prompts found'));
+            process.exit(1);
+        }
+        const defaultTest = {
+            options: {
                 prefix: cmdObj.promptPrefix,
                 suffix: cmdObj.promptSuffix,
+                provider: cmdObj.grader,
+                // rubricPrompt:
             },
-            ...config,
+            ...config.defaultTest,
         };
-        if (cmdObj.grader) {
-            options.grading = {
-                provider: await (0, providers_js_1.loadApiProvider)(cmdObj.grader),
-            };
+        const testSuite = {
+            description: config.description,
+            prompts: parsedPrompts,
+            providers: parsedProviders,
+            tests: parsedTests,
+            defaultTest,
+        };
+        const options = {
+            showProgressBar: true,
+            maxConcurrency: !isNaN(maxConcurrency) && maxConcurrency > 0 ? maxConcurrency : undefined,
+            ...evaluateOptions,
+        };
+        if (cmdObj.grader && testSuite.defaultTest) {
+            testSuite.defaultTest.options = testSuite.defaultTest.options || {};
+            testSuite.defaultTest.options.provider = await (0, providers_js_1.loadApiProvider)(cmdObj.grader);
         }
         if (cmdObj.generateSuggestions) {
-            options.prompt.generateSuggestions = true;
+            options.generateSuggestions = true;
         }
-        const summary = await (0, evaluator_js_1.evaluate)(options);
+        const summary = await (0, evaluator_js_1.evaluate)(testSuite, options);
         if (cmdObj.output) {
             logger_js_1.default.info(chalk_1.default.yellow(`Writing output to ${cmdObj.output}`));
             (0, util_js_1.writeOutput)(cmdObj.output, summary);
@@ -228,7 +240,7 @@ async function main() {
             }
             logger_js_1.default.info('\n' + table.toString());
         }
-        if (cmdObj.noWrite || cmdObj.view) {
+        if (cmdObj.view || !cmdObj.write) {
             logger_js_1.default.info('Evaluation complete');
         }
         else {
@@ -237,7 +249,7 @@ async function main() {
         }
         logger_js_1.default.info(chalk_1.default.green.bold(`Successes: ${summary.stats.successes}`));
         logger_js_1.default.info(chalk_1.default.red.bold(`Failures: ${summary.stats.failures}`));
-        logger_js_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total} Prompt ${summary.stats.tokenUsage.prompt} Completion ${summary.stats.tokenUsage.completion}`);
+        logger_js_1.default.info(`Token usage: Total ${summary.stats.tokenUsage.total}, Prompt ${summary.stats.tokenUsage.prompt}, Completion ${summary.stats.tokenUsage.completion}, Cached ${summary.stats.tokenUsage.cached}`);
         logger_js_1.default.info('Done.');
         if (cmdObj.view) {
             (0, server_js_1.init)(parseInt(cmdObj.view, 10) || 15500);