promptfoo 0.5.1 → 0.7.0

package/dist/evaluator.js

@@ -1,10 +1,40 @@
-import async from 'async';
-import nunjucks from 'nunjucks';
-import { DEFAULT_GRADING_PROMPT } from './prompts.js';
-import { DefaultEmbeddingProvider } from './providers/openai.js';
-import { cosineSimilarity } from './util.js';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.evaluate = void 0;
+const node_readline_1 = __importDefault(require("node:readline"));
+const async_1 = __importDefault(require("async"));
+const chalk_1 = __importDefault(require("chalk"));
+const nunjucks_1 = __importDefault(require("nunjucks"));
+const logger_js_1 = __importDefault(require("./logger.js"));
+const assertions_js_1 = require("./assertions.js");
+const suggestions_js_1 = require("./suggestions.js");
 const DEFAULT_MAX_CONCURRENCY = 4;
-const SIMILAR_REGEX = /similar(?::|\((\d+(\.\d+)?)\):)/;
 class Evaluator {
     constructor(options) {
         this.options = options;
@@ -15,114 +45,13 @@ class Evaluator {
                 total: 0,
                 prompt: 0,
                 completion: 0,
+                cached: 0,
             },
         };
     }
-    async gradeOutput(expected, output) {
-        const { grading } = this.options;
-        if (!grading) {
-            throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
-        }
-        const prompt = nunjucks.renderString(grading.prompt || DEFAULT_GRADING_PROMPT, {
-            content: output,
-            rubric: expected,
-        });
-        const resp = await grading.provider.callApi(prompt);
-        if (resp.error || !resp.output) {
-            return {
-                pass: false,
-                reason: resp.error || 'No output',
-                tokensUsed: {
-                    total: resp.tokenUsage?.total || 0,
-                    prompt: resp.tokenUsage?.prompt || 0,
-                    completion: resp.tokenUsage?.completion || 0,
-                },
-            };
-        }
-        try {
-            const parsed = JSON.parse(resp.output);
-            parsed.tokensUsed = {
-                total: resp.tokenUsage?.total || 0,
-                prompt: resp.tokenUsage?.prompt || 0,
-                completion: resp.tokenUsage?.completion || 0,
-            };
-            return parsed;
-        }
-        catch (err) {
-            return {
-                pass: false,
-                reason: `Output is not valid JSON: ${resp.output}`,
-                tokensUsed: {
-                    total: resp.tokenUsage?.total || 0,
-                    prompt: resp.tokenUsage?.prompt || 0,
-                    completion: resp.tokenUsage?.completion || 0,
-                },
-            };
-        }
-    }
-    async checkSimilarity(expected, output, threshold) {
-        const expectedEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(expected);
-        const outputEmbedding = await DefaultEmbeddingProvider.callEmbeddingApi(output);
-        const tokensUsed = {
-            total: (expectedEmbedding.tokenUsage?.total || 0) + (outputEmbedding.tokenUsage?.total || 0),
-            prompt: (expectedEmbedding.tokenUsage?.prompt || 0) + (outputEmbedding.tokenUsage?.prompt || 0),
-            completion: (expectedEmbedding.tokenUsage?.completion || 0) +
-                (outputEmbedding.tokenUsage?.completion || 0),
-        };
-        if (expectedEmbedding.error || outputEmbedding.error) {
-            return {
-                pass: false,
-                reason: expectedEmbedding.error || outputEmbedding.error || 'Unknown error fetching embeddings',
-                tokensUsed,
-            };
-        }
-        if (!expectedEmbedding.embedding || !outputEmbedding.embedding) {
-            return {
-                pass: false,
-                reason: 'Embedding not found',
-                tokensUsed,
-            };
-        }
-        const similarity = cosineSimilarity(expectedEmbedding.embedding, outputEmbedding.embedding);
-        if (similarity < threshold) {
-            return {
-                pass: false,
-                reason: `Similarity ${similarity} is less than threshold ${threshold}`,
-                tokensUsed,
-            };
-        }
-        return {
-            pass: true,
-            reason: `Similarity ${similarity} is greater than threshold ${threshold}`,
-            tokensUsed,
-        };
-    }
-    async checkExpectedValue(expected, output) {
-        const match = expected.match(SIMILAR_REGEX);
-        if (match) {
-            const threshold = parseFloat(match[1]) || 0.8;
-            const rest = expected.replace(SIMILAR_REGEX, '').trim();
-            return this.checkSimilarity(rest, output, threshold);
-        }
-        else if (expected.startsWith('eval:')) {
-            const evalBody = expected.slice(5);
-            const evalFunction = new Function('output', `return ${evalBody}`);
-            return { pass: evalFunction(output) };
-        }
-        else if (expected.startsWith('grade:')) {
-            return this.gradeOutput(expected.slice(6), output);
-        }
-        else {
-            const pass = expected === output;
-            return {
-                pass,
-                reason: pass ? undefined : `Expected: ${expected}, Output: ${output}`,
-            };
-        }
-    }
     async runEval({ provider, prompt, vars, includeProviderId, }) {
         vars = vars || {};
-        const renderedPrompt = nunjucks.renderString(prompt, vars);
+        const renderedPrompt = nunjucks_1.default.renderString(prompt, vars);
         // Note that we're using original prompt, not renderedPrompt
         const promptDisplay = includeProviderId ? `[${provider.id()}] ${prompt}` : prompt;
         const setup = {
@@ -144,7 +73,7 @@ class Evaluator {
             }
             else if (response.output) {
                 const checkResult = vars.__expected
-                    ? await this.checkExpectedValue(vars.__expected, response.output)
+                    ? await (0, assertions_js_1.matchesExpectedValue)(vars.__expected, response.output, this.options)
                     : { pass: true };
                 if (!checkResult.pass) {
                     ret.error = checkResult.reason || `Expected: ${vars.__expected}`;
@@ -159,6 +88,7 @@ class Evaluator {
             this.stats.tokenUsage.total += response.tokenUsage?.total || 0;
             this.stats.tokenUsage.prompt += response.tokenUsage?.prompt || 0;
             this.stats.tokenUsage.completion += response.tokenUsage?.completion || 0;
+            this.stats.tokenUsage.cached += response.tokenUsage?.cached || 0;
             if (ret.success) {
                 this.stats.successes++;
             }
@@ -178,6 +108,42 @@ class Evaluator {
     async evaluate() {
         const options = this.options;
         const prompts = [];
+        if (options.prompt?.generateSuggestions) {
+            logger_js_1.default.info(`Generating prompt variations...`);
+            const { prompts: newPrompts, error } = await (0, suggestions_js_1.generatePrompts)(options.prompts[0], 1);
+            if (error || !newPrompts) {
+                throw new Error(`Failed to generate prompts: ${error}`);
+            }
+            logger_js_1.default.info(chalk_1.default.blue('Generated prompts:'));
+            let numAdded = 0;
+            for (const prompt of newPrompts) {
+                logger_js_1.default.info('--------------------------------------------------------');
+                logger_js_1.default.info(`${prompt}`);
+                logger_js_1.default.info('--------------------------------------------------------');
+                // Ask the user if they want to continue
+                await new Promise((resolve) => {
+                    const rl = node_readline_1.default.createInterface({
+                        input: process.stdin,
+                        output: process.stdout,
+                    });
+                    rl.question(`${chalk_1.default.blue('Do you want to test this prompt?')} (y/N): `, async (answer) => {
+                        rl.close();
+                        if (answer.toLowerCase().startsWith('y')) {
+                            options.prompts.push(prompt);
+                            numAdded++;
+                        }
+                        else {
+                            logger_js_1.default.info('Skipping this prompt.');
+                        }
+                        resolve(true);
+                    });
+                });
+            }
+            if (numAdded < 1) {
+                logger_js_1.default.info(chalk_1.default.red('No prompts selected. Aborting.'));
+                process.exit(1);
+            }
+        }
         for (const promptContent of options.prompts) {
             for (const provider of options.providers) {
                 const display = options.providers.length > 1 ? `[${provider.id()}] ${promptContent}` : promptContent;
@@ -188,23 +154,27 @@ class Evaluator {
             }
         }
         const vars = options.vars && options.vars.length > 0 ? options.vars : [{}];
-        const varsWithExpectedKeyRemoved = vars.map((v) => {
+        const varsWithSpecialColsRemoved = vars.map((v) => {
             const ret = { ...v };
-            delete ret.__expected;
+            Object.keys(ret).forEach((key) => {
+                if (key.startsWith('__')) {
+                    delete ret[key];
+                }
+            });
             return ret;
         });
         const isTest = vars[0].__expected;
         const table = {
             head: {
                 prompts: prompts.map((p) => p.display),
-                vars: Object.keys(varsWithExpectedKeyRemoved[0]),
+                vars: Object.keys(varsWithSpecialColsRemoved[0]),
             },
             body: [],
         };
         let progressbar;
         if (options.showProgressBar) {
             const totalNumRuns = options.prompts.length * options.providers.length * (options.vars?.length || 1);
-            const cliProgress = await import('cli-progress');
+            const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
             progressbar = new cliProgress.SingleBar({
                 format: 'Eval: [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {provider} "{prompt}" {vars}',
             }, cliProgress.Presets.shades_classic);
@@ -218,11 +188,13 @@ class Evaluator {
         let rowIndex = 0;
         for (const row of vars) {
             let colIndex = 0;
+            const prependToPrompt = row.__prefix || options.prompt?.prefix || '';
+            const appendToPrompt = row.__suffix || options.prompt?.suffix || '';
             for (const promptContent of options.prompts) {
                 for (const provider of options.providers) {
                     runEvalOptions.push({
                         provider,
-                        prompt: promptContent,
+                        prompt: prependToPrompt + promptContent + appendToPrompt,
                         vars: row,
                         includeProviderId: options.providers.length > 1,
                         rowIndex,
@@ -234,7 +206,7 @@ class Evaluator {
             rowIndex++;
         }
         const results = [];
-        await async.forEachOfLimit(runEvalOptions, options.maxConcurrency || DEFAULT_MAX_CONCURRENCY, async (options, index) => {
+        await async_1.default.forEachOfLimit(runEvalOptions, options.maxConcurrency || DEFAULT_MAX_CONCURRENCY, async (options, index) => {
             const row = await this.runEval(options);
             results.push(row);
             if (progressbar) {
@@ -283,8 +255,9 @@ class Evaluator {
         return { version: 1, results, stats: this.stats, table };
     }
 }
-export function evaluate(options) {
+function evaluate(options) {
     const ev = new Evaluator(options);
     return ev.evaluate();
 }
+exports.evaluate = evaluate;
 //# sourceMappingURL=evaluator.js.map

package/dist/evaluator.js.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,QAAQ,MAAM,UAAU,CAAC;AAEhC,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,wBAAwB,EAAE,MAAM,uBAAuB,CAAC;AAajE,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAkB7C,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,aAAa,GAAG,iCAAiC,CAAC;AAExD,MAAM,SAAS;IAIb,YAAY,OAAwB;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;aACd;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,MAAc;QAChD,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAEjC,IAAI,CAAC,OAAO,EAAE;YACZ,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;SACH;QAED,MAAM,MAAM,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,IAAI,sBAAsB,EAAE;YAC7E,OAAO,EAAE,MAAM;YACf,MAAM,EAAE,QAAQ;SACjB,CAAC,CAAC;QAEH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YAC9B,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,WAAW;gBACjC,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;oBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;oBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;iBAC7C;aACF,CAAC;SACH;QAED,IAAI;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAkB,CAAC;YACxD,MAAM,CAAC,UAAU,GAAG;gBAClB,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C,CAAC;YACF,OAAO,MAAM,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE;gBAClD,UAAU,EAAE;oBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;oBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;oBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;iBAC7C;aACF,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,QAAgB,EAChB,MAAc,EACd,SAAiB;QAEjB,MAAM,iBAAiB,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QACpF,MAAM,eAAe,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEhF,MAAM,UAAU,GAAG;YACjB,KAAK,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC5F,MAAM,EACJ,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACzF,UAAU,EACR,CAAC,iBAAiB,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;gBAC/C,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;SAChD,CAAC;QAEF,IAAI,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,EAAE;YACpD,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EACJ,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,IAAI,mCAAmC;gBACzF,UAAU;aACX,CAAC;SACH;QAED,IAAI,CAAC,iBAAiB,CAAC,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;YAC9D,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,qBAAqB;gBAC7B,UAAU;aACX,CAAC;SACH;QAED,MAAM,UAAU,GAAG,gBAAgB,CAAC,iBAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;QAC5F,IAAI,UAAU,GAAG,SAAS,EAAE;YAC1B,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,cAAc,UAAU,2BAA2B,SAAS,EAAE;gBACtE,UAAU;aACX,CAAC;SACH;QACD,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,cAAc,UAAU,8BAA8B,SAAS,EAAE;YACzE,UAAU;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,kBAAkB,CACtB,QAAgB,EAChB,MAAc;QAEd,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAE5C,IAAI,KAAK,EAAE;YACT,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC;YAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACxD,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;SACtD;aAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;YACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,YAAY,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,UAAU,QAAQ,EAAE,CAAC,CAAC;YAClE,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;SACvC;aAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;YACxC,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;SACpD;aAAM;YACL,MAAM,IAAI,GAAG,QAAQ,KAAK,MAAM,CAAC;YACjC,OAAO;gBACL,IAAI;gBACJ,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,QAAQ,aAAa,MAAM,EAAE;aACtE,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3D,4DAA4D;QAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAElF,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU;oBACjC,CAAC,CAAC,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,CAAC;oBACjE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,IAAI,aAAa,IAAI,CAAC,UAAU,EAAE,CAAC;iBAClE;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;aAChC;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YAEzE,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,MAAM,OAAO,GACX,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,aAAa;oBAClB,OAAO;iBACR,CAAC,CAAC;aACJ;SACF;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,GAAG,CAAC,UAAU,CAAC;YACtB,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAClC,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;aACjD;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,cAAc,CAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;gBAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;oBACxC,cAAc,CAAC,IAAI,CAAC;wBAClB,QAAQ;wBACR,MAAM,EAAE,aAAa;wBACrB,IAAI,EAAE,GAAG;wBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;wBAC/C,QAAQ;wBACR,QAAQ;qBACT,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC;SACZ;QAED,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,KAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;iBACxC,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,MAAM,UAAU,QAAQ,CAAC,OAAwB;IAC/C,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC"}
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,4DAAiC;AACjC,mDAAuD;AAYvD,qDAAmD;AAYnD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,MAAM,SAAS;IAIb,YAAY,OAAwB;QAClC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;aACV;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QAClB,MAAM,cAAc,GAAG,kBAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE3D,4DAA4D;QAC5D,MAAM,aAAa,GAAG,iBAAiB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAElF,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU;oBACjC,CAAC,CAAC,MAAM,IAAA,oCAAoB,EAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC;oBAC5E,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBACnB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,IAAI,aAAa,IAAI,CAAC,UAAU,EAAE,CAAC;iBAClE;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;aAChC;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YACjE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YACzE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;YAEjE,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC;gBAClB,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,MAAM,EAAE,mBAAmB,EAAE;YACvC,mBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,gCAAe,EAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACpF,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;aACzD;YAED,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gBAC/B,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBACxE,mBAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzB,mBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAExE,wCAAwC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,CAAC,CAAC;oBACH,EAAE,CAAC,QAAQ,CACT,GAAG,eAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,UAAU,EAC3D,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;4BACxC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;4BAC7B,QAAQ,EAAE,CAAC;yBACZ;6BAAM;4BACL,mBAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;yBACtC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC,CACF,CAAC;gBACJ,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE;gBAChB,mBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;YAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;gBACxC,MAAM,OAAO,GACX,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;gBACvF,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,aAAa;oBAClB,OAAO;iBACR,CAAC,CAAC;aACJ;SACF;QAED,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3E,MAAM,0BAA0B,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAChD,MAAM,GAAG,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE;oBACxB,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;iBACjB;YACH,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;QAClC,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC,CAAC,CAAC;aACjD;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,MAAM,YAAY,GAChB,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC;YAClF,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;YACrE,MAAM,cAAc,GAAG,GAAG,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,CAAC;YAEpE,KAAK,MAAM,aAAa,IAAI,OAAO,CAAC,OAAO,EAAE;gBAC3C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE;oBACxC,cAAc,CAAC,IAAI,CAAC;wBAClB,QAAQ;wBACR,MAAM,EAAE,eAAe,GAAG,aAAa,GAAG,cAAc;wBACxD,IAAI,EAAE,GAAG;wBACT,iBAAiB,EAAE,OAAO,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;wBAC/C,QAAQ;wBACR,QAAQ;qBACT,CAAC,CAAC;oBACH,QAAQ,EAAE,CAAC;iBACZ;aACF;YACD,QAAQ,EAAE,CAAC;SACZ;QAED,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,eAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACnC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;iBACxC,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,SAAgB,QAAQ,CAAC,OAAwB;IAC/C,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAHD,4BAGC"}

package/dist/index.d.ts

@@ -1,7 +1,20 @@
+import { loadApiProvider } from './providers.js';
 import type { ApiProvider, EvaluateOptions, EvaluateSummary } from './types.js';
+export * from './types.js';
 declare function evaluate(providers: (string | ApiProvider)[] | (string | ApiProvider), options: Omit<EvaluateOptions, 'providers'>): Promise<EvaluateSummary>;
 declare const _default: {
     evaluate: typeof evaluate;
+    assertions: {
+        matchesSimilarity: typeof import("./assertions.js").matchesSimilarity;
+        matchesLlmRubric: typeof import("./assertions.js").matchesLlmRubric;
+    };
+    providers: {
+        OpenAiCompletionProvider: typeof import("./providers/openai.js").OpenAiCompletionProvider;
+        OpenAiChatCompletionProvider: typeof import("./providers/openai.js").OpenAiChatCompletionProvider;
+        LocalAiCompletionProvider: typeof import("./providers/localai.js").LocalAiCompletionProvider;
+        LocalAiChatProvider: typeof import("./providers/localai.js").LocalAiChatProvider;
+        loadApiProvider: typeof loadApiProvider;
+    };
 };
 export default _default;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhF,iBAAe,QAAQ,CACrB,SAAS,EAAE,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC,EAC5D,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,WAAW,CAAC,GAC1C,OAAO,CAAC,eAAe,CAAC,CAsB1B;;;;AAED,wBAEE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAIjD,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhF,cAAc,YAAY,CAAC;AAE3B,iBAAe,QAAQ,CACrB,SAAS,EAAE,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC,EAC5D,OAAO,EAAE,IAAI,CAAC,eAAe,EAAE,WAAW,CAAC,GAC1C,OAAO,CAAC,eAAe,CAAC,CAsB1B;;;;;;;;;;;;;;;AAQD,wBAIE"}

package/dist/index.js

@@ -1,10 +1,32 @@
-import { evaluate as doEvaluate } from './evaluator.js';
-import { loadApiProvider } from './providers.js';
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const evaluator_js_1 = require("./evaluator.js");
+const providers_js_1 = require("./providers.js");
+const assertions_js_1 = __importDefault(require("./assertions.js"));
+const providers_js_2 = __importDefault(require("./providers.js"));
+__exportStar(require("./types.js"), exports);
 async function evaluate(providers, options) {
     let apiProviders = [];
     const addProvider = async (provider) => {
         if (typeof provider === 'string') {
-            apiProviders.push(await loadApiProvider(provider));
+            apiProviders.push(await (0, providers_js_1.loadApiProvider)(provider));
         }
         else {
             apiProviders.push(provider);
@@ -18,12 +40,19 @@ async function evaluate(providers, options) {
     else {
         await addProvider(providers);
     }
-    return doEvaluate({
+    return (0, evaluator_js_1.evaluate)({
         ...options,
         providers: apiProviders,
     });
 }
-export default {
+module.exports = {
+    evaluate,
+    assertions: assertions_js_1.default,
+    providers: providers_js_2.default,
+};
+exports.default = {
     evaluate,
+    assertions: assertions_js_1.default,
+    providers: providers_js_2.default,
 };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAIjD,KAAK,UAAU,QAAQ,CACrB,SAA4D,EAC5D,OAA2C;IAE3C,IAAI,YAAY,GAAkB,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,KAAK,EAAE,QAA8B,EAAE,EAAE;QAC3D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;YAChC,YAAY,CAAC,IAAI,CAAC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;SACpD;aAAM;YACL,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC7B;IACH,CAAC,CAAC;IAEF,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;SAC7B;KACF;SAAM;QACL,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;KAC9B;IAED,OAAO,UAAU,CAAC;QAChB,GAAG,OAAO;QACV,SAAS,EAAE,YAAY;KACxB,CAAC,CAAC;AACL,CAAC;AAED,eAAe;IACb,QAAQ;CACT,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;AAAA,iDAAwD;AACxD,iDAAiD;AACjD,oEAAyC;AACzC,kEAAuC;AAIvC,6CAA2B;AAE3B,KAAK,UAAU,QAAQ,CACrB,SAA4D,EAC5D,OAA2C;IAE3C,IAAI,YAAY,GAAkB,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,KAAK,EAAE,QAA8B,EAAE,EAAE;QAC3D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;YAChC,YAAY,CAAC,IAAI,CAAC,MAAM,IAAA,8BAAe,EAAC,QAAQ,CAAC,CAAC,CAAC;SACpD;aAAM;YACL,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;SAC7B;IACH,CAAC,CAAC;IAEF,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE;QAC5B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAC;SAC7B;KACF;SAAM;QACL,MAAM,WAAW,CAAC,SAAS,CAAC,CAAC;KAC9B;IAED,OAAO,IAAA,uBAAU,EAAC;QAChB,GAAG,OAAO;QACV,SAAS,EAAE,YAAY;KACxB,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,OAAO,GAAG;IACf,QAAQ;IACR,UAAU,EAAV,uBAAU;IACV,SAAS,EAAT,sBAAS;CACV,CAAC;AAEF,kBAAe;IACb,QAAQ;IACR,UAAU,EAAV,uBAAU;IACV,SAAS,EAAT,sBAAS;CACV,CAAC"}

package/dist/logger.js

@@ -1,32 +1,38 @@
-import chalk from 'chalk';
-import winston from 'winston';
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setLogLevel = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const winston_1 = __importDefault(require("winston"));
 const logLevels = {
     error: 0,
     warn: 1,
     info: 2,
     debug: 3,
 };
-const customFormatter = winston.format.printf(({ level, message, ...args }) => {
+const customFormatter = winston_1.default.format.printf(({ level, message, ...args }) => {
     if (level === 'error') {
-        return chalk.red(message);
+        return chalk_1.default.red(message);
     }
     else if (level === 'warn') {
-        return chalk.yellow(message);
+        return chalk_1.default.yellow(message);
     }
     else if (level === 'info') {
         return message;
     }
     else if (level === 'debug') {
-        return chalk.cyan(message);
+        return chalk_1.default.cyan(message);
     }
     throw new Error(`Invalid log level: ${level}`);
 });
-const logger = winston.createLogger({
+const logger = winston_1.default.createLogger({
     levels: logLevels,
-    format: winston.format.combine(winston.format.simple(), customFormatter),
-    transports: [new winston.transports.Console()],
+    format: winston_1.default.format.combine(winston_1.default.format.simple(), customFormatter),
+    transports: [new winston_1.default.transports.Console()],
 });
-export function setLogLevel(level) {
+function setLogLevel(level) {
     if (logLevels.hasOwnProperty(level)) {
         logger.transports[0].level = level;
     }
@@ -34,5 +40,6 @@ export function setLogLevel(level) {
         throw new Error(`Invalid log level: ${level}`);
     }
 }
-export default logger;
+exports.setLogLevel = setLogLevel;
+exports.default = logger;
 //# sourceMappingURL=logger.js.map

package/dist/logger.js.map

@@ -1 +1 @@
-{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,MAAM,SAAS,GAAG;IAChB,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;IAC5E,IAAI,KAAK,KAAK,OAAO,EAAE;QACrB,OAAO,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;KAC3B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAC9B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,OAAO,CAAC;KAChB;SAAM,IAAI,KAAK,KAAK,OAAO,EAAE;QAC5B,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KAC5B;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAClC,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC;IACxE,UAAU,EAAE,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;CAC/C,CAAC,CAAC;AAEH,MAAM,UAAU,WAAW,CAAC,KAA6B;IACvD,IAAI,SAAS,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE;QACnC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;KACpC;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;KAChD;AACH,CAAC;AAED,eAAe,MAAM,CAAC"}
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAC1B,sDAA8B;AAE9B,MAAM,SAAS,GAAG;IAChB,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,MAAM,eAAe,GAAG,iBAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;IAC5E,IAAI,KAAK,KAAK,OAAO,EAAE;QACrB,OAAO,eAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;KAC3B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,eAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAC9B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,OAAO,CAAC;KAChB;SAAM,IAAI,KAAK,KAAK,OAAO,EAAE;QAC5B,OAAO,eAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KAC5B;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,iBAAO,CAAC,YAAY,CAAC;IAClC,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,iBAAO,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC;IACxE,UAAU,EAAE,CAAC,IAAI,iBAAO,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;CAC/C,CAAC,CAAC;AAEH,SAAgB,WAAW,CAAC,KAA6B;IACvD,IAAI,SAAS,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE;QACnC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;KACpC;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;KAChD;AACH,CAAC;AAND,kCAMC;AAED,kBAAe,MAAM,CAAC"}