promptfoo 0.91.3 → 0.92.0

package/dist/src/models/eval.js

@@ -0,0 +1,390 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EvalQueries = void 0;
+exports.createEvalId = createEvalId;
+exports.getSummaryofLatestEvals = getSummaryofLatestEvals;
+const crypto_1 = require("crypto");
+const drizzle_orm_1 = require("drizzle-orm");
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const constants_1 = require("../constants");
+const database_1 = require("../database");
+const tables_1 = require("../database/tables");
+const logger_1 = __importDefault(require("../logger"));
+const utils_1 = require("../prompts/utils");
+const convertEvalResultsToTable_1 = require("../util/convertEvalResultsToTable");
+const createHash_1 = require("../util/createHash");
+const time_1 = require("../util/time");
+const evalResult_1 = __importDefault(require("./evalResult"));
+function createEvalId(createdAt = new Date()) {
+    return `eval-${(0, createHash_1.randomSequence)(3)}-${createdAt.toISOString().slice(0, 19)}`;
+}
+class EvalQueries {
+    static async getVarsFromEvals(evals) {
+        const db = (0, database_1.getDb)();
+        const query = drizzle_orm_1.sql.raw(`SELECT DISTINCT j.key, eval_id from (SELECT eval_id, json_extract(eval_results.test_case, '$.vars') as vars
+FROM eval_results where eval_id IN (${evals.map((e) => `'${e.id}'`).join(',')})) t, json_each(t.vars) j;`);
+        // @ts-ignore
+        const results = await db.all(query);
+        const vars = results.reduce((acc, r) => {
+            acc[r.eval_id] = acc[r.eval_id] || [];
+            acc[r.eval_id].push(r.key);
+            return acc;
+        }, {});
+        return vars;
+    }
+}
+exports.EvalQueries = EvalQueries;
+class Eval {
+    static async latest() {
+        const db = (0, database_1.getDb)();
+        const db_results = await db
+            .select({
+            id: tables_1.evals.id,
+        })
+            .from(tables_1.evals)
+            .orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt))
+            .limit(1);
+        if (db_results.length === 0) {
+            return undefined;
+        }
+        return await Eval.findById(db_results[0].id);
+    }
+    static async findById(id) {
+        const db = (0, database_1.getDb)();
+        const { evals, datasetResults } = await db.transaction(async (tx) => {
+            const evals = await tx.select().from(tables_1.evals).where((0, drizzle_orm_1.eq)(tables_1.evals.id, id));
+            const datasetResults = await tx
+                .select({
+                datasetId: tables_1.evalsToDatasets.datasetId,
+            })
+                .from(tables_1.evalsToDatasets)
+                .where((0, drizzle_orm_1.eq)(tables_1.evalsToDatasets.evalId, id))
+                .limit(1);
+            return { evals, datasetResults };
+        });
+        if (evals.length === 0) {
+            return undefined;
+        }
+        const eval_ = evals[0];
+        const datasetId = datasetResults[0]?.datasetId;
+        const evalInstance = new Eval(eval_.config, {
+            id: eval_.id,
+            createdAt: new Date(eval_.createdAt),
+            author: eval_.author || undefined,
+            description: eval_.description || undefined,
+            prompts: eval_.prompts || [],
+            datasetId,
+            persisted: true,
+        });
+        if (eval_.results && 'table' in eval_.results) {
+            evalInstance.oldResults = eval_.results;
+        }
+        return evalInstance;
+    }
+    static async getMany(limit = constants_1.DEFAULT_QUERY_LIMIT) {
+        const db = (0, database_1.getDb)();
+        const evals = await db
+            .select()
+            .from(tables_1.evals)
+            .limit(limit)
+            .orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt))
+            .all();
+        return evals.map((e) => new Eval(e.config, {
+            id: e.id,
+            createdAt: new Date(e.createdAt),
+            author: e.author || undefined,
+            description: e.description || undefined,
+            prompts: e.prompts || [],
+            persisted: true,
+        }));
+    }
+    static async create(config, renderedPrompts, // The config doesn't contain the actual prompts, so we need to pass them in separately
+    opts) {
+        const createdAt = opts?.createdAt || new Date();
+        const evalId = opts?.id || createEvalId(createdAt);
+        const db = (0, database_1.getDb)();
+        await db.transaction((tx) => {
+            tx.insert(tables_1.evals)
+                .values({
+                id: evalId,
+                createdAt: createdAt.getTime(),
+                author: opts?.author,
+                description: config.description,
+                config,
+                results: {},
+            })
+                .run();
+            if (opts?.results) {
+                const res = tx
+                    .insert(tables_1.evalResultsTable)
+                    .values(opts.results?.map((r) => ({ ...r, evalId, id: (0, crypto_1.randomUUID)() })))
+                    .run();
+                logger_1.default.debug(`Inserted ${res.changes} eval results`);
+            }
+            for (const prompt of renderedPrompts) {
+                const label = prompt.label || prompt.display || prompt.raw;
+                const promptId = (0, utils_1.hashPrompt)(prompt);
+                tx.insert(tables_1.prompts)
+                    .values({
+                    id: promptId,
+                    prompt: label,
+                })
+                    .onConflictDoNothing()
+                    .run();
+                tx.insert(tables_1.evalsToPrompts)
+                    .values({
+                    evalId,
+                    promptId,
+                })
+                    .onConflictDoNothing()
+                    .run();
+                logger_1.default.debug(`Inserting prompt ${promptId}`);
+            }
+            // Record dataset relation
+            const datasetId = (0, createHash_1.sha256)(JSON.stringify(config.tests || []));
+            tx.insert(tables_1.datasets)
+                .values({
+                id: datasetId,
+                tests: config.tests,
+            })
+                .onConflictDoNothing()
+                .run();
+            tx.insert(tables_1.evalsToDatasets)
+                .values({
+                evalId,
+                datasetId,
+            })
+                .onConflictDoNothing()
+                .run();
+            logger_1.default.debug(`Inserting dataset ${datasetId}`);
+            // Record tags
+            if (config.tags) {
+                for (const [tagKey, tagValue] of Object.entries(config.tags)) {
+                    const tagId = (0, createHash_1.sha256)(`${tagKey}:${tagValue}`);
+                    tx.insert(tables_1.tags)
+                        .values({
+                        id: tagId,
+                        name: tagKey,
+                        value: tagValue,
+                    })
+                        .onConflictDoNothing()
+                        .run();
+                    tx.insert(tables_1.evalsToTags)
+                        .values({
+                        evalId,
+                        tagId,
+                    })
+                        .onConflictDoNothing()
+                        .run();
+                    logger_1.default.debug(`Inserting tag ${tagId}`);
+                }
+            }
+        });
+        return new Eval(config, { id: evalId, author: opts?.author, createdAt, persisted: true });
+    }
+    constructor(config, opts) {
+        const createdAt = opts?.createdAt || new Date();
+        this.createdAt = createdAt.getTime();
+        this.id = opts?.id || createEvalId(createdAt);
+        this.author = opts?.author;
+        this.config = config;
+        this.results = [];
+        this.prompts = opts?.prompts || [];
+        this.datasetId = opts?.datasetId;
+        this.persisted = opts?.persisted || false;
+    }
+    version() {
+        /**
+         * Version 3 is the denormalized version of where the table and results are stored on the eval object.
+         * Version 4 is the normalized version where the results are stored in another databse table and the table for vizualization is generated by the app.
+         */
+        return this.oldResults && 'table' in this.oldResults ? 3 : 4;
+    }
+    useOldResults() {
+        return this.version() < 4;
+    }
+    setTable(table) {
+        (0, tiny_invariant_1.default)(this.version() < 4, 'Eval is not version 3');
+        (0, tiny_invariant_1.default)(this.oldResults, 'Old results not found');
+        this.oldResults.table = table;
+    }
+    async save() {
+        const db = (0, database_1.getDb)();
+        const updateObj = {
+            config: this.config,
+            prompts: this.prompts,
+            description: this.config.description,
+            author: this.author,
+            updatedAt: (0, time_1.getCurrentTimestamp)(),
+        };
+        if (this.useOldResults()) {
+            (0, tiny_invariant_1.default)(this.oldResults, 'Old results not found');
+            updateObj.results = this.oldResults;
+        }
+        await db.update(tables_1.evals).set(updateObj).where((0, drizzle_orm_1.eq)(tables_1.evals.id, this.id)).run();
+        this.persisted = true;
+    }
+    async getVars() {
+        if (this.useOldResults()) {
+            (0, tiny_invariant_1.default)(this.oldResults, 'Old results not found');
+            return this.oldResults.table?.head.vars || [];
+        }
+        const db = (0, database_1.getDb)();
+        const query = (0, drizzle_orm_1.sql) `SELECT DISTINCT j.key from (SELECT json_extract(test_case_results.test_case, '$.vars') as vars
+    FROM test_case_results where test_case_results.eval_id = ${this.id}) t, json_each(t.vars) j;`;
+        // @ts-ignore
+        const results = await db.all(query);
+        return results.map((r) => r.key) || [];
+    }
+    getPrompts() {
+        if (this.useOldResults()) {
+            (0, tiny_invariant_1.default)(this.oldResults, 'Old results not found');
+            return this.oldResults.table?.head.prompts || [];
+        }
+        return this.prompts;
+    }
+    async getTable() {
+        if (this.useOldResults()) {
+            return this.oldResults?.table || { head: { prompts: [], vars: [] }, body: [] };
+        }
+        return (0, convertEvalResultsToTable_1.convertResultsToTable)(await this.toResultsFile());
+    }
+    async addResult(result, test) {
+        const newResult = await evalResult_1.default.createFromEvaluateResult(this.id, result, test, {
+            persist: this.persisted,
+        });
+        this.results.push(newResult);
+    }
+    async addPrompts(prompts) {
+        this.prompts = prompts;
+        if (this.persisted) {
+            const db = (0, database_1.getDb)();
+            await db.update(tables_1.evals).set({ prompts }).where((0, drizzle_orm_1.eq)(tables_1.evals.id, this.id)).run();
+        }
+    }
+    async addProviders(providers) {
+        if (this.persisted) {
+            const db = (0, database_1.getDb)();
+            await db.transaction(async (tx) => {
+                for (const provider of providers) {
+                    const id = provider.id;
+                    tx.insert(tables_1.evalsToProviders)
+                        .values({
+                        evalId: this.id,
+                        providerId: id,
+                    })
+                        .onConflictDoNothing()
+                        .run();
+                }
+            });
+        }
+    }
+    async loadResults() {
+        this.results = await evalResult_1.default.findManyByEvalId(this.id);
+    }
+    async getResults() {
+        if (this.useOldResults()) {
+            (0, tiny_invariant_1.default)(this.oldResults, 'Old results not found');
+            return this.oldResults.results;
+        }
+        await this.loadResults();
+        return this.results;
+    }
+    async toEvaluateSummary() {
+        if (this.useOldResults()) {
+            (0, tiny_invariant_1.default)(this.oldResults, 'Old results not found');
+            return {
+                version: 2,
+                timestamp: new Date(this.createdAt).toISOString(),
+                results: this.oldResults.results,
+                table: this.oldResults.table,
+                stats: this.oldResults.stats,
+            };
+        }
+        if (this.results.length === 0) {
+            await this.loadResults();
+        }
+        const stats = {
+            successes: 0,
+            failures: 0,
+            tokenUsage: {
+                cached: 0,
+                completion: 0,
+                prompt: 0,
+                total: 0,
+            },
+        };
+        for (const prompt of this.prompts) {
+            stats.successes += prompt.metrics?.testPassCount || 0;
+            stats.failures += prompt.metrics?.testFailCount || 0;
+            stats.tokenUsage.prompt += prompt.metrics?.tokenUsage.prompt || 0;
+            stats.tokenUsage.cached += prompt.metrics?.tokenUsage.cached || 0;
+            stats.tokenUsage.completion += prompt.metrics?.tokenUsage.completion || 0;
+            stats.tokenUsage.total += prompt.metrics?.tokenUsage.total || 0;
+        }
+        return {
+            version: 3,
+            timestamp: new Date(this.createdAt).toISOString(),
+            prompts: this.prompts,
+            results: this.results.map((r) => r.toEvaluateResult()),
+            stats,
+        };
+    }
+    async toResultsFile() {
+        const results = {
+            version: this.version(),
+            createdAt: new Date(this.createdAt).toISOString(),
+            results: await this.toEvaluateSummary(),
+            config: this.config,
+            author: this.author || null,
+            prompts: this.getPrompts(),
+            datasetId: this.datasetId || null,
+        };
+        return results;
+    }
+    async delete() {
+        const db = (0, database_1.getDb)();
+        await db.transaction(() => {
+            db.delete(tables_1.evalsToDatasets).where((0, drizzle_orm_1.eq)(tables_1.evalsToDatasets.evalId, this.id)).run();
+            db.delete(tables_1.evalsToPrompts).where((0, drizzle_orm_1.eq)(tables_1.evalsToPrompts.evalId, this.id)).run();
+            db.delete(tables_1.evalsToTags).where((0, drizzle_orm_1.eq)(tables_1.evalsToTags.evalId, this.id)).run();
+            db.delete(tables_1.evalsToProviders).where((0, drizzle_orm_1.eq)(tables_1.evalsToProviders.evalId, this.id)).run();
+            db.delete(tables_1.evalResultsTable).where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.evalId, this.id)).run();
+            db.delete(tables_1.evals).where((0, drizzle_orm_1.eq)(tables_1.evals.id, this.id)).run();
+        });
+    }
+}
+exports.default = Eval;
+async function getSummaryofLatestEvals(limit = constants_1.DEFAULT_QUERY_LIMIT, filterDescription, datasetId) {
+    const db = (0, database_1.getDb)();
+    const startTime = performance.now();
+    const query = db
+        .select({
+        evalId: tables_1.evals.id,
+        createdAt: tables_1.evals.createdAt,
+        description: tables_1.evals.description,
+        numTests: (0, drizzle_orm_1.sql) `MAX(${tables_1.evalResultsTable.testIdx} + 1)`.as('numTests'),
+        datasetId: tables_1.evalsToDatasets.datasetId,
+    })
+        .from(tables_1.evals)
+        .leftJoin(tables_1.evalsToDatasets, (0, drizzle_orm_1.eq)(tables_1.evals.id, tables_1.evalsToDatasets.evalId))
+        .leftJoin(tables_1.evalResultsTable, (0, drizzle_orm_1.eq)(tables_1.evals.id, tables_1.evalResultsTable.evalId))
+        .where((0, drizzle_orm_1.and)(datasetId ? (0, drizzle_orm_1.eq)(tables_1.evalsToDatasets.datasetId, datasetId) : undefined, filterDescription ? (0, drizzle_orm_1.like)(tables_1.evals.description, `%${filterDescription}%`) : undefined, (0, drizzle_orm_1.eq)(tables_1.evals.results, {})))
+        .groupBy(tables_1.evals.id);
+    const results = query.orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt)).limit(limit).all();
+    const mappedResults = results.map((result) => ({
+        evalId: result.evalId,
+        createdAt: result.createdAt,
+        description: result.description,
+        numTests: result.numTests || 0,
+        datasetId: result.datasetId,
+    }));
+    const endTime = performance.now();
+    const executionTime = endTime - startTime;
+    logger_1.default.debug(`listPreviousResults execution time: ${executionTime.toFixed(2)}ms`);
+    return mappedResults;
+}
+//# sourceMappingURL=eval.js.map

package/dist/src/models/eval.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../src/models/eval.ts"],"names":[],"mappings":";;;;;;AAoCA,oCAEC;AAkaD,0DA0CC;AAlfD,mCAAoC;AACpC,6CAAuD;AACvD,oEAAuC;AACvC,4CAAmD;AACnD,0CAAoC;AACpC,+CAU4B;AAC5B,uDAA+B;AAC/B,4CAA8C;AAa9C,iFAA0E;AAC1E,mDAA4D;AAC5D,uCAAmD;AACnD,8DAAsC;AAGtC,SAAgB,YAAY,CAAC,YAAkB,IAAI,IAAI,EAAE;IACvD,OAAO,QAAQ,IAAA,2BAAc,EAAC,CAAC,CAAC,IAAI,SAAS,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAC7E,CAAC;AAED,MAAa,WAAW;IACtB,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,KAAa;QACzC,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,KAAK,GAAG,iBAAG,CAAC,GAAG,CACnB;sCACgC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,4BAA4B,CACpG,CAAC;QACF,aAAa;QACb,MAAM,OAAO,GAAuC,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAA6B,EAAE,CAAC,EAAE,EAAE;YAC/D,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACtC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAC3B,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAhBD,kCAgBC;AAED,MAAqB,IAAI;IAavB,MAAM,CAAC,KAAK,CAAC,MAAM;QACjB,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,UAAU,GAAG,MAAM,EAAE;aACxB,MAAM,CAAC;YACN,EAAE,EAAE,cAAU,CAAC,EAAE;SAClB,CAAC;aACD,IAAI,CAAC,cAAU,CAAC;aAChB,OAAO,CAAC,IAAA,kBAAI,EAAC,cAAU,CAAC,SAAS,CAAC,CAAC;aACnC,KAAK,CAAC,CAAC,CAAC,CAAC;QAEZ,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAU;QAC9B,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QAEnB,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YAClE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,cAAU,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAU,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAC9E,MAAM,cAAc,GAAG,MAAM,EAAE;iBAC5B,MAAM,CAAC;gBACN,SAAS,EAAE,wBAAe,CAAC,SAAS;aACrC,CAAC;iBACD,IAAI,CAAC,wBAAe,CAAC;iBACrB,KAAK,CAAC,IAAA,gBAAE,EAAC,wBAAe,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;iBACrC,KAAK,CAAC,CAAC,CAAC,CAAC;YAEZ,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvB,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;QAE/C,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAC1C,EAAE,EAAE,KAAK,CAAC,EAAE;YACZ,SAAS,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YACpC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,SAAS;YACjC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,SAAS;YAC3C,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,EAAE;YAC5B,SAAS;YACT,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,OAAO,IAAI,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAC9C,YAAY,CAAC,UAAU,GAAG,KAAK,CAAC,OAA4B,CAAC;QAC/D,CAAC;QAED,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAgB,+BAAmB;QACtD,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,KAAK,GAAG,MAAM,EAAE;aACnB,MAAM,EAAE;aACR,IAAI,CAAC,cAAU,CAAC;aAChB,KAAK,CAAC,KAAK,CAAC;aACZ,OAAO,CAAC,IAAA,kBAAI,EAAC,cAAU,CAAC,SAAS,CAAC,CAAC;aACnC,GAAG,EAAE,CAAC;QACT,OAAO,KAAK,CAAC,GAAG,CACd,CAAC,CAAC,EAAE,EAAE,CACJ,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE;YACjB,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAChC,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,SAAS;YAC7B,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,SAAS;YACvC,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;YACxB,SAAS,EAAE,IAAI;SAChB,CAAC,CACL,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CACjB,MAA8B,EAC9B,eAAyB,EAAE,uFAAuF;IAClH,IAMC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,IAAI,EAAE,EAAE,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;QACnD,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE;YAC1B,EAAE,CAAC,MAAM,CAAC,cAAU,CAAC;iBAClB,MAAM,CAAC;gBACN,EAAE,EAAE,MAAM;gBACV,SAAS,EAAE,SAAS,CAAC,OAAO,EAAE;gBAC9B,MAAM,EAAE,IAAI,EAAE,MAAM;gBACpB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,MAAM;gBACN,OAAO,EAAE,EAAE;aACZ,CAAC;iBACD,GAAG,EAAE,CAAC;YACT,IAAI,IAAI,EAAE,OAAO,EAAE,CAAC;gBAClB,MAAM,GAAG,GAAG,EAAE;qBACX,MAAM,CAAC,yBAAgB,CAAC;qBACxB,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,IAAA,mBAAU,GAAE,EAAE,CAAC,CAAC,CAAC;qBACtE,GAAG,EAAE,CAAC;gBACT,gBAAM,CAAC,KAAK,CAAC,YAAY,GAAG,CAAC,OAAO,eAAe,CAAC,CAAC;YACvD,CAAC;YAED,KAAK,MAAM,MAAM,IAAI,eAAe,EAAE,CAAC;gBACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;gBAC3D,MAAM,QAAQ,GAAG,IAAA,kBAAU,EAAC,MAAM,CAAC,CAAC;gBAEpC,EAAE,CAAC,MAAM,CAAC,gBAAY,CAAC;qBACpB,MAAM,CAAC;oBACN,EAAE,EAAE,QAAQ;oBACZ,MAAM,EAAE,KAAK;iBACd,CAAC;qBACD,mBAAmB,EAAE;qBACrB,GAAG,EAAE,CAAC;gBAET,EAAE,CAAC,MAAM,CAAC,uBAAc,CAAC;qBACtB,MAAM,CAAC;oBACN,MAAM;oBACN,QAAQ;iBACT,CAAC;qBACD,mBAAmB,EAAE;qBACrB,GAAG,EAAE,CAAC;gBAET,gBAAM,CAAC,KAAK,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;YAC/C,CAAC;YAED,0BAA0B;YAC1B,MAAM,SAAS,GAAG,IAAA,mBAAM,EAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7D,EAAE,CAAC,MAAM,CAAC,iBAAQ,CAAC;iBAChB,MAAM,CAAC;gBACN,EAAE,EAAE,SAAS;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;iBACD,mBAAmB,EAAE;iBACrB,GAAG,EAAE,CAAC;YAET,EAAE,CAAC,MAAM,CAAC,wBAAe,CAAC;iBACvB,MAAM,CAAC;gBACN,MAAM;gBACN,SAAS;aACV,CAAC;iBACD,mBAAmB,EAAE;iBACrB,GAAG,EAAE,CAAC;YAET,gBAAM,CAAC,KAAK,CAAC,qBAAqB,SAAS,EAAE,CAAC,CAAC;YAE/C,cAAc;YACd,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7D,MAAM,KAAK,GAAG,IAAA,mBAAM,EAAC,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC;oBAE9C,EAAE,CAAC,MAAM,CAAC,aAAS,CAAC;yBACjB,MAAM,CAAC;wBACN,EAAE,EAAE,KAAK;wBACT,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,QAAQ;qBAChB,CAAC;yBACD,mBAAmB,EAAE;yBACrB,GAAG,EAAE,CAAC;oBAET,EAAE,CAAC,MAAM,CAAC,oBAAW,CAAC;yBACnB,MAAM,CAAC;wBACN,MAAM;wBACN,KAAK;qBACN,CAAC;yBACD,mBAAmB,EAAE;yBACrB,GAAG,EAAE,CAAC;oBAET,gBAAM,CAAC,KAAK,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,YACE,MAA8B,EAC9B,IAQC;QAED,MAAM,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,IAAI,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,EAAE,IAAI,YAAY,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,IAAI,EAAE,SAAS,IAAI,KAAK,CAAC;IAC5C,CAAC;IAED,OAAO;QACL;;;WAGG;QACH,OAAO,IAAI,CAAC,UAAU,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC5B,CAAC;IAED,QAAQ,CAAC,KAAoB;QAC3B,IAAA,wBAAS,EAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACvD,IAAA,wBAAS,EAAC,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;QACpD,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,KAAK,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,SAAS,GAAwB;YACrC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;YACpC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,SAAS,EAAE,IAAA,0BAAmB,GAAE;SACjC,CAAC;QAEF,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YACzB,IAAA,wBAAS,EAAC,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;YACpD,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC;QACtC,CAAC;QACD,MAAM,EAAE,CAAC,MAAM,CAAC,cAAU,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAU,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACnF,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YACzB,IAAA,wBAAS,EAAC,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;YACpD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAChD,CAAC;QACD,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,KAAK,GAAG,IAAA,iBAAG,EAAA;+DAC0C,IAAI,CAAC,EAAE,2BAA2B,CAAC;QAC9F,aAAa;QACb,MAAM,OAAO,GAAsB,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEvD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;IACzC,CAAC;IAED,UAAU;QACR,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YACzB,IAAA,wBAAS,EAAC,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;YACpD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACnD,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QACjF,CAAC;QACD,OAAO,IAAA,iDAAqB,EAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAsB,EAAE,IAAoB;QAC1D,MAAM,SAAS,GAAG,MAAM,oBAAU,CAAC,wBAAwB,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;YACjF,OAAO,EAAE,IAAI,CAAC,SAAS;SACxB,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,OAA0B;QACzC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;YACnB,MAAM,EAAE,CAAC,MAAM,CAAC,cAAU,CAAC,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAU,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACvF,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,SAAqB;QACtC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;YACnB,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAChC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;oBACvB,EAAE,CAAC,MAAM,CAAC,yBAAgB,CAAC;yBACxB,MAAM,CAAC;wBACN,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,UAAU,EAAE,EAAE;qBACf,CAAC;yBACD,mBAAmB,EAAE;yBACrB,GAAG,EAAE,CAAC;gBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW;QACf,IAAI,CAAC,OAAO,GAAG,MAAM,oBAAU,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YACzB,IAAA,wBAAS,EAAC,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;YACpD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QACjC,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IACD,KAAK,CAAC,iBAAiB;QACrB,IAAI,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC;YACzB,IAAA,wBAAS,EAAC,IAAI,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;YACpD,OAAO;gBACL,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;gBACjD,OAAO,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO;gBAChC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK;gBAC5B,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK;aAC7B,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3B,CAAC;QACD,MAAM,KAAK,GAAkB;YAC3B,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;gBACT,KAAK,EAAE,CAAC;aACT;SACF,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,KAAK,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,IAAI,CAAC,CAAC;YACtD,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,OAAO,EAAE,aAAa,IAAI,CAAC,CAAC;YACrD,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;YAClE,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;YAClE,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;YAC1E,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;QAClE,CAAC;QAED,OAAO;YACL,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;YACjD,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,EAAE,CAAC;YACtD,KAAK;SACN,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa;QACjB,MAAM,OAAO,GAAgB;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;YACvB,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;YACjD,OAAO,EAAE,MAAM,IAAI,CAAC,iBAAiB,EAAE;YACvC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;YAC3B,OAAO,EAAE,IAAI,CAAC,UAAU,EAAE;YAC1B,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,IAAI;SAClC,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YACxB,EAAE,CAAC,MAAM,CAAC,wBAAe,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,wBAAe,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YAC5E,EAAE,CAAC,MAAM,CAAC,uBAAc,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,uBAAc,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YAC1E,EAAE,CAAC,MAAM,CAAC,oBAAW,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,oBAAW,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YACpE,EAAE,CAAC,MAAM,CAAC,yBAAgB,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,yBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9E,EAAE,CAAC,MAAM,CAAC,yBAAgB,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,yBAAgB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YAC9E,EAAE,CAAC,MAAM,CAAC,cAAU,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,cAAU,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QAChE,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AA5YD,uBA4YC;AAEM,KAAK,UAAU,uBAAuB,CAC3C,QAAgB,+BAAmB,EACnC,iBAA0B,EAC1B,SAAkB;IAElB,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;IACnB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,EAAE;SACb,MAAM,CAAC;QACN,MAAM,EAAE,cAAU,CAAC,EAAE;QACrB,SAAS,EAAE,cAAU,CAAC,SAAS;QAC/B,WAAW,EAAE,cAAU,CAAC,WAAW;QACnC,QAAQ,EAAE,IAAA,iBAAG,EAAA,OAAO,yBAAgB,CAAC,OAAO,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC;QAClE,SAAS,EAAE,wBAAe,CAAC,SAAS;KACrC,CAAC;SACD,IAAI,CAAC,cAAU,CAAC;SAChB,QAAQ,CAAC,wBAAe,EAAE,IAAA,gBAAE,EAAC,cAAU,CAAC,EAAE,EAAE,wBAAe,CAAC,MAAM,CAAC,CAAC;SACpE,QAAQ,CAAC,yBAAgB,EAAE,IAAA,gBAAE,EAAC,cAAU,CAAC,EAAE,EAAE,yBAAgB,CAAC,MAAM,CAAC,CAAC;SACtE,KAAK,CACJ,IAAA,iBAAG,EACD,SAAS,CAAC,CAAC,CAAC,IAAA,gBAAE,EAAC,wBAAe,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAChE,iBAAiB,CAAC,CAAC,CAAC,IAAA,kBAAI,EAAC,cAAU,CAAC,WAAW,EAAE,IAAI,iBAAiB,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EACtF,IAAA,gBAAE,EAAC,cAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAC3B,CACF;SACA,OAAO,CAAC,cAAU,CAAC,EAAE,CAAC,CAAC;IAE1B,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAA,kBAAI,EAAC,cAAU,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC;IAE7E,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,QAAQ,EAAG,MAAM,CAAC,QAAmB,IAAI,CAAC;QAC1C,SAAS,EAAE,MAAM,CAAC,SAAS;KAC5B,CAAC,CAAC,CAAC;IAEJ,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,aAAa,GAAG,OAAO,GAAG,SAAS,CAAC;IAC1C,gBAAM,CAAC,KAAK,CAAC,uCAAuC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAElF,OAAO,aAAa,CAAC;AACvB,CAAC"}

package/dist/src/models/evalResult.d.ts

@@ -0,0 +1,50 @@
+import type { AtomicTestCase, GradingResult, Prompt, ProviderOptions, ProviderResponse } from '../types';
+import { type EvaluateResult } from '../types';
+export default class EvalResult {
+    static createFromEvaluateResult(evalId: string, result: EvaluateResult, testCase: AtomicTestCase, opts?: {
+        persist: boolean;
+    }): Promise<EvalResult>;
+    static createManyFromEvaluateResult(results: EvaluateResult[], evalId: string): Promise<EvalResult[]>;
+    static findById(id: string): Promise<EvalResult | null>;
+    static findManyByEvalId(evalId: string): Promise<EvalResult[]>;
+    id: string;
+    evalId: string;
+    description?: string | null;
+    promptIdx: number;
+    testIdx: number;
+    testCase: AtomicTestCase;
+    prompt: Prompt;
+    promptId: string;
+    error?: string | null;
+    success: boolean;
+    score: number;
+    response: ProviderResponse | null;
+    gradingResult: GradingResult | null;
+    namedScores: Record<string, number>;
+    provider: ProviderOptions;
+    latencyMs: number;
+    cost: number;
+    persisted: boolean;
+    constructor(opts: {
+        id: string;
+        evalId: string;
+        promptIdx: number;
+        testIdx: number;
+        testCase: AtomicTestCase;
+        prompt: Prompt;
+        promptId?: string | null;
+        error?: string | null;
+        success: boolean;
+        score: number;
+        response: ProviderResponse | null;
+        gradingResult: GradingResult | null;
+        namedScores?: Record<string, number> | null;
+        provider: ProviderOptions;
+        latencyMs?: number | null;
+        cost?: number | null;
+        persisted?: boolean;
+    });
+    save(): Promise<void>;
+    toEvaluateResult(): EvaluateResult;
+}
+//# sourceMappingURL=evalResult.d.ts.map

package/dist/src/models/evalResult.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalResult.d.ts","sourceRoot":"","sources":["../../../src/models/evalResult.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,MAAM,EACN,eAAe,EACf,gBAAgB,EACjB,MAAM,UAAU,CAAC;AAClB,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,UAAU,CAAC;AAG/C,MAAM,CAAC,OAAO,OAAO,UAAU;WAChB,wBAAwB,CACnC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,cAAc,EACtB,QAAQ,EAAE,cAAc,EACxB,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE;WAgChB,4BAA4B,CAAC,OAAO,EAAE,cAAc,EAAE,EAAE,MAAM,EAAE,MAAM;WAetE,QAAQ,CAAC,EAAE,EAAE,MAAM;WAMnB,gBAAgB,CAAC,MAAM,EAAE,MAAM;IAS5C,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,cAAc,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAC;IAClC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,QAAQ,EAAE,eAAe,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,OAAO,CAAC;gBAEP,IAAI,EAAE;QAChB,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,cAAc,CAAC;QACzB,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACzB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACtB,OAAO,EAAE,OAAO,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,gBAAgB,GAAG,IAAI,CAAC;QAClC,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;QACpC,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;QAC5C,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;IAqBK,IAAI;IAeV,gBAAgB,IAAI,cAAc;CAqBnC"}

package/dist/src/models/evalResult.js

@@ -0,0 +1,122 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto_1 = require("crypto");
+const drizzle_orm_1 = require("drizzle-orm");
+const database_1 = require("../database");
+const tables_1 = require("../database/tables");
+const utils_1 = require("../prompts/utils");
+const time_1 = require("../util/time");
+class EvalResult {
+    static async createFromEvaluateResult(evalId, result, testCase, opts) {
+        const persist = opts?.persist == null ? true : opts.persist;
+        const { prompt, error, score, latencyMs, success, provider, gradingResult, namedScores, cost } = result;
+        const args = {
+            id: (0, crypto_1.randomUUID)(),
+            evalId,
+            testCase,
+            promptIdx: result.promptIdx,
+            testIdx: result.testIdx,
+            prompt,
+            promptId: (0, utils_1.hashPrompt)(prompt),
+            error: error?.toString(),
+            success,
+            score: score == null ? 0 : score,
+            response: result.response || null,
+            gradingResult: gradingResult || null,
+            namedScores,
+            provider,
+            latencyMs,
+            cost,
+        };
+        if (persist) {
+            const db = (0, database_1.getDb)();
+            const dbResult = await db.insert(tables_1.evalResultsTable).values(args).returning();
+            return new EvalResult({ ...dbResult[0], persisted: true });
+        }
+        return new EvalResult(args);
+    }
+    static async createManyFromEvaluateResult(results, evalId) {
+        const db = (0, database_1.getDb)();
+        const returnResults = [];
+        await db.transaction(async (tx) => {
+            for (const result of results) {
+                const dbResult = await tx
+                    .insert(tables_1.evalResultsTable)
+                    .values({ ...result, evalId, id: (0, crypto_1.randomUUID)() })
+                    .returning();
+                returnResults.push(new EvalResult({ ...dbResult[0], persisted: true }));
+            }
+        });
+        return returnResults;
+    }
+    static async findById(id) {
+        const db = (0, database_1.getDb)();
+        const result = await db.select().from(tables_1.evalResultsTable).where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.id, id));
+        return result.length > 0 ? new EvalResult({ ...result[0], persisted: true }) : null;
+    }
+    static async findManyByEvalId(evalId) {
+        const db = (0, database_1.getDb)();
+        const results = await db
+            .select()
+            .from(tables_1.evalResultsTable)
+            .where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.evalId, evalId));
+        return results.map((result) => new EvalResult({ ...result, persisted: true }));
+    }
+    constructor(opts) {
+        this.id = opts.id;
+        this.evalId = opts.evalId;
+        this.promptIdx = opts.promptIdx;
+        this.testIdx = opts.testIdx;
+        this.testCase = opts.testCase;
+        this.prompt = opts.prompt;
+        this.promptId = opts.promptId || (0, utils_1.hashPrompt)(opts.prompt);
+        this.error = opts.error;
+        this.score = opts.score;
+        this.success = opts.success;
+        this.response = opts.response;
+        this.gradingResult = opts.gradingResult;
+        this.namedScores = opts.namedScores || {};
+        this.provider = opts.provider;
+        this.latencyMs = opts.latencyMs || 0;
+        this.cost = opts.cost || 0;
+        this.persisted = opts.persisted || false;
+    }
+    async save() {
+        const db = (0, database_1.getDb)();
+        //check if this exists in the db
+        if (this.persisted) {
+            await db
+                .update(tables_1.evalResultsTable)
+                .set({ ...this, updatedAt: (0, time_1.getCurrentTimestamp)() })
+                .where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.id, this.id));
+        }
+        else {
+            const result = await db.insert(tables_1.evalResultsTable).values(this).returning();
+            this.id = result[0].id;
+            this.persisted = true;
+        }
+    }
+    toEvaluateResult() {
+        return {
+            cost: this.cost,
+            description: this.description || undefined,
+            error: this.error || undefined,
+            gradingResult: this.gradingResult,
+            id: this.id,
+            latencyMs: this.latencyMs,
+            namedScores: this.namedScores,
+            prompt: this.prompt,
+            promptId: this.promptId,
+            promptIdx: this.promptIdx,
+            provider: { id: this.provider.id, label: this.provider.label },
+            response: this.response || undefined,
+            score: this.score,
+            success: this.success,
+            testCase: this.testCase,
+            testIdx: this.testIdx,
+            vars: this.testCase.vars || {},
+        };
+    }
+}
+exports.default = EvalResult;
+//# sourceMappingURL=evalResult.js.map

package/dist/src/models/evalResult.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evalResult.js","sourceRoot":"","sources":["../../../src/models/evalResult.ts"],"names":[],"mappings":";;AAAA,mCAAoC;AACpC,6CAAiC;AACjC,0CAAoC;AACpC,+CAAsD;AACtD,4CAA8C;AAS9C,uCAAmD;AAEnD,MAAqB,UAAU;IAC7B,MAAM,CAAC,KAAK,CAAC,wBAAwB,CACnC,MAAc,EACd,MAAsB,EACtB,QAAwB,EACxB,IAA2B;QAE3B,MAAM,OAAO,GAAG,IAAI,EAAE,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;QAC5D,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,WAAW,EAAE,IAAI,EAAE,GAC5F,MAAM,CAAC;QACT,MAAM,IAAI,GAAG;YACX,EAAE,EAAE,IAAA,mBAAU,GAAE;YAChB,MAAM;YACN,QAAQ;YACR,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM;YACN,QAAQ,EAAE,IAAA,kBAAU,EAAC,MAAM,CAAC;YAC5B,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE;YACxB,OAAO;YACP,KAAK,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;YAChC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,aAAa,EAAE,aAAa,IAAI,IAAI;YACpC,WAAW;YACX,QAAQ;YACR,SAAS;YACT,IAAI;SACL,CAAC;QACF,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;YAEnB,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAgB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;YAC5E,OAAO,IAAI,UAAU,CAAC,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,4BAA4B,CAAC,OAAyB,EAAE,MAAc;QACjF,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,aAAa,GAAiB,EAAE,CAAC;QACvC,MAAM,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YAChC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,EAAE;qBACtB,MAAM,CAAC,yBAAgB,CAAC;qBACxB,MAAM,CAAC,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,IAAA,mBAAU,GAAE,EAAE,CAAC;qBAC/C,SAAS,EAAE,CAAC;gBACf,aAAa,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC1E,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,aAAa,CAAC;IACvB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAU;QAC9B,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,yBAAgB,CAAC,CAAC,KAAK,CAAC,IAAA,gBAAE,EAAC,yBAAgB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAC3F,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACtF,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,MAAc;QAC1C,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,EAAE;aACrB,MAAM,EAAE;aACR,IAAI,CAAC,yBAAgB,CAAC;aACtB,KAAK,CAAC,IAAA,gBAAE,EAAC,yBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9C,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,EAAE,GAAG,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACjF,CAAC;IAqBD,YAAY,IAkBX;QACC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAClB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE1B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAA,kBAAU,EAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;QACnB,gCAAgC;QAChC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,MAAM,EAAE;iBACL,MAAM,CAAC,yBAAgB,CAAC;iBACxB,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,SAAS,EAAE,IAAA,0BAAmB,GAAE,EAAE,CAAC;iBAClD,KAAK,CAAC,IAAA,gBAAE,EAAC,yBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7C,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,yBAAgB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;YAC1E,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;IACH,CAAC;IAED,gBAAgB;QACd,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,SAAS;YAC1C,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,SAAS;YAC9B,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,QAAQ,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;YAC9D,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS;YACpC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE;SAC/B,CAAC;IACJ,CAAC;CACF;AAjKD,6BAiKC"}

package/dist/src/models/provider.d.ts

@@ -0,0 +1,9 @@
+import type { ApiProvider } from '../types';
+export default class Provider {
+    id: string;
+    providerId: string;
+    config: Record<string, any>;
+    static createMultiple(providers: ApiProvider[]): Promise<Provider[]>;
+    constructor(id: string, providerId: string, config: Record<string, any>);
+}
+//# sourceMappingURL=provider.d.ts.map

package/dist/src/models/provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/models/provider.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAO5C,MAAM,CAAC,OAAO,OAAO,QAAQ;IA8BlB,EAAE,EAAE,MAAM;IACV,UAAU,EAAE,MAAM;IAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;WA/BvB,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE;gBA6B3C,EAAE,EAAE,MAAM,EACV,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;CAMrC"}

package/dist/src/models/provider.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const drizzle_orm_1 = require("drizzle-orm");
+const database_1 = require("../database");
+const tables_1 = require("../database/tables");
+const createHash_1 = require("../util/createHash");
+function getProviderId(provider) {
+    return (0, createHash_1.sha256)(provider.id() + JSON.stringify(provider.config));
+}
+class Provider {
+    static async createMultiple(providers) {
+        const db = (0, database_1.getDb)();
+        const ret = [];
+        for (const provider of providers) {
+            const id = getProviderId(provider);
+            let providerResult;
+            let results = await db.select().from(tables_1.providers).where((0, drizzle_orm_1.eq)(tables_1.providers.id, id));
+            if (results.length > 0) {
+                providerResult = results[0];
+            }
+            else {
+                results = await db
+                    .insert(tables_1.providers)
+                    .values({
+                    id,
+                    providerId: provider.id(),
+                    config: provider.config || {},
+                })
+                    .onConflictDoNothing()
+                    .returning();
+                providerResult = results[0];
+            }
+            ret.push(new Provider(providerResult.id, providerResult.providerId, providerResult.config));
+        }
+        return ret;
+    }
+    constructor(id, providerId, config) {
+        this.id = id;
+        this.providerId = providerId;
+        this.config = config;
+        this.id = id;
+        this.providerId = providerId;
+        this.config = config;
+    }
+}
+exports.default = Provider;
+//# sourceMappingURL=provider.js.map