promptfoo 0.69.2 → 0.70.1

package/dist/src/config.js

@@ -0,0 +1,424 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.dereferenceConfig = dereferenceConfig;
+exports.readConfig = readConfig;
+exports.maybeReadConfig = maybeReadConfig;
+exports.readConfigs = readConfigs;
+exports.resolveConfigs = resolveConfigs;
+const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
+const chalk_1 = __importDefault(require("chalk"));
+const dedent_1 = __importDefault(require("dedent"));
+const fs = __importStar(require("fs"));
+const glob_1 = require("glob");
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const path = __importStar(require("path"));
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const assertions_1 = require("./assertions");
+const validateAssertions_1 = require("./assertions/validateAssertions");
+const filterTests_1 = require("./commands/eval/filterTests");
+const esm_1 = require("./esm");
+const logger_1 = __importDefault(require("./logger"));
+const prompts_1 = require("./prompts");
+const providers_1 = require("./providers");
+const testCases_1 = require("./testCases");
+const types_1 = require("./types");
+const util_1 = require("./util");
+async function dereferenceConfig(rawConfig) {
+    if (process.env.PROMPTFOO_DISABLE_REF_PARSER) {
+        return rawConfig;
+    }
+    // Track and delete tools[i].function for each tool, preserving the rest of the properties
+    // https://github.com/promptfoo/promptfoo/issues/364
+    // Remove parameters from functions and tools to prevent dereferencing
+    const extractFunctionParameters = (functions) => {
+        return functions.map((func) => {
+            const { parameters } = func;
+            delete func.parameters;
+            return { parameters };
+        });
+    };
+    const extractToolParameters = (tools) => {
+        return tools.map((tool) => {
+            const { parameters } = tool.function || {};
+            if (tool.function?.parameters) {
+                delete tool.function.parameters;
+            }
+            return { parameters };
+        });
+    };
+    // Restore parameters to functions and tools after dereferencing
+    const restoreFunctionParameters = (functions, parametersList) => {
+        functions.forEach((func, index) => {
+            if (parametersList[index]?.parameters) {
+                func.parameters = parametersList[index].parameters;
+            }
+        });
+    };
+    const restoreToolParameters = (tools, parametersList) => {
+        tools.forEach((tool, index) => {
+            if (parametersList[index]?.parameters) {
+                tool.function = tool.function || {};
+                tool.function.parameters = parametersList[index].parameters;
+            }
+        });
+    };
+    const functionsParametersList = [];
+    const toolsParametersList = [];
+    if (Array.isArray(rawConfig.providers)) {
+        rawConfig.providers.forEach((provider, providerIndex) => {
+            if (typeof provider === 'string')
+                return;
+            if (typeof provider === 'function')
+                return;
+            if (!provider.config) {
+                // Handle when provider is a map
+                provider = Object.values(provider)[0];
+            }
+            if (provider.config?.functions) {
+                functionsParametersList[providerIndex] = extractFunctionParameters(provider.config.functions);
+            }
+            if (provider.config?.tools) {
+                toolsParametersList[providerIndex] = extractToolParameters(provider.config.tools);
+            }
+        });
+    }
+    // Dereference JSON
+    const config = (await json_schema_ref_parser_1.default.dereference(rawConfig));
+    // Restore functions and tools parameters
+    if (Array.isArray(config.providers)) {
+        config.providers.forEach((provider, index) => {
+            if (typeof provider === 'string')
+                return;
+            if (typeof provider === 'function')
+                return;
+            if (!provider.config) {
+                // Handle when provider is a map
+                provider = Object.values(provider)[0];
+            }
+            if (functionsParametersList[index]) {
+                provider.config.functions = provider.config.functions || [];
+                restoreFunctionParameters(provider.config.functions, functionsParametersList[index]);
+            }
+            if (toolsParametersList[index]) {
+                provider.config.tools = provider.config.tools || [];
+                restoreToolParameters(provider.config.tools, toolsParametersList[index]);
+            }
+        });
+    }
+    return config;
+}
+async function readConfig(configPath) {
+    const ext = path.parse(configPath).ext;
+    switch (ext) {
+        case '.json':
+        case '.yaml':
+        case '.yml':
+            const rawConfig = js_yaml_1.default.load(fs.readFileSync(configPath, 'utf-8'));
+            return dereferenceConfig(rawConfig);
+        case '.js':
+        case '.cjs':
+        case '.mjs':
+            return (await (0, esm_1.importModule)(configPath));
+        default:
+            throw new Error(`Unsupported configuration file format: ${ext}`);
+    }
+}
+async function maybeReadConfig(configPath) {
+    if (!fs.existsSync(configPath)) {
+        return undefined;
+    }
+    return readConfig(configPath);
+}
+/**
+ * Reads multiple configuration files and combines them into a single UnifiedConfig.
+ *
+ * @param {string[]} configPaths - An array of paths to configuration files. Supports glob patterns.
+ * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
+ */
+async function readConfigs(configPaths) {
+    const configs = [];
+    for (const configPath of configPaths) {
+        const globPaths = (0, glob_1.globSync)(configPath, {
+            windowsPathsNoEscape: true,
+        });
+        if (globPaths.length === 0) {
+            throw new Error(`No configuration file found at ${configPath}`);
+        }
+        for (const globPath of globPaths) {
+            const config = await readConfig(globPath);
+            configs.push(config);
+        }
+    }
+    const providers = [];
+    const seenProviders = new Set();
+    configs.forEach((config) => {
+        (0, tiny_invariant_1.default)(typeof config.providers !== 'function', 'Providers cannot be a function for multiple configs');
+        if (typeof config.providers === 'string') {
+            if (!seenProviders.has(config.providers)) {
+                providers.push(config.providers);
+                seenProviders.add(config.providers);
+            }
+        }
+        else if (Array.isArray(config.providers)) {
+            config.providers.forEach((provider) => {
+                if (!seenProviders.has(JSON.stringify(provider))) {
+                    providers.push(provider);
+                    seenProviders.add(JSON.stringify(provider));
+                }
+            });
+        }
+    });
+    const tests = [];
+    for (const config of configs) {
+        if (typeof config.tests === 'string') {
+            const newTests = await (0, testCases_1.readTests)(config.tests, path.dirname(configPaths[0]));
+            tests.push(...newTests);
+        }
+        else if (Array.isArray(config.tests)) {
+            tests.push(...config.tests);
+        }
+    }
+    const configsAreStringOrArray = configs.every((config) => typeof config.prompts === 'string' || Array.isArray(config.prompts));
+    const configsAreObjects = configs.every((config) => typeof config.prompts === 'object');
+    let prompts = configsAreStringOrArray ? [] : {};
+    const makeAbsolute = (configPath, relativePath) => {
+        if (typeof relativePath === 'string') {
+            if (relativePath.startsWith('file://')) {
+                relativePath =
+                    'file://' + path.resolve(path.dirname(configPath), relativePath.slice('file://'.length));
+            }
+            return relativePath;
+        }
+        else if (typeof relativePath === 'object' && relativePath.id) {
+            if (relativePath.id.startsWith('file://')) {
+                relativePath.id =
+                    'file://' +
+                        path.resolve(path.dirname(configPath), relativePath.id.slice('file://'.length));
+            }
+            return relativePath;
+        }
+        else {
+            throw new Error('Invalid prompt object');
+        }
+    };
+    const seenPrompts = new Set();
+    const addSeenPrompt = (prompt) => {
+        if (typeof prompt === 'string') {
+            seenPrompts.add(prompt);
+        }
+        else if (typeof prompt === 'object' && prompt.id) {
+            seenPrompts.add(prompt);
+        }
+        else {
+            throw new Error('Invalid prompt object');
+        }
+    };
+    configs.forEach((config, idx) => {
+        if (typeof config.prompts === 'string') {
+            (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix string and map-type prompts');
+            const absolutePrompt = makeAbsolute(configPaths[idx], config.prompts);
+            addSeenPrompt(absolutePrompt);
+        }
+        else if (Array.isArray(config.prompts)) {
+            (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix configs with map and array-type prompts');
+            config.prompts
+                .map((prompt) => makeAbsolute(configPaths[idx], prompt))
+                .forEach((prompt) => addSeenPrompt(prompt));
+        }
+        else {
+            // Object format such as { 'prompts/prompt1.txt': 'foo', 'prompts/prompt2.txt': 'bar' }
+            (0, tiny_invariant_1.default)(typeof prompts === 'object', 'Cannot mix configs with map and array-type prompts');
+            prompts = { ...prompts, ...config.prompts };
+        }
+    });
+    if (Array.isArray(prompts)) {
+        prompts.push(...Array.from(seenPrompts));
+    }
+    // Combine all configs into a single UnifiedConfig
+    const combinedConfig = {
+        description: configs.map((config) => config.description).join(', '),
+        providers,
+        prompts,
+        tests,
+        scenarios: configs.flatMap((config) => config.scenarios || []),
+        defaultTest: configs.reduce((prev, curr) => {
+            return {
+                ...prev,
+                ...curr.defaultTest,
+                vars: { ...prev?.vars, ...curr.defaultTest?.vars },
+                assert: [...(prev?.assert || []), ...(curr.defaultTest?.assert || [])],
+                options: { ...prev?.options, ...curr.defaultTest?.options },
+            };
+        }, {}),
+        nunjucksFilters: configs.reduce((prev, curr) => ({ ...prev, ...curr.nunjucksFilters }), {}),
+        env: configs.reduce((prev, curr) => ({ ...prev, ...curr.env }), {}),
+        evaluateOptions: configs.reduce((prev, curr) => ({ ...prev, ...curr.evaluateOptions }), {}),
+        commandLineOptions: configs.reduce((prev, curr) => ({ ...prev, ...curr.commandLineOptions }), {}),
+        metadata: configs.reduce((prev, curr) => ({ ...prev, ...curr.metadata }), {}),
+        sharing: !configs.some((config) => config.sharing === false),
+    };
+    return combinedConfig;
+}
+async function resolveConfigs(cmdObj, defaultConfig) {
+    // Config parsing
+    let fileConfig = {};
+    const configPaths = cmdObj.config;
+    if (configPaths) {
+        fileConfig = await readConfigs(configPaths);
+    }
+    // Standalone assertion mode
+    if (cmdObj.assertions) {
+        if (!cmdObj.modelOutputs) {
+            logger_1.default.error(chalk_1.default.red('You must provide --model-outputs when using --assertions'));
+            process.exit(1);
+        }
+        const modelOutputs = JSON.parse(fs.readFileSync(path.join(process.cwd(), cmdObj.modelOutputs), 'utf8'));
+        const assertions = await (0, assertions_1.readAssertions)(cmdObj.assertions);
+        fileConfig.prompts = ['{{output}}'];
+        fileConfig.providers = ['echo'];
+        fileConfig.tests = modelOutputs.map((output) => {
+            if (typeof output === 'string') {
+                return {
+                    vars: {
+                        output,
+                    },
+                    assert: assertions,
+                };
+            }
+            return {
+                vars: {
+                    output: output.output,
+                    ...(output.tags === undefined ? {} : { tags: output.tags.join(', ') }),
+                },
+                assert: assertions,
+            };
+        });
+    }
+    // Use basepath in cases where path was supplied in the config file
+    const basePath = configPaths ? path.dirname(configPaths[0]) : '';
+    const defaultTestRaw = fileConfig.defaultTest || defaultConfig.defaultTest;
+    const config = {
+        description: fileConfig.description || defaultConfig.description,
+        prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts || [],
+        providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers || [],
+        tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests || [],
+        scenarios: fileConfig.scenarios || defaultConfig.scenarios,
+        env: fileConfig.env || defaultConfig.env,
+        sharing: process.env.PROMPTFOO_DISABLE_SHARING === '1'
+            ? false
+            : fileConfig.sharing ?? defaultConfig.sharing ?? true,
+        defaultTest: defaultTestRaw ? await (0, testCases_1.readTest)(defaultTestRaw, basePath) : undefined,
+        derivedMetrics: fileConfig.derivedMetrics || defaultConfig.derivedMetrics,
+        outputPath: cmdObj.output || fileConfig.outputPath || defaultConfig.outputPath,
+        metadata: fileConfig.metadata || defaultConfig.metadata,
+    };
+    // Validation
+    if (!config.prompts || config.prompts.length === 0) {
+        logger_1.default.error(chalk_1.default.red('You must provide at least 1 prompt'));
+        process.exit(1);
+    }
+    if (!config.providers || config.providers.length === 0) {
+        logger_1.default.error(chalk_1.default.red('You must specify at least 1 provider (for example, openai:gpt-4o)'));
+        process.exit(1);
+    }
+    (0, tiny_invariant_1.default)(Array.isArray(config.providers), 'providers must be an array');
+    config.providers.forEach((provider) => {
+        const result = types_1.ProviderSchema.safeParse(provider);
+        if (!result.success) {
+            const errors = result.error.errors
+                .map((err) => {
+                return `- ${err.message}`;
+            })
+                .join('\n');
+            const providerString = typeof provider === 'string' ? provider : JSON.stringify(provider);
+            logger_1.default.warn(chalk_1.default.yellow((0, dedent_1.default) `
+              Provider: ${providerString} encountered errors during schema validation:
+
+                ${errors}
+
+              Please double check your configuration.` + '\n'));
+        }
+    });
+    // Parse prompts, providers, and tests
+    const parsedPrompts = await (0, prompts_1.readPrompts)(config.prompts, cmdObj.prompts ? undefined : basePath);
+    const parsedProviders = await (0, providers_1.loadApiProviders)(config.providers, {
+        env: config.env,
+        basePath,
+    });
+    const parsedTests = await (0, testCases_1.readTests)(config.tests || [], cmdObj.tests ? undefined : basePath);
+    // Parse testCases for each scenario
+    if (fileConfig.scenarios) {
+        for (const scenario of fileConfig.scenarios) {
+            const parsedScenarioTests = await (0, testCases_1.readTests)(scenario.tests, cmdObj.tests ? undefined : basePath);
+            scenario.tests = parsedScenarioTests;
+            const filteredTests = await (0, filterTests_1.filterTests)({
+                ...scenario,
+                providers: parsedProviders,
+                prompts: parsedPrompts,
+            }, {
+                firstN: cmdObj.filterFirstN,
+                pattern: cmdObj.filterPattern,
+                failing: cmdObj.filterFailing,
+            });
+            (0, tiny_invariant_1.default)(filteredTests, 'filteredTests are undefined');
+            scenario.tests = filteredTests;
+        }
+    }
+    const parsedProviderPromptMap = (0, prompts_1.readProviderPromptMap)(config, parsedPrompts);
+    if (parsedPrompts.length === 0) {
+        logger_1.default.error(chalk_1.default.red('No prompts found'));
+        process.exit(1);
+    }
+    const defaultTest = {
+        options: {
+            prefix: cmdObj.promptPrefix,
+            suffix: cmdObj.promptSuffix,
+            provider: cmdObj.grader,
+            // rubricPrompt
+            ...(config.defaultTest?.options || {}),
+        },
+        ...config.defaultTest,
+    };
+    const testSuite = {
+        description: config.description,
+        prompts: parsedPrompts,
+        providers: parsedProviders,
+        providerPromptMap: parsedProviderPromptMap,
+        tests: parsedTests,
+        scenarios: config.scenarios,
+        defaultTest,
+        derivedMetrics: config.derivedMetrics,
+        nunjucksFilters: await (0, util_1.readFilters)(fileConfig.nunjucksFilters || defaultConfig.nunjucksFilters || {}),
+    };
+    if (testSuite.tests) {
+        (0, validateAssertions_1.validateAssertions)(testSuite.tests);
+    }
+    return { config, testSuite, basePath };
+}
+//# sourceMappingURL=config.js.map

package/dist/src/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/config.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2BA,8CAoGC;AAED,gCAeC;AAED,0CAKC;AAQD,kCAkIC;AAED,wCAyKC;AA5cD,iGAA6D;AAC7D,kDAA0B;AAC1B,oDAA4B;AAC5B,uCAAyB;AACzB,+BAAgC;AAChC,sDAA2B;AAC3B,2CAA6B;AAC7B,oEAAuC;AACvC,6CAA8C;AAC9C,wEAAqE;AACrE,6DAA0D;AAC1D,+BAAqC;AACrC,sDAA8B;AAC9B,uCAA+D;AAC/D,2CAA+C;AAC/C,2CAAkD;AAClD,mCAQiB;AACjB,iCAAqC;AAE9B,KAAK,UAAU,iBAAiB,CAAC,SAAwB;IAC9D,IAAI,OAAO,CAAC,GAAG,CAAC,4BAA4B,EAAE,CAAC;QAC7C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,0FAA0F;IAC1F,oDAAoD;IAEpD,sEAAsE;IACtE,MAAM,yBAAyB,GAAG,CAAC,SAAoC,EAAE,EAAE;QACzE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YAC5B,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;YAC5B,OAAO,IAAI,CAAC,UAAU,CAAC;YACvB,OAAO,EAAE,UAAU,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,CAAC,KAA+C,EAAE,EAAE;QAChF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACxB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAClC,CAAC;YACD,OAAO,EAAE,UAAU,EAAE,CAAC;QACxB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,gEAAgE;IAChE,MAAM,yBAAyB,GAAG,CAChC,SAAoC,EACpC,cAAyC,EACzC,EAAE;QACF,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAChC,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;gBACtC,IAAI,CAAC,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC;YACrD,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,qBAAqB,GAAG,CAC5B,KAA+C,EAC/C,cAAyC,EACzC,EAAE;QACF,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC5B,IAAI,cAAc,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;gBACtC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;gBACpC,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,uBAAuB,GAAgC,EAAE,CAAC;IAChE,MAAM,mBAAmB,GAAgC,EAAE,CAAC;IAE5D,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC;QACvC,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,aAAa,EAAE,EAAE;YACtD,IAAI,OAAO,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACzC,IAAI,OAAO,QAAQ,KAAK,UAAU;gBAAE,OAAO;YAC3C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrB,gCAAgC;gBAChC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAoB,CAAC;YAC3D,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;gBAC/B,uBAAuB,CAAC,aAAa,CAAC,GAAG,yBAAyB,CAChE,QAAQ,CAAC,MAAM,CAAC,SAAS,CAC1B,CAAC;YACJ,CAAC;YAED,IAAI,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC;gBAC3B,mBAAmB,CAAC,aAAa,CAAC,GAAG,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpF,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,mBAAmB;IACnB,MAAM,MAAM,GAAG,CAAC,MAAM,gCAAU,CAAC,WAAW,CAAC,SAAS,CAAC,CAA6B,CAAC;IAErF,yCAAyC;IACzC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;YAC3C,IAAI,OAAO,QAAQ,KAAK,QAAQ;gBAAE,OAAO;YACzC,IAAI,OAAO,QAAQ,KAAK,UAAU;gBAAE,OAAO;YAC3C,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACrB,gCAAgC;gBAChC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAoB,CAAC;YAC3D,CAAC;YAED,IAAI,uBAAuB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,QAAQ,CAAC,MAAM,CAAC,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;gBAC5D,yBAAyB,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,uBAAuB,CAAC,KAAK,CAAC,CAAC,CAAC;YACvF,CAAC;YAED,IAAI,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,QAAQ,CAAC,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;gBACpD,qBAAqB,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAEM,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;IACvC,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,OAAO,CAAC;QACb,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,MAAM,SAAS,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAkB,CAAC;YACnF,OAAO,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACtC,KAAK,KAAK,CAAC;QACX,KAAK,MAAM,CAAC;QACZ,KAAK,MAAM;YACT,OAAO,CAAC,MAAM,IAAA,kBAAY,EAAC,UAAU,CAAC,CAAkB,CAAC;QAC3D;YACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,eAAe,CAAC,UAAkB;IACtD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;AAChC,CAAC;AAED;;;;;GAKG;AACI,KAAK,UAAU,WAAW,CAAC,WAAqB;IACrD,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,SAAS,GAAG,IAAA,eAAQ,EAAC,UAAU,EAAE;YACrC,oBAAoB,EAAE,IAAI;SAC3B,CAAC,CAAC;QACH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,kCAAkC,UAAU,EAAE,CAAC,CAAC;QAClE,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC1C,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAA+B,EAAE,CAAC;IACjD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;QACzB,IAAA,wBAAS,EACP,OAAO,MAAM,CAAC,SAAS,KAAK,UAAU,EACtC,qDAAqD,CACtD,CAAC;QACF,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;YACzC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBACjC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACpC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;oBACjD,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBACzB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,KAAK,GAA2B,EAAE,CAAC;IACzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,MAAM,IAAA,qBAAS,EAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC1B,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,MAAM,uBAAuB,GAAG,OAAO,CAAC,KAAK,CAC3C,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAChF,CAAC;IACF,MAAM,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC;IACxF,IAAI,OAAO,GAA6B,uBAAuB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAE1E,MAAM,YAAY,GAAG,CAAC,UAAkB,EAAE,YAA6B,EAAE,EAAE;QACzE,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE,CAAC;YACrC,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBACvC,YAAY;oBACV,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,OAAO,YAAY,CAAC;QACtB,CAAC;aAAM,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,YAAY,CAAC,EAAE,EAAE,CAAC;YAC/D,IAAI,YAAY,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1C,YAAY,CAAC,EAAE;oBACb,SAAS;wBACT,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;YACpF,CAAC;YACD,OAAO,YAAY,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC/C,MAAM,aAAa,GAAG,CAAC,MAAuB,EAAE,EAAE;QAChD,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;YACnD,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC,CAAC;IACF,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE;QAC9B,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YACvC,IAAA,wBAAS,EAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,wCAAwC,CAAC,CAAC;YAC5E,MAAM,cAAc,GAAG,YAAY,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YACtE,aAAa,CAAC,cAAc,CAAC,CAAC;QAChC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,IAAA,wBAAS,EAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,oDAAoD,CAAC,CAAC;YACxF,MAAM,CAAC,OAAO;iBACX,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;iBACvD,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,uFAAuF;YACvF,IAAA,wBAAS,EAAC,OAAO,OAAO,KAAK,QAAQ,EAAE,oDAAoD,CAAC,CAAC;YAC7F,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;QAC9C,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,kDAAkD;IAClD,MAAM,cAAc,GAAkB;QACpC,WAAW,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACnE,SAAS;QACT,OAAO;QACP,KAAK;QACL,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;QAC9D,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAmC,EAAE,IAAI,EAAE,EAAE;YACxE,OAAO;gBACL,GAAG,IAAI;gBACP,GAAG,IAAI,CAAC,WAAW;gBACnB,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE;gBAClD,MAAM,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,IAAI,EAAE,CAAC,CAAC;gBACtE,OAAO,EAAE,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE;aAC5D,CAAC;QACJ,CAAC,EAAE,EAAE,CAAC;QACN,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3F,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QACnE,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3F,kBAAkB,EAAE,OAAO,CAAC,MAAM,CAChC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC,EACzD,EAAE,CACH;QACD,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7E,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC;KAC7D,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC;AAEM,KAAK,UAAU,cAAc,CAClC,MAAmC,EACnC,aAAqC;IAErC,iBAAiB;IACjB,IAAI,UAAU,GAA2B,EAAE,CAAC;IAC5C,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,IAAI,WAAW,EAAE,CAAC;QAChB,UAAU,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,CAAC;IAC9C,CAAC;IAED,4BAA4B;IAC5B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACzB,gBAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC,CAAC;YACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAC7B,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,CACnB,CAAC;QACtD,MAAM,UAAU,GAAG,MAAM,IAAA,2BAAc,EAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC3D,UAAU,CAAC,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC;QACpC,UAAU,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC;QAChC,UAAU,CAAC,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC7C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;gBAC/B,OAAO;oBACL,IAAI,EAAE;wBACJ,MAAM;qBACP;oBACD,MAAM,EAAE,UAAU;iBACnB,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE;oBACJ,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;iBACvE;gBACD,MAAM,EAAE,UAAU;aACnB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,mEAAmE;IACnE,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEjE,MAAM,cAAc,GAAG,UAAU,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW,CAAC;IAC3E,MAAM,MAAM,GAAkE;QAC5E,WAAW,EAAE,UAAU,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW;QAChE,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,UAAU,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO,IAAI,EAAE;QAC5E,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS,IAAI,EAAE;QACpF,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,IAAI,IAAI,UAAU,CAAC,KAAK,IAAI,aAAa,CAAC,KAAK,IAAI,EAAE;QACnF,SAAS,EAAE,UAAU,CAAC,SAAS,IAAI,aAAa,CAAC,SAAS;QAC1D,GAAG,EAAE,UAAU,CAAC,GAAG,IAAI,aAAa,CAAC,GAAG;QACxC,OAAO,EACL,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,GAAG;YAC3C,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,UAAU,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO,IAAI,IAAI;QACzD,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC,MAAM,IAAA,oBAAQ,EAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS;QAClF,cAAc,EAAE,UAAU,CAAC,cAAc,IAAI,aAAa,CAAC,cAAc;QACzE,UAAU,EAAE,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,UAAU,IAAI,aAAa,CAAC,UAAU;QAC9E,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;KACxD,CAAC;IAEF,aAAa;IACb,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnD,gBAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvD,gBAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,mEAAmE,CAAC,CAAC,CAAC;QAC7F,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,IAAA,wBAAS,EAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,4BAA4B,CAAC,CAAC;IACzE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,sBAAc,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM;iBAC/B,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBACX,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;YAC5B,CAAC,CAAC;iBACD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,cAAc,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;YAC1F,gBAAM,CAAC,IAAI,CACT,eAAK,CAAC,MAAM,CACV,IAAA,gBAAM,EAAA;0BACU,cAAc;;kBAEtB,MAAM;;sDAE8B,GAAG,IAAI,CACpD,CACF,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,sCAAsC;IACtC,MAAM,aAAa,GAAG,MAAM,IAAA,qBAAW,EAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC/F,MAAM,eAAe,GAAG,MAAM,IAAA,4BAAgB,EAAC,MAAM,CAAC,SAAS,EAAE;QAC/D,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,QAAQ;KACT,CAAC,CAAC;IACH,MAAM,WAAW,GAAe,MAAM,IAAA,qBAAS,EAC7C,MAAM,CAAC,KAAK,IAAI,EAAE,EAClB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CACpC,CAAC;IAEF,oCAAoC;IACpC,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;QACzB,KAAK,MAAM,QAAQ,IAAI,UAAU,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,mBAAmB,GAAe,MAAM,IAAA,qBAAS,EACrD,QAAQ,CAAC,KAAK,EACd,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CACpC,CAAC;YACF,QAAQ,CAAC,KAAK,GAAG,mBAAmB,CAAC;YACrC,MAAM,aAAa,GAAG,MAAM,IAAA,yBAAW,EACrC;gBACE,GAAG,QAAQ;gBACX,SAAS,EAAE,eAAe;gBAC1B,OAAO,EAAE,aAAa;aACvB,EACD;gBACE,MAAM,EAAE,MAAM,CAAC,YAAY;gBAC3B,OAAO,EAAE,MAAM,CAAC,aAAa;gBAC7B,OAAO,EAAE,MAAM,CAAC,aAAa;aAC9B,CACF,CAAC;YACF,IAAA,wBAAS,EAAC,aAAa,EAAE,6BAA6B,CAAC,CAAC;YACxD,QAAQ,CAAC,KAAK,GAAG,aAAa,CAAC;QACjC,CAAC;IACH,CAAC;IAED,MAAM,uBAAuB,GAAG,IAAA,+BAAqB,EAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAE7E,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,gBAAM,CAAC,KAAK,CAAC,eAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAC5C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,WAAW,GAAa;QAC5B,OAAO,EAAE;YACP,MAAM,EAAE,MAAM,CAAC,YAAY;YAC3B,MAAM,EAAE,MAAM,CAAC,YAAY;YAC3B,QAAQ,EAAE,MAAM,CAAC,MAAM;YACvB,eAAe;YACf,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,IAAI,EAAE,CAAC;SACvC;QACD,GAAG,MAAM,CAAC,WAAW;KACtB,CAAC;IAEF,MAAM,SAAS,GAAc;QAC3B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,OAAO,EAAE,aAAa;QACtB,SAAS,EAAE,eAAe;QAC1B,iBAAiB,EAAE,uBAAuB;QAC1C,KAAK,EAAE,WAAW;QAClB,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,WAAW;QACX,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,eAAe,EAAE,MAAM,IAAA,kBAAW,EAChC,UAAU,CAAC,eAAe,IAAI,aAAa,CAAC,eAAe,IAAI,EAAE,CAClE;KACF,CAAC;IAEF,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;QACpB,IAAA,uCAAkB,EAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC"}

package/dist/src/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EACV,WAAW,EAEX,eAAe,EAGf,eAAe,EAEf,iBAAiB,EACjB,MAAM,EAEN,SAAS,EAGV,MAAM,SAAS,CAAC;AAGjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,OAAO,CAM7F;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GACzC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CA4BjC;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EACrC,eAAe,CAAC,EAAE,iBAAiB,EACnC,QAAQ,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,MAAM,CAAC,CAuHjB;AA+wBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EACV,WAAW,EAEX,eAAe,EAGf,eAAe,EAEf,iBAAiB,EACjB,MAAM,EAEN,SAAS,EAGV,MAAM,SAAS,CAAC;AAGjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,OAAO,CAM7F;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GACzC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CA4BjC;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EACrC,eAAe,CAAC,EAAE,iBAAiB,EACnC,QAAQ,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,MAAM,CAAC,CAuHjB;AAixBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}

package/dist/src/evaluator.js

@@ -356,11 +356,9 @@ class Evaluator {
                 const checkResult = await (0, assertions_1.runAssertions)({
                     prompt: renderedPrompt,
                     provider,
+                    providerResponse: processedResponse,
                     test,
-                    output: processedResponse.output,
                     latencyMs: response.cached ? undefined : latencyMs,
-                    logProbs: response.logProbs,
-                    cost: processedResponse.cost,
                 });
                 if (!checkResult.pass) {
                     ret.error = checkResult.reason;
@@ -453,16 +451,18 @@ class Evaluator {
             }
         }
         // Split prompts by provider
-        for (const prompt of testSuite.prompts) {
-            for (const provider of testSuite.providers) {
+        // Order matters - keep provider in outer loop to reduce need to swap models during local inference.
+        for (const provider of testSuite.providers) {
+            for (const prompt of testSuite.prompts) {
                 // Check if providerPromptMap exists and if it contains the current prompt's label
-                if (!isAllowedPrompt(prompt, testSuite.providerPromptMap?.[provider.id()])) {
+                const providerKey = provider.label || provider.id();
+                if (!isAllowedPrompt(prompt, testSuite.providerPromptMap?.[providerKey])) {
                     continue;
                 }
                 const completedPrompt = {
                     ...prompt,
                     id: (0, util_1.sha256)(typeof prompt.raw === 'object' ? JSON.stringify(prompt.raw) : prompt.raw),
-                    provider: provider.label || provider.id(),
+                    provider: providerKey,
                     label: prompt.label,
                     metrics: {
                         score: 0,
@@ -560,9 +560,11 @@ class Evaluator {
             for (let repeatIndex = 0; repeatIndex < numRepeat; repeatIndex++) {
                 for (const vars of varCombinations) {
                     let colIndex = 0;
-                    for (const prompt of testSuite.prompts) {
-                        for (const provider of testSuite.providers) {
-                            if (!isAllowedPrompt(prompt, testSuite.providerPromptMap?.[provider.id()])) {
+                    // Order matters - keep provider in outer loop to reduce need to swap models during local inference.
+                    for (const provider of testSuite.providers) {
+                        for (const prompt of testSuite.prompts) {
+                            const providerKey = provider.label || provider.id();
+                            if (!isAllowedPrompt(prompt, testSuite.providerPromptMap?.[providerKey])) {
                                 continue;
                             }
                             runEvalOptions.push({