promptfoo 0.96.0 → 0.96.1

package/dist/src/assertions/index.js

@@ -1,27 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -32,36 +9,50 @@ exports.runAssertions = runAssertions;
 exports.runCompareAssertion = runCompareAssertion;
 exports.readAssertions = readAssertions;
 const async_1 = __importDefault(require("async"));
-const fastest_levenshtein_1 = require("fastest-levenshtein");
 const fs_1 = __importDefault(require("fs"));
-const rouge = __importStar(require("js-rouge"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
-const node_util_1 = __importDefault(require("node:util"));
 const path_1 = __importDefault(require("path"));
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const cliState_1 = __importDefault(require("../cliState"));
 const envars_1 = require("../envars");
 const esm_1 = require("../esm");
-const fetch_1 = require("../fetch");
 const logger_1 = __importDefault(require("../logger"));
 const matchers_1 = require("../matchers");
-const openaiUtil_1 = require("../providers/openaiUtil");
 const packageParser_1 = require("../providers/packageParser");
-const shared_1 = require("../providers/shared");
 const pythonUtils_1 = require("../python/pythonUtils");
-const wrapper_1 = require("../python/wrapper");
-const graders_1 = require("../redteam/graders");
 const telemetry_1 = __importDefault(require("../telemetry"));
-const types_1 = require("../types");
 const file_1 = require("../util/file");
 const templates_1 = require("../util/templates");
 const transform_1 = require("../util/transform");
 const AssertionsResult_1 = require("./AssertionsResult");
+const answerRelevance_1 = require("./answerRelevance");
 const bleu_1 = require("./bleu");
+const classifier_1 = require("./classifier");
+const contains_1 = require("./contains");
+const contextFaithfulness_1 = require("./contextFaithfulness");
+const contextRecall_1 = require("./contextRecall");
+const contextRelevance_1 = require("./contextRelevance");
+const cost_1 = require("./cost");
+const equals_1 = require("./equals");
+const factuality_1 = require("./factuality");
+const javascript_1 = require("./javascript");
 const json_1 = require("./json");
+const latency_1 = require("./latency");
+const levenshtein_1 = require("./levenshtein");
+const llmRubric_1 = require("./llmRubric");
+const modelGradedClosedQa_1 = require("./modelGradedClosedQa");
+const moderation_1 = require("./moderation");
+const openai_1 = require("./openai");
 const perplexity_1 = require("./perplexity");
+const python_1 = require("./python");
+const redteam_1 = require("./redteam");
+const regex_1 = require("./regex");
+const rouge_1 = require("./rouge");
+const similar_1 = require("./similar");
 const sql_1 = require("./sql");
+const startsWith_1 = require("./startsWith");
 const utils_1 = require("./utils");
+const webhook_1 = require("./webhook");
 const xml_1 = require("./xml");
 const ASSERTIONS_MAX_CONCURRENCY = (0, envars_1.getEnvInt)('PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY', 3);
 exports.MODEL_GRADED_ASSERTION_TYPES = new Set([
@@ -69,40 +60,20 @@ exports.MODEL_GRADED_ASSERTION_TYPES = new Set([
     'context-faithfulness',
     'context-recall',
     'context-relevance',
+    'factuality',
     'llm-rubric',
     'model-graded-closedqa',
-    'factuality',
     'model-graded-factuality',
 ]);
 const nunjucks = (0, templates_1.getNunjucksEngine)();
-function coerceString(value) {
-    if (typeof value === 'string') {
-        return value;
-    }
-    return JSON.stringify(value);
-}
-function handleRougeScore(baseType, assertion, expected, output, inverted) {
-    const fnName = baseType[baseType.length - 1];
-    const rougeMethod = rouge[fnName];
-    const score = rougeMethod(output, expected, {});
-    const pass = score >= (assertion.threshold || 0.75) != inverted;
-    return {
-        pass,
-        score: inverted ? 1 - score : score,
-        reason: pass
-            ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${assertion.threshold || 0.75}`
-            : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${assertion.threshold || 0.75}`,
-        assertion,
-    };
-}
 async function runAssertion({ prompt, provider, assertion, test, latencyMs, providerResponse, }) {
     const { cost, logProbs, output: originalOutput } = providerResponse;
     let output = originalOutput;
-    let pass = false;
-    let score = 0.0;
     (0, tiny_invariant_1.default)(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
     const inverse = assertion.type.startsWith('not-');
-    const baseType = inverse ? assertion.type.slice(4) : assertion.type;
+    const baseType = inverse
+        ? assertion.type.slice(4)
+        : assertion.type;
     telemetry_1.default.record('assertion_used', {
         type: baseType,
     });
@@ -112,7 +83,6 @@ async function runAssertion({ prompt, provider, assertion, test, latencyMs, prov
             prompt: { label: prompt },
         });
     }
-    const outputString = coerceString(output);
     const context = {
         prompt,
         vars: test.vars || {},
@@ -186,703 +156,70 @@ async function runAssertion({ prompt, provider, assertion, test, latencyMs, prov
             return v;
         });
     }
-    // Transform test
-    test = (0, utils_1.getFinalTest)(test, assertion);
-    if (baseType === 'equals') {
-        if (typeof renderedValue === 'object') {
-            pass = node_util_1.default.isDeepStrictEqual(renderedValue, JSON.parse(outputString)) !== inverse;
-            renderedValue = JSON.stringify(renderedValue);
-        }
-        else {
-            pass = (renderedValue == outputString) !== inverse;
-        }
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output "${outputString}" to ${inverse ? 'not ' : ''}equal "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'is-json') {
-        return (0, json_1.handleIsJson)(assertion, renderedValue, valueFromScript, outputString, inverse);
-    }
-    if (baseType === 'contains-json') {
-        return (0, json_1.handleContainsJson)(assertion, renderedValue, valueFromScript, outputString, inverse);
-    }
-    if (baseType === 'is-xml' || baseType === 'contains-xml') {
-        let requiredElements;
-        if (typeof renderedValue === 'string') {
-            requiredElements = renderedValue.split(',').map((el) => el.trim());
-        }
-        else if (Array.isArray(renderedValue) && renderedValue.length > 0) {
-            requiredElements = renderedValue.map((el) => el.toString());
-        }
-        else if (typeof renderedValue === 'object' && Object.keys(renderedValue).length > 0) {
-            if ('requiredElements' in renderedValue && Array.isArray(renderedValue.requiredElements)) {
-                requiredElements = renderedValue.requiredElements.map((el) => el.toString());
-            }
-            else {
-                throw new Error('xml assertion must contain a string, array value, or no value');
-            }
-        }
-        const result = (baseType === 'is-xml' ? xml_1.validateXml : xml_1.containsXml)(outputString, requiredElements);
-        pass = result.isValid !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass ? 'Assertion passed' : result.reason,
-            assertion,
-        };
-    }
-    if (baseType === 'is-sql') {
-        return (0, sql_1.isSql)(outputString, renderedValue, inverse, assertion);
-    }
-    if (baseType === 'contains-sql') {
-        const match = outputString.match(/```(?:sql)?([^`]+)```/);
-        if (match) {
-            const sqlCode = match[1].trim();
-            return (0, sql_1.isSql)(sqlCode, renderedValue, inverse, assertion);
-        }
-        else {
-            return (0, sql_1.isSql)(outputString, renderedValue, inverse, assertion);
-        }
-    }
-    if (baseType === 'contains') {
-        (0, tiny_invariant_1.default)(renderedValue, '"contains" assertion type must have a string or number value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"contains" assertion type must have a string or number value');
-        pass = outputString.includes(String(renderedValue)) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'contains-any') {
-        (0, tiny_invariant_1.default)(renderedValue, '"contains-any" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-any" assertion type must have an array value');
-        pass = renderedValue.some((value) => outputString.includes(String(value))) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'icontains-any') {
-        (0, tiny_invariant_1.default)(renderedValue, '"icontains-any" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-any" assertion type must have an array value');
-        pass =
-            renderedValue.some((value) => outputString.toLowerCase().includes(String(value).toLowerCase())) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'contains-all') {
-        (0, tiny_invariant_1.default)(renderedValue, '"contains-all" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-all" assertion type must have an array value');
-        const missingStrings = renderedValue.filter((value) => !outputString.includes(String(value)));
-        pass = (missingStrings.length === 0) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain all of [${renderedValue.join(', ')}]. Missing: [${missingStrings.join(', ')}]`,
-            assertion,
-        };
-    }
-    if (baseType === 'icontains-all') {
-        (0, tiny_invariant_1.default)(renderedValue, '"icontains-all" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-all" assertion type must have an array value');
-        const missingStrings = renderedValue.filter((value) => !outputString.toLowerCase().includes(String(value).toLowerCase()));
-        pass = (missingStrings.length === 0) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain all of [${renderedValue.join(', ')}]. Missing: [${missingStrings.join(', ')}]`,
-            assertion,
-        };
-    }
-    if (baseType === 'regex') {
-        (0, tiny_invariant_1.default)(renderedValue, '"regex" assertion type must have a string value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"regex" assertion type must have a string value');
-        const regex = new RegExp(renderedValue);
-        pass = regex.test(outputString) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'icontains') {
-        (0, tiny_invariant_1.default)(renderedValue, '"icontains" assertion type must have a string or number value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"icontains" assertion type must have a string or number value');
-        pass = outputString.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'starts-with') {
-        (0, tiny_invariant_1.default)(renderedValue, '"starts-with" assertion type must have a string value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"starts-with" assertion type must have a string value');
-        pass = outputString.startsWith(String(renderedValue)) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'is-valid-openai-tools-call') {
-        const toolsOutput = output;
-        if (!Array.isArray(toolsOutput) ||
-            toolsOutput.length === 0 ||
-            typeof toolsOutput[0].function.name !== 'string' ||
-            typeof toolsOutput[0].function.arguments !== 'string') {
-            return {
-                pass: false,
-                score: 0,
-                reason: `OpenAI did not return a valid-looking tools response: ${JSON.stringify(toolsOutput)}`,
-                assertion,
-            };
-        }
-        try {
-            toolsOutput.forEach((toolOutput) => (0, openaiUtil_1.validateFunctionCall)(toolOutput.function, provider.config.tools?.map((tool) => tool.function), test.vars));
-            return {
-                pass: true,
-                score: 1,
-                reason: 'Assertion passed',
-                assertion,
-            };
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: err.message,
-                assertion,
-            };
-        }
-    }
-    if (baseType === 'is-valid-openai-function-call') {
-        const functionOutput = output;
-        if (typeof functionOutput !== 'object' ||
-            typeof functionOutput.name !== 'string' ||
-            typeof functionOutput.arguments !== 'string') {
-            return {
-                pass: false,
-                score: 0,
-                reason: `OpenAI did not return a valid-looking function call: ${JSON.stringify(functionOutput)}`,
-                assertion,
-            };
-        }
-        try {
-            (0, openaiUtil_1.validateFunctionCall)(functionOutput, provider.config.functions, test.vars);
-            return {
-                pass: true,
-                score: 1,
-                reason: 'Assertion passed',
-                assertion,
-            };
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: err.message,
-                assertion,
-            };
-        }
-    }
-    if (baseType === 'javascript') {
-        try {
-            const validateResult = async (result) => {
-                result = await Promise.resolve(result);
-                if (typeof result === 'boolean' || typeof result === 'number' || (0, types_1.isGradingResult)(result)) {
-                    return result;
-                }
-                else {
-                    throw new Error(`Custom function must return a boolean, number, or GradingResult object. Got type ${typeof result}: ${JSON.stringify(result)}`);
-                }
-            };
-            if (typeof assertion.value === 'function') {
-                let ret = assertion.value(outputString, context);
-                ret = await validateResult(ret);
-                if (!ret.assertion) {
-                    // Populate the assertion object if the custom function didn't return it.
-                    const functionString = assertion.value.toString();
-                    ret.assertion = {
-                        type: 'javascript',
-                        value: functionString.length > 50 ? functionString.slice(0, 50) + '...' : functionString,
-                    };
-                }
-                return ret;
-            }
-            (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'javascript assertion must have a string value');
-            /**
-             * Removes trailing newline from the rendered value.
-             * This is necessary for handling multi-line string literals in YAML
-             * that are defined on a single line in the YAML file.
-             *
-             * @example
-             * value: |
-             *   output === 'true'
-             */
-            renderedValue = renderedValue.trimEnd();
-            let result;
-            if (typeof valueFromScript === 'undefined') {
-                const functionBody = renderedValue.includes('\n')
-                    ? renderedValue
-                    : `return ${renderedValue}`;
-                const customFunction = new Function('output', 'context', functionBody);
-                result = await validateResult(customFunction(output, context));
-            }
-            else {
-                (0, tiny_invariant_1.default)(typeof valueFromScript === 'boolean' ||
-                    typeof valueFromScript === 'number' ||
-                    typeof valueFromScript === 'object', `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
-                result = await validateResult(valueFromScript);
-            }
-            if (typeof result === 'boolean') {
-                pass = result !== inverse;
-                score = pass ? 1 : 0;
-            }
-            else if (typeof result === 'number') {
-                pass = assertion.threshold ? result >= assertion.threshold : result > 0;
-                score = result;
-            }
-            else if (typeof result === 'object') {
-                return result;
-            }
-            else {
-                throw new Error('Custom function must return a boolean or number');
-            }
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: `Custom function threw error: ${err.message}
-Stack Trace: ${err.stack}
-${renderedValue}`,
-                assertion,
-            };
-        }
-        return {
-            pass,
-            score,
-            reason: pass
-                ? 'Assertion passed'
-                : `Custom function returned ${inverse ? 'true' : 'false'}
-${renderedValue}`,
-            assertion,
-        };
-    }
-    if (baseType === 'python') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'python assertion must have a string value');
-        try {
-            let result;
-            if (typeof valueFromScript === 'undefined') {
-                const isMultiline = renderedValue.includes('\n');
-                let indentStyle = '    ';
-                if (isMultiline) {
-                    // Detect the indentation style of the first indented line
-                    const match = renderedValue.match(/^(?!\s*$)\s+/m);
-                    if (match) {
-                        indentStyle = match[0];
-                    }
-                }
-                const pythonScript = `import json
-
-def main(output, context):
-${isMultiline
-                    ? renderedValue
-                        .split('\n')
-                        .map((line) => `${indentStyle}${line}`)
-                        .join('\n')
-                    : `    return ${renderedValue}`}
-`;
-                result = await (0, wrapper_1.runPythonCode)(pythonScript, 'main', [output, context]);
-            }
-            else {
-                result = valueFromScript;
-            }
-            if ((typeof result === 'boolean' && result) ||
-                (typeof result === 'string' && result.toLowerCase() === 'true')) {
-                pass = true;
-                score = 1.0;
-            }
-            else if ((typeof result === 'boolean' && !result) ||
-                (typeof result === 'string' && result.toLowerCase() === 'false')) {
-                pass = false;
-                score = 0.0;
-            }
-            else if (typeof result === 'string' && result.startsWith('{')) {
-                let parsed;
-                try {
-                    parsed = JSON.parse(result);
-                }
-                catch (err) {
-                    throw new Error(`Invalid JSON: ${err} when parsing result: ${result}`);
-                }
-                if (!(0, types_1.isGradingResult)(parsed)) {
-                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead: ${result}`);
-                }
-                return parsed;
-            }
-            else if (typeof result === 'object') {
-                if (!(0, types_1.isGradingResult)(result)) {
-                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead:\n${JSON.stringify(result, null, 2)}`);
-                }
-                const pythonGradingResult = result;
-                if (assertion.threshold && pythonGradingResult.score < assertion.threshold) {
-                    pythonGradingResult.pass = false;
-                    pythonGradingResult.reason = `Python score ${pythonGradingResult.score} is less than threshold ${assertion.threshold}`;
-                }
-                return {
-                    ...pythonGradingResult,
-                    assertion,
-                };
-            }
-            else {
-                score = Number.parseFloat(String(result));
-                pass = assertion.threshold ? score >= assertion.threshold : score > 0;
-                if (Number.isNaN(score)) {
-                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-                }
-                if (typeof assertion.threshold !== 'undefined' && score < assertion.threshold) {
-                    pass = false;
-                }
-            }
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: `Python code execution failed: ${err.message}`,
-                assertion,
-            };
-        }
-        return {
-            pass,
-            score,
-            reason: pass
-                ? 'Assertion passed'
-                : `Python code returned ${pass ? 'true' : 'false'}\n${assertion.value}`,
-            assertion,
-        };
-    }
-    if (baseType === 'similar') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || Array.isArray(renderedValue), 'Similarity assertion type must have a string or array of strings value');
-        if (Array.isArray(renderedValue)) {
-            let minScore = Infinity;
-            for (const value of renderedValue) {
-                const result = await (0, matchers_1.matchesSimilarity)(value, outputString, assertion.threshold || 0.75, inverse, test.options);
-                if (result.pass) {
-                    return {
-                        assertion,
-                        ...result,
-                    };
-                }
-                if (result.score < minScore) {
-                    minScore = result.score;
-                }
-            }
-            return {
-                assertion,
-                pass: false,
-                score: minScore,
-                reason: `None of the provided values met the similarity threshold`,
-            };
-        }
-        else {
-            return {
-                assertion,
-                ...(await (0, matchers_1.matchesSimilarity)(renderedValue, outputString, assertion.threshold || 0.75, inverse, test.options)),
-            };
-        }
-    }
-    if (baseType === 'llm-rubric') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"llm-rubric" assertion type must have a string value');
-        if (test.options?.rubricPrompt && typeof test.options.rubricPrompt === 'object') {
-            test.options.rubricPrompt = JSON.stringify(test.options.rubricPrompt);
-        }
-        // Update the assertion value. This allows the web view to display the prompt.
-        assertion.value = assertion.value || test.options?.rubricPrompt;
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesLlmRubric)(renderedValue || '', outputString, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'model-graded-factuality' || baseType === 'factuality') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'factuality assertion type must have a string value');
-        (0, tiny_invariant_1.default)(prompt, 'factuality assertion type must have a prompt');
-        if (test.options?.rubricPrompt) {
-            // Substitute vars in prompt
-            (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
-            test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
-        }
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesFactuality)(prompt, renderedValue, outputString, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'model-graded-closedqa') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'model-graded-closedqa assertion type must have a string value');
-        (0, tiny_invariant_1.default)(prompt, 'model-graded-closedqa assertion type must have a prompt');
-        if (test.options?.rubricPrompt) {
-            // Substitute vars in prompt
-            (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
-            test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
-        }
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesClosedQa)(prompt, renderedValue, outputString, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'answer-relevance') {
-        (0, tiny_invariant_1.default)(typeof output === 'string', 'answer-relevance assertion type must evaluate a string output');
-        (0, tiny_invariant_1.default)(prompt, 'answer-relevance assertion type must have a prompt');
-        const input = typeof test.vars?.query === 'string' ? test.vars.query : prompt;
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesAnswerRelevance)(input, output, assertion.threshold || 0, test.options)),
-        };
-    }
-    if (baseType === 'context-recall') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'context-recall assertion type must have a string value');
-        (0, tiny_invariant_1.default)(prompt, 'context-recall assertion type must have a prompt');
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesContextRecall)(typeof test.vars?.context === 'string' ? test.vars.context : prompt, renderedValue, assertion.threshold || 0, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'context-relevance') {
-        (0, tiny_invariant_1.default)(test.vars, 'context-relevance assertion type must have a vars object');
-        (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-relevance assertion type must have a query var');
-        (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-relevance assertion type must have a context var');
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesContextRelevance)(test.vars.query, test.vars.context, assertion.threshold || 0, test.options)),
-        };
-    }
-    if (baseType === 'context-faithfulness') {
-        (0, tiny_invariant_1.default)(test.vars, 'context-faithfulness assertion type must have a vars object');
-        (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-faithfulness assertion type must have a query var');
-        (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-faithfulness assertion type must have a context var');
-        (0, tiny_invariant_1.default)(typeof output === 'string', 'context-faithfulness assertion type must have a string output');
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesContextFaithfulness)(test.vars.query, output, test.vars.context, assertion.threshold || 0, test.options)),
-        };
-    }
-    if (baseType === 'moderation') {
-        // Some redteam techniques override the actual prompt that is used, so we need to assess that prompt for moderation.
-        const promptToModerate = providerResponse.metadata?.redteamFinalPrompt || prompt;
-        const outputString = typeof output === 'string' ? output : JSON.stringify(output);
-        (0, tiny_invariant_1.default)(promptToModerate, 'moderation assertion type must have a prompt');
-        (0, tiny_invariant_1.default)(!assertion.value ||
-            (Array.isArray(assertion.value) && typeof assertion.value[0] === 'string'), 'moderation assertion value must be a string array if set');
-        if (promptToModerate[0] === '[' || promptToModerate[0] === '{') {
-            // Try to extract the last user message from OpenAI-style prompts.
-            try {
-                const parsedPrompt = (0, shared_1.parseChatPrompt)(promptToModerate, null);
-                if (parsedPrompt && parsedPrompt.length > 0) {
-                    prompt = parsedPrompt[parsedPrompt.length - 1].content;
-                }
-            }
-            catch {
-                // Ignore error
-            }
-        }
-        const moderationResult = await (0, matchers_1.matchesModeration)({
-            userPrompt: promptToModerate,
-            assistantResponse: outputString,
-            categories: Array.isArray(assertion.value) ? assertion.value : [],
-        }, test.options);
-        pass = moderationResult.pass;
-        return {
-            pass,
-            score: moderationResult.score,
-            reason: moderationResult.reason,
-            assertion,
-        };
-    }
-    if (baseType === 'webhook') {
-        (0, tiny_invariant_1.default)(renderedValue, '"webhook" assertion type must have a URL value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
-        try {
-            const context = {
-                prompt,
-                vars: test.vars || {},
-            };
-            const response = await (0, fetch_1.fetchWithRetries)(renderedValue, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                },
-                body: JSON.stringify({ output, context }),
-            }, (0, envars_1.getEnvInt)('WEBHOOK_TIMEOUT', 5000));
-            if (!response.ok) {
-                throw new Error(`Webhook response status: ${response.status}`);
-            }
-            const jsonResponse = await response.json();
-            pass = jsonResponse.pass !== inverse;
-            score =
-                typeof jsonResponse.score === 'undefined'
-                    ? pass
-                        ? 1
-                        : 0
-                    : inverse
-                        ? 1 - jsonResponse.score
-                        : jsonResponse.score;
-            const reason = jsonResponse.reason ||
-                (pass ? 'Assertion passed' : `Webhook returned ${inverse ? 'true' : 'false'}`);
-            return {
-                pass,
-                score,
-                reason,
-                assertion,
-            };
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: `Webhook error: ${err.message}`,
-                assertion,
-            };
-        }
-    }
-    if (baseType === 'rouge-n') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"rouge" assertion type must be a string value');
-        return handleRougeScore(baseType, assertion, renderedValue, outputString, inverse);
-    }
-    if (baseType === 'bleu') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' ||
-            (Array.isArray(renderedValue) && renderedValue.every((v) => typeof v === 'string')), '"bleu" assertion type must have a string or array of strings value');
-        return (0, bleu_1.handleBleuScore)(assertion, renderedValue, outputString, inverse);
-    }
-    if (baseType === 'levenshtein') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"levenshtein" assertion type must have a string value');
-        const levDistance = (0, fastest_levenshtein_1.distance)(outputString, renderedValue);
-        pass = levDistance <= (assertion.threshold || 5);
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Levenshtein distance ${levDistance} is greater than threshold ${assertion.threshold || 5}`,
-            assertion,
-        };
-    }
-    if (baseType === 'classifier') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"classifier" assertion type must have a string value or be undefined');
-        // Assertion provider overrides test provider
-        const classificationResult = await (0, matchers_1.matchesClassification)(renderedValue, outputString, assertion.threshold ?? 1, test.options);
-        if (inverse) {
-            classificationResult.pass = !classificationResult.pass;
-            classificationResult.score = 1 - classificationResult.score;
-        }
-        return {
-            assertion,
-            ...classificationResult,
-        };
-    }
-    if (baseType === 'latency') {
-        if (!assertion.threshold) {
-            throw new Error('Latency assertion must have a threshold in milliseconds');
-        }
-        if (latencyMs === undefined) {
-            throw new Error('Latency assertion does not support cached results. Rerun the eval with --no-cache');
-        }
-        pass = latencyMs <= assertion.threshold;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Latency ${latencyMs}ms is greater than threshold ${assertion.threshold}ms`,
-            assertion,
-        };
-    }
-    if (baseType === 'perplexity') {
-        return (0, perplexity_1.handlePerplexity)(logProbs, assertion);
-    }
-    if (baseType === 'perplexity-score') {
-        return (0, perplexity_1.handlePerplexityScore)(logProbs, assertion);
-    }
-    if (baseType === 'cost') {
-        if (!assertion.threshold) {
-            throw new Error('Cost assertion must have a threshold');
-        }
-        if (typeof cost === 'undefined') {
-            throw new Error('Cost assertion does not support providers that do not return cost');
-        }
-        pass = cost <= assertion.threshold;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Cost ${cost.toPrecision(2)} is greater than threshold ${assertion.threshold}`,
-            assertion,
-        };
+    const assertionParams = {
+        assertion,
+        baseType,
+        context,
+        cost,
+        inverse,
+        latencyMs,
+        logProbs,
+        output,
+        outputString: (0, utils_1.coerceString)(output),
+        prompt,
+        provider,
+        providerResponse,
+        renderedValue,
+        test: (0, utils_1.getFinalTest)(test, assertion),
+        valueFromScript,
+    };
+    // Map assertion types to their handler functions>
+    const assertionHandlers = {
+        'answer-relevance': answerRelevance_1.handleAnswerRelevance,
+        bleu: bleu_1.handleBleuScore,
+        classifier: classifier_1.handleClassifier,
+        contains: contains_1.handleContains,
+        'contains-all': contains_1.handleContainsAll,
+        'contains-any': contains_1.handleContainsAny,
+        'contains-json': json_1.handleContainsJson,
+        'contains-sql': sql_1.handleContainsSql,
+        'contains-xml': xml_1.handleIsXml,
+        'context-faithfulness': contextFaithfulness_1.handleContextFaithfulness,
+        'context-recall': contextRecall_1.handleContextRecall,
+        'context-relevance': contextRelevance_1.handleContextRelevance,
+        cost: cost_1.handleCost,
+        equals: equals_1.handleEquals,
+        factuality: factuality_1.handleFactuality,
+        icontains: contains_1.handleIContains,
+        'icontains-all': contains_1.handleIContainsAll,
+        'icontains-any': contains_1.handleIContainsAny,
+        'is-json': json_1.handleIsJson,
+        'is-sql': sql_1.handleIsSql,
+        'is-valid-openai-function-call': openai_1.handleIsValidOpenAiFunctionCall,
+        'is-valid-openai-tools-call': openai_1.handleIsValidOpenAiToolsCall,
+        'is-xml': xml_1.handleIsXml,
+        javascript: javascript_1.handleJavascript,
+        latency: latency_1.handleLatency,
+        levenshtein: levenshtein_1.handleLevenshtein,
+        'llm-rubric': llmRubric_1.handleLlmRubric,
+        'model-graded-closedqa': modelGradedClosedQa_1.handleModelGradedClosedQa,
+        'model-graded-factuality': factuality_1.handleFactuality,
+        moderation: moderation_1.handleModeration,
+        perplexity: perplexity_1.handlePerplexity,
+        'perplexity-score': perplexity_1.handlePerplexityScore,
+        python: python_1.handlePython,
+        regex: regex_1.handleRegex,
+        'rouge-n': rouge_1.handleRougeScore,
+        similar: similar_1.handleSimilar,
+        'starts-with': startsWith_1.handleStartsWith,
+        webhook: webhook_1.handleWebhook,
+    };
+    const handler = assertionHandlers[baseType];
+    if (handler) {
+        return handler(assertionParams);
     }
     if (baseType.startsWith('promptfoo:redteam:')) {
-        const grader = (0, graders_1.getGraderById)(baseType);
-        (0, tiny_invariant_1.default)(grader, `Unknown promptfoo grader: ${baseType}`);
-        (0, tiny_invariant_1.default)(prompt, `Promptfoo grader ${baseType} must have a prompt`);
-        const { grade, rubric, suggestions } = await grader.getResult(prompt, outputString, test, provider, renderedValue);
-        return {
-            assertion: {
-                ...assertion,
-                value: rubric,
-            },
-            ...grade,
-            suggestions,
-            metadata: {
-                // Pass through all test metadata for redteam
-                ...test.metadata,
-                ...grade.metadata,
-            },
-        };
+        return (0, redteam_1.handleRedteam)(assertionParams);
     }
     throw new Error('Unknown assertion type: ' + assertion.type);
 }