promptfoo 0.95.0 → 0.96.1

package/dist/src/assertions/index.js

@@ -1,231 +1,79 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MODEL_GRADED_ASSERTION_TYPES = void 0;
-exports.createAjv = createAjv;
-exports.validateXml = validateXml;
-exports.containsXml = containsXml;
-exports.isSql = isSql;
 exports.runAssertion = runAssertion;
 exports.runAssertions = runAssertions;
 exports.runCompareAssertion = runCompareAssertion;
 exports.readAssertions = readAssertions;
-const ajv_1 = __importDefault(require("ajv"));
-const ajv_formats_1 = __importDefault(require("ajv-formats"));
 const async_1 = __importDefault(require("async"));
-const fast_xml_parser_1 = require("fast-xml-parser");
-const fastest_levenshtein_1 = require("fastest-levenshtein");
 const fs_1 = __importDefault(require("fs"));
-const rouge = __importStar(require("js-rouge"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
-const node_util_1 = __importDefault(require("node:util"));
 const path_1 = __importDefault(require("path"));
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const cliState_1 = __importDefault(require("../cliState"));
 const envars_1 = require("../envars");
 const esm_1 = require("../esm");
-const fetch_1 = require("../fetch");
 const logger_1 = __importDefault(require("../logger"));
 const matchers_1 = require("../matchers");
-const openaiUtil_1 = require("../providers/openaiUtil");
 const packageParser_1 = require("../providers/packageParser");
-const shared_1 = require("../providers/shared");
 const pythonUtils_1 = require("../python/pythonUtils");
-const wrapper_1 = require("../python/wrapper");
-const graders_1 = require("../redteam/graders");
 const telemetry_1 = __importDefault(require("../telemetry"));
-const types_1 = require("../types");
 const file_1 = require("../util/file");
-const json_1 = require("../util/json");
 const templates_1 = require("../util/templates");
 const transform_1 = require("../util/transform");
 const AssertionsResult_1 = require("./AssertionsResult");
+const answerRelevance_1 = require("./answerRelevance");
+const bleu_1 = require("./bleu");
+const classifier_1 = require("./classifier");
+const contains_1 = require("./contains");
+const contextFaithfulness_1 = require("./contextFaithfulness");
+const contextRecall_1 = require("./contextRecall");
+const contextRelevance_1 = require("./contextRelevance");
+const cost_1 = require("./cost");
+const equals_1 = require("./equals");
+const factuality_1 = require("./factuality");
+const javascript_1 = require("./javascript");
+const json_1 = require("./json");
+const latency_1 = require("./latency");
+const levenshtein_1 = require("./levenshtein");
+const llmRubric_1 = require("./llmRubric");
+const modelGradedClosedQa_1 = require("./modelGradedClosedQa");
+const moderation_1 = require("./moderation");
+const openai_1 = require("./openai");
+const perplexity_1 = require("./perplexity");
+const python_1 = require("./python");
+const redteam_1 = require("./redteam");
+const regex_1 = require("./regex");
+const rouge_1 = require("./rouge");
+const similar_1 = require("./similar");
+const sql_1 = require("./sql");
+const startsWith_1 = require("./startsWith");
 const utils_1 = require("./utils");
+const webhook_1 = require("./webhook");
+const xml_1 = require("./xml");
 const ASSERTIONS_MAX_CONCURRENCY = (0, envars_1.getEnvInt)('PROMPTFOO_ASSERTIONS_MAX_CONCURRENCY', 3);
 exports.MODEL_GRADED_ASSERTION_TYPES = new Set([
     'answer-relevance',
     'context-faithfulness',
     'context-recall',
     'context-relevance',
+    'factuality',
     'llm-rubric',
     'model-graded-closedqa',
-    'factuality',
     'model-graded-factuality',
 ]);
-function createAjv() {
-    const ajvOptions = {
-        strictSchema: !(0, envars_1.getEnvBool)('PROMPTFOO_DISABLE_AJV_STRICT_MODE'),
-    };
-    const ajv = new ajv_1.default(ajvOptions);
-    (0, ajv_formats_1.default)(ajv);
-    return ajv;
-}
-const ajv = createAjv();
 const nunjucks = (0, templates_1.getNunjucksEngine)();
-function coerceString(value) {
-    if (typeof value === 'string') {
-        return value;
-    }
-    return JSON.stringify(value);
-}
-function handleRougeScore(baseType, assertion, expected, output, inverted) {
-    const fnName = baseType[baseType.length - 1];
-    const rougeMethod = rouge[fnName];
-    const score = rougeMethod(output, expected, {});
-    const pass = score >= (assertion.threshold || 0.75) != inverted;
-    return {
-        pass,
-        score: inverted ? 1 - score : score,
-        reason: pass
-            ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${assertion.threshold || 0.75}`
-            : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${assertion.threshold || 0.75}`,
-        assertion,
-    };
-}
-function validateXml(xmlString, requiredElements) {
-    if (!xmlString.startsWith('<')) {
-        return { isValid: false, reason: 'XML is missing opening tag' };
-    }
-    const parser = new fast_xml_parser_1.XMLParser({
-        allowBooleanAttributes: true,
-        ignoreAttributes: false,
-        parseAttributeValue: true,
-        parseTagValue: true,
-    });
-    try {
-        const parsedXml = parser.parse(xmlString);
-        if (requiredElements && requiredElements.length > 0) {
-            const missingElements = requiredElements.filter((element) => {
-                const path = element.split('.');
-                let current = parsedXml;
-                for (const key of path) {
-                    if (current[key] === undefined) {
-                        return true;
-                    }
-                    current = current[key];
-                }
-                return false;
-            });
-            if (missingElements.length > 0) {
-                return {
-                    isValid: false,
-                    reason: `XML is missing required elements: ${missingElements.join(', ')}`,
-                };
-            }
-        }
-        return { isValid: true, reason: 'XML is valid and contains all required elements' };
-    }
-    catch (err) {
-        return { isValid: false, reason: `XML parsing failed: ${err.message}` };
-    }
-}
-function containsXml(outputString, requiredElements) {
-    const xmlRegex = /<\?xml.*?>[\s\S]*<\/[^>]+>|\S*<[^>]+>[\s\S]*<\/[^>]+>/;
-    const xmlMatches = outputString.match(xmlRegex);
-    if (!xmlMatches) {
-        return { isValid: false, reason: 'No XML content found in the output' };
-    }
-    for (const xmlMatch of xmlMatches) {
-        const { isValid, reason } = validateXml(xmlMatch, requiredElements);
-        if (isValid) {
-            return { isValid: true, reason };
-        }
-    }
-    return { isValid: false, reason: 'No valid XML content found matching the requirements' };
-}
-async function isSql(outputString, renderedValue, inverse, assertion) {
-    let pass = false;
-    let databaseType = 'MySQL';
-    let whiteTableList;
-    let whiteColumnList;
-    if (renderedValue && typeof renderedValue === 'object') {
-        const value = renderedValue;
-        databaseType = value.databaseType || 'MySQL';
-        whiteTableList = value.allowedTables;
-        whiteColumnList = value.allowedColumns;
-    }
-    if (renderedValue && typeof renderedValue !== 'object') {
-        throw new Error('is-sql assertion must have a object value.');
-    }
-    const { Parser: SqlParser } = await Promise.resolve().then(() => __importStar(require('node-sql-parser'))).catch(() => {
-        throw new Error('node-sql-parser is not installed. Please install it first');
-    });
-    const sqlParser = new SqlParser();
-    const opt = { database: databaseType };
-    const failureReasons = [];
-    try {
-        sqlParser.astify(outputString, opt);
-        pass = !inverse;
-    }
-    catch {
-        pass = inverse;
-        failureReasons.push(`SQL statement does not conform to the provided ${databaseType} database syntax.`);
-    }
-    if (whiteTableList) {
-        opt.type = 'table';
-        try {
-            sqlParser.whiteListCheck(outputString, whiteTableList, opt);
-        }
-        catch (err) {
-            pass = inverse;
-            const error = err;
-            failureReasons.push(`SQL validation failed: ${error.message}.`);
-        }
-    }
-    if (whiteColumnList) {
-        opt.type = 'column';
-        try {
-            sqlParser.whiteListCheck(outputString, whiteColumnList, opt);
-        }
-        catch (err) {
-            pass = inverse;
-            const error = err;
-            failureReasons.push(`SQL validation failed: ${error.message}.`);
-        }
-    }
-    if (inverse && pass === false && failureReasons.length === 0) {
-        failureReasons.push('The output SQL statement is valid');
-    }
-    return {
-        pass,
-        score: pass ? 1 : 0,
-        reason: pass ? 'Assertion passed' : failureReasons.join(' '),
-        assertion,
-    };
-}
 async function runAssertion({ prompt, provider, assertion, test, latencyMs, providerResponse, }) {
     const { cost, logProbs, output: originalOutput } = providerResponse;
     let output = originalOutput;
-    let pass = false;
-    let score = 0.0;
     (0, tiny_invariant_1.default)(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
     const inverse = assertion.type.startsWith('not-');
-    const baseType = inverse ? assertion.type.slice(4) : assertion.type;
+    const baseType = inverse
+        ? assertion.type.slice(4)
+        : assertion.type;
     telemetry_1.default.record('assertion_used', {
         type: baseType,
     });
@@ -235,7 +83,6 @@ async function runAssertion({ prompt, provider, assertion, test, latencyMs, prov
             prompt: { label: prompt },
         });
     }
-    const outputString = coerceString(output);
     const context = {
         prompt,
         vars: test.vars || {},
@@ -309,810 +156,70 @@ async function runAssertion({ prompt, provider, assertion, test, latencyMs, prov
             return v;
         });
     }
-    // Transform test
-    test = (0, utils_1.getFinalTest)(test, assertion);
-    if (baseType === 'equals') {
-        if (typeof renderedValue === 'object') {
-            pass = node_util_1.default.isDeepStrictEqual(renderedValue, JSON.parse(outputString)) !== inverse;
-            renderedValue = JSON.stringify(renderedValue);
-        }
-        else {
-            pass = (renderedValue == outputString) !== inverse;
-        }
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output "${outputString}" to ${inverse ? 'not ' : ''}equal "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'is-json') {
-        let parsedJson;
-        try {
-            parsedJson = JSON.parse(outputString);
-            pass = !inverse;
-        }
-        catch {
-            pass = inverse;
-        }
-        if (pass && renderedValue) {
-            let validate;
-            if (typeof renderedValue === 'string') {
-                if (renderedValue.startsWith('file://')) {
-                    // Reference the JSON schema from external file
-                    const schema = valueFromScript;
-                    (0, tiny_invariant_1.default)(schema, 'is-json references a file that does not export a JSON schema');
-                    validate = ajv.compile(schema);
-                }
-                else {
-                    const scheme = js_yaml_1.default.load(renderedValue);
-                    validate = ajv.compile(scheme);
-                }
-            }
-            else if (typeof renderedValue === 'object') {
-                // Value is JSON schema
-                validate = ajv.compile(renderedValue);
-            }
-            else {
-                throw new Error('is-json assertion must have a string or object value');
-            }
-            pass = validate(parsedJson);
-            if (!pass) {
-                return {
-                    pass,
-                    score: 0,
-                    reason: `JSON does not conform to the provided schema. Errors: ${ajv.errorsText(validate.errors)}`,
-                    assertion,
-                };
-            }
-        }
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass ? 'Assertion passed' : 'Expected output to be valid JSON',
-            assertion,
-        };
-    }
-    if (baseType === 'is-xml' || baseType === 'contains-xml') {
-        let requiredElements;
-        if (typeof renderedValue === 'string') {
-            requiredElements = renderedValue.split(',').map((el) => el.trim());
-        }
-        else if (Array.isArray(renderedValue) && renderedValue.length > 0) {
-            requiredElements = renderedValue.map((el) => el.toString());
-        }
-        else if (typeof renderedValue === 'object' && Object.keys(renderedValue).length > 0) {
-            if ('requiredElements' in renderedValue && Array.isArray(renderedValue.requiredElements)) {
-                requiredElements = renderedValue.requiredElements.map((el) => el.toString());
-            }
-            else {
-                throw new Error('xml assertion must contain a string, array value, or no value');
-            }
-        }
-        const result = (baseType === 'is-xml' ? validateXml : containsXml)(outputString, requiredElements);
-        pass = result.isValid !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass ? 'Assertion passed' : result.reason,
-            assertion,
-        };
-    }
-    if (baseType === 'is-sql') {
-        return isSql(outputString, renderedValue, inverse, assertion);
-    }
-    if (baseType === 'contains-sql') {
-        const match = outputString.match(/```(?:sql)?([^`]+)```/);
-        if (match) {
-            const sqlCode = match[1].trim();
-            return isSql(sqlCode, renderedValue, inverse, assertion);
-        }
-        else {
-            return isSql(outputString, renderedValue, inverse, assertion);
-        }
-    }
-    if (baseType === 'contains') {
-        (0, tiny_invariant_1.default)(renderedValue, '"contains" assertion type must have a string or number value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"contains" assertion type must have a string or number value');
-        pass = outputString.includes(String(renderedValue)) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'contains-any') {
-        (0, tiny_invariant_1.default)(renderedValue, '"contains-any" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-any" assertion type must have an array value');
-        pass = renderedValue.some((value) => outputString.includes(String(value))) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'icontains-any') {
-        (0, tiny_invariant_1.default)(renderedValue, '"icontains-any" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-any" assertion type must have an array value');
-        pass =
-            renderedValue.some((value) => outputString.toLowerCase().includes(String(value).toLowerCase())) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'contains-all') {
-        (0, tiny_invariant_1.default)(renderedValue, '"contains-all" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-all" assertion type must have an array value');
-        const missingStrings = renderedValue.filter((value) => !outputString.includes(String(value)));
-        pass = (missingStrings.length === 0) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain all of [${renderedValue.join(', ')}]. Missing: [${missingStrings.join(', ')}]`,
-            assertion,
-        };
-    }
-    if (baseType === 'icontains-all') {
-        (0, tiny_invariant_1.default)(renderedValue, '"icontains-all" assertion type must have a value');
-        if (typeof renderedValue === 'string') {
-            renderedValue = renderedValue.split(',').map((v) => v.trim());
-        }
-        (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-all" assertion type must have an array value');
-        const missingStrings = renderedValue.filter((value) => !outputString.toLowerCase().includes(String(value).toLowerCase()));
-        pass = (missingStrings.length === 0) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain all of [${renderedValue.join(', ')}]. Missing: [${missingStrings.join(', ')}]`,
-            assertion,
-        };
-    }
-    if (baseType === 'regex') {
-        (0, tiny_invariant_1.default)(renderedValue, '"regex" assertion type must have a string value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"regex" assertion type must have a string value');
-        const regex = new RegExp(renderedValue);
-        pass = regex.test(outputString) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'icontains') {
-        (0, tiny_invariant_1.default)(renderedValue, '"icontains" assertion type must have a string or number value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"icontains" assertion type must have a string or number value');
-        pass = outputString.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'starts-with') {
-        (0, tiny_invariant_1.default)(renderedValue, '"starts-with" assertion type must have a string value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"starts-with" assertion type must have a string value');
-        pass = outputString.startsWith(String(renderedValue)) !== inverse;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
-            assertion,
-        };
-    }
-    if (baseType === 'contains-json') {
-        let errorMessage = 'Expected output to contain valid JSON';
-        const jsonObjects = (0, json_1.extractJsonObjects)(outputString);
-        pass = inverse ? jsonObjects.length === 0 : jsonObjects.length > 0;
-        for (const jsonObject of jsonObjects) {
-            if (renderedValue) {
-                let validate;
-                if (typeof renderedValue === 'string') {
-                    if (renderedValue.startsWith('file://')) {
-                        // Reference the JSON schema from external file
-                        const schema = valueFromScript;
-                        (0, tiny_invariant_1.default)(schema, 'contains-json references a file that does not export a JSON schema');
-                        validate = ajv.compile(schema);
-                    }
-                    else {
-                        const scheme = js_yaml_1.default.load(renderedValue);
-                        validate = ajv.compile(scheme);
-                    }
-                }
-                else if (typeof renderedValue === 'object') {
-                    // Value is JSON schema
-                    validate = ajv.compile(renderedValue);
-                }
-                else {
-                    throw new Error('contains-json assertion must have a string or object value');
-                }
-                pass = validate(jsonObject);
-                if (pass) {
-                    break;
-                }
-                else {
-                    errorMessage = `JSON does not conform to the provided schema. Errors: ${ajv.errorsText(validate.errors)}`;
-                }
-            }
-        }
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass ? 'Assertion passed' : errorMessage,
-            assertion,
-        };
-    }
-    if (baseType === 'is-valid-openai-tools-call') {
-        const toolsOutput = output;
-        if (!Array.isArray(toolsOutput) ||
-            toolsOutput.length === 0 ||
-            typeof toolsOutput[0].function.name !== 'string' ||
-            typeof toolsOutput[0].function.arguments !== 'string') {
-            return {
-                pass: false,
-                score: 0,
-                reason: `OpenAI did not return a valid-looking tools response: ${JSON.stringify(toolsOutput)}`,
-                assertion,
-            };
-        }
-        try {
-            toolsOutput.forEach((toolOutput) => (0, openaiUtil_1.validateFunctionCall)(toolOutput.function, provider.config.tools?.map((tool) => tool.function), test.vars));
-            return {
-                pass: true,
-                score: 1,
-                reason: 'Assertion passed',
-                assertion,
-            };
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: err.message,
-                assertion,
-            };
-        }
-    }
-    if (baseType === 'is-valid-openai-function-call') {
-        const functionOutput = output;
-        if (typeof functionOutput !== 'object' ||
-            typeof functionOutput.name !== 'string' ||
-            typeof functionOutput.arguments !== 'string') {
-            return {
-                pass: false,
-                score: 0,
-                reason: `OpenAI did not return a valid-looking function call: ${JSON.stringify(functionOutput)}`,
-                assertion,
-            };
-        }
-        try {
-            (0, openaiUtil_1.validateFunctionCall)(functionOutput, provider.config.functions, test.vars);
-            return {
-                pass: true,
-                score: 1,
-                reason: 'Assertion passed',
-                assertion,
-            };
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: err.message,
-                assertion,
-            };
-        }
-    }
-    if (baseType === 'javascript') {
-        try {
-            const validateResult = async (result) => {
-                result = await Promise.resolve(result);
-                if (typeof result === 'boolean' || typeof result === 'number' || (0, types_1.isGradingResult)(result)) {
-                    return result;
-                }
-                else {
-                    throw new Error(`Custom function must return a boolean, number, or GradingResult object. Got type ${typeof result}: ${JSON.stringify(result)}`);
-                }
-            };
-            if (typeof assertion.value === 'function') {
-                let ret = assertion.value(outputString, context);
-                ret = await validateResult(ret);
-                if (!ret.assertion) {
-                    // Populate the assertion object if the custom function didn't return it.
-                    const functionString = assertion.value.toString();
-                    ret.assertion = {
-                        type: 'javascript',
-                        value: functionString.length > 50 ? functionString.slice(0, 50) + '...' : functionString,
-                    };
-                }
-                return ret;
-            }
-            (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'javascript assertion must have a string value');
-            /**
-             * Removes trailing newline from the rendered value.
-             * This is necessary for handling multi-line string literals in YAML
-             * that are defined on a single line in the YAML file.
-             *
-             * @example
-             * value: |
-             *   output === 'true'
-             */
-            renderedValue = renderedValue.trimEnd();
-            let result;
-            if (typeof valueFromScript === 'undefined') {
-                const functionBody = renderedValue.includes('\n')
-                    ? renderedValue
-                    : `return ${renderedValue}`;
-                const customFunction = new Function('output', 'context', functionBody);
-                result = await validateResult(customFunction(output, context));
-            }
-            else {
-                (0, tiny_invariant_1.default)(typeof valueFromScript === 'boolean' ||
-                    typeof valueFromScript === 'number' ||
-                    typeof valueFromScript === 'object', `Javascript assertion script must return a boolean, number, or object (${assertion.value})`);
-                result = await validateResult(valueFromScript);
-            }
-            if (typeof result === 'boolean') {
-                pass = result !== inverse;
-                score = pass ? 1 : 0;
-            }
-            else if (typeof result === 'number') {
-                pass = assertion.threshold ? result >= assertion.threshold : result > 0;
-                score = result;
-            }
-            else if (typeof result === 'object') {
-                return result;
-            }
-            else {
-                throw new Error('Custom function must return a boolean or number');
-            }
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: `Custom function threw error: ${err.message}
-Stack Trace: ${err.stack}
-${renderedValue}`,
-                assertion,
-            };
-        }
-        return {
-            pass,
-            score,
-            reason: pass
-                ? 'Assertion passed'
-                : `Custom function returned ${inverse ? 'true' : 'false'}
-${renderedValue}`,
-            assertion,
-        };
-    }
-    if (baseType === 'python') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'python assertion must have a string value');
-        try {
-            let result;
-            if (typeof valueFromScript === 'undefined') {
-                const isMultiline = renderedValue.includes('\n');
-                let indentStyle = '    ';
-                if (isMultiline) {
-                    // Detect the indentation style of the first indented line
-                    const match = renderedValue.match(/^(?!\s*$)\s+/m);
-                    if (match) {
-                        indentStyle = match[0];
-                    }
-                }
-                const pythonScript = `import json
-
-def main(output, context):
-${isMultiline
-                    ? renderedValue
-                        .split('\n')
-                        .map((line) => `${indentStyle}${line}`)
-                        .join('\n')
-                    : `    return ${renderedValue}`}
-`;
-                result = await (0, wrapper_1.runPythonCode)(pythonScript, 'main', [output, context]);
-            }
-            else {
-                result = valueFromScript;
-            }
-            if ((typeof result === 'boolean' && result) ||
-                (typeof result === 'string' && result.toLowerCase() === 'true')) {
-                pass = true;
-                score = 1.0;
-            }
-            else if ((typeof result === 'boolean' && !result) ||
-                (typeof result === 'string' && result.toLowerCase() === 'false')) {
-                pass = false;
-                score = 0.0;
-            }
-            else if (typeof result === 'string' && result.startsWith('{')) {
-                let parsed;
-                try {
-                    parsed = JSON.parse(result);
-                }
-                catch (err) {
-                    throw new Error(`Invalid JSON: ${err} when parsing result: ${result}`);
-                }
-                if (!(0, types_1.isGradingResult)(parsed)) {
-                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead: ${result}`);
-                }
-                return parsed;
-            }
-            else if (typeof result === 'object') {
-                if (!(0, types_1.isGradingResult)(result)) {
-                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead:\n${JSON.stringify(result, null, 2)}`);
-                }
-                const pythonGradingResult = result;
-                if (assertion.threshold && pythonGradingResult.score < assertion.threshold) {
-                    pythonGradingResult.pass = false;
-                    pythonGradingResult.reason = `Python score ${pythonGradingResult.score} is less than threshold ${assertion.threshold}`;
-                }
-                return {
-                    ...pythonGradingResult,
-                    assertion,
-                };
-            }
-            else {
-                score = Number.parseFloat(String(result));
-                pass = assertion.threshold ? score >= assertion.threshold : score > 0;
-                if (Number.isNaN(score)) {
-                    throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
-                }
-                if (typeof assertion.threshold !== 'undefined' && score < assertion.threshold) {
-                    pass = false;
-                }
-            }
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: `Python code execution failed: ${err.message}`,
-                assertion,
-            };
-        }
-        return {
-            pass,
-            score,
-            reason: pass
-                ? 'Assertion passed'
-                : `Python code returned ${pass ? 'true' : 'false'}\n${assertion.value}`,
-            assertion,
-        };
-    }
-    if (baseType === 'similar') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || Array.isArray(renderedValue), 'Similarity assertion type must have a string or array of strings value');
-        if (Array.isArray(renderedValue)) {
-            let minScore = Infinity;
-            for (const value of renderedValue) {
-                const result = await (0, matchers_1.matchesSimilarity)(value, outputString, assertion.threshold || 0.75, inverse, test.options);
-                if (result.pass) {
-                    return {
-                        assertion,
-                        ...result,
-                    };
-                }
-                if (result.score < minScore) {
-                    minScore = result.score;
-                }
-            }
-            return {
-                assertion,
-                pass: false,
-                score: minScore,
-                reason: `None of the provided values met the similarity threshold`,
-            };
-        }
-        else {
-            return {
-                assertion,
-                ...(await (0, matchers_1.matchesSimilarity)(renderedValue, outputString, assertion.threshold || 0.75, inverse, test.options)),
-            };
-        }
-    }
-    if (baseType === 'llm-rubric') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"llm-rubric" assertion type must have a string value');
-        if (test.options?.rubricPrompt && typeof test.options.rubricPrompt === 'object') {
-            test.options.rubricPrompt = JSON.stringify(test.options.rubricPrompt);
-        }
-        // Update the assertion value. This allows the web view to display the prompt.
-        assertion.value = assertion.value || test.options?.rubricPrompt;
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesLlmRubric)(renderedValue || '', outputString, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'model-graded-factuality' || baseType === 'factuality') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'factuality assertion type must have a string value');
-        (0, tiny_invariant_1.default)(prompt, 'factuality assertion type must have a prompt');
-        if (test.options?.rubricPrompt) {
-            // Substitute vars in prompt
-            (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
-            test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
-        }
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesFactuality)(prompt, renderedValue, outputString, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'model-graded-closedqa') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'model-graded-closedqa assertion type must have a string value');
-        (0, tiny_invariant_1.default)(prompt, 'model-graded-closedqa assertion type must have a prompt');
-        if (test.options?.rubricPrompt) {
-            // Substitute vars in prompt
-            (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
-            test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
-        }
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesClosedQa)(prompt, renderedValue, outputString, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'answer-relevance') {
-        (0, tiny_invariant_1.default)(typeof output === 'string', 'answer-relevance assertion type must evaluate a string output');
-        (0, tiny_invariant_1.default)(prompt, 'answer-relevance assertion type must have a prompt');
-        const input = typeof test.vars?.query === 'string' ? test.vars.query : prompt;
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesAnswerRelevance)(input, output, assertion.threshold || 0, test.options)),
-        };
-    }
-    if (baseType === 'context-recall') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'context-recall assertion type must have a string value');
-        (0, tiny_invariant_1.default)(prompt, 'context-recall assertion type must have a prompt');
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesContextRecall)(typeof test.vars?.context === 'string' ? test.vars.context : prompt, renderedValue, assertion.threshold || 0, test.options, test.vars)),
-        };
-    }
-    if (baseType === 'context-relevance') {
-        (0, tiny_invariant_1.default)(test.vars, 'context-relevance assertion type must have a vars object');
-        (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-relevance assertion type must have a query var');
-        (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-relevance assertion type must have a context var');
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesContextRelevance)(test.vars.query, test.vars.context, assertion.threshold || 0, test.options)),
-        };
-    }
-    if (baseType === 'context-faithfulness') {
-        (0, tiny_invariant_1.default)(test.vars, 'context-faithfulness assertion type must have a vars object');
-        (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-faithfulness assertion type must have a query var');
-        (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-faithfulness assertion type must have a context var');
-        (0, tiny_invariant_1.default)(typeof output === 'string', 'context-faithfulness assertion type must have a string output');
-        return {
-            assertion,
-            ...(await (0, matchers_1.matchesContextFaithfulness)(test.vars.query, output, test.vars.context, assertion.threshold || 0, test.options)),
-        };
-    }
-    if (baseType === 'moderation') {
-        // Some redteam techniques override the actual prompt that is used, so we need to assess that prompt for moderation.
-        const promptToModerate = providerResponse.metadata?.redteamFinalPrompt || prompt;
-        const outputString = typeof output === 'string' ? output : JSON.stringify(output);
-        (0, tiny_invariant_1.default)(promptToModerate, 'moderation assertion type must have a prompt');
-        (0, tiny_invariant_1.default)(!assertion.value ||
-            (Array.isArray(assertion.value) && typeof assertion.value[0] === 'string'), 'moderation assertion value must be a string array if set');
-        if (promptToModerate[0] === '[' || promptToModerate[0] === '{') {
-            // Try to extract the last user message from OpenAI-style prompts.
-            try {
-                const parsedPrompt = (0, shared_1.parseChatPrompt)(promptToModerate, null);
-                if (parsedPrompt && parsedPrompt.length > 0) {
-                    prompt = parsedPrompt[parsedPrompt.length - 1].content;
-                }
-            }
-            catch {
-                // Ignore error
-            }
-        }
-        const moderationResult = await (0, matchers_1.matchesModeration)({
-            userPrompt: promptToModerate,
-            assistantResponse: outputString,
-            categories: Array.isArray(assertion.value) ? assertion.value : [],
-        }, test.options);
-        pass = moderationResult.pass;
-        return {
-            pass,
-            score: moderationResult.score,
-            reason: moderationResult.reason,
-            assertion,
-        };
-    }
-    if (baseType === 'webhook') {
-        (0, tiny_invariant_1.default)(renderedValue, '"webhook" assertion type must have a URL value');
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
-        try {
-            const context = {
-                prompt,
-                vars: test.vars || {},
-            };
-            const response = await (0, fetch_1.fetchWithRetries)(renderedValue, {
-                method: 'POST',
-                headers: {
-                    'Content-Type': 'application/json',
-                },
-                body: JSON.stringify({ output, context }),
-            }, (0, envars_1.getEnvInt)('WEBHOOK_TIMEOUT', 5000));
-            if (!response.ok) {
-                throw new Error(`Webhook response status: ${response.status}`);
-            }
-            const jsonResponse = await response.json();
-            pass = jsonResponse.pass !== inverse;
-            score =
-                typeof jsonResponse.score === 'undefined'
-                    ? pass
-                        ? 1
-                        : 0
-                    : inverse
-                        ? 1 - jsonResponse.score
-                        : jsonResponse.score;
-            const reason = jsonResponse.reason ||
-                (pass ? 'Assertion passed' : `Webhook returned ${inverse ? 'true' : 'false'}`);
-            return {
-                pass,
-                score,
-                reason,
-                assertion,
-            };
-        }
-        catch (err) {
-            return {
-                pass: false,
-                score: 0,
-                reason: `Webhook error: ${err.message}`,
-                assertion,
-            };
-        }
-    }
-    if (baseType === 'rouge-n') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"rouge" assertion type must be a string value');
-        return handleRougeScore(baseType, assertion, renderedValue, outputString, inverse);
-    }
-    if (baseType === 'levenshtein') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"levenshtein" assertion type must have a string value');
-        const levDistance = (0, fastest_levenshtein_1.distance)(outputString, renderedValue);
-        pass = levDistance <= (assertion.threshold || 5);
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Levenshtein distance ${levDistance} is greater than threshold ${assertion.threshold || 5}`,
-            assertion,
-        };
-    }
-    if (baseType === 'classifier') {
-        (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'undefined', '"classifier" assertion type must have a string value or be undefined');
-        // Assertion provider overrides test provider
-        const classificationResult = await (0, matchers_1.matchesClassification)(renderedValue, outputString, assertion.threshold ?? 1, test.options);
-        if (inverse) {
-            classificationResult.pass = !classificationResult.pass;
-            classificationResult.score = 1 - classificationResult.score;
-        }
-        return {
-            assertion,
-            ...classificationResult,
-        };
-    }
-    if (baseType === 'latency') {
-        if (!assertion.threshold) {
-            throw new Error('Latency assertion must have a threshold in milliseconds');
-        }
-        if (latencyMs === undefined) {
-            throw new Error('Latency assertion does not support cached results. Rerun the eval with --no-cache');
-        }
-        pass = latencyMs <= assertion.threshold;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Latency ${latencyMs}ms is greater than threshold ${assertion.threshold}ms`,
-            assertion,
-        };
-    }
-    if (baseType === 'perplexity') {
-        if (!logProbs || logProbs.length === 0) {
-            throw new Error('Perplexity assertion does not support providers that do not return logProbs');
-        }
-        const sumLogProbs = logProbs.reduce((acc, logProb) => acc + logProb, 0);
-        const avgLogProb = sumLogProbs / logProbs.length;
-        const perplexity = Math.exp(-avgLogProb);
-        pass = assertion.threshold ? perplexity <= assertion.threshold : true;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Perplexity ${perplexity.toFixed(2)} is greater than threshold ${assertion.threshold}`,
-            assertion,
-        };
-    }
-    if (baseType === 'perplexity-score') {
-        if (!logProbs || logProbs.length === 0) {
-            throw new Error('perplexity-score assertion does not support providers that do not return logProbs');
-        }
-        const sumLogProbs = logProbs.reduce((acc, logProb) => acc + logProb, 0);
-        const avgLogProb = sumLogProbs / logProbs.length;
-        const perplexity = Math.exp(-avgLogProb);
-        const perplexityNorm = 1 / (1 + perplexity);
-        pass = assertion.threshold ? perplexityNorm >= assertion.threshold : true;
-        return {
-            pass,
-            score: perplexityNorm,
-            reason: pass
-                ? 'Assertion passed'
-                : `Perplexity score ${perplexityNorm.toFixed(2)} is less than threshold ${assertion.threshold}`,
-            assertion,
-        };
-    }
-    if (baseType === 'cost') {
-        if (!assertion.threshold) {
-            throw new Error('Cost assertion must have a threshold');
-        }
-        if (typeof cost === 'undefined') {
-            throw new Error('Cost assertion does not support providers that do not return cost');
-        }
-        pass = cost <= assertion.threshold;
-        return {
-            pass,
-            score: pass ? 1 : 0,
-            reason: pass
-                ? 'Assertion passed'
-                : `Cost ${cost.toPrecision(2)} is greater than threshold ${assertion.threshold}`,
-            assertion,
-        };
+    const assertionParams = {
+        assertion,
+        baseType,
+        context,
+        cost,
+        inverse,
+        latencyMs,
+        logProbs,
+        output,
+        outputString: (0, utils_1.coerceString)(output),
+        prompt,
+        provider,
+        providerResponse,
+        renderedValue,
+        test: (0, utils_1.getFinalTest)(test, assertion),
+        valueFromScript,
+    };
+    // Map assertion types to their handler functions>
+    const assertionHandlers = {
+        'answer-relevance': answerRelevance_1.handleAnswerRelevance,
+        bleu: bleu_1.handleBleuScore,
+        classifier: classifier_1.handleClassifier,
+        contains: contains_1.handleContains,
+        'contains-all': contains_1.handleContainsAll,
+        'contains-any': contains_1.handleContainsAny,
+        'contains-json': json_1.handleContainsJson,
+        'contains-sql': sql_1.handleContainsSql,
+        'contains-xml': xml_1.handleIsXml,
+        'context-faithfulness': contextFaithfulness_1.handleContextFaithfulness,
+        'context-recall': contextRecall_1.handleContextRecall,
+        'context-relevance': contextRelevance_1.handleContextRelevance,
+        cost: cost_1.handleCost,
+        equals: equals_1.handleEquals,
+        factuality: factuality_1.handleFactuality,
+        icontains: contains_1.handleIContains,
+        'icontains-all': contains_1.handleIContainsAll,
+        'icontains-any': contains_1.handleIContainsAny,
+        'is-json': json_1.handleIsJson,
+        'is-sql': sql_1.handleIsSql,
+        'is-valid-openai-function-call': openai_1.handleIsValidOpenAiFunctionCall,
+        'is-valid-openai-tools-call': openai_1.handleIsValidOpenAiToolsCall,
+        'is-xml': xml_1.handleIsXml,
+        javascript: javascript_1.handleJavascript,
+        latency: latency_1.handleLatency,
+        levenshtein: levenshtein_1.handleLevenshtein,
+        'llm-rubric': llmRubric_1.handleLlmRubric,
+        'model-graded-closedqa': modelGradedClosedQa_1.handleModelGradedClosedQa,
+        'model-graded-factuality': factuality_1.handleFactuality,
+        moderation: moderation_1.handleModeration,
+        perplexity: perplexity_1.handlePerplexity,
+        'perplexity-score': perplexity_1.handlePerplexityScore,
+        python: python_1.handlePython,
+        regex: regex_1.handleRegex,
+        'rouge-n': rouge_1.handleRougeScore,
+        similar: similar_1.handleSimilar,
+        'starts-with': startsWith_1.handleStartsWith,
+        webhook: webhook_1.handleWebhook,
+    };
+    const handler = assertionHandlers[baseType];
+    if (handler) {
+        return handler(assertionParams);
     }
     if (baseType.startsWith('promptfoo:redteam:')) {
-        const grader = (0, graders_1.getGraderById)(baseType);
-        (0, tiny_invariant_1.default)(grader, `Unknown promptfoo grader: ${baseType}`);
-        (0, tiny_invariant_1.default)(prompt, `Promptfoo grader ${baseType} must have a prompt`);
-        const { grade, rubric, suggestions } = await grader.getResult(prompt, outputString, test, provider, renderedValue);
-        return {
-            assertion: {
-                ...assertion,
-                value: rubric,
-            },
-            ...grade,
-            suggestions,
-            metadata: {
-                // Pass through all test metadata for redteam
-                ...test.metadata,
-                ...grade.metadata,
-            },
-        };
+        return (0, redteam_1.handleRedteam)(assertionParams);
     }
     throw new Error('Unknown assertion type: ' + assertion.type);
 }