promptfoo 0.95.0 → 0.96.1

package/dist/src/assertions/openai.js

@@ -0,0 +1,68 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleIsValidOpenAiToolsCall = exports.handleIsValidOpenAiFunctionCall = void 0;
+const openaiUtil_1 = require("../providers/openaiUtil");
+const handleIsValidOpenAiFunctionCall = ({ assertion, output, provider, test, }) => {
+    const functionOutput = output;
+    if (typeof functionOutput !== 'object' ||
+        typeof functionOutput.name !== 'string' ||
+        typeof functionOutput.arguments !== 'string') {
+        return {
+            pass: false,
+            score: 0,
+            reason: `OpenAI did not return a valid-looking function call: ${JSON.stringify(functionOutput)}`,
+            assertion,
+        };
+    }
+    try {
+        (0, openaiUtil_1.validateFunctionCall)(functionOutput, provider.config.functions, test.vars);
+        return {
+            pass: true,
+            score: 1,
+            reason: 'Assertion passed',
+            assertion,
+        };
+    }
+    catch (err) {
+        return {
+            pass: false,
+            score: 0,
+            reason: err.message,
+            assertion,
+        };
+    }
+};
+exports.handleIsValidOpenAiFunctionCall = handleIsValidOpenAiFunctionCall;
+const handleIsValidOpenAiToolsCall = ({ assertion, output, provider, test, }) => {
+    const toolsOutput = output;
+    if (!Array.isArray(toolsOutput) ||
+        toolsOutput.length === 0 ||
+        typeof toolsOutput[0].function.name !== 'string' ||
+        typeof toolsOutput[0].function.arguments !== 'string') {
+        return {
+            pass: false,
+            score: 0,
+            reason: `OpenAI did not return a valid-looking tools response: ${JSON.stringify(toolsOutput)}`,
+            assertion,
+        };
+    }
+    try {
+        toolsOutput.forEach((toolOutput) => (0, openaiUtil_1.validateFunctionCall)(toolOutput.function, provider.config.tools?.map((tool) => tool.function), test.vars));
+        return {
+            pass: true,
+            score: 1,
+            reason: 'Assertion passed',
+            assertion,
+        };
+    }
+    catch (err) {
+        return {
+            pass: false,
+            score: 0,
+            reason: err.message,
+            assertion,
+        };
+    }
+};
+exports.handleIsValidOpenAiToolsCall = handleIsValidOpenAiToolsCall;
+//# sourceMappingURL=openai.js.map

package/dist/src/assertions/openai.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../../src/assertions/openai.ts"],"names":[],"mappings":";;;AACA,wDAA+D;AAIxD,MAAM,+BAA+B,GAAG,CAAC,EAC9C,SAAS,EACT,MAAM,EACN,QAAQ,EACR,IAAI,GACY,EAAiB,EAAE;IACnC,MAAM,cAAc,GAAG,MAA6C,CAAC;IACrE,IACE,OAAO,cAAc,KAAK,QAAQ;QAClC,OAAO,cAAc,CAAC,IAAI,KAAK,QAAQ;QACvC,OAAO,cAAc,CAAC,SAAS,KAAK,QAAQ,EAC5C,CAAC;QACD,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,wDAAwD,IAAI,CAAC,SAAS,CAC5E,cAAc,CACf,EAAE;YACH,SAAS;SACV,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,IAAA,iCAAoB,EAClB,cAAc,EACb,QAAyC,CAAC,MAAM,CAAC,SAAS,EAC3D,IAAI,CAAC,IAAI,CACV,CAAC;QACF,OAAO;YACL,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,kBAAkB;YAC1B,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAG,GAAa,CAAC,OAAO;YAC9B,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAzCW,QAAA,+BAA+B,mCAyC1C;AAEK,MAAM,4BAA4B,GAAG,CAAC,EAC3C,SAAS,EACT,MAAM,EACN,QAAQ,EACR,IAAI,GACY,EAAiB,EAAE;IACnC,MAAM,WAAW,GAAG,MAGjB,CAAC;IACJ,IACE,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;QAC3B,WAAW,CAAC,MAAM,KAAK,CAAC;QACxB,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,KAAK,QAAQ;QAChD,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,QAAQ,EACrD,CAAC;QACD,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,yDAAyD,IAAI,CAAC,SAAS,CAC7E,WAAW,CACZ,EAAE;YACH,SAAS;SACV,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,WAAW,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CACjC,IAAA,iCAAoB,EAClB,UAAU,CAAC,QAAQ,EAClB,QAAyC,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EACrF,IAAI,CAAC,IAAI,CACV,CACF,CAAC;QACF,OAAO;YACL,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,kBAAkB;YAC1B,SAAS;SACV,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAG,GAAa,CAAC,OAAO;YAC9B,SAAS;SACV,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAhDW,QAAA,4BAA4B,gCAgDvC"}

package/dist/src/assertions/perplexity.d.ts

@@ -0,0 +1,4 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare function handlePerplexity({ logProbs, assertion }: AssertionParams): GradingResult;
+export declare function handlePerplexityScore({ logProbs, assertion }: AssertionParams): GradingResult;
+//# sourceMappingURL=perplexity.d.ts.map

package/dist/src/assertions/perplexity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"perplexity.d.ts","sourceRoot":"","sources":["../../../src/assertions/perplexity.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,wBAAgB,gBAAgB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,eAAe,GAAG,aAAa,CAiBxF;AAED,wBAAgB,qBAAqB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,eAAe,GAAG,aAAa,CAsB7F"}

package/dist/src/assertions/perplexity.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handlePerplexity = handlePerplexity;
+exports.handlePerplexityScore = handlePerplexityScore;
+function handlePerplexity({ logProbs, assertion }) {
+    if (!logProbs || logProbs.length === 0) {
+        throw new Error('Perplexity assertion does not support providers that do not return logProbs');
+    }
+    const sumLogProbs = logProbs.reduce((acc, logProb) => acc + logProb, 0);
+    const avgLogProb = sumLogProbs / logProbs.length;
+    const perplexity = Math.exp(-avgLogProb);
+    const pass = assertion.threshold ? perplexity <= assertion.threshold : true;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Perplexity ${perplexity.toFixed(2)} is greater than threshold ${assertion.threshold}`,
+        assertion,
+    };
+}
+function handlePerplexityScore({ logProbs, assertion }) {
+    if (!logProbs || logProbs.length === 0) {
+        throw new Error('perplexity-score assertion does not support providers that do not return logProbs');
+    }
+    const sumLogProbs = logProbs.reduce((acc, logProb) => acc + logProb, 0);
+    const avgLogProb = sumLogProbs / logProbs.length;
+    const perplexity = Math.exp(-avgLogProb);
+    const perplexityNorm = 1 / (1 + perplexity);
+    const pass = assertion.threshold ? perplexityNorm >= assertion.threshold : true;
+    return {
+        pass,
+        score: perplexityNorm,
+        reason: pass
+            ? 'Assertion passed'
+            : `Perplexity score ${perplexityNorm.toFixed(2)} is less than threshold ${assertion.threshold}`,
+        assertion,
+    };
+}
+//# sourceMappingURL=perplexity.js.map

package/dist/src/assertions/perplexity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"perplexity.js","sourceRoot":"","sources":["../../../src/assertions/perplexity.ts"],"names":[],"mappings":";;AAEA,4CAiBC;AAED,sDAsBC;AAzCD,SAAgB,gBAAgB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAmB;IACvE,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,cAAc,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,8BAA8B,SAAS,CAAC,SAAS,EAAE;QAC1F,SAAS;KACV,CAAC;AACJ,CAAC;AAED,SAAgB,qBAAqB,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAmB;IAC5E,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CACb,mFAAmF,CACpF,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,cAAc,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC;IAE5C,MAAM,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;IAChF,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,cAAc;QACrB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,oBAAoB,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,2BAC3C,SAAS,CAAC,SACZ,EAAE;QACN,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/src/assertions/python.d.ts

@@ -0,0 +1,4 @@
+import type { AssertionParams } from '../types';
+import { type GradingResult } from '../types';
+export declare const handlePython: ({ assertion, renderedValue, valueFromScript, context, output, }: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=python.d.ts.map

package/dist/src/assertions/python.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"python.d.ts","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,EAAE,KAAK,aAAa,EAAmB,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,YAAY,oEAMtB,eAAe,KAAG,OAAO,CAAC,aAAa,CA0GzC,CAAC"}

package/dist/src/assertions/python.js

@@ -0,0 +1,107 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handlePython = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const wrapper_1 = require("../python/wrapper");
+const types_1 = require("../types");
+const handlePython = async ({ assertion, renderedValue, valueFromScript, context, output, }) => {
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'python assertion must have a string value');
+    let pass;
+    let score;
+    try {
+        let result;
+        if (typeof valueFromScript === 'undefined') {
+            const isMultiline = renderedValue.includes('\n');
+            let indentStyle = '    ';
+            if (isMultiline) {
+                // Detect the indentation style of the first indented line
+                const match = renderedValue.match(/^(?!\s*$)\s+/m);
+                if (match) {
+                    indentStyle = match[0];
+                }
+            }
+            const pythonScript = `import json
+
+def main(output, context):
+${isMultiline
+                ? renderedValue
+                    .split('\n')
+                    .map((line) => `${indentStyle}${line}`)
+                    .join('\n')
+                : `    return ${renderedValue}`}
+`;
+            result = await (0, wrapper_1.runPythonCode)(pythonScript, 'main', [output, context]);
+        }
+        else {
+            result = valueFromScript;
+        }
+        if ((typeof result === 'boolean' && result) ||
+            (typeof result === 'string' && result.toLowerCase() === 'true')) {
+            pass = true;
+            score = 1.0;
+        }
+        else if ((typeof result === 'boolean' && !result) ||
+            (typeof result === 'string' && result.toLowerCase() === 'false')) {
+            pass = false;
+            score = 0.0;
+        }
+        else if (typeof result === 'string' && result.startsWith('{')) {
+            let parsed;
+            try {
+                parsed = JSON.parse(result);
+            }
+            catch (err) {
+                throw new Error(`Invalid JSON: ${err} when parsing result: ${result}`);
+            }
+            if (!(0, types_1.isGradingResult)(parsed)) {
+                throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead: ${result}`);
+            }
+            return parsed;
+        }
+        else if (typeof result === 'object') {
+            if (!(0, types_1.isGradingResult)(result)) {
+                throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Got instead:\n${JSON.stringify(result, null, 2)}`);
+            }
+            const pythonGradingResult = result;
+            if (assertion.threshold && pythonGradingResult.score < assertion.threshold) {
+                pythonGradingResult.pass = false;
+                pythonGradingResult.reason = `Python score ${pythonGradingResult.score} is less than threshold ${assertion.threshold}`;
+            }
+            return {
+                ...pythonGradingResult,
+                assertion,
+            };
+        }
+        else {
+            score = Number.parseFloat(String(result));
+            pass = assertion.threshold ? score >= assertion.threshold : score > 0;
+            if (Number.isNaN(score)) {
+                throw new Error(`Python assertion must return a boolean, number, or {pass, score, reason} object. Instead got:\n${result}`);
+            }
+            if (typeof assertion.threshold !== 'undefined' && score < assertion.threshold) {
+                pass = false;
+            }
+        }
+    }
+    catch (err) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Python code execution failed: ${err.message}`,
+            assertion,
+        };
+    }
+    return {
+        pass,
+        score,
+        reason: pass
+            ? 'Assertion passed'
+            : `Python code returned ${pass ? 'true' : 'false'}\n${assertion.value}`,
+        assertion,
+    };
+};
+exports.handlePython = handlePython;
+//# sourceMappingURL=python.js.map

package/dist/src/assertions/python.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"python.js","sourceRoot":"","sources":["../../../src/assertions/python.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,+CAAkD;AAElD,oCAA+D;AAExD,MAAM,YAAY,GAAG,KAAK,EAAE,EACjC,SAAS,EACT,aAAa,EACb,eAAe,EACf,OAAO,EACP,MAAM,GACU,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,2CAA2C,CAAC,CAAC;IAC1F,IAAI,IAAI,CAAC;IACT,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,IAAI,MAAsE,CAAC;QAC3E,IAAI,OAAO,eAAe,KAAK,WAAW,EAAE,CAAC;YAC3C,MAAM,WAAW,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACjD,IAAI,WAAW,GAAG,MAAM,CAAC;YACzB,IAAI,WAAW,EAAE,CAAC;gBAChB,0DAA0D;gBAC1D,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;gBACnD,IAAI,KAAK,EAAE,CAAC;oBACV,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,MAAM,YAAY,GAAG;;;EAIzB,WAAW;gBACT,CAAC,CAAC,aAAa;qBACV,KAAK,CAAC,IAAI,CAAC;qBACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,WAAW,GAAG,IAAI,EAAE,CAAC;qBACtC,IAAI,CAAC,IAAI,CAAC;gBACf,CAAC,CAAC,cAAc,aAAa,EACjC;CACC,CAAC;YACI,MAAM,GAAG,MAAM,IAAA,uBAAa,EAAC,YAAY,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;QACxE,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,eAAe,CAAC;QAC3B,CAAC;QAED,IACE,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC;YACvC,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,MAAM,CAAC,EAC/D,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;YACZ,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;aAAM,IACL,CAAC,OAAO,MAAM,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC;YACxC,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,EAChE,CAAC;YACD,IAAI,GAAG,KAAK,CAAC;YACb,KAAK,GAAG,GAAG,CAAC;QACd,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,IAAI,MAAM,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,yBAAyB,MAAM,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,IAAI,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,iGAAiG,MAAM,EAAE,CAC1G,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;aAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACtC,IAAI,CAAC,IAAA,uBAAe,EAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,kGAAkG,IAAI,CAAC,SAAS,CAC9G,MAAM,EACN,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,mBAAmB,GAAG,MAA0C,CAAC;YACvE,IAAI,SAAS,CAAC,SAAS,IAAI,mBAAmB,CAAC,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC3E,mBAAmB,CAAC,IAAI,GAAG,KAAK,CAAC;gBACjC,mBAAmB,CAAC,MAAM,GAAG,gBAAgB,mBAAmB,CAAC,KAAK,2BAA2B,SAAS,CAAC,SAAS,EAAE,CAAC;YACzH,CAAC;YACD,OAAO;gBACL,GAAG,mBAAmB;gBACtB,SAAS;aACV,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAC1C,IAAI,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC;YACtE,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,kGAAkG,MAAM,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,SAAS,CAAC,SAAS,KAAK,WAAW,IAAI,KAAK,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC;gBAC9E,IAAI,GAAG,KAAK,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,iCAAkC,GAAa,CAAC,OAAO,EAAE;YACjE,SAAS;SACV,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI;QACJ,KAAK;QACL,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,wBAAwB,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,CAAC,KAAK,EAAE;QACzE,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAhHW,QAAA,YAAY,gBAgHvB"}

package/dist/src/assertions/redteam.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleRedteam: ({ assertion, baseType, test, prompt, outputString, provider, renderedValue, }: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=redteam.d.ts.map

package/dist/src/assertions/redteam.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.d.ts","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,aAAa,kFAQvB,eAAe,KAAG,OAAO,CAAC,aAAa,CAwBzC,CAAC"}

package/dist/src/assertions/redteam.js

@@ -0,0 +1,29 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleRedteam = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const graders_1 = require("../redteam/graders");
+const handleRedteam = async ({ assertion, baseType, test, prompt, outputString, provider, renderedValue, }) => {
+    const grader = (0, graders_1.getGraderById)(assertion.type);
+    (0, tiny_invariant_1.default)(grader, `Unknown promptfoo grader: ${baseType}`);
+    (0, tiny_invariant_1.default)(prompt, `Promptfoo grader ${baseType} must have a prompt`);
+    const { grade, rubric, suggestions } = await grader.getResult(prompt, outputString, test, provider, renderedValue);
+    return {
+        assertion: {
+            ...assertion,
+            value: rubric,
+        },
+        ...grade,
+        suggestions,
+        metadata: {
+            // Pass through all test metadata for redteam
+            ...test.metadata,
+            ...grade.metadata,
+        },
+    };
+};
+exports.handleRedteam = handleRedteam;
+//# sourceMappingURL=redteam.js.map

package/dist/src/assertions/redteam.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"redteam.js","sourceRoot":"","sources":["../../../src/assertions/redteam.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,gDAAmD;AAG5C,MAAM,aAAa,GAAG,KAAK,EAAE,EAClC,SAAS,EACT,QAAQ,EACR,IAAI,EACJ,MAAM,EACN,YAAY,EACZ,QAAQ,EACR,aAAa,GACG,EAA0B,EAAE;IAC5C,MAAM,MAAM,GAAG,IAAA,uBAAa,EAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAA,wBAAS,EAAC,MAAM,EAAE,6BAA6B,QAAQ,EAAE,CAAC,CAAC;IAC3D,IAAA,wBAAS,EAAC,MAAM,EAAE,oBAAoB,QAAQ,qBAAqB,CAAC,CAAC;IACrE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAC3D,MAAM,EACN,YAAY,EACZ,IAAI,EACJ,QAAQ,EACR,aAAa,CACd,CAAC;IACF,OAAO;QACL,SAAS,EAAE;YACT,GAAG,SAAS;YACZ,KAAK,EAAE,MAAM;SACd;QACD,GAAG,KAAK;QACR,WAAW;QACX,QAAQ,EAAE;YACR,6CAA6C;YAC7C,GAAG,IAAI,CAAC,QAAQ;YAChB,GAAG,KAAK,CAAC,QAAQ;SAClB;KACF,CAAC;AACJ,CAAC,CAAC;AAhCW,QAAA,aAAa,iBAgCxB"}

package/dist/src/assertions/regex.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleRegex: ({ assertion, renderedValue, outputString, inverse, }: AssertionParams) => GradingResult;
+//# sourceMappingURL=regex.d.ts.map

package/dist/src/assertions/regex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex.d.ts","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,WAAW,yDAKrB,eAAe,KAAG,aAuBpB,CAAC"}

package/dist/src/assertions/regex.js

@@ -0,0 +1,34 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleRegex = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const handleRegex = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"regex" assertion type must have a string value');
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"regex" assertion type must have a string value');
+    let regex;
+    try {
+        regex = new RegExp(renderedValue);
+    }
+    catch (error) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Invalid regex pattern: ${error instanceof Error ? error.message : 'unknown error'}`,
+            assertion,
+        };
+    }
+    const pass = regex.test(outputString) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
+        assertion,
+    };
+};
+exports.handleRegex = handleRegex;
+//# sourceMappingURL=regex.js.map

package/dist/src/assertions/regex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"regex.js","sourceRoot":"","sources":["../../../src/assertions/regex.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AAGhC,MAAM,WAAW,GAAG,CAAC,EAC1B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,iDAAiD,CAAC,CAAC;IAC5E,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,iDAAiD,CAAC,CAAC;IAChG,IAAI,KAAa,CAAC;IAClB,IAAI,CAAC;QACH,KAAK,GAAG,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE;YAC5F,SAAS;SACV,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,KAAK,OAAO,CAAC;IAClD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,gBAAgB,aAAa,GAAG;QAC/E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA5BW,QAAA,WAAW,eA4BtB"}

package/dist/src/assertions/rouge.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare function handleRougeScore({ baseType, assertion, renderedValue, outputString, inverse, }: AssertionParams): GradingResult;
+//# sourceMappingURL=rouge.d.ts.map

package/dist/src/assertions/rouge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rouge.d.ts","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,wBAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACR,EAAE,eAAe,GAAG,aAAa,CAkBjC"}

package/dist/src/assertions/rouge.js

@@ -0,0 +1,47 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleRougeScore = handleRougeScore;
+const rouge = __importStar(require("js-rouge"));
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+function handleRougeScore({ baseType, assertion, renderedValue, outputString, inverse, }) {
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"rouge" assertion type must be a string value');
+    const fnName = baseType[baseType.length - 1];
+    const rougeMethod = rouge[fnName];
+    const score = rougeMethod(outputString, renderedValue, {});
+    const pass = score >= (assertion.threshold || 0.75) != inverse;
+    return {
+        pass,
+        score: inverse ? 1 - score : score,
+        reason: pass
+            ? `${baseType.toUpperCase()} score ${score.toFixed(2)} is greater than or equal to threshold ${assertion.threshold || 0.75}`
+            : `${baseType.toUpperCase()} score ${score.toFixed(2)} is less than threshold ${assertion.threshold || 0.75}`,
+        assertion,
+    };
+}
+//# sourceMappingURL=rouge.js.map

package/dist/src/assertions/rouge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rouge.js","sourceRoot":"","sources":["../../../src/assertions/rouge.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,4CAwBC;AA5BD,gDAAkC;AAClC,oEAAuC;AAGvC,SAAgB,gBAAgB,CAAC,EAC/B,QAAQ,EACR,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS;IAChB,IAAA,wBAAS,EAAC,OAAO,aAAa,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAC;IAChE,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,YAAY,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC;IAC/D,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK;QAClC,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,CAAC,OAAO,CAC9C,CAAC,CACF,0CAA0C,SAAS,CAAC,SAAS,IAAI,IAAI,EAAE;YAC1E,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,2BACjD,SAAS,CAAC,SAAS,IAAI,IACzB,EAAE;QACN,SAAS;KACV,CAAC;AACJ,CAAC"}

package/dist/src/assertions/similar.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleSimilar: ({ assertion, renderedValue, outputString, inverse, test, }: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=similar.d.ts.map

package/dist/src/assertions/similar.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"similar.d.ts","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,aAAa,+DAMvB,eAAe,KAAG,OAAO,CAAC,aAAa,CA2CzC,CAAC"}

package/dist/src/assertions/similar.js

@@ -0,0 +1,40 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleSimilar = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const matchers_1 = require("../matchers");
+const handleSimilar = async ({ assertion, renderedValue, outputString, inverse, test, }) => {
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || Array.isArray(renderedValue), 'Similarity assertion type must have a string or array of strings value');
+    if (Array.isArray(renderedValue)) {
+        let minScore = Infinity;
+        for (const value of renderedValue) {
+            const result = await (0, matchers_1.matchesSimilarity)(value, outputString, assertion.threshold || 0.75, inverse, test.options);
+            if (result.pass) {
+                return {
+                    assertion,
+                    ...result,
+                };
+            }
+            if (result.score < minScore) {
+                minScore = result.score;
+            }
+        }
+        return {
+            assertion,
+            pass: false,
+            score: minScore,
+            reason: `None of the provided values met the similarity threshold`,
+        };
+    }
+    else {
+        return {
+            assertion,
+            ...(await (0, matchers_1.matchesSimilarity)(renderedValue, outputString, assertion.threshold || 0.75, inverse, test.options)),
+        };
+    }
+};
+exports.handleSimilar = handleSimilar;
+//# sourceMappingURL=similar.js.map

package/dist/src/assertions/similar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"similar.js","sourceRoot":"","sources":["../../../src/assertions/similar.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,0CAAgD;AAGzC,MAAM,aAAa,GAAG,KAAK,EAAE,EAClC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,EACP,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EACjE,wEAAwE,CACzE,CAAC;IACF,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QACjC,IAAI,QAAQ,GAAG,QAAQ,CAAC;QACxB,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,IAAA,4BAAiB,EACpC,KAAK,EACL,YAAY,EACZ,SAAS,CAAC,SAAS,IAAI,IAAI,EAC3B,OAAO,EACP,IAAI,CAAC,OAAO,CACb,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,OAAO;oBACL,SAAS;oBACT,GAAG,MAAM;iBACV,CAAC;YACJ,CAAC;YACD,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,EAAE,CAAC;gBAC5B,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC;QACD,OAAO;YACL,SAAS;YACT,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,0DAA0D;SACnE,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,OAAO;YACL,SAAS;YACT,GAAG,CAAC,MAAM,IAAA,4BAAiB,EACzB,aAAa,EACb,YAAY,EACZ,SAAS,CAAC,SAAS,IAAI,IAAI,EAC3B,OAAO,EACP,IAAI,CAAC,OAAO,CACb,CAAC;SACH,CAAC;IACJ,CAAC;AACH,CAAC,CAAC;AAjDW,QAAA,aAAa,iBAiDxB"}

package/dist/src/assertions/sql.d.ts

@@ -0,0 +1,4 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleIsSql: ({ assertion, renderedValue, outputString, inverse, }: AssertionParams) => Promise<GradingResult>;
+export declare const handleContainsSql: (assertionParams: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=sql.d.ts.map

package/dist/src/assertions/sql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../../src/assertions/sql.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,WAAW,yDAKrB,eAAe,KAAG,OAAO,CAAC,aAAa,CAyEzC,CAAC;AAEF,eAAO,MAAM,iBAAiB,oBACX,eAAe,KAC/B,OAAO,CAAC,aAAa,CAOvB,CAAC"}

package/dist/src/assertions/sql.js

@@ -0,0 +1,98 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleContainsSql = exports.handleIsSql = void 0;
+const utils_1 = require("./utils");
+const handleIsSql = async ({ assertion, renderedValue, outputString, inverse, }) => {
+    let pass = false;
+    let databaseType = 'MySQL';
+    let whiteTableList;
+    let whiteColumnList;
+    if (renderedValue && typeof renderedValue === 'object') {
+        const value = renderedValue;
+        databaseType = value.databaseType || 'MySQL';
+        whiteTableList = value.allowedTables;
+        whiteColumnList = value.allowedColumns;
+    }
+    if (renderedValue && typeof renderedValue !== 'object') {
+        throw new Error('is-sql assertion must have a object value.');
+    }
+    const { Parser: SqlParser } = await Promise.resolve().then(() => __importStar(require('node-sql-parser'))).catch(() => {
+        throw new Error('node-sql-parser is not installed. Please install it first');
+    });
+    const sqlParser = new SqlParser();
+    const opt = { database: databaseType };
+    const failureReasons = [];
+    try {
+        sqlParser.astify(outputString, opt);
+        pass = !inverse;
+    }
+    catch {
+        pass = inverse;
+        failureReasons.push(`SQL statement does not conform to the provided ${databaseType} database syntax.`);
+    }
+    if (whiteTableList) {
+        opt.type = 'table';
+        try {
+            sqlParser.whiteListCheck(outputString, whiteTableList, opt);
+        }
+        catch (err) {
+            pass = inverse;
+            const error = err;
+            failureReasons.push(`SQL validation failed: ${error.message}.`);
+        }
+    }
+    if (whiteColumnList) {
+        opt.type = 'column';
+        try {
+            sqlParser.whiteListCheck(outputString, whiteColumnList, opt);
+        }
+        catch (err) {
+            pass = inverse;
+            const error = err;
+            failureReasons.push(`SQL validation failed: ${error.message}.`);
+        }
+    }
+    if (inverse && pass === false && failureReasons.length === 0) {
+        failureReasons.push('The output SQL statement is valid');
+    }
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass ? 'Assertion passed' : failureReasons.join(' '),
+        assertion,
+    };
+};
+exports.handleIsSql = handleIsSql;
+const handleContainsSql = async (assertionParams) => {
+    const match = (0, utils_1.coerceString)(assertionParams.outputString).match(/```(?:sql)?([^`]+)```/);
+    if (match) {
+        const sqlCode = match[1].trim();
+        return (0, exports.handleIsSql)({ ...assertionParams, outputString: sqlCode });
+    }
+    return (0, exports.handleIsSql)(assertionParams);
+};
+exports.handleContainsSql = handleContainsSql;
+//# sourceMappingURL=sql.js.map

package/dist/src/assertions/sql.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql.js","sourceRoot":"","sources":["../../../src/assertions/sql.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAEA,mCAAuC;AAEhC,MAAM,WAAW,GAAG,KAAK,EAAE,EAChC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAA0B,EAAE;IAC5C,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,YAAY,GAAW,OAAO,CAAC;IACnC,IAAI,cAAoC,CAAC;IACzC,IAAI,eAAqC,CAAC;IAC1C,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,aAIb,CAAC;QAEF,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,OAAO,CAAC;QAC7C,cAAc,GAAG,KAAK,CAAC,aAAa,CAAC;QACrC,eAAe,GAAG,KAAK,CAAC,cAAc,CAAC;IACzC,CAAC;IAED,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,kDAAO,iBAAiB,IAAE,KAAK,CAAC,GAAG,EAAE;QACvE,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,IAAI,SAAS,EAAE,CAAC;IAElC,MAAM,GAAG,GAAoB,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC;IAExD,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,IAAI,CAAC;QACH,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;QACpC,IAAI,GAAG,CAAC,OAAO,CAAC;IAClB,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,GAAG,OAAO,CAAC;QACf,cAAc,CAAC,IAAI,CACjB,kDAAkD,YAAY,mBAAmB,CAClF,CAAC;IACJ,CAAC;IAED,IAAI,cAAc,EAAE,CAAC;QACnB,GAAG,CAAC,IAAI,GAAG,OAAO,CAAC;QACnB,IAAI,CAAC;YACH,SAAS,CAAC,cAAc,CAAC,YAAY,EAAE,cAAc,EAAE,GAAG,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,OAAO,CAAC;YACf,MAAM,KAAK,GAAG,GAAY,CAAC;YAC3B,cAAc,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,IAAI,eAAe,EAAE,CAAC;QACpB,GAAG,CAAC,IAAI,GAAG,QAAQ,CAAC;QACpB,IAAI,CAAC;YACH,SAAS,CAAC,cAAc,CAAC,YAAY,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,OAAO,CAAC;YACf,MAAM,KAAK,GAAG,GAAY,CAAC;YAC3B,cAAc,CAAC,IAAI,CAAC,0BAA0B,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,IAAI,OAAO,IAAI,IAAI,KAAK,KAAK,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,cAAc,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;QAC5D,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA9EW,QAAA,WAAW,eA8EtB;AAEK,MAAM,iBAAiB,GAAG,KAAK,EACpC,eAAgC,EACR,EAAE;IAC1B,MAAM,KAAK,GAAG,IAAA,oBAAY,EAAC,eAAe,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;IACxF,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,IAAA,mBAAW,EAAC,EAAE,GAAG,eAAe,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,IAAA,mBAAW,EAAC,eAAe,CAAC,CAAC;AACtC,CAAC,CAAC;AATW,QAAA,iBAAiB,qBAS5B"}

package/dist/src/assertions/startsWith.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleStartsWith: ({ assertion, renderedValue, outputString, inverse, }: AssertionParams) => GradingResult;
+//# sourceMappingURL=startsWith.d.ts.map

package/dist/src/assertions/startsWith.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"startsWith.d.ts","sourceRoot":"","sources":["../../../src/assertions/startsWith.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,gBAAgB,yDAK1B,eAAe,KAAG,aAepB,CAAC"}

package/dist/src/assertions/startsWith.js

@@ -0,0 +1,22 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleStartsWith = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const handleStartsWith = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"starts-with" assertion type must have a string value');
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string', '"starts-with" assertion type must have a string value');
+    const pass = outputString.startsWith(String(renderedValue)) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
+        assertion,
+    };
+};
+exports.handleStartsWith = handleStartsWith;
+//# sourceMappingURL=startsWith.js.map

package/dist/src/assertions/startsWith.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"startsWith.js","sourceRoot":"","sources":["../../../src/assertions/startsWith.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AAGhC,MAAM,gBAAgB,GAAG,CAAC,EAC/B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,uDAAuD,CAAC,CAAC;IAClF,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,uDAAuD,CACxD,CAAC;IACF,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,OAAO,CAAC;IACxE,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,eAAe,aAAa,GAAG;QAC9E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AApBW,QAAA,gBAAgB,oBAoB3B"}

package/dist/src/assertions/utils.d.ts

@@ -1,4 +1,5 @@
 import { type Assertion, type TestCase } from '../types';
 export declare function getFinalTest(test: TestCase, assertion: Assertion): Readonly<TestCase>;
 export declare function processFileReference(fileRef: string): object | string;
+export declare function coerceString(value: string | object): string;
 //# sourceMappingURL=utils.d.ts.map

package/dist/src/assertions/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/assertions/utils.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,QAAQ,EAAE,MAAM,UAAU,CAAC;AAIzD,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,sBAUhE;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAYrE"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/assertions/utils.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,QAAQ,EAAE,MAAM,UAAU,CAAC;AAIzD,wBAAgB,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,sBAUhE;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAYrE;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAK3D"}