promptfoo 0.96.0 → 0.96.2

package/dist/src/assertions/contains.js

@@ -0,0 +1,110 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleIContainsAll = exports.handleContainsAll = exports.handleIContainsAny = exports.handleContainsAny = exports.handleIContains = exports.handleContains = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const handleContains = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"contains" assertion type must have a string or number value');
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"contains" assertion type must have a string or number value');
+    const pass = outputString.includes(String(renderedValue)) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
+        assertion,
+    };
+};
+exports.handleContains = handleContains;
+const handleIContains = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"icontains" assertion type must have a string or number value');
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string' || typeof renderedValue === 'number', '"icontains" assertion type must have a string or number value');
+    const pass = outputString.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
+        assertion,
+    };
+};
+exports.handleIContains = handleIContains;
+const handleContainsAny = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"contains-any" assertion type must have a value');
+    if (typeof renderedValue === 'string') {
+        // Handle quoted values and escaped commas
+        renderedValue =
+            renderedValue
+                .match(/(".*?"|[^,]+)(?=\s*,|\s*$)/g)
+                ?.map((v) => v.trim().replace(/^"|"$/g, '')) ?? [];
+    }
+    (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-any" assertion type must have an array value');
+    const pass = renderedValue.some((value) => outputString.includes(String(value))) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
+        assertion,
+    };
+};
+exports.handleContainsAny = handleContainsAny;
+const handleIContainsAny = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"icontains-any" assertion type must have a value');
+    if (typeof renderedValue === 'string') {
+        renderedValue = renderedValue.split(',').map((v) => v.trim());
+    }
+    (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-any" assertion type must have an array value');
+    const pass = renderedValue.some((value) => outputString.toLowerCase().includes(String(value).toLowerCase())) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
+        assertion,
+    };
+};
+exports.handleIContainsAny = handleIContainsAny;
+const handleContainsAll = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"contains-all" assertion type must have a value');
+    if (typeof renderedValue === 'string') {
+        renderedValue = renderedValue.split(',').map((v) => v.trim());
+    }
+    (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"contains-all" assertion type must have an array value');
+    const missingStrings = renderedValue.filter((value) => !outputString.includes(String(value)));
+    const pass = (missingStrings.length === 0) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}contain all of [${renderedValue.join(', ')}]. Missing: [${missingStrings.join(', ')}]`,
+        assertion,
+    };
+};
+exports.handleContainsAll = handleContainsAll;
+const handleIContainsAll = ({ assertion, renderedValue, outputString, inverse, }) => {
+    (0, tiny_invariant_1.default)(renderedValue, '"icontains-all" assertion type must have a value');
+    if (typeof renderedValue === 'string') {
+        renderedValue = renderedValue.split(',').map((v) => v.trim());
+    }
+    (0, tiny_invariant_1.default)(Array.isArray(renderedValue), '"icontains-all" assertion type must have an array value');
+    const missingStrings = renderedValue.filter((value) => !outputString.toLowerCase().includes(String(value).toLowerCase()));
+    const pass = (missingStrings.length === 0) !== inverse;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output to ${inverse ? 'not ' : ''}contain all of [${renderedValue.join(', ')}]. Missing: [${missingStrings.join(', ')}]`,
+        assertion,
+    };
+};
+exports.handleIContainsAll = handleIContainsAll;
+//# sourceMappingURL=contains.js.map

package/dist/src/assertions/contains.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contains.js","sourceRoot":"","sources":["../../../src/assertions/contains.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AAGhC,MAAM,cAAc,GAAG,CAAC,EAC7B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,8DAA8D,CAAC,CAAC;IACzF,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,IAAI,OAAO,aAAa,KAAK,QAAQ,EACtE,8DAA8D,CAC/D,CAAC;IACF,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,KAAK,OAAO,CAAC;IACtE,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,aAAa,GAAG;QAC3E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AApBW,QAAA,cAAc,kBAoBzB;AAEK,MAAM,eAAe,GAAG,CAAC,EAC9B,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,+DAA+D,CAAC,CAAC;IAC1F,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,IAAI,OAAO,aAAa,KAAK,QAAQ,EACtE,+DAA+D,CAChE,CAAC;IACF,MAAM,IAAI,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,OAAO,CAAC;IAClG,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,aAAa,GAAG;QAC3E,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AApBW,QAAA,eAAe,mBAoB1B;AAEK,MAAM,iBAAiB,GAAG,CAAC,EAChC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,iDAAiD,CAAC,CAAC;IAC5E,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,0CAA0C;QAC1C,aAAa;YACX,aAAa;iBACV,KAAK,CAAC,6BAA6B,CAAC;gBACrC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,CAAC;IACD,IAAA,wBAAS,EAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,wDAAwD,CAAC,CAAC;IAClG,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC;IAC7F,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,mBAAmB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QAC7F,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAxBW,QAAA,iBAAiB,qBAwB5B;AAEK,MAAM,kBAAkB,GAAG,CAAC,EACjC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,kDAAkD,CAAC,CAAC;IAC7E,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAA,wBAAS,EACP,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAC5B,yDAAyD,CAC1D,CAAC;IACF,MAAM,IAAI,GACR,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAC3B,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CACjE,KAAK,OAAO,CAAC;IAChB,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,mBAAmB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QAC7F,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA1BW,QAAA,kBAAkB,sBA0B7B;AAEK,MAAM,iBAAiB,GAAG,CAAC,EAChC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,iDAAiD,CAAC,CAAC;IAC5E,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAA,wBAAS,EAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,wDAAwD,CAAC,CAAC;IAClG,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9F,MAAM,IAAI,GAAG,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC;IACvD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,mBAAmB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QACtI,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AArBW,QAAA,iBAAiB,qBAqB5B;AAEK,MAAM,kBAAkB,GAAG,CAAC,EACjC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GACS,EAAiB,EAAE;IACnC,IAAA,wBAAS,EAAC,aAAa,EAAE,kDAAkD,CAAC,CAAC;IAC7E,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,aAAa,GAAG,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAA,wBAAS,EACP,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAC5B,yDAAyD,CAC1D,CAAC;IACF,MAAM,cAAc,GAAG,aAAa,CAAC,MAAM,CACzC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAC7E,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC;IACvD,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,mBAAmB,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;QACtI,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA1BW,QAAA,kBAAkB,sBA0B7B"}

package/dist/src/assertions/contextFaithfulness.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare function handleContextFaithfulness({ assertion, test, output, }: AssertionParams): Promise<GradingResult>;
+//# sourceMappingURL=contextFaithfulness.d.ts.map

package/dist/src/assertions/contextFaithfulness.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextFaithfulness.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextFaithfulness.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,wBAAsB,yBAAyB,CAAC,EAC9C,SAAS,EACT,IAAI,EACJ,MAAM,GACP,EAAE,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAyB1C"}

package/dist/src/assertions/contextFaithfulness.js

@@ -0,0 +1,19 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleContextFaithfulness = handleContextFaithfulness;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const matchers_1 = require("../matchers");
+async function handleContextFaithfulness({ assertion, test, output, }) {
+    (0, tiny_invariant_1.default)(test.vars, 'context-faithfulness assertion type must have a vars object');
+    (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-faithfulness assertion type must have a query var');
+    (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-faithfulness assertion type must have a context var');
+    (0, tiny_invariant_1.default)(typeof output === 'string', 'context-faithfulness assertion type must have a string output');
+    return {
+        assertion,
+        ...(await (0, matchers_1.matchesContextFaithfulness)(test.vars.query, output, test.vars.context, assertion.threshold || 0, test.options)),
+    };
+}
+//# sourceMappingURL=contextFaithfulness.js.map

package/dist/src/assertions/contextFaithfulness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextFaithfulness.js","sourceRoot":"","sources":["../../../src/assertions/contextFaithfulness.ts"],"names":[],"mappings":";;;;;AAIA,8DA6BC;AAjCD,oEAAuC;AACvC,0CAAyD;AAGlD,KAAK,UAAU,yBAAyB,CAAC,EAC9C,SAAS,EACT,IAAI,EACJ,MAAM,GACU;IAChB,IAAA,wBAAS,EAAC,IAAI,CAAC,IAAI,EAAE,6DAA6D,CAAC,CAAC;IACpF,IAAA,wBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,QAAQ,EACnC,2DAA2D,CAC5D,CAAC;IACF,IAAA,wBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,EACrC,6DAA6D,CAC9D,CAAC;IACF,IAAA,wBAAS,EACP,OAAO,MAAM,KAAK,QAAQ,EAC1B,+DAA+D,CAChE,CAAC;IAEF,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,qCAA0B,EAClC,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,MAAM,EACN,IAAI,CAAC,IAAI,CAAC,OAAO,EACjB,SAAS,CAAC,SAAS,IAAI,CAAC,EACxB,IAAI,CAAC,OAAO,CACb,CAAC;KACH,CAAC;AACJ,CAAC"}

package/dist/src/assertions/contextRecall.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleContextRecall: ({ assertion, renderedValue, prompt, test, }: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=contextRecall.d.ts.map

package/dist/src/assertions/contextRecall.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRecall.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextRecall.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,mBAAmB,gDAK7B,eAAe,KAAG,OAAO,CAAC,aAAa,CAgBzC,CAAC"}

package/dist/src/assertions/contextRecall.js

@@ -0,0 +1,18 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleContextRecall = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const matchers_1 = require("../matchers");
+const handleContextRecall = async ({ assertion, renderedValue, prompt, test, }) => {
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'context-recall assertion type must have a string value');
+    (0, tiny_invariant_1.default)(prompt, 'context-recall assertion type must have a prompt');
+    return {
+        assertion,
+        ...(await (0, matchers_1.matchesContextRecall)(typeof test.vars?.context === 'string' ? test.vars.context : prompt, renderedValue, assertion.threshold || 0, test.options, test.vars)),
+    };
+};
+exports.handleContextRecall = handleContextRecall;
+//# sourceMappingURL=contextRecall.js.map

package/dist/src/assertions/contextRecall.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRecall.js","sourceRoot":"","sources":["../../../src/assertions/contextRecall.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,0CAAmD;AAG5C,MAAM,mBAAmB,GAAG,KAAK,EAAE,EACxC,SAAS,EACT,aAAa,EACb,MAAM,EACN,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,wDAAwD,CACzD,CAAC;IACF,IAAA,wBAAS,EAAC,MAAM,EAAE,kDAAkD,CAAC,CAAC;IACtE,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,+BAAoB,EAC5B,OAAO,IAAI,CAAC,IAAI,EAAE,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,EACnE,aAAa,EACb,SAAS,CAAC,SAAS,IAAI,CAAC,EACxB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,IAAI,CACV,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AArBW,QAAA,mBAAmB,uBAqB9B"}

package/dist/src/assertions/contextRelevance.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleContextRelevance: ({ assertion, test, }: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=contextRelevance.d.ts.map

package/dist/src/assertions/contextRelevance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRelevance.d.ts","sourceRoot":"","sources":["../../../src/assertions/contextRelevance.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,sBAAsB,yBAGhC,eAAe,KAAG,OAAO,CAAC,aAAa,CAoBzC,CAAC"}

package/dist/src/assertions/contextRelevance.js

@@ -0,0 +1,19 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleContextRelevance = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const matchers_1 = require("../matchers");
+const handleContextRelevance = async ({ assertion, test, }) => {
+    (0, tiny_invariant_1.default)(test.vars, 'context-relevance assertion type must have a vars object');
+    (0, tiny_invariant_1.default)(typeof test.vars.query === 'string', 'context-relevance assertion type must have a query var');
+    (0, tiny_invariant_1.default)(typeof test.vars.context === 'string', 'context-relevance assertion type must have a context var');
+    return {
+        assertion,
+        ...(await (0, matchers_1.matchesContextRelevance)(test.vars.query, test.vars.context, assertion.threshold || 0, test.options)),
+    };
+};
+exports.handleContextRelevance = handleContextRelevance;
+//# sourceMappingURL=contextRelevance.js.map

package/dist/src/assertions/contextRelevance.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRelevance.js","sourceRoot":"","sources":["../../../src/assertions/contextRelevance.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,0CAAsD;AAG/C,MAAM,sBAAsB,GAAG,KAAK,EAAE,EAC3C,SAAS,EACT,IAAI,GACY,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EAAC,IAAI,CAAC,IAAI,EAAE,0DAA0D,CAAC,CAAC;IACjF,IAAA,wBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,QAAQ,EACnC,wDAAwD,CACzD,CAAC;IACF,IAAA,wBAAS,EACP,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,QAAQ,EACrC,0DAA0D,CAC3D,CAAC;IAEF,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,kCAAuB,EAC/B,IAAI,CAAC,IAAI,CAAC,KAAK,EACf,IAAI,CAAC,IAAI,CAAC,OAAO,EACjB,SAAS,CAAC,SAAS,IAAI,CAAC,EACxB,IAAI,CAAC,OAAO,CACb,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AAvBW,QAAA,sBAAsB,0BAuBjC"}

package/dist/src/assertions/cost.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleCost: ({ cost, assertion }: AssertionParams) => GradingResult;
+//# sourceMappingURL=cost.d.ts.map

package/dist/src/assertions/cost.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.d.ts","sourceRoot":"","sources":["../../../src/assertions/cost.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,eAAO,MAAM,UAAU,wBAAyB,eAAe,KAAG,aAiBjE,CAAC"}

package/dist/src/assertions/cost.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleCost = void 0;
+const handleCost = ({ cost, assertion }) => {
+    if (!assertion.threshold) {
+        throw new Error('Cost assertion must have a threshold');
+    }
+    if (typeof cost === 'undefined') {
+        throw new Error('Cost assertion does not support providers that do not return cost');
+    }
+    const pass = cost <= assertion.threshold;
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Cost ${cost.toPrecision(2)} is greater than threshold ${assertion.threshold}`,
+        assertion,
+    };
+};
+exports.handleCost = handleCost;
+//# sourceMappingURL=cost.js.map

package/dist/src/assertions/cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cost.js","sourceRoot":"","sources":["../../../src/assertions/cost.ts"],"names":[],"mappings":";;;AAEO,MAAM,UAAU,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAmB,EAAiB,EAAE;IAChF,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IACD,IAAI,OAAO,IAAI,KAAK,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;IACvF,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,IAAI,SAAS,CAAC,SAAS,CAAC;IACzC,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,8BAA8B,SAAS,CAAC,SAAS,EAAE;QAClF,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AAjBW,QAAA,UAAU,cAiBrB"}

package/dist/src/assertions/equals.d.ts

@@ -0,0 +1,4 @@
+import type { AssertionParams } from '../types';
+import type { GradingResult } from '../types';
+export declare const handleEquals: ({ assertion, renderedValue, outputString, inverse, }: Pick<AssertionParams, "assertion" | "renderedValue" | "outputString" | "inverse">) => Promise<GradingResult>;
+//# sourceMappingURL=equals.d.ts.map

package/dist/src/assertions/equals.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"equals.d.ts","sourceRoot":"","sources":["../../../src/assertions/equals.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAChD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE9C,eAAO,MAAM,YAAY,yDAKtB,IAAI,CACL,eAAe,EACf,WAAW,GAAG,eAAe,GAAG,cAAc,GAAG,SAAS,CAC3D,KAAG,OAAO,CAAC,aAAa,CAqBxB,CAAC"}

package/dist/src/assertions/equals.js

@@ -0,0 +1,32 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleEquals = void 0;
+const util_1 = __importDefault(require("util"));
+const handleEquals = async ({ assertion, renderedValue, outputString, inverse, }) => {
+    let pass;
+    if (typeof renderedValue === 'object') {
+        try {
+            pass = util_1.default.isDeepStrictEqual(renderedValue, JSON.parse(outputString)) !== inverse;
+        }
+        catch {
+            pass = false;
+        }
+        renderedValue = JSON.stringify(renderedValue);
+    }
+    else {
+        pass = (String(renderedValue) === outputString) !== inverse;
+    }
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason: pass
+            ? 'Assertion passed'
+            : `Expected output "${outputString}" to ${inverse ? 'not ' : ''}equal "${renderedValue}"`,
+        assertion,
+    };
+};
+exports.handleEquals = handleEquals;
+//# sourceMappingURL=equals.js.map

package/dist/src/assertions/equals.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"equals.js","sourceRoot":"","sources":["../../../src/assertions/equals.ts"],"names":[],"mappings":";;;;;;AAAA,gDAAwB;AAIjB,MAAM,YAAY,GAAG,KAAK,EAAE,EACjC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,OAAO,GAIR,EAA0B,EAAE;IAC3B,IAAI,IAAa,CAAC;IAClB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,IAAI,GAAG,cAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,KAAK,OAAO,CAAC;QACrF,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,GAAG,KAAK,CAAC;QACf,CAAC;QACD,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,YAAY,CAAC,KAAK,OAAO,CAAC;IAC9D,CAAC;IAED,OAAO;QACL,IAAI;QACJ,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,kBAAkB;YACpB,CAAC,CAAC,oBAAoB,YAAY,QAAQ,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,aAAa,GAAG;QAC3F,SAAS;KACV,CAAC;AACJ,CAAC,CAAC;AA7BW,QAAA,YAAY,gBA6BvB"}

package/dist/src/assertions/factuality.d.ts

@@ -0,0 +1,3 @@
+import type { AssertionParams, GradingResult } from '../types';
+export declare const handleFactuality: ({ assertion, renderedValue, outputString, test, prompt, }: AssertionParams) => Promise<GradingResult>;
+//# sourceMappingURL=factuality.d.ts.map

package/dist/src/assertions/factuality.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"factuality.d.ts","sourceRoot":"","sources":["../../../src/assertions/factuality.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG/D,eAAO,MAAM,gBAAgB,8DAM1B,eAAe,KAAG,OAAO,CAAC,aAAa,CAiBzC,CAAC"}

package/dist/src/assertions/factuality.js

@@ -0,0 +1,25 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.handleFactuality = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const matchers_1 = require("../matchers");
+const templates_1 = require("../util/templates");
+const handleFactuality = async ({ assertion, renderedValue, outputString, test, prompt, }) => {
+    (0, tiny_invariant_1.default)(typeof renderedValue === 'string', 'factuality assertion type must have a string value');
+    (0, tiny_invariant_1.default)(prompt, 'factuality assertion type must have a prompt');
+    if (test.options?.rubricPrompt) {
+        // Substitute vars in prompt
+        (0, tiny_invariant_1.default)(typeof test.options.rubricPrompt === 'string', 'rubricPrompt must be a string');
+        const nunjucks = (0, templates_1.getNunjucksEngine)();
+        test.options.rubricPrompt = nunjucks.renderString(test.options.rubricPrompt, test.vars || {});
+    }
+    return {
+        assertion,
+        ...(await (0, matchers_1.matchesFactuality)(prompt, renderedValue, outputString, test.options, test.vars)),
+    };
+};
+exports.handleFactuality = handleFactuality;
+//# sourceMappingURL=factuality.js.map

package/dist/src/assertions/factuality.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"factuality.js","sourceRoot":"","sources":["../../../src/assertions/factuality.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AACvC,0CAAgD;AAEhD,iDAAsD;AAE/C,MAAM,gBAAgB,GAAG,KAAK,EAAE,EACrC,SAAS,EACT,aAAa,EACb,YAAY,EACZ,IAAI,EACJ,MAAM,GACU,EAA0B,EAAE;IAC5C,IAAA,wBAAS,EACP,OAAO,aAAa,KAAK,QAAQ,EACjC,oDAAoD,CACrD,CAAC;IACF,IAAA,wBAAS,EAAC,MAAM,EAAE,8CAA8C,CAAC,CAAC;IAClE,IAAI,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;QAC/B,4BAA4B;QAC5B,IAAA,wBAAS,EAAC,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,KAAK,QAAQ,EAAE,+BAA+B,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAG,IAAA,6BAAiB,GAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAChG,CAAC;IAED,OAAO;QACL,SAAS;QACT,GAAG,CAAC,MAAM,IAAA,4BAAiB,EAAC,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;KAC3F,CAAC;AACJ,CAAC,CAAC;AAvBW,QAAA,gBAAgB,oBAuB3B"}

package/dist/src/assertions/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/assertions/index.ts"],"names":[],"mappings":"AAaA,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,aAAa,CAAC;AASrB,OAAO,KAAK,EAAiC,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAChF,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,aAAa,EAEnB,MAAM,UAAU,CAAC;AAclB,eAAO,MAAM,4BAA4B,oBASvC,CAAC;AAqCH,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,SAAS,EACT,gBAAgB,GACjB,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,cAAc,CAAC;IACrB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CAs/BzB;AAED,wBAAsB,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,gBAAgB,EAChB,IAAI,EACJ,SAAS,GACV,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,EAAE,cAAc,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CAkFzB;AAED,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,aAAa,EAAE,CAAC,CAa1B;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAU3E;;;;;;;;;;;;;;;;AAGD,wBAcE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/assertions/index.ts"],"names":[],"mappings":"AASA,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACvB,0BAA0B,EAC1B,iBAAiB,EACjB,iBAAiB,EAClB,MAAM,aAAa,CAAC;AAIrB,OAAO,KAAK,EAIV,gBAAgB,EACjB,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,cAAc,EACnB,KAAK,aAAa,EACnB,MAAM,UAAU,CAAC;AA4ClB,eAAO,MAAM,4BAA4B,oBASvC,CAAC;AAIH,wBAAsB,YAAY,CAAC,EACjC,MAAM,EACN,QAAQ,EACR,SAAS,EACT,IAAI,EACJ,SAAS,EACT,gBAAgB,GACjB,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,IAAI,EAAE,cAAc,CAAC;IACrB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CAuKzB;AAED,wBAAsB,aAAa,CAAC,EAClC,MAAM,EACN,QAAQ,EACR,gBAAgB,EAChB,IAAI,EACJ,SAAS,GACV,EAAE;IACD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,IAAI,EAAE,cAAc,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAAC,aAAa,CAAC,CAkFzB;AAED,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,cAAc,EACpB,SAAS,EAAE,SAAS,EACpB,OAAO,EAAE,MAAM,EAAE,GAChB,OAAO,CAAC,aAAa,EAAE,CAAC,CAa1B;AAED,wBAAsB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAU3E;;;;;;;;;;;;;;;;AAGD,wBAcE"}