promptfoo 0.103.13 → 0.103.15

package/dist/test/assertions/answerRelevance.test.js

@@ -0,0 +1,177 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const answerRelevance_1 = require("../../src/assertions/answerRelevance");
+const matchers_1 = require("../../src/matchers");
+const invariant_1 = __importDefault(require("../../src/util/invariant"));
+jest.mock('../../src/matchers');
+jest.mock('../../src/util/invariant');
+describe('handleAnswerRelevance', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+        jest.mocked(invariant_1.default).mockImplementation((condition, message) => {
+            if (!condition) {
+                throw new Error(typeof message === 'function' ? message() : message);
+            }
+        });
+    });
+    it('should call matchesAnswerRelevance with correct parameters', async () => {
+        const mockMatchesAnswerRelevance = jest.mocked(matchers_1.matchesAnswerRelevance);
+        mockMatchesAnswerRelevance.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        const result = await (0, answerRelevance_1.handleAnswerRelevance)({
+            assertion: {
+                type: 'answer-relevance',
+                threshold: 0.7,
+            },
+            output: 'test output',
+            prompt: 'test prompt',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'answer-relevance',
+            context: {},
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(mockMatchesAnswerRelevance).toHaveBeenCalledWith('test prompt', 'test output', 0.7, {});
+        expect(result).toEqual({
+            assertion: {
+                type: 'answer-relevance',
+                threshold: 0.7,
+            },
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+    });
+    it('should use query from vars if available', async () => {
+        const mockMatchesAnswerRelevance = jest.mocked(matchers_1.matchesAnswerRelevance);
+        mockMatchesAnswerRelevance.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        const result = await (0, answerRelevance_1.handleAnswerRelevance)({
+            assertion: {
+                type: 'answer-relevance',
+                threshold: 0.7,
+            },
+            output: 'test output',
+            prompt: 'test prompt',
+            test: {
+                vars: {
+                    query: 'test query',
+                },
+                options: {},
+            },
+            baseType: 'answer-relevance',
+            context: {},
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(mockMatchesAnswerRelevance).toHaveBeenCalledWith('test query', 'test output', 0.7, {});
+        expect(result).toEqual({
+            assertion: {
+                type: 'answer-relevance',
+                threshold: 0.7,
+            },
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+    });
+    it('should throw error if output is not string', async () => {
+        await expect((0, answerRelevance_1.handleAnswerRelevance)({
+            assertion: {
+                type: 'answer-relevance',
+            },
+            output: {},
+            prompt: 'test prompt',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'answer-relevance',
+            context: {},
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('answer-relevance assertion type must evaluate a string output');
+    });
+    it('should throw error if prompt is missing', async () => {
+        await expect((0, answerRelevance_1.handleAnswerRelevance)({
+            assertion: {
+                type: 'answer-relevance',
+            },
+            output: 'test output',
+            prompt: '',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'answer-relevance',
+            context: {},
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('answer-relevance assertion type must have a prompt');
+    });
+    it('should use default threshold of 0 if not specified', async () => {
+        const mockMatchesAnswerRelevance = jest.mocked(matchers_1.matchesAnswerRelevance);
+        mockMatchesAnswerRelevance.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        const result = await (0, answerRelevance_1.handleAnswerRelevance)({
+            assertion: {
+                type: 'answer-relevance',
+            },
+            output: 'test output',
+            prompt: 'test prompt',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'answer-relevance',
+            context: {},
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(mockMatchesAnswerRelevance).toHaveBeenCalledWith('test prompt', 'test output', 0, {});
+        expect(result).toEqual({
+            assertion: {
+                type: 'answer-relevance',
+            },
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+    });
+});
+//# sourceMappingURL=answerRelevance.test.js.map

package/dist/test/assertions/answerRelevance.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"answerRelevance.test.js","sourceRoot":"","sources":["../../../test/assertions/answerRelevance.test.ts"],"names":[],"mappings":";;;;;AAAA,0EAA6E;AAC7E,iDAA4D;AAE5D,yEAAiD;AAEjD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAChC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;AAEtC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,mBAAS,CAAC,CAAC,kBAAkB,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE;YAC/D,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,OAAO,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACvE,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,iCAAsB,CAAC,CAAC;QACvE,0BAA0B,CAAC,iBAAiB,CAAC;YAC3C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,uCAAqB,EAAC;YACzC,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;gBACxB,SAAS,EAAE,GAAG;aACf;YACD,MAAM,EAAE,aAAa;YACrB,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,kBAAkB;YAC5B,OAAO,EAAE,EAAmC;YAC5C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAAC,aAAa,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC/F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;gBACxB,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,iCAAsB,CAAC,CAAC;QACvE,0BAA0B,CAAC,iBAAiB,CAAC;YAC3C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,uCAAqB,EAAC;YACzC,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;gBACxB,SAAS,EAAE,GAAG;aACf;YACD,MAAM,EAAE,aAAa;YACrB,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;iBACpB;gBACD,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,kBAAkB;YAC5B,OAAO,EAAE,EAAmC;YAC5C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;gBACxB,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,MAAM,CACV,IAAA,uCAAqB,EAAC;YACpB,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;aACzB;YACD,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,kBAAkB;YAC5B,OAAO,EAAE,EAAmC;YAC5C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACF,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,+DAA+D,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;QACvD,MAAM,MAAM,CACV,IAAA,uCAAqB,EAAC;YACpB,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;aACzB;YACD,MAAM,EAAE,aAAa;YACrB,MAAM,EAAE,EAAE;YACV,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,kBAAkB;YAC5B,OAAO,EAAE,EAAmC;YAC5C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACF,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,0BAA0B,GAAG,IAAI,CAAC,MAAM,CAAC,iCAAsB,CAAC,CAAC;QACvE,0BAA0B,CAAC,iBAAiB,CAAC;YAC3C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,uCAAqB,EAAC;YACzC,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;aACzB;YACD,MAAM,EAAE,aAAa;YACrB,MAAM,EAAE,aAAa;YACrB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,kBAAkB;YAC5B,OAAO,EAAE,EAAmC;YAC5C,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAAC,aAAa,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7F,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,kBAAkB;aACzB;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/contextFaithfulness.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=contextFaithfulness.test.d.ts.map

package/dist/test/assertions/contextFaithfulness.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextFaithfulness.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/contextFaithfulness.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/contextFaithfulness.test.js

@@ -0,0 +1,226 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const contextFaithfulness_1 = require("../../src/assertions/contextFaithfulness");
+const matchers = __importStar(require("../../src/matchers"));
+jest.mock('../../src/matchers');
+describe('handleContextFaithfulness', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should throw error if vars is missing', async () => {
+        const params = {
+            assertion: { type: 'context-faithfulness' },
+            test: {},
+            output: 'test output',
+            baseType: 'context-faithfulness',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {},
+                logProbs: null,
+                tokenUsage: null,
+                cached: false,
+                provider: null,
+                providerResponse: null,
+            },
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: null,
+        };
+        await expect((0, contextFaithfulness_1.handleContextFaithfulness)(params)).rejects.toThrow('context-faithfulness assertion type must have a vars object');
+    });
+    it('should throw error if query is missing', async () => {
+        const params = {
+            assertion: { type: 'context-faithfulness' },
+            test: {
+                vars: {
+                    context: 'test context',
+                },
+            },
+            output: 'test output',
+            baseType: 'context-faithfulness',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {},
+                logProbs: null,
+                tokenUsage: null,
+                cached: false,
+                provider: null,
+                providerResponse: null,
+            },
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: null,
+        };
+        await expect((0, contextFaithfulness_1.handleContextFaithfulness)(params)).rejects.toThrow('context-faithfulness assertion type must have a query var');
+    });
+    it('should throw error if context is missing', async () => {
+        const params = {
+            assertion: { type: 'context-faithfulness' },
+            test: {
+                vars: {
+                    query: 'test query',
+                },
+            },
+            output: 'test output',
+            baseType: 'context-faithfulness',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {},
+                logProbs: null,
+                tokenUsage: null,
+                cached: false,
+                provider: null,
+                providerResponse: null,
+            },
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: null,
+        };
+        await expect((0, contextFaithfulness_1.handleContextFaithfulness)(params)).rejects.toThrow('context-faithfulness assertion type must have a context var');
+    });
+    it('should throw error if output is not a string', async () => {
+        const params = {
+            assertion: { type: 'context-faithfulness' },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+            },
+            output: 123,
+            baseType: 'context-faithfulness',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {},
+                logProbs: null,
+                tokenUsage: null,
+                cached: false,
+                provider: null,
+                providerResponse: null,
+            },
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: null,
+        };
+        await expect((0, contextFaithfulness_1.handleContextFaithfulness)(params)).rejects.toThrow('context-faithfulness assertion type must have a string output');
+    });
+    it('should call matchesContextFaithfulness with correct params', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.9,
+            reason: 'test reason',
+        };
+        jest.mocked(matchers.matchesContextFaithfulness).mockResolvedValue(mockResult);
+        const params = {
+            assertion: {
+                type: 'context-faithfulness',
+                threshold: 0.8,
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+                options: {
+                    provider: 'test-provider',
+                },
+            },
+            output: 'test output',
+            baseType: 'context-faithfulness',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {},
+                logProbs: null,
+                tokenUsage: null,
+                cached: false,
+                provider: null,
+                providerResponse: null,
+            },
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: null,
+        };
+        const result = await (0, contextFaithfulness_1.handleContextFaithfulness)(params);
+        expect(matchers.matchesContextFaithfulness).toHaveBeenCalledWith('test query', 'test output', 'test context', 0.8, { provider: 'test-provider' });
+        expect(result).toEqual({
+            assertion: params.assertion,
+            ...mockResult,
+        });
+    });
+    it('should use default threshold of 0 if not provided', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.9,
+            reason: 'test reason',
+        };
+        jest.mocked(matchers.matchesContextFaithfulness).mockResolvedValue(mockResult);
+        const params = {
+            assertion: {
+                type: 'context-faithfulness',
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+                options: {},
+            },
+            output: 'test output',
+            baseType: 'context-faithfulness',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {},
+                logProbs: null,
+                tokenUsage: null,
+                cached: false,
+                provider: null,
+                providerResponse: null,
+            },
+            inverse: false,
+            outputString: 'test output',
+            providerResponse: null,
+        };
+        await (0, contextFaithfulness_1.handleContextFaithfulness)(params);
+        expect(matchers.matchesContextFaithfulness).toHaveBeenCalledWith('test query', 'test output', 'test context', 0, {});
+    });
+});
+//# sourceMappingURL=contextFaithfulness.test.js.map

package/dist/test/assertions/contextFaithfulness.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextFaithfulness.test.js","sourceRoot":"","sources":["../../../test/assertions/contextFaithfulness.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kFAAqF;AACrF,6DAA+C;AAE/C,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;YAC3C,IAAI,EAAE,EAAE;YACR,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,sBAA+B;YACzC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE;gBACR,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,IAAI;aACvB;YACD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAa,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACpE,6DAA6D,CAC9D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;YAC3C,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,OAAO,EAAE,cAAc;iBACxB;aACF;YACD,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,sBAA+B;YACzC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE;gBACR,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,IAAI;aACvB;YACD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAa,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACpE,2DAA2D,CAC5D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;YAC3C,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;iBACpB;aACF;YACD,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,sBAA+B;YACzC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE;gBACR,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,IAAI;aACvB;YACD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAa,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACpE,6DAA6D,CAC9D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;YAC3C,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;aACF;YACD,MAAM,EAAE,GAAG;YACX,QAAQ,EAAE,sBAA+B;YACzC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE;gBACR,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,IAAI;aACvB;YACD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAa,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACpE,+DAA+D,CAChE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAE/E,MAAM,MAAM,GAAG;YACb,SAAS,EAAE;gBACT,IAAI,EAAE,sBAA+B;gBACrC,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;gBACD,OAAO,EAAE;oBACP,QAAQ,EAAE,eAAe;iBAC1B;aACF;YACD,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,sBAA+B;YACzC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE;gBACR,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,IAAI;aACvB;YACD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,+CAAyB,EAAC,MAAa,CAAC,CAAC;QAE9D,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAC9D,YAAY,EACZ,aAAa,EACb,cAAc,EACd,GAAG,EACH,EAAE,QAAQ,EAAE,eAAe,EAAE,CAC9B,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,GAAG,UAAU;SACd,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAE/E,MAAM,MAAM,GAAG;YACb,SAAS,EAAE;gBACT,IAAI,EAAE,sBAA+B;aACtC;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;gBACD,OAAO,EAAE,EAAE;aACZ;YACD,MAAM,EAAE,aAAa;YACrB,QAAQ,EAAE,sBAA+B;YACzC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE;gBACR,QAAQ,EAAE,IAAI;gBACd,UAAU,EAAE,IAAI;gBAChB,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,IAAI;gBACd,gBAAgB,EAAE,IAAI;aACvB;YACD,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE,IAAI;SACvB,CAAC;QAEF,MAAM,IAAA,+CAAyB,EAAC,MAAa,CAAC,CAAC;QAE/C,MAAM,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC,oBAAoB,CAC9D,YAAY,EACZ,aAAa,EACb,cAAc,EACd,CAAC,EACD,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/contextRecall.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=contextRecall.test.d.ts.map

package/dist/test/assertions/contextRecall.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRecall.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/contextRecall.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/contextRecall.test.js

@@ -0,0 +1,243 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const contextRecall_1 = require("../../src/assertions/contextRecall");
+const matchers = __importStar(require("../../src/matchers"));
+jest.mock('../../src/matchers');
+describe('handleContextRecall', () => {
+    const mockMatchesContextRecall = jest.spyOn(matchers, 'matchesContextRecall');
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should handle context recall with prompt context', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.9,
+            reason: 'Good recall',
+        };
+        mockMatchesContextRecall.mockResolvedValue(mockResult);
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall', threshold: 0.8 },
+            renderedValue: 'test output',
+            prompt: 'test context',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'test context',
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        const result = await (0, contextRecall_1.handleContextRecall)(params);
+        expect(result).toEqual({
+            assertion: { type: 'context-recall', threshold: 0.8 },
+            ...mockResult,
+        });
+        expect(mockMatchesContextRecall).toHaveBeenCalledWith('test context', 'test output', 0.8, {}, {});
+    });
+    it('should handle context recall with vars context', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.85,
+            reason: 'Good recall from vars',
+        };
+        mockMatchesContextRecall.mockResolvedValue(mockResult);
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall', threshold: 0.7 },
+            renderedValue: 'test output',
+            prompt: 'original context',
+            test: {
+                vars: { context: 'context from vars' },
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'original context',
+                vars: { context: 'context from vars' },
+                test: {
+                    vars: { context: 'context from vars' },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        const result = await (0, contextRecall_1.handleContextRecall)(params);
+        expect(result).toEqual({
+            assertion: { type: 'context-recall', threshold: 0.7 },
+            ...mockResult,
+        });
+        expect(mockMatchesContextRecall).toHaveBeenCalledWith('context from vars', 'test output', 0.7, {}, { context: 'context from vars' });
+    });
+    it('should use default threshold of 0 when not provided', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.5,
+            reason: 'Default threshold test',
+        };
+        mockMatchesContextRecall.mockResolvedValue(mockResult);
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall' },
+            renderedValue: 'test output',
+            prompt: 'test context',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'test context',
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        const result = await (0, contextRecall_1.handleContextRecall)(params);
+        expect(result).toEqual({
+            assertion: { type: 'context-recall' },
+            ...mockResult,
+        });
+        expect(mockMatchesContextRecall).toHaveBeenCalledWith('test context', 'test output', 0, {}, {});
+    });
+    it('should throw error when renderedValue is not a string', async () => {
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall' },
+            renderedValue: { value: 123 }, // Changed to object to match AssertionValue type
+            prompt: 'test context',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'test context',
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: '123',
+            outputString: '123',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        await expect((0, contextRecall_1.handleContextRecall)(params)).rejects.toThrow('context-recall assertion type must have a string value');
+    });
+    it('should throw error when prompt is missing', async () => {
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall' },
+            renderedValue: 'test output',
+            prompt: undefined,
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: undefined,
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        await expect((0, contextRecall_1.handleContextRecall)(params)).rejects.toThrow('context-recall assertion type must have a prompt');
+    });
+});
+//# sourceMappingURL=contextRecall.test.js.map