promptfoo 0.103.13 → 0.103.14

package/dist/test/assertions/contextRecall.test.js

@@ -0,0 +1,243 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const contextRecall_1 = require("../../src/assertions/contextRecall");
+const matchers = __importStar(require("../../src/matchers"));
+jest.mock('../../src/matchers');
+describe('handleContextRecall', () => {
+    const mockMatchesContextRecall = jest.spyOn(matchers, 'matchesContextRecall');
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should handle context recall with prompt context', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.9,
+            reason: 'Good recall',
+        };
+        mockMatchesContextRecall.mockResolvedValue(mockResult);
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall', threshold: 0.8 },
+            renderedValue: 'test output',
+            prompt: 'test context',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'test context',
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        const result = await (0, contextRecall_1.handleContextRecall)(params);
+        expect(result).toEqual({
+            assertion: { type: 'context-recall', threshold: 0.8 },
+            ...mockResult,
+        });
+        expect(mockMatchesContextRecall).toHaveBeenCalledWith('test context', 'test output', 0.8, {}, {});
+    });
+    it('should handle context recall with vars context', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.85,
+            reason: 'Good recall from vars',
+        };
+        mockMatchesContextRecall.mockResolvedValue(mockResult);
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall', threshold: 0.7 },
+            renderedValue: 'test output',
+            prompt: 'original context',
+            test: {
+                vars: { context: 'context from vars' },
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'original context',
+                vars: { context: 'context from vars' },
+                test: {
+                    vars: { context: 'context from vars' },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        const result = await (0, contextRecall_1.handleContextRecall)(params);
+        expect(result).toEqual({
+            assertion: { type: 'context-recall', threshold: 0.7 },
+            ...mockResult,
+        });
+        expect(mockMatchesContextRecall).toHaveBeenCalledWith('context from vars', 'test output', 0.7, {}, { context: 'context from vars' });
+    });
+    it('should use default threshold of 0 when not provided', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.5,
+            reason: 'Default threshold test',
+        };
+        mockMatchesContextRecall.mockResolvedValue(mockResult);
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall' },
+            renderedValue: 'test output',
+            prompt: 'test context',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'test context',
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        const result = await (0, contextRecall_1.handleContextRecall)(params);
+        expect(result).toEqual({
+            assertion: { type: 'context-recall' },
+            ...mockResult,
+        });
+        expect(mockMatchesContextRecall).toHaveBeenCalledWith('test context', 'test output', 0, {}, {});
+    });
+    it('should throw error when renderedValue is not a string', async () => {
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall' },
+            renderedValue: { value: 123 }, // Changed to object to match AssertionValue type
+            prompt: 'test context',
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: 'test context',
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: '123',
+            outputString: '123',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        await expect((0, contextRecall_1.handleContextRecall)(params)).rejects.toThrow('context-recall assertion type must have a string value');
+    });
+    it('should throw error when prompt is missing', async () => {
+        const mockProvider = {
+            id: () => 'test-provider',
+            callApi: jest.fn(),
+        };
+        const params = {
+            assertion: { type: 'context-recall' },
+            renderedValue: 'test output',
+            prompt: undefined,
+            test: {
+                vars: {},
+                options: {},
+            },
+            baseType: 'context-recall',
+            context: {
+                prompt: undefined,
+                vars: {},
+                test: {
+                    vars: {},
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: mockProvider,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            provider: mockProvider,
+            providerResponse: {},
+        };
+        await expect((0, contextRecall_1.handleContextRecall)(params)).rejects.toThrow('context-recall assertion type must have a prompt');
+    });
+});
+//# sourceMappingURL=contextRecall.test.js.map

package/dist/test/assertions/contextRecall.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRecall.test.js","sourceRoot":"","sources":["../../../test/assertions/contextRecall.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sEAAyE;AACzE,6DAA+C;AAG/C,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,MAAM,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAE9E,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QACF,wBAAwB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,GAAG,UAAU;SACd,CAAC,CAAC;QACH,MAAM,CAAC,wBAAwB,CAAC,CAAC,oBAAoB,CACnD,cAAc,EACd,aAAa,EACb,GAAG,EACH,EAAE,EACF,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,uBAAuB;SAChC,CAAC;QACF,wBAAwB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,kBAAkB;YAC1B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;gBACtC,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,IAAI,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;gBACtC,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;oBACtC,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,GAAG,UAAU;SACd,CAAC,CAAC;QACH,MAAM,CAAC,wBAAwB,CAAC,CAAC,oBAAoB,CACnD,mBAAmB,EACnB,aAAa,EACb,GAAG,EACH,EAAE,EACF,EAAE,OAAO,EAAE,mBAAmB,EAAE,CACjC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,wBAAwB;SACjC,CAAC;QACF,wBAAwB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,GAAG,UAAU;SACd,CAAC,CAAC;QACH,MAAM,CAAC,wBAAwB,CAAC,CAAC,oBAAoB,CAAC,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAClG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,aAAa,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,iDAAiD;YAChF,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,KAAK;YACb,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACvD,wDAAwD,CACzD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,SAAS;YACjB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACvD,kDAAkD,CACnD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/contextRelevance.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=contextRelevance.test.d.ts.map

package/dist/test/assertions/contextRelevance.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRelevance.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/contextRelevance.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/contextRelevance.test.js

@@ -0,0 +1,238 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const contextRelevance_1 = require("../../src/assertions/contextRelevance");
+const matchers_1 = require("../../src/matchers");
+jest.mock('../../src/matchers');
+describe('handleContextRelevance', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should handle valid input', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        };
+        jest.mocked(matchers_1.matchesContextRelevance).mockResolvedValue(mockResult);
+        const result = await (0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+                threshold: 0.7,
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+                options: {},
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(result).toEqual({
+            assertion: {
+                type: 'context-relevance',
+                threshold: 0.7,
+            },
+            ...mockResult,
+        });
+        expect(matchers_1.matchesContextRelevance).toHaveBeenCalledWith('test query', 'test context', 0.7, {});
+    });
+    it('should throw error if vars is missing', async () => {
+        await expect((0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {},
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('context-relevance assertion type must have a vars object');
+    });
+    it('should throw error if query is missing', async () => {
+        await expect((0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {
+                vars: {
+                    context: 'test context',
+                },
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('context-relevance assertion type must have a query var');
+    });
+    it('should throw error if context is missing', async () => {
+        await expect((0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                },
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('context-relevance assertion type must have a context var');
+    });
+    it('should use default threshold of 0 if not specified', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.5,
+            reason: 'test reason',
+        };
+        jest.mocked(matchers_1.matchesContextRelevance).mockResolvedValue(mockResult);
+        await (0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+                options: {},
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(matchers_1.matchesContextRelevance).toHaveBeenCalledWith('test query', 'test context', 0, {});
+    });
+});
+//# sourceMappingURL=contextRelevance.test.js.map

package/dist/test/assertions/contextRelevance.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRelevance.test.js","sourceRoot":"","sources":["../../../test/assertions/contextRelevance.test.ts"],"names":[],"mappings":";;AAAA,4EAA+E;AAC/E,iDAA6D;AAE7D,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,kCAAuB,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,MAAM,IAAA,yCAAsB,EAAC;YAC1C,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;gBACzB,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;gBACD,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CAAC;QAEV,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;gBACzB,SAAS,EAAE,GAAG;aACf;YACD,GAAG,UAAU;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,kCAAuB,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,CACV,IAAA,yCAAsB,EAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CACV,CAAC,OAAO,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,MAAM,CACV,IAAA,yCAAsB,EAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,OAAO,EAAE,cAAc;iBACxB;aACF;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CACV,CAAC,OAAO,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,CACV,IAAA,yCAAsB,EAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;iBACpB;aACF;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CACV,CAAC,OAAO,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,kCAAuB,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEnE,MAAM,IAAA,yCAAsB,EAAC;YAC3B,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;gBACD,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CAAC;QAEV,MAAM,CAAC,kCAAuB,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/geval.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=geval.test.d.ts.map

package/dist/test/assertions/geval.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"geval.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/geval.test.ts"],"names":[],"mappings":""}