promptfoo 0.103.13 → 0.103.15

package/dist/test/assertions/contextRecall.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRecall.test.js","sourceRoot":"","sources":["../../../test/assertions/contextRecall.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sEAAyE;AACzE,6DAA+C;AAG/C,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,MAAM,wBAAwB,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAE9E,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;QAChE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QACF,wBAAwB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,GAAG,UAAU;SACd,CAAC,CAAC;QACH,MAAM,CAAC,wBAAwB,CAAC,CAAC,oBAAoB,CACnD,cAAc,EACd,aAAa,EACb,GAAG,EACH,EAAE,EACF,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,uBAAuB;SAChC,CAAC;QACF,wBAAwB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,kBAAkB;YAC1B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;gBACtC,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,kBAAkB;gBAC1B,IAAI,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;gBACtC,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE,OAAO,EAAE,mBAAmB,EAAE;oBACtC,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,SAAS,EAAE,GAAG,EAAE;YACrD,GAAG,UAAU;SACd,CAAC,CAAC;QACH,MAAM,CAAC,wBAAwB,CAAC,CAAC,oBAAoB,CACnD,mBAAmB,EACnB,aAAa,EACb,GAAG,EACH,EAAE,EACF,EAAE,OAAO,EAAE,mBAAmB,EAAE,CACjC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,wBAAwB;SACjC,CAAC;QACF,wBAAwB,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEvD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC;QAEjD,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,GAAG,UAAU;SACd,CAAC,CAAC;QACH,MAAM,CAAC,wBAAwB,CAAC,CAAC,oBAAoB,CAAC,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IAClG,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,aAAa,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,iDAAiD;YAChF,MAAM,EAAE,cAAc;YACtB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,cAAc;gBACtB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,KAAK;YACb,YAAY,EAAE,KAAK;YACnB,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACvD,wDAAwD,CACzD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,YAAY,GAAgB;YAChC,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;YACzB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;SACnB,CAAC;QAEF,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;YACrC,aAAa,EAAE,aAAa;YAC5B,MAAM,EAAE,SAAS;YACjB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,gBAAgB;YAC1B,OAAO,EAAE;gBACP,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,YAAY;gBACtB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,EAAsB;SACzC,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,mCAAmB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACvD,kDAAkD,CACnD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/contextRelevance.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=contextRelevance.test.d.ts.map

package/dist/test/assertions/contextRelevance.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRelevance.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/contextRelevance.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/contextRelevance.test.js

@@ -0,0 +1,238 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const contextRelevance_1 = require("../../src/assertions/contextRelevance");
+const matchers_1 = require("../../src/matchers");
+jest.mock('../../src/matchers');
+describe('handleContextRelevance', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should handle valid input', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        };
+        jest.mocked(matchers_1.matchesContextRelevance).mockResolvedValue(mockResult);
+        const result = await (0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+                threshold: 0.7,
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+                options: {},
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(result).toEqual({
+            assertion: {
+                type: 'context-relevance',
+                threshold: 0.7,
+            },
+            ...mockResult,
+        });
+        expect(matchers_1.matchesContextRelevance).toHaveBeenCalledWith('test query', 'test context', 0.7, {});
+    });
+    it('should throw error if vars is missing', async () => {
+        await expect((0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {},
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('context-relevance assertion type must have a vars object');
+    });
+    it('should throw error if query is missing', async () => {
+        await expect((0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {
+                vars: {
+                    context: 'test context',
+                },
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('context-relevance assertion type must have a query var');
+    });
+    it('should throw error if context is missing', async () => {
+        await expect((0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                },
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        })).rejects.toThrow('context-relevance assertion type must have a context var');
+    });
+    it('should use default threshold of 0 if not specified', async () => {
+        const mockResult = {
+            pass: true,
+            score: 0.5,
+            reason: 'test reason',
+        };
+        jest.mocked(matchers_1.matchesContextRelevance).mockResolvedValue(mockResult);
+        await (0, contextRelevance_1.handleContextRelevance)({
+            assertion: {
+                type: 'context-relevance',
+            },
+            test: {
+                vars: {
+                    query: 'test query',
+                    context: 'test context',
+                },
+                options: {},
+            },
+            baseType: 'context-relevance',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {
+                        query: 'test query',
+                        context: 'test context',
+                    },
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    config: {},
+                    callApi: jest.fn(),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    tokenUsage: {},
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            providerResponse: {
+                output: 'test output',
+                tokenUsage: {},
+            },
+        });
+        expect(matchers_1.matchesContextRelevance).toHaveBeenCalledWith('test query', 'test context', 0, {});
+    });
+});
+//# sourceMappingURL=contextRelevance.test.js.map

package/dist/test/assertions/contextRelevance.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contextRelevance.test.js","sourceRoot":"","sources":["../../../test/assertions/contextRelevance.test.ts"],"names":[],"mappings":";;AAAA,4EAA+E;AAC/E,iDAA6D;AAE7D,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,kCAAuB,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAG,MAAM,IAAA,yCAAsB,EAAC;YAC1C,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;gBACzB,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;gBACD,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CAAC;QAEV,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;gBACzB,SAAS,EAAE,GAAG;aACf;YACD,GAAG,UAAU;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,kCAAuB,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,cAAc,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,MAAM,MAAM,CACV,IAAA,yCAAsB,EAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CACV,CAAC,OAAO,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,MAAM,CACV,IAAA,yCAAsB,EAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,OAAO,EAAE,cAAc;iBACxB;aACF;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CACV,CAAC,OAAO,CAAC,OAAO,CAAC,wDAAwD,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,CACV,IAAA,yCAAsB,EAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;iBACpB;aACF;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CACV,CAAC,OAAO,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,UAAU,GAAG;YACjB,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC;QAEF,IAAI,CAAC,MAAM,CAAC,kCAAuB,CAAC,CAAC,iBAAiB,CAAC,UAAU,CAAC,CAAC;QAEnE,MAAM,IAAA,yCAAsB,EAAC;YAC3B,SAAS,EAAE;gBACT,IAAI,EAAE,mBAAmB;aAC1B;YACD,IAAI,EAAE;gBACJ,IAAI,EAAE;oBACJ,KAAK,EAAE,YAAY;oBACnB,OAAO,EAAE,cAAc;iBACxB;gBACD,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,mBAAmB;YAC7B,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE;wBACJ,KAAK,EAAE,YAAY;wBACnB,OAAO,EAAE,cAAc;qBACxB;oBACD,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;iBACnB;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,UAAU,EAAE,EAAE;iBACf;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,UAAU,EAAE,EAAE;aACf;SACK,CAAC,CAAC;QAEV,MAAM,CAAC,kCAAuB,CAAC,CAAC,oBAAoB,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/geval.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=geval.test.d.ts.map

package/dist/test/assertions/geval.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"geval.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/geval.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/geval.test.js

@@ -0,0 +1,222 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const geval_1 = require("../../src/assertions/geval");
+const matchers_1 = require("../../src/matchers");
+jest.mock('../../src/matchers');
+describe('handleGEval', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should handle string renderedValue', async () => {
+        const mockMatchesGEval = jest.mocked(matchers_1.matchesGEval);
+        mockMatchesGEval.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        const result = await (0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: 'test criteria',
+                threshold: 0.7,
+            },
+            renderedValue: 'test criteria',
+            prompt: 'test prompt',
+            outputString: 'test output',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            providerResponse: {
+                output: 'test output',
+                error: undefined,
+            },
+        });
+        expect(result).toEqual({
+            assertion: {
+                type: 'g-eval',
+                value: 'test criteria',
+                threshold: 0.7,
+            },
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        expect(mockMatchesGEval).toHaveBeenCalledWith('test criteria', 'test prompt', 'test output', 0.7, {});
+    });
+    it('should handle array renderedValue', async () => {
+        const mockMatchesGEval = jest.mocked(matchers_1.matchesGEval);
+        mockMatchesGEval.mockResolvedValueOnce({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason 1',
+        });
+        mockMatchesGEval.mockResolvedValueOnce({
+            pass: false,
+            score: 0.6,
+            reason: 'test reason 2',
+        });
+        const result = await (0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: ['criteria1', 'criteria2'],
+                threshold: 0.7,
+            },
+            renderedValue: ['criteria1', 'criteria2'],
+            prompt: 'test prompt',
+            outputString: 'test output',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            providerResponse: {
+                output: 'test output',
+                error: undefined,
+            },
+        });
+        expect(result).toEqual({
+            assertion: {
+                type: 'g-eval',
+                value: ['criteria1', 'criteria2'],
+                threshold: 0.7,
+            },
+            pass: true,
+            score: 0.7,
+            reason: 'test reason 2',
+        });
+    });
+    it('should use default threshold if not provided', async () => {
+        const mockMatchesGEval = jest.mocked(matchers_1.matchesGEval);
+        mockMatchesGEval.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        await (0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: 'test criteria',
+            },
+            renderedValue: 'test criteria',
+            prompt: 'test prompt',
+            outputString: 'test output',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            providerResponse: {
+                output: 'test output',
+                error: undefined,
+            },
+        });
+        expect(mockMatchesGEval).toHaveBeenCalledWith('test criteria', 'test prompt', 'test output', 0.7, {});
+    });
+    it('should throw error for invalid renderedValue type', async () => {
+        await expect((0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: 'test',
+            },
+            renderedValue: undefined,
+            prompt: 'test',
+            outputString: 'test',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test',
+            providerResponse: {
+                output: 'test',
+                error: undefined,
+            },
+        })).rejects.toThrow('G-Eval assertion type must have a string or array of strings value');
+    });
+});
+//# sourceMappingURL=geval.test.js.map

package/dist/test/assertions/geval.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geval.test.js","sourceRoot":"","sources":["../../../test/assertions/geval.test.ts"],"names":[],"mappings":";;AAAA,sDAAyD;AACzD,iDAAkD;AAElD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAY,CAAC,CAAC;QACnD,gBAAgB,CAAC,iBAAiB,CAAC;YACjC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAW,EAAC;YAC/B,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,eAAe;gBACtB,SAAS,EAAE,GAAG;aACf;YACD,aAAa,EAAE,eAAe;YAC9B,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,eAAe;gBACtB,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,eAAe,EACf,aAAa,EACb,aAAa,EACb,GAAG,EACH,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAY,CAAC,CAAC;QACnD,gBAAgB,CAAC,qBAAqB,CAAC;YACrC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QACH,gBAAgB,CAAC,qBAAqB,CAAC;YACrC,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAW,EAAC;YAC/B,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;gBACjC,SAAS,EAAE,GAAG;aACf;YACD,aAAa,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;YACzC,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;gBACjC,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAY,CAAC,CAAC;QACnD,gBAAgB,CAAC,iBAAiB,CAAC;YACjC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,IAAA,mBAAW,EAAC;YAChB,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,eAAe;aACvB;YACD,aAAa,EAAE,eAAe;YAC9B,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,eAAe,EACf,aAAa,EACb,aAAa,EACb,GAAG,EACH,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,CACV,IAAA,mBAAW,EAAC;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,MAAM;aACd;YACD,aAAa,EAAE,SAAS;YACxB,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,MAAM;YACpB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,MAAM;YACd,gBAAgB,EAAE;gBAChB,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC;IAC1F,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/modelGradedClosedQa.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=modelGradedClosedQa.test.d.ts.map

package/dist/test/assertions/modelGradedClosedQa.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelGradedClosedQa.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/modelGradedClosedQa.test.ts"],"names":[],"mappings":""}