promptfoo 0.103.13 → 0.103.14

package/dist/test/assertions/geval.test.js

@@ -0,0 +1,222 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const geval_1 = require("../../src/assertions/geval");
+const matchers_1 = require("../../src/matchers");
+jest.mock('../../src/matchers');
+describe('handleGEval', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+    });
+    it('should handle string renderedValue', async () => {
+        const mockMatchesGEval = jest.mocked(matchers_1.matchesGEval);
+        mockMatchesGEval.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        const result = await (0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: 'test criteria',
+                threshold: 0.7,
+            },
+            renderedValue: 'test criteria',
+            prompt: 'test prompt',
+            outputString: 'test output',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            providerResponse: {
+                output: 'test output',
+                error: undefined,
+            },
+        });
+        expect(result).toEqual({
+            assertion: {
+                type: 'g-eval',
+                value: 'test criteria',
+                threshold: 0.7,
+            },
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        expect(mockMatchesGEval).toHaveBeenCalledWith('test criteria', 'test prompt', 'test output', 0.7, {});
+    });
+    it('should handle array renderedValue', async () => {
+        const mockMatchesGEval = jest.mocked(matchers_1.matchesGEval);
+        mockMatchesGEval.mockResolvedValueOnce({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason 1',
+        });
+        mockMatchesGEval.mockResolvedValueOnce({
+            pass: false,
+            score: 0.6,
+            reason: 'test reason 2',
+        });
+        const result = await (0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: ['criteria1', 'criteria2'],
+                threshold: 0.7,
+            },
+            renderedValue: ['criteria1', 'criteria2'],
+            prompt: 'test prompt',
+            outputString: 'test output',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            providerResponse: {
+                output: 'test output',
+                error: undefined,
+            },
+        });
+        expect(result).toEqual({
+            assertion: {
+                type: 'g-eval',
+                value: ['criteria1', 'criteria2'],
+                threshold: 0.7,
+            },
+            pass: true,
+            score: 0.7,
+            reason: 'test reason 2',
+        });
+    });
+    it('should use default threshold if not provided', async () => {
+        const mockMatchesGEval = jest.mocked(matchers_1.matchesGEval);
+        mockMatchesGEval.mockResolvedValue({
+            pass: true,
+            score: 0.8,
+            reason: 'test reason',
+        });
+        await (0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: 'test criteria',
+            },
+            renderedValue: 'test criteria',
+            prompt: 'test prompt',
+            outputString: 'test output',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test output',
+            providerResponse: {
+                output: 'test output',
+                error: undefined,
+            },
+        });
+        expect(mockMatchesGEval).toHaveBeenCalledWith('test criteria', 'test prompt', 'test output', 0.7, {});
+    });
+    it('should throw error for invalid renderedValue type', async () => {
+        await expect((0, geval_1.handleGEval)({
+            assertion: {
+                type: 'g-eval',
+                value: 'test',
+            },
+            renderedValue: undefined,
+            prompt: 'test',
+            outputString: 'test',
+            test: {
+                vars: {},
+                assert: [],
+                options: {},
+            },
+            baseType: 'g-eval',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: {
+                    vars: {},
+                    assert: [],
+                    options: {},
+                },
+                logProbs: undefined,
+                provider: {
+                    id: () => 'test-provider',
+                    callApi: async () => ({ output: 'test' }),
+                },
+                providerResponse: {
+                    output: 'test output',
+                    error: undefined,
+                },
+            },
+            inverse: false,
+            output: 'test',
+            providerResponse: {
+                output: 'test',
+                error: undefined,
+            },
+        })).rejects.toThrow('G-Eval assertion type must have a string or array of strings value');
+    });
+});
+//# sourceMappingURL=geval.test.js.map

package/dist/test/assertions/geval.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geval.test.js","sourceRoot":"","sources":["../../../test/assertions/geval.test.ts"],"names":[],"mappings":";;AAAA,sDAAyD;AACzD,iDAAkD;AAElD,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAEhC,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;QAClD,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAY,CAAC,CAAC;QACnD,gBAAgB,CAAC,iBAAiB,CAAC;YACjC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAW,EAAC;YAC/B,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,eAAe;gBACtB,SAAS,EAAE,GAAG;aACf;YACD,aAAa,EAAE,eAAe;YAC9B,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,eAAe;gBACtB,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,eAAe,EACf,aAAa,EACb,aAAa,EACb,GAAG,EACH,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAY,CAAC,CAAC;QACnD,gBAAgB,CAAC,qBAAqB,CAAC;YACrC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QACH,gBAAgB,CAAC,qBAAqB,CAAC;YACrC,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,IAAA,mBAAW,EAAC;YAC/B,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;gBACjC,SAAS,EAAE,GAAG;aACf;YACD,aAAa,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;YACzC,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,CAAC,WAAW,EAAE,WAAW,CAAC;gBACjC,SAAS,EAAE,GAAG;aACf;YACD,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,eAAe;SACxB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;QAC5D,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,uBAAY,CAAC,CAAC;QACnD,gBAAgB,CAAC,iBAAiB,CAAC;YACjC,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,GAAG;YACV,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;QAEH,MAAM,IAAA,mBAAW,EAAC;YAChB,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,eAAe;aACvB;YACD,aAAa,EAAE,eAAe;YAC9B,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE;gBAChB,MAAM,EAAE,aAAa;gBACrB,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CAAC;QAEH,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAC3C,eAAe,EACf,aAAa,EACb,aAAa,EACb,GAAG,EACH,EAAE,CACH,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,MAAM,MAAM,CACV,IAAA,mBAAW,EAAC;YACV,SAAS,EAAE;gBACT,IAAI,EAAE,QAAQ;gBACd,KAAK,EAAE,MAAM;aACd;YACD,aAAa,EAAE,SAAS;YACxB,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,MAAM;YACpB,IAAI,EAAE;gBACJ,IAAI,EAAE,EAAE;gBACR,MAAM,EAAE,EAAE;gBACV,OAAO,EAAE,EAAE;aACZ;YACD,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE;oBACJ,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,EAAE;oBACV,OAAO,EAAE,EAAE;iBACZ;gBACD,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE;oBACR,EAAE,EAAE,GAAG,EAAE,CAAC,eAAe;oBACzB,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;iBAC1C;gBACD,gBAAgB,EAAE;oBAChB,MAAM,EAAE,aAAa;oBACrB,KAAK,EAAE,SAAS;iBACjB;aACF;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,MAAM;YACd,gBAAgB,EAAE;gBAChB,MAAM,EAAE,MAAM;gBACd,KAAK,EAAE,SAAS;aACjB;SACF,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC;IAC1F,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/assertions/modelGradedClosedQa.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=modelGradedClosedQa.test.d.ts.map

package/dist/test/assertions/modelGradedClosedQa.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelGradedClosedQa.test.d.ts","sourceRoot":"","sources":["../../../test/assertions/modelGradedClosedQa.test.ts"],"names":[],"mappings":""}

package/dist/test/assertions/modelGradedClosedQa.test.js

@@ -0,0 +1,200 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const modelGradedClosedQa_1 = require("../../src/assertions/modelGradedClosedQa");
+const matchers_1 = require("../../src/matchers");
+const templates_1 = require("../../src/util/templates");
+jest.mock('../../src/matchers');
+jest.mock('../../src/util/templates');
+describe('handleModelGradedClosedQa', () => {
+    beforeEach(() => {
+        jest.resetAllMocks();
+        const mockNunjucksEnv = {
+            options: { autoescape: true },
+            render: jest.fn(),
+            renderString: jest.fn().mockImplementation((str) => str),
+            addFilter: jest.fn(),
+            getFilter: jest.fn(),
+            hasExtension: jest.fn(),
+            addExtension: jest.fn(),
+            removeExtension: jest.fn(),
+            getExtension: jest.fn(),
+            addGlobal: jest.fn(),
+            getGlobal: jest.fn(),
+            getTemplate: jest.fn(),
+            express: jest.fn(),
+            on: jest.fn(),
+        };
+        jest.mocked(templates_1.getNunjucksEngine).mockReturnValue(mockNunjucksEnv);
+        jest.mocked(matchers_1.matchesClosedQa).mockResolvedValue({
+            pass: true,
+            score: 1,
+            reason: 'test reason',
+        });
+    });
+    it('should validate string value', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: { vars: {} },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: {},
+            test: {
+                options: {},
+                vars: {},
+            },
+        };
+        await expect((0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params)).rejects.toThrow('model-graded-closedqa assertion type must have a string value');
+    });
+    it('should validate prompt exists', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: undefined,
+                vars: {},
+                test: { vars: {} },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: undefined,
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {},
+                vars: {},
+            },
+        };
+        await expect((0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params)).rejects.toThrow('model-graded-closedqa assertion type must have a prompt');
+    });
+    it('should validate rubricPrompt is string if provided', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: {},
+                test: { vars: {} },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {
+                    rubricPrompt: {},
+                },
+                vars: {},
+            },
+        };
+        await expect((0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params)).rejects.toThrow('rubricPrompt must be a string');
+    });
+    it('should process rubricPrompt with nunjucks if provided', async () => {
+        const mockRenderString = jest.fn().mockReturnValue('rendered rubric');
+        const mockNunjucksEnv = {
+            options: { autoescape: true },
+            render: jest.fn(),
+            renderString: mockRenderString,
+            addFilter: jest.fn(),
+            getFilter: jest.fn(),
+            hasExtension: jest.fn(),
+            addExtension: jest.fn(),
+            removeExtension: jest.fn(),
+            getExtension: jest.fn(),
+            addGlobal: jest.fn(),
+            getGlobal: jest.fn(),
+            getTemplate: jest.fn(),
+            express: jest.fn(),
+            on: jest.fn(),
+        };
+        jest.mocked(templates_1.getNunjucksEngine).mockReturnValue(mockNunjucksEnv);
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: { var: 'value' },
+                test: { vars: { var: 'value' } },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {
+                    rubricPrompt: 'test rubric {{ var }}',
+                },
+                vars: {
+                    var: 'value',
+                },
+            },
+        };
+        await (0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params);
+        expect(mockRenderString).toHaveBeenCalledWith('test rubric {{ var }}', { var: 'value' });
+    });
+    it('should call matchesClosedQa with correct parameters', async () => {
+        const params = {
+            assertion: { type: 'model-graded-closedqa' },
+            baseType: 'model-graded-closedqa',
+            context: {
+                prompt: 'test prompt',
+                vars: { var: 'value' },
+                test: { vars: { var: 'value' } },
+                logProbs: undefined,
+                provider: undefined,
+                providerResponse: undefined,
+            },
+            inverse: false,
+            output: 'test output',
+            outputString: 'test output',
+            prompt: 'test prompt',
+            providerResponse: {},
+            renderedValue: 'test value',
+            test: {
+                options: {
+                    rubricPrompt: 'test rubric',
+                },
+                vars: {
+                    var: 'value',
+                },
+            },
+        };
+        const result = await (0, modelGradedClosedQa_1.handleModelGradedClosedQa)(params);
+        expect(matchers_1.matchesClosedQa).toHaveBeenCalledWith('test prompt', 'test value', 'test output', {
+            rubricPrompt: 'test rubric',
+        }, {
+            var: 'value',
+        });
+        expect(result).toEqual({
+            assertion: { type: 'model-graded-closedqa' },
+            pass: true,
+            score: 1,
+            reason: 'test reason',
+        });
+    });
+});
+//# sourceMappingURL=modelGradedClosedQa.test.js.map

package/dist/test/assertions/modelGradedClosedQa.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"modelGradedClosedQa.test.js","sourceRoot":"","sources":["../../../test/assertions/modelGradedClosedQa.test.ts"],"names":[],"mappings":";;AACA,kFAAqF;AACrF,iDAAqD;AAErD,wDAA6D;AAE7D,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;AAChC,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;AAEtC,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,eAAe,GAAG;YACtB,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE;YACjB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC;YACxD,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,eAAe,EAAE,IAAI,CAAC,EAAE,EAAE;YAC1B,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE;YACtB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;SACqB,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,6BAAiB,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,CAAC,0BAAe,CAAC,CAAC,iBAAiB,CAAC;YAC7C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;QAC5C,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBAClB,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,EAAE;YACjB,IAAI,EAAE;gBACJ,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,EAAE;aACT;SACF,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC7D,+DAA+D,CAChE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;QAC7C,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBAClB,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,SAAS;YACjB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,EAAE;gBACX,IAAI,EAAE,EAAE;aACT;SACF,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC7D,yDAAyD,CAC1D,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE;gBACR,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;gBAClB,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,YAAY,EAAE,EAAS;iBACxB;gBACD,IAAI,EAAE,EAAE;aACT;SACF,CAAC;QAEF,MAAM,MAAM,CAAC,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC7D,+BAA+B,CAChC,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,gBAAgB,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC;QACtE,MAAM,eAAe,GAAG;YACtB,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,EAAE,EAAE;YACjB,YAAY,EAAE,gBAAgB;YAC9B,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,eAAe,EAAE,IAAI,CAAC,EAAE,EAAE;YAC1B,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE;YACvB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE;YACpB,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE;YACtB,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE;YAClB,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE;SACqB,CAAC;QAErC,IAAI,CAAC,MAAM,CAAC,6BAAiB,CAAC,CAAC,eAAe,CAAC,eAAe,CAAC,CAAC;QAEhE,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;gBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;gBAChC,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,YAAY,EAAE,uBAAuB;iBACtC;gBACD,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO;iBACb;aACF;SACF,CAAC;QAEF,MAAM,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC;QAExC,MAAM,CAAC,gBAAgB,CAAC,CAAC,oBAAoB,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;QACnE,MAAM,MAAM,GAAoB;YAC9B,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,QAAQ,EAAE,uBAAuB;YACjC,OAAO,EAAE;gBACP,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE;gBACtB,IAAI,EAAE,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE;gBAChC,QAAQ,EAAE,SAAS;gBACnB,QAAQ,EAAE,SAAS;gBACnB,gBAAgB,EAAE,SAAS;aAC5B;YACD,OAAO,EAAE,KAAK;YACd,MAAM,EAAE,aAAa;YACrB,YAAY,EAAE,aAAa;YAC3B,MAAM,EAAE,aAAa;YACrB,gBAAgB,EAAE,EAAE;YACpB,aAAa,EAAE,YAAY;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE;oBACP,YAAY,EAAE,aAAa;iBAC5B;gBACD,IAAI,EAAE;oBACJ,GAAG,EAAE,OAAO;iBACb;aACF;SACF,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAA,+CAAyB,EAAC,MAAM,CAAC,CAAC;QAEvD,MAAM,CAAC,0BAAe,CAAC,CAAC,oBAAoB,CAC1C,aAAa,EACb,YAAY,EACZ,aAAa,EACb;YACE,YAAY,EAAE,aAAa;SAC5B,EACD;YACE,GAAG,EAAE,OAAO;SACb,CACF,CAAC;QAEF,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,SAAS,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;YAC5C,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,aAAa;SACtB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/fetch.test.js

@@ -30,7 +30,7 @@ jest.mock('../src/logger');
 jest.mock('../src/envars', () => ({
     getEnvString: jest.fn().mockImplementation((key, defaultValue = '') => ''),
     getEnvBool: jest.fn().mockImplementation((key, defaultValue = false) => false),
-    getEnvInt: jest.fn().mockReturnValue(100),
+    getEnvInt: jest.fn().mockImplementation((key, defaultValue = 0) => defaultValue),
 }));
 jest.mock('fs', () => ({
     readFileSync: jest.fn(),
@@ -266,7 +266,6 @@ describe('fetchWithProxy', () => {
         const mockCertContent = 'mock-cert-content';
         const mockProxyUrl = 'http://proxy.example.com';
         process.env.HTTPS_PROXY = mockProxyUrl;
-        // Update basePath in cliState
         cliState_1.default.basePath = mockBasePath;
         jest.mocked(envars_1.getEnvString).mockImplementation((key, defaultValue = '') => {
             if (key === 'PROMPTFOO_CA_CERT_PATH') {
@@ -307,7 +306,6 @@ describe('fetchWithProxy', () => {
             },
         });
         expect(undici_1.setGlobalDispatcher).toHaveBeenCalledWith(expect.any(undici_1.ProxyAgent));
-        // Reset basePath
         cliState_1.default.basePath = undefined;
     });
     it('should not create ProxyAgent when no proxy URL is found', async () => {
@@ -324,6 +322,10 @@ describe('fetchWithProxy', () => {
         };
         const allProxyVars = ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy'];
         const testCases = [
+            {
+                env: { HTTPS_PROXY: mockProxyUrls.HTTPS_PROXY },
+                expected: { url: mockProxyUrls.HTTPS_PROXY },
+            },
             {
                 env: { https_proxy: mockProxyUrls.https_proxy },
                 expected: { url: mockProxyUrls.https_proxy },
@@ -342,10 +344,6 @@ describe('fetchWithProxy', () => {
             allProxyVars.forEach((key) => {
                 delete process.env[key];
             });
-            const existingProxyVars = allProxyVars.filter((key) => process.env[key]);
-            if (existingProxyVars.length > 0) {
-                console.warn('Found unexpected proxy variables:', existingProxyVars);
-            }
             Object.entries(testCase.env).forEach(([key, value]) => {
                 process.env[key] = value;
             });
@@ -361,12 +359,15 @@ describe('fetchWithProxy', () => {
             });
             expect(undici_1.setGlobalDispatcher).toHaveBeenCalledWith(expect.any(undici_1.ProxyAgent));
             const debugCalls = jest.mocked(logger_1.default.debug).mock.calls;
-            expect(debugCalls).toEqual(expect.arrayContaining([
-                [
-                    expect.stringMatching(new RegExp(`Found proxy configuration in .*https?_proxy.*: ${testCase.expected.url}/`, 'i')),
-                ],
-                [`Using proxy: ${testCase.expected.url}/`],
-            ]));
+            const normalizedCalls = debugCalls.map((call) => call[0].replace(/\/$/, ''));
+            // On Windows, environment variables are case-insensitive
+            const isWindows = process.platform === 'win32';
+            const expectedEnvVar = Object.keys(testCase.env)[0];
+            const expectedUrl = testCase.expected.url;
+            expect(normalizedCalls).toEqual([
+                expect.stringMatching(new RegExp(`Found proxy configuration in ${isWindows ? '.*' : expectedEnvVar}: ${expectedUrl}`, isWindows ? 'i' : '')),
+                `Using proxy: ${expectedUrl}`,
+            ]);
             allProxyVars.forEach((key) => {
                 delete process.env[key];
             });
@@ -508,14 +509,14 @@ describe('fetchWithRetries', () => {
     it('should make retries+1 total attempts', async () => {
         jest.mocked(global.fetch).mockRejectedValue(new Error('Network error'));
         await expect((0, fetch_1.fetchWithRetries)('https://example.com', {}, 1000, 2)).rejects.toThrow('Request failed after 2 retries: Network error');
-        expect(global.fetch).toHaveBeenCalledTimes(3); // Initial attempt + 2 retries
-        expect(time_1.sleep).toHaveBeenCalledTimes(2); // Should sleep between attempts, but not after last attempt
+        expect(global.fetch).toHaveBeenCalledTimes(3);
+        expect(time_1.sleep).toHaveBeenCalledTimes(2);
     });
     it('should not sleep after the final attempt', async () => {
         jest.mocked(global.fetch).mockRejectedValue(new Error('Network error'));
         await expect((0, fetch_1.fetchWithRetries)('https://example.com', {}, 1000, 1)).rejects.toThrow('Request failed after 1 retries: Network error');
-        expect(global.fetch).toHaveBeenCalledTimes(2); // Initial attempt + 1 retry
-        expect(time_1.sleep).toHaveBeenCalledTimes(1); // Should only sleep once between attempts
+        expect(global.fetch).toHaveBeenCalledTimes(2);
+        expect(time_1.sleep).toHaveBeenCalledTimes(1);
     });
     it('should handle 5XX errors when PROMPTFOO_RETRY_5XX is true', async () => {
         jest.mocked(envars_1.getEnvBool).mockImplementation((key) => {
@@ -560,8 +561,55 @@ describe('fetchWithRetries', () => {
         const mockFetch = jest.fn().mockRejectedValue(new Error('Network error'));
         global.fetch = mockFetch;
         await expect((0, fetch_1.fetchWithRetries)('https://example.com', {}, 1000, 2)).rejects.toThrow('Request failed after 2 retries: Network error');
-        expect(mockFetch).toHaveBeenCalledTimes(3); // Initial attempt + 2 retries
+        expect(mockFetch).toHaveBeenCalledTimes(3);
         expect(time_1.sleep).toHaveBeenCalledTimes(2);
     });
+    it('should handle detailed error information', async () => {
+        const error = new Error('Network error');
+        error.code = 'ECONNREFUSED';
+        error.cause = 'Connection refused';
+        jest.mocked(global.fetch).mockRejectedValue(error);
+        await expect((0, fetch_1.fetchWithRetries)('https://example.com', {}, 1000, 1)).rejects.toThrow('Request failed after 1 retries: Network error');
+        expect(global.fetch).toHaveBeenCalledTimes(2);
+        expect(time_1.sleep).toHaveBeenCalledTimes(1);
+    });
+    it('should handle non-Error objects in rejection', async () => {
+        jest.mocked(global.fetch).mockRejectedValue('String error');
+        await expect((0, fetch_1.fetchWithRetries)('https://example.com', {}, 1000, 1)).rejects.toThrow('Request failed after 1 retries: undefined');
+        expect(global.fetch).toHaveBeenCalledTimes(2);
+        expect(time_1.sleep).toHaveBeenCalledTimes(1);
+    });
+    it('should handle rate limits with OpenAI specific headers', async () => {
+        const rateLimitedResponse = (0, utils_1.createMockResponse)({
+            status: 429,
+            headers: new Headers({
+                'x-ratelimit-reset-tokens': '5',
+            }),
+        });
+        const successResponse = (0, utils_1.createMockResponse)();
+        const mockFetch = jest
+            .fn()
+            .mockResolvedValueOnce(rateLimitedResponse)
+            .mockResolvedValueOnce(successResponse);
+        global.fetch = mockFetch;
+        await (0, fetch_1.fetchWithRetries)('https://example.com', {}, 1000, 2);
+        expect(mockFetch).toHaveBeenCalledTimes(2);
+        expect(logger_1.default.debug).toHaveBeenCalledWith(expect.stringContaining('Rate limited on URL'));
+        expect(time_1.sleep).toHaveBeenCalledTimes(1);
+    });
+});
+describe('sanitizeUrl', () => {
+    it('should mask credentials in URLs', () => {
+        const url = 'https://username:password@example.com/api';
+        expect((0, fetch_1.sanitizeUrl)(url)).toBe('https://***:***@example.com/api');
+    });
+    it('should handle URLs without credentials', () => {
+        const url = 'https://example.com/api';
+        expect((0, fetch_1.sanitizeUrl)(url)).toBe(url);
+    });
+    it('should return original string for invalid URLs', () => {
+        const invalidUrl = 'not-a-url';
+        expect((0, fetch_1.sanitizeUrl)(invalidUrl)).toBe(invalidUrl);
+    });
 });
 //# sourceMappingURL=fetch.test.js.map