promptfoo 0.103.2 → 0.103.4

package/dist/test/assertions/index.test.js

@@ -38,15 +38,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const dedent_1 = __importDefault(require("dedent"));
 const fs = __importStar(require("fs"));
-const node_module_1 = require("node:module");
 const path = __importStar(require("path"));
 const assertions_1 = require("../../src/assertions");
 const xml_1 = require("../../src/assertions/xml");
 const fetch_1 = require("../../src/fetch");
 const openai_1 = require("../../src/providers/openai");
 const replicate_1 = require("../../src/providers/replicate");
-const pythonUtils_1 = require("../../src/python/pythonUtils");
-const wrapper_1 = require("../../src/python/wrapper");
 const utils_1 = require("../util/utils");
 jest.mock('../../src/redteam/remoteGeneration', () => ({
     shouldGenerateRemote: jest.fn().mockReturnValue(false),
@@ -69,20 +66,6 @@ jest.mock('../../src/fetch', () => {
         fetchWithRetries: jest.fn(actual.fetchWithRetries),
     };
 });
-jest.mock('../../src/python/wrapper', () => {
-    const actual = jest.requireActual('../../src/python/wrapper');
-    return {
-        ...actual,
-        runPythonCode: jest.fn(actual.runPythonCode),
-    };
-});
-jest.mock('../../src/python/pythonUtils', () => {
-    const actual = jest.requireActual('../../src/python/pythonUtils');
-    return {
-        ...actual,
-        runPython: jest.fn(actual.runPython),
-    };
-});
 jest.mock('glob', () => ({
     globSync: jest.fn(),
 }));
@@ -551,22 +534,6 @@ describe('runAssertion', () => {
         type: 'javascript',
         value: 'output === "Expected output"',
     };
-    const javascriptMultilineStringAssertion = {
-        type: 'javascript',
-        value: `
-      if (output === "Expected output") {
-        return {
-          pass: true,
-          score: 0.5,
-          reason: 'Assertion passed',
-        };
-      }
-      return {
-        pass: false,
-        score: 0,
-        reason: 'Assertion failed',
-      };`,
-    };
     const javascriptStringAssertionWithNumber = {
         type: 'javascript',
         value: 'output.length * 10',
@@ -578,29 +545,15 @@ describe('runAssertion', () => {
             maximumOutputSize: 20,
         },
     };
+    const javascriptBooleanAssertionWithoutConfig = {
+        type: 'javascript',
+        value: 'output.length <= context.config.maximumOutputSize',
+    };
     const javascriptStringAssertionWithNumberAndThreshold = {
         type: 'javascript',
         value: 'output.length * 10',
         threshold: 0.5,
     };
-    const javascriptFunctionAssertion = {
-        type: 'javascript',
-        value: async (output) => ({
-            pass: true,
-            score: 0.5,
-            reason: 'Assertion passed',
-            assertion: null,
-        }),
-    };
-    const javascriptFunctionFailAssertion = {
-        type: 'javascript',
-        value: async (output) => ({
-            pass: false,
-            score: 0.5,
-            reason: 'Assertion failed',
-            assertion: null,
-        }),
-    };
     it('should pass when the equality assertion passes', async () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
@@ -1419,6 +1372,58 @@ describe('runAssertion', () => {
             reason: 'Assertion passed',
         });
     });
+    it('should disregard invalid inputs for assert index', async () => {
+        const output = 'Expected output';
+        const result = await (0, assertions_1.runAssertion)({
+            prompt: 'Some prompt',
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            assertion: javascriptBooleanAssertionWithConfig,
+            test: {
+                assert: [
+                    {
+                        type: 'javascript',
+                        value: 'output.length <= context.config.maximumOutputSize',
+                        config: {
+                            maximumOutputSize: 1,
+                        },
+                    },
+                ],
+            },
+            providerResponse: { output },
+            assertIndex: 45,
+        });
+        expect(result).toMatchObject({
+            pass: true,
+            score: 1.0,
+            reason: 'Assertion passed',
+        });
+    });
+    it('should correctly set configuration from the test case when assert index is valid', async () => {
+        const output = 'Expected output';
+        const result = await (0, assertions_1.runAssertion)({
+            prompt: 'Some prompt',
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            assertion: javascriptBooleanAssertionWithoutConfig,
+            test: {
+                assert: [
+                    {
+                        type: 'javascript',
+                        value: 'output.length <= context.config.maximumOutputSize',
+                        config: {
+                            maximumOutputSize: 50,
+                        },
+                    },
+                ],
+            },
+            providerResponse: { output },
+            assertIndex: 0,
+        });
+        expect(result).toMatchObject({
+            pass: true,
+            score: 1.0,
+            reason: 'Assertion passed',
+        });
+    });
     it('should fail when javascript returns an output string that is larger than the maximum size threshold', async () => {
         const output = 'Expected output with some extra characters';
         const result = await (0, assertions_1.runAssertion)({
@@ -1515,100 +1520,6 @@ describe('runAssertion', () => {
             reason: 'Assertion passed',
         });
     });
-    it('should pass when javascript function assertion passes - with vars', async () => {
-        const output = 'Expected output';
-        const javascriptStringAssertionWithVars = {
-            type: 'javascript',
-            value: 'output === "Expected output" && context.vars.foo === "bar"',
-        };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
-            assertion: javascriptStringAssertionWithVars,
-            test: { vars: { foo: 'bar' } },
-            providerResponse: { output },
-        });
-        expect(result).toMatchObject({
-            pass: true,
-            reason: 'Assertion passed',
-        });
-    });
-    it('should fail when the javascript does not match vars', async () => {
-        const output = 'Expected output';
-        const javascriptStringAssertionWithVars = {
-            type: 'javascript',
-            value: 'output === "Expected output" && context.vars.foo === "something else"',
-        };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
-            assertion: javascriptStringAssertionWithVars,
-            test: { vars: { foo: 'bar' } },
-            providerResponse: { output },
-        });
-        expect(result).toMatchObject({
-            pass: false,
-            reason: 'Custom function returned false\noutput === "Expected output" && context.vars.foo === "something else"',
-        });
-    });
-    it('should pass when the function returns pass', async () => {
-        const output = 'Expected output';
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
-            assertion: javascriptFunctionAssertion,
-            test: {},
-            providerResponse: { output },
-        });
-        expect(result).toMatchObject({
-            pass: true,
-            score: 0.5,
-            reason: 'Assertion passed',
-        });
-    });
-    it('should fail when the function returns fail', async () => {
-        const output = 'Expected output';
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
-            assertion: javascriptFunctionFailAssertion,
-            test: {},
-            providerResponse: { output },
-        });
-        expect(result).toMatchObject({
-            pass: false,
-            score: 0.5,
-            reason: 'Assertion failed',
-        });
-    });
-    it('should pass when the multiline javascript assertion passes', async () => {
-        const output = 'Expected output';
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            assertion: javascriptMultilineStringAssertion,
-            test: {},
-            providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
-        });
-        expect(result).toMatchObject({
-            pass: true,
-            reason: 'Assertion passed',
-        });
-    });
-    it('should pass when the multiline javascript assertion fails', async () => {
-        const output = 'Not the expected output';
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            assertion: javascriptMultilineStringAssertion,
-            test: {},
-            providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
-        });
-        expect(result).toMatchObject({
-            pass: false,
-            reason: 'Assertion failed',
-        });
-    });
     const notContainsAssertion = {
         type: 'not-contains',
         value: 'Unexpected output',
@@ -2103,336 +2014,6 @@ describe('runAssertion', () => {
             reason: 'Levenshtein distance 8 is greater than threshold 5',
         });
     });
-    it.each([
-        [
-            'boolean',
-            jest.fn((output) => output === 'Expected output'),
-            true,
-            'Assertion passed',
-        ],
-        ['number', jest.fn((output) => output.length), true, 'Assertion passed'],
-        [
-            'GradingResult',
-            jest.fn((output) => ({ pass: true, score: 1, reason: 'Custom reason' })),
-            true,
-            'Custom reason',
-        ],
-        [
-            'boolean',
-            jest.fn((output) => output !== 'Expected output'),
-            false,
-            'Custom function returned false',
-        ],
-        ['number', jest.fn((output) => 0), false, 'Custom function returned false'],
-        [
-            'GradingResult',
-            jest.fn((output) => ({ pass: false, score: 0.1, reason: 'Custom reason' })),
-            false,
-            'Custom reason',
-        ],
-        [
-            'boolean Promise',
-            jest.fn((output) => Promise.resolve(true)),
-            true,
-            'Assertion passed',
-        ],
-    ])('should pass when the file:// assertion with .js file returns a %s', async (type, mockFn, expectedPass, expectedReason) => {
-        const output = 'Expected output';
-        jest.doMock(path.resolve('/path/to/assert.js'), () => mockFn, { virtual: true });
-        const fileAssertion = {
-            type: 'javascript',
-            value: 'file:///path/to/assert.js',
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider,
-            assertion: fileAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(mockFn).toHaveBeenCalledWith('Expected output', {
-            prompt: 'Some prompt',
-            vars: {},
-            test: {},
-            provider,
-            providerResponse,
-        });
-        expect(result).toMatchObject({
-            pass: expectedPass,
-            reason: expect.stringContaining(expectedReason),
-        });
-    });
-    it.each([
-        [
-            'boolean',
-            jest.fn((output) => output === 'Expected output'),
-            true,
-            'Assertion passed',
-        ],
-        ['number', jest.fn((output) => output.length), true, 'Assertion passed'],
-        [
-            'GradingResult',
-            jest.fn((output) => ({ pass: true, score: 1, reason: 'Custom reason' })),
-            true,
-            'Custom reason',
-        ],
-        [
-            'boolean',
-            jest.fn((output) => output !== 'Expected output'),
-            false,
-            'Custom function returned false',
-        ],
-        ['number', jest.fn((output) => 0), false, 'Custom function returned false'],
-        [
-            'GradingResult',
-            jest.fn((output) => ({ pass: false, score: 0.1, reason: 'Custom reason' })),
-            false,
-            'Custom reason',
-        ],
-        [
-            'boolean Promise',
-            jest.fn((output) => Promise.resolve(true)),
-            true,
-            'Assertion passed',
-        ],
-    ])('should pass when assertion is a package path', async (type, mockFn, expectedPass, expectedReason) => {
-        const output = 'Expected output';
-        const require = (0, node_module_1.createRequire)('');
-        jest.spyOn(require, 'resolve').mockReturnValueOnce('/node_modules/@promptfoo/fake/index.js');
-        jest.doMock(path.resolve('/node_modules/@promptfoo/fake/index.js'), () => {
-            return {
-                assertionFunction: mockFn,
-            };
-        }, { virtual: true });
-        const fileAssertion = {
-            type: 'javascript',
-            value: 'package:@promptfoo/fake:assertionFunction',
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider,
-            assertion: fileAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(mockFn).toHaveBeenCalledWith('Expected output', {
-            prompt: 'Some prompt',
-            vars: {},
-            test: {},
-            provider,
-            providerResponse,
-        });
-        expect(result).toMatchObject({
-            pass: expectedPass,
-            reason: expect.stringContaining(expectedReason),
-        });
-    });
-    it('should resolve js paths relative to the configuration file', async () => {
-        const output = 'Expected output';
-        const mockFn = jest.fn((output) => output === 'Expected output');
-        jest.doMock(path.resolve('/base/path/path/to/assert.js'), () => mockFn, { virtual: true });
-        const fileAssertion = {
-            type: 'javascript',
-            value: 'file://./path/to/assert.js',
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider,
-            assertion: fileAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(mockFn).toHaveBeenCalledWith('Expected output', {
-            prompt: 'Some prompt',
-            vars: {},
-            test: {},
-            provider,
-            providerResponse,
-        });
-        expect(result).toMatchObject({
-            pass: true,
-            reason: 'Assertion passed',
-        });
-    });
-    it('should handle output strings with both single and double quotes correctly in python assertion', async () => {
-        const expectedPythonValue = '0.5';
-        jest.mocked(wrapper_1.runPythonCode).mockResolvedValueOnce(expectedPythonValue);
-        const output = 'This is a string with "double quotes"\n and \'single quotes\' \n\n and some \n\t newlines.';
-        const pythonAssertion = {
-            type: 'python',
-            value: expectedPythonValue,
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider,
-            assertion: pythonAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(wrapper_1.runPythonCode).toHaveBeenCalledTimes(1);
-        expect(wrapper_1.runPythonCode).toHaveBeenCalledWith(expect.anything(), 'main', [
-            output,
-            { prompt: 'Some prompt', test: {}, vars: {}, provider, providerResponse },
-        ]);
-        expect(result).toMatchObject({
-            pass: true,
-            reason: 'Assertion passed',
-            score: Number(expectedPythonValue),
-        });
-    });
-    it.each([
-        ['boolean', false, 0, 'Python code returned false', false, undefined],
-        ['number', 0, 0, 'Python code returned false', false, undefined],
-        [
-            'GradingResult',
-            `{"pass": false, "score": 0, "reason": "Custom error"}`,
-            0,
-            'Custom error',
-            false,
-            undefined,
-        ],
-        ['boolean', true, 1, 'Assertion passed', true, undefined],
-        ['number', 1, 1, 'Assertion passed', true, undefined],
-        [
-            'GradingResult',
-            `{"pass": true, "score": 1, "reason": "Custom success"}`,
-            1,
-            'Custom success',
-            true,
-            undefined,
-        ],
-        [
-            'GradingResult',
-            // This score is less than the assertion threshold in the test
-            `{"pass": true, "score": 0.4, "reason": "Foo bar"}`,
-            0.4,
-            'Python score 0.4 is less than threshold 0.5',
-            false,
-            0.5,
-        ],
-    ])('should handle inline return type %s with return value: %p', async (type, returnValue, expectedScore, expectedReason, expectedPass, threshold) => {
-        const output = 'This is a string with "double quotes"\n and \'single quotes\' \n\n and some \n\t newlines.';
-        let resolvedValue;
-        if (type === 'GradingResult') {
-            resolvedValue = JSON.parse(returnValue);
-        }
-        else {
-            resolvedValue = returnValue;
-        }
-        const pythonAssertion = {
-            type: 'python',
-            value: returnValue.toString(),
-            threshold,
-        };
-        jest.mocked(wrapper_1.runPythonCode).mockResolvedValueOnce(resolvedValue);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt',
-            provider,
-            assertion: pythonAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(wrapper_1.runPythonCode).toHaveBeenCalledTimes(1);
-        expect(wrapper_1.runPythonCode).toHaveBeenCalledWith(expect.anything(), 'main', [
-            output,
-            { prompt: 'Some prompt', test: {}, vars: {}, provider, providerResponse },
-        ]);
-        expect(result).toMatchObject({
-            pass: expectedPass,
-            reason: expect.stringMatching(expectedReason),
-            score: expectedScore,
-        });
-    });
-    it.each([
-        ['boolean', 'True', true, 'Assertion passed'],
-        ['number', '0.5', true, 'Assertion passed'],
-        ['boolean', true, true, 'Assertion passed'],
-        ['number', 0.5, true, 'Assertion passed'],
-        [
-            'GradingResult',
-            '{"pass": true, "score": 1, "reason": "Custom reason"}',
-            true,
-            'Custom reason',
-        ],
-        ['boolean', 'False', false, 'Python code returned false'],
-        ['number', '0', false, 'Python code returned false'],
-        [
-            'GradingResult',
-            '{"pass": false, "score": 0, "reason": "Custom reason"}',
-            false,
-            'Custom reason',
-        ],
-    ])('should handle when the file:// assertion with .py file returns a %s', async (type, pythonOutput, expectedPass, expectedReason) => {
-        const output = 'Expected output';
-        jest.mocked(pythonUtils_1.runPython).mockResolvedValueOnce(pythonOutput);
-        const fileAssertion = {
-            type: 'python',
-            value: 'file:///path/to/assert.py',
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider,
-            assertion: fileAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(pythonUtils_1.runPython).toHaveBeenCalledWith(path.resolve('/path/to/assert.py'), 'get_assert', [
-            output,
-            {
-                prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-                vars: {},
-                test: {},
-                provider,
-                providerResponse,
-            },
-        ]);
-        expect(result).toMatchObject({
-            pass: expectedPass,
-            reason: expect.stringContaining(expectedReason),
-        });
-        expect(pythonUtils_1.runPython).toHaveBeenCalledTimes(1);
-    });
-    it('should handle when python file assertions throw an error', async () => {
-        const output = 'Expected output';
-        jest
-            .mocked(pythonUtils_1.runPython)
-            .mockRejectedValue(new Error('The Python script `call_api` function must return a dict with an `output`'));
-        const fileAssertion = {
-            type: 'python',
-            value: 'file:///path/to/assert.py',
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const providerResponse = { output };
-        const result = await (0, assertions_1.runAssertion)({
-            prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider,
-            assertion: fileAssertion,
-            test: {},
-            providerResponse,
-        });
-        expect(pythonUtils_1.runPython).toHaveBeenCalledTimes(1);
-        expect(result).toEqual({
-            assertion: {
-                type: 'python',
-                value: 'file:///path/to/assert.py',
-            },
-            pass: false,
-            reason: 'The Python script `call_api` function must return a dict with an `output`',
-            score: 0,
-        });
-    });
     describe('latency assertion', () => {
         it('should pass when the latency assertion passes', async () => {
             const output = 'Expected output';