promptfoo 0.94.5 → 0.95.0

package/dist/test/assertions/index.test.js

@@ -2151,17 +2151,21 @@ describe('runAssertion', () => {
             type: 'javascript',
             value: 'file:///path/to/assert.js',
         };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: fileAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(mockFn).toHaveBeenCalledWith('Expected output', {
             prompt: 'Some prompt',
             vars: {},
             test: {},
+            provider,
+            providerResponse,
         });
         expect(result).toMatchObject({
             pass: expectedPass,
@@ -2214,17 +2218,21 @@ describe('runAssertion', () => {
             type: 'javascript',
             value: 'package:@promptfoo/fake:assertionFunction',
         };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: fileAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(mockFn).toHaveBeenCalledWith('Expected output', {
             prompt: 'Some prompt',
             vars: {},
             test: {},
+            provider,
+            providerResponse,
         });
         expect(result).toMatchObject({
             pass: expectedPass,
@@ -2239,17 +2247,21 @@ describe('runAssertion', () => {
             type: 'javascript',
             value: 'file://./path/to/assert.js',
         };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: fileAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(mockFn).toHaveBeenCalledWith('Expected output', {
             prompt: 'Some prompt',
             vars: {},
             test: {},
+            provider,
+            providerResponse,
         });
         expect(result).toMatchObject({
             pass: true,
@@ -2264,17 +2276,19 @@ describe('runAssertion', () => {
             type: 'python',
             value: expectedPythonValue,
         };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: pythonAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(wrapper_1.runPythonCode).toHaveBeenCalledTimes(1);
         expect(wrapper_1.runPythonCode).toHaveBeenCalledWith(expect.anything(), 'main', [
             output,
-            { prompt: 'Some prompt', test: {}, vars: {} },
+            { prompt: 'Some prompt', test: {}, vars: {}, provider, providerResponse },
         ]);
         expect(result).toMatchObject({
             pass: true,
@@ -2327,17 +2341,19 @@ describe('runAssertion', () => {
             threshold,
         };
         jest.mocked(wrapper_1.runPythonCode).mockResolvedValueOnce(resolvedValue);
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: pythonAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(wrapper_1.runPythonCode).toHaveBeenCalledTimes(1);
         expect(wrapper_1.runPythonCode).toHaveBeenCalledWith(expect.anything(), 'main', [
             output,
-            { prompt: 'Some prompt', test: {}, vars: {} },
+            { prompt: 'Some prompt', test: {}, vars: {}, provider, providerResponse },
         ]);
         expect(result).toMatchObject({
             pass: expectedPass,
@@ -2371,12 +2387,14 @@ describe('runAssertion', () => {
             type: 'python',
             value: 'file:///path/to/assert.py',
         };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: fileAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(pythonUtils_1.runPython).toHaveBeenCalledWith(path.resolve('/path/to/assert.py'), 'get_assert', [
             output,
@@ -2384,6 +2402,8 @@ describe('runAssertion', () => {
                 prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
                 vars: {},
                 test: {},
+                provider,
+                providerResponse,
             },
         ]);
         expect(result).toMatchObject({
@@ -2401,12 +2421,14 @@ describe('runAssertion', () => {
             type: 'python',
             value: 'file:///path/to/assert.py',
         };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+        const providerResponse = { output };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+            provider,
             assertion: fileAssertion,
             test: {},
-            providerResponse: { output },
+            providerResponse,
         });
         expect(pythonUtils_1.runPython).toHaveBeenCalledTimes(1);
         expect(result).toEqual({
@@ -2422,16 +2444,18 @@ describe('runAssertion', () => {
     describe('latency assertion', () => {
         it('should pass when the latency assertion passes', async () => {
             const output = 'Expected output';
+            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+            const providerResponse = { output };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                provider,
                 assertion: {
                     type: 'latency',
                     threshold: 100,
                 },
                 latencyMs: 50,
                 test: {},
-                providerResponse: { output },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,
@@ -2440,16 +2464,18 @@ describe('runAssertion', () => {
         });
         it('should fail when the latency assertion fails', async () => {
             const output = 'Expected output';
+            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+            const providerResponse = { output };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                provider,
                 assertion: {
                     type: 'latency',
                     threshold: 100,
                 },
                 latencyMs: 1000,
                 test: {},
-                providerResponse: { output },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: false,
@@ -2471,16 +2497,18 @@ describe('runAssertion', () => {
         });
         it('should pass when the latency is 0ms', async () => {
             const output = 'Expected output';
+            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+            const providerResponse = { output };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                provider,
                 assertion: {
                     type: 'latency',
                     threshold: 100,
                 },
                 latencyMs: 0,
                 test: {},
-                providerResponse: { output },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,
@@ -2502,16 +2530,18 @@ describe('runAssertion', () => {
         });
         it('should handle latency equal to threshold', async () => {
             const output = 'Expected output';
+            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
+            const providerResponse = { output };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                provider,
                 assertion: {
                     type: 'latency',
                     threshold: 100,
                 },
                 latencyMs: 100,
                 test: {},
-                providerResponse: { output },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,
@@ -2525,6 +2555,7 @@ describe('runAssertion', () => {
             const provider = {
                 callApi: jest.fn().mockResolvedValue({ logProbs }),
             };
+            const providerResponse = { output: 'Some output', logProbs };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
                 provider,
@@ -2533,7 +2564,7 @@ describe('runAssertion', () => {
                     threshold: 2,
                 },
                 test: {},
-                providerResponse: { output: 'Some output', logProbs },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,
@@ -2545,6 +2576,7 @@ describe('runAssertion', () => {
             const provider = {
                 callApi: jest.fn().mockResolvedValue({ logProbs }),
             };
+            const providerResponse = { output: 'Some output', logProbs };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
                 provider,
@@ -2553,7 +2585,7 @@ describe('runAssertion', () => {
                     threshold: 0.2,
                 },
                 test: {},
-                providerResponse: { output: 'Some output', logProbs },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: false,
@@ -2567,6 +2599,7 @@ describe('runAssertion', () => {
             const provider = {
                 callApi: jest.fn().mockResolvedValue({ logProbs }),
             };
+            const providerResponse = { output: 'Some output', logProbs };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
                 provider,
@@ -2575,7 +2608,7 @@ describe('runAssertion', () => {
                     threshold: 0.25,
                 },
                 test: {},
-                providerResponse: { output: 'Some output', logProbs },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,
@@ -2587,6 +2620,7 @@ describe('runAssertion', () => {
             const provider = {
                 callApi: jest.fn().mockResolvedValue({ logProbs }),
             };
+            const providerResponse = { output: 'Some output', logProbs };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
                 provider,
@@ -2595,7 +2629,7 @@ describe('runAssertion', () => {
                     threshold: 0.5,
                 },
                 test: {},
-                providerResponse: { output: 'Some output', logProbs },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: false,
@@ -2609,6 +2643,7 @@ describe('runAssertion', () => {
             const provider = {
                 callApi: jest.fn().mockResolvedValue({ cost }),
             };
+            const providerResponse = { output: 'Some output', cost };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
                 provider,
@@ -2617,7 +2652,7 @@ describe('runAssertion', () => {
                     threshold: 0.001,
                 },
                 test: {},
-                providerResponse: { output: 'Some output', cost },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,
@@ -2629,6 +2664,7 @@ describe('runAssertion', () => {
             const provider = {
                 callApi: jest.fn().mockResolvedValue({ cost }),
             };
+            const providerResponse = { output: 'Some output', cost };
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
                 provider,
@@ -2637,7 +2673,7 @@ describe('runAssertion', () => {
                     threshold: 0.001,
                 },
                 test: {},
-                providerResponse: { output: 'Some output', cost },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: false,
@@ -2648,30 +2684,32 @@ describe('runAssertion', () => {
     describe('is-valid-openai-function-call assertion', () => {
         it('should pass for a valid function call with correct arguments', async () => {
             const output = { arguments: '{"x": 10, "y": 20}', name: 'add' };
-            const result = await (0, assertions_1.runAssertion)({
-                prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('foo', {
-                    config: {
-                        functions: [
-                            {
-                                name: 'add',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        x: { type: 'number' },
-                                        y: { type: 'number' },
-                                    },
-                                    required: ['x', 'y'],
+            const provider = new openai_1.OpenAiChatCompletionProvider('foo', {
+                config: {
+                    functions: [
+                        {
+                            name: 'add',
+                            parameters: {
+                                type: 'object',
+                                properties: {
+                                    x: { type: 'number' },
+                                    y: { type: 'number' },
                                 },
+                                required: ['x', 'y'],
                             },
-                        ],
-                    },
-                }),
+                        },
+                    ],
+                },
+            });
+            const providerResponse = { output };
+            const result = await (0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider,
                 assertion: {
                     type: 'is-valid-openai-function-call',
                 },
                 test: {},
-                providerResponse: { output },
+                providerResponse,
             });
             expect(result).toMatchObject({
                 pass: true,