promptfoo 0.103.13 → 0.103.14

package/dist/test/providers/index.test.js

@@ -122,199 +122,6 @@ describe('call provider apis', () => {
         jest.clearAllMocks();
         await (0, cache_1.clearCache)();
     });
-    it('OpenAiCompletionProvider callApi', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ text: 'Test output' }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiCompletionProvider('text-davinci-003');
-        const result = await provider.callApi('Test prompt');
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-    });
-    it('OpenAiChatCompletionProvider callApi', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: 'Test output' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-    });
-    it('OpenAiChatCompletionProvider callApi with caching', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: 'Test output 2' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output 2');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result2.output).toBe('Test output 2');
-        expect(result2.tokenUsage).toEqual({ total: 10, cached: 10 });
-    });
-    it('OpenAiChatCompletionProvider callApi with cache disabled', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: 'Test output' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        (0, cache_1.disableCache)();
-        const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(2);
-        expect(result2.output).toBe('Test output');
-        expect(result2.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        (0, cache_1.enableCache)();
-    });
-    it('OpenAiChatCompletionProvider constructor with config', async () => {
-        const config = {
-            temperature: 3.1415926,
-            max_tokens: 201,
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', { config });
-        const prompt = 'Test prompt';
-        await provider.callApi(prompt);
-        expect(mockFetch).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
-            body: expect.stringMatching(`temperature\":3.1415926`),
-        }));
-        expect(provider.config.temperature).toBe(config.temperature);
-        expect(provider.config.max_tokens).toBe(config.max_tokens);
-    });
-    it('OpenAiChatCompletionProvider callApi with structured output', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: '{"name": "John", "age": 30}' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-            config: {
-                response_format: {
-                    type: 'json_schema',
-                    json_schema: {
-                        name: 'person',
-                        strict: true,
-                        schema: {
-                            type: 'object',
-                            properties: {
-                                name: { type: 'string' },
-                                age: { type: 'number' },
-                            },
-                            required: ['name', 'age'],
-                            additionalProperties: false,
-                        },
-                    },
-                },
-            },
-        });
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Get me a person' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toEqual({ name: 'John', age: 30 });
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-    });
-    it('OpenAiChatCompletionProvider callApi handles model refusals', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { refusal: 'Content policy violation' } }],
-                usage: { total_tokens: 5, prompt_tokens: 5, completion_tokens: 0 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Generate inappropriate content' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Content policy violation');
-        expect(result.tokenUsage).toEqual({ total: 5, prompt: 5, completion: 0 });
-        expect(result.isRefusal).toBe(true);
-    });
-    it('OpenAiChatCompletionProvider callApi with function tool callbacks', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [
-                    {
-                        message: {
-                            content: null,
-                            tool_calls: [
-                                {
-                                    function: {
-                                        name: 'get_weather',
-                                        arguments: '{"location":"New York"}',
-                                    },
-                                },
-                            ],
-                        },
-                    },
-                ],
-                usage: { total_tokens: 15, prompt_tokens: 10, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const mockWeatherFunction = jest.fn().mockResolvedValue('Sunny, 25°C');
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-            config: {
-                tools: [
-                    {
-                        type: 'function',
-                        function: {
-                            name: 'get_weather',
-                            description: 'Get the weather for a location',
-                            parameters: {
-                                type: 'object',
-                                properties: {
-                                    location: { type: 'string' },
-                                },
-                                required: ['location'],
-                            },
-                        },
-                    },
-                ],
-                functionToolCallbacks: {
-                    get_weather: mockWeatherFunction,
-                },
-            },
-        });
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: "What's the weather in New York?" }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(mockWeatherFunction).toHaveBeenCalledWith('{"location":"New York"}');
-        expect(result.output).toBe('Sunny, 25°C');
-        expect(result.tokenUsage).toEqual({ total: 15, prompt: 10, completion: 5 });
-    });
     it('AzureOpenAiCompletionProvider callApi', async () => {
         const mockResponse = {
             ...defaultMockResponse,
@@ -748,267 +555,6 @@ describe('call provider apis', () => {
             jest.restoreAllMocks();
         });
     });
-    describe('OpenAiChatCompletionProvider with functionToolCallbacks', () => {
-        it('should call function tool and return result', async () => {
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [
-                        {
-                            message: {
-                                content: null,
-                                function_call: {
-                                    name: 'addNumbers',
-                                    arguments: '{"a":5,"b":6}',
-                                },
-                            },
-                        },
-                    ],
-                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-                })),
-                ok: true,
-            };
-            mockFetch.mockResolvedValue(mockResponse);
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    tools: [
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'addNumbers',
-                                description: 'Add two numbers together',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        a: { type: 'number' },
-                                        b: { type: 'number' },
-                                    },
-                                    required: ['a', 'b'],
-                                },
-                            },
-                        },
-                    ],
-                    functionToolCallbacks: {
-                        addNumbers: (parametersJsonString) => {
-                            const { a, b } = JSON.parse(parametersJsonString);
-                            return Promise.resolve(JSON.stringify(a + b));
-                        },
-                    },
-                },
-            });
-            const result = await provider.callApi('Add 5 and 6');
-            expect(mockFetch).toHaveBeenCalledTimes(1);
-            expect(result.output).toBe('11');
-            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        });
-        it('should handle multiple function tool calls', async () => {
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [
-                        {
-                            message: {
-                                content: null,
-                                tool_calls: [
-                                    {
-                                        function: {
-                                            name: 'addNumbers',
-                                            arguments: '{"a":5,"b":6}',
-                                        },
-                                    },
-                                    {
-                                        function: {
-                                            name: 'multiplyNumbers',
-                                            arguments: '{"x":2,"y":3}',
-                                        },
-                                    },
-                                ],
-                            },
-                        },
-                    ],
-                    usage: { total_tokens: 15, prompt_tokens: 7, completion_tokens: 8 },
-                })),
-                ok: true,
-            };
-            mockFetch.mockResolvedValue(mockResponse);
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    tools: [
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'addNumbers',
-                                description: 'Add two numbers together',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        a: { type: 'number' },
-                                        b: { type: 'number' },
-                                    },
-                                    required: ['a', 'b'],
-                                },
-                            },
-                        },
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'multiplyNumbers',
-                                description: 'Multiply two numbers',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        x: { type: 'number' },
-                                        y: { type: 'number' },
-                                    },
-                                    required: ['x', 'y'],
-                                },
-                            },
-                        },
-                    ],
-                    functionToolCallbacks: {
-                        addNumbers: (parametersJsonString) => {
-                            const { a, b } = JSON.parse(parametersJsonString);
-                            return Promise.resolve(JSON.stringify(a + b));
-                        },
-                        multiplyNumbers: (parametersJsonString) => {
-                            const { x, y } = JSON.parse(parametersJsonString);
-                            return Promise.resolve(JSON.stringify(x * y));
-                        },
-                    },
-                },
-            });
-            const result = await provider.callApi('Add 5 and 6, then multiply 2 and 3');
-            expect(mockFetch).toHaveBeenCalledTimes(1);
-            expect(result.output).toBe('11\n6');
-            expect(result.tokenUsage).toEqual({ total: 15, prompt: 7, completion: 8 });
-        });
-        it('should handle errors in function tool callbacks', async () => {
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [
-                        {
-                            message: {
-                                content: null,
-                                function_call: {
-                                    name: 'errorFunction',
-                                    arguments: '{}',
-                                },
-                            },
-                        },
-                    ],
-                    usage: { total_tokens: 5, prompt_tokens: 2, completion_tokens: 3 },
-                })),
-                ok: true,
-            };
-            mockFetch.mockResolvedValue(mockResponse);
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    tools: [
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'errorFunction',
-                                description: 'A function that always throws an error',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {},
-                                },
-                            },
-                        },
-                    ],
-                    functionToolCallbacks: {
-                        errorFunction: () => {
-                            throw new Error('Test error');
-                        },
-                    },
-                },
-            });
-            const result = await provider.callApi('Call the error function');
-            expect(mockFetch).toHaveBeenCalledTimes(1);
-            expect(result.output).toEqual({ arguments: '{}', name: 'errorFunction' });
-            expect(result.tokenUsage).toEqual({ total: 5, prompt: 2, completion: 3 });
-        });
-    });
-    describe('OpenAiChatCompletionProvider response_format', () => {
-        it('should prioritize response_format from prompt config over provider config', async () => {
-            const providerResponseFormat = {
-                type: 'json_object',
-            };
-            const promptResponseFormat = {
-                type: 'json_schema',
-                json_schema: {
-                    name: 'test_schema',
-                    strict: true,
-                    schema: {
-                        type: 'object',
-                        properties: { key2: { type: 'string' } },
-                        additionalProperties: false,
-                    },
-                },
-            };
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    response_format: providerResponseFormat,
-                },
-            });
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [{ message: { content: '{"key2": "value2"}' } }],
-                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-                })),
-            };
-            jest.mocked(fetch).mockResolvedValue(mockResponse);
-            const result = await provider.callApi('Test prompt', {
-                vars: {},
-                prompt: {
-                    raw: 'Test prompt',
-                    label: 'Test prompt',
-                    config: {
-                        response_format: promptResponseFormat,
-                    },
-                },
-            });
-            expect(fetch).toHaveBeenCalledTimes(1);
-            const fetchArgs = jest.mocked(fetch).mock.calls[0];
-            const requestBody = JSON.parse(fetchArgs[1]?.body);
-            expect(requestBody.response_format).toEqual(promptResponseFormat);
-            expect(result.output).toEqual({ key2: 'value2' });
-            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        });
-        it('should use provider config response_format when prompt config is not provided', async () => {
-            const providerResponseFormat = {
-                type: 'json_object',
-            };
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    response_format: providerResponseFormat,
-                },
-            });
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [{ message: { content: '{"key1": "value1"}' } }],
-                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-                })),
-            };
-            jest.mocked(fetch).mockResolvedValue(mockResponse);
-            const result = await provider.callApi('Test prompt', {
-                vars: {},
-                prompt: {
-                    raw: 'Test prompt',
-                    label: 'Test prompt',
-                },
-            });
-            expect(fetch).toHaveBeenCalledTimes(1);
-            const fetchArgs = jest.mocked(fetch).mock.calls[0];
-            const requestBody = JSON.parse(fetchArgs[1]?.body);
-            expect(requestBody.response_format).toEqual(providerResponseFormat);
-            expect(result.output).toBe('{"key1": "value1"}');
-            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        });
-    });
 });
 describe('loadApiProvider', () => {
     beforeEach(() => {