promptfoo 0.103.13 → 0.103.15

package/dist/test/providers/index.test.js

@@ -107,6 +107,7 @@ jest.mock('../../src/redteam/remoteGeneration', () => ({
     neverGenerateRemote: jest.fn().mockReturnValue(false),
     getRemoteGenerationUrl: jest.fn().mockReturnValue('http://test-url'),
 }));
+jest.mock('../../src/providers/websocket');
 const mockFetch = jest.mocked(jest.fn());
 global.fetch = mockFetch;
 const defaultMockResponse = {
@@ -117,204 +118,22 @@ const defaultMockResponse = {
         entries: jest.fn().mockReturnValue([]),
     },
 };
+// Dynamic import
+jest.mock('../../src/providers/adaline.gateway', () => ({
+    AdalineGatewayChatProvider: jest.fn().mockImplementation((providerName, modelName) => ({
+        id: () => `adaline:${providerName}:chat:${modelName}`,
+        constructor: { name: 'AdalineGatewayChatProvider' },
+    })),
+    AdalineGatewayEmbeddingProvider: jest.fn().mockImplementation((providerName, modelName) => ({
+        id: () => `adaline:${providerName}:embedding:${modelName}`,
+        constructor: { name: 'AdalineGatewayEmbeddingProvider' },
+    })),
+}));
 describe('call provider apis', () => {
     afterEach(async () => {
         jest.clearAllMocks();
         await (0, cache_1.clearCache)();
     });
-    it('OpenAiCompletionProvider callApi', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ text: 'Test output' }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiCompletionProvider('text-davinci-003');
-        const result = await provider.callApi('Test prompt');
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-    });
-    it('OpenAiChatCompletionProvider callApi', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: 'Test output' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-    });
-    it('OpenAiChatCompletionProvider callApi with caching', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: 'Test output 2' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output 2');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result2.output).toBe('Test output 2');
-        expect(result2.tokenUsage).toEqual({ total: 10, cached: 10 });
-    });
-    it('OpenAiChatCompletionProvider callApi with cache disabled', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: 'Test output' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Test output');
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        (0, cache_1.disableCache)();
-        const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(2);
-        expect(result2.output).toBe('Test output');
-        expect(result2.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        (0, cache_1.enableCache)();
-    });
-    it('OpenAiChatCompletionProvider constructor with config', async () => {
-        const config = {
-            temperature: 3.1415926,
-            max_tokens: 201,
-        };
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', { config });
-        const prompt = 'Test prompt';
-        await provider.callApi(prompt);
-        expect(mockFetch).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
-            body: expect.stringMatching(`temperature\":3.1415926`),
-        }));
-        expect(provider.config.temperature).toBe(config.temperature);
-        expect(provider.config.max_tokens).toBe(config.max_tokens);
-    });
-    it('OpenAiChatCompletionProvider callApi with structured output', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { content: '{"name": "John", "age": 30}' } }],
-                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-            config: {
-                response_format: {
-                    type: 'json_schema',
-                    json_schema: {
-                        name: 'person',
-                        strict: true,
-                        schema: {
-                            type: 'object',
-                            properties: {
-                                name: { type: 'string' },
-                                age: { type: 'number' },
-                            },
-                            required: ['name', 'age'],
-                            additionalProperties: false,
-                        },
-                    },
-                },
-            },
-        });
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Get me a person' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toEqual({ name: 'John', age: 30 });
-        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-    });
-    it('OpenAiChatCompletionProvider callApi handles model refusals', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [{ message: { refusal: 'Content policy violation' } }],
-                usage: { total_tokens: 5, prompt_tokens: 5, completion_tokens: 0 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Generate inappropriate content' }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(result.output).toBe('Content policy violation');
-        expect(result.tokenUsage).toEqual({ total: 5, prompt: 5, completion: 0 });
-        expect(result.isRefusal).toBe(true);
-    });
-    it('OpenAiChatCompletionProvider callApi with function tool callbacks', async () => {
-        const mockResponse = {
-            ...defaultMockResponse,
-            text: jest.fn().mockResolvedValue(JSON.stringify({
-                choices: [
-                    {
-                        message: {
-                            content: null,
-                            tool_calls: [
-                                {
-                                    function: {
-                                        name: 'get_weather',
-                                        arguments: '{"location":"New York"}',
-                                    },
-                                },
-                            ],
-                        },
-                    },
-                ],
-                usage: { total_tokens: 15, prompt_tokens: 10, completion_tokens: 5 },
-            })),
-            ok: true,
-        };
-        mockFetch.mockResolvedValue(mockResponse);
-        const mockWeatherFunction = jest.fn().mockResolvedValue('Sunny, 25°C');
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-            config: {
-                tools: [
-                    {
-                        type: 'function',
-                        function: {
-                            name: 'get_weather',
-                            description: 'Get the weather for a location',
-                            parameters: {
-                                type: 'object',
-                                properties: {
-                                    location: { type: 'string' },
-                                },
-                                required: ['location'],
-                            },
-                        },
-                    },
-                ],
-                functionToolCallbacks: {
-                    get_weather: mockWeatherFunction,
-                },
-            },
-        });
-        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: "What's the weather in New York?" }]));
-        expect(mockFetch).toHaveBeenCalledTimes(1);
-        expect(mockWeatherFunction).toHaveBeenCalledWith('{"location":"New York"}');
-        expect(result.output).toBe('Sunny, 25°C');
-        expect(result.tokenUsage).toEqual({ total: 15, prompt: 10, completion: 5 });
-    });
     it('AzureOpenAiCompletionProvider callApi', async () => {
         const mockResponse = {
             ...defaultMockResponse,
@@ -748,267 +567,6 @@ describe('call provider apis', () => {
             jest.restoreAllMocks();
         });
     });
-    describe('OpenAiChatCompletionProvider with functionToolCallbacks', () => {
-        it('should call function tool and return result', async () => {
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [
-                        {
-                            message: {
-                                content: null,
-                                function_call: {
-                                    name: 'addNumbers',
-                                    arguments: '{"a":5,"b":6}',
-                                },
-                            },
-                        },
-                    ],
-                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-                })),
-                ok: true,
-            };
-            mockFetch.mockResolvedValue(mockResponse);
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    tools: [
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'addNumbers',
-                                description: 'Add two numbers together',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        a: { type: 'number' },
-                                        b: { type: 'number' },
-                                    },
-                                    required: ['a', 'b'],
-                                },
-                            },
-                        },
-                    ],
-                    functionToolCallbacks: {
-                        addNumbers: (parametersJsonString) => {
-                            const { a, b } = JSON.parse(parametersJsonString);
-                            return Promise.resolve(JSON.stringify(a + b));
-                        },
-                    },
-                },
-            });
-            const result = await provider.callApi('Add 5 and 6');
-            expect(mockFetch).toHaveBeenCalledTimes(1);
-            expect(result.output).toBe('11');
-            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        });
-        it('should handle multiple function tool calls', async () => {
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [
-                        {
-                            message: {
-                                content: null,
-                                tool_calls: [
-                                    {
-                                        function: {
-                                            name: 'addNumbers',
-                                            arguments: '{"a":5,"b":6}',
-                                        },
-                                    },
-                                    {
-                                        function: {
-                                            name: 'multiplyNumbers',
-                                            arguments: '{"x":2,"y":3}',
-                                        },
-                                    },
-                                ],
-                            },
-                        },
-                    ],
-                    usage: { total_tokens: 15, prompt_tokens: 7, completion_tokens: 8 },
-                })),
-                ok: true,
-            };
-            mockFetch.mockResolvedValue(mockResponse);
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    tools: [
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'addNumbers',
-                                description: 'Add two numbers together',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        a: { type: 'number' },
-                                        b: { type: 'number' },
-                                    },
-                                    required: ['a', 'b'],
-                                },
-                            },
-                        },
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'multiplyNumbers',
-                                description: 'Multiply two numbers',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {
-                                        x: { type: 'number' },
-                                        y: { type: 'number' },
-                                    },
-                                    required: ['x', 'y'],
-                                },
-                            },
-                        },
-                    ],
-                    functionToolCallbacks: {
-                        addNumbers: (parametersJsonString) => {
-                            const { a, b } = JSON.parse(parametersJsonString);
-                            return Promise.resolve(JSON.stringify(a + b));
-                        },
-                        multiplyNumbers: (parametersJsonString) => {
-                            const { x, y } = JSON.parse(parametersJsonString);
-                            return Promise.resolve(JSON.stringify(x * y));
-                        },
-                    },
-                },
-            });
-            const result = await provider.callApi('Add 5 and 6, then multiply 2 and 3');
-            expect(mockFetch).toHaveBeenCalledTimes(1);
-            expect(result.output).toBe('11\n6');
-            expect(result.tokenUsage).toEqual({ total: 15, prompt: 7, completion: 8 });
-        });
-        it('should handle errors in function tool callbacks', async () => {
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [
-                        {
-                            message: {
-                                content: null,
-                                function_call: {
-                                    name: 'errorFunction',
-                                    arguments: '{}',
-                                },
-                            },
-                        },
-                    ],
-                    usage: { total_tokens: 5, prompt_tokens: 2, completion_tokens: 3 },
-                })),
-                ok: true,
-            };
-            mockFetch.mockResolvedValue(mockResponse);
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    tools: [
-                        {
-                            type: 'function',
-                            function: {
-                                name: 'errorFunction',
-                                description: 'A function that always throws an error',
-                                parameters: {
-                                    type: 'object',
-                                    properties: {},
-                                },
-                            },
-                        },
-                    ],
-                    functionToolCallbacks: {
-                        errorFunction: () => {
-                            throw new Error('Test error');
-                        },
-                    },
-                },
-            });
-            const result = await provider.callApi('Call the error function');
-            expect(mockFetch).toHaveBeenCalledTimes(1);
-            expect(result.output).toEqual({ arguments: '{}', name: 'errorFunction' });
-            expect(result.tokenUsage).toEqual({ total: 5, prompt: 2, completion: 3 });
-        });
-    });
-    describe('OpenAiChatCompletionProvider response_format', () => {
-        it('should prioritize response_format from prompt config over provider config', async () => {
-            const providerResponseFormat = {
-                type: 'json_object',
-            };
-            const promptResponseFormat = {
-                type: 'json_schema',
-                json_schema: {
-                    name: 'test_schema',
-                    strict: true,
-                    schema: {
-                        type: 'object',
-                        properties: { key2: { type: 'string' } },
-                        additionalProperties: false,
-                    },
-                },
-            };
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    response_format: providerResponseFormat,
-                },
-            });
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [{ message: { content: '{"key2": "value2"}' } }],
-                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-                })),
-            };
-            jest.mocked(fetch).mockResolvedValue(mockResponse);
-            const result = await provider.callApi('Test prompt', {
-                vars: {},
-                prompt: {
-                    raw: 'Test prompt',
-                    label: 'Test prompt',
-                    config: {
-                        response_format: promptResponseFormat,
-                    },
-                },
-            });
-            expect(fetch).toHaveBeenCalledTimes(1);
-            const fetchArgs = jest.mocked(fetch).mock.calls[0];
-            const requestBody = JSON.parse(fetchArgs[1]?.body);
-            expect(requestBody.response_format).toEqual(promptResponseFormat);
-            expect(result.output).toEqual({ key2: 'value2' });
-            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        });
-        it('should use provider config response_format when prompt config is not provided', async () => {
-            const providerResponseFormat = {
-                type: 'json_object',
-            };
-            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
-                config: {
-                    response_format: providerResponseFormat,
-                },
-            });
-            const mockResponse = {
-                ...defaultMockResponse,
-                text: jest.fn().mockResolvedValue(JSON.stringify({
-                    choices: [{ message: { content: '{"key1": "value1"}' } }],
-                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
-                })),
-            };
-            jest.mocked(fetch).mockResolvedValue(mockResponse);
-            const result = await provider.callApi('Test prompt', {
-                vars: {},
-                prompt: {
-                    raw: 'Test prompt',
-                    label: 'Test prompt',
-                },
-            });
-            expect(fetch).toHaveBeenCalledTimes(1);
-            const fetchArgs = jest.mocked(fetch).mock.calls[0];
-            const requestBody = JSON.parse(fetchArgs[1]?.body);
-            expect(requestBody.response_format).toEqual(providerResponseFormat);
-            expect(result.output).toBe('{"key1": "value1"}');
-            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
-        });
-    });
 });
 describe('loadApiProvider', () => {
     beforeEach(() => {
@@ -1141,6 +699,13 @@ describe('loadApiProvider', () => {
         // Intentionally openai, because it's just a wrapper around openai
         expect(provider.id()).toBe('gpt-4o-mini');
     });
+    it('loadApiProvider with perplexity', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('perplexity:llama-3-sonar-large-32k-online');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        expect(provider.id()).toBe('llama-3-sonar-large-32k-online');
+        expect(provider.config.apiBaseUrl).toBe('https://api.perplexity.ai');
+        expect(provider.config.apiKeyEnvar).toBe('PERPLEXITY_API_KEY');
+    });
     it('loadApiProvider with togetherai', async () => {
         const provider = await (0, providers_1.loadApiProvider)('togetherai:chat:meta/meta-llama/Meta-Llama-3-8B-Instruct');
         expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
@@ -1360,9 +925,9 @@ describe('loadApiProvider', () => {
         expect(provider.delay).toBe(500);
     });
     it('supports templating in provider URL', async () => {
-        process.env.MYHOST = 'api.example.com';
-        process.env.MYPORT = '8080';
-        const provider = await (0, providers_1.loadApiProvider)('https://{{ env.MYHOST }}:{{ env.MYPORT }}/query', {
+        process.env.MY_HOST = 'api.example.com';
+        process.env.MY_PORT = '8080';
+        const provider = await (0, providers_1.loadApiProvider)('https://{{ env.MY_HOST }}:{{ env.MY_PORT }}/query', {
             options: {
                 config: {
                     body: {},
@@ -1370,8 +935,8 @@ describe('loadApiProvider', () => {
             },
         });
         expect(provider.id()).toBe('https://api.example.com:8080/query');
-        delete process.env.MYHOST;
-        delete process.env.MYPORT;
+        delete process.env.MY_HOST;
+        delete process.env.MY_PORT;
     });
     it('throws an error for unidentified providers', async () => {
         const mockExit = jest.spyOn(process, 'exit').mockImplementation(() => {
@@ -1407,5 +972,16 @@ describe('loadApiProvider', () => {
         expect(provider.config.apiBaseUrl).toBe('https://api.x.ai/v1');
         expect(provider.config.apiKeyEnvar).toBe('XAI_API_KEY');
     });
+    it('loadApiProvider with adaline:openai:chat', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('adaline:openai:chat:gpt-4');
+        expect(provider.id()).toBe('adaline:openai:chat:gpt-4');
+    });
+    it('loadApiProvider with adaline:openai:embedding', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('adaline:openai:embedding:text-embedding-3-large');
+        expect(provider.id()).toBe('adaline:openai:embedding:text-embedding-3-large');
+    });
+    it('should throw error for invalid adaline provider path', async () => {
+        await expect((0, providers_1.loadApiProvider)('adaline:invalid')).rejects.toThrow("Invalid adaline provider path: adaline:invalid. path format should be 'adaline:<provider_name>:<model_type>:<model_name>' eg. 'adaline:openai:chat:gpt-4o'");
+    });
 });
 //# sourceMappingURL=index.test.js.map