promptfoo 0.93.3 → 0.94.1

package/dist/test/{providers.test.js → providers/index.test.js}

@@ -30,30 +30,29 @@ const chalk_1 = __importDefault(require("chalk"));
 const child_process_1 = __importDefault(require("child_process"));
 const dedent_1 = __importDefault(require("dedent"));
 const fs = __importStar(require("fs"));
-const node_fetch_1 = __importDefault(require("node-fetch"));
 const path = __importStar(require("path"));
 const stream_1 = __importDefault(require("stream"));
-const cache_1 = require("../src/cache");
-const esm_1 = require("../src/esm");
-const logger_1 = __importDefault(require("../src/logger"));
-const providers_1 = require("../src/providers");
-const anthropic_1 = require("../src/providers/anthropic");
-const azureopenai_1 = require("../src/providers/azureopenai");
-const bedrock_1 = require("../src/providers/bedrock");
-const cloudflare_ai_1 = require("../src/providers/cloudflare-ai");
-const huggingface_1 = require("../src/providers/huggingface");
-const llama_1 = require("../src/providers/llama");
-const ollama_1 = require("../src/providers/ollama");
-const openai_1 = require("../src/providers/openai");
-const pythonCompletion_1 = require("../src/providers/pythonCompletion");
-const replicate_1 = require("../src/providers/replicate");
-const scriptCompletion_1 = require("../src/providers/scriptCompletion");
-const vertex_1 = require("../src/providers/vertex");
-const voyage_1 = require("../src/providers/voyage");
-const webhook_1 = require("../src/providers/webhook");
-const iterative_1 = __importDefault(require("../src/redteam/providers/iterative"));
-const iterativeImage_1 = __importDefault(require("../src/redteam/providers/iterativeImage"));
-const iterativeTree_1 = __importDefault(require("../src/redteam/providers/iterativeTree"));
+const cache_1 = require("../../src/cache");
+const esm_1 = require("../../src/esm");
+const logger_1 = __importDefault(require("../../src/logger"));
+const providers_1 = require("../../src/providers");
+const anthropic_1 = require("../../src/providers/anthropic");
+const azureopenai_1 = require("../../src/providers/azureopenai");
+const bedrock_1 = require("../../src/providers/bedrock");
+const cloudflare_ai_1 = require("../../src/providers/cloudflare-ai");
+const huggingface_1 = require("../../src/providers/huggingface");
+const llama_1 = require("../../src/providers/llama");
+const ollama_1 = require("../../src/providers/ollama");
+const openai_1 = require("../../src/providers/openai");
+const pythonCompletion_1 = require("../../src/providers/pythonCompletion");
+const replicate_1 = require("../../src/providers/replicate");
+const scriptCompletion_1 = require("../../src/providers/scriptCompletion");
+const vertex_1 = require("../../src/providers/vertex");
+const voyage_1 = require("../../src/providers/voyage");
+const webhook_1 = require("../../src/providers/webhook");
+const iterative_1 = __importDefault(require("../../src/redteam/providers/iterative"));
+const iterativeImage_1 = __importDefault(require("../../src/redteam/providers/iterativeImage"));
+const iterativeTree_1 = __importDefault(require("../../src/redteam/providers/iterativeTree"));
 jest.mock('fs', () => ({
     readFileSync: jest.fn(),
     writeFileSync: jest.fn(),
@@ -68,12 +67,11 @@ jest.mock('fs', () => ({
 jest.mock('glob', () => ({
     globSync: jest.fn(),
 }));
-jest.mock('node-fetch', () => jest.fn());
 jest.mock('proxy-agent', () => ({
     ProxyAgent: jest.fn().mockImplementation(() => ({})),
 }));
-jest.mock('../src/esm', () => ({
-    ...jest.requireActual('../src/esm'),
+jest.mock('../../src/esm', () => ({
+    ...jest.requireActual('../../src/esm'),
     importModule: jest.fn(),
 }));
 jest.mock('fs', () => ({
@@ -84,15 +82,17 @@ jest.mock('fs', () => ({
 jest.mock('glob', () => ({
     globSync: jest.fn(),
 }));
-jest.mock('../src/database', () => ({
+jest.mock('../../src/database', () => ({
     getDb: jest.fn(),
 }));
-jest.mock('../src/logger', () => ({
+jest.mock('../../src/logger', () => ({
     error: jest.fn(),
     debug: jest.fn(),
     info: jest.fn(),
     warn: jest.fn(),
 }));
+const mockFetch = jest.mocked(jest.fn());
+global.fetch = mockFetch;
 const defaultMockResponse = {
     status: 200,
     statusText: 'OK',
@@ -113,10 +113,10 @@ describe('call provider apis', () => {
                 usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new openai_1.OpenAiCompletionProvider('text-davinci-003');
         const result = await provider.callApi('Test prompt');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
     });
@@ -129,10 +129,10 @@ describe('call provider apis', () => {
             })),
             ok: true,
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
     });
@@ -145,14 +145,14 @@ describe('call provider apis', () => {
             })),
             ok: true,
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output 2');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
         const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result2.output).toBe('Test output 2');
         expect(result2.tokenUsage).toEqual({ total: 10, cached: 10 });
     });
@@ -165,15 +165,15 @@ describe('call provider apis', () => {
             })),
             ok: true,
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
         (0, cache_1.disableCache)();
         const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(2);
+        expect(mockFetch).toHaveBeenCalledTimes(2);
         expect(result2.output).toBe('Test output');
         expect(result2.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
         (0, cache_1.enableCache)();
@@ -186,7 +186,7 @@ describe('call provider apis', () => {
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', { config });
         const prompt = 'Test prompt';
         await provider.callApi(prompt);
-        expect(node_fetch_1.default).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
+        expect(mockFetch).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
             body: expect.stringMatching(`temperature\":3.1415926`),
         }));
         expect(provider.config.temperature).toBe(config.temperature);
@@ -201,7 +201,7 @@ describe('call provider apis', () => {
             })),
             ok: true,
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
             config: {
                 response_format: {
@@ -223,7 +223,7 @@ describe('call provider apis', () => {
             },
         });
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Get me a person' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toEqual({ name: 'John', age: 30 });
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
     });
@@ -236,12 +236,13 @@ describe('call provider apis', () => {
             })),
             ok: true,
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Generate inappropriate content' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
-        expect(result.error).toBe('Model refused to generate a response: Content policy violation');
+        expect(mockFetch).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Content policy violation');
         expect(result.tokenUsage).toEqual({ total: 5, prompt: 5, completion: 0 });
+        expect(result.isRefusal).toBe(true);
     });
     it('OpenAiChatCompletionProvider callApi with function tool callbacks', async () => {
         const mockResponse = {
@@ -266,7 +267,7 @@ describe('call provider apis', () => {
             })),
             ok: true,
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const mockWeatherFunction = jest.fn().mockResolvedValue('Sunny, 25°C');
         const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
             config: {
@@ -292,7 +293,7 @@ describe('call provider apis', () => {
             },
         });
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: "What's the weather in New York?" }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(mockWeatherFunction).toHaveBeenCalledWith('{"location":"New York"}');
         expect(result.output).toBe('Sunny, 25°C');
         expect(result.tokenUsage).toEqual({ total: 15, prompt: 10, completion: 5 });
@@ -305,10 +306,10 @@ describe('call provider apis', () => {
                 usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new azureopenai_1.AzureOpenAiCompletionProvider('text-davinci-003');
         const result = await provider.callApi('Test prompt');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
     });
@@ -320,10 +321,10 @@ describe('call provider apis', () => {
                 usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new azureopenai_1.AzureOpenAiChatCompletionProvider('gpt-4o-mini');
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
     });
@@ -348,7 +349,7 @@ describe('call provider apis', () => {
                 usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new azureopenai_1.AzureOpenAiChatCompletionProvider('gpt-4o-mini', {
             config: { dataSources },
         });
@@ -356,7 +357,7 @@ describe('call provider apis', () => {
             { role: 'system', content: 'System prompt' },
             { role: 'user', content: 'Test prompt' },
         ]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test response');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
     });
@@ -369,10 +370,10 @@ describe('call provider apis', () => {
                 usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new azureopenai_1.AzureOpenAiChatCompletionProvider('gpt-4o-mini');
         const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
         expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
         (0, cache_1.enableCache)();
@@ -384,10 +385,10 @@ describe('call provider apis', () => {
                 content: 'Test output',
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new llama_1.LlamaProvider('llama.cpp');
         const result = await provider.callApi('Test prompt');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
     });
     it('OllamaCompletionProvider callApi', async () => {
@@ -404,10 +405,10 @@ describe('call provider apis', () => {
 {"model":"llama2:13b","created_at":"2023-08-08T21:50:35.117166Z","response":" blue","done":false}
 {"model":"llama2:13b","created_at":"2023-08-08T21:50:41.695299Z","done":true,"context":[1,29871,1,13,9314],"total_duration":10411943458,"load_duration":458333,"sample_count":217,"sample_duration":154566000,"prompt_eval_count":11,"prompt_eval_duration":3334582000,"eval_count":216,"eval_duration":6905134000}`),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new ollama_1.OllamaCompletionProvider('llama');
         const result = await provider.callApi('Test prompt');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Great question! The sky appears blue');
     });
     it('OllamaChatProvider callApi', async () => {
@@ -422,10 +423,10 @@ describe('call provider apis', () => {
 {"model":"orca-mini","created_at":"2023-12-16T01:46:19.324309782Z","message":{"role":"assistant","content":".","images":null},"done":false}
 {"model":"orca-mini","created_at":"2023-12-16T01:46:19.337165395Z","done":true,"total_duration":1486443841,"load_duration":1280794143,"prompt_eval_count":35,"prompt_eval_duration":142384000,"eval_count":6,"eval_duration":61912000}`),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new ollama_1.OllamaChatProvider('llama');
         const result = await provider.callApi('Test prompt');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe(' Because of Rayleigh scattering.');
     });
     it('WebhookProvider callApi', async () => {
@@ -435,10 +436,10 @@ describe('call provider apis', () => {
                 output: 'Test output',
             })),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new webhook_1.WebhookProvider('http://example.com/webhook');
         const result = await provider.callApi('Test prompt');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.output).toBe('Test output');
     });
     describe.each([
@@ -450,10 +451,10 @@ describe('call provider apis', () => {
                 ...defaultMockResponse,
                 text: jest.fn().mockResolvedValue(JSON.stringify(mockedData)),
             };
-            jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+            mockFetch.mockResolvedValue(mockResponse);
             const provider = new huggingface_1.HuggingfaceTextGenerationProvider('gpt2');
             const result = await provider.callApi('Test prompt');
-            expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+            expect(mockFetch).toHaveBeenCalledTimes(1);
             expect(result.output).toBe('Test output');
         });
     });
@@ -462,10 +463,10 @@ describe('call provider apis', () => {
             ...defaultMockResponse,
             text: jest.fn().mockResolvedValue(JSON.stringify([0.1, 0.2, 0.3, 0.4, 0.5])),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new huggingface_1.HuggingfaceFeatureExtractionProvider('distilbert-base-uncased');
         const result = await provider.callEmbeddingApi('Test text');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.embedding).toEqual([0.1, 0.2, 0.3, 0.4, 0.5]);
     });
     it('HuggingfaceTextClassificationProvider callClassificationApi', async () => {
@@ -485,10 +486,10 @@ describe('call provider apis', () => {
             ...defaultMockResponse,
             text: jest.fn().mockResolvedValue(JSON.stringify(mockClassification)),
         };
-        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        mockFetch.mockResolvedValue(mockResponse);
         const provider = new huggingface_1.HuggingfaceTextClassificationProvider('foo');
         const result = await provider.callClassificationApi('Test text');
-        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(mockFetch).toHaveBeenCalledTimes(1);
         expect(result.classification).toEqual({
             nothate: 0.9,
             hate: 0.1,
@@ -498,7 +499,6 @@ describe('call provider apis', () => {
         beforeAll(() => {
             (0, cache_1.enableCache)();
         });
-        const fetchMock = jest.mocked(node_fetch_1.default);
         const cloudflareMinimumConfig = {
             accountId: 'testAccountId',
             apiKey: 'testApiKey',
@@ -530,13 +530,13 @@ describe('call provider apis', () => {
                     text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
                     ok: true,
                 };
-                fetchMock.mockResolvedValue(mockResponse);
+                mockFetch.mockResolvedValue(mockResponse);
                 const result = await provider.callApi(PROMPT);
-                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(mockFetch).toHaveBeenCalledTimes(1);
                 expect(result.output).toBe(responsePayload.result.response);
                 expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
                 const resultFromCache = await provider.callApi(PROMPT);
-                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(mockFetch).toHaveBeenCalledTimes(1);
                 expect(resultFromCache.output).toBe(responsePayload.result.response);
                 expect(resultFromCache.tokenUsage).toEqual(tokenUsageDefaultResponse);
             });
@@ -560,13 +560,13 @@ describe('call provider apis', () => {
                         text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
                         ok: true,
                     };
-                    fetchMock.mockResolvedValue(mockResponse);
+                    mockFetch.mockResolvedValue(mockResponse);
                     const result = await provider.callApi(PROMPT);
-                    expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                    expect(mockFetch).toHaveBeenCalledTimes(1);
                     expect(result.output).toBe(responsePayload.result.response);
                     expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
                     const resultFromCache = await provider.callApi(PROMPT);
-                    expect(node_fetch_1.default).toHaveBeenCalledTimes(2);
+                    expect(mockFetch).toHaveBeenCalledTimes(2);
                     expect(resultFromCache.output).toBe(responsePayload.result.response);
                     expect(resultFromCache.tokenUsage).toEqual(tokenUsageDefaultResponse);
                 }
@@ -589,7 +589,7 @@ describe('call provider apis', () => {
                     text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
                     ok: true,
                 };
-                fetchMock.mockResolvedValue(mockResponse);
+                mockFetch.mockResolvedValue(mockResponse);
                 const result = await provider.callApi(PROMPT);
                 expect(result.error).toContain(JSON.stringify(responsePayload.errors));
             });
@@ -625,19 +625,14 @@ describe('call provider apis', () => {
                     text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
                     ok: true,
                 };
-                fetchMock.mockResolvedValue(mockResponse);
+                mockFetch.mockResolvedValue(mockResponse);
                 await cfProvider.callApi(PROMPT);
-                expect(fetchMock).toHaveBeenCalledTimes(1);
-                expect(fetchMock.mock.calls).toHaveLength(1);
-                const [url, options] = fetchMock.mock.calls[0] ?? [undefined, {}];
-                const { body, headers, method } = options;
-                expect(url).toContain(cloudflareChatConfig.accountId);
-                expect(method).toBe('POST');
-                expect(headers && 'Authorization' in headers ? headers['Authorization'] : '').toContain(cloudflareChatConfig.apiKey);
-                const hydratedBody = typeof body === 'string' ? JSON.parse(body) : body;
-                expect(hydratedBody.prompt).toBe(PROMPT);
+                expect(mockFetch).toHaveBeenCalledTimes(1);
+                expect(mockFetch).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
+                    body: expect.stringMatching(`"prompt":"${PROMPT}"`),
+                }));
                 const { accountId: _accountId, apiKey: _apiKey, ...passThroughConfig } = cloudflareChatConfig;
-                const { prompt: _prompt, ...bodyWithoutPrompt } = hydratedBody;
+                const { prompt: _prompt, ...bodyWithoutPrompt } = JSON.parse(jest.mocked(mockFetch).mock.calls[0][1].body);
                 expect(bodyWithoutPrompt).toEqual(passThroughConfig);
             });
         });
@@ -659,9 +654,9 @@ describe('call provider apis', () => {
                     text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
                     ok: true,
                 };
-                fetchMock.mockResolvedValue(mockResponse);
+                mockFetch.mockResolvedValue(mockResponse);
                 const result = await provider.callApi('Test chat prompt');
-                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(mockFetch).toHaveBeenCalledTimes(1);
                 expect(result.output).toBe(responsePayload.result.response);
                 expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
             });
@@ -685,9 +680,9 @@ describe('call provider apis', () => {
                     text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
                     ok: true,
                 };
-                fetchMock.mockResolvedValue(mockResponse);
+                mockFetch.mockResolvedValue(mockResponse);
                 const result = await provider.callEmbeddingApi('Create embeddings from this');
-                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(mockFetch).toHaveBeenCalledTimes(1);
                 expect(result.embedding).toEqual(responsePayload.result.data[0]);
                 expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
             });
@@ -756,7 +751,7 @@ describe('call provider apis', () => {
                 })),
                 ok: true,
             };
-            jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+            mockFetch.mockResolvedValue(mockResponse);
             const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
                 config: {
                     tools: [
@@ -785,7 +780,7 @@ describe('call provider apis', () => {
                 },
             });
             const result = await provider.callApi('Add 5 and 6');
-            expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+            expect(mockFetch).toHaveBeenCalledTimes(1);
             expect(result.output).toBe('11');
             expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
         });
@@ -818,7 +813,7 @@ describe('call provider apis', () => {
                 })),
                 ok: true,
             };
-            jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+            mockFetch.mockResolvedValue(mockResponse);
             const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
                 config: {
                     tools: [
@@ -866,7 +861,7 @@ describe('call provider apis', () => {
                 },
             });
             const result = await provider.callApi('Add 5 and 6, then multiply 2 and 3');
-            expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+            expect(mockFetch).toHaveBeenCalledTimes(1);
             expect(result.output).toBe('11\n6');
             expect(result.tokenUsage).toEqual({ total: 15, prompt: 7, completion: 8 });
         });
@@ -889,7 +884,7 @@ describe('call provider apis', () => {
                 })),
                 ok: true,
             };
-            jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+            mockFetch.mockResolvedValue(mockResponse);
             const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
                 config: {
                     tools: [
@@ -913,17 +908,97 @@ describe('call provider apis', () => {
                 },
             });
             const result = await provider.callApi('Call the error function');
-            expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+            expect(mockFetch).toHaveBeenCalledTimes(1);
             expect(result.output).toEqual({ arguments: '{}', name: 'errorFunction' });
             expect(result.tokenUsage).toEqual({ total: 5, prompt: 2, completion: 3 });
         });
     });
+    describe('OpenAiChatCompletionProvider response_format', () => {
+        it('should prioritize response_format from prompt config over provider config', async () => {
+            const providerResponseFormat = {
+                type: 'json_object',
+            };
+            const promptResponseFormat = {
+                type: 'json_schema',
+                json_schema: {
+                    name: 'test_schema',
+                    strict: true,
+                    schema: {
+                        type: 'object',
+                        properties: { key2: { type: 'string' } },
+                        additionalProperties: false,
+                    },
+                },
+            };
+            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
+                config: {
+                    response_format: providerResponseFormat,
+                },
+            });
+            const mockResponse = {
+                ...defaultMockResponse,
+                text: jest.fn().mockResolvedValue(JSON.stringify({
+                    choices: [{ message: { content: '{"key2": "value2"}' } }],
+                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+                })),
+            };
+            jest.mocked(fetch).mockResolvedValue(mockResponse);
+            const result = await provider.callApi('Test prompt', {
+                vars: {},
+                prompt: {
+                    raw: 'Test prompt',
+                    label: 'Test prompt',
+                    config: {
+                        response_format: promptResponseFormat,
+                    },
+                },
+            });
+            expect(fetch).toHaveBeenCalledTimes(1);
+            const fetchArgs = jest.mocked(fetch).mock.calls[0];
+            const requestBody = JSON.parse(fetchArgs[1]?.body);
+            expect(requestBody.response_format).toEqual(promptResponseFormat);
+            expect(result.output).toEqual({ key2: 'value2' });
+            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+        });
+        it('should use provider config response_format when prompt config is not provided', async () => {
+            const providerResponseFormat = {
+                type: 'json_object',
+            };
+            const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini', {
+                config: {
+                    response_format: providerResponseFormat,
+                },
+            });
+            const mockResponse = {
+                ...defaultMockResponse,
+                text: jest.fn().mockResolvedValue(JSON.stringify({
+                    choices: [{ message: { content: '{"key1": "value1"}' } }],
+                    usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+                })),
+            };
+            jest.mocked(fetch).mockResolvedValue(mockResponse);
+            const result = await provider.callApi('Test prompt', {
+                vars: {},
+                prompt: {
+                    raw: 'Test prompt',
+                    label: 'Test prompt',
+                },
+            });
+            expect(fetch).toHaveBeenCalledTimes(1);
+            const fetchArgs = jest.mocked(fetch).mock.calls[0];
+            const requestBody = JSON.parse(fetchArgs[1]?.body);
+            expect(requestBody.response_format).toEqual(providerResponseFormat);
+            expect(result.output).toBe('{"key1": "value1"}');
+            expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+        });
+    });
 });
 describe('loadApiProvider', () => {
     it('loadApiProvider with yaml filepath', async () => {
-        const mockYamlContent = `id: 'openai:gpt-4'
-config:
-  key: 'value'`;
+        const mockYamlContent = (0, dedent_1.default) `
+    id: 'openai:gpt-4'
+    config:
+      key: 'value'`;
         jest.mocked(fs.readFileSync).mockReturnValueOnce(mockYamlContent);
         const provider = await (0, providers_1.loadApiProvider)('file://path/to/mock-provider-file.yaml');
         expect(provider.id()).toBe('openai:gpt-4');
@@ -1040,6 +1115,17 @@ config:
         // Intentionally openai, because it's just a wrapper around openai
         expect(provider.id()).toBe('mistralai/mistral-medium');
     });
+    it('loadApiProvider with github', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('github:gpt-4o-mini');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        // Intentionally openai, because it's just a wrapper around openai
+        expect(provider.id()).toBe('gpt-4o-mini');
+    });
+    it('loadApiProvider with togetherai', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('togetherai:chat:meta/meta-llama/Meta-Llama-3-8B-Instruct');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        expect(provider.id()).toBe('meta/meta-llama/Meta-Llama-3-8B-Instruct');
+    });
     it('loadApiProvider with voyage', async () => {
         const provider = await (0, providers_1.loadApiProvider)('voyage:voyage-2');
         expect(provider).toBeInstanceOf(voyage_1.VoyageEmbeddingProvider);
@@ -1287,5 +1373,12 @@ config:
         const provider = await (0, providers_1.loadApiProvider)('openai:chat:gpt-4o', { options: providerOptions });
         expect(provider.label).toBe('foo');
     });
+    it('loadApiProvider with xai', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('xai:grok-2');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        expect(provider.id()).toBe('grok-2');
+        expect(provider.config.apiBaseUrl).toBe('https://api.x.ai/v1');
+        expect(provider.config.apiKeyEnvar).toBe('XAI_API_KEY');
+    });
 });
-//# sourceMappingURL=providers.test.js.map
+//# sourceMappingURL=index.test.js.map