promptfoo 0.72.1 → 0.73.0

package/dist/test/providers.test.js

@@ -0,0 +1,883 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const child_process_1 = __importDefault(require("child_process"));
+const fs = __importStar(require("fs"));
+const node_fetch_1 = __importDefault(require("node-fetch"));
+const path = __importStar(require("path"));
+const stream_1 = __importDefault(require("stream"));
+const cache_1 = require("../src/cache");
+const esm_1 = require("../src/esm");
+const providers_1 = require("../src/providers");
+const anthropic_1 = require("../src/providers/anthropic");
+const azureopenai_1 = require("../src/providers/azureopenai");
+const bedrock_1 = require("../src/providers/bedrock");
+const cloudflare_ai_1 = require("../src/providers/cloudflare-ai");
+const huggingface_1 = require("../src/providers/huggingface");
+const llama_1 = require("../src/providers/llama");
+const ollama_1 = require("../src/providers/ollama");
+const openai_1 = require("../src/providers/openai");
+const replicate_1 = require("../src/providers/replicate");
+const scriptCompletion_1 = require("../src/providers/scriptCompletion");
+const vertex_1 = require("../src/providers/vertex");
+const voyage_1 = require("../src/providers/voyage");
+const webhook_1 = require("../src/providers/webhook");
+const iterative_1 = __importDefault(require("../src/redteam/providers/iterative"));
+const iterativeImage_1 = __importDefault(require("../src/redteam/providers/iterativeImage"));
+jest.mock('fs', () => ({
+    readFileSync: jest.fn(),
+    writeFileSync: jest.fn(),
+    statSync: jest.fn(),
+    readdirSync: jest.fn(),
+    existsSync: jest.fn(),
+    mkdirSync: jest.fn(),
+    promises: {
+        readFile: jest.fn(),
+    },
+}));
+jest.mock('glob', () => ({
+    globSync: jest.fn(),
+}));
+jest.mock('node-fetch', () => jest.fn());
+jest.mock('proxy-agent', () => ({
+    ProxyAgent: jest.fn().mockImplementation(() => ({})),
+}));
+jest.mock('../src/esm', () => ({
+    ...jest.requireActual('../src/esm'),
+    importModule: jest.fn(),
+}));
+jest.mock('fs', () => ({
+    readFileSync: jest.fn(),
+    existsSync: jest.fn(),
+    mkdirSync: jest.fn(),
+}));
+jest.mock('glob', () => ({
+    globSync: jest.fn(),
+}));
+jest.mock('../src/database', () => ({
+    getDb: jest.fn(),
+}));
+describe('call provider apis', () => {
+    afterEach(async () => {
+        jest.clearAllMocks();
+        await (0, cache_1.clearCache)();
+    });
+    it('OpenAiCompletionProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ text: 'Test output' }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new openai_1.OpenAiCompletionProvider('text-davinci-003');
+        const result = await provider.callApi('Test prompt');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+    });
+    it('OpenAiChatCompletionProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ message: { content: 'Test output' } }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+            ok: true,
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-3.5-turbo');
+        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+    });
+    it('OpenAiChatCompletionProvider callApi with caching', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ message: { content: 'Test output 2' } }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+            ok: true,
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-3.5-turbo');
+        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output 2');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+        const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt 2' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result2.output).toBe('Test output 2');
+        expect(result2.tokenUsage).toEqual({ total: 10, cached: 10 });
+    });
+    it('OpenAiChatCompletionProvider callApi with cache disabled', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ message: { content: 'Test output' } }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+            ok: true,
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-3.5-turbo');
+        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+        (0, cache_1.disableCache)();
+        const result2 = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(2);
+        expect(result2.output).toBe('Test output');
+        expect(result2.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+        (0, cache_1.enableCache)();
+    });
+    it('OpenAiChatCompletionProvider constructor with config', async () => {
+        const config = {
+            temperature: 3.1415926,
+            max_tokens: 201,
+        };
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-3.5-turbo', { config });
+        const prompt = 'Test prompt';
+        await provider.callApi(prompt);
+        expect(node_fetch_1.default).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({
+            body: expect.stringMatching(`temperature\":3.1415926`),
+        }));
+        expect(provider.config.temperature).toBe(config.temperature);
+        expect(provider.config.max_tokens).toBe(config.max_tokens);
+    });
+    it('AzureOpenAiCompletionProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ text: 'Test output' }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new azureopenai_1.AzureOpenAiCompletionProvider('text-davinci-003');
+        const result = await provider.callApi('Test prompt');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+    });
+    it('AzureOpenAiChatCompletionProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ message: { content: 'Test output' } }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new azureopenai_1.AzureOpenAiChatCompletionProvider('gpt-3.5-turbo');
+        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+    });
+    it('AzureOpenAiChatCompletionProvider callApi with dataSources', async () => {
+        const dataSources = [
+            {
+                type: 'AzureCognitiveSearch',
+                endpoint: 'https://search.windows.net',
+                indexName: 'search-test',
+                semanticConfiguration: 'default',
+                queryType: 'vectorSimpleHybrid',
+            },
+        ];
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [
+                    { message: { role: 'system', content: 'System prompt' } },
+                    { message: { role: 'user', content: 'Test prompt' } },
+                    { message: { role: 'assistant', content: 'Test response' } },
+                ],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new azureopenai_1.AzureOpenAiChatCompletionProvider('gpt-3.5-turbo', {
+            config: { dataSources },
+        });
+        const result = await provider.callApi(JSON.stringify([
+            { role: 'system', content: 'System prompt' },
+            { role: 'user', content: 'Test prompt' },
+        ]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test response');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+    });
+    it('AzureOpenAiChatCompletionProvider callApi with cache disabled', async () => {
+        (0, cache_1.disableCache)();
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                choices: [{ message: { content: 'Test output' } }],
+                usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new azureopenai_1.AzureOpenAiChatCompletionProvider('gpt-3.5-turbo');
+        const result = await provider.callApi(JSON.stringify([{ role: 'user', content: 'Test prompt' }]));
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+        expect(result.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+        (0, cache_1.enableCache)();
+    });
+    it('LlamaProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                content: 'Test output',
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new llama_1.LlamaProvider('llama.cpp');
+        const result = await provider.callApi('Test prompt');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+    });
+    it('OllamaCompletionProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn()
+                .mockResolvedValue(`{"model":"llama2:13b","created_at":"2023-08-08T21:50:34.898068Z","response":"Gre","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:34.929199Z","response":"at","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:34.959989Z","response":" question","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:34.992117Z","response":"!","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:35.023658Z","response":" The","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:35.0551Z","response":" sky","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:35.086103Z","response":" appears","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:35.117166Z","response":" blue","done":false}
+{"model":"llama2:13b","created_at":"2023-08-08T21:50:41.695299Z","done":true,"context":[1,29871,1,13,9314],"total_duration":10411943458,"load_duration":458333,"sample_count":217,"sample_duration":154566000,"prompt_eval_count":11,"prompt_eval_duration":3334582000,"eval_count":216,"eval_duration":6905134000}`),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new ollama_1.OllamaCompletionProvider('llama');
+        const result = await provider.callApi('Test prompt');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Great question! The sky appears blue');
+    });
+    it('OllamaChatProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn()
+                .mockResolvedValue(`{"model":"orca-mini","created_at":"2023-12-16T01:46:19.263682972Z","message":{"role":"assistant","content":" Because","images":null},"done":false}
+{"model":"orca-mini","created_at":"2023-12-16T01:46:19.275143974Z","message":{"role":"assistant","content":" of","images":null},"done":false}
+{"model":"orca-mini","created_at":"2023-12-16T01:46:19.288137727Z","message":{"role":"assistant","content":" Ray","images":null},"done":false}
+{"model":"orca-mini","created_at":"2023-12-16T01:46:19.301139709Z","message":{"role":"assistant","content":"leigh","images":null},"done":false}
+{"model":"orca-mini","created_at":"2023-12-16T01:46:19.311364699Z","message":{"role":"assistant","content":" scattering","images":null},"done":false}
+{"model":"orca-mini","created_at":"2023-12-16T01:46:19.324309782Z","message":{"role":"assistant","content":".","images":null},"done":false}
+{"model":"orca-mini","created_at":"2023-12-16T01:46:19.337165395Z","done":true,"total_duration":1486443841,"load_duration":1280794143,"prompt_eval_count":35,"prompt_eval_duration":142384000,"eval_count":6,"eval_duration":61912000}`),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new ollama_1.OllamaChatProvider('llama');
+        const result = await provider.callApi('Test prompt');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe(' Because of Rayleigh scattering.');
+    });
+    it('WebhookProvider callApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify({
+                output: 'Test output',
+            })),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new webhook_1.WebhookProvider('http://example.com/webhook');
+        const result = await provider.callApi('Test prompt');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.output).toBe('Test output');
+    });
+    describe.each([
+        ['Array format', [{ generated_text: 'Test output' }]], // Array format
+        ['Object format', { generated_text: 'Test output' }], // Object format
+    ])('HuggingfaceTextGenerationProvider callApi with %s', (format, mockedData) => {
+        it('returns expected output', async () => {
+            const mockResponse = {
+                text: jest.fn().mockResolvedValue(JSON.stringify(mockedData)),
+            };
+            jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+            const provider = new huggingface_1.HuggingfaceTextGenerationProvider('gpt2');
+            const result = await provider.callApi('Test prompt');
+            expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+            expect(result.output).toBe('Test output');
+        });
+    });
+    it('HuggingfaceFeatureExtractionProvider callEmbeddingApi', async () => {
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify([0.1, 0.2, 0.3, 0.4, 0.5])),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new huggingface_1.HuggingfaceFeatureExtractionProvider('distilbert-base-uncased');
+        const result = await provider.callEmbeddingApi('Test text');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.embedding).toEqual([0.1, 0.2, 0.3, 0.4, 0.5]);
+    });
+    it('HuggingfaceTextClassificationProvider callClassificationApi', async () => {
+        const mockClassification = [
+            [
+                {
+                    label: 'nothate',
+                    score: 0.9,
+                },
+                {
+                    label: 'hate',
+                    score: 0.1,
+                },
+            ],
+        ];
+        const mockResponse = {
+            text: jest.fn().mockResolvedValue(JSON.stringify(mockClassification)),
+        };
+        jest.mocked(node_fetch_1.default).mockResolvedValue(mockResponse);
+        const provider = new huggingface_1.HuggingfaceTextClassificationProvider('foo');
+        const result = await provider.callClassificationApi('Test text');
+        expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+        expect(result.classification).toEqual({
+            nothate: 0.9,
+            hate: 0.1,
+        });
+    });
+    describe('CloudflareAi', () => {
+        beforeAll(() => {
+            (0, cache_1.enableCache)();
+        });
+        const fetchMock = jest.mocked(node_fetch_1.default);
+        const cloudflareMinimumConfig = {
+            accountId: 'testAccountId',
+            apiKey: 'testApiKey',
+        };
+        const testModelName = '@cf/meta/llama-2-7b-chat-fp16';
+        // Token usage is not implemented for cloudflare so this is the default that
+        // is returned
+        const tokenUsageDefaultResponse = {
+            total: undefined,
+            prompt: undefined,
+            completion: undefined,
+        };
+        describe('CloudflareAiCompletionProvider', () => {
+            it('callApi with caching enabled', async () => {
+                const PROMPT = 'Test prompt for caching';
+                const provider = new cloudflare_ai_1.CloudflareAiCompletionProvider(testModelName, {
+                    config: cloudflareMinimumConfig,
+                });
+                const responsePayload = {
+                    success: true,
+                    errors: [],
+                    messages: [],
+                    result: {
+                        response: 'Test text output',
+                    },
+                };
+                const mockResponse = {
+                    text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
+                    ok: true,
+                };
+                fetchMock.mockResolvedValue(mockResponse);
+                const result = await provider.callApi(PROMPT);
+                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(result.output).toBe(responsePayload.result.response);
+                expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
+                const resultFromCache = await provider.callApi(PROMPT);
+                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(resultFromCache.output).toBe(responsePayload.result.response);
+                expect(resultFromCache.tokenUsage).toEqual(tokenUsageDefaultResponse);
+            });
+            it('callApi with caching disabled', async () => {
+                const PROMPT = 'test prompt without caching';
+                try {
+                    (0, cache_1.disableCache)();
+                    const provider = new cloudflare_ai_1.CloudflareAiCompletionProvider(testModelName, {
+                        config: cloudflareMinimumConfig,
+                    });
+                    const responsePayload = {
+                        success: true,
+                        errors: [],
+                        messages: [],
+                        result: {
+                            response: 'Test text output',
+                        },
+                    };
+                    const mockResponse = {
+                        text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
+                        ok: true,
+                    };
+                    fetchMock.mockResolvedValue(mockResponse);
+                    const result = await provider.callApi(PROMPT);
+                    expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                    expect(result.output).toBe(responsePayload.result.response);
+                    expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
+                    const resultFromCache = await provider.callApi(PROMPT);
+                    expect(node_fetch_1.default).toHaveBeenCalledTimes(2);
+                    expect(resultFromCache.output).toBe(responsePayload.result.response);
+                    expect(resultFromCache.tokenUsage).toEqual(tokenUsageDefaultResponse);
+                }
+                finally {
+                    (0, cache_1.enableCache)();
+                }
+            });
+            it('callApi handles cloudflare error properly', async () => {
+                const PROMPT = 'Test prompt for caching';
+                const provider = new cloudflare_ai_1.CloudflareAiCompletionProvider(testModelName, {
+                    config: cloudflareMinimumConfig,
+                });
+                const responsePayload = {
+                    success: false,
+                    errors: ['Some error occurred'],
+                    messages: [],
+                };
+                const mockResponse = {
+                    text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
+                    ok: true,
+                };
+                fetchMock.mockResolvedValue(mockResponse);
+                const result = await provider.callApi(PROMPT);
+                expect(result.error).toContain(JSON.stringify(responsePayload.errors));
+            });
+            it('Can be invoked with custom configuration', async () => {
+                const cloudflareChatConfig = {
+                    accountId: 'MADE_UP_ACCOUNT_ID',
+                    apiKey: 'MADE_UP_API_KEY',
+                    frequency_penalty: 10,
+                };
+                const rawProviderConfigs = [
+                    {
+                        [`cloudflare-ai:completion:${testModelName}`]: {
+                            config: cloudflareChatConfig,
+                        },
+                    },
+                ];
+                const providers = await (0, providers_1.loadApiProviders)(rawProviderConfigs);
+                expect(providers).toHaveLength(1);
+                expect(providers[0]).toBeInstanceOf(cloudflare_ai_1.CloudflareAiCompletionProvider);
+                const cfProvider = providers[0];
+                expect(cfProvider.config).toEqual(cloudflareChatConfig);
+                const PROMPT = 'Test prompt for custom configuration';
+                const responsePayload = {
+                    success: true,
+                    errors: [],
+                    messages: [],
+                    result: {
+                        response: 'Test text output',
+                    },
+                };
+                const mockResponse = {
+                    text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
+                    ok: true,
+                };
+                fetchMock.mockResolvedValue(mockResponse);
+                await cfProvider.callApi(PROMPT);
+                expect(fetchMock).toHaveBeenCalledTimes(1);
+                expect(fetchMock.mock.calls).toHaveLength(1);
+                const [url, options] = fetchMock.mock.calls[0] ?? [undefined, {}];
+                const { body, headers, method } = options;
+                expect(url).toContain(cloudflareChatConfig.accountId);
+                expect(method).toBe('POST');
+                expect(headers && 'Authorization' in headers ? headers['Authorization'] : '').toContain(cloudflareChatConfig.apiKey);
+                const hydratedBody = typeof body === 'string' ? JSON.parse(body) : body;
+                expect(hydratedBody.prompt).toBe(PROMPT);
+                const { accountId, apiKey, ...passThroughConfig } = cloudflareChatConfig;
+                const { prompt, ...bodyWithoutPrompt } = hydratedBody;
+                expect(bodyWithoutPrompt).toEqual(passThroughConfig);
+            });
+        });
+        describe('CloudflareAiChatCompletionProvider', () => {
+            it('Should handle chat provider', async () => {
+                const provider = new cloudflare_ai_1.CloudflareAiChatCompletionProvider(testModelName, {
+                    config: cloudflareMinimumConfig,
+                });
+                const responsePayload = {
+                    success: true,
+                    errors: [],
+                    messages: [],
+                    result: {
+                        response: 'Test text output',
+                    },
+                };
+                const mockResponse = {
+                    text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
+                    ok: true,
+                };
+                fetchMock.mockResolvedValue(mockResponse);
+                const result = await provider.callApi('Test chat prompt');
+                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(result.output).toBe(responsePayload.result.response);
+                expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
+            });
+        });
+        describe('CloudflareAiEmbeddingProvider', () => {
+            it('Should return embeddings in the proper format', async () => {
+                const provider = new cloudflare_ai_1.CloudflareAiEmbeddingProvider(testModelName, {
+                    config: cloudflareMinimumConfig,
+                });
+                const responsePayload = {
+                    success: true,
+                    errors: [],
+                    messages: [],
+                    result: {
+                        shape: [1, 3],
+                        data: [[0.02055364102125168, -0.013749595731496811, 0.0024201320484280586]],
+                    },
+                };
+                const mockResponse = {
+                    text: jest.fn().mockResolvedValue(JSON.stringify(responsePayload)),
+                    ok: true,
+                };
+                fetchMock.mockResolvedValue(mockResponse);
+                const result = await provider.callEmbeddingApi('Create embeddings from this');
+                expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+                expect(result.embedding).toEqual(responsePayload.result.data[0]);
+                expect(result.tokenUsage).toEqual(tokenUsageDefaultResponse);
+            });
+        });
+    });
+    describe.each([
+        ['python rag.py', 'python', ['rag.py']],
+        ['echo "hello world"', 'echo', ['hello world']],
+        ['./path/to/file.py run', './path/to/file.py', ['run']],
+        ['"/Path/To/My File.py"', '/Path/To/My File.py', []],
+    ])('ScriptCompletionProvider callApi with script %s', (script, inputFile, inputArgs) => {
+        it('returns expected output', async () => {
+            const mockResponse = 'Test script output';
+            const mockChildProcess = {
+                stdout: new stream_1.default.Readable(),
+                stderr: new stream_1.default.Readable(),
+            };
+            const execFileSpy = jest
+                .spyOn(child_process_1.default, 'execFile')
+                .mockImplementation((file, args, options, callback) => {
+                process.nextTick(() => callback && callback(null, Buffer.from(mockResponse), Buffer.from('')));
+                return mockChildProcess;
+            });
+            const provider = new scriptCompletion_1.ScriptCompletionProvider(script, {
+                config: {
+                    some_config_val: 42,
+                },
+            });
+            const result = await provider.callApi('Test prompt', {
+                vars: {
+                    var1: 'value 1',
+                    var2: 'value 2 "with some double "quotes""',
+                },
+            });
+            expect(result.output).toBe(mockResponse);
+            expect(execFileSpy).toHaveBeenCalledTimes(1);
+            expect(execFileSpy).toHaveBeenCalledWith(expect.stringContaining(inputFile), expect.arrayContaining(inputArgs.concat([
+                'Test prompt',
+                '{"config":{"some_config_val":42}}',
+                '{"vars":{"var1":"value 1","var2":"value 2 \\"with some double \\"quotes\\"\\""}}',
+            ])), expect.any(Object), expect.any(Function));
+            jest.restoreAllMocks();
+        });
+    });
+});
+describe('loadApiProvider', () => {
+    it('loadApiProvider with filepath', async () => {
+        const mockYamlContent = `id: 'openai:gpt-4'
+config:
+  key: 'value'`;
+        jest.mocked(fs.readFileSync).mockReturnValueOnce(mockYamlContent);
+        const provider = await (0, providers_1.loadApiProvider)('file://path/to/mock-provider-file.yaml');
+        expect(provider.id()).toBe('openai:gpt-4');
+        expect(fs.readFileSync).toHaveBeenCalledTimes(1);
+        expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringMatching(/path[\\\/]to[\\\/]mock-provider-file\.yaml/), 'utf8');
+    });
+    it('loadApiProvider with openai:chat', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openai:chat');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+    });
+    it('loadApiProvider with openai:completion', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openai:completion');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiCompletionProvider);
+    });
+    it('loadApiProvider with openai:assistant', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openai:assistant:foobar');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiAssistantProvider);
+    });
+    it('loadApiProvider with openai:chat:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openai:chat:gpt-3.5-turbo');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+    });
+    it('loadApiProvider with openai:completion:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openai:completion:text-davinci-003');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiCompletionProvider);
+    });
+    it('loadApiProvider with OpenAI finetuned model', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openai:chat:ft:gpt-3.5-turbo-0613:company-name::ID:');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        expect(provider.id()).toBe('openai:ft:gpt-3.5-turbo-0613:company-name::ID:');
+    });
+    it('loadApiProvider with azureopenai:completion:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('azureopenai:completion:text-davinci-003');
+        expect(provider).toBeInstanceOf(azureopenai_1.AzureOpenAiCompletionProvider);
+    });
+    it('loadApiProvider with azureopenai:chat:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('azureopenai:chat:gpt-3.5-turbo');
+        expect(provider).toBeInstanceOf(azureopenai_1.AzureOpenAiChatCompletionProvider);
+    });
+    it('loadApiProvider with anthropic:completion', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('anthropic:completion');
+        expect(provider).toBeInstanceOf(anthropic_1.AnthropicCompletionProvider);
+    });
+    it('loadApiProvider with anthropic:completion:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('anthropic:completion:claude-1');
+        expect(provider).toBeInstanceOf(anthropic_1.AnthropicCompletionProvider);
+    });
+    it('loadApiProvider with ollama:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('ollama:llama2:13b');
+        expect(provider).toBeInstanceOf(ollama_1.OllamaCompletionProvider);
+        expect(provider.id()).toBe('ollama:completion:llama2:13b');
+    });
+    it('loadApiProvider with ollama:completion:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('ollama:completion:llama2:13b');
+        expect(provider).toBeInstanceOf(ollama_1.OllamaCompletionProvider);
+        expect(provider.id()).toBe('ollama:completion:llama2:13b');
+    });
+    it('loadApiProvider with ollama:embedding:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('ollama:embedding:llama2:13b');
+        expect(provider).toBeInstanceOf(ollama_1.OllamaEmbeddingProvider);
+    });
+    it('loadApiProvider with ollama:embeddings:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('ollama:embeddings:llama2:13b');
+        expect(provider).toBeInstanceOf(ollama_1.OllamaEmbeddingProvider);
+    });
+    it('loadApiProvider with ollama:chat:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('ollama:chat:llama2:13b');
+        expect(provider).toBeInstanceOf(ollama_1.OllamaChatProvider);
+        expect(provider.id()).toBe('ollama:chat:llama2:13b');
+    });
+    it('loadApiProvider with llama:modelName', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('llama');
+        expect(provider).toBeInstanceOf(llama_1.LlamaProvider);
+    });
+    it('loadApiProvider with webhook', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('webhook:http://example.com/webhook');
+        expect(provider).toBeInstanceOf(webhook_1.WebhookProvider);
+    });
+    it('loadApiProvider with huggingface:text-generation', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('huggingface:text-generation:foobar/baz');
+        expect(provider).toBeInstanceOf(huggingface_1.HuggingfaceTextGenerationProvider);
+    });
+    it('loadApiProvider with huggingface:feature-extraction', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('huggingface:feature-extraction:foobar/baz');
+        expect(provider).toBeInstanceOf(huggingface_1.HuggingfaceFeatureExtractionProvider);
+    });
+    it('loadApiProvider with huggingface:text-classification', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('huggingface:text-classification:foobar/baz');
+        expect(provider).toBeInstanceOf(huggingface_1.HuggingfaceTextClassificationProvider);
+    });
+    it('loadApiProvider with hf:text-classification', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('hf:text-classification:foobar/baz');
+        expect(provider).toBeInstanceOf(huggingface_1.HuggingfaceTextClassificationProvider);
+    });
+    it('loadApiProvider with bedrock:completion', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('bedrock:completion:anthropic.claude-v2:1');
+        expect(provider).toBeInstanceOf(bedrock_1.AwsBedrockCompletionProvider);
+    });
+    it('loadApiProvider with openrouter', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('openrouter:mistralai/mistral-medium');
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        // Intentionally openai, because it's just a wrapper around openai
+        expect(provider.id()).toBe('mistralai/mistral-medium');
+    });
+    it('loadApiProvider with voyage', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('voyage:voyage-2');
+        expect(provider).toBeInstanceOf(voyage_1.VoyageEmbeddingProvider);
+        expect(provider.id()).toBe('voyage:voyage-2');
+    });
+    it('loadApiProvider with vertex:chat', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('vertex:chat:vertex-chat-model');
+        expect(provider).toBeInstanceOf(vertex_1.VertexChatProvider);
+        expect(provider.id()).toBe('vertex:vertex-chat-model');
+    });
+    it('loadApiProvider with vertex:embedding', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('vertex:embedding:vertex-embedding-model');
+        expect(provider).toBeInstanceOf(vertex_1.VertexEmbeddingProvider);
+        expect(provider.id()).toBe('vertex:vertex-embedding-model');
+    });
+    it('loadApiProvider with vertex:embeddings', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('vertex:embeddings:vertex-embedding-model');
+        expect(provider).toBeInstanceOf(vertex_1.VertexEmbeddingProvider);
+        expect(provider.id()).toBe('vertex:vertex-embedding-model');
+    });
+    it('loadApiProvider with vertex:modelname', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('vertex:vertex-chat-model');
+        expect(provider).toBeInstanceOf(vertex_1.VertexChatProvider);
+        expect(provider.id()).toBe('vertex:vertex-chat-model');
+    });
+    it('loadApiProvider with replicate:modelname', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('replicate:meta/llama3');
+        expect(provider).toBeInstanceOf(replicate_1.ReplicateProvider);
+        expect(provider.id()).toBe('replicate:meta/llama3');
+    });
+    it('loadApiProvider with replicate:modelname:version', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('replicate:meta/llama3:abc123');
+        expect(provider).toBeInstanceOf(replicate_1.ReplicateProvider);
+        expect(provider.id()).toBe('replicate:meta/llama3:abc123');
+    });
+    it('loadApiProvider with replicate:image', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('replicate:image:stability-ai/sdxl');
+        expect(provider).toBeInstanceOf(replicate_1.ReplicateImageProvider);
+        expect(provider.id()).toBe('replicate:stability-ai/sdxl');
+    });
+    it('loadApiProvider with replicate:image:version', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('replicate:image:stability-ai/sdxl:abc123');
+        expect(provider).toBeInstanceOf(replicate_1.ReplicateImageProvider);
+        expect(provider.id()).toBe('replicate:stability-ai/sdxl:abc123');
+    });
+    it('loadApiProvider with replicate:moderation', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('replicate:moderation:foo/bar');
+        expect(provider).toBeInstanceOf(replicate_1.ReplicateModerationProvider);
+        expect(provider.id()).toBe('replicate:foo/bar');
+    });
+    it('loadApiProvider with replicate:moderation:version', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('replicate:moderation:foo/bar:abc123');
+        expect(provider).toBeInstanceOf(replicate_1.ReplicateModerationProvider);
+        expect(provider.id()).toBe('replicate:foo/bar:abc123');
+    });
+    it('loadApiProvider with cloudflare-ai', async () => {
+        const supportedModelTypes = [
+            { modelType: 'chat', providerKlass: cloudflare_ai_1.CloudflareAiChatCompletionProvider },
+            { modelType: 'embedding', providerKlass: cloudflare_ai_1.CloudflareAiEmbeddingProvider },
+            { modelType: 'embeddings', providerKlass: cloudflare_ai_1.CloudflareAiEmbeddingProvider },
+            { modelType: 'completion', providerKlass: cloudflare_ai_1.CloudflareAiCompletionProvider },
+        ];
+        const unsupportedModelTypes = ['assistant'];
+        const modelName = 'mistralai/mistral-medium';
+        // Without any model type should throw an error
+        await expect((0, providers_1.loadApiProvider)(`cloudflare-ai:${modelName}`)).rejects.toThrow(/Unknown Cloudflare AI model type/);
+        for (const unsupportedModelType of unsupportedModelTypes) {
+            await expect((0, providers_1.loadApiProvider)(`cloudflare-ai:${unsupportedModelType}:${modelName}`)).rejects.toThrow(/Unknown Cloudflare AI model type/);
+        }
+        for (const { modelType, providerKlass } of supportedModelTypes) {
+            const cfProvider = await (0, providers_1.loadApiProvider)(`cloudflare-ai:${modelType}:${modelName}`);
+            const modelTypeForId = modelType === 'embeddings' ? 'embedding' : modelType;
+            expect(cfProvider.id()).toMatch(`cloudflare-ai:${modelTypeForId}:${modelName}`);
+            expect(cfProvider).toBeInstanceOf(providerKlass);
+        }
+    });
+    it('loadApiProvider with promptfoo:redteam:iterative', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('promptfoo:redteam:iterative');
+        expect(provider).toBeInstanceOf(iterative_1.default);
+        expect(provider.id()).toBe('promptfoo:redteam:iterative');
+    });
+    it('loadApiProvider with promptfoo:redteam:iterative:image', async () => {
+        const provider = await (0, providers_1.loadApiProvider)('promptfoo:redteam:iterative:image');
+        expect(provider).toBeInstanceOf(iterativeImage_1.default);
+        expect(provider.id()).toBe('promptfoo:redteam:iterative:image');
+    });
+    it('loadApiProvider with RawProviderConfig', async () => {
+        const rawProviderConfig = {
+            'openai:chat': {
+                id: 'test',
+                config: { foo: 'bar' },
+            },
+        };
+        const provider = await (0, providers_1.loadApiProvider)('openai:chat', {
+            options: rawProviderConfig['openai:chat'],
+        });
+        expect(provider).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+    });
+    it('loadApiProviders with ProviderFunction', async () => {
+        const providerFunction = async (prompt) => {
+            return {
+                output: `Output for ${prompt}`,
+                tokenUsage: { total: 10, prompt: 5, completion: 5 },
+            };
+        };
+        const providers = await (0, providers_1.loadApiProviders)(providerFunction);
+        expect(providers).toHaveLength(1);
+        expect(providers[0].id()).toBe('custom-function');
+        const response = await providers[0].callApi('Test prompt');
+        expect(response.output).toBe('Output for Test prompt');
+        expect(response.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
+    });
+    it('loadApiProviders with CustomApiProvider', async () => {
+        const providerPath = 'file://path/to/file.js';
+        class CustomApiProvider {
+            id() {
+                return 'custom-api-provider';
+            }
+            async callApi(input) {
+                return { output: `Processed ${input}` };
+            }
+        }
+        jest.mocked(esm_1.importModule).mockResolvedValue(CustomApiProvider);
+        const providers = await (0, providers_1.loadApiProviders)(providerPath);
+        expect(esm_1.importModule).toHaveBeenCalledWith(path.resolve('path/to/file.js'));
+        expect(providers).toHaveLength(1);
+        expect(providers[0].id()).toBe('custom-api-provider');
+        const response = await providers[0].callApi('Test input');
+        expect(response.output).toBe('Processed Test input');
+    });
+    it('loadApiProviders with CustomApiProvider, absolute path', async () => {
+        const providerPath = 'file:///absolute/path/to/file.js';
+        class CustomApiProvider {
+            id() {
+                return 'custom-api-provider';
+            }
+            async callApi(input) {
+                return { output: `Processed ${input}` };
+            }
+        }
+        jest.mocked(esm_1.importModule).mockResolvedValue(CustomApiProvider);
+        const providers = await (0, providers_1.loadApiProviders)(providerPath);
+        expect(esm_1.importModule).toHaveBeenCalledWith('/absolute/path/to/file.js');
+        expect(providers).toHaveLength(1);
+        expect(providers[0].id()).toBe('custom-api-provider');
+        const response = await providers[0].callApi('Test input');
+        expect(response.output).toBe('Processed Test input');
+    });
+    it('loadApiProviders with RawProviderConfig[]', async () => {
+        const rawProviderConfigs = [
+            {
+                'openai:chat:abc123': {
+                    config: { foo: 'bar' },
+                },
+            },
+            {
+                'openai:completion:def456': {
+                    config: { foo: 'bar' },
+                },
+            },
+            {
+                'anthropic:completion:ghi789': {
+                    config: { foo: 'bar' },
+                },
+            },
+        ];
+        const providers = await (0, providers_1.loadApiProviders)(rawProviderConfigs);
+        expect(providers).toHaveLength(3);
+        expect(providers[0]).toBeInstanceOf(openai_1.OpenAiChatCompletionProvider);
+        expect(providers[1]).toBeInstanceOf(openai_1.OpenAiCompletionProvider);
+        expect(providers[2]).toBeInstanceOf(anthropic_1.AnthropicCompletionProvider);
+    });
+    it('loadApiProvider sets provider.delay', async () => {
+        const providerOptions = {
+            id: 'test-delay',
+            config: {},
+            delay: 500,
+        };
+        const provider = await (0, providers_1.loadApiProvider)('echo', { options: providerOptions });
+        expect(provider.delay).toBe(500);
+    });
+});
+//# sourceMappingURL=providers.test.js.map