promptfoo 0.72.1 → 0.73.0

package/dist/test/evaluator.test.js

@@ -0,0 +1,922 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs"));
+const glob_1 = __importDefault(require("glob"));
+const path = __importStar(require("path"));
+const evaluator_1 = require("../src/evaluator");
+jest.mock('node-fetch', () => jest.fn());
+jest.mock('proxy-agent', () => ({
+    ProxyAgent: jest.fn().mockImplementation(() => ({})),
+}));
+jest.mock('glob', () => ({
+    globSync: jest.fn(),
+}));
+jest.mock('fs', () => ({
+    readFileSync: jest.fn(),
+    writeFileSync: jest.fn(),
+    statSync: jest.fn(),
+    readdirSync: jest.fn(),
+    existsSync: jest.fn(),
+    mkdirSync: jest.fn(),
+    promises: {
+        readFile: jest.fn(),
+    },
+}));
+jest.mock('../src/esm');
+jest.mock('../src/database', () => ({
+    getDb: jest.fn(),
+}));
+const mockApiProvider = {
+    id: jest.fn().mockReturnValue('test-provider'),
+    callApi: jest.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+    }),
+};
+const mockGradingApiProviderPasses = {
+    id: jest.fn().mockReturnValue('test-grading-provider'),
+    callApi: jest.fn().mockResolvedValue({
+        output: JSON.stringify({ pass: true, reason: 'Test grading output' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+    }),
+};
+const mockGradingApiProviderFails = {
+    id: jest.fn().mockReturnValue('test-grading-provider'),
+    callApi: jest.fn().mockResolvedValue({
+        output: JSON.stringify({ pass: false, reason: 'Grading failed reason' }),
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+    }),
+};
+function toPrompt(text) {
+    return { raw: text, label: text };
+}
+describe('evaluator', () => {
+    beforeEach(() => {
+        jest.clearAllMocks();
+    });
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+    it('evaluate with vars', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+            tests: [
+                {
+                    vars: { var1: 'value1', var2: 'value2' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+        expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with vars - no escaping', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+            tests: [
+                {
+                    vars: { var1: '1 < 2', var2: 'he said "hello world"...' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt 1 < 2 he said "hello world"...');
+        expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with vars as object', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ var1.prop1 }} {{ var2 }}')],
+            tests: [
+                {
+                    vars: { var1: { prop1: 'value1' }, var2: 'value2' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+        expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1.prop1 }} {{ var2 }}');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with named prompt', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [{ raw: 'Test prompt {{ var1 }} {{ var2 }}', label: 'test display name' }],
+            tests: [
+                {
+                    vars: { var1: 'value1', var2: 'value2' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+        expect(summary.results[0].prompt.label).toBe('test display name');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with multiple vars', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+            tests: [
+                {
+                    vars: { var1: ['value1', 'value3'], var2: ['value2', 'value4'] },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+        expect(summary.stats.successes).toBe(4);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 40, prompt: 20, completion: 20, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+        expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with multiple providers', async () => {
+        const testSuite = {
+            providers: [mockApiProvider, mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ var1 }} {{ var2 }}')],
+            tests: [
+                {
+                    vars: { var1: 'value1', var2: 'value2' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+        expect(summary.stats.successes).toBe(2);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 20, prompt: 10, completion: 10, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
+        expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate without tests', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt');
+        expect(summary.results[0].prompt.label).toBe('Test prompt');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate without tests with multiple providers', async () => {
+        const testSuite = {
+            providers: [mockApiProvider, mockApiProvider, mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
+        expect(summary.stats.successes).toBe(3);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 30, prompt: 15, completion: 15, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt');
+        expect(summary.results[0].prompt.label).toBe('Test prompt');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with expected value matching output', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'equals',
+                            value: 'Test output',
+                        },
+                    ],
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].success).toBe(true);
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with expected value not matching output', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'equals',
+                            value: 'Different output',
+                        },
+                    ],
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(0);
+        expect(summary.stats.failures).toBe(1);
+        expect(summary.results[0].success).toBe(false);
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with fn: expected value', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'javascript',
+                            value: 'output === "Test output";',
+                        },
+                    ],
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].success).toBe(true);
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with fn: expected value not matching output', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'javascript',
+                            value: 'output === "Different output";',
+                        },
+                    ],
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(0);
+        expect(summary.stats.failures).toBe(1);
+        expect(summary.results[0].success).toBe(false);
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with grading expected value', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'llm-rubric',
+                            value: 'output is a test output',
+                        },
+                    ],
+                },
+            ],
+            defaultTest: {
+                options: {
+                    provider: mockGradingApiProviderPasses,
+                },
+            },
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].success).toBe(true);
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with grading expected value does not pass', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'llm-rubric',
+                            value: 'output is a test output',
+                        },
+                    ],
+                },
+            ],
+            defaultTest: {
+                options: {
+                    provider: mockGradingApiProviderFails,
+                },
+            },
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(0);
+        expect(summary.stats.failures).toBe(1);
+        expect(summary.results[0].success).toBe(false);
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with transform option - default test', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            defaultTest: {
+                options: {
+                    transform: 'output + " postprocessed"',
+                },
+            },
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+    });
+    it('evaluate with transform option - single test', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'equals',
+                            value: 'Test output postprocessed',
+                        },
+                    ],
+                    options: {
+                        transform: 'output + " postprocessed"',
+                    },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+    });
+    it('evaluate with transform option - json provider', async () => {
+        const mockApiJsonProvider = {
+            id: jest.fn().mockReturnValue('test-provider-json'),
+            callApi: jest.fn().mockResolvedValue({
+                output: '{"output": "testing", "value": 123}',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+        };
+        const testSuite = {
+            providers: [mockApiJsonProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'equals',
+                            value: '123',
+                        },
+                    ],
+                    options: {
+                        transform: `JSON.parse(output).value`,
+                    },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiJsonProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe(123);
+    });
+    it('evaluate with provider transform', async () => {
+        const mockApiProviderWithTransform = {
+            id: jest.fn().mockReturnValue('test-provider-transform'),
+            callApi: jest.fn().mockResolvedValue({
+                output: 'Original output',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+            transform: '`Transformed: ${output}`',
+        };
+        const testSuite = {
+            providers: [mockApiProviderWithTransform],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    assert: [
+                        {
+                            type: 'equals',
+                            value: 'Transformed: Original output',
+                        },
+                    ],
+                    options: {}, // No test transform, relying on provider's transform
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('Transformed: Original output');
+    });
+    it('evaluate with providerPromptMap', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt 1'), toPrompt('Test prompt 2')],
+            providerPromptMap: {
+                'test-provider': ['Test prompt 1'],
+            },
+            tests: [
+                {
+                    vars: { var1: 'value1', var2: 'value2' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.results[0].prompt.raw).toBe('Test prompt 1');
+        expect(summary.results[0].prompt.label).toBe('Test prompt 1');
+        expect(summary.results[0].response?.output).toBe('Test output');
+    });
+    it('evaluate with allowed prompts filtering', async () => {
+        const mockApiProvider = {
+            id: jest.fn().mockReturnValue('test-provider'),
+            callApi: jest.fn().mockResolvedValue({
+                output: 'Test output',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+        };
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [
+                { raw: 'Test prompt 1', label: 'prompt1' },
+                { raw: 'Test prompt 2', label: 'prompt2' },
+                { raw: 'Test prompt 3', label: 'group1:prompt3' },
+            ],
+            providerPromptMap: {
+                'test-provider': ['prompt1', 'group1'],
+            },
+            tests: [
+                {
+                    vars: { var1: 'value1', var2: 'value2' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+        expect(summary).toMatchObject({
+            stats: {
+                successes: 2,
+                failures: 0,
+            },
+            results: [{ prompt: { label: 'prompt1' } }, { prompt: { label: 'group1:prompt3' } }],
+        });
+    });
+    it('evaluate with scenarios', async () => {
+        const mockApiProvider = {
+            id: jest.fn().mockReturnValue('test-provider'),
+            callApi: jest
+                .fn()
+                .mockResolvedValueOnce({
+                output: 'Hola mundo',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            })
+                .mockResolvedValueOnce({
+                output: 'Bonjour le monde',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+        };
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ language }}')],
+            scenarios: [
+                {
+                    config: [
+                        {
+                            vars: {
+                                language: 'Spanish',
+                                expectedHelloWorld: 'Hola mundo',
+                            },
+                        },
+                        {
+                            vars: {
+                                language: 'French',
+                                expectedHelloWorld: 'Bonjour le monde',
+                            },
+                        },
+                    ],
+                    tests: [
+                        {
+                            assert: [
+                                {
+                                    type: 'equals',
+                                    value: '{{expectedHelloWorld}}',
+                                },
+                            ],
+                        },
+                    ],
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+        expect(summary.stats.successes).toBe(2);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('Hola mundo');
+        expect(summary.results[1].response?.output).toBe('Bonjour le monde');
+    });
+    it('evaluate with scenarios and multiple vars', async () => {
+        const mockApiProvider = {
+            id: jest.fn().mockReturnValue('test-provider'),
+            callApi: jest
+                .fn()
+                .mockResolvedValueOnce({
+                output: 'Spanish Hola',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            })
+                .mockResolvedValueOnce({
+                output: 'Spanish Bonjour',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            })
+                .mockResolvedValueOnce({
+                output: 'French Hola',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            })
+                .mockResolvedValueOnce({
+                output: 'French Bonjour',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+        };
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt {{ language }} {{ greeting }}')],
+            scenarios: [
+                {
+                    config: [
+                        {
+                            vars: {
+                                language: ['Spanish', 'French'],
+                                greeting: ['Hola', 'Bonjour'],
+                            },
+                        },
+                    ],
+                    tests: [
+                        {
+                            assert: [
+                                {
+                                    type: 'equals',
+                                    value: '{{language}} {{greeting}}',
+                                },
+                            ],
+                        },
+                    ],
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
+        expect(summary.stats.successes).toBe(4);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('Spanish Hola');
+        expect(summary.results[1].response?.output).toBe('Spanish Bonjour');
+        expect(summary.results[2].response?.output).toBe('French Hola');
+        expect(summary.results[3].response?.output).toBe('French Bonjour');
+    });
+    it('evaluate with _conversation variable', async () => {
+        const mockApiProvider = {
+            id: jest.fn().mockReturnValue('test-provider'),
+            callApi: jest.fn().mockImplementation((prompt) => {
+                return Promise.resolve({
+                    output: prompt,
+                    tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                });
+            }),
+        };
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('{{ var1 }} {{ _conversation[0].output }}')],
+            tests: [
+                {
+                    vars: { var1: 'First run' },
+                },
+                {
+                    vars: { var1: 'Second run' },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
+        expect(summary.stats.successes).toBe(2);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('First run ');
+        expect(summary.results[1].response?.output).toBe('Second run First run ');
+    });
+    it('evaluate with labeled and unlabeled providers and providerPromptMap', async () => {
+        const mockLabeledProvider = {
+            id: () => 'labeled-provider-id',
+            label: 'Labeled Provider',
+            callApi: jest.fn().mockResolvedValue({
+                output: 'Labeled Provider Output',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+        };
+        const mockUnlabeledProvider = {
+            id: () => 'unlabeled-provider-id',
+            callApi: jest.fn().mockResolvedValue({
+                output: 'Unlabeled Provider Output',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+            }),
+        };
+        const testSuite = {
+            providers: [mockLabeledProvider, mockUnlabeledProvider],
+            prompts: [
+                {
+                    raw: 'Prompt 1',
+                    label: 'prompt1',
+                },
+                {
+                    raw: 'Prompt 2',
+                    label: 'prompt2',
+                },
+            ],
+            providerPromptMap: {
+                'Labeled Provider': ['prompt1'],
+                'unlabeled-provider-id': ['prompt2'],
+            },
+        };
+        await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
+            stats: expect.objectContaining({
+                successes: 2,
+                failures: 0,
+            }),
+            results: [
+                expect.objectContaining({
+                    provider: expect.objectContaining({
+                        id: 'labeled-provider-id',
+                        label: 'Labeled Provider',
+                    }),
+                    response: expect.objectContaining({
+                        output: 'Labeled Provider Output',
+                    }),
+                }),
+                expect.objectContaining({
+                    provider: expect.objectContaining({
+                        id: 'unlabeled-provider-id',
+                        label: undefined,
+                    }),
+                    response: expect.objectContaining({
+                        output: 'Unlabeled Provider Output',
+                    }),
+                }),
+            ],
+            table: expect.objectContaining({
+                head: expect.objectContaining({
+                    prompts: [
+                        expect.objectContaining({
+                            provider: 'Labeled Provider',
+                        }),
+                        expect.objectContaining({
+                            provider: 'unlabeled-provider-id',
+                        }),
+                    ],
+                }),
+            }),
+        }));
+        expect(mockLabeledProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(mockUnlabeledProvider.callApi).toHaveBeenCalledTimes(1);
+    });
+    it('should use the options from the test if they exist', async () => {
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Test prompt')],
+            tests: [
+                {
+                    vars: { var1: 'value1', var2: 'value2' },
+                    options: {
+                        transform: 'output + " postprocessed"',
+                    },
+                },
+            ],
+        };
+        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(summary.stats.successes).toBe(1);
+        expect(summary.stats.failures).toBe(0);
+        expect(summary.results[0].response?.output).toBe('Test output postprocessed');
+    });
+});
+describe('renderPrompt', () => {
+    it('should render a prompt with a single variable', async () => {
+        const prompt = toPrompt('Test prompt {{ var1 }}');
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, { var1: 'value1' }, {});
+        expect(renderedPrompt).toBe('Test prompt value1');
+    });
+    it('should render a JSON prompt', async () => {
+        const prompt = toPrompt('[{"text": "Test prompt "}, {"text": "{{ var1 }}"}]');
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, { var1: 'value1' }, {});
+        expect(renderedPrompt).toBe(JSON.stringify(JSON.parse('[{"text":"Test prompt "},{"text":"value1"}]'), null, 2));
+    });
+    it('should render a JSON prompt and escape the var string', async () => {
+        const prompt = toPrompt('[{"text": "Test prompt "}, {"text": "{{ var1 }}"}]');
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, { var1: 'He said "hello world!"' }, {});
+        expect(renderedPrompt).toBe(JSON.stringify(JSON.parse('[{"text":"Test prompt "},{"text":"He said \\"hello world!\\""}]'), null, 2));
+    });
+    it('should render a JSON prompt with nested JSON', async () => {
+        const prompt = toPrompt('[{"text": "Test prompt "}, {"text": "{{ var1 }}"}]');
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, { var1: '{"nested": "value1"}' }, {});
+        expect(renderedPrompt).toBe(JSON.stringify(JSON.parse('[{"text":"Test prompt "},{"text":"{\\"nested\\": \\"value1\\"}"}]'), null, 2));
+    });
+    it('should load external yaml files in renderPrompt', async () => {
+        const prompt = toPrompt('Test prompt with {{ var1 }}');
+        const vars = { var1: 'file://test.txt' };
+        const evaluateOptions = {};
+        // Mock fs.readFileSync to simulate loading a YAML file
+        jest.spyOn(fs, 'readFileSync').mockReturnValueOnce('loaded from file');
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, vars, evaluateOptions);
+        expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('test.txt'), 'utf8');
+        expect(renderedPrompt).toBe('Test prompt with loaded from file');
+    });
+    it('should load external js files in renderPrompt and execute the exported function', async () => {
+        const prompt = toPrompt('Test prompt with {{ var1 }}');
+        const vars = { var1: 'file:///path/to/testFunction.js' };
+        const evaluateOptions = {};
+        jest.doMock(path.resolve('/path/to/testFunction.js'), () => (varName, prompt, vars) => ({ output: `Dynamic value for ${varName}` }), { virtual: true });
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, vars, evaluateOptions);
+        expect(renderedPrompt).toBe('Test prompt with Dynamic value for var1');
+    });
+    it('should load external json files in renderPrompt and parse the JSON content', async () => {
+        const prompt = toPrompt('Test prompt with {{ var1 }}');
+        const vars = { var1: 'file:///path/to/testData.json' };
+        const evaluateOptions = {};
+        jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(JSON.stringify({ key: 'valueFromJson' }));
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, vars, evaluateOptions);
+        expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('testData.json'), 'utf8');
+        expect(renderedPrompt).toBe('Test prompt with {"key":"valueFromJson"}');
+    });
+    it('should load external yaml files in renderPrompt and parse the YAML content', async () => {
+        const prompt = toPrompt('Test prompt with {{ var1 }}');
+        const vars = { var1: 'file:///path/to/testData.yaml' };
+        const evaluateOptions = {};
+        jest.spyOn(fs, 'readFileSync').mockReturnValueOnce('key: valueFromYaml');
+        const renderedPrompt = await (0, evaluator_1.renderPrompt)(prompt, vars, evaluateOptions);
+        expect(fs.readFileSync).toHaveBeenCalledWith(expect.stringContaining('testData.yaml'), 'utf8');
+        expect(renderedPrompt).toBe('Test prompt with {"key":"valueFromYaml"}');
+    });
+});
+describe('resolveVariables', () => {
+    it('should replace placeholders with corresponding variable values', () => {
+        const variables = { final: '{{ my_greeting }}, {{name}}!', my_greeting: 'Hello', name: 'John' };
+        const expected = { final: 'Hello, John!', my_greeting: 'Hello', name: 'John' };
+        expect((0, evaluator_1.resolveVariables)(variables)).toEqual(expected);
+    });
+    it('should handle nested variable substitutions', () => {
+        const variables = { first: '{{second}}', second: '{{third}}', third: 'value' };
+        const expected = { first: 'value', second: 'value', third: 'value' };
+        expect((0, evaluator_1.resolveVariables)(variables)).toEqual(expected);
+    });
+    it('should not modify variables without placeholders', () => {
+        const variables = { greeting: 'Hello, world!', name: 'John' };
+        const expected = { greeting: 'Hello, world!', name: 'John' };
+        expect((0, evaluator_1.resolveVariables)(variables)).toEqual(expected);
+    });
+    it('should not fail if a variable is not found', () => {
+        const variables = { greeting: 'Hello, {{name}}!' };
+        expect((0, evaluator_1.resolveVariables)(variables)).toEqual({ greeting: 'Hello, {{name}}!' });
+    });
+    it('should not fail for unresolved placeholders', () => {
+        const variables = { greeting: 'Hello, {{name}}!', name: '{{unknown}}' };
+        expect((0, evaluator_1.resolveVariables)(variables)).toEqual({
+            greeting: 'Hello, {{unknown}}!',
+            name: '{{unknown}}',
+        });
+    });
+});
+describe('generateVarCombinations', () => {
+    it('should generate combinations for simple variables', () => {
+        const vars = { language: 'English', greeting: 'Hello' };
+        const expected = [{ language: 'English', greeting: 'Hello' }];
+        expect((0, evaluator_1.generateVarCombinations)(vars)).toEqual(expected);
+    });
+    it('should generate combinations for array variables', () => {
+        const vars = { language: ['English', 'French'], greeting: 'Hello' };
+        const expected = [
+            { language: 'English', greeting: 'Hello' },
+            { language: 'French', greeting: 'Hello' },
+        ];
+        expect((0, evaluator_1.generateVarCombinations)(vars)).toEqual(expected);
+    });
+    it('should handle file paths and expand them into combinations', () => {
+        const vars = { language: 'English', greeting: 'file:///path/to/greetings/*.txt' };
+        jest.spyOn(glob_1.default, 'globSync').mockReturnValue(['greeting1.txt', 'greeting2.txt']);
+        const expected = [
+            { language: 'English', greeting: 'file://greeting1.txt' },
+            { language: 'English', greeting: 'file://greeting2.txt' },
+        ];
+        expect((0, evaluator_1.generateVarCombinations)(vars)).toEqual(expected);
+    });
+    it('should correctly handle nested array variables', () => {
+        const vars = {
+            options: [
+                ['opt1', 'opt2'],
+                ['opt3', 'opt4'],
+            ],
+        };
+        const expected = [
+            {
+                options: [
+                    ['opt1', 'opt2'],
+                    ['opt3', 'opt4'],
+                ],
+            },
+        ];
+        expect((0, evaluator_1.generateVarCombinations)(vars)).toEqual(expected);
+    });
+    it('should return an empty array for empty input', () => {
+        const vars = {};
+        const expected = [{}];
+        expect((0, evaluator_1.generateVarCombinations)(vars)).toEqual(expected);
+    });
+});
+describe('isAllowedPrompt', () => {
+    const prompt1 = {
+        label: 'prompt1',
+        raw: '',
+    };
+    const prompt2 = {
+        label: 'group1:prompt2',
+        raw: '',
+    };
+    const prompt3 = {
+        label: 'group2:prompt3',
+        raw: '',
+    };
+    it('should return true if allowedPrompts is undefined', () => {
+        expect((0, evaluator_1.isAllowedPrompt)(prompt1, undefined)).toBe(true);
+    });
+    it('should return true if allowedPrompts includes the prompt label', () => {
+        expect((0, evaluator_1.isAllowedPrompt)(prompt1, ['prompt1', 'prompt2'])).toBe(true);
+    });
+    it('should return true if allowedPrompts includes a label that matches the start of the prompt label followed by a colon', () => {
+        expect((0, evaluator_1.isAllowedPrompt)(prompt2, ['group1'])).toBe(true);
+    });
+    it('should return false if allowedPrompts does not include the prompt label or any matching start label with a colon', () => {
+        expect((0, evaluator_1.isAllowedPrompt)(prompt3, ['group1', 'prompt2'])).toBe(false);
+    });
+    // TODO: What should the expected behavior of this test be?
+    it('should return false if allowedPrompts is an empty array', () => {
+        expect((0, evaluator_1.isAllowedPrompt)(prompt1, [])).toBe(false);
+    });
+});
+//# sourceMappingURL=evaluator.test.js.map