promptfoo 0.91.3 → 0.92.1

package/dist/test/evaluator.test.js

@@ -3,9 +3,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const crypto_1 = require("crypto");
 const glob_1 = __importDefault(require("glob"));
 const evaluator_1 = require("../src/evaluator");
 const evaluatorHelpers_1 = require("../src/evaluatorHelpers");
+const migrate_1 = require("../src/migrate");
+const eval_1 = __importDefault(require("../src/models/eval"));
 jest.mock('node-fetch', () => jest.fn());
 jest.mock('proxy-agent', () => ({
     ProxyAgent: jest.fn().mockImplementation(() => ({})),
@@ -13,21 +16,7 @@ jest.mock('proxy-agent', () => ({
 jest.mock('glob', () => ({
     globSync: jest.fn(),
 }));
-jest.mock('fs', () => ({
-    readFileSync: jest.fn(),
-    writeFileSync: jest.fn(),
-    statSync: jest.fn(),
-    readdirSync: jest.fn(),
-    existsSync: jest.fn(),
-    mkdirSync: jest.fn(),
-    promises: {
-        readFile: jest.fn(),
-    },
-}));
 jest.mock('../src/esm');
-jest.mock('../src/database', () => ({
-    getDb: jest.fn(),
-}));
 jest.mock('../src/logger');
 jest.mock('../src/evaluatorHelpers', () => ({
     ...jest.requireActual('../src/evaluatorHelpers'),
@@ -40,6 +29,13 @@ const mockApiProvider = {
         tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
     }),
 };
+const mockApiProvider2 = {
+    id: jest.fn().mockReturnValue('test-provider-2'),
+    callApi: jest.fn().mockResolvedValue({
+        output: 'Test output',
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+    }),
+};
 const mockGradingApiProviderPasses = {
     id: jest.fn().mockReturnValue('test-grading-provider'),
     callApi: jest.fn().mockResolvedValue({
@@ -58,6 +54,9 @@ function toPrompt(text) {
     return { raw: text, label: text };
 }
 describe('evaluator', () => {
+    beforeAll(async () => {
+        await (0, migrate_1.runDbMigrations)();
+    });
     beforeEach(() => {
         jest.clearAllMocks();
     });
@@ -74,7 +73,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -93,7 +94,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -112,7 +115,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -131,7 +136,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -150,7 +157,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
         expect(summary.stats.successes).toBe(4);
         expect(summary.stats.failures).toBe(0);
@@ -169,7 +178,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
         expect(summary.stats.successes).toBe(2);
         expect(summary.stats.failures).toBe(0);
@@ -183,7 +194,9 @@ describe('evaluator', () => {
             providers: [mockApiProvider],
             prompts: [toPrompt('Test prompt')],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -197,7 +210,9 @@ describe('evaluator', () => {
             providers: [mockApiProvider, mockApiProvider, mockApiProvider],
             prompts: [toPrompt('Test prompt')],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
         expect(summary.stats.successes).toBe(3);
         expect(summary.stats.failures).toBe(0);
@@ -221,7 +236,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -243,7 +260,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(0);
         expect(summary.stats.failures).toBe(1);
@@ -265,7 +284,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -287,7 +308,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(0);
         expect(summary.stats.failures).toBe(1);
@@ -314,7 +337,9 @@ describe('evaluator', () => {
                 },
             },
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -341,7 +366,9 @@ describe('evaluator', () => {
                 },
             },
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(0);
         expect(summary.stats.failures).toBe(1);
@@ -358,7 +385,9 @@ describe('evaluator', () => {
                 },
             },
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -382,7 +411,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -413,7 +444,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiJsonProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -443,7 +476,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -470,7 +505,10 @@ describe('evaluator', () => {
                 },
             },
         };
-        await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toEqual(expect.objectContaining({
             stats: expect.objectContaining({
                 successes: 2,
                 failures: 0,
@@ -534,7 +572,10 @@ describe('evaluator', () => {
                 },
             ],
         };
-        await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toEqual(expect.objectContaining({
             stats: expect.objectContaining({
                 successes: 1,
                 failures: 0,
@@ -562,7 +603,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -595,7 +638,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
         expect(summary).toMatchObject({
             stats: {
@@ -651,7 +696,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
         expect(summary.stats.successes).toBe(2);
         expect(summary.stats.failures).toBe(0);
@@ -706,7 +753,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
         expect(summary.stats.successes).toBe(4);
         expect(summary.stats.failures).toBe(0);
@@ -764,8 +813,10 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const result = await (0, evaluator_1.evaluate)(testSuite, {});
-        expect(result).toMatchObject({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toMatchObject({
             stats: {
                 successes: 2,
                 failures: 0,
@@ -787,7 +838,7 @@ describe('evaluator', () => {
                 }),
             ]),
         });
-        expect(result.table.body[0].test.metadata).toEqual({
+        expect(summary.results[0].testCase.metadata).toEqual({
             defaultKey: 'defaultValue',
             configKey: 'configValue',
             testKey: 'testValue',
@@ -807,8 +858,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
-        expect(summary.table.body[0].test.metadata).toEqual({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        expect(evalRecord.results[0].testCase.metadata).toEqual({
             defaultKey: 'defaultValue',
             testKey: 'testValue',
         });
@@ -835,7 +887,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
         expect(summary.stats.successes).toBe(2);
         expect(summary.stats.failures).toBe(0);
@@ -875,7 +929,10 @@ describe('evaluator', () => {
                 'unlabeled-provider-id': ['prompt2'],
             },
         };
-        await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toMatchObject({
             stats: expect.objectContaining({
                 successes: 2,
                 failures: 0,
@@ -900,19 +957,15 @@ describe('evaluator', () => {
                     }),
                 }),
             ],
-            table: expect.objectContaining({
-                head: expect.objectContaining({
-                    prompts: [
-                        expect.objectContaining({
-                            provider: 'Labeled Provider',
-                        }),
-                        expect.objectContaining({
-                            provider: 'unlabeled-provider-id',
-                        }),
-                    ],
-                }),
+        });
+        expect(evalRecord.prompts).toEqual(expect.arrayContaining([
+            expect.objectContaining({
+                provider: 'Labeled Provider',
             }),
-        }));
+            expect.objectContaining({
+                provider: 'unlabeled-provider-id',
+            }),
+        ]));
         expect(mockLabeledProvider.callApi).toHaveBeenCalledTimes(1);
         expect(mockUnlabeledProvider.callApi).toHaveBeenCalledTimes(1);
     });
@@ -929,7 +982,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -961,7 +1016,9 @@ describe('evaluator', () => {
                 },
             ],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
@@ -993,19 +1050,16 @@ describe('evaluator', () => {
                 },
             ],
         };
-        await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toMatchObject({
             stats: expect.objectContaining({
                 successes: 1,
                 failures: 0,
             }),
-            results: expect.arrayContaining([
-                expect.objectContaining({
-                    response: expect.objectContaining({
-                        output: 'Postprocess: Provider: Original output',
-                    }),
-                }),
-            ]),
-        }));
+        });
+        expect(summary.results[0].response?.output).toBe('Postprocess: Provider: Original output');
         expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
     });
     it('evaluate with provider transform, test transform, and test postprocess (deprecated)', async () => {
@@ -1035,7 +1089,10 @@ describe('evaluator', () => {
                 },
             ],
         };
-        await expect((0, evaluator_1.evaluate)(testSuite, {})).resolves.toEqual(expect.objectContaining({
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toMatchObject({
             stats: expect.objectContaining({
                 successes: 1,
                 failures: 0,
@@ -1047,7 +1104,7 @@ describe('evaluator', () => {
                     }),
                 }),
             ]),
-        }));
+        });
         expect(mockApiProviderWithTransform.callApi).toHaveBeenCalledTimes(1);
     });
     it('evaluate with no output', async () => {
@@ -1063,7 +1120,9 @@ describe('evaluator', () => {
             prompts: [toPrompt('Test prompt')],
             tests: [],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(summary.stats.successes).toBe(0);
         expect(summary.stats.failures).toBe(1);
         expect(summary.results[0].error).toBe('No output');
@@ -1084,7 +1143,9 @@ describe('evaluator', () => {
             prompts: [toPrompt('Test prompt')],
             tests: [],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
         expect(summary.results[0].success).toBe(true);
@@ -1126,7 +1187,9 @@ describe('evaluator', () => {
             ],
             tests: [{ vars: { problem: '8x + 31 = 2' } }],
         };
-        const summary = await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
@@ -1167,7 +1230,8 @@ describe('evaluator', () => {
         };
         const mockedRunExtensionHook = jest.mocked(evaluatorHelpers_1.runExtensionHook);
         mockedRunExtensionHook.mockClear();
-        await (0, evaluator_1.evaluate)(testSuite, {});
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
         // Check if runExtensionHook was called 4 times (beforeAll, beforeEach, afterEach, afterAll)
         expect(mockedRunExtensionHook).toHaveBeenCalledTimes(4);
         // Check beforeAll call
@@ -1207,6 +1271,20 @@ describe('evaluator', () => {
             suite: testSuite,
         }));
     });
+    it('should handle multiple providers', async () => {
+        const testSuite = {
+            providers: [mockApiProvider, mockApiProvider2],
+            prompts: [toPrompt('Test prompt')],
+            tests: [],
+        };
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary.stats.successes).toBe(2);
+        expect(summary.stats.failures).toBe(0);
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+        expect(mockApiProvider2.callApi).toHaveBeenCalledTimes(1);
+    });
 });
 describe('generateVarCombinations', () => {
     it('should generate combinations for simple variables', () => {