promptfoo 0.93.2 → 0.94.0

package/dist/test/assertions.test.js

@@ -28,7 +28,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const dedent_1 = __importDefault(require("dedent"));
 const fs = __importStar(require("fs"));
-const node_fetch_1 = require("node-fetch");
 const path = __importStar(require("path"));
 const assertions_1 = require("../src/assertions");
 const fetch_1 = require("../src/fetch");
@@ -74,8 +73,13 @@ jest.mock('../src/esm');
 jest.mock('../src/database', () => ({
     getDb: jest.fn(),
 }));
+jest.mock('path', () => ({
+    ...jest.requireActual('path'),
+    resolve: jest.fn(jest.requireActual('path').resolve),
+    extname: jest.fn(jest.requireActual('path').extname),
+}));
 jest.mock('../src/cliState', () => ({
-    basePath: '/config_path',
+    basePath: '/base/path',
 }));
 jest.mock('../src/matchers', () => {
     const actual = jest.requireActual('../src/matchers');
@@ -133,7 +137,7 @@ describe('runAssertions', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test,
             providerResponse: { output },
         });
@@ -146,7 +150,7 @@ describe('runAssertions', () => {
         const output = 'Actual output';
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test,
             providerResponse: { output },
         });
@@ -159,7 +163,7 @@ describe('runAssertions', () => {
         const output = { key: 'value' };
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test,
             providerResponse: { output },
         });
@@ -171,7 +175,7 @@ describe('runAssertions', () => {
     it('should fail when combined score is less than threshold', async () => {
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test: {
                 threshold: 0.5,
                 assert: [
@@ -197,7 +201,7 @@ describe('runAssertions', () => {
     it('should pass when combined score is greater than threshold', async () => {
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test: {
                 threshold: 0.25,
                 assert: [
@@ -222,7 +226,7 @@ describe('runAssertions', () => {
     });
     describe('assert-set', () => {
         const prompt = 'Some prompt';
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4');
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
         it('assert-set success', async () => {
             const output = 'Expected output';
             const test = {
@@ -602,7 +606,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertion,
             test: {},
             providerResponse: { output },
@@ -616,7 +620,7 @@ describe('runAssertion', () => {
         const output = 'Actual output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertion,
             test: {},
             providerResponse: { output },
@@ -637,7 +641,7 @@ describe('runAssertion', () => {
             assertion: notEqualsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -651,7 +655,7 @@ describe('runAssertion', () => {
             assertion: notEqualsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -662,7 +666,7 @@ describe('runAssertion', () => {
         const output = { key: 'value' };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertion,
             test: {},
             providerResponse: { output },
@@ -676,7 +680,7 @@ describe('runAssertion', () => {
         const output = { key: 'value' };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertionWithObject,
             test: {},
             providerResponse: { output },
@@ -690,7 +694,7 @@ describe('runAssertion', () => {
         const output = { key: 'not value' };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertionWithObject,
             test: {},
             providerResponse: { output },
@@ -709,7 +713,7 @@ describe('runAssertion', () => {
         const output = '{"key":"value"}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -729,7 +733,7 @@ describe('runAssertion', () => {
         const output = '{"key":"not value"}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -744,7 +748,7 @@ describe('runAssertion', () => {
         const output = '{"key":"value"}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -758,7 +762,7 @@ describe('runAssertion', () => {
         const output = 'Not valid JSON';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -772,7 +776,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": 80.123, "longitude": -1}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -786,7 +790,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": "high", "longitude": [-1]}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -800,7 +804,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": 80.123, "longitude": -1}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchemaYamlString,
             test: {},
             providerResponse: { output },
@@ -814,7 +818,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": "high", "longitude": [-1]}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchemaYamlString,
             test: {},
             providerResponse: { output },
@@ -838,7 +842,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: { type: 'is-json', value: schemaWithFormat },
             test: {},
             providerResponse: { output },
@@ -862,7 +866,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: { type: 'is-json', value: schemaWithFormat },
             test: {},
             providerResponse: { output },
@@ -896,7 +900,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": 80.123, "longitude": -1}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -931,7 +935,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": "high", "longitude": [-1]}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -946,7 +950,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -960,7 +964,7 @@ describe('runAssertion', () => {
         const output = 'SELECT * FROM orders ORDERY BY order_date';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -974,7 +978,7 @@ describe('runAssertion', () => {
         const output = 'SELECT * FROM orders ORDERY BY order_date';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: notIsSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -988,7 +992,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: notIsSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -1002,7 +1006,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabase,
             test: {},
             providerResponse: { output },
@@ -1016,7 +1020,7 @@ describe('runAssertion', () => {
         const output = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabase,
             test: {},
             providerResponse: { output },
@@ -1030,7 +1034,7 @@ describe('runAssertion', () => {
         const output = 'SELECT * FROM departments WHERE department_id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
             test: {},
             providerResponse: { output },
@@ -1044,7 +1048,7 @@ describe('runAssertion', () => {
         const output = 'UPDATE employees SET department_id = 2 WHERE employee_id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
             test: {},
             providerResponse: { output },
@@ -1058,7 +1062,7 @@ describe('runAssertion', () => {
         const output = 'SELECT name FROM t';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
             test: {},
             providerResponse: { output },
@@ -1072,7 +1076,7 @@ describe('runAssertion', () => {
         const output = 'SELECT age FROM a WHERE id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
             test: {},
             providerResponse: { output },
@@ -1086,7 +1090,7 @@ describe('runAssertion', () => {
         const output = 'SELECT name FROM departments';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1100,7 +1104,7 @@ describe('runAssertion', () => {
         const output = `INSERT INTO departments (name) VALUES ('HR')`;
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1114,7 +1118,7 @@ describe('runAssertion', () => {
         const output = 'UPDATE a SET id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1128,7 +1132,7 @@ describe('runAssertion', () => {
         const output = `DELETE FROM employees;`;
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1142,7 +1146,7 @@ describe('runAssertion', () => {
         const output = 'wassup\n```\nSELECT id, name FROM users\n```\nyolo';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1158,7 +1162,7 @@ describe('runAssertion', () => {
         const output = 'wassup\n```sql\nSELECT id, name FROM users\n```\nyolo';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1174,7 +1178,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1190,7 +1194,7 @@ describe('runAssertion', () => {
         const output = 'nothin';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1205,7 +1209,7 @@ describe('runAssertion', () => {
         const output = '```python\nprint("Hello, World!")\n```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1220,7 +1224,7 @@ describe('runAssertion', () => {
         const output = 'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} \n\n blah blah';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -1234,7 +1238,7 @@ describe('runAssertion', () => {
         const output = 'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} another {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}}\n\n blah blah';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -1248,7 +1252,7 @@ describe('runAssertion', () => {
         const output = 'Not valid JSON';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -1262,7 +1266,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -1276,7 +1280,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -1310,7 +1314,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -1345,7 +1349,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -1360,7 +1364,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -1374,7 +1378,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertion,
             test: {},
             providerResponse: { output },
@@ -1388,7 +1392,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithNumber,
             test: {},
             providerResponse: { output },
@@ -1403,7 +1407,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptBooleanAssertionWithConfig,
             test: {},
             providerResponse: { output },
@@ -1418,7 +1422,7 @@ describe('runAssertion', () => {
         const output = 'Expected output with some extra characters';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptBooleanAssertionWithConfig,
             test: {},
             providerResponse: { output },
@@ -1433,7 +1437,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithNumberAndThreshold,
             test: {},
             providerResponse: { output },
@@ -1448,7 +1452,7 @@ describe('runAssertion', () => {
         const output = '';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithNumberAndThreshold,
             test: {},
             providerResponse: { output },
@@ -1467,7 +1471,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -1482,7 +1486,7 @@ describe('runAssertion', () => {
         const output = 'Different output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertion,
             test: {},
             providerResponse: { output },
@@ -1500,7 +1504,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'variable value',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: { vars: { foo: 'Expected output' } },
             providerResponse: { output },
@@ -1518,7 +1522,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithVars,
             test: { vars: { foo: 'bar' } },
             providerResponse: { output },
@@ -1536,7 +1540,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithVars,
             test: { vars: { foo: 'bar' } },
             providerResponse: { output },
@@ -1550,7 +1554,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptFunctionAssertion,
             test: {},
             providerResponse: { output },
@@ -1565,7 +1569,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptFunctionFailAssertion,
             test: {},
             providerResponse: { output },
@@ -1583,7 +1587,7 @@ describe('runAssertion', () => {
             assertion: javascriptMultilineStringAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1597,7 +1601,7 @@ describe('runAssertion', () => {
             assertion: javascriptMultilineStringAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1615,7 +1619,7 @@ describe('runAssertion', () => {
             assertion: notContainsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1629,7 +1633,7 @@ describe('runAssertion', () => {
             assertion: notContainsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1648,7 +1652,7 @@ describe('runAssertion', () => {
             assertion: containsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1662,7 +1666,7 @@ describe('runAssertion', () => {
             assertion: containsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1681,7 +1685,7 @@ describe('runAssertion', () => {
             assertion: notContainsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1695,7 +1699,7 @@ describe('runAssertion', () => {
             assertion: notContainsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1714,7 +1718,7 @@ describe('runAssertion', () => {
             assertion: containsAnyAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1728,7 +1732,7 @@ describe('runAssertion', () => {
             assertion: containsAnyAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1745,7 +1749,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1762,7 +1766,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1781,7 +1785,7 @@ describe('runAssertion', () => {
             assertion: containsAllAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1795,7 +1799,7 @@ describe('runAssertion', () => {
             assertion: containsAllAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1812,7 +1816,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1829,7 +1833,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1848,7 +1852,7 @@ describe('runAssertion', () => {
             assertion: containsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1862,7 +1866,7 @@ describe('runAssertion', () => {
             assertion: containsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1881,7 +1885,7 @@ describe('runAssertion', () => {
             assertion: notContainsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1895,7 +1899,7 @@ describe('runAssertion', () => {
             assertion: notContainsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1909,7 +1913,7 @@ describe('runAssertion', () => {
     };
     it('should pass when the webhook assertion passes', async () => {
         const output = 'Expected output';
-        jest.mocked(fetch_1.fetchWithRetries).mockImplementation(() => Promise.resolve(new node_fetch_1.Response(JSON.stringify({ pass: true }), {
+        jest.mocked(fetch_1.fetchWithRetries).mockImplementation(() => Promise.resolve(new Response(JSON.stringify({ pass: true }), {
             status: 200,
             headers: { 'Content-Type': 'application/json' },
         })));
@@ -1918,7 +1922,7 @@ describe('runAssertion', () => {
             assertion: webhookAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1927,7 +1931,7 @@ describe('runAssertion', () => {
     });
     it('should fail when the webhook assertion fails', async () => {
         const output = 'Different output';
-        jest.mocked(fetch_1.fetchWithRetries).mockImplementation(() => Promise.resolve(new node_fetch_1.Response(JSON.stringify({ pass: false }), {
+        jest.mocked(fetch_1.fetchWithRetries).mockImplementation(() => Promise.resolve(new Response(JSON.stringify({ pass: false }), {
             status: 200,
             headers: { 'Content-Type': 'application/json' },
         })));
@@ -1936,7 +1940,7 @@ describe('runAssertion', () => {
             assertion: webhookAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1945,7 +1949,7 @@ describe('runAssertion', () => {
     });
     it('should fail when the webhook returns an error', async () => {
         const output = 'Expected output';
-        jest.mocked(fetch_1.fetchWithRetries).mockImplementation(() => Promise.resolve(new node_fetch_1.Response('', {
+        jest.mocked(fetch_1.fetchWithRetries).mockImplementation(() => Promise.resolve(new Response('', {
             status: 500,
             headers: { 'Content-Type': 'application/json' },
         })));
@@ -1954,7 +1958,7 @@ describe('runAssertion', () => {
             assertion: webhookAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1974,7 +1978,7 @@ describe('runAssertion', () => {
             assertion: rougeNAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1988,7 +1992,7 @@ describe('runAssertion', () => {
             assertion: rougeNAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -2007,7 +2011,7 @@ describe('runAssertion', () => {
             assertion: startsWithAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -2021,7 +2025,7 @@ describe('runAssertion', () => {
             assertion: startsWithAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -2057,7 +2061,7 @@ describe('runAssertion', () => {
             assertion,
             test,
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -2074,7 +2078,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: levenshteinAssertion,
             test: {},
             providerResponse: { output },
@@ -2088,7 +2092,7 @@ describe('runAssertion', () => {
         const output = 'Different output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: levenshteinAssertion,
             test: {},
             providerResponse: { output },
@@ -2140,7 +2144,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2158,14 +2162,14 @@ describe('runAssertion', () => {
     it('should resolve js paths relative to the configuration file', async () => {
         const output = 'Expected output';
         const mockFn = jest.fn((output) => output === 'Expected output');
-        jest.doMock(path.resolve('/config_path/path/to/assert.js'), () => mockFn, { virtual: true });
+        jest.doMock(path.resolve('/base/path/path/to/assert.js'), () => mockFn, { virtual: true });
         const fileAssertion = {
             type: 'javascript',
             value: 'file://./path/to/assert.js',
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2190,7 +2194,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: pythonAssertion,
             test: {},
             providerResponse: { output },
@@ -2253,7 +2257,7 @@ describe('runAssertion', () => {
         jest.mocked(wrapper_1.runPythonCode).mockResolvedValueOnce(resolvedValue);
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: pythonAssertion,
             test: {},
             providerResponse: { output },
@@ -2297,7 +2301,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2327,7 +2331,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2348,7 +2352,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2366,7 +2370,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2384,7 +2388,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             await expect((0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2397,7 +2401,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2415,7 +2419,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             await expect((0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                 },
@@ -2428,7 +2432,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -3277,6 +3281,80 @@ describe('runAssertion', () => {
             })).rejects.toThrow('context-faithfulness assertion type must have a string output');
         });
     });
+    describe('file references', () => {
+        it('should handle file reference in string value', async () => {
+            const assertion = {
+                type: 'equals',
+                value: 'file://expected_output.txt',
+            };
+            const expectedContent = 'Expected output';
+            jest.mocked(fs.readFileSync).mockReturnValue(expectedContent);
+            jest.mocked(path.resolve).mockReturnValue('/base/path/expected_output.txt');
+            jest.mocked(path.extname).mockReturnValue('.txt');
+            const result = await (0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: 'Expected output' },
+            });
+            expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/expected_output.txt', 'utf8');
+            expect(result.pass).toBe(true);
+        });
+        it('should handle file references in array values', async () => {
+            const assertion = {
+                type: 'contains-any',
+                value: ['The expected output', 'string output', 'file://my_expected_output.txt'],
+            };
+            const fileContent = 'file content';
+            jest.mocked(fs.readFileSync).mockReturnValue(fileContent);
+            jest.mocked(path.resolve).mockReturnValue('/base/path/my_expected_output.txt');
+            jest.mocked(path.extname).mockReturnValue('.txt');
+            await expect((0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: 'file content' },
+            })).resolves.toEqual(expect.objectContaining({
+                pass: true,
+            }));
+            expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/my_expected_output.txt', 'utf8');
+            await expect((0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: 'string output' },
+            })).resolves.toEqual(expect.objectContaining({
+                pass: true,
+            }));
+        });
+        it('should handle file reference in object value', async () => {
+            const assertion = {
+                type: 'is-json',
+                value: 'file://schema.json',
+            };
+            const schemaContent = JSON.stringify({
+                type: 'object',
+                properties: {
+                    key: { type: 'string' },
+                },
+            });
+            jest.mocked(fs.readFileSync).mockReturnValue(schemaContent);
+            jest.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
+            jest.mocked(path.extname).mockReturnValue('.json');
+            const result = await (0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: '{"key": "value"}' },
+            });
+            expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
+            expect(result.pass).toBe(true);
+        });
+    });
 });
 describe('validateXml', () => {
     it('should validate a simple valid XML string', () => {