promptfoo 0.93.1 → 0.93.3

package/dist/test/assertions.test.js

@@ -74,8 +74,13 @@ jest.mock('../src/esm');
 jest.mock('../src/database', () => ({
     getDb: jest.fn(),
 }));
+jest.mock('path', () => ({
+    ...jest.requireActual('path'),
+    resolve: jest.fn(jest.requireActual('path').resolve),
+    extname: jest.fn(jest.requireActual('path').extname),
+}));
 jest.mock('../src/cliState', () => ({
-    basePath: '/config_path',
+    basePath: '/base/path',
 }));
 jest.mock('../src/matchers', () => {
     const actual = jest.requireActual('../src/matchers');
@@ -133,7 +138,7 @@ describe('runAssertions', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test,
             providerResponse: { output },
         });
@@ -146,7 +151,7 @@ describe('runAssertions', () => {
         const output = 'Actual output';
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test,
             providerResponse: { output },
         });
@@ -159,7 +164,7 @@ describe('runAssertions', () => {
         const output = { key: 'value' };
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test,
             providerResponse: { output },
         });
@@ -171,7 +176,7 @@ describe('runAssertions', () => {
     it('should fail when combined score is less than threshold', async () => {
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test: {
                 threshold: 0.5,
                 assert: [
@@ -197,7 +202,7 @@ describe('runAssertions', () => {
     it('should pass when combined score is greater than threshold', async () => {
         const result = await (0, assertions_1.runAssertions)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             test: {
                 threshold: 0.25,
                 assert: [
@@ -222,7 +227,7 @@ describe('runAssertions', () => {
     });
     describe('assert-set', () => {
         const prompt = 'Some prompt';
-        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4');
+        const provider = new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini');
         it('assert-set success', async () => {
             const output = 'Expected output';
             const test = {
@@ -602,7 +607,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertion,
             test: {},
             providerResponse: { output },
@@ -616,7 +621,7 @@ describe('runAssertion', () => {
         const output = 'Actual output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertion,
             test: {},
             providerResponse: { output },
@@ -637,7 +642,7 @@ describe('runAssertion', () => {
             assertion: notEqualsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -651,7 +656,7 @@ describe('runAssertion', () => {
             assertion: notEqualsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -662,7 +667,7 @@ describe('runAssertion', () => {
         const output = { key: 'value' };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertion,
             test: {},
             providerResponse: { output },
@@ -676,7 +681,7 @@ describe('runAssertion', () => {
         const output = { key: 'value' };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertionWithObject,
             test: {},
             providerResponse: { output },
@@ -690,7 +695,7 @@ describe('runAssertion', () => {
         const output = { key: 'not value' };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: equalityAssertionWithObject,
             test: {},
             providerResponse: { output },
@@ -709,7 +714,7 @@ describe('runAssertion', () => {
         const output = '{"key":"value"}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -729,7 +734,7 @@ describe('runAssertion', () => {
         const output = '{"key":"not value"}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -744,7 +749,7 @@ describe('runAssertion', () => {
         const output = '{"key":"value"}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -758,7 +763,7 @@ describe('runAssertion', () => {
         const output = 'Not valid JSON';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -772,7 +777,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": 80.123, "longitude": -1}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -786,7 +791,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": "high", "longitude": [-1]}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -800,7 +805,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": 80.123, "longitude": -1}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchemaYamlString,
             test: {},
             providerResponse: { output },
@@ -814,7 +819,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": "high", "longitude": [-1]}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isJsonAssertionWithSchemaYamlString,
             test: {},
             providerResponse: { output },
@@ -838,7 +843,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: { type: 'is-json', value: schemaWithFormat },
             test: {},
             providerResponse: { output },
@@ -862,7 +867,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: { type: 'is-json', value: schemaWithFormat },
             test: {},
             providerResponse: { output },
@@ -896,7 +901,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": 80.123, "longitude": -1}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -931,7 +936,7 @@ describe('runAssertion', () => {
         const output = '{"latitude": "high", "longitude": [-1]}';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -946,7 +951,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -960,7 +965,7 @@ describe('runAssertion', () => {
         const output = 'SELECT * FROM orders ORDERY BY order_date';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -974,7 +979,7 @@ describe('runAssertion', () => {
         const output = 'SELECT * FROM orders ORDERY BY order_date';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: notIsSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -988,7 +993,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: notIsSqlAssertion,
             test: {},
             providerResponse: { output },
@@ -1002,7 +1007,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabase,
             test: {},
             providerResponse: { output },
@@ -1016,7 +1021,7 @@ describe('runAssertion', () => {
         const output = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabase,
             test: {},
             providerResponse: { output },
@@ -1030,7 +1035,7 @@ describe('runAssertion', () => {
         const output = 'SELECT * FROM departments WHERE department_id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
             test: {},
             providerResponse: { output },
@@ -1044,7 +1049,7 @@ describe('runAssertion', () => {
         const output = 'UPDATE employees SET department_id = 2 WHERE employee_id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
             test: {},
             providerResponse: { output },
@@ -1058,7 +1063,7 @@ describe('runAssertion', () => {
         const output = 'SELECT name FROM t';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
             test: {},
             providerResponse: { output },
@@ -1072,7 +1077,7 @@ describe('runAssertion', () => {
         const output = 'SELECT age FROM a WHERE id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
             test: {},
             providerResponse: { output },
@@ -1086,7 +1091,7 @@ describe('runAssertion', () => {
         const output = 'SELECT name FROM departments';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1100,7 +1105,7 @@ describe('runAssertion', () => {
         const output = `INSERT INTO departments (name) VALUES ('HR')`;
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1114,7 +1119,7 @@ describe('runAssertion', () => {
         const output = 'UPDATE a SET id = 1';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1128,7 +1133,7 @@ describe('runAssertion', () => {
         const output = `DELETE FROM employees;`;
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: isSqlAssertionWithDatabaseAndBothList,
             test: {},
             providerResponse: { output },
@@ -1142,7 +1147,7 @@ describe('runAssertion', () => {
         const output = 'wassup\n```\nSELECT id, name FROM users\n```\nyolo';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1158,7 +1163,7 @@ describe('runAssertion', () => {
         const output = 'wassup\n```sql\nSELECT id, name FROM users\n```\nyolo';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1174,7 +1179,7 @@ describe('runAssertion', () => {
         const output = 'SELECT id, name FROM users';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1190,7 +1195,7 @@ describe('runAssertion', () => {
         const output = 'nothin';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1205,7 +1210,7 @@ describe('runAssertion', () => {
         const output = '```python\nprint("Hello, World!")\n```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: {
                 type: 'contains-sql',
             },
@@ -1220,7 +1225,7 @@ describe('runAssertion', () => {
         const output = 'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} \n\n blah blah';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -1234,7 +1239,7 @@ describe('runAssertion', () => {
         const output = 'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} another {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}}\n\n blah blah';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -1248,7 +1253,7 @@ describe('runAssertion', () => {
         const output = 'Not valid JSON';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertion,
             test: {},
             providerResponse: { output },
@@ -1262,7 +1267,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -1276,7 +1281,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -1310,7 +1315,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -1345,7 +1350,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -1360,7 +1365,7 @@ describe('runAssertion', () => {
         const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: containsJsonAssertionWithSchema,
             test: {},
             providerResponse: { output },
@@ -1374,7 +1379,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertion,
             test: {},
             providerResponse: { output },
@@ -1388,7 +1393,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithNumber,
             test: {},
             providerResponse: { output },
@@ -1403,7 +1408,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptBooleanAssertionWithConfig,
             test: {},
             providerResponse: { output },
@@ -1418,7 +1423,7 @@ describe('runAssertion', () => {
         const output = 'Expected output with some extra characters';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptBooleanAssertionWithConfig,
             test: {},
             providerResponse: { output },
@@ -1433,7 +1438,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithNumberAndThreshold,
             test: {},
             providerResponse: { output },
@@ -1448,7 +1453,7 @@ describe('runAssertion', () => {
         const output = '';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithNumberAndThreshold,
             test: {},
             providerResponse: { output },
@@ -1467,7 +1472,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: {},
             providerResponse: { output },
@@ -1482,7 +1487,7 @@ describe('runAssertion', () => {
         const output = 'Different output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertion,
             test: {},
             providerResponse: { output },
@@ -1500,7 +1505,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'variable value',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion,
             test: { vars: { foo: 'Expected output' } },
             providerResponse: { output },
@@ -1518,7 +1523,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithVars,
             test: { vars: { foo: 'bar' } },
             providerResponse: { output },
@@ -1536,7 +1541,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptStringAssertionWithVars,
             test: { vars: { foo: 'bar' } },
             providerResponse: { output },
@@ -1550,7 +1555,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptFunctionAssertion,
             test: {},
             providerResponse: { output },
@@ -1565,7 +1570,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: javascriptFunctionFailAssertion,
             test: {},
             providerResponse: { output },
@@ -1583,7 +1588,7 @@ describe('runAssertion', () => {
             assertion: javascriptMultilineStringAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1597,7 +1602,7 @@ describe('runAssertion', () => {
             assertion: javascriptMultilineStringAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1615,7 +1620,7 @@ describe('runAssertion', () => {
             assertion: notContainsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1629,7 +1634,7 @@ describe('runAssertion', () => {
             assertion: notContainsAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1648,7 +1653,7 @@ describe('runAssertion', () => {
             assertion: containsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1662,7 +1667,7 @@ describe('runAssertion', () => {
             assertion: containsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1681,7 +1686,7 @@ describe('runAssertion', () => {
             assertion: notContainsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1695,7 +1700,7 @@ describe('runAssertion', () => {
             assertion: notContainsLowerAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1714,7 +1719,7 @@ describe('runAssertion', () => {
             assertion: containsAnyAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1728,7 +1733,7 @@ describe('runAssertion', () => {
             assertion: containsAnyAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1745,7 +1750,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1762,7 +1767,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1781,7 +1786,7 @@ describe('runAssertion', () => {
             assertion: containsAllAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1795,7 +1800,7 @@ describe('runAssertion', () => {
             assertion: containsAllAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1812,7 +1817,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1829,7 +1834,7 @@ describe('runAssertion', () => {
             },
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1848,7 +1853,7 @@ describe('runAssertion', () => {
             assertion: containsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1862,7 +1867,7 @@ describe('runAssertion', () => {
             assertion: containsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1881,7 +1886,7 @@ describe('runAssertion', () => {
             assertion: notContainsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1895,7 +1900,7 @@ describe('runAssertion', () => {
             assertion: notContainsRegexAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1918,7 +1923,7 @@ describe('runAssertion', () => {
             assertion: webhookAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1936,7 +1941,7 @@ describe('runAssertion', () => {
             assertion: webhookAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1954,7 +1959,7 @@ describe('runAssertion', () => {
             assertion: webhookAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -1974,7 +1979,7 @@ describe('runAssertion', () => {
             assertion: rougeNAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -1988,7 +1993,7 @@ describe('runAssertion', () => {
             assertion: rougeNAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -2007,7 +2012,7 @@ describe('runAssertion', () => {
             assertion: startsWithAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -2021,7 +2026,7 @@ describe('runAssertion', () => {
             assertion: startsWithAssertion,
             test: {},
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: false,
@@ -2057,7 +2062,7 @@ describe('runAssertion', () => {
             assertion,
             test,
             providerResponse: { output },
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
         });
         expect(result).toMatchObject({
             pass: true,
@@ -2074,7 +2079,7 @@ describe('runAssertion', () => {
         const output = 'Expected output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: levenshteinAssertion,
             test: {},
             providerResponse: { output },
@@ -2088,7 +2093,7 @@ describe('runAssertion', () => {
         const output = 'Different output';
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: levenshteinAssertion,
             test: {},
             providerResponse: { output },
@@ -2140,7 +2145,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2158,14 +2163,14 @@ describe('runAssertion', () => {
     it('should resolve js paths relative to the configuration file', async () => {
         const output = 'Expected output';
         const mockFn = jest.fn((output) => output === 'Expected output');
-        jest.doMock(path.resolve('/config_path/path/to/assert.js'), () => mockFn, { virtual: true });
+        jest.doMock(path.resolve('/base/path/path/to/assert.js'), () => mockFn, { virtual: true });
         const fileAssertion = {
             type: 'javascript',
             value: 'file://./path/to/assert.js',
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2190,7 +2195,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: pythonAssertion,
             test: {},
             providerResponse: { output },
@@ -2253,7 +2258,7 @@ describe('runAssertion', () => {
         jest.mocked(wrapper_1.runPythonCode).mockResolvedValueOnce(resolvedValue);
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: pythonAssertion,
             test: {},
             providerResponse: { output },
@@ -2297,7 +2302,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2327,7 +2332,7 @@ describe('runAssertion', () => {
         };
         const result = await (0, assertions_1.runAssertion)({
             prompt: 'Some prompt that includes "double quotes" and \'single quotes\'',
-            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+            provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
             assertion: fileAssertion,
             test: {},
             providerResponse: { output },
@@ -2348,7 +2353,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2366,7 +2371,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2384,7 +2389,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             await expect((0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2397,7 +2402,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -2415,7 +2420,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             await expect((0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                 },
@@ -2428,7 +2433,7 @@ describe('runAssertion', () => {
             const output = 'Expected output';
             const result = await (0, assertions_1.runAssertion)({
                 prompt: 'Some prompt',
-                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4'),
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
                 assertion: {
                     type: 'latency',
                     threshold: 100,
@@ -3277,6 +3282,80 @@ describe('runAssertion', () => {
             })).rejects.toThrow('context-faithfulness assertion type must have a string output');
         });
     });
+    describe('file references', () => {
+        it('should handle file reference in string value', async () => {
+            const assertion = {
+                type: 'equals',
+                value: 'file://expected_output.txt',
+            };
+            const expectedContent = 'Expected output';
+            jest.mocked(fs.readFileSync).mockReturnValue(expectedContent);
+            jest.mocked(path.resolve).mockReturnValue('/base/path/expected_output.txt');
+            jest.mocked(path.extname).mockReturnValue('.txt');
+            const result = await (0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: 'Expected output' },
+            });
+            expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/expected_output.txt', 'utf8');
+            expect(result.pass).toBe(true);
+        });
+        it('should handle file references in array values', async () => {
+            const assertion = {
+                type: 'contains-any',
+                value: ['The expected output', 'string output', 'file://my_expected_output.txt'],
+            };
+            const fileContent = 'file content';
+            jest.mocked(fs.readFileSync).mockReturnValue(fileContent);
+            jest.mocked(path.resolve).mockReturnValue('/base/path/my_expected_output.txt');
+            jest.mocked(path.extname).mockReturnValue('.txt');
+            await expect((0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: 'file content' },
+            })).resolves.toEqual(expect.objectContaining({
+                pass: true,
+            }));
+            expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/my_expected_output.txt', 'utf8');
+            await expect((0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: 'string output' },
+            })).resolves.toEqual(expect.objectContaining({
+                pass: true,
+            }));
+        });
+        it('should handle file reference in object value', async () => {
+            const assertion = {
+                type: 'is-json',
+                value: 'file://schema.json',
+            };
+            const schemaContent = JSON.stringify({
+                type: 'object',
+                properties: {
+                    key: { type: 'string' },
+                },
+            });
+            jest.mocked(fs.readFileSync).mockReturnValue(schemaContent);
+            jest.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
+            jest.mocked(path.extname).mockReturnValue('.json');
+            const result = await (0, assertions_1.runAssertion)({
+                prompt: 'Some prompt',
+                provider: new openai_1.OpenAiChatCompletionProvider('gpt-4o-mini'),
+                assertion,
+                test: {},
+                providerResponse: { output: '{"key": "value"}' },
+            });
+            expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
+            expect(result.pass).toBe(true);
+        });
+    });
 });
 describe('validateXml', () => {
     it('should validate a simple valid XML string', () => {