promptfoo 0.95.0 → 0.96.0

package/dist/test/{is-sql-tests/node-sql-parser.test.js → assertions/sql.test.js}

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 /* eslint-disable jest/no-commented-out-tests */
-const assertions_1 = require("../../src/assertions");
+const sql_1 = require("../../src/assertions/sql");
 const assertion = {
     type: 'is-sql',
 };
@@ -10,28 +10,28 @@ describe('Basic tests', () => {
     it('should pass when the output string is a valid SQL statement', async () => {
         const renderedValue = undefined;
         const outputString = 'SELECT id, name FROM users';
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(true);
         expect(result.reason).toBe('Assertion passed');
     });
     it('should fail when the SQL statement contains a syntax error in the ORDER BY clause', async () => {
         const renderedValue = undefined;
         const outputString = 'SELECT * FROM orders ORDERY BY order_date';
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
     });
     it('should fail when the SQL statement uses a reserved keyword as a table name', async () => {
         const renderedValue = undefined;
         const outputString = 'SELECT * FROM select WHERE id = 1';
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
     });
     it('should fail when the SQL statement has an incorrect DELETE syntax', async () => {
         const renderedValue = undefined;
         const outputString = 'DELETE employees WHERE id = 1';
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
     });
@@ -65,7 +65,7 @@ describe('Database Specific Syntax Tests', () => {
             databaseType: 'PostgreSQL',
         };
         const outputString = `SELECT * FROM employees WHERE id = 1 LOCK IN SHARE MODE`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe('SQL statement does not conform to the provided PostgreSQL database syntax.');
     });
@@ -74,7 +74,7 @@ describe('Database Specific Syntax Tests', () => {
             databaseType: 'MySQL',
         };
         const outputString = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe('SQL statement does not conform to the provided MySQL database syntax.');
     });
@@ -83,7 +83,7 @@ describe('Database Specific Syntax Tests', () => {
             databaseType: 'PostgreSQL',
         };
         const outputString = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(true);
         expect(result.reason).toBe('Assertion passed');
     });
@@ -109,7 +109,7 @@ describe('White Table/Column List Tests', () => {
             allowedTables: ['(select|update|insert|delete)::null::departments'],
         };
         const outputString = `SELECT * FROM employees`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe(`SQL validation failed: authority = 'select::null::employees' is required in table whiteList to execute SQL = 'SELECT * FROM employees'.`);
     });
@@ -119,7 +119,7 @@ describe('White Table/Column List Tests', () => {
             allowedTables: ['(select|update|insert|delete)::null::departments'],
         };
         const outputString = `SELECT * FROM departments`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(true);
         expect(result.reason).toBe('Assertion passed');
     });
@@ -129,7 +129,7 @@ describe('White Table/Column List Tests', () => {
             allowedColumns: ['select::null::name', 'update::null::id'],
         };
         const outputString = `SELECT id FROM t`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(false);
         expect(result.reason).toBe(`SQL validation failed: authority = 'select::null::id' is required in column whiteList to execute SQL = 'SELECT id FROM t'.`);
     });
@@ -139,7 +139,7 @@ describe('White Table/Column List Tests', () => {
             allowedColumns: ['insert::department::dept_name', 'insert::department::location'],
         };
         const outputString = `INSERT INTO department (dept_name, location) VALUES ('Sales', 'New York')`;
-        const result = await (0, assertions_1.isSql)(outputString, renderedValue, false, assertion);
+        const result = await (0, sql_1.isSql)(outputString, renderedValue, false, assertion);
         expect(result.pass).toBe(true);
         expect(result.reason).toBe('Assertion passed');
     });
@@ -176,4 +176,4 @@ describe('White Table/Column List Tests', () => {
     // });
 });
 /* eslint-enable jest/no-commented-out-tests */
-//# sourceMappingURL=node-sql-parser.test.js.map
+//# sourceMappingURL=sql.test.js.map

package/dist/test/assertions/sql.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sql.test.js","sourceRoot":"","sources":["../../../test/assertions/sql.test.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,kDAAiD;AAGjD,MAAM,SAAS,GAAc;IAC3B,IAAI,EAAE,QAAQ;CACf,CAAC;AAEF,2HAA2H;AAC3H,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;QAC3E,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,MAAM,YAAY,GAAG,4BAA4B,CAAC;QAClD,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mFAAmF,EAAE,KAAK,IAAI,EAAE;QACjG,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,MAAM,YAAY,GAAG,2CAA2C,CAAC;QACjE,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,uEAAuE,CACxE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,KAAK,IAAI,EAAE;QAC1F,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,MAAM,YAAY,GAAG,mCAAmC,CAAC;QACzD,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,uEAAuE,CACxE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;QACjF,MAAM,aAAa,GAAG,SAAS,CAAC;QAChC,MAAM,YAAY,GAAG,+BAA+B,CAAC;QACrD,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,uEAAuE,CACxE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,+EAA+E;IAC/E,qCAAqC;IACrC,uEAAuE;IACvE,qEAAqE;IACrE,qCAAqC;IACrC,yGAAyG;IACzG,MAAM;IAEN;;;OAGG;IACH,+EAA+E;IAC/E,qCAAqC;IACrC,sDAAsD;IACtD,qEAAqE;IACrE,qCAAqC;IACrC,yGAAyG;IACzG,MAAM;AACR,CAAC,CAAC,CAAC;AAEH,2HAA2H;AAC3H,QAAQ,CAAC,gCAAgC,EAAE,GAAG,EAAE;IAC9C,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;QAC5F,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,YAAY;SAC3B,CAAC;QACF,MAAM,YAAY,GAAG,yDAAyD,CAAC;QAC/E,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,4EAA4E,CAC7E,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;QAC5F,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,OAAO;SACtB,CAAC;QACF,MAAM,YAAY,GAAG,4EAA4E,CAAC;QAClG,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,uEAAuE,CACxE,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,KAAK,IAAI,EAAE;QAC5F,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,YAAY;SAC3B,CAAC;QACF,MAAM,YAAY,GAAG,4EAA4E,CAAC;QAClG,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH;;;OAGG;IACH,6FAA6F;IAC7F,4BAA4B;IAC5B,6BAA6B;IAC7B,OAAO;IACP,2DAA2D;IAC3D,qEAAqE;IACrE,qCAAqC;IACrC,yGAAyG;IACzG,MAAM;AACR,CAAC,CAAC,CAAC;AAEH,2HAA2H;AAC3H,QAAQ,CAAC,+BAA+B,EAAE,GAAG,EAAE;IAC7C,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,OAAO;YACrB,aAAa,EAAE,CAAC,kDAAkD,CAAC;SACpE,CAAC;QACF,MAAM,YAAY,GAAG,yBAAyB,CAAC;QAC/C,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,yIAAyI,CAC1I,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wEAAwE,EAAE,KAAK,IAAI,EAAE;QACtF,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,OAAO;YACrB,aAAa,EAAE,CAAC,kDAAkD,CAAC;SACpE,CAAC;QACF,MAAM,YAAY,GAAG,2BAA2B,CAAC;QACjD,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAC9E,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,OAAO;YACrB,cAAc,EAAE,CAAC,oBAAoB,EAAE,kBAAkB,CAAC;SAC3D,CAAC;QACF,MAAM,YAAY,GAAG,kBAAkB,CAAC;QACxC,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CACxB,4HAA4H,CAC7H,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,KAAK,IAAI,EAAE;QACvF,MAAM,aAAa,GAAG;YACpB,YAAY,EAAE,OAAO;YACrB,cAAc,EAAE,CAAC,+BAA+B,EAAE,8BAA8B,CAAC;SAClF,CAAC;QACF,MAAM,YAAY,GAAG,2EAA2E,CAAC;QACjG,MAAM,MAAM,GAAkB,MAAM,IAAA,WAAK,EAAC,YAAY,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACzF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH;;;;;;OAMG;IACH,wFAAwF;IACxF,4BAA4B;IAC5B,6BAA6B;IAC7B,yCAAyC;IACzC,OAAO;IACP,gDAAgD;IAChD,qEAAqE;IACrE,oCAAoC;IACpC,oDAAoD;IACpD,MAAM;IAEN;;;OAGG;IACH,wFAAwF;IACxF,4BAA4B;IAC5B,6BAA6B;IAC7B,2EAA2E;IAC3E,OAAO;IACP,4EAA4E;IAC5E,qEAAqE;IACrE,oCAAoC;IACpC,oDAAoD;IACpD,MAAM;AACR,CAAC,CAAC,CAAC;AAEH,+CAA+C"}

package/dist/test/commands/eval/filterTests.test.js

@@ -47,5 +47,35 @@ describe('filterTests', () => {
             expect(result).not.toBe(tests);
         });
     });
+    describe('sample', () => {
+        it('should return the requested number of random tests', async () => {
+            const result = await (0, filterTests_1.filterTests)(testSuite, { sample: 2 });
+            expect(result).toHaveLength(2);
+            // Each result should be one of the original tests
+            result?.forEach((test) => {
+                expect(tests).toContain(test);
+            });
+        });
+        it('accepts sample as string', async () => {
+            const result = await (0, filterTests_1.filterTests)(testSuite, { sample: '2' });
+            expect(result).toHaveLength(2);
+        });
+        it('throws an exception when sample is not a number', async () => {
+            await expect(() => (0, filterTests_1.filterTests)(testSuite, { sample: 'NOPE' })).rejects.toEqual(new Error('sample must be a number, got: NOPE'));
+        });
+        it('returns all tests when sample size exceeds test count', async () => {
+            const result = await (0, filterTests_1.filterTests)(testSuite, { sample: 10 });
+            expect(result).toHaveLength(tests.length);
+            result?.forEach((test) => {
+                expect(tests).toContain(test);
+            });
+        });
+        it('can combine with pattern filter', async () => {
+            const result = await (0, filterTests_1.filterTests)(testSuite, { sample: 1, pattern: 'ey$' });
+            expect(result).toBeDefined();
+            expect(result).toHaveLength(1);
+            expect(result?.[0].description).toMatch(/ey$/);
+        });
+    });
 });
 //# sourceMappingURL=filterTests.test.js.map

package/dist/test/commands/eval/filterTests.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"filterTests.test.js","sourceRoot":"","sources":["../../../../test/commands/eval/filterTests.test.ts"],"names":[],"mappings":";;AAAA,wEAAqE;AAGrE,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG;QACf,WAAW,EAAE,MAAM;KACpB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG;QAChB,KAAK;KACO,CAAC;IAEf,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,EAAe,EAAE,EAAE,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5E,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"filterTests.test.js","sourceRoot":"","sources":["../../../../test/commands/eval/filterTests.test.ts"],"names":[],"mappings":";;AAAA,wEAAqE;AAGrE,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG;QACf,WAAW,EAAE,MAAM;KACpB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,SAAS,GAAG;QAChB,WAAW,EAAE,OAAO;KACrB,CAAC;IACF,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,iBAAiB,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,SAAS,GAAG;QAChB,KAAK;KACO,CAAC;IAEf,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;QACjD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAEhD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QAChC,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,EAAe,EAAE,EAAE,CAAC,CAAC;QAEtD,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,EAAE,CAAC;IACjC,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5E,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;YACvE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAEhE,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE7E,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;YAE3D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,kDAAkD;YAClD,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBACvB,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAE7D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC5E,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAE5D,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;gBACvB,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,iCAAiC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,MAAM,GAAG,MAAM,IAAA,yBAAW,EAAC,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAE3E,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/factories/evalFactory.d.ts

@@ -27,12 +27,26 @@ export default class EvalFactory {
                 AZURE_OPENAI_API_BASE_URL?: string | undefined;
                 AZURE_OPENAI_API_HOST?: string | undefined;
                 AZURE_OPENAI_API_KEY?: string | undefined;
+                AZURE_API_BASE_URL?: string | undefined;
+                AZURE_API_HOST?: string | undefined;
+                AZURE_API_KEY?: string | undefined;
+                AZURE_DEPLOYMENT_NAME?: string | undefined;
+                AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                 AZURE_OPENAI_BASE_URL?: string | undefined;
+                AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                AZURE_CLIENT_SECRET?: string | undefined;
+                AZURE_CLIENT_ID?: string | undefined;
+                AZURE_TENANT_ID?: string | undefined;
+                AZURE_AUTHORITY_HOST?: string | undefined;
+                AZURE_TOKEN_SCOPE?: string | undefined;
                 BAM_API_HOST?: string | undefined;
                 BAM_API_KEY?: string | undefined;
                 CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                 CLOUDFLARE_API_KEY?: string | undefined;
+                PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                 COHERE_API_KEY?: string | undefined;
+                FAL_KEY?: string | undefined;
                 GOOGLE_API_HOST?: string | undefined;
                 GOOGLE_API_KEY?: string | undefined;
                 GROQ_API_KEY?: string | undefined;
@@ -54,6 +68,9 @@ export default class EvalFactory {
                 VERTEX_PROJECT_ID?: string | undefined;
                 VERTEX_PUBLISHER?: string | undefined;
                 VERTEX_REGION?: string | undefined;
+                WATSONX_AI_APIKEY?: string | undefined;
+                WATSONX_AI_PROJECT_ID?: string | undefined;
+                WATSONX_AI_BEARER_TOKEN?: string | undefined;
             } | undefined;
             metadata?: Record<string, any> | undefined;
             description?: string | undefined;
@@ -94,12 +111,26 @@ export default class EvalFactory {
                     AZURE_OPENAI_API_BASE_URL?: string | undefined;
                     AZURE_OPENAI_API_HOST?: string | undefined;
                     AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_API_BASE_URL?: string | undefined;
+                    AZURE_API_HOST?: string | undefined;
+                    AZURE_API_KEY?: string | undefined;
+                    AZURE_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                     AZURE_OPENAI_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_CLIENT_SECRET?: string | undefined;
+                    AZURE_CLIENT_ID?: string | undefined;
+                    AZURE_TENANT_ID?: string | undefined;
+                    AZURE_AUTHORITY_HOST?: string | undefined;
+                    AZURE_TOKEN_SCOPE?: string | undefined;
                     BAM_API_HOST?: string | undefined;
                     BAM_API_KEY?: string | undefined;
                     CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                     CLOUDFLARE_API_KEY?: string | undefined;
+                    PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                     COHERE_API_KEY?: string | undefined;
+                    FAL_KEY?: string | undefined;
                     GOOGLE_API_HOST?: string | undefined;
                     GOOGLE_API_KEY?: string | undefined;
                     GROQ_API_KEY?: string | undefined;
@@ -121,6 +152,9 @@ export default class EvalFactory {
                     VERTEX_PROJECT_ID?: string | undefined;
                     VERTEX_PUBLISHER?: string | undefined;
                     VERTEX_REGION?: string | undefined;
+                    WATSONX_AI_APIKEY?: string | undefined;
+                    WATSONX_AI_PROJECT_ID?: string | undefined;
+                    WATSONX_AI_BEARER_TOKEN?: string | undefined;
                 } | undefined;
             } | (((args_0: string, args_1: {
                 vars: Record<string, string | {}>;
@@ -159,12 +193,26 @@ export default class EvalFactory {
                     AZURE_OPENAI_API_BASE_URL?: string | undefined;
                     AZURE_OPENAI_API_HOST?: string | undefined;
                     AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_API_BASE_URL?: string | undefined;
+                    AZURE_API_HOST?: string | undefined;
+                    AZURE_API_KEY?: string | undefined;
+                    AZURE_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                     AZURE_OPENAI_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_CLIENT_SECRET?: string | undefined;
+                    AZURE_CLIENT_ID?: string | undefined;
+                    AZURE_TENANT_ID?: string | undefined;
+                    AZURE_AUTHORITY_HOST?: string | undefined;
+                    AZURE_TOKEN_SCOPE?: string | undefined;
                     BAM_API_HOST?: string | undefined;
                     BAM_API_KEY?: string | undefined;
                     CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                     CLOUDFLARE_API_KEY?: string | undefined;
+                    PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                     COHERE_API_KEY?: string | undefined;
+                    FAL_KEY?: string | undefined;
                     GOOGLE_API_HOST?: string | undefined;
                     GOOGLE_API_KEY?: string | undefined;
                     GROQ_API_KEY?: string | undefined;
@@ -186,6 +234,9 @@ export default class EvalFactory {
                     VERTEX_PROJECT_ID?: string | undefined;
                     VERTEX_PUBLISHER?: string | undefined;
                     VERTEX_REGION?: string | undefined;
+                    WATSONX_AI_APIKEY?: string | undefined;
+                    WATSONX_AI_PROJECT_ID?: string | undefined;
+                    WATSONX_AI_BEARER_TOKEN?: string | undefined;
                 } | undefined;
             }>)[] | undefined;
             tests?: string | (string | {
@@ -231,12 +282,26 @@ export default class EvalFactory {
                         AZURE_OPENAI_API_BASE_URL?: string | undefined;
                         AZURE_OPENAI_API_HOST?: string | undefined;
                         AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_API_BASE_URL?: string | undefined;
+                        AZURE_API_HOST?: string | undefined;
+                        AZURE_API_KEY?: string | undefined;
+                        AZURE_DEPLOYMENT_NAME?: string | undefined;
+                        AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                         AZURE_OPENAI_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                        AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                        AZURE_CLIENT_SECRET?: string | undefined;
+                        AZURE_CLIENT_ID?: string | undefined;
+                        AZURE_TENANT_ID?: string | undefined;
+                        AZURE_AUTHORITY_HOST?: string | undefined;
+                        AZURE_TOKEN_SCOPE?: string | undefined;
                         BAM_API_HOST?: string | undefined;
                         BAM_API_KEY?: string | undefined;
                         CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                         CLOUDFLARE_API_KEY?: string | undefined;
+                        PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                         COHERE_API_KEY?: string | undefined;
+                        FAL_KEY?: string | undefined;
                         GOOGLE_API_HOST?: string | undefined;
                         GOOGLE_API_KEY?: string | undefined;
                         GROQ_API_KEY?: string | undefined;
@@ -258,6 +323,9 @@ export default class EvalFactory {
                         VERTEX_PROJECT_ID?: string | undefined;
                         VERTEX_PUBLISHER?: string | undefined;
                         VERTEX_REGION?: string | undefined;
+                        WATSONX_AI_APIKEY?: string | undefined;
+                        WATSONX_AI_PROJECT_ID?: string | undefined;
+                        WATSONX_AI_BEARER_TOKEN?: string | undefined;
                     } | undefined;
                 } | {
                     callApi: import("../../src").CallApiFunction;
@@ -353,12 +421,26 @@ export default class EvalFactory {
                             AZURE_OPENAI_API_BASE_URL?: string | undefined;
                             AZURE_OPENAI_API_HOST?: string | undefined;
                             AZURE_OPENAI_API_KEY?: string | undefined;
+                            AZURE_API_BASE_URL?: string | undefined;
+                            AZURE_API_HOST?: string | undefined;
+                            AZURE_API_KEY?: string | undefined;
+                            AZURE_DEPLOYMENT_NAME?: string | undefined;
+                            AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                             AZURE_OPENAI_BASE_URL?: string | undefined;
+                            AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                            AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                            AZURE_CLIENT_SECRET?: string | undefined;
+                            AZURE_CLIENT_ID?: string | undefined;
+                            AZURE_TENANT_ID?: string | undefined;
+                            AZURE_AUTHORITY_HOST?: string | undefined;
+                            AZURE_TOKEN_SCOPE?: string | undefined;
                             BAM_API_HOST?: string | undefined;
                             BAM_API_KEY?: string | undefined;
                             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                             CLOUDFLARE_API_KEY?: string | undefined;
+                            PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                             COHERE_API_KEY?: string | undefined;
+                            FAL_KEY?: string | undefined;
                             GOOGLE_API_HOST?: string | undefined;
                             GOOGLE_API_KEY?: string | undefined;
                             GROQ_API_KEY?: string | undefined;
@@ -380,6 +462,9 @@ export default class EvalFactory {
                             VERTEX_PROJECT_ID?: string | undefined;
                             VERTEX_PUBLISHER?: string | undefined;
                             VERTEX_REGION?: string | undefined;
+                            WATSONX_AI_APIKEY?: string | undefined;
+                            WATSONX_AI_PROJECT_ID?: string | undefined;
+                            WATSONX_AI_BEARER_TOKEN?: string | undefined;
                         } | undefined;
                     } | {
                         callApi: import("../../src").CallApiFunction;
@@ -472,12 +557,26 @@ export default class EvalFactory {
                             AZURE_OPENAI_API_BASE_URL?: string | undefined;
                             AZURE_OPENAI_API_HOST?: string | undefined;
                             AZURE_OPENAI_API_KEY?: string | undefined;
+                            AZURE_API_BASE_URL?: string | undefined;
+                            AZURE_API_HOST?: string | undefined;
+                            AZURE_API_KEY?: string | undefined;
+                            AZURE_DEPLOYMENT_NAME?: string | undefined;
+                            AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                             AZURE_OPENAI_BASE_URL?: string | undefined;
+                            AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                            AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                            AZURE_CLIENT_SECRET?: string | undefined;
+                            AZURE_CLIENT_ID?: string | undefined;
+                            AZURE_TENANT_ID?: string | undefined;
+                            AZURE_AUTHORITY_HOST?: string | undefined;
+                            AZURE_TOKEN_SCOPE?: string | undefined;
                             BAM_API_HOST?: string | undefined;
                             BAM_API_KEY?: string | undefined;
                             CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                             CLOUDFLARE_API_KEY?: string | undefined;
+                            PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                             COHERE_API_KEY?: string | undefined;
+                            FAL_KEY?: string | undefined;
                             GOOGLE_API_HOST?: string | undefined;
                             GOOGLE_API_KEY?: string | undefined;
                             GROQ_API_KEY?: string | undefined;
@@ -499,6 +598,9 @@ export default class EvalFactory {
                             VERTEX_PROJECT_ID?: string | undefined;
                             VERTEX_PUBLISHER?: string | undefined;
                             VERTEX_REGION?: string | undefined;
+                            WATSONX_AI_APIKEY?: string | undefined;
+                            WATSONX_AI_PROJECT_ID?: string | undefined;
+                            WATSONX_AI_BEARER_TOKEN?: string | undefined;
                         } | undefined;
                     } | {
                         callApi: import("../../src").CallApiFunction;
@@ -593,12 +695,26 @@ export default class EvalFactory {
                         AZURE_OPENAI_API_BASE_URL?: string | undefined;
                         AZURE_OPENAI_API_HOST?: string | undefined;
                         AZURE_OPENAI_API_KEY?: string | undefined;
+                        AZURE_API_BASE_URL?: string | undefined;
+                        AZURE_API_HOST?: string | undefined;
+                        AZURE_API_KEY?: string | undefined;
+                        AZURE_DEPLOYMENT_NAME?: string | undefined;
+                        AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                         AZURE_OPENAI_BASE_URL?: string | undefined;
+                        AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                        AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                        AZURE_CLIENT_SECRET?: string | undefined;
+                        AZURE_CLIENT_ID?: string | undefined;
+                        AZURE_TENANT_ID?: string | undefined;
+                        AZURE_AUTHORITY_HOST?: string | undefined;
+                        AZURE_TOKEN_SCOPE?: string | undefined;
                         BAM_API_HOST?: string | undefined;
                         BAM_API_KEY?: string | undefined;
                         CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                         CLOUDFLARE_API_KEY?: string | undefined;
+                        PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                         COHERE_API_KEY?: string | undefined;
+                        FAL_KEY?: string | undefined;
                         GOOGLE_API_HOST?: string | undefined;
                         GOOGLE_API_KEY?: string | undefined;
                         GROQ_API_KEY?: string | undefined;
@@ -620,6 +736,9 @@ export default class EvalFactory {
                         VERTEX_PROJECT_ID?: string | undefined;
                         VERTEX_PUBLISHER?: string | undefined;
                         VERTEX_REGION?: string | undefined;
+                        WATSONX_AI_APIKEY?: string | undefined;
+                        WATSONX_AI_PROJECT_ID?: string | undefined;
+                        WATSONX_AI_BEARER_TOKEN?: string | undefined;
                     } | undefined;
                 } | {
                     callApi: import("../../src").CallApiFunction;
@@ -697,6 +816,7 @@ export default class EvalFactory {
                 maxConcurrency?: number | undefined;
                 output?: string[] | undefined;
                 write?: boolean | undefined;
+                verbose?: boolean | undefined;
                 vars?: string | undefined;
                 prompts?: string[] | undefined;
                 delay?: number | undefined;
@@ -706,7 +826,6 @@ export default class EvalFactory {
                 tests?: string | undefined;
                 assertions?: string | undefined;
                 modelOutputs?: string | undefined;
-                verbose?: boolean | undefined;
                 grader?: string | undefined;
                 tableCellMaxLength?: number | undefined;
                 table?: boolean | undefined;
@@ -715,6 +834,7 @@ export default class EvalFactory {
                 watch?: boolean | undefined;
                 filterFailing?: string | undefined;
                 filterFirstN?: number | undefined;
+                filterSample?: number | undefined;
                 filterPattern?: string | undefined;
                 filterProviders?: string | undefined;
                 filterTargets?: string | undefined;
@@ -761,12 +881,26 @@ export default class EvalFactory {
                     AZURE_OPENAI_API_BASE_URL?: string | undefined;
                     AZURE_OPENAI_API_HOST?: string | undefined;
                     AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_API_BASE_URL?: string | undefined;
+                    AZURE_API_HOST?: string | undefined;
+                    AZURE_API_KEY?: string | undefined;
+                    AZURE_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                     AZURE_OPENAI_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_CLIENT_SECRET?: string | undefined;
+                    AZURE_CLIENT_ID?: string | undefined;
+                    AZURE_TENANT_ID?: string | undefined;
+                    AZURE_AUTHORITY_HOST?: string | undefined;
+                    AZURE_TOKEN_SCOPE?: string | undefined;
                     BAM_API_HOST?: string | undefined;
                     BAM_API_KEY?: string | undefined;
                     CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                     CLOUDFLARE_API_KEY?: string | undefined;
+                    PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                     COHERE_API_KEY?: string | undefined;
+                    FAL_KEY?: string | undefined;
                     GOOGLE_API_HOST?: string | undefined;
                     GOOGLE_API_KEY?: string | undefined;
                     GROQ_API_KEY?: string | undefined;
@@ -788,6 +922,9 @@ export default class EvalFactory {
                     VERTEX_PROJECT_ID?: string | undefined;
                     VERTEX_PUBLISHER?: string | undefined;
                     VERTEX_REGION?: string | undefined;
+                    WATSONX_AI_APIKEY?: string | undefined;
+                    WATSONX_AI_PROJECT_ID?: string | undefined;
+                    WATSONX_AI_BEARER_TOKEN?: string | undefined;
                 } | undefined;
             } | (((args_0: string, args_1: {
                 vars: Record<string, string | {}>;
@@ -826,12 +963,26 @@ export default class EvalFactory {
                     AZURE_OPENAI_API_BASE_URL?: string | undefined;
                     AZURE_OPENAI_API_HOST?: string | undefined;
                     AZURE_OPENAI_API_KEY?: string | undefined;
+                    AZURE_API_BASE_URL?: string | undefined;
+                    AZURE_API_HOST?: string | undefined;
+                    AZURE_API_KEY?: string | undefined;
+                    AZURE_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
                     AZURE_OPENAI_BASE_URL?: string | undefined;
+                    AZURE_OPENAI_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME?: string | undefined;
+                    AZURE_CLIENT_SECRET?: string | undefined;
+                    AZURE_CLIENT_ID?: string | undefined;
+                    AZURE_TENANT_ID?: string | undefined;
+                    AZURE_AUTHORITY_HOST?: string | undefined;
+                    AZURE_TOKEN_SCOPE?: string | undefined;
                     BAM_API_HOST?: string | undefined;
                     BAM_API_KEY?: string | undefined;
                     CLOUDFLARE_ACCOUNT_ID?: string | undefined;
                     CLOUDFLARE_API_KEY?: string | undefined;
+                    PROMPTFOO_REMOTE_GENERATION_URL?: string | undefined;
                     COHERE_API_KEY?: string | undefined;
+                    FAL_KEY?: string | undefined;
                     GOOGLE_API_HOST?: string | undefined;
                     GOOGLE_API_KEY?: string | undefined;
                     GROQ_API_KEY?: string | undefined;
@@ -853,6 +1004,9 @@ export default class EvalFactory {
                     VERTEX_PROJECT_ID?: string | undefined;
                     VERTEX_PUBLISHER?: string | undefined;
                     VERTEX_REGION?: string | undefined;
+                    WATSONX_AI_APIKEY?: string | undefined;
+                    WATSONX_AI_PROJECT_ID?: string | undefined;
+                    WATSONX_AI_BEARER_TOKEN?: string | undefined;
                 } | undefined;
             }>)[] | undefined;
         }>;

package/dist/test/factories/evalFactory.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evalFactory.d.ts","sourceRoot":"","sources":["../../../test/factories/evalFactory.ts"],"names":[],"mappings":"AAGA,OAAO,IAAI,MAAM,uBAAuB,CAAC;AAGzC,MAAM,CAAC,OAAO,OAAO,WAAW;WACjB,MAAM;WAgJN,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAI7B"}
+{"version":3,"file":"evalFactory.d.ts","sourceRoot":"","sources":["../../../test/factories/evalFactory.ts"],"names":[],"mappings":"AAGA,OAAO,IAAI,MAAM,uBAAuB,CAAC;AAGzC,MAAM,CAAC,OAAO,OAAO,WAAW;WACjB,MAAM;WAgJN,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAI7B"}