promptfoo 0.55.0 → 0.57.0

package/README.md

@@ -199,7 +199,7 @@ npx promptfoo eval -p prompts.txt -r openai:gpt-3.5-turbo -t tests.csv
 ![Side-by-side evaluation of LLM prompt quality, html output](https://user-images.githubusercontent.com/310310/235483444-4ddb832d-e103-4b9c-a862-b0d6cc11cdc0.png)
 -->
 
-This command will evaluate the prompts in `prompts.txt`, substituing the variable values from `vars.csv`, and output results in your terminal.
+This command will evaluate the prompts in `prompts.txt`, substituting the variable values from `vars.csv`, and output results in your terminal.
 
 You can also output a nice [spreadsheet](https://docs.google.com/spreadsheets/d/1nanoj3_TniWrDl1Sj-qYqIMD6jwm5FBy15xPFdUTsmI/edit?usp=sharing), [JSON](https://github.com/typpo/promptfoo/blob/main/examples/simple-cli/output.json), YAML, or an HTML file:
 
@@ -248,6 +248,9 @@ You can also use `promptfoo` as a library in your project by importing the `eval
 
     // The required score for this test case.  If not provided, the test case is graded pass/fail.
     threshold?: number;
+    
+    // Override the provider for this test
+    provider?: string | ProviderOptions | ApiProvider;
   }
 
   interface Assertion {

package/dist/package.json

@@ -2,7 +2,7 @@
     "name": "promptfoo",
     "description": "LLM eval & testing toolkit",
     "author": "Ian Webster",
-    "version": "0.55.0",
+    "version": "0.57.0",
     "license": "MIT",
     "type": "commonjs",
     "repository": "promptfoo/promptfoo",
@@ -45,8 +45,10 @@
         "@aws-sdk/client-bedrock-runtime": "^3.458.0",
         "@azure/identity": "^4.0.0",
         "@azure/openai-assistants": "^1.0.0-beta.5",
+        "@ibm-generative-ai/node-sdk": "^2.0.6",
         "google-auth-library": "^9.7.0",
-        "googleapis": "^134.0.0"
+        "googleapis": "^134.0.0",
+        "langfuse": "^3.7.0"
     },
     "devDependencies": {
         "@aws-sdk/client-bedrock-runtime": "^3.458.0",
@@ -100,13 +102,14 @@
         "dotenv": "^16.4.5",
         "drizzle-orm": "^0.29.3",
         "express": "^4.18.2",
+        "fast-deep-equal": "^3.1.3",
         "fastest-levenshtein": "^1.0.16",
         "glob": "^10.2.6",
         "js-yaml": "^4.1.0",
         "mathjs": "^12.4.1",
         "node-fetch": "^2.6.7",
         "nunjucks": "^3.2.4",
-        "openai": "^4.19.0",
+        "openai": "^4.38.5",
         "opener": "^1.5.2",
         "proxy-agent": "^6.3.1",
         "python-shell": "^5.0.0",

package/dist/src/cache.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":"AAGA,OAAO,YAAY,MAAM,eAAe,CAAC;AAQzC,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAc3D,wBAAgB,QAAQ,uBAoBvB;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAM,GAAG,MAAe,EAChC,IAAI,GAAE,OAAe,GACpB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAAC,CAuDzC;AAED,wBAAgB,WAAW,SAE1B;AAED,wBAAgB,YAAY,SAE3B;AAED,wBAAsB,UAAU,kBAE/B;AAED,wBAAgB,cAAc,YAE7B"}
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":"AAGA,OAAO,YAAY,MAAM,eAAe,CAAC;AAQzC,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAc3D,wBAAgB,QAAQ,uBAsBvB;AAED,wBAAsB,cAAc,CAClC,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,EACf,MAAM,GAAE,MAAM,GAAG,MAAe,EAChC,IAAI,GAAE,OAAe,GACpB,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAAC,CA2DzC;AAED,wBAAgB,WAAW,SAE1B;AAED,wBAAgB,YAAY,SAE3B;AAED,wBAAsB,UAAU,kBAE/B;AAED,wBAAgB,cAAc,YAE7B"}

package/dist/src/cache.js

@@ -17,16 +17,19 @@ let enabled = typeof process.env.PROMPTFOO_CACHE_ENABLED === 'undefined'
     : process.env.PROMPTFOO_CACHE_ENABLED === '1' ||
         process.env.PROMPTFOO_CACHE_ENABLED === 'true' ||
         process.env.PROMPTFOO_CACHE_ENABLED === 'yes';
-const cacheType = process.env.PROMPTFOO_CACHE_TYPE || (process.env.NODE_ENV === 'test' ? 'memory' : 'disk');
+let cacheType = process.env.PROMPTFOO_CACHE_TYPE || (process.env.NODE_ENV === 'test' ? 'memory' : 'disk');
 function getCache() {
     if (!cacheInstance) {
-        const cachePath = process.env.PROMPTFOO_CACHE_PATH || path_1.default.join((0, util_1.getConfigDirectoryPath)(), 'cache');
-        if (!fs_1.default.existsSync(cachePath)) {
-            logger_1.default.info(`Creating cache folder at ${cachePath}.`);
-            fs_1.default.mkdirSync(cachePath, { recursive: true });
+        let cachePath = '';
+        if (cacheType === 'disk' && enabled) {
+            cachePath = process.env.PROMPTFOO_CACHE_PATH || path_1.default.join((0, util_1.getConfigDirectoryPath)(), 'cache');
+            if (!fs_1.default.existsSync(cachePath)) {
+                logger_1.default.info(`Creating cache folder at ${cachePath}.`);
+                fs_1.default.mkdirSync(cachePath, { recursive: true });
+            }
         }
         cacheInstance = cache_manager_1.default.caching({
-            store: cacheType === 'disk' ? cache_manager_fs_hash_1.default : 'memory',
+            store: cacheType === 'disk' && enabled ? cache_manager_fs_hash_1.default : 'memory',
             options: {
                 max: process.env.PROMPTFOO_CACHE_MAX_FILE_COUNT || 10000,
                 path: cachePath,
@@ -72,6 +75,10 @@ async function fetchWithCache(url, options = {}, timeout, format = 'json', bust
                 // Don't cache error responses
                 return;
             }
+            if (!data) {
+                // Don't cache empty responses
+                return;
+            }
             logger_1.default.debug(`Storing ${url} response in cache: ${data}`);
             return data;
         }
@@ -79,7 +86,7 @@ async function fetchWithCache(url, options = {}, timeout, format = 'json', bust
             throw new Error(`Error parsing response from ${url}: ${err.message}. Received text: ${responseText}`);
         }
     });
-    if (cached) {
+    if (cached && cachedResponse) {
         logger_1.default.debug(`Returning cached response for ${url}: ${cachedResponse}`);
     }
     return {

package/dist/src/cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,gDAAwB;AAExB,kEAAyC;AACzC,kFAA4C;AAE5C,sDAA8B;AAC9B,mCAA2C;AAC3C,iCAAgD;AAKhD,IAAI,aAAgC,CAAC;AAErC,IAAI,OAAO,GACT,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,WAAW;IACxD,CAAC,CAAC,IAAI;IACN,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,GAAG;QAC3C,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,MAAM;QAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,KAAK,CAAC;AAEpD,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5F,SAAgB,QAAQ;IACtB,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAI,CAAC,IAAI,CAAC,IAAA,6BAAsB,GAAE,EAAE,OAAO,CAAC,CAAC;QACnF,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC7B,gBAAM,CAAC,IAAI,CAAC,4BAA4B,SAAS,GAAG,CAAC,CAAC;YACtD,YAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SAC9C;QACD,aAAa,GAAG,uBAAY,CAAC,OAAO,CAAC;YACnC,KAAK,EAAE,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,+BAAO,CAAC,CAAC,CAAC,QAAQ;YAChD,OAAO,EAAE;gBACP,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,KAAM;gBACzD,IAAI,EAAE,SAAS;gBACf,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;gBACzD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,EAAE,iBAAiB;gBACvE,+CAA+C;aAChD;SACF,CAAC,CAAC;KACJ;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AApBD,4BAoBC;AAEM,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe,EACf,SAA0B,MAAM,EAChC,OAAgB,KAAK;IAErB,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;QACpB,MAAM,IAAI,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ;aAC1D,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;SAChE;KACF;IAED,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;IAE/B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,OAAO,CAAC;IACpB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;IAExD,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,aAAa,GAAG,IAAI,CAAC;IAEzB,sFAAsF;IACtF,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;QAC3D,kDAAkD;QAClD,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI;YACF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YACzF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAChB,aAAa,GAAG,IAAI,CAAC;gBACrB,8BAA8B;gBAC9B,OAAO;aACR;YACD,gBAAM,CAAC,KAAK,CAAC,WAAW,GAAG,uBAAuB,IAAI,EAAE,CAAC,CAAC;YAC1D,OAAO,IAAI,CAAC;SACb;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,IAAI,KAAK,CACb,+BAA+B,GAAG,KAC/B,GAAa,CAAC,OACjB,oBAAoB,YAAY,EAAE,CACnC,CAAC;SACH;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,MAAM,EAAE;QACV,gBAAM,CAAC,KAAK,CAAC,iCAAiC,GAAG,KAAK,cAAc,EAAE,CAAC,CAAC;KACzE;IAED,OAAO;QACL,MAAM;QACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,IAAI,aAAa,CAAW,CAAC;KAC9D,CAAC;AACJ,CAAC;AA7DD,wCA6DC;AAED,SAAgB,WAAW;IACzB,OAAO,GAAG,IAAI,CAAC;AACjB,CAAC;AAFD,kCAEC;AAED,SAAgB,YAAY;IAC1B,OAAO,GAAG,KAAK,CAAC;AAClB,CAAC;AAFD,oCAEC;AAEM,KAAK,UAAU,UAAU;IAC9B,OAAO,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;AAC5B,CAAC;AAFD,gCAEC;AAED,SAAgB,cAAc;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC;AAFD,wCAEC"}
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,gDAAwB;AAExB,kEAAyC;AACzC,kFAA4C;AAE5C,sDAA8B;AAC9B,mCAA2C;AAC3C,iCAAgD;AAKhD,IAAI,aAAgC,CAAC;AAErC,IAAI,OAAO,GACT,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,WAAW;IACxD,CAAC,CAAC,IAAI;IACN,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,GAAG;QAC3C,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,MAAM;QAC9C,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,KAAK,CAAC;AAEpD,IAAI,SAAS,GACX,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5F,SAAgB,QAAQ;IACtB,IAAI,CAAC,aAAa,EAAE;QAClB,IAAI,SAAS,GAAG,EAAE,CAAC;QACnB,IAAI,SAAS,KAAK,MAAM,IAAI,OAAO,EAAE;YACnC,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,cAAI,CAAC,IAAI,CAAC,IAAA,6BAAsB,GAAE,EAAE,OAAO,CAAC,CAAC;YAC7F,IAAI,CAAC,YAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC7B,gBAAM,CAAC,IAAI,CAAC,4BAA4B,SAAS,GAAG,CAAC,CAAC;gBACtD,YAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;aAC9C;SACF;QACD,aAAa,GAAG,uBAAY,CAAC,OAAO,CAAC;YACnC,KAAK,EAAE,SAAS,KAAK,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,+BAAO,CAAC,CAAC,CAAC,QAAQ;YAC3D,OAAO,EAAE;gBACP,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,KAAM;gBACzD,IAAI,EAAE,SAAS;gBACf,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;gBACzD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,EAAE,iBAAiB;gBACvE,+CAA+C;aAChD;SACF,CAAC,CAAC;KACJ;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAtBD,4BAsBC;AAEM,KAAK,UAAU,cAAc,CAClC,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe,EACf,SAA0B,MAAM,EAChC,OAAgB,KAAK;IAErB,IAAI,CAAC,OAAO,IAAI,IAAI,EAAE;QACpB,MAAM,IAAI,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,KAAK;gBACb,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ;aAC1D,CAAC;SACH;QAAC,OAAO,KAAK,EAAE;YACd,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,EAAE,CAAC,CAAC;SAChE;KACF;IAED,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;IAE/B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,OAAO,CAAC;IACpB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;IAExD,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,IAAI,aAAa,GAAG,IAAI,CAAC;IAEzB,sFAAsF;IACtF,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;QAC3D,kDAAkD;QAClD,MAAM,GAAG,KAAK,CAAC;QACf,MAAM,QAAQ,GAAG,MAAM,IAAA,wBAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI;YACF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YACzF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAChB,aAAa,GAAG,IAAI,CAAC;gBACrB,8BAA8B;gBAC9B,OAAO;aACR;YACD,IAAI,CAAC,IAAI,EAAE;gBACT,8BAA8B;gBAC9B,OAAO;aACR;YACD,gBAAM,CAAC,KAAK,CAAC,WAAW,GAAG,uBAAuB,IAAI,EAAE,CAAC,CAAC;YAC1D,OAAO,IAAI,CAAC;SACb;QAAC,OAAO,GAAG,EAAE;YACZ,MAAM,IAAI,KAAK,CACb,+BAA+B,GAAG,KAC/B,GAAa,CAAC,OACjB,oBAAoB,YAAY,EAAE,CACnC,CAAC;SACH;IACH,CAAC,CAAC,CAAC;IAEH,IAAI,MAAM,IAAI,cAAc,EAAE;QAC5B,gBAAM,CAAC,KAAK,CAAC,iCAAiC,GAAG,KAAK,cAAc,EAAE,CAAC,CAAC;KACzE;IAED,OAAO;QACL,MAAM;QACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,IAAI,aAAa,CAAW,CAAC;KAC9D,CAAC;AACJ,CAAC;AAjED,wCAiEC;AAED,SAAgB,WAAW;IACzB,OAAO,GAAG,IAAI,CAAC;AACjB,CAAC;AAFD,kCAEC;AAED,SAAgB,YAAY;IAC1B,OAAO,GAAG,KAAK,CAAC;AAClB,CAAC;AAFD,oCAEC;AAEM,KAAK,UAAU,UAAU;IAC9B,OAAO,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;AAC5B,CAAC;AAFD,gCAEC;AAED,SAAgB,cAAc;IAC5B,OAAO,OAAO,CAAC;AACjB,CAAC;AAFD,wCAEC"}

package/dist/src/commands/eval/filterFailingTests.d.ts

@@ -0,0 +1,5 @@
+import { TestSuite } from "../../types";
+type Tests = NonNullable<TestSuite['tests']>;
+export declare function filterFailingTests(testSuite: TestSuite, outputPath: string): Promise<Tests>;
+export {};
+//# sourceMappingURL=filterFailingTests.d.ts.map

package/dist/src/commands/eval/filterFailingTests.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterFailingTests.d.ts","sourceRoot":"","sources":["../../../../src/commands/eval/filterFailingTests.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,aAAa,CAAC;AAGtC,KAAK,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AAE7C,wBAAsB,kBAAkB,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAejG"}

package/dist/src/commands/eval/filterFailingTests.js

@@ -0,0 +1,19 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.filterFailingTests = void 0;
+const util_1 = require("../../util");
+async function filterFailingTests(testSuite, outputPath) {
+    if (!testSuite.tests) {
+        return [];
+    }
+    const { results } = await (0, util_1.readOutput)(outputPath);
+    const failingResults = results.results.filter((result) => !result.success);
+    if (failingResults.length === 0) {
+        return [];
+    }
+    return [...testSuite.tests].filter((test) => {
+        return failingResults.some((result) => (0, util_1.resultIsForTestCase)(result, test));
+    });
+}
+exports.filterFailingTests = filterFailingTests;
+//# sourceMappingURL=filterFailingTests.js.map

package/dist/src/commands/eval/filterFailingTests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterFailingTests.js","sourceRoot":"","sources":["../../../../src/commands/eval/filterFailingTests.ts"],"names":[],"mappings":";;;AACA,qCAA2D;AAIpD,KAAK,UAAU,kBAAkB,CAAC,SAAoB,EAAE,UAAkB;IAC/E,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE;QACpB,OAAO,EAAE,CAAC;KACX;IAED,MAAM,EAAC,OAAO,EAAC,GAAG,MAAM,IAAA,iBAAU,EAAC,UAAU,CAAC,CAAC;IAC/C,MAAM,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE3E,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,OAAO,EAAE,CAAC;KACX;IAED,OAAO,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QAC1C,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAA,0BAAmB,EAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5E,CAAC,CAAU,CAAC;AACd,CAAC;AAfD,gDAeC"}

package/dist/src/commands/eval/filterTests.d.ts

@@ -0,0 +1,10 @@
+import { TestSuite } from "../../types";
+interface Args {
+    firstN?: string;
+    pattern?: string;
+    failing?: string;
+}
+type Tests = TestSuite['tests'];
+export declare function filterTests(testSuite: TestSuite, args: Args): Promise<Tests>;
+export {};
+//# sourceMappingURL=filterTests.d.ts.map

package/dist/src/commands/eval/filterTests.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterTests.d.ts","sourceRoot":"","sources":["../../../../src/commands/eval/filterTests.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,SAAS,EAAC,MAAM,aAAa,CAAC;AAGtC,UAAU,IAAI;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,KAAK,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;AAEhC,wBAAsB,WAAW,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAmClF"}

package/dist/src/commands/eval/filterTests.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.filterTests = void 0;
+const filterFailingTests_1 = require("./filterFailingTests");
+async function filterTests(testSuite, args) {
+    const tests = testSuite.tests;
+    if (!tests) {
+        return tests;
+    }
+    if (Object.keys(args).length === 0) {
+        return tests;
+    }
+    const { firstN, pattern, failing } = args;
+    let newTests;
+    if (failing) {
+        newTests = await (0, filterFailingTests_1.filterFailingTests)(testSuite, failing);
+    }
+    else {
+        newTests = [...tests];
+    }
+    if (pattern) {
+        newTests = newTests.filter((test) => test.description && test.description.match(pattern));
+    }
+    if (firstN) {
+        const count = parseInt(firstN);
+        if (isNaN(count)) {
+            throw new Error(`firstN must be a number, got: ${firstN}`);
+        }
+        newTests = newTests.slice(0, count);
+    }
+    return newTests;
+}
+exports.filterTests = filterTests;
+//# sourceMappingURL=filterTests.js.map

package/dist/src/commands/eval/filterTests.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"filterTests.js","sourceRoot":"","sources":["../../../../src/commands/eval/filterTests.ts"],"names":[],"mappings":";;;AACA,6DAAwD;AAUjD,KAAK,UAAU,WAAW,CAAC,SAAoB,EAAE,IAAU;IAChE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAE9B,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,KAAK,CAAC;KACd;IAED,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE;QAClC,OAAO,KAAK,CAAC;KACd;IAED,MAAM,EAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAC,GAAG,IAAI,CAAC;IACxC,IAAI,QAA4B,CAAC;IAEjC,IAAI,OAAO,EAAE;QACX,QAAQ,GAAG,MAAM,IAAA,uCAAkB,EAAC,SAAS,EAAE,OAAO,CAAC,CAAC;KACzD;SAAM;QACL,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;KACvB;IAED,IAAI,OAAO,EAAG;QACZ,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;KAC3F;IAED,IAAI,MAAM,EAAE;QACV,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;QAE/B,IAAI,KAAK,CAAC,KAAK,CAAC,EAAE;YAChB,MAAM,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAAC,CAAC;SAC5D;QAED,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;KACrC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAnCD,kCAmCC"}

package/dist/src/commands/export.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare function exportCommand(program: Command): void;
+//# sourceMappingURL=export.d.ts.map

package/dist/src/commands/export.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"export.d.ts","sourceRoot":"","sources":["../../../src/commands/export.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAQpC,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,QA6C7C"}

package/dist/src/commands/export.js

@@ -0,0 +1,56 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.exportCommand = void 0;
+const drizzle_orm_1 = require("drizzle-orm");
+const database_1 = require("../database");
+const fs_1 = __importDefault(require("fs"));
+const logger_1 = __importDefault(require("../logger"));
+const telemetry_1 = __importDefault(require("../telemetry"));
+function exportCommand(program) {
+    program
+        .command('export <evalId>')
+        .description('Export an eval record to a JSON file')
+        .option('-o, --output [outputPath]', 'Output path for the exported file')
+        .action(async (evalId, cmdObj) => {
+        try {
+            const db = (0, database_1.getDb)();
+            const result = await db
+                .select({
+                id: database_1.evals.id,
+                createdAt: database_1.evals.createdAt,
+                description: database_1.evals.description,
+                results: database_1.evals.results,
+                config: database_1.evals.config,
+            })
+                .from(database_1.evals)
+                .where((0, drizzle_orm_1.eq)(database_1.evals.id, evalId))
+                .execute();
+            if (!result || result.length === 0) {
+                logger_1.default.error(`No eval found with ID ${evalId}`);
+                process.exit(1);
+            }
+            const jsonData = JSON.stringify(result[0], null, 2);
+            if (cmdObj.output) {
+                fs_1.default.writeFileSync(cmdObj.output, jsonData);
+                logger_1.default.info(`Eval with ID ${evalId} has been successfully exported to ${cmdObj.output}.`);
+            }
+            else {
+                console.log(jsonData);
+            }
+            telemetry_1.default.record('command_used', {
+                name: 'export',
+                evalId: evalId,
+            });
+            await telemetry_1.default.send();
+        }
+        catch (error) {
+            logger_1.default.error(`Failed to export eval: ${error}`);
+            process.exit(1);
+        }
+    });
+}
+exports.exportCommand = exportCommand;
+//# sourceMappingURL=export.js.map

package/dist/src/commands/export.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"export.js","sourceRoot":"","sources":["../../../src/commands/export.ts"],"names":[],"mappings":";;;;;;AACA,6CAAiC;AAEjC,0CAA2C;AAC3C,4CAAoB;AACpB,uDAA+B;AAC/B,6DAAqC;AAErC,SAAgB,aAAa,CAAC,OAAgB;IAC5C,OAAO;SACJ,OAAO,CAAC,iBAAiB,CAAC;SAC1B,WAAW,CAAC,sCAAsC,CAAC;SACnD,MAAM,CAAC,2BAA2B,EAAE,mCAAmC,CAAC;SACxE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE;QAC/B,IAAI;YACF,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;YACnB,MAAM,MAAM,GAAG,MAAM,EAAE;iBACpB,MAAM,CAAC;gBACN,EAAE,EAAE,gBAAK,CAAC,EAAE;gBACZ,SAAS,EAAE,gBAAK,CAAC,SAAS;gBAC1B,WAAW,EAAE,gBAAK,CAAC,WAAW;gBAC9B,OAAO,EAAE,gBAAK,CAAC,OAAO;gBACtB,MAAM,EAAE,gBAAK,CAAC,MAAM;aACrB,CAAC;iBACD,IAAI,CAAC,gBAAK,CAAC;iBACX,KAAK,CAAC,IAAA,gBAAE,EAAC,gBAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;iBAC3B,OAAO,EAAE,CAAC;YAEb,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;gBAClC,gBAAM,CAAC,KAAK,CAAC,yBAAyB,MAAM,EAAE,CAAC,CAAC;gBAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;YAED,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACpD,IAAI,MAAM,CAAC,MAAM,EAAE;gBACjB,YAAE,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBAC1C,gBAAM,CAAC,IAAI,CACT,gBAAgB,MAAM,sCAAsC,MAAM,CAAC,MAAM,GAAG,CAC7E,CAAC;aACH;iBAAM;gBACL,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;aACvB;YAED,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;gBAC/B,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,MAAM;aACf,CAAC,CAAC;YACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;SACxB;QAAC,OAAO,KAAK,EAAE;YACd,gBAAM,CAAC,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AA7CD,sCA6CC"}

package/dist/src/commands/import.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare function importCommand(program: Command): void;
+//# sourceMappingURL=import.d.ts.map

package/dist/src/commands/import.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.d.ts","sourceRoot":"","sources":["../../../src/commands/import.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,QAgC7C"}

package/dist/src/commands/import.js

@@ -0,0 +1,44 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.importCommand = void 0;
+const database_1 = require("../database");
+const fs_1 = __importDefault(require("fs"));
+const logger_1 = __importDefault(require("../logger"));
+const telemetry_1 = __importDefault(require("../telemetry"));
+function importCommand(program) {
+    program
+        .command('import <file>')
+        .description('Import an eval record from a JSON file')
+        .action(async (file) => {
+        try {
+            const fileContent = fs_1.default.readFileSync(file, 'utf-8');
+            const evalData = JSON.parse(fileContent);
+            const db = (0, database_1.getDb)();
+            await db
+                .insert(database_1.evals)
+                .values({
+                id: evalData.id,
+                createdAt: evalData.createdAt,
+                description: evalData.description,
+                results: evalData.results,
+                config: evalData.config,
+            })
+                .run();
+            logger_1.default.info(`Eval with ID ${evalData.id} has been successfully imported.`);
+            telemetry_1.default.record('command_used', {
+                name: 'import',
+                evalId: evalData.id,
+            });
+            await telemetry_1.default.send();
+        }
+        catch (error) {
+            logger_1.default.error(`Failed to import eval: ${error}`);
+            process.exit(1);
+        }
+    });
+}
+exports.importCommand = importCommand;
+//# sourceMappingURL=import.js.map

package/dist/src/commands/import.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"import.js","sourceRoot":"","sources":["../../../src/commands/import.ts"],"names":[],"mappings":";;;;;;AACA,0CAA2C;AAC3C,4CAAoB;AACpB,uDAA+B;AAC/B,6DAAqC;AAErC,SAAgB,aAAa,CAAC,OAAgB;IAC5C,OAAO;SACJ,OAAO,CAAC,eAAe,CAAC;SACxB,WAAW,CAAC,wCAAwC,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACrB,IAAI;YACF,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAEzC,MAAM,EAAE,GAAG,IAAA,gBAAK,GAAE,CAAC;YACnB,MAAM,EAAE;iBACL,MAAM,CAAC,gBAAK,CAAC;iBACb,MAAM,CAAC;gBACN,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,SAAS,EAAE,QAAQ,CAAC,SAAS;gBAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,MAAM,EAAE,QAAQ,CAAC,MAAM;aACxB,CAAC;iBACD,GAAG,EAAE,CAAC;YAET,gBAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,CAAC,EAAE,kCAAkC,CAAC,CAAC;YAC3E,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;gBAC/B,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,QAAQ,CAAC,EAAE;aACpB,CAAC,CAAC;YACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;SACxB;QAAC,OAAO,KAAK,EAAE;YACd,gBAAM,CAAC,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YAChD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SACjB;IACH,CAAC,CAAC,CAAC;AACP,CAAC;AAhCD,sCAgCC"}

package/dist/src/commands/list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QAuG3C"}
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAOpC,wBAAgB,WAAW,CAAC,OAAO,EAAE,OAAO,QA0G3C"}

package/dist/src/commands/list.js

@@ -15,6 +15,7 @@ function listCommand(program) {
         .command('evals')
         .description('List evaluations.')
         .option('--env-path <path>', 'Path to the environment file')
+        .option('-n <limit>', 'Number of evals to display')
         .action(async (cmdObj) => {
         (0, util_1.setupEnv)(cmdObj.envPath);
         telemetry_1.default.maybeShowNotice();
@@ -22,7 +23,7 @@ function listCommand(program) {
             name: 'list evals',
         });
         await telemetry_1.default.send();
-        const evals = await (0, util_1.getEvals)();
+        const evals = await (0, util_1.getEvals)(Number(cmdObj.n) || undefined);
         const tableData = evals.map((evl) => ({
             'Eval ID': evl.id,
             Description: evl.description || '',
@@ -38,6 +39,7 @@ function listCommand(program) {
         .command('prompts')
         .description('List prompts used')
         .option('--env-path <path>', 'Path to the environment file')
+        .option('-n <limit>', 'Number of prompts to display')
         .action(async (cmdObj) => {
         (0, util_1.setupEnv)(cmdObj.envPath);
         telemetry_1.default.maybeShowNotice();
@@ -45,7 +47,7 @@ function listCommand(program) {
             name: 'list prompts',
         });
         await telemetry_1.default.send();
-        const prompts = (await (0, util_1.getPrompts)()).sort((a, b) => b.recentEvalId.localeCompare(a.recentEvalId));
+        const prompts = (await (0, util_1.getPrompts)(Number(cmdObj.n) || undefined)).sort((a, b) => b.recentEvalId.localeCompare(a.recentEvalId));
         const tableData = prompts.map((prompt) => ({
             'Prompt ID': prompt.id.slice(0, 6),
             Raw: prompt.prompt.raw.slice(0, 100) + (prompt.prompt.raw.length > 100 ? '...' : ''),
@@ -61,6 +63,7 @@ function listCommand(program) {
         .command('datasets')
         .description('List datasets used')
         .option('--env-path <path>', 'Path to the environment file')
+        .option('-n <limit>', 'Number of datasets to display')
         .action(async (cmdObj) => {
         (0, util_1.setupEnv)(cmdObj.envPath);
         telemetry_1.default.maybeShowNotice();
@@ -68,7 +71,7 @@ function listCommand(program) {
             name: 'list datasets',
         });
         await telemetry_1.default.send();
-        const datasets = (await (0, util_1.getTestCases)()).sort((a, b) => b.recentEvalId.localeCompare(a.recentEvalId));
+        const datasets = (await (0, util_1.getTestCases)(Number(cmdObj.n) || undefined)).sort((a, b) => b.recentEvalId.localeCompare(a.recentEvalId));
         const tableData = datasets.map((dataset) => ({
             'Dataset ID': dataset.id.slice(0, 6),
             'Highest scoring prompt': dataset.prompts

package/dist/src/commands/list.js.map

@@ -1 +1 @@
-{"version":3,"file":"list.js","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAG1B,kCAA4F;AAC5F,oCAAqC;AACrC,uDAA+B;AAC/B,6DAAqC;AAErC,SAAgB,WAAW,CAAC,OAAgB;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC;IAElF,WAAW;SACR,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,mBAAmB,CAAC;SAChC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,KAAK,EAAE,MAA4B,EAAE,EAAE;QAC7C,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,YAAY;SACnB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,IAAA,eAAQ,GAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,SAAS,EAAE,GAAG,CAAC,EAAE;YACjB,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,aAAM,EAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACxF,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3D,CAAC,CAAC,CAAC;QAEJ,gBAAM,CAAC,IAAI,CAAC,IAAA,iBAAS,EAAC,SAAS,CAAC,CAAC,CAAC;QAClC,IAAA,kBAAW,GAAE,CAAC;QAEd,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,2CAA2C,CAC1F,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,4BAA4B,CAAC,uCAAuC,CACxF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,WAAW;SACR,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,mBAAmB,CAAC;SAChC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,KAAK,EAAE,MAA4B,EAAE,EAAE;QAC7C,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAG,CAAC,MAAM,IAAA,iBAAU,GAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACjD,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAC7C,CAAC;QACF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAClC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACpF,SAAS,EAAE,MAAM,CAAC,KAAK;YACvB,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACpD,CAAC,CAAC,CAAC;QAEJ,gBAAM,CAAC,IAAI,CAAC,IAAA,iBAAS,EAAC,SAAS,CAAC,CAAC,CAAC;QAClC,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,4BAA4B,CAAC,uCAAuC,CACxF,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,2CAA2C,CAC1F,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,WAAW;SACR,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,oBAAoB,CAAC;SACjC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,KAAK,EAAE,MAA4B,EAAE,EAAE;QAC7C,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,eAAe;SACtB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAA,mBAAY,GAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACpD,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAC7C,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC3C,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACpC,wBAAwB,EAAE,OAAO,CAAC,OAAO;iBACtC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAClF,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,KAAK;YACxB,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;YACnC,kBAAkB,EAAE,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACrD,CAAC,CAAC,CAAC;QAEJ,gBAAM,CAAC,IAAI,CAAC,IAAA,iBAAS,EAAC,SAAS,CAAC,CAAC,CAAC;QAClC,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,6BAA6B,CAAC,wCAAwC,CAC1F,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,4BAA4B,CAAC,uCAAuC,CACxF,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,2CAA2C,CAC1F,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AAvGD,kCAuGC"}
+{"version":3,"file":"list.js","sourceRoot":"","sources":["../../../src/commands/list.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAG1B,kCAA4F;AAC5F,oCAAqC;AACrC,uDAA+B;AAC/B,6DAAqC;AAErC,SAAgB,WAAW,CAAC,OAAgB;IAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,wBAAwB,CAAC,CAAC;IAElF,WAAW;SACR,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,mBAAmB,CAAC;SAChC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,YAAY,EAAE,4BAA4B,CAAC;SAClD,MAAM,CAAC,KAAK,EAAE,MAAwC,EAAE,EAAE;QACzD,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,YAAY;SACnB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,KAAK,GAAG,MAAM,IAAA,eAAQ,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC;QAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACpC,SAAS,EAAE,GAAG,CAAC,EAAE;YACjB,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;YAClC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,aAAM,EAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YACxF,IAAI,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;SAC3D,CAAC,CAAC,CAAC;QAEJ,gBAAM,CAAC,IAAI,CAAC,IAAA,iBAAS,EAAC,SAAS,CAAC,CAAC,CAAC;QAClC,IAAA,kBAAW,GAAE,CAAC;QAEd,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,2CAA2C,CAC1F,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,4BAA4B,CAAC,uCAAuC,CACxF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,WAAW;SACR,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,mBAAmB,CAAC;SAChC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,YAAY,EAAE,8BAA8B,CAAC;SACpD,MAAM,CAAC,KAAK,EAAE,MAAwC,EAAE,EAAE;QACzD,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,cAAc;SACrB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAG,CAAC,MAAM,IAAA,iBAAU,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC9E,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAC7C,CAAC;QACF,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YAClC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACpF,SAAS,EAAE,MAAM,CAAC,KAAK;YACvB,kBAAkB,EAAE,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACpD,CAAC,CAAC,CAAC;QAEJ,gBAAM,CAAC,IAAI,CAAC,IAAA,iBAAS,EAAC,SAAS,CAAC,CAAC,CAAC;QAClC,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,4BAA4B,CAAC,uCAAuC,CACxF,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,2CAA2C,CAC1F,CAAC;IACJ,CAAC,CAAC,CAAC;IAEL,WAAW;SACR,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,oBAAoB,CAAC;SACjC,MAAM,CAAC,mBAAmB,EAAE,8BAA8B,CAAC;SAC3D,MAAM,CAAC,YAAY,EAAE,+BAA+B,CAAC;SACrD,MAAM,CAAC,KAAK,EAAE,MAAwC,EAAE,EAAE;QACzD,IAAA,eAAQ,EAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzB,mBAAS,CAAC,eAAe,EAAE,CAAC;QAC5B,mBAAS,CAAC,MAAM,CAAC,cAAc,EAAE;YAC/B,IAAI,EAAE,eAAe;SACtB,CAAC,CAAC;QACH,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;QAEvB,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAA,mBAAY,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACjF,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAC7C,CAAC;QACF,MAAM,SAAS,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC3C,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACpC,wBAAwB,EAAE,OAAO,CAAC,OAAO;iBACtC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAClF,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,KAAK;YACxB,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM;YACnC,kBAAkB,EAAE,OAAO,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACrD,CAAC,CAAC,CAAC;QAEJ,gBAAM,CAAC,IAAI,CAAC,IAAA,iBAAS,EAAC,SAAS,CAAC,CAAC,CAAC;QAClC,IAAA,kBAAW,GAAE,CAAC;QACd,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,6BAA6B,CAAC,wCAAwC,CAC1F,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,4BAA4B,CAAC,uCAAuC,CACxF,CAAC;QACF,gBAAM,CAAC,IAAI,CACT,OAAO,eAAK,CAAC,KAAK,CAAC,0BAA0B,CAAC,2CAA2C,CAC1F,CAAC;IACJ,CAAC,CAAC,CAAC;AACP,CAAC;AA1GD,kCA0GC"}

package/dist/src/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAoBA,OAAO,KAAK,EACV,WAAW,EAGX,eAAe,EAGf,eAAe,EAEf,iBAAiB,EACjB,MAAM,EAEN,SAAS,EAEV,MAAM,SAAS,CAAC;AAGjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAkClC;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GACzC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CA4BjC;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EACrC,eAAe,CAAC,EAAE,iBAAiB,EACnC,QAAQ,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,MAAM,CAAC,CA2GjB;AA0uBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EACV,WAAW,EAEX,eAAe,EAGf,eAAe,EAEf,iBAAiB,EACjB,MAAM,EAEN,SAAS,EAEV,MAAM,SAAS,CAAC;AACjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AAED,wBAAgB,gBAAgB,CAC9B,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GACzC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CA4BjC;AAED,wBAAsB,YAAY,CAChC,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EACrC,eAAe,CAAC,EAAE,iBAAiB,EACnC,QAAQ,CAAC,EAAE,WAAW,GACrB,OAAO,CAAC,MAAM,CAAC,CAsHjB;AA+uBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}

package/dist/src/evaluator.js

@@ -43,7 +43,8 @@ const suggestions_1 = require("./suggestions");
 const util_1 = require("./util");
 const azureopenaiUtil_1 = require("./providers/azureopenaiUtil");
 const wrapper_1 = require("./python/wrapper");
-const portkey_1 = require("./integrations/portkey");
+const esm_1 = require("./esm");
+const cache_1 = require("./cache");
 exports.DEFAULT_MAX_CONCURRENCY = 4;
 function generateVarCombinations(vars) {
     const keys = Object.keys(vars);
@@ -55,6 +56,9 @@ function generateVarCombinations(vars) {
             const resolvedPath = path.resolve(cliState_1.default.basePath || '', filePath);
             const filePaths = (0, glob_1.globSync)(resolvedPath.replace(/\\/g, '/'));
             values = filePaths.map((path) => `file://${path}`);
+            if (values.length === 0) {
+                throw new Error(`No files found for variable ${key} at path ${resolvedPath}`);
+            }
         }
         else {
             values = Array.isArray(vars[key]) ? vars[key] : [vars[key]];
@@ -116,12 +120,12 @@ async function renderPrompt(prompt, vars, nunjucksFilters, provider) {
             logger_1.default.debug(`Loading var ${varName} from file: ${filePath}`);
             switch (fileExtension) {
                 case 'js':
-                    const javascriptOutput = require(filePath)(varName, basePrompt, vars);
+                    const javascriptOutput = (await (await (0, esm_1.importModule)(filePath))(varName, basePrompt, vars, provider));
                     if (javascriptOutput.error) {
-                        throw new Error(`Error running JS script ${filePath}: ${javascriptOutput.error}`);
+                        throw new Error(`Error running ${filePath}: ${javascriptOutput.error}`);
                     }
                     if (!javascriptOutput.output) {
-                        throw new Error(`JS script ${filePath} did not return any output`);
+                        throw new Error(`Expected ${filePath} to return { output: string } but got ${javascriptOutput}`);
                     }
                     vars[varName] = javascriptOutput.output;
                     break;
@@ -167,9 +171,17 @@ async function renderPrompt(prompt, vars, nunjucksFilters, provider) {
     resolveVariables(vars);
     // Third party integrations
     if (prompt.raw.startsWith('portkey://')) {
-        const portKeyResult = await (0, portkey_1.getPrompt)(prompt.raw.slice('portkey://'.length), vars);
+        const { getPrompt } = await Promise.resolve().then(() => __importStar(require('./integrations/portkey')));
+        const portKeyResult = await getPrompt(prompt.raw.slice('portkey://'.length), vars);
         return JSON.stringify(portKeyResult.messages);
     }
+    else if (prompt.raw.startsWith('langfuse://')) {
+        const { getPrompt } = await Promise.resolve().then(() => __importStar(require('./integrations/langfuse')));
+        const langfusePrompt = prompt.raw.slice('langfuse://'.length);
+        const [helper, version] = langfusePrompt.split(':');
+        const langfuseResult = await getPrompt(helper, Number(version));
+        return langfuseResult;
+    }
     // Render prompt
     try {
         if (process.env.PROMPTFOO_DISABLE_JSON_AUTOESCAPE) {
@@ -258,16 +270,20 @@ class Evaluator {
                 cost: 0,
                 cached: false,
             };
-            if (!test.providerOutput) {
-                response = await provider.callApi(renderedPrompt, {
+            if (test.providerOutput) {
+                response.output = test.providerOutput;
+            }
+            else {
+                response = await (test.provider || provider).callApi(renderedPrompt, {
                     vars,
+                    logger: logger_1.default,
+                    fetchWithCache: cache_1.fetchWithCache,
+                    getCache: cache_1.getCache,
                 }, {
+                    originalProvider: provider,
                     includeLogProbs: test.assert?.some((a) => a.type === 'perplexity'),
                 });
             }
-            else {
-                response.output = test.providerOutput;
-            }
             const endTime = Date.now();
             latencyMs = endTime - startTime;
             let conversationLastInput = undefined;