promptfoo 0.11.0 → 0.13.0

package/dist/package.json

@@ -0,0 +1,87 @@
+{
+    "name": "promptfoo",
+    "description": "Prompt engineering toolkit",
+    "author": "Ian Webster",
+    "version": "0.13.0",
+    "license": "MIT",
+    "type": "commonjs",
+    "main": "dist/src/index.js",
+    "exports": {
+        ".": {
+            "import": "./dist/src/index.js",
+            "require": "./dist/src/index.js"
+        }
+    },
+    "types": "dist/src/index.d.ts",
+    "typings": "dist/src/index.d.ts",
+    "files": [
+        "dist",
+        "src"
+    ],
+    "engines": {
+        "node": ">=12"
+    },
+    "bin": {
+        "promptfoo": "dist/src/main.js"
+    },
+    "scripts": {
+        "local": "ts-node --esm src/main.ts",
+        "install:client": "cd src/web/client && npm install",
+        "build:clean": "rm -rf dist",
+        "build:client": "cd src/web/client && npm run build && cp -r dist/ ../../../dist/src/web/client",
+        "build:watch": "tsc --watch",
+        "build": "tsc && cp src/*.html dist/src && npm run build:client && chmod +x dist/src/main.js",
+        "prepare": "npm run install:client && npm run build:clean && npm run build",
+        "test": "jest",
+        "test:watch": "jest --watch",
+        "format": "prettier -w ."
+    },
+    "devDependencies": {
+        "@types/async": "^3.2.20",
+        "@types/cache-manager": "^4.0.2",
+        "@types/cache-manager-fs-hash": "^0.0.1",
+        "@types/cli-progress": "^3.11.0",
+        "@types/cors": "^2.8.13",
+        "@types/debounce": "^1.2.1",
+        "@types/express": "^4.17.17",
+        "@types/glob": "^8.1.0",
+        "@types/jest": "^29.5.1",
+        "@types/js-yaml": "^4.0.5",
+        "@types/node-fetch": "^2.6.4",
+        "@types/nunjucks": "^3.2.2",
+        "@types/opener": "^1.4.0",
+        "@types/semver": "^7.5.0",
+        "babel-jest": "^29.5.0",
+        "jest": "^29.5.0",
+        "jest-watch-typeahead": "^2.2.2",
+        "prettier": "^2.8.8",
+        "ts-jest": "^29.1.0",
+        "ts-node": "^10.9.1",
+        "typescript": "^5.0.4"
+    },
+    "dependencies": {
+        "@apidevtools/json-schema-ref-parser": "^10.1.0",
+        "async": "^3.2.4",
+        "cache-manager": "^4.1.0",
+        "cache-manager-fs-hash": "^1.0.0",
+        "chalk": "^4.1.2",
+        "cli-progress": "^3.12.0",
+        "cli-table3": "^0.6.3",
+        "commander": "^10.0.1",
+        "cors": "^2.8.5",
+        "csv-parse": "^5.3.8",
+        "csv-stringify": "^6.3.2",
+        "debounce": "^1.2.1",
+        "express": "^4.18.2",
+        "glob": "^10.2.6",
+        "js-yaml": "^4.1.0",
+        "node-fetch": "^2.6.7",
+        "nunjucks": "^3.2.4",
+        "opener": "^1.5.2",
+        "rouge": "^1.0.3",
+        "semver": "^7.5.1",
+        "socket.io": "^4.6.1",
+        "tiny-invariant": "^1.3.1",
+        "winston": "^3.8.2"
+    }
+}

package/dist/src/__mocks__/esm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"esm.d.ts","sourceRoot":"","sources":["../../../src/__mocks__/esm.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,WAE3B"}

package/dist/src/__mocks__/esm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../../src/__mocks__/esm.ts"],"names":[],"mappings":";;;AAAA,SAAgB,YAAY;IAC1B,OAAO,WAAW,CAAC;AACrB,CAAC;AAFD,oCAEC"}

package/dist/src/assertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.d.ts","sourceRoot":"","sources":["../../src/assertions.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,SAAS,EAET,aAAa,EACb,aAAa,EACb,cAAc,EACf,MAAM,SAAS,CAAC;AA4BjB,wBAAsB,aAAa,CAAC,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAyBhG;AAED,wBAAsB,YAAY,CAChC,SAAS,EAAE,SAAS,EACpB,IAAI,EAAE,cAAc,EACpB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,aAAa,CAAC,CAiNxB;AAoBD,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,OAAO,GAAE,OAAe,GACvB,OAAO,CAAC,aAAa,CAAC,CA6CxB;AAED,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,CAAC,CAgDxB;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAwD/D;;;;;AAED,wBAGE"}

package/dist/{assertions.js → src/assertions.js}

@@ -7,17 +7,16 @@ exports.assertionFromString = exports.matchesLlmRubric = exports.matchesSimilari
 const rouge_1 = __importDefault(require("rouge"));
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const nunjucks_1 = __importDefault(require("nunjucks"));
+const telemetry_1 = __importDefault(require("./telemetry"));
 const openai_1 = require("./providers/openai");
 const util_1 = require("./util");
 const providers_1 = require("./providers");
 const prompts_1 = require("./prompts");
-const SIMILAR_REGEX = /similar(?::|\((\d+(\.\d+)?)\):)/;
 const DEFAULT_SEMANTIC_SIMILARITY_THRESHOLD = 0.8;
 function handleRougeScore(baseType, assertion, expected, output, inverted) {
     const fnName = baseType[baseType.length - 1];
     const rougeMethod = rouge_1.default[fnName];
     const score = rougeMethod(output, expected);
-    console.log(output, expected, score);
     const pass = score >= (assertion.threshold || 0.75) != inverted;
     return {
         pass,
@@ -54,6 +53,9 @@ async function runAssertion(assertion, test, output) {
     (0, tiny_invariant_1.default)(assertion.type, `Assertion must have a type: ${JSON.stringify(assertion)}`);
     const inverse = assertion.type.startsWith('not-');
     const baseType = inverse ? assertion.type.slice(4) : assertion.type;
+    telemetry_1.default.record('assertion_used', {
+        type: baseType,
+    });
     if (baseType === 'equals') {
         pass = assertion.value === output;
         return {
@@ -327,17 +329,30 @@ function assertionFromString(expected) {
         };
     }
     // New options
-    const assertionRegex = /^(not-)?(equals|contains|contains-any|contains-all|regex|icontains):(.+)$/;
+    const assertionRegex = /^(not-)?(equals|contains-any|contains-all|contains-json|is-json|regex|icontains|contains|webhook|rouge-n|similar)(?::|\((\d+(\.\d+)?)\):)?(.*)$/;
     const regexMatch = expected.match(assertionRegex);
     if (regexMatch) {
-        const [_, notPrefix, type, value] = regexMatch;
+        const [_, notPrefix, type, __, thresholdStr, value] = regexMatch;
         const fullType = notPrefix ? `not-${type}` : type;
+        const threshold = parseFloat(thresholdStr);
         if (type === 'contains-any' || type === 'contains-all') {
             return {
                 type: fullType,
                 value: value.split(',').map((s) => s.trim()),
             };
         }
+        else if (type === 'contains-json' || type === 'is-json') {
+            return {
+                type: fullType,
+            };
+        }
+        else if (type === 'rouge-n' || type === 'similar') {
+            return {
+                type: fullType,
+                value,
+                threshold: threshold || (type === 'similar' ? DEFAULT_SEMANTIC_SIMILARITY_THRESHOLD : 0.75),
+            };
+        }
         else {
             return {
                 type: fullType,
@@ -345,22 +360,6 @@ function assertionFromString(expected) {
             };
         }
     }
-    // Options that require some special handling
-    const match = expected.match(SIMILAR_REGEX);
-    if (match) {
-        const threshold = parseFloat(match[1]) || DEFAULT_SEMANTIC_SIMILARITY_THRESHOLD;
-        const rest = expected.replace(SIMILAR_REGEX, '').trim();
-        return {
-            type: 'similar',
-            value: rest,
-            threshold,
-        };
-    }
-    if (expected === 'is-json' || expected === 'contains-json') {
-        return {
-            type: expected,
-        };
-    }
     // Default to equality
     return {
         type: 'equals',

package/dist/src/assertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"assertions.js","sourceRoot":"","sources":["../../src/assertions.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAC1B,oEAAuC;AACvC,wDAAgC;AAEhC,4DAAoC;AACpC,+CAAsF;AACtF,iCAA4D;AAC5D,2CAA8C;AAC9C,uCAAmD;AAUnD,MAAM,qCAAqC,GAAG,GAAG,CAAC;AAElD,SAAS,gBAAgB,CACvB,QAAmB,EACnB,SAAoB,EACpB,QAA2B,EAC3B,MAAc,EACd,QAAiB;IAEjB,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAoB,CAAC;IAChE,MAAM,WAAW,GAAG,eAAK,CAAC,MAAM,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,IAAI,CAAC,IAAI,QAAQ,CAAC;IAEhE,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,IAAI;YACV,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,0CACtC,SAAS,CAAC,SAAS,IAAI,IACzB,EAAE;YACJ,CAAC,CAAC,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,KAAK,2BACtC,SAAS,CAAC,SAAS,IAAI,IACzB,EAAE;KACP,CAAC;AACJ,CAAC;AAEM,KAAK,UAAU,aAAa,CAAC,IAAoB,EAAE,MAAc;IACtE,MAAM,UAAU,GAAG;QACjB,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,CAAC;QACT,UAAU,EAAE,CAAC;KACd,CAAC;IAEF,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,UAAU,EAAE,CAAC;KAC5D;IAED,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE;QACnC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YAChB,OAAO,MAAM,CAAC;SACf;QAED,IAAI,MAAM,CAAC,UAAU,EAAE;YACrB,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;YAC5C,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YAC9C,UAAU,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC;SACvD;KACF;IAED,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,uBAAuB,EAAE,UAAU,EAAE,CAAC;AACrE,CAAC;AAzBD,sCAyBC;AAEM,KAAK,UAAU,YAAY,CAChC,SAAoB,EACpB,IAAoB,EACpB,MAAc;IAEd,IAAI,IAAI,GAAY,KAAK,CAAC;IAE1B,IAAA,wBAAS,EAAC,SAAS,CAAC,IAAI,EAAE,+BAA+B,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAEtF,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;IAEpE,mBAAS,CAAC,MAAM,CAAC,gBAAgB,EAAE;QACjC,IAAI,EAAE,QAAQ;KACf,CAAC,CAAC;IAEH,IAAI,QAAQ,KAAK,QAAQ,EAAE;QACzB,IAAI,GAAG,SAAS,CAAC,KAAK,KAAK,MAAM,CAAC;QAClC,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,oBAAoB,SAAS,CAAC,KAAK,GAAG;SAC3E,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,SAAS,EAAE;QAC1B,IAAI;YACF,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACnB,IAAI,GAAG,CAAC,OAAO,CAAC;SACjB;QAAC,OAAO,GAAG,EAAE;YACZ,IAAI,GAAG,OAAO,CAAC;SAChB;QACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,kCAAkC,EAAE,CAAC;KACzF;IAED,IAAI,QAAQ,KAAK,UAAU,EAAE;QAC3B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,oDAAoD,CAAC,CAAC;QACjF,IAAA,wBAAS,EACP,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EACnC,oDAAoD,CACrD,CAAC;QACF,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC;QACpD,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,SAAS,CAAC,KAAK,GAAG;SAC9E,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,cAAc,EAAE;QAC/B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,iDAAiD,CAAC,CAAC;QAC9E,IAAA,wBAAS,EACP,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAC9B,wDAAwD,CACzD,CAAC;QACF,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC;QAC3E,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,mBAAmB,SAAS,CAAC,KAAK,CAAC,IAAI,CAChF,IAAI,CACL,GAAG;SACT,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,cAAc,EAAE;QAC/B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,iDAAiD,CAAC,CAAC;QAC9E,IAAA,wBAAS,EACP,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,EAC9B,wDAAwD,CACzD,CAAC;QACF,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC;QAC5E,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,mBAAmB,SAAS,CAAC,KAAK,CAAC,IAAI,CAChF,IAAI,CACL,GAAG;SACT,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,OAAO,EAAE;QACxB,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,iDAAiD,CAAC,CAAC;QAC9E,IAAA,wBAAS,EACP,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EACnC,oDAAoD,CACrD,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,OAAO,CAAC;QACtC,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,gBAAgB,SAAS,CAAC,KAAK,GAAG;SAClF,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,WAAW,EAAE;QAC5B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,qDAAqD,CAAC,CAAC;QAClF,IAAA,wBAAS,EACP,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EACnC,qDAAqD,CACtD,CAAC;QACF,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,OAAO,CAAC;QAChF,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,YAAY,SAAS,CAAC,KAAK,GAAG;SAC9E,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,eAAe,EAAE;QAChC,IAAI,GAAG,YAAY,CAAC,MAAM,CAAC,KAAK,OAAO,CAAC;QACxC,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,sBAAsB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,oBAAoB;SACpE,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,YAAY,EAAE;QAC7B,IAAI;YACF,MAAM,cAAc,GAAG,IAAI,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC;YACtF,MAAM,OAAO,GAAG;gBACd,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB,CAAC;YACF,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC;SACpD;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,gCAAiC,GAAa,CAAC,OAAO;EACpE,SAAS,CAAC,KAAK,EAAE;aACZ,CAAC;SACH;QACD,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI;gBACV,CAAC,CAAC,kBAAkB;gBACpB,CAAC,CAAC,4BAA4B,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;EAC9D,SAAS,CAAC,KAAK,EAAE;SACd,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,SAAS,EAAE;QAC1B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,+CAA+C,CAAC,CAAC;QAC5E,IAAA,wBAAS,EACP,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EACnC,oDAAoD,CACrD,CAAC;QACF,OAAO,iBAAiB,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,SAAS,IAAI,IAAI,EAAE,OAAO,CAAC,CAAC;KACzF;IAED,IAAI,QAAQ,KAAK,YAAY,EAAE;QAC7B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,+CAA+C,CAAC,CAAC;QAC5E,IAAA,wBAAS,EACP,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EACnC,oDAAoD,CACrD,CAAC;QACF,OAAO,gBAAgB,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;KAChE;IAED,IAAI,QAAQ,KAAK,SAAS,EAAE;QAC1B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,gDAAgD,CAAC,CAAC;QAC7E,IAAA,wBAAS,EACP,OAAO,SAAS,CAAC,KAAK,KAAK,QAAQ,EACnC,gDAAgD,CACjD,CAAC;QAEF,IAAI;YACF,MAAM,OAAO,GAAG;gBACd,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE;aACtB,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAgB,EACrC,SAAS,CAAC,KAAK,EACf;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;aAC1C,EACD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAC/E,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE;gBAChB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;aAChE;YAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3C,IAAI,GAAG,YAAY,CAAC,IAAI,KAAK,OAAO,CAAC;SACtC;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,kBAAmB,GAAa,CAAC,OAAO,EAAE;aACnD,CAAC;SACH;QAED,OAAO;YACL,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,oBAAoB,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE;SACrF,CAAC;KACH;IAED,IAAI,QAAQ,KAAK,SAAS,EAAE;QAC1B,IAAA,wBAAS,EAAC,SAAS,CAAC,KAAK,EAAE,8DAA8D,CAAC,CAAC;QAC3F,OAAO,gBAAgB,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;KAChF;IAED,MAAM,IAAI,KAAK,CAAC,0BAA0B,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;AAC/D,CAAC;AArND,oCAqNC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,oDAAoD;IACpD,MAAM,WAAW,GAAG,wBAAwB,CAAC;IAE7C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAErC,IAAI,CAAC,KAAK,EAAE;QACV,OAAO,KAAK,CAAC;KACd;IAED,IAAI;QACF,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,OAAO,KAAK,CAAC;KACd;AACH,CAAC;AAEM,KAAK,UAAU,iBAAiB,CACrC,QAAgB,EAChB,MAAc,EACd,SAAiB,EACjB,UAAmB,KAAK;IAExB,MAAM,iBAAiB,GAAG,MAAM,iCAAwB,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IACpF,MAAM,eAAe,GAAG,MAAM,iCAAwB,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAEhF,MAAM,UAAU,GAAG;QACjB,KAAK,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC,CAAC;QAC5F,MAAM,EAAE,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,eAAe,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;QAC/F,UAAU,EACR,CAAC,iBAAiB,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;YAC/C,CAAC,eAAe,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;KAChD,CAAC;IAEF,IAAI,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,EAAE;QACpD,OAAO;YACL,IAAI,EAAE,KAAK;YACX,MAAM,EACJ,iBAAiB,CAAC,KAAK,IAAI,eAAe,CAAC,KAAK,IAAI,mCAAmC;YACzF,UAAU;SACX,CAAC;KACH;IAED,IAAI,CAAC,iBAAiB,CAAC,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE;QAC9D,OAAO;YACL,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,qBAAqB;YAC7B,UAAU;SACX,CAAC;KACH;IAED,MAAM,UAAU,GAAG,IAAA,uBAAgB,EAAC,iBAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,SAAS,CAAC,CAAC;IAC5F,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC;IACzE,MAAM,iBAAiB,GAAG,cAAc,UAAU,8BAA8B,SAAS,EAAE,CAAC;IAC5F,MAAM,cAAc,GAAG,cAAc,UAAU,2BAA2B,SAAS,EAAE,CAAC;IACtF,IAAI,IAAI,EAAE;QACR,OAAO;YACL,IAAI,EAAE,IAAI;YACV,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB;YACpD,UAAU;SACX,CAAC;KACH;IACD,OAAO;QACL,IAAI,EAAE,KAAK;QACX,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,cAAc;QACpD,UAAU;KACX,CAAC;AACJ,CAAC;AAlDD,8CAkDC;AAEM,KAAK,UAAU,gBAAgB,CACpC,QAAgB,EAChB,MAAc,EACd,OAAuB;IAEvB,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,KAAK,CACb,wFAAwF,CACzF,CAAC;KACH;IAED,MAAM,MAAM,GAAG,kBAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,YAAY,IAAI,gCAAsB,EAAE;QACnF,OAAO,EAAE,MAAM;QACf,MAAM,EAAE,QAAQ;KACjB,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,+BAAsB,CAAC;IAC1D,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;QAChC,QAAQ,GAAG,MAAM,IAAA,2BAAe,EAAC,QAAQ,CAAC,CAAC;KAC5C;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;QAC9B,OAAO;YACL,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,IAAI,CAAC,KAAK,IAAI,WAAW;YACjC,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;IAED,IAAI;QACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAkB,CAAC;QACxD,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;YAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;YACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;SAC7C,CAAC;QACF,OAAO,MAAM,CAAC;KACf;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO;YACL,IAAI,EAAE,KAAK;YACX,MAAM,EAAE,6BAA6B,IAAI,CAAC,MAAM,EAAE;YAClD,UAAU,EAAE;gBACV,KAAK,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,IAAI,CAAC;gBAClC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC;gBACpC,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,IAAI,CAAC;aAC7C;SACF,CAAC;KACH;AACH,CAAC;AApDD,4CAoDC;AAED,SAAgB,mBAAmB,CAAC,QAAgB;IAClD,iBAAiB;IACjB,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC9D,wCAAwC;QACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QAC/E,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACjD,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,YAAY;SACpB,CAAC;KACH;IACD,IAAI,QAAQ,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACjC,OAAO;YACL,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;SACzB,CAAC;KACH;IAED,cAAc;IACd,MAAM,cAAc,GAClB,iJAAiJ,CAAC;IACpJ,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IAElD,IAAI,UAAU,EAAE;QACd,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,KAAK,CAAC,GAAG,UAAU,CAAC;QACjE,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAClD,MAAM,SAAS,GAAG,UAAU,CAAC,YAAY,CAAC,CAAC;QAE3C,IAAI,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,cAAc,EAAE;YACtD,OAAO;gBACL,IAAI,EAAE,QAAyB;gBAC/B,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aAC7C,CAAC;SACH;aAAM,IAAI,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,SAAS,EAAE;YACzD,OAAO;gBACL,IAAI,EAAE,QAAyB;aAChC,CAAC;SACH;aAAM,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE;YACnD,OAAO;gBACL,IAAI,EAAE,QAAyB;gBAC/B,KAAK;gBACL,SAAS,EAAE,SAAS,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,qCAAqC,CAAC,CAAC,CAAC,IAAI,CAAC;aAC5F,CAAC;SACH;aAAM;YACL,OAAO;gBACL,IAAI,EAAE,QAAyB;gBAC/B,KAAK;aACN,CAAC;SACH;KACF;IAED,sBAAsB;IACtB,OAAO;QACL,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,QAAQ;KAChB,CAAC;AACJ,CAAC;AAxDD,kDAwDC;AAED,kBAAe;IACb,iBAAiB;IACjB,gBAAgB;CACjB,CAAC"}

package/dist/src/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAkC3D,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE,CAAC,CAyCzC;AAED,wBAAgB,WAAW,SAE1B;AAED,wBAAgB,YAAY,SAG3B"}

package/dist/{cache.js → src/cache.js}

@@ -61,7 +61,7 @@ async function fetchJsonWithCache(url, options = {}, timeout) {
     try {
         const data = await response.json();
         if (response.ok) {
-            logger_1.default.debug(`Storing ${url} response in cache: ${data}`);
+            logger_1.default.debug(`Storing ${url} response in cache: ${JSON.stringify(data)}`);
             await cache.set(cacheKey, JSON.stringify(data));
         }
         return {

package/dist/src/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/cache.ts"],"names":[],"mappings":";;;;;;AAAA,sDAAyB;AACzB,0DAA6B;AAE7B,kEAAyC;AACzC,kFAA4C;AAE5C,sDAA8B;AAC9B,iCAAkE;AAKlE,IAAI,aAAgC,CAAC;AAErC,IAAI,OAAO,GACT,OAAO,OAAO,CAAC,GAAG,CAAC,uBAAuB,KAAK,WAAW;IACxD,CAAC,CAAC,IAAI;IACN,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;AAEnD,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;AAE5F,SAAS,QAAQ;IACf,IAAI,CAAC,aAAa,EAAE;QAClB,MAAM,SAAS,GACb,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,mBAAI,CAAC,IAAI,CAAC,IAAA,6BAAsB,GAAE,EAAE,OAAO,CAAC,CAAC;QACnF,IAAI,CAAC,iBAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YAC7B,gBAAM,CAAC,IAAI,CAAC,4BAA4B,SAAS,GAAG,CAAC,CAAC;YACtD,iBAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;SAC9C;QACD,aAAa,GAAG,uBAAY,CAAC,OAAO,CAAC;YACnC,KAAK,EAAE,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,+BAAO,CAAC,CAAC,CAAC,QAAQ;YAChD,OAAO,EAAE;gBACP,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,KAAM;gBACzD,IAAI,EAAE,SAAS;gBACf,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE;gBACzD,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,GAAG,EAAE,iBAAiB;gBACvE,+CAA+C;aAChD;SACF,CAAC,CAAC;KACJ;IACD,OAAO,aAAa,CAAC;AACvB,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACtC,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,IAAI,CAAC,OAAO,EAAE;QACZ,MAAM,IAAI,GAAG,MAAM,IAAA,uBAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3D,OAAO;YACL,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,MAAM,IAAI,CAAC,IAAI,EAAE;SACxB,CAAC;KACH;IAED,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;IAE/B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,OAAO,CAAC;IACpB,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;IAExD,iCAAiC;IACjC,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAEjD,IAAI,cAAc,EAAE;QAClB,gBAAM,CAAC,KAAK,CAAC,iCAAiC,GAAG,KAAK,cAAc,EAAE,CAAC,CAAC;QACxE,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,cAAwB,CAAC;SAC3C,CAAC;KACH;IAED,kDAAkD;IAClD,MAAM,QAAQ,GAAG,MAAM,IAAA,uBAAgB,EAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/D,IAAI;QACF,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,QAAQ,CAAC,EAAE,EAAE;YACf,gBAAM,CAAC,KAAK,CAAC,WAAW,GAAG,uBAAuB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1E,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;SACjD;QACD,OAAO;YACL,MAAM,EAAE,KAAK;YACb,IAAI;SACL,CAAC;KACH;IAAC,OAAO,GAAG,EAAE;QACZ,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC;KAC/D;AACH,CAAC;AA7CD,gDA6CC;AAED,SAAgB,WAAW;IACzB,OAAO,GAAG,IAAI,CAAC;AACjB,CAAC;AAFD,kCAEC;AAED,SAAgB,YAAY;IAC1B,gBAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,OAAO,GAAG,KAAK,CAAC;AAClB,CAAC;AAHD,oCAGC"}

package/dist/src/esm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"esm.d.ts","sourceRoot":"","sources":["../../src/esm.ts"],"names":[],"mappings":"AAKA,wBAAgB,YAAY,IAAI,MAAM,CAOrC"}

package/dist/src/esm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../src/esm.ts"],"names":[],"mappings":";AAAA,0DAA0D;;;AAE1D,0BAA0B;AAC1B,sCAAsC;AAEtC,SAAgB,YAAY;IAC1B;;;;KAIC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAPD,oCAOC"}

package/dist/src/evaluator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAEV,eAAe,EAGf,eAAe,EAEf,SAAS,EAIV,MAAM,SAAS,CAAC;AA+VjB,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}

package/dist/{evaluator.js → src/evaluator.js}

@@ -32,6 +32,7 @@ const async_1 = __importDefault(require("async"));
 const chalk_1 = __importDefault(require("chalk"));
 const nunjucks_1 = __importDefault(require("nunjucks"));
 const logger_1 = __importDefault(require("./logger"));
+const telemetry_1 = __importDefault(require("./telemetry"));
 const assertions_1 = require("./assertions");
 const suggestions_1 = require("./suggestions");
 const DEFAULT_MAX_CONCURRENCY = 4;
@@ -201,33 +202,9 @@ class Evaluator {
                 varsWithSpecialColsRemoved.push(varWithSpecialColsRemoved);
             }
         }
-        // Set up table...
-        const isTest = tests.some((t) => !!t.assert);
-        const table = {
-            head: {
-                prompts: prompts.map((p) => p.display),
-                vars: Array.from(varNames).sort(),
-                // TODO(ian): add assertions to table?
-            },
-            body: [],
-        };
-        // And progress bar...
-        let progressbar;
-        if (options.showProgressBar) {
-            // FIXME(ian): Add var combinations too
-            const totalNumRuns = testSuite.prompts.length * testSuite.providers.length * (tests.length || 1);
-            const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
-            progressbar = new cliProgress.SingleBar({
-                format: 'Eval: [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {provider} "{prompt}" {vars}',
-            }, cliProgress.Presets.shades_classic);
-            progressbar.start(totalNumRuns, 0, {
-                provider: '',
-                prompt: '',
-                vars: '',
-            });
-        }
         // Set up eval cases
         const runEvalOptions = [];
+        let totalVarCombinations = 0;
         let rowIndex = 0;
         for (const testCase of tests) {
             // Handle default properties
@@ -240,6 +217,7 @@ class Evaluator {
             const appendToPrompt = testCase.options?.suffix || testSuite.defaultTest?.options?.suffix || '';
             // Finalize test case eval
             const varCombinations = generateVarCombinations(testCase.vars || {});
+            totalVarCombinations += varCombinations.length;
             for (const vars of varCombinations) {
                 let colIndex = 0;
                 for (const prompt of testSuite.prompts) {
@@ -261,6 +239,30 @@ class Evaluator {
                 rowIndex++;
             }
         }
+        // Set up table...
+        const isTest = tests.some((t) => !!t.assert);
+        const table = {
+            head: {
+                prompts: prompts.map((p) => p.display),
+                vars: Array.from(varNames).sort(),
+                // TODO(ian): add assertions to table?
+            },
+            body: [],
+        };
+        // Set up progress bar...
+        let progressbar;
+        if (options.showProgressBar) {
+            const totalNumRuns = testSuite.prompts.length * testSuite.providers.length * (totalVarCombinations || 1);
+            const cliProgress = await Promise.resolve().then(() => __importStar(require('cli-progress')));
+            progressbar = new cliProgress.SingleBar({
+                format: 'Eval: [{bar}] {percentage}% | ETA: {eta}s | {value}/{total} | {provider} "{prompt}" {vars}',
+            }, cliProgress.Presets.shades_classic);
+            progressbar.start(totalNumRuns, 0, {
+                provider: '',
+                prompt: '',
+                vars: '',
+            });
+        }
         // Actually run the eval
         const results = [];
         await async_1.default.forEachOfLimit(runEvalOptions, options.maxConcurrency || DEFAULT_MAX_CONCURRENCY, async (options, index) => {
@@ -309,6 +311,7 @@ class Evaluator {
         if (progressbar) {
             progressbar.stop();
         }
+        telemetry_1.default.record('eval_ran', {});
         return { version: 1, results, stats: this.stats, table };
     }
 }

package/dist/src/evaluator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kEAAqC;AAErC,kDAA0B;AAC1B,kDAA0B;AAC1B,wDAAgC;AAEhC,sDAA8B;AAC9B,4DAAoC;AACpC,6CAA6C;AAC7C,+CAAgD;AA4BhD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC,SAAS,uBAAuB,CAC9B,IAAuC;IAEvC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,YAAY,GAA6B,CAAC,EAAE,CAAC,CAAC;IAEpD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,eAAe,GAA6B,EAAE,CAAC;QAErD,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACtC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;gBAC1B,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,KAAe,EAAE,CAAC,CAAC;aAClE;SACF;QAED,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,YAAY,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;KACvC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,SAAS;IAKb,YAAY,SAAoB,EAAE,OAAwB;QACxD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,GAAG;YACX,SAAS,EAAE,CAAC;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,CAAC;gBACT,UAAU,EAAE,CAAC;gBACb,MAAM,EAAE,CAAC;aACV;SACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EACZ,QAAQ,EACR,MAAM,EACN,IAAI,EACJ,iBAAiB,GACF;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7B,MAAM,cAAc,GAAG,kBAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE/D,4DAA4D;QAC5D,IAAI,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;QACnC,IAAI,iBAAiB,EAAE;YACrB,aAAa,GAAG,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,aAAa,EAAE,CAAC;SACvD;QAED,MAAM,KAAK,GAAG;YACZ,MAAM,EAAE;gBACN,GAAG,EAAE,cAAc;gBACnB,OAAO,EAAE,aAAa;aACvB;YACD,IAAI;SACL,CAAC;QAEF,IAAI;YACF,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;YACxD,MAAM,GAAG,GAAmB;gBAC1B,GAAG,KAAK;gBACR,QAAQ;gBACR,OAAO,EAAE,KAAK;aACf,CAAC;YACF,IAAI,QAAQ,CAAC,KAAK,EAAE;gBAClB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC;aAC5B;iBAAM,IAAI,QAAQ,CAAC,MAAM,EAAE;gBAC1B,MAAM,WAAW,GAAG,MAAM,IAAA,0BAAa,EAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC/D,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE;oBACrB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC;iBAChC;gBACD,GAAG,CAAC,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC/B,IAAI,WAAW,CAAC,UAAU,EAAE;oBAC1B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC;oBAC5D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,WAAW,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC9D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC;iBACvE;aACF;iBAAM;gBACL,GAAG,CAAC,OAAO,GAAG,KAAK,CAAC;gBACpB,GAAG,CAAC,KAAK,GAAG,WAAW,CAAC;aACzB;YAED,2BAA2B;YAC3B,IAAI,QAAQ,CAAC,UAAU,EAAE;gBACvB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;gBAC9D,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;gBAChE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;gBACxE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;aACjE;YAED,IAAI,GAAG,CAAC,OAAO,EAAE;gBACf,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;aACxB;iBAAM;gBACL,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;aACvB;YAED,OAAO,GAAG,CAAC;SACZ;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,GAAG,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,GAAI,GAAa,CAAC,KAAK;gBAClD,OAAO,EAAE,KAAK;aACf,CAAC;SACH;IACH,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QACpC,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,IAAI,OAAO,CAAC,mBAAmB,EAAE;YAC/B,iDAAiD;YACjD,gBAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,MAAM,IAAA,6BAAe,EAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAC1F,IAAI,KAAK,IAAI,CAAC,UAAU,EAAE;gBACxB,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;aACzD;YAED,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC;YAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE;gBAC/B,gBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBACxE,gBAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC;gBACzB,gBAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;gBAExE,wCAAwC;gBACxC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,MAAM,EAAE,GAAG,uBAAQ,CAAC,eAAe,CAAC;wBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;wBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;qBACvB,CAAC,CAAC;oBACH,EAAE,CAAC,QAAQ,CACT,GAAG,eAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,UAAU,EAC3D,KAAK,EAAE,MAAM,EAAE,EAAE;wBACf,EAAE,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE;4BACxC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;4BACzD,QAAQ,EAAE,CAAC;yBACZ;6BAAM;4BACL,gBAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;yBACtC;wBACD,OAAO,CAAC,IAAI,CAAC,CAAC;oBAChB,CAAC,CACF,CAAC;gBACJ,CAAC,CAAC,CAAC;aACJ;YAED,IAAI,QAAQ,GAAG,CAAC,EAAE;gBAChB,gBAAM,CAAC,IAAI,CAAC,eAAK,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC,CAAC;gBACzD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;aACjB;SACF;QAED,4BAA4B;QAC5B,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE;YACtC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE;gBAC1C,MAAM,cAAc,GAClB,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,EAAE,EAAE,KAAK,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;gBAC3F,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,MAAM;oBACT,OAAO,EAAE,cAAc;iBACxB,CAAC,CAAC;aACJ;SACF;QAED,uCAAuC;QAEvC,MAAM,KAAK,GAAG,CACZ,SAAS,CAAC,KAAK,IAAI;YACjB;YACE,8DAA8D;aAC/D;SACF,CACF,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACb,MAAM,aAAa,GAAa,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;YACzE,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAgB,IAAI,GAAG,EAAE,CAAC;QACxC,MAAM,0BAA0B,GAAwC,EAAE,CAAC;QAC3E,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE;YAC5B,IAAI,QAAQ,CAAC,IAAI,EAAE;gBACjB,MAAM,yBAAyB,GAAsC,EAAE,CAAC;gBACxE,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBAChD,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtB,yBAAyB,CAAC,OAAO,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;iBAC7D;gBACD,0BAA0B,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;aAC5D;SACF;QAED,oBAAoB;QACpB,MAAM,cAAc,GAAqB,EAAE,CAAC;QAC5C,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE;YAC5B,4BAA4B;YAC5B,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC9E,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,WAAW,EAAE,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YACzF,QAAQ,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;YAC1C,QAAQ,CAAC,OAAO,CAAC,QAAQ;gBACvB,QAAQ,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC;YACxE,MAAM,eAAe,GACnB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;YAC3E,MAAM,cAAc,GAClB,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,IAAI,EAAE,CAAC;YAE3E,0BAA0B;YAC1B,MAAM,eAAe,GAAG,uBAAuB,CAAC,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YACrE,oBAAoB,IAAI,eAAe,CAAC,MAAM,CAAC;YAC/C,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE;gBAClC,IAAI,QAAQ,GAAG,CAAC,CAAC;gBACjB,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE;oBACtC,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,SAAS,EAAE;wBAC1C,cAAc,CAAC,IAAI,CAAC;4BAClB,QAAQ;4BACR,MAAM,EAAE;gCACN,GAAG,MAAM;gCACT,GAAG,EAAE,eAAe,GAAG,MAAM,CAAC,GAAG,GAAG,cAAc;6BACnD;4BACD,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE;4BAC3B,iBAAiB,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;4BACjD,QAAQ;4BACR,QAAQ;yBACT,CAAC,CAAC;wBACH,QAAQ,EAAE,CAAC;qBACZ;iBACF;gBACD,QAAQ,EAAE,CAAC;aACZ;SACF;QAED,kBAAkB;QAClB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAE7C,MAAM,KAAK,GAAkB;YAC3B,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACtC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE;gBACjC,sCAAsC;aACvC;YACD,IAAI,EAAE,EAAE;SACT,CAAC;QAEF,yBAAyB;QACzB,IAAI,WAAkC,CAAC;QACvC,IAAI,OAAO,CAAC,eAAe,EAAE;YAC3B,MAAM,YAAY,GAChB,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,oBAAoB,IAAI,CAAC,CAAC,CAAC;YACtF,MAAM,WAAW,GAAG,wDAAa,cAAc,GAAC,CAAC;YACjD,WAAW,GAAG,IAAI,WAAW,CAAC,SAAS,CACrC;gBACE,MAAM,EACJ,4FAA4F;aAC/F,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;YACF,WAAW,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE;gBACjC,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE;gBACV,IAAI,EAAE,EAAE;aACT,CAAC,CAAC;SACJ;QAED,wBAAwB;QACxB,MAAM,OAAO,GAAqB,EAAE,CAAC;QACrC,MAAM,eAAK,CAAC,cAAc,CACxB,cAAc,EACd,OAAO,CAAC,cAAc,IAAI,uBAAuB,EACjD,KAAK,EAAE,OAAuB,EAAE,KAAsB,EAAE,EAAE;YACxD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YAExC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAElB,IAAI,WAAW,EAAE;gBACf,WAAW,CAAC,SAAS,CAAC;oBACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE;oBAC/B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;oBACvC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;yBAC1C,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;yBAC5B,IAAI,CAAC,GAAG,CAAC;yBACT,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;iBAChB,CAAC,CAAC;aACJ;YAED,wBAAwB;YACxB,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;gBAC7B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;aAClD;YAED,IAAI,UAA8B,CAAC;YACnC,IAAI,MAAM,EAAE;gBACV,IAAI,GAAG,CAAC,OAAO,EAAE;oBACf,UAAU,GAAG,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBAClE;qBAAM;oBACL,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,UAAU,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;iBACrF;aACF;iBAAM,IAAI,GAAG,CAAC,KAAK,EAAE;gBACpB,UAAU,GAAG,UAAU,GAAG,CAAC,KAAK,EAAE,CAAC;aACpC;iBAAM;gBACL,UAAU,GAAG,GAAG,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;aACtD;YAED,sEAAsE;YACtE,uCAAuC;YACvC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE;gBACzB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG;oBACrB,OAAO,EAAE,EAAE;oBACX,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;iBAClF,CAAC;aACH;YACD,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,UAAU,CAAC;QACtD,CAAC,CACF,CAAC;QAEF,IAAI,WAAW,EAAE;YACf,WAAW,CAAC,IAAI,EAAE,CAAC;SACpB;QAED,mBAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAEjC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;IAC3D,CAAC;CACF;AAED,SAAgB,QAAQ,CAAC,SAAoB,EAAE,OAAwB;IACrE,MAAM,EAAE,GAAG,IAAI,SAAS,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAC7C,OAAO,EAAE,CAAC,QAAQ,EAAE,CAAC;AACvB,CAAC;AAHD,4BAGC"}

package/dist/{index.d.ts → src/index.d.ts}

@@ -1,9 +1,13 @@
+import assertions from './assertions';
+import providers from './providers';
 import type { EvaluateOptions, TestSuiteConfig } from './types';
 export * from './types';
+export { generateTable } from './table';
 interface EvaluateTestSuite extends TestSuiteConfig {
     prompts: string[];
 }
 declare function evaluate(testSuite: EvaluateTestSuite, options?: EvaluateOptions): Promise<import("./types").EvaluateSummary>;
+export { evaluate, assertions, providers };
 declare const _default: {
     evaluate: typeof evaluate;
     assertions: {

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,cAAc,CAAC;AACtC,OAAO,SAAS,MAAM,aAAa,CAAC;AAKpC,OAAO,KAAK,EAAE,eAAe,EAAa,eAAe,EAAE,MAAM,SAAS,CAAC;AAE3E,cAAc,SAAS,CAAC;AAExB,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,UAAU,iBAAkB,SAAQ,eAAe;IACjD,OAAO,EAAE,MAAM,EAAE,CAAC;CACnB;AAED,iBAAe,QAAQ,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,GAAE,eAAoB,8CAelF;AAED,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;;;;;;;;;;;;;;;AAE3C,wBAIE"}

package/dist/{index.js → src/index.js}

@@ -17,16 +17,22 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const evaluator_1 = require("./evaluator");
-const providers_1 = require("./providers");
+exports.providers = exports.assertions = exports.evaluate = exports.generateTable = void 0;
 const assertions_1 = __importDefault(require("./assertions"));
-const providers_2 = __importDefault(require("./providers"));
+exports.assertions = assertions_1.default;
+const providers_1 = __importDefault(require("./providers"));
+exports.providers = providers_1.default;
+const telemetry_1 = __importDefault(require("./telemetry"));
+const evaluator_1 = require("./evaluator");
+const providers_2 = require("./providers");
 const util_1 = require("./util");
 __exportStar(require("./types"), exports);
+var table_1 = require("./table");
+Object.defineProperty(exports, "generateTable", { enumerable: true, get: function () { return table_1.generateTable; } });
 async function evaluate(testSuite, options = {}) {
     const constructedTestSuite = {
         ...testSuite,
-        providers: await (0, providers_1.loadApiProviders)(testSuite.providers),
+        providers: await (0, providers_2.loadApiProviders)(testSuite.providers),
         tests: await (0, util_1.readTests)(testSuite.tests),
         // Full prompts expected (not filepaths)
         prompts: testSuite.prompts.map((promptContent) => ({
@@ -34,16 +40,14 @@ async function evaluate(testSuite, options = {}) {
             display: promptContent,
         })),
     };
-    return (0, evaluator_1.evaluate)(constructedTestSuite, options);
+    const ret = await (0, evaluator_1.evaluate)(constructedTestSuite, options);
+    await telemetry_1.default.send();
+    return ret;
 }
-module.exports = {
-    evaluate,
-    assertions: assertions_1.default,
-    providers: providers_2.default,
-};
+exports.evaluate = evaluate;
 exports.default = {
     evaluate,
     assertions: assertions_1.default,
-    providers: providers_2.default,
+    providers: providers_1.default,
 };
 //# sourceMappingURL=index.js.map

package/dist/src/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,8DAAsC;AAiCnB,qBAjCZ,oBAAU,CAiCY;AAhC7B,4DAAoC;AAgCL,oBAhCxB,mBAAS,CAgCwB;AA/BxC,4DAAoC;AACpC,2CAAqD;AACrD,2CAA+C;AAC/C,iCAAmC;AAGnC,0CAAwB;AAExB,iCAAwC;AAA/B,sGAAA,aAAa,OAAA;AAMtB,KAAK,UAAU,QAAQ,CAAC,SAA4B,EAAE,UAA2B,EAAE;IACjF,MAAM,oBAAoB,GAAc;QACtC,GAAG,SAAS;QACZ,SAAS,EAAE,MAAM,IAAA,4BAAgB,EAAC,SAAS,CAAC,SAAS,CAAC;QACtD,KAAK,EAAE,MAAM,IAAA,gBAAS,EAAC,SAAS,CAAC,KAAK,CAAC;QAEvC,wCAAwC;QACxC,OAAO,EAAE,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;YACjD,GAAG,EAAE,aAAa;YAClB,OAAO,EAAE,aAAa;SACvB,CAAC,CAAC;KACJ,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,IAAA,oBAAU,EAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC;IAC5D,MAAM,mBAAS,CAAC,IAAI,EAAE,CAAC;IACvB,OAAO,GAAG,CAAC;AACb,CAAC;AAEQ,4BAAQ;AAEjB,kBAAe;IACb,QAAQ;IACR,UAAU,EAAV,oBAAU;IACV,SAAS,EAAT,mBAAS;CACV,CAAC"}

package/dist/src/logger.d.ts

@@ -0,0 +1,12 @@
+import winston from 'winston';
+export declare const LOG_LEVELS: {
+    error: number;
+    warn: number;
+    info: number;
+    debug: number;
+};
+declare const logger: winston.Logger;
+export declare function getLogLevel(): string | undefined;
+export declare function setLogLevel(level: keyof typeof LOG_LEVELS): void;
+export default logger;
+//# sourceMappingURL=logger.d.ts.map

package/dist/src/logger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/logger.ts"],"names":[],"mappings":"AACA,OAAO,OAAO,MAAM,SAAS,CAAC;AAE9B,eAAO,MAAM,UAAU;;;;;CAKtB,CAAC;AAeF,QAAA,MAAM,MAAM,gBAQV,CAAC;AAEH,wBAAgB,WAAW,uBAE1B;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,OAAO,UAAU,QAMzD;AAED,eAAe,MAAM,CAAC"}

package/dist/{logger.js → src/logger.js}

@@ -3,10 +3,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setLogLevel = void 0;
+exports.setLogLevel = exports.getLogLevel = exports.LOG_LEVELS = void 0;
 const chalk_1 = __importDefault(require("chalk"));
 const winston_1 = __importDefault(require("winston"));
-const logLevels = {
+exports.LOG_LEVELS = {
     error: 0,
     warn: 1,
     info: 2,
@@ -28,12 +28,20 @@ const customFormatter = winston_1.default.format.printf(({ level, message, ...ar
     throw new Error(`Invalid log level: ${level}`);
 });
 const logger = winston_1.default.createLogger({
-    levels: logLevels,
+    levels: exports.LOG_LEVELS,
     format: winston_1.default.format.combine(winston_1.default.format.simple(), customFormatter),
-    transports: [new winston_1.default.transports.Console()],
+    transports: [
+        new winston_1.default.transports.Console({
+            level: process.env.LOG_LEVEL || 'info',
+        }),
+    ],
 });
+function getLogLevel() {
+    return logger.transports[0].level;
+}
+exports.getLogLevel = getLogLevel;
 function setLogLevel(level) {
-    if (logLevels.hasOwnProperty(level)) {
+    if (exports.LOG_LEVELS.hasOwnProperty(level)) {
         logger.transports[0].level = level;
     }
     else {

package/dist/src/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/logger.ts"],"names":[],"mappings":";;;;;;AAAA,kDAA0B;AAC1B,sDAA8B;AAEjB,QAAA,UAAU,GAAG;IACxB,KAAK,EAAE,CAAC;IACR,IAAI,EAAE,CAAC;IACP,IAAI,EAAE,CAAC;IACP,KAAK,EAAE,CAAC;CACT,CAAC;AAEF,MAAM,eAAe,GAAG,iBAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE;IAC5E,IAAI,KAAK,KAAK,OAAO,EAAE;QACrB,OAAO,eAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;KAC3B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,eAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAC9B;SAAM,IAAI,KAAK,KAAK,MAAM,EAAE;QAC3B,OAAO,OAAO,CAAC;KAChB;SAAM,IAAI,KAAK,KAAK,OAAO,EAAE;QAC5B,OAAO,eAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;KAC5B;IACD,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,iBAAO,CAAC,YAAY,CAAC;IAClC,MAAM,EAAE,kBAAU;IAClB,MAAM,EAAE,iBAAO,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,eAAe,CAAC;IACxE,UAAU,EAAE;QACV,IAAI,iBAAO,CAAC,UAAU,CAAC,OAAO,CAAC;YAC7B,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;SACvC,CAAC;KACH;CACF,CAAC,CAAC;AAEH,SAAgB,WAAW;IACzB,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACpC,CAAC;AAFD,kCAEC;AAED,SAAgB,WAAW,CAAC,KAA8B;IACxD,IAAI,kBAAU,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE;QACpC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;KACpC;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,sBAAsB,KAAK,EAAE,CAAC,CAAC;KAChD;AACH,CAAC;AAND,kCAMC;AAED,kBAAe,MAAM,CAAC"}

package/dist/src/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/main.ts"],"names":[],"mappings":""}