promptfoo 0.68.3 → 0.69.1

package/dist/drizzle/0002_tidy_juggernaut.sql

@@ -0,0 +1 @@
+ALTER TABLE evals ADD `author` text;

package/dist/drizzle/meta/0002_snapshot.json

@@ -0,0 +1,244 @@
+{
+  "version": "5",
+  "dialect": "sqlite",
+  "id": "6325e41a-40cd-4db1-9e99-df139a264a14",
+  "prevId": "476c887f-edcb-4b29-b054-50a8383cc6c2",
+  "tables": {
+    "datasets": {
+      "name": "datasets",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "tests": {
+          "name": "tests",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "evals": {
+      "name": "evals",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        },
+        "author": {
+          "name": "author",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "description": {
+          "name": "description",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": false,
+          "autoincrement": false
+        },
+        "results": {
+          "name": "results",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "config": {
+          "name": "config",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    },
+    "evals_to_datasets": {
+      "name": "evals_to_datasets",
+      "columns": {
+        "eval_id": {
+          "name": "eval_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "dataset_id": {
+          "name": "dataset_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "evals_to_datasets_eval_id_evals_id_fk": {
+          "name": "evals_to_datasets_eval_id_evals_id_fk",
+          "tableFrom": "evals_to_datasets",
+          "tableTo": "evals",
+          "columnsFrom": [
+            "eval_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "evals_to_datasets_dataset_id_datasets_id_fk": {
+          "name": "evals_to_datasets_dataset_id_datasets_id_fk",
+          "tableFrom": "evals_to_datasets",
+          "tableTo": "datasets",
+          "columnsFrom": [
+            "dataset_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "evals_to_datasets_eval_id_dataset_id_pk": {
+          "columns": [
+            "dataset_id",
+            "eval_id"
+          ],
+          "name": "evals_to_datasets_eval_id_dataset_id_pk"
+        }
+      },
+      "uniqueConstraints": {}
+    },
+    "evals_to_prompts": {
+      "name": "evals_to_prompts",
+      "columns": {
+        "eval_id": {
+          "name": "eval_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "prompt_id": {
+          "name": "prompt_id",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {
+        "evals_to_prompts_eval_id_evals_id_fk": {
+          "name": "evals_to_prompts_eval_id_evals_id_fk",
+          "tableFrom": "evals_to_prompts",
+          "tableTo": "evals",
+          "columnsFrom": [
+            "eval_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        },
+        "evals_to_prompts_prompt_id_prompts_id_fk": {
+          "name": "evals_to_prompts_prompt_id_prompts_id_fk",
+          "tableFrom": "evals_to_prompts",
+          "tableTo": "prompts",
+          "columnsFrom": [
+            "prompt_id"
+          ],
+          "columnsTo": [
+            "id"
+          ],
+          "onDelete": "no action",
+          "onUpdate": "no action"
+        }
+      },
+      "compositePrimaryKeys": {
+        "evals_to_prompts_eval_id_prompt_id_pk": {
+          "columns": [
+            "eval_id",
+            "prompt_id"
+          ],
+          "name": "evals_to_prompts_eval_id_prompt_id_pk"
+        }
+      },
+      "uniqueConstraints": {}
+    },
+    "prompts": {
+      "name": "prompts",
+      "columns": {
+        "id": {
+          "name": "id",
+          "type": "text",
+          "primaryKey": true,
+          "notNull": true,
+          "autoincrement": false
+        },
+        "created_at": {
+          "name": "created_at",
+          "type": "integer",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false,
+          "default": "CURRENT_TIMESTAMP"
+        },
+        "prompt": {
+          "name": "prompt",
+          "type": "text",
+          "primaryKey": false,
+          "notNull": true,
+          "autoincrement": false
+        }
+      },
+      "indexes": {},
+      "foreignKeys": {},
+      "compositePrimaryKeys": {},
+      "uniqueConstraints": {}
+    }
+  },
+  "enums": {},
+  "_meta": {
+    "schemas": {},
+    "tables": {},
+    "columns": {}
+  }
+}

package/dist/drizzle/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1710706957023,
       "tag": "0001_wide_calypso",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "5",
+      "when": 1719636070237,
+      "tag": "0002_tidy_juggernaut",
+      "breakpoints": true
     }
   ]
 }

package/dist/package.json

@@ -2,7 +2,7 @@
     "name": "promptfoo",
     "description": "LLM eval & testing toolkit",
     "author": "Ian Webster",
-    "version": "0.68.3",
+    "version": "0.69.1",
     "license": "MIT",
     "type": "commonjs",
     "repository": {
@@ -62,18 +62,18 @@
         "node-sql-parser": "^5.2.0"
     },
     "devDependencies": {
-        "@aws-sdk/client-bedrock-runtime": "^3.602.0",
+        "@aws-sdk/client-bedrock-runtime": "^3.609.0",
         "@azure/identity": "^4.3.0",
-        "@eslint/js": "^9.5.0",
-        "@swc/cli": "^0.3.12",
-        "@swc/core": "^1.6.1",
+        "@eslint/js": "^9.6.0",
+        "@swc/cli": "^0.4.0",
+        "@swc/core": "^1.6.13",
         "@swc/jest": "^0.2.36",
         "@trivago/prettier-plugin-sort-imports": "^4.3.0",
         "@types/async": "^3.2.24",
-        "@types/better-sqlite3": "^7.6.10",
+        "@types/better-sqlite3": "^7.6.11",
         "@types/cache-manager": "^4.0.6",
         "@types/cache-manager-fs-hash": "^0.0.4",
-        "@types/cli-progress": "^3.11.5",
+        "@types/cli-progress": "^3.11.6",
         "@types/compression": "^1.7.5",
         "@types/cors": "^2.8.17",
         "@types/debounce": "^1.2.4",
@@ -87,7 +87,7 @@
         "@types/nunjucks": "^3.2.6",
         "@types/opener": "^1.4.3",
         "@types/semver": "^7.5.8",
-        "@types/uuid": "^9.0.8",
+        "@types/uuid": "^10.0.0",
         "check-dependency-version-consistency": "^4.1.0",
         "cloudflare": "^3.4.0",
         "drizzle-kit": "^0.20.13",
@@ -101,17 +101,17 @@
         "prettier": "^3.3.2",
         "shx": "^0.3.4",
         "ts-node": "^10.9.2",
-        "typescript": "^5.4.5",
-        "typescript-eslint": "^7.13.1"
+        "typescript": "^5.5.3",
+        "typescript-eslint": "^7.15.0"
     },
     "dependencies": {
-        "@anthropic-ai/sdk": "^0.24.0",
+        "@anthropic-ai/sdk": "^0.24.3",
         "@apidevtools/json-schema-ref-parser": "^11.6.4",
-        "@googleapis/sheets": "^8.0.0",
+        "@googleapis/sheets": "^9.0.0",
         "ajv": "^8.16.0",
         "ajv-formats": "^2.1.1",
         "async": "^3.2.5",
-        "better-sqlite3": "^11.0.0",
+        "better-sqlite3": "^11.1.2",
         "cache-manager": "^4.1.0",
         "cache-manager-fs-hash": "^1.0.0",
         "chalk": "^4.1.2",
@@ -130,23 +130,24 @@
         "express": "^4.19.2",
         "fast-deep-equal": "^3.1.3",
         "fastest-levenshtein": "^1.0.16",
-        "glob": "^10.4.1",
+        "glob": "^10.4.3",
         "inquirer": "^8.2.6",
         "js-yaml": "^4.1.0",
-        "mathjs": "^13.0.0",
+        "mathjs": "^13.0.2",
         "node-fetch": "^2.6.7",
         "nunjucks": "^3.2.4",
         "openai": "^4.52.3",
         "opener": "^1.5.2",
         "proxy-agent": "^6.4.0",
         "python-shell": "^5.0.0",
-        "replicate": "^0.30.2",
+        "replicate": "^0.31.1",
         "rfdc": "^1.4.1",
         "rouge": "git+https://github.com/kenlimmj/rouge.git#f35111b599aca55f1d4dc1d4a3d15e28e7f7c55f",
         "semver": "^7.6.2",
         "socket.io": "^4.7.5",
         "tiny-invariant": "^1.3.3",
         "uuid": "^10.0.0",
-        "winston": "^3.13.0"
+        "winston": "^3.13.0",
+        "zod": "^3.23.8"
     }
 }

package/dist/src/__mocks__/database.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="jest" />
 declare const mockDbInstance: {};
 declare const mockRelations: jest.Mock<any, any, any>;
 declare const mockSqliteTable: jest.Mock<any, any, any>;

package/dist/src/__mocks__/database.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":";AAAA,QAAA,MAAM,cAAc,IAInB,CAAC;AAEF,QAAA,MAAM,aAAa,0BAAY,CAAC;AAEhC,QAAA,MAAM,eAAe,0BAGnB,CAAC"}
+{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../../src/__mocks__/database.ts"],"names":[],"mappings":"AAAA,QAAA,MAAM,cAAc,IAInB,CAAC;AAEF,QAAA,MAAM,aAAa,0BAAY,CAAC;AAEhC,QAAA,MAAM,eAAe,0BAGnB,CAAC"}

package/dist/src/__mocks__/esm.js

@@ -23,12 +23,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.importModule = exports.getDirectory = void 0;
+exports.getDirectory = getDirectory;
+exports.importModule = importModule;
 const path = __importStar(require("path"));
 function getDirectory() {
     return '/test/dir';
 }
-exports.getDirectory = getDirectory;
 function importModule(filePath, functionName) {
     const mod = require(path.resolve(filePath));
     if (functionName) {
@@ -36,5 +36,4 @@ function importModule(filePath, functionName) {
     }
     return mod;
 }
-exports.importModule = importModule;
 //# sourceMappingURL=esm.js.map

package/dist/src/__mocks__/esm.js.map

@@ -1 +1 @@
-{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../../src/__mocks__/esm.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAE7B,SAAgB,YAAY;IAC1B,OAAO,WAAW,CAAC;AACrB,CAAC;AAFD,oCAEC;AAED,SAAgB,YAAY,CAAC,QAAgB,EAAE,YAAqB;IAClE,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAND,oCAMC"}
+{"version":3,"file":"esm.js","sourceRoot":"","sources":["../../../src/__mocks__/esm.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAEA,oCAEC;AAED,oCAMC;AAZD,2CAA6B;AAE7B,SAAgB,YAAY;IAC1B,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAgB,YAAY,CAAC,QAAgB,EAAE,YAAqB;IAClE,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC5C,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC;IAC3B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/src/accounts.d.ts

@@ -1,3 +1,4 @@
-export declare function getUserEmail(): string | undefined;
+export declare function getUserEmail(): string | null;
 export declare function setUserEmail(email: string): void;
+export declare function getAuthor(): string | null;
 //# sourceMappingURL=accounts.d.ts.map

package/dist/src/accounts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"accounts.d.ts","sourceRoot":"","sources":["../../src/accounts.ts"],"names":[],"mappings":"AAEA,wBAAgB,YAAY,IAAI,MAAM,GAAG,SAAS,CAGjD;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,QAEzC"}
+{"version":3,"file":"accounts.d.ts","sourceRoot":"","sources":["../../src/accounts.ts"],"names":[],"mappings":"AAEA,wBAAgB,YAAY,IAAI,MAAM,GAAG,IAAI,CAG5C;AAED,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,QAEzC;AAED,wBAAgB,SAAS,IAAI,MAAM,GAAG,IAAI,CAEzC"}

package/dist/src/accounts.js

@@ -1,14 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.setUserEmail = exports.getUserEmail = void 0;
+exports.getUserEmail = getUserEmail;
+exports.setUserEmail = setUserEmail;
+exports.getAuthor = getAuthor;
 const globalConfig_1 = require("./globalConfig");
 function getUserEmail() {
     const globalConfig = (0, globalConfig_1.readGlobalConfig)();
-    return globalConfig.account?.email;
+    return globalConfig.account?.email || null;
 }
-exports.getUserEmail = getUserEmail;
 function setUserEmail(email) {
     (0, globalConfig_1.writeGlobalConfig)({ account: { email } });
 }
-exports.setUserEmail = setUserEmail;
+function getAuthor() {
+    return process.env.PROMPTFOO_AUTHOR || getUserEmail() || null;
+}
 //# sourceMappingURL=accounts.js.map

package/dist/src/accounts.js.map

@@ -1 +1 @@
-{"version":3,"file":"accounts.js","sourceRoot":"","sources":["../../src/accounts.ts"],"names":[],"mappings":";;;AAAA,iDAAqE;AAErE,SAAgB,YAAY;IAC1B,MAAM,YAAY,GAAG,IAAA,+BAAgB,GAAE,CAAC;IACxC,OAAO,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC;AACrC,CAAC;AAHD,oCAGC;AAED,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAA,gCAAiB,EAAC,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAFD,oCAEC"}
+{"version":3,"file":"accounts.js","sourceRoot":"","sources":["../../src/accounts.ts"],"names":[],"mappings":";;AAEA,oCAGC;AAED,oCAEC;AAED,8BAEC;AAbD,iDAAqE;AAErE,SAAgB,YAAY;IAC1B,MAAM,YAAY,GAAG,IAAA,+BAAgB,GAAE,CAAC;IACxC,OAAO,YAAY,CAAC,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC;AAC7C,CAAC;AAED,SAAgB,YAAY,CAAC,KAAa;IACxC,IAAA,gCAAiB,EAAC,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,SAAgB,SAAS;IACvB,OAAO,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,YAAY,EAAE,IAAI,IAAI,CAAC;AAChE,CAAC"}

package/dist/src/assertions/AssertionsResult.js

@@ -39,9 +39,9 @@ class AssertionsResult {
             this.namedScores[metric] = (this.namedScores[metric] || 0) + result.score;
         }
         if (result.tokensUsed) {
-            this.tokensUsed.total += result.tokensUsed.total;
-            this.tokensUsed.prompt += result.tokensUsed.prompt;
-            this.tokensUsed.completion += result.tokensUsed.completion;
+            this.tokensUsed.total += result.tokensUsed.total || 0;
+            this.tokensUsed.prompt += result.tokensUsed.prompt || 0;
+            this.tokensUsed.completion += result.tokensUsed.completion || 0;
         }
         if (result.pass) {
             return;

package/dist/src/assertions/AssertionsResult.js.map

@@ -1 +1 @@
-{"version":3,"file":"AssertionsResult.js","sourceRoot":"","sources":["../../../src/assertions/AssertionsResult.ts"],"names":[],"mappings":";;;AAEA,MAAM,mBAAmB,GAAG;IAC1B,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,CAAC;CACd,CAAC;AAOF,MAAa,gBAAgB;IAC3B,MAAM,CAAC,eAAe;QACpB,OAAO;YACL,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,EAAE,GAAG,mBAAmB,EAAE;YACtC,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAcD,YAAY,EACV,SAAS,EACT,kBAAkB,MAIhB,EAAE;QAlBE,eAAU,GAAG;YACnB,GAAG,mBAAmB;SACvB,CAAC;QAGM,eAAU,GAAW,CAAC,CAAC;QACvB,gBAAW,GAAW,CAAC,CAAC;QAExB,qBAAgB,GAAoB,EAAE,CAAC;QACvC,gBAAW,GAA2B,EAAE,CAAC;QACzC,WAAM,GAAyB,IAAI,CAAC;QAS1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,CAAC;IAChD,CAAC;IAED,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,SAAS,CAAC,EACR,KAAK,EACL,MAAM,EACN,MAAM,EACN,MAAM,GAAG,CAAC,GAMX;QACC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;QACzC,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;QAC3B,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;QAEtC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QAC5E,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;YACjD,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;YACnD,IAAI,CAAC,UAAU,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,CAAC;QAC7D,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QAElC,IAAI,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,UAAU;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC;QACjD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC;QAE9B,IAAI,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;QAE9E,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,wFAAwF;YACxF,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;YAE/B,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,GAAG,mBAAmB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,SAAS,YAAY,CAAC;YAC/E,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,mBAAmB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,SAAS,YAAY,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG;YACZ,IAAI;YACJ,KAAK;YACL,MAAM;YACN,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,SAAS,EAAE,IAAI;SAChB,CAAC;QAEF,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;CACF;AA3GD,4CA2GC"}
+{"version":3,"file":"AssertionsResult.js","sourceRoot":"","sources":["../../../src/assertions/AssertionsResult.ts"],"names":[],"mappings":";;;AAEA,MAAM,mBAAmB,GAAG;IAC1B,KAAK,EAAE,CAAC;IACR,MAAM,EAAE,CAAC;IACT,UAAU,EAAE,CAAC;CACd,CAAC;AAOF,MAAa,gBAAgB;IAC3B,MAAM,CAAC,eAAe;QACpB,OAAO;YACL,IAAI,EAAE,IAAI;YACV,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,EAAE,GAAG,mBAAmB,EAAE;YACtC,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IAcD,YAAY,EACV,SAAS,EACT,kBAAkB,MAIhB,EAAE;QAlBE,eAAU,GAAG;YACnB,GAAG,mBAAmB;SACvB,CAAC;QAGM,eAAU,GAAW,CAAC,CAAC;QACvB,gBAAW,GAAW,CAAC,CAAC;QAExB,qBAAgB,GAAoB,EAAE,CAAC;QACvC,gBAAW,GAA2B,EAAE,CAAC;QACzC,WAAM,GAAyB,IAAI,CAAC;QAS1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,mBAAmB,GAAG,kBAAkB,CAAC;IAChD,CAAC;IAED,IAAI,kBAAkB;QACpB,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED,SAAS,CAAC,EACR,KAAK,EACL,MAAM,EACN,MAAM,EACN,MAAM,GAAG,CAAC,GAMX;QACC,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC;QACzC,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;QAC3B,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;QAEtC,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QAC5E,CAAC;QAED,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,IAAI,CAAC,UAAU,CAAC,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,CAAC;YACtD,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,IAAI,CAAC,CAAC;YACxD,IAAI,CAAC,UAAU,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC;QAElC,IAAI,OAAO,CAAC,GAAG,CAAC,qCAAqC,EAAE,CAAC;YACtD,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAED,UAAU;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC;QACjD,IAAI,IAAI,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC;QAE9B,IAAI,MAAM,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;QAE9E,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,wFAAwF;YACxF,IAAI,GAAG,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC;YAE/B,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,GAAG,mBAAmB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,SAAS,YAAY,CAAC;YAC/E,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,mBAAmB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,SAAS,YAAY,CAAC;YAC/E,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,GAAG;YACZ,IAAI;YACJ,KAAK;YACL,MAAM;YACN,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,SAAS,EAAE,IAAI;SAChB,CAAC;QAEF,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;CACF;AA3GD,4CA2GC"}

package/dist/src/assertions/validateAssertions.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateAssertions = exports.AssertValiationError = void 0;
+exports.AssertValiationError = void 0;
+exports.validateAssertions = validateAssertions;
 class AssertValiationError extends Error {
     constructor(message, testCase) {
         const testCaseDescription = testCase.description || JSON.stringify(testCase);
@@ -31,5 +32,4 @@ function validateAssertions(tests) {
         }
     }
 }
-exports.validateAssertions = validateAssertions;
 //# sourceMappingURL=validateAssertions.js.map

package/dist/src/assertions/validateAssertions.js.map

@@ -1 +1 @@
-{"version":3,"file":"validateAssertions.js","sourceRoot":"","sources":["../../../src/assertions/validateAssertions.ts"],"names":[],"mappings":";;;AAEA,MAAa,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe,EAAE,QAAkB;QAC7C,MAAM,mBAAmB,GAAG,QAAQ,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAE7E,KAAK,CAAC,GAAG,OAAO,SAAS,mBAAmB,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAPD,oDAOC;AAED,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAc;IAC1D,IAAI,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAAC,2CAA2C,EAAE,IAAI,CAAC,CAAC;IACpF,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,oBAAoB,CAAC,oDAAoD,EAAE,IAAI,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,EAAE,IAAI,CAAC,CAAC;IACrF,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAA6D;IAC9F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpC,IAAI,SAAS,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACpC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAVD,gDAUC"}
+{"version":3,"file":"validateAssertions.js","sourceRoot":"","sources":["../../../src/assertions/validateAssertions.ts"],"names":[],"mappings":";;;AAyBA,gDAUC;AAjCD,MAAa,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe,EAAE,QAAkB;QAC7C,MAAM,mBAAmB,GAAG,QAAQ,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAE7E,KAAK,CAAC,GAAG,OAAO,SAAS,mBAAmB,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAPD,oDAOC;AAED,SAAS,iBAAiB,CAAC,SAAiB,EAAE,IAAc;IAC1D,IAAI,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAAC,2CAA2C,EAAE,IAAI,CAAC,CAAC;IACpF,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,oBAAoB,CAAC,oDAAoD,EAAE,IAAI,CAAC,CAAC;IAC7F,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,YAAY,CAAC,EAAE,CAAC;QAC1E,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,EAAE,IAAI,CAAC,CAAC;IACrF,CAAC;AACH,CAAC;AAED,SAAgB,kBAAkB,CAAC,KAA6D;IAC9F,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpC,IAAI,SAAS,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACpC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/src/assertions.js

@@ -26,7 +26,12 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.readAssertions = exports.runCompareAssertion = exports.runAssertions = exports.runAssertion = exports.isSql = exports.MODEL_GRADED_ASSERTION_TYPES = void 0;
+exports.MODEL_GRADED_ASSERTION_TYPES = void 0;
+exports.isSql = isSql;
+exports.runAssertion = runAssertion;
+exports.runAssertions = runAssertions;
+exports.runCompareAssertion = runCompareAssertion;
+exports.readAssertions = readAssertions;
 const ajv_1 = __importDefault(require("ajv"));
 const ajv_formats_1 = __importDefault(require("ajv-formats"));
 const async_1 = __importDefault(require("async"));
@@ -157,7 +162,6 @@ async function isSql(outputString, renderedValue, inverse, assertion) {
         assertion,
     };
 }
-exports.isSql = isSql;
 async function runAssertion({ prompt, provider, assertion, test, output, latencyMs, logProbs, cost, }) {
     let pass = false;
     let score = 0.0;
@@ -990,7 +994,6 @@ ${isMultiline
     }
     throw new Error('Unknown assertion type: ' + assertion.type);
 }
-exports.runAssertion = runAssertion;
 async function runAssertions({ prompt, provider, test, output, latencyMs, logProbs, cost, }) {
     if (!test.assert || test.assert.length < 1) {
         return AssertionsResult_1.AssertionsResult.noAssertsResult();
@@ -1055,7 +1058,6 @@ async function runAssertions({ prompt, provider, test, output, latencyMs, logPro
     });
     return mainAssertResult.testResult();
 }
-exports.runAssertions = runAssertions;
 async function runCompareAssertion(test, assertion, outputs) {
     (0, tiny_invariant_1.default)(typeof assertion.value === 'string', 'select-best must have a string value');
     test = getFinalTest(test, assertion);
@@ -1065,7 +1067,6 @@ async function runCompareAssertion(test, assertion, outputs) {
         assertion,
     }));
 }
-exports.runCompareAssertion = runCompareAssertion;
 async function readAssertions(filePath) {
     try {
         const assertions = js_yaml_1.default.load(fs_1.default.readFileSync(filePath, 'utf-8'));
@@ -1078,7 +1079,6 @@ async function readAssertions(filePath) {
         throw new Error(`Failed to read assertions from ${filePath}:\n${err}`);
     }
 }
-exports.readAssertions = readAssertions;
 // These exports are used by the node.js package (index.ts)
 exports.default = {
     runAssertion,