promptfoo 0.91.2 → 0.92.0

package/dist/src/database/tables.js

@@ -1,14 +1,20 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.evalsToDatasetsRelations = exports.evalsToPromptsRelations = exports.evalsRelations = exports.datasetsRelations = exports.evalsToDatasets = exports.datasets = exports.evalsToTagsRelations = exports.tagsRelations = exports.evalsToTags = exports.promptsRelations = exports.evalsToPrompts = exports.evals = exports.tags = exports.prompts = void 0;
+exports.evalsToDatasetsRelations = exports.evalsToPromptsRelations = exports.evalsRelations = exports.datasetsRelations = exports.evalsToDatasets = exports.datasets = exports.evalsToProvidersRelations = exports.evalsToProviders = exports.evalsToTagsRelations = exports.tagsRelations = exports.evalsToTags = exports.promptsRelations = exports.evalsToPrompts = exports.evalResultsTable = exports.evals = exports.tags = exports.prompts = exports.providers = void 0;
 const drizzle_orm_1 = require("drizzle-orm");
 const sqlite_core_1 = require("drizzle-orm/sqlite-core");
+// ------------ Providers ------------
+exports.providers = (0, sqlite_core_1.sqliteTable)('providers', {
+    id: (0, sqlite_core_1.text)('id').primaryKey(),
+    providerId: (0, sqlite_core_1.text)('provider_id').notNull(),
+    config: (0, sqlite_core_1.text)('options', { mode: 'json' }).$type().notNull(),
+});
 // ------------ Prompts ------------
 exports.prompts = (0, sqlite_core_1.sqliteTable)('prompts', {
     id: (0, sqlite_core_1.text)('id').primaryKey(),
     createdAt: (0, sqlite_core_1.integer)('created_at')
         .notNull()
-        .default((0, drizzle_orm_1.sql) `CURRENT_TIMESTAMP`),
+        .default((0, drizzle_orm_1.sql) `cast(unixepoch() as int)`),
     prompt: (0, sqlite_core_1.text)('prompt').notNull(),
 }, (table) => ({
     createdAtIdx: (0, sqlite_core_1.index)('prompts_created_at_idx').on(table.createdAt),
@@ -27,21 +33,54 @@ exports.evals = (0, sqlite_core_1.sqliteTable)('evals', {
     id: (0, sqlite_core_1.text)('id').primaryKey(),
     createdAt: (0, sqlite_core_1.integer)('created_at')
         .notNull()
-        .default((0, drizzle_orm_1.sql) `CURRENT_TIMESTAMP`),
+        .default((0, drizzle_orm_1.sql) `cast(unixepoch() as int)`),
     author: (0, sqlite_core_1.text)('author'),
     description: (0, sqlite_core_1.text)('description'),
     results: (0, sqlite_core_1.text)('results', { mode: 'json' }).$type().notNull(),
     config: (0, sqlite_core_1.text)('config', { mode: 'json' }).$type().notNull(),
+    prompts: (0, sqlite_core_1.text)('prompts', { mode: 'json' }).$type(),
 }, (table) => ({
     createdAtIdx: (0, sqlite_core_1.index)('evals_created_at_idx').on(table.createdAt),
     authorIdx: (0, sqlite_core_1.index)('evals_author_idx').on(table.author),
 }));
-exports.evalsToPrompts = (0, sqlite_core_1.sqliteTable)('evals_to_prompts', {
+exports.evalResultsTable = (0, sqlite_core_1.sqliteTable)('eval_results', {
+    id: (0, sqlite_core_1.text)('id').primaryKey(),
+    createdAt: (0, sqlite_core_1.integer)('created_at')
+        .notNull()
+        .default((0, drizzle_orm_1.sql) `cast(unixepoch() as int)`),
+    updatedAt: (0, sqlite_core_1.integer)('updated_at')
+        .notNull()
+        .default((0, drizzle_orm_1.sql) `cast(unixepoch() as int)`),
     evalId: (0, sqlite_core_1.text)('eval_id')
         .notNull()
         .references(() => exports.evals.id),
-    // Drizzle doesn't support this migration for sqlite, so we remove foreign keys manually.
-    //.references(() => evals.id, { onDelete: 'cascade' }),
+    promptIdx: (0, sqlite_core_1.integer)('prompt_idx').notNull(),
+    testIdx: (0, sqlite_core_1.integer)('test_idx').notNull(),
+    testCase: (0, sqlite_core_1.text)('test_case', { mode: 'json' }).$type().notNull(),
+    prompt: (0, sqlite_core_1.text)('prompt', { mode: 'json' }).$type().notNull(),
+    promptId: (0, sqlite_core_1.text)('prompt_id').references(() => exports.prompts.id),
+    // Provider-related fields
+    provider: (0, sqlite_core_1.text)('provider', { mode: 'json' }).$type().notNull(),
+    providerId: (0, sqlite_core_1.text)('provider_id').references(() => exports.providers.id),
+    latencyMs: (0, sqlite_core_1.integer)('latency_ms'),
+    cost: (0, sqlite_core_1.real)('cost'),
+    // Output-related fields
+    response: (0, sqlite_core_1.text)('response', { mode: 'json' }).$type(),
+    error: (0, sqlite_core_1.text)('error'),
+    // Result-related fields
+    success: (0, sqlite_core_1.integer)('success', { mode: 'boolean' }).notNull(),
+    score: (0, sqlite_core_1.real)('score').notNull(),
+    gradingResult: (0, sqlite_core_1.text)('grading_result', { mode: 'json' }).$type(),
+    namedScores: (0, sqlite_core_1.text)('named_scores', { mode: 'json' }).$type(),
+    // Metadata fields
+    metadata: (0, sqlite_core_1.text)('metadata', { mode: 'json' }).$type(),
+}, (table) => ({
+    evalIdIdx: (0, sqlite_core_1.index)('eval_result_eval_id_idx').on(table.evalId),
+}));
+exports.evalsToPrompts = (0, sqlite_core_1.sqliteTable)('evals_to_prompts', {
+    evalId: (0, sqlite_core_1.text)('eval_id')
+        .notNull()
+        .references(() => exports.evals.id, { onDelete: 'cascade' }),
     promptId: (0, sqlite_core_1.text)('prompt_id')
         .notNull()
         .references(() => exports.prompts.id),
@@ -78,13 +117,33 @@ exports.evalsToTagsRelations = (0, drizzle_orm_1.relations)(exports.evalsToTags,
         references: [exports.tags.id],
     }),
 }));
+exports.evalsToProviders = (0, sqlite_core_1.sqliteTable)('evals_to_providers', {
+    providerId: (0, sqlite_core_1.text)('provider_id')
+        .notNull()
+        .references(() => exports.providers.id),
+    evalId: (0, sqlite_core_1.text)('eval_id')
+        .notNull()
+        .references(() => exports.evals.id),
+}, (t) => ({
+    pk: (0, sqlite_core_1.primaryKey)({ columns: [t.providerId, t.evalId] }),
+}));
+exports.evalsToProvidersRelations = (0, drizzle_orm_1.relations)(exports.evalsToProviders, ({ one }) => ({
+    provider: one(exports.providers, {
+        fields: [exports.evalsToProviders.providerId],
+        references: [exports.providers.id],
+    }),
+    eval: one(exports.evals, {
+        fields: [exports.evalsToProviders.evalId],
+        references: [exports.evals.id],
+    }),
+}));
 // ------------ Datasets ------------
 exports.datasets = (0, sqlite_core_1.sqliteTable)('datasets', {
     id: (0, sqlite_core_1.text)('id').primaryKey(),
     tests: (0, sqlite_core_1.text)('tests', { mode: 'json' }).$type(),
     createdAt: (0, sqlite_core_1.integer)('created_at')
         .notNull()
-        .default((0, drizzle_orm_1.sql) `CURRENT_TIMESTAMP`),
+        .default((0, drizzle_orm_1.sql) `cast(unixepoch() as int)`),
 }, (table) => ({
     createdAtIdx: (0, sqlite_core_1.index)('datasets_created_at_idx').on(table.createdAt),
 }));
@@ -140,7 +199,7 @@ export const llmOutputs = sqliteTable(
     id: text('id')
       .notNull()
       .unique(),
-    createdAt: integer('created_at').notNull().default(sql`CURRENT_TIMESTAMP`),
+    createdAt: integer('created_at').notNull().default(sql`cast(unixepoch() as int)`),
     evalId: text('eval_id')
       .notNull()
       .references(() => evals.id),

package/dist/src/database/tables.js.map

@@ -1 +1 @@
-{"version":3,"file":"tables.js","sourceRoot":"","sources":["../../../src/database/tables.ts"],"names":[],"mappings":";;;AAAA,6CAA6C;AAC7C,yDAOiC;AAGjC,oCAAoC;AAEvB,QAAA,OAAO,GAAG,IAAA,yBAAW,EAChC,SAAS,EACT;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,mBAAmB,CAAC;IAClC,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;CACjC,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,YAAY,EAAE,IAAA,mBAAK,EAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;CAClE,CAAC,CACH,CAAC;AAEF,iCAAiC;AAEpB,QAAA,IAAI,GAAG,IAAA,yBAAW,EAC7B,MAAM,EACN;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,IAAI,EAAE,IAAA,kBAAI,EAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,CAAC,CAAC,OAAO,EAAE;CAC/B,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,OAAO,EAAE,IAAA,mBAAK,EAAC,eAAe,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;IAC9C,eAAe,EAAE,IAAA,yBAAW,EAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;CACnF,CAAC,CACH,CAAC;AAEF,kCAAkC;AAErB,QAAA,KAAK,GAAG,IAAA,yBAAW,EAC9B,OAAO,EACP;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,mBAAmB,CAAC;IAClC,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC;IACtB,WAAW,EAAE,IAAA,kBAAI,EAAC,aAAa,CAAC;IAChC,OAAO,EAAE,IAAA,kBAAI,EAAC,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAmB,CAAC,OAAO,EAAE;IAC7E,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B,CAAC,OAAO,EAAE;CACnF,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,YAAY,EAAE,IAAA,mBAAK,EAAC,sBAAsB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;IAC/D,SAAS,EAAE,IAAA,mBAAK,EAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;CACtD,CAAC,CACH,CAAC;AAEW,QAAA,cAAc,GAAG,IAAA,yBAAW,EACvC,kBAAkB,EAClB;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,yFAAyF;IACzF,uDAAuD;IACvD,QAAQ,EAAE,IAAA,kBAAI,EAAC,WAAW,CAAC;SACxB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;CAChC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;IACnD,SAAS,EAAE,IAAA,mBAAK,EAAC,8BAA8B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7D,WAAW,EAAE,IAAA,mBAAK,EAAC,gCAAgC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;CACpE,CAAC,CACH,CAAC;AAEW,QAAA,gBAAgB,GAAG,IAAA,uBAAS,EAAC,eAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAChE,cAAc,EAAE,IAAI,CAAC,sBAAc,CAAC;CACrC,CAAC,CAAC,CAAC;AAES,QAAA,WAAW,GAAG,IAAA,yBAAW,EACpC,eAAe,EACf;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,KAAK,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC;SAClB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,YAAI,CAAC,EAAE,CAAC;CAC7B,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;IAChD,SAAS,EAAE,IAAA,mBAAK,EAAC,2BAA2B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,QAAQ,EAAE,IAAA,mBAAK,EAAC,0BAA0B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;CACxD,CAAC,CACH,CAAC;AAEW,QAAA,aAAa,GAAG,IAAA,uBAAS,EAAC,YAAI,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1D,WAAW,EAAE,IAAI,CAAC,mBAAW,CAAC;CAC/B,CAAC,CAAC,CAAC;AAES,QAAA,oBAAoB,GAAG,IAAA,uBAAS,EAAC,mBAAW,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACvE,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,mBAAW,CAAC,MAAM,CAAC;QAC5B,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,GAAG,EAAE,GAAG,CAAC,YAAI,EAAE;QACb,MAAM,EAAE,CAAC,mBAAW,CAAC,KAAK,CAAC;QAC3B,UAAU,EAAE,CAAC,YAAI,CAAC,EAAE,CAAC;KACtB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,qCAAqC;AAExB,QAAA,QAAQ,GAAG,IAAA,yBAAW,EACjC,UAAU,EACV;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B;IACtE,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,mBAAmB,CAAC;CACnC,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,YAAY,EAAE,IAAA,mBAAK,EAAC,yBAAyB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;CACnE,CAAC,CACH,CAAC;AAEW,QAAA,eAAe,GAAG,IAAA,yBAAW,EACxC,mBAAmB,EACnB;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,yFAAyF;IACzF,uDAAuD;IACvD,SAAS,EAAE,IAAA,kBAAI,EAAC,YAAY,CAAC;SAC1B,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,gBAAQ,CAAC,EAAE,CAAC;CACjC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;IACpD,SAAS,EAAE,IAAA,mBAAK,EAAC,+BAA+B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9D,YAAY,EAAE,IAAA,mBAAK,EAAC,kCAAkC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;CACxE,CAAC,CACH,CAAC;AAEW,QAAA,iBAAiB,GAAG,IAAA,uBAAS,EAAC,gBAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClE,eAAe,EAAE,IAAI,CAAC,uBAAe,CAAC;CACvC,CAAC,CAAC,CAAC;AAEJ,kCAAkC;AAErB,QAAA,cAAc,GAAG,IAAA,uBAAS,EAAC,aAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5D,cAAc,EAAE,IAAI,CAAC,sBAAc,CAAC;IACpC,eAAe,EAAE,IAAI,CAAC,uBAAe,CAAC;IACtC,WAAW,EAAE,IAAI,CAAC,mBAAW,CAAC;CAC/B,CAAC,CAAC,CAAC;AAES,QAAA,uBAAuB,GAAG,IAAA,uBAAS,EAAC,sBAAc,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7E,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,sBAAc,CAAC,MAAM,CAAC;QAC/B,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,EAAE,GAAG,CAAC,eAAO,EAAE;QACnB,MAAM,EAAE,CAAC,sBAAc,CAAC,QAAQ,CAAC;QACjC,UAAU,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;KACzB,CAAC;CACH,CAAC,CAAC,CAAC;AAES,QAAA,wBAAwB,GAAG,IAAA,uBAAS,EAAC,uBAAe,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/E,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,uBAAe,CAAC,MAAM,CAAC;QAChC,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,OAAO,EAAE,GAAG,CAAC,gBAAQ,EAAE;QACrB,MAAM,EAAE,CAAC,uBAAe,CAAC,SAAS,CAAC;QACnC,UAAU,EAAE,CAAC,gBAAQ,CAAC,EAAE,CAAC;KAC1B,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,oCAAoC;AACpC,wDAAwD;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCE"}
+{"version":3,"file":"tables.js","sourceRoot":"","sources":["../../../src/database/tables.ts"],"names":[],"mappings":";;;AAAA,6CAA6C;AAC7C,yDAQiC;AAYjC,sCAAsC;AAEzB,QAAA,SAAS,GAAG,IAAA,yBAAW,EAAC,WAAW,EAAE;IAChD,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,UAAU,EAAE,IAAA,kBAAI,EAAC,aAAa,CAAC,CAAC,OAAO,EAAE;IACzC,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAuB,CAAC,OAAO,EAAE;CACjF,CAAC,CAAC;AAEH,oCAAoC;AAEvB,QAAA,OAAO,GAAG,IAAA,yBAAW,EAChC,SAAS,EACT;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,0BAA0B,CAAC;IACzC,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;CACjC,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,YAAY,EAAE,IAAA,mBAAK,EAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;CAClE,CAAC,CACH,CAAC;AAEF,iCAAiC;AAEpB,QAAA,IAAI,GAAG,IAAA,yBAAW,EAC7B,MAAM,EACN;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,IAAI,EAAE,IAAA,kBAAI,EAAC,MAAM,CAAC,CAAC,OAAO,EAAE;IAC5B,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,CAAC,CAAC,OAAO,EAAE;CAC/B,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,OAAO,EAAE,IAAA,mBAAK,EAAC,eAAe,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;IAC9C,eAAe,EAAE,IAAA,yBAAW,EAAC,wBAAwB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;CACnF,CAAC,CACH,CAAC;AAEF,kCAAkC;AAErB,QAAA,KAAK,GAAG,IAAA,yBAAW,EAC9B,OAAO,EACP;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,0BAA0B,CAAC;IACzC,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC;IACtB,WAAW,EAAE,IAAA,kBAAI,EAAC,aAAa,CAAC;IAChC,OAAO,EAAE,IAAA,kBAAI,EAAC,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA8B,CAAC,OAAO,EAAE;IACxF,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B,CAAC,OAAO,EAAE;IAClF,OAAO,EAAE,IAAA,kBAAI,EAAC,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAqB;CACtE,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,YAAY,EAAE,IAAA,mBAAK,EAAC,sBAAsB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;IAC/D,SAAS,EAAE,IAAA,mBAAK,EAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;CACtD,CAAC,CACH,CAAC;AAEW,QAAA,gBAAgB,GAAG,IAAA,yBAAW,EACzC,cAAc,EACd;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,0BAA0B,CAAC;IACzC,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,0BAA0B,CAAC;IACzC,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC,CAAC,OAAO,EAAE;IAC1C,OAAO,EAAE,IAAA,qBAAO,EAAC,UAAU,CAAC,CAAC,OAAO,EAAE;IAEtC,QAAQ,EAAE,IAAA,kBAAI,EAAC,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAkB,CAAC,OAAO,EAAE;IAC/E,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAU,CAAC,OAAO,EAAE;IAClE,QAAQ,EAAE,IAAA,kBAAI,EAAC,WAAW,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;IAExD,0BAA0B;IAC1B,QAAQ,EAAE,IAAA,kBAAI,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAmB,CAAC,OAAO,EAAE;IAC/E,UAAU,EAAE,IAAA,kBAAI,EAAC,aAAa,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,iBAAS,CAAC,EAAE,CAAC;IAE9D,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;IAChC,IAAI,EAAE,IAAA,kBAAI,EAAC,MAAM,CAAC;IAElB,wBAAwB;IACxB,QAAQ,EAAE,IAAA,kBAAI,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAoB;IACtE,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,CAAC;IAEpB,wBAAwB;IACxB,OAAO,EAAE,IAAA,qBAAO,EAAC,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,OAAO,EAAE;IAC1D,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,CAAC,CAAC,OAAO,EAAE;IAC9B,aAAa,EAAE,IAAA,kBAAI,EAAC,gBAAgB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAiB;IAC9E,WAAW,EAAE,IAAA,kBAAI,EAAC,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B;IAEnF,kBAAkB;IAClB,QAAQ,EAAE,IAAA,kBAAI,EAAC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B;CAC7E,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,SAAS,EAAE,IAAA,mBAAK,EAAC,yBAAyB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;CAC7D,CAAC,CACH,CAAC;AAEW,QAAA,cAAc,GAAG,IAAA,yBAAW,EACvC,kBAAkB,EAClB;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACtD,QAAQ,EAAE,IAAA,kBAAI,EAAC,WAAW,CAAC;SACxB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;CAChC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;IACnD,SAAS,EAAE,IAAA,mBAAK,EAAC,8BAA8B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC7D,WAAW,EAAE,IAAA,mBAAK,EAAC,gCAAgC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;CACpE,CAAC,CACH,CAAC;AAEW,QAAA,gBAAgB,GAAG,IAAA,uBAAS,EAAC,eAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAChE,cAAc,EAAE,IAAI,CAAC,sBAAc,CAAC;CACrC,CAAC,CAAC,CAAC;AAES,QAAA,WAAW,GAAG,IAAA,yBAAW,EACpC,eAAe,EACf;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,KAAK,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC;SAClB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,YAAI,CAAC,EAAE,CAAC;CAC7B,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;IAChD,SAAS,EAAE,IAAA,mBAAK,EAAC,2BAA2B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,QAAQ,EAAE,IAAA,mBAAK,EAAC,0BAA0B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;CACxD,CAAC,CACH,CAAC;AAEW,QAAA,aAAa,GAAG,IAAA,uBAAS,EAAC,YAAI,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1D,WAAW,EAAE,IAAI,CAAC,mBAAW,CAAC;CAC/B,CAAC,CAAC,CAAC;AAES,QAAA,oBAAoB,GAAG,IAAA,uBAAS,EAAC,mBAAW,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACvE,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,mBAAW,CAAC,MAAM,CAAC;QAC5B,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,GAAG,EAAE,GAAG,CAAC,YAAI,EAAE;QACb,MAAM,EAAE,CAAC,mBAAW,CAAC,KAAK,CAAC;QAC3B,UAAU,EAAE,CAAC,YAAI,CAAC,EAAE,CAAC;KACtB,CAAC;CACH,CAAC,CAAC,CAAC;AAES,QAAA,gBAAgB,GAAG,IAAA,yBAAW,EACzC,oBAAoB,EACpB;IACE,UAAU,EAAE,IAAA,kBAAI,EAAC,aAAa,CAAC;SAC5B,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,iBAAS,CAAC,EAAE,CAAC;IACjC,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;CAC9B,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;CACtD,CAAC,CACH,CAAC;AAEW,QAAA,yBAAyB,GAAG,IAAA,uBAAS,EAAC,wBAAgB,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACjF,QAAQ,EAAE,GAAG,CAAC,iBAAS,EAAE;QACvB,MAAM,EAAE,CAAC,wBAAgB,CAAC,UAAU,CAAC;QACrC,UAAU,EAAE,CAAC,iBAAS,CAAC,EAAE,CAAC;KAC3B,CAAC;IACF,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,wBAAgB,CAAC,MAAM,CAAC;QACjC,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,qCAAqC;AAExB,QAAA,QAAQ,GAAG,IAAA,yBAAW,EACjC,UAAU,EACV;IACE,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B;IACtE,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,0BAA0B,CAAC;CAC1C,EACD,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACV,YAAY,EAAE,IAAA,mBAAK,EAAC,yBAAyB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC;CACnE,CAAC,CACH,CAAC;AAEW,QAAA,eAAe,GAAG,IAAA,yBAAW,EACxC,mBAAmB,EACnB;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,yFAAyF;IACzF,uDAAuD;IACvD,SAAS,EAAE,IAAA,kBAAI,EAAC,YAAY,CAAC;SAC1B,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,gBAAQ,CAAC,EAAE,CAAC;CACjC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;IACpD,SAAS,EAAE,IAAA,mBAAK,EAAC,+BAA+B,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9D,YAAY,EAAE,IAAA,mBAAK,EAAC,kCAAkC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;CACxE,CAAC,CACH,CAAC;AAEW,QAAA,iBAAiB,GAAG,IAAA,uBAAS,EAAC,gBAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClE,eAAe,EAAE,IAAI,CAAC,uBAAe,CAAC;CACvC,CAAC,CAAC,CAAC;AAEJ,kCAAkC;AAErB,QAAA,cAAc,GAAG,IAAA,uBAAS,EAAC,aAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5D,cAAc,EAAE,IAAI,CAAC,sBAAc,CAAC;IACpC,eAAe,EAAE,IAAI,CAAC,uBAAe,CAAC;IACtC,WAAW,EAAE,IAAI,CAAC,mBAAW,CAAC;CAC/B,CAAC,CAAC,CAAC;AAES,QAAA,uBAAuB,GAAG,IAAA,uBAAS,EAAC,sBAAc,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7E,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,sBAAc,CAAC,MAAM,CAAC;QAC/B,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,EAAE,GAAG,CAAC,eAAO,EAAE;QACnB,MAAM,EAAE,CAAC,sBAAc,CAAC,QAAQ,CAAC;QACjC,UAAU,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;KACzB,CAAC;CACH,CAAC,CAAC,CAAC;AAES,QAAA,wBAAwB,GAAG,IAAA,uBAAS,EAAC,uBAAe,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/E,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,uBAAe,CAAC,MAAM,CAAC;QAChC,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,OAAO,EAAE,GAAG,CAAC,gBAAQ,EAAE;QACrB,MAAM,EAAE,CAAC,uBAAe,CAAC,SAAS,CAAC;QACnC,UAAU,EAAE,CAAC,gBAAQ,CAAC,EAAE,CAAC;KAC1B,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,oCAAoC;AACpC,wDAAwD;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCE"}

package/dist/src/database/types.d.ts

@@ -1,9 +1,9 @@
-import type { EvaluateSummary, UnifiedConfig } from '../types';
+import type { EvaluateSummaryV3, UnifiedConfig } from '../types';
 export interface ResultsFile {
     version: number;
     createdAt: string;
     author: string | null;
-    results: EvaluateSummary;
+    results: EvaluateSummaryV3;
     config: Partial<UnifiedConfig>;
     datasetId?: string;
 }
@@ -11,7 +11,7 @@ export interface EvalWithMetadata {
     id: string;
     date: Date;
     config: Partial<UnifiedConfig>;
-    results: EvaluateSummary;
+    results: EvaluateSummaryV3;
     description?: string;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/src/database/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/database/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEjE,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB"}

package/dist/src/evaluator.d.ts

@@ -1,4 +1,5 @@
-import type { EvaluateOptions, EvaluateSummary, Prompt, TestSuite } from './types';
+import type Eval from './models/eval';
+import type { EvaluateOptions, Prompt, TestSuite } from './types';
 export declare const DEFAULT_MAX_CONCURRENCY = 4;
 /**
  * Validates if a given prompt is allowed based on the provided list of allowed
@@ -19,5 +20,5 @@ export declare const DEFAULT_MAX_CONCURRENCY = 4;
  */
 export declare function isAllowedPrompt(prompt: Prompt, allowedPrompts: string[] | undefined): boolean;
 export declare function generateVarCombinations(vars: Record<string, string | string[] | any>): Record<string, string | any[]>[];
-export declare function evaluate(testSuite: TestSuite, options: EvaluateOptions): Promise<EvaluateSummary>;
+export declare function evaluate(testSuite: TestSuite, evalRecord: Eval, options: EvaluateOptions): Promise<Eval>;
 //# sourceMappingURL=evaluator.d.ts.map

package/dist/src/evaluator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAIV,eAAe,EAGf,eAAe,EAEf,MAAM,EAGN,SAAS,EACV,MAAM,SAAS,CAAC;AAGjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,OAAO,CAM7F;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AAuyBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,4BAGtE"}
+{"version":3,"file":"evaluator.d.ts","sourceRoot":"","sources":["../../src/evaluator.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,IAAI,MAAM,eAAe,CAAC;AAMtC,OAAO,KAAK,EAIV,eAAe,EAGf,MAAM,EAGN,SAAS,EACV,MAAM,SAAS,CAAC;AAGjB,eAAO,MAAM,uBAAuB,IAAI,CAAC;AAEzC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,SAAS,GAAG,OAAO,CAM7F;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,GAC5C,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC,EAAE,CAqClC;AAixBD,wBAAgB,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,iBAGxF"}

package/dist/src/evaluator.js

@@ -43,6 +43,7 @@ const envars_1 = require("./envars");
 const evaluatorHelpers_1 = require("./evaluatorHelpers");
 const logger_1 = __importDefault(require("./logger"));
 const prompt_1 = require("./models/prompt");
+const provider_1 = __importDefault(require("./models/provider"));
 const azureopenaiUtil_1 = require("./providers/azureopenaiUtil");
 const suggestions_1 = require("./suggestions");
 const telemetry_1 = __importDefault(require("./telemetry"));
@@ -103,8 +104,9 @@ function generateVarCombinations(vars) {
     return combinations;
 }
 class Evaluator {
-    constructor(testSuite, options) {
+    constructor(testSuite, evalRecord, options) {
         this.testSuite = testSuite;
+        this.evalRecord = evalRecord;
         this.options = options;
         this.stats = {
             successes: 0,
@@ -120,7 +122,7 @@ class Evaluator {
         this.registers = {};
     }
     async runEval({ provider, prompt, // raw prompt
-    test, delay, nunjucksFilters: filters, evaluateOptions, }) {
+    test, delay, nunjucksFilters: filters, evaluateOptions, testIdx, promptIdx, }) {
         // Use the original prompt to set the label, not renderedPrompt
         const promptLabel = prompt.label;
         // Set up the special _conversation variable
@@ -145,6 +147,7 @@ class Evaluator {
             provider: {
                 id: provider.id(),
                 label: provider.label,
+                config: provider.config,
             },
             prompt: {
                 raw: renderedPrompt,
@@ -215,6 +218,10 @@ class Evaluator {
                 latencyMs,
                 cost: response.cost,
                 metadata: response.metadata,
+                promptIdx,
+                testIdx,
+                testCase: test,
+                promptId: prompt.id || '',
             };
             if (response.error) {
                 ret.error = response.error;
@@ -291,12 +298,17 @@ class Evaluator {
                 score: 0,
                 namedScores: {},
                 latencyMs,
+                promptIdx,
+                testIdx,
+                testCase: test,
+                promptId: prompt.id || '',
             };
         }
     }
     async evaluate() {
         const { testSuite, options } = this;
         const prompts = [];
+        const rowsWithSelectBestAssertion = new Set();
         await (0, evaluatorHelpers_1.runExtensionHook)(testSuite.extensions, 'beforeAll', { suite: testSuite });
         if (options.generateSuggestions) {
             // TODO(ian): Move this into its own command/file
@@ -449,7 +461,7 @@ class Evaluator {
         }
         // Set up eval cases
         const runEvalOptions = [];
-        let rowIndex = 0;
+        let testIdx = 0;
         for (let index = 0; index < tests.length; index++) {
             const testCase = tests[index];
             (0, tiny_invariant_1.default)(Array.isArray(testSuite.defaultTest?.assert || []), `defaultTest.assert is not an array in test case #${index + 1}`);
@@ -469,7 +481,7 @@ class Evaluator {
             const numRepeat = this.options.repeat || 1;
             for (let repeatIndex = 0; repeatIndex < numRepeat; repeatIndex++) {
                 for (const vars of varCombinations) {
-                    let colIndex = 0;
+                    let promptIdx = 0;
                     // Order matters - keep provider in outer loop to reduce need to swap models during local inference.
                     for (const provider of testSuite.providers) {
                         for (const prompt of testSuite.prompts) {
@@ -486,30 +498,18 @@ class Evaluator {
                                 },
                                 test: { ...testCase, vars, options: testCase.options },
                                 nunjucksFilters: testSuite.nunjucksFilters,
-                                rowIndex,
-                                colIndex,
+                                testIdx,
+                                promptIdx,
                                 repeatIndex,
                                 evaluateOptions: options,
                             });
-                            colIndex++;
+                            promptIdx++;
                         }
                     }
-                    rowIndex++;
+                    testIdx++;
                 }
             }
         }
-        // Set up table...
-        const isTest = tests.some((t) => !!t.assert);
-        const table = {
-            head: {
-                prompts,
-                vars: [
-                    ...Object.keys(testSuite.defaultTest?.vars || {}).sort(),
-                    ...Array.from(varNames).sort(),
-                ],
-            },
-            body: [],
-        };
         // Determine run parameters
         let concurrency = options.maxConcurrency || exports.DEFAULT_MAX_CONCURRENCY;
         if (concurrency > 1) {
@@ -525,7 +525,6 @@ class Evaluator {
             }
         }
         // Actually run the eval
-        const results = [];
         let numComplete = 0;
         const processEvalStep = async (evalStep, index) => {
             if (typeof index !== 'number') {
@@ -535,61 +534,21 @@ class Evaluator {
                 test: evalStep.test,
             });
             const row = await this.runEval(evalStep);
-            results.push(row);
+            if (evalStep.test.assert?.some((a) => a.type === 'select-best')) {
+                rowsWithSelectBestAssertion.add(row.testIdx);
+            }
             numComplete++;
             if (options.progressCallback) {
-                options.progressCallback(results.length, runEvalOptions.length, index, evalStep);
-            }
-            // Bookkeeping for table
-            let resultText;
-            const outputTextDisplay = (typeof row.response?.output === 'object'
-                ? JSON.stringify(row.response.output)
-                : row.response?.output || row.error || '');
-            if (isTest) {
-                if (row.success) {
-                    resultText = `${outputTextDisplay || row.error || ''}`;
-                }
-                else {
-                    resultText = `${row.error}\n---\n${outputTextDisplay}`;
-                }
+                options.progressCallback(this.evalRecord.results.length, runEvalOptions.length, index, evalStep);
             }
-            else if (row.error) {
-                resultText = `${row.error}`;
+            try {
+                await this.evalRecord.addResult(row, evalStep.test);
             }
-            else {
-                resultText = outputTextDisplay;
-            }
-            const { rowIndex, colIndex } = evalStep;
-            if (!table.body[rowIndex]) {
-                table.body[rowIndex] = {
-                    description: evalStep.test.description,
-                    outputs: [],
-                    test: evalStep.test,
-                    vars: table.head.vars
-                        .map((varName) => {
-                        const varValue = evalStep.test.vars?.[varName] || '';
-                        if (typeof varValue === 'string') {
-                            return varValue;
-                        }
-                        return JSON.stringify(varValue);
-                    })
-                        .flat(),
-                };
+            catch (error) {
+                logger_1.default.error(`Error saving result: ${error} ${JSON.stringify(row)}`);
             }
-            table.body[rowIndex].outputs[colIndex] = {
-                pass: row.success,
-                score: row.score,
-                namedScores: row.namedScores,
-                text: resultText,
-                prompt: row.prompt.raw,
-                provider: row.provider.label || row.provider.id,
-                latencyMs: row.latencyMs,
-                tokenUsage: row.response?.tokenUsage,
-                gradingResult: row.gradingResult,
-                cost: row.cost || 0,
-                metadata: row.metadata,
-            };
-            const metrics = table.head.prompts[colIndex].metrics;
+            const { promptIdx } = row;
+            const metrics = prompts[promptIdx].metrics;
             (0, tiny_invariant_1.default)(metrics, 'Expected prompt.metrics to be set');
             metrics.score += row.score;
             for (const [key, value] of Object.entries(row.namedScores)) {
@@ -705,7 +664,7 @@ class Evaluator {
         // Then run concurrent evaluations
         await async_1.default.forEachOfLimit(concurrentRunEvalOptions, concurrency, processEvalStep);
         // Do we have to run comparisons between row outputs?
-        const compareRowsCount = table.body.reduce((count, row) => count + (row.test.assert?.some((a) => a.type === 'select-best') ? 1 : 0), 0);
+        const compareRowsCount = rowsWithSelectBestAssertion.size;
         let progressBar;
         if (compareRowsCount > 0 && multibar) {
             progressBar = multibar.create(compareRowsCount, 0, {
@@ -714,60 +673,72 @@ class Evaluator {
                 vars: '',
             });
         }
-        for (let index = 0; index < table.body.length; index++) {
-            const row = table.body[index];
-            const compareAssertion = row.test.assert?.find((a) => a.type === 'select-best');
+        let compareCount = 0;
+        for (const testIdx of rowsWithSelectBestAssertion) {
+            compareCount++;
+            const resultsToCompare = this.evalRecord.results.filter((r) => r.testIdx === testIdx);
+            if (resultsToCompare.length === 0) {
+                logger_1.default.warn(`Expected results to be found for test index ${testIdx}`);
+                continue;
+            }
+            const compareAssertion = resultsToCompare[0].testCase.assert?.find((a) => a.type === 'select-best');
             if (compareAssertion) {
-                const outputs = row.outputs.map((o) => o.text);
-                const gradingResults = await (0, assertions_1.runCompareAssertion)(row.test, compareAssertion, outputs);
-                row.outputs.forEach((output, index) => {
+                const outputs = resultsToCompare.map((r) => r.response?.output || '');
+                const gradingResults = await (0, assertions_1.runCompareAssertion)(resultsToCompare[0].testCase, compareAssertion, outputs);
+                for (let index = 0; index < resultsToCompare.length; index++) {
+                    const result = resultsToCompare[index];
                     const gradingResult = gradingResults[index];
-                    if (output.gradingResult) {
-                        output.gradingResult.tokensUsed = output.gradingResult.tokensUsed || {
+                    if (result.gradingResult) {
+                        result.gradingResult.tokensUsed = result.gradingResult.tokensUsed || {
                             total: 0,
                             prompt: 0,
                             completion: 0,
                         };
-                        output.gradingResult.tokensUsed = output.gradingResult.tokensUsed || {
+                        result.gradingResult.tokensUsed = result.gradingResult.tokensUsed || {
                             total: 0,
                             prompt: 0,
                             completion: 0,
                         };
-                        output.gradingResult.tokensUsed.total =
-                            (output.gradingResult.tokensUsed.total || 0) + (gradingResult.tokensUsed?.total || 0);
-                        output.gradingResult.tokensUsed.prompt =
-                            (output.gradingResult.tokensUsed.prompt || 0) +
+                        result.gradingResult.tokensUsed.total =
+                            (result.gradingResult.tokensUsed.total || 0) + (gradingResult.tokensUsed?.total || 0);
+                        result.gradingResult.tokensUsed.prompt =
+                            (result.gradingResult.tokensUsed.prompt || 0) +
                                 (gradingResult.tokensUsed?.prompt || 0);
-                        output.gradingResult.tokensUsed.completion =
-                            (output.gradingResult.tokensUsed.completion || 0) +
+                        result.gradingResult.tokensUsed.completion =
+                            (result.gradingResult.tokensUsed.completion || 0) +
                                 (gradingResult.tokensUsed?.completion || 0);
-                        output.pass = output.gradingResult.pass =
-                            output.gradingResult.pass && gradingResult.pass;
+                        result.success = result.gradingResult.pass =
+                            result.gradingResult.pass && gradingResult.pass;
                         if (!gradingResult.pass) {
                             // Failure overrides the reason and the score
-                            output.gradingResult.reason = gradingResult.reason;
-                            output.score = output.gradingResult.score = gradingResult.score;
-                            output.text = `${gradingResult.reason}\n---\n${output.text}`;
+                            result.gradingResult.reason = gradingResult.reason;
+                            result.score = result.gradingResult.score = gradingResult.score;
                         }
-                        if (!output.gradingResult.componentResults) {
-                            output.gradingResult.componentResults = [];
+                        if (!result.gradingResult.componentResults) {
+                            result.gradingResult.componentResults = [];
                         }
-                        output.gradingResult.componentResults.push(gradingResult);
+                        result.gradingResult.componentResults.push(gradingResult);
                     }
                     else {
-                        output.gradingResult = gradingResult;
+                        result.gradingResult = gradingResult;
                     }
-                });
+                    if (this.evalRecord.persisted) {
+                        await result.save();
+                    }
+                }
                 if (progressBar) {
                     progressBar.increment({
-                        prompt: row.outputs[0].text.slice(0, 10).replace(/\n/g, ''),
+                        prompt: resultsToCompare[0].prompt.raw.slice(0, 10).replace(/\n/g, ''),
                     });
                 }
                 else {
-                    logger_1.default.debug(`Model-graded comparison #${index + 1} of ${compareRowsCount} complete`);
+                    logger_1.default.debug(`Model-graded comparison #${compareCount} of ${compareRowsCount} complete`);
                 }
             }
         }
+        await this.evalRecord.addPrompts(prompts);
+        const providers = await provider_1.default.createMultiple(testSuite.providers);
+        await this.evalRecord.addProviders(providers);
         // Finish up
         if (multibar) {
             multibar.stop();
@@ -776,7 +747,7 @@ class Evaluator {
             progressBar.stop();
         }
         await (0, evaluatorHelpers_1.runExtensionHook)(testSuite.extensions, 'afterAll', {
-            results,
+            results: this.evalRecord.results.map((r) => r.toEvaluateResult()),
             suite: testSuite,
         });
         telemetry_1.default.record('eval_ran', {
@@ -792,14 +763,14 @@ class Evaluator {
             assertionTypes: Array.from(new Set(tests.flatMap((t) => t.assert || []).map((a) => a.type))).sort(),
             eventSource: options.eventSource || 'default',
             ci: (0, envars_1.isCI)(),
-            hasAnyPass: results.some((r) => r.success),
+            hasAnyPass: this.evalRecord.results.some((r) => r.success),
             isRedteam: Boolean(testSuite.redteam),
         });
-        return { version: 2, timestamp: new Date().toISOString(), results, stats: this.stats, table };
+        return this.evalRecord;
     }
 }
-function evaluate(testSuite, options) {
-    const ev = new Evaluator(testSuite, options);
+function evaluate(testSuite, evalRecord, options) {
+    const ev = new Evaluator(testSuite, evalRecord, options);
     return ev.evaluate();
 }
 //# sourceMappingURL=evaluator.js.map