promptfoo 0.72.2 → 0.73.1

package/dist/src/database/index.js

@@ -0,0 +1,50 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDbPath = getDbPath;
+exports.getDbSignalPath = getDbSignalPath;
+exports.getDb = getDb;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const better_sqlite3_2 = require("drizzle-orm/better-sqlite3");
+const path = __importStar(require("path"));
+const config_1 = require("../util/config");
+let dbInstance = null;
+function getDbPath() {
+    return path.resolve((0, config_1.getConfigDirectoryPath)(true /* createIfNotExists */), 'promptfoo.db');
+}
+function getDbSignalPath() {
+    return path.resolve((0, config_1.getConfigDirectoryPath)(true /* createIfNotExists */), 'evalLastWritten');
+}
+function getDb() {
+    if (!dbInstance) {
+        const sqlite = new better_sqlite3_1.default(getDbPath());
+        dbInstance = (0, better_sqlite3_2.drizzle)(sqlite);
+    }
+    return dbInstance;
+}
+//# sourceMappingURL=index.js.map

package/dist/src/database/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/database/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAOA,8BAEC;AAED,0CAEC;AAED,sBAMC;AArBD,oEAAsC;AACtC,+DAAqD;AACrD,2CAA6B;AAC7B,2CAAwD;AAExD,IAAI,UAAU,GAAsC,IAAI,CAAC;AAEzD,SAAgB,SAAS;IACvB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAA,+BAAsB,EAAC,IAAI,CAAC,uBAAuB,CAAC,EAAE,cAAc,CAAC,CAAC;AAC5F,CAAC;AAED,SAAgB,eAAe;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,IAAA,+BAAsB,EAAC,IAAI,CAAC,uBAAuB,CAAC,EAAE,iBAAiB,CAAC,CAAC;AAC/F,CAAC;AAED,SAAgB,KAAK;IACnB,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,IAAI,wBAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;QACzC,UAAU,GAAG,IAAA,wBAAO,EAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}

package/dist/src/{database.d.ts → database/operations.d.ts}

@@ -1,4 +1,4 @@
-import type { EvaluateSummary } from './types';
+import type { EvaluateSummary } from '../types';
 export declare const prompts: import("drizzle-orm/sqlite-core").SQLiteTableWithColumns<{
     name: "prompts";
     schema: undefined;
@@ -265,10 +265,10 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 }>) & {
                     label?: string | undefined;
                 }) | (string | {
+                    id?: string | undefined;
+                    config?: any;
                     prompts?: string[] | undefined;
                     delay?: number | undefined;
-                    config?: any;
-                    id?: string | undefined;
                     label?: string | undefined;
                     transform?: string | undefined;
                     env?: {
@@ -350,10 +350,10 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 }>) & {
                     label?: string | undefined;
                 }) | Record<string, {
+                    id?: string | undefined;
+                    config?: any;
                     prompts?: string[] | undefined;
                     delay?: number | undefined;
-                    config?: any;
-                    id?: string | undefined;
                     label?: string | undefined;
                     transform?: string | undefined;
                     env?: {
@@ -399,11 +399,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         postprocess?: string | undefined;
                         storeOutputAs?: string | undefined;
                     } & {
+                        provider?: any;
                         rubricPrompt?: string | string[] | {
                             role: string;
                             content: string;
                         }[] | undefined;
-                        provider?: any;
                         factuality?: {
                             subset?: number | undefined;
                             superset?: number | undefined;
@@ -415,12 +415,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         disableVarExpansion?: boolean | undefined;
                         disableConversationVar?: boolean | undefined;
                     }) | undefined;
-                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
                     provider?: string | {
+                        id?: string | undefined;
+                        config?: any;
                         prompts?: string[] | undefined;
                         delay?: number | undefined;
-                        config?: any;
-                        id?: string | undefined;
                         label?: string | undefined;
                         transform?: string | undefined;
                         env?: {
@@ -522,31 +521,33 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             classification?: Record<string, number> | undefined;
                         }>) | undefined;
                     } | undefined;
+                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                    metadata?: Record<string, any> | undefined;
                     description?: string | undefined;
                     providerOutput?: string | {} | undefined;
                     assert?: ({
-                        type: import("./types").AssertionType;
-                        value?: import("./types").AssertionValue | undefined;
+                        type: import("../types").AssertionType;
+                        value?: import("../types").AssertionValue | undefined;
+                        provider?: any;
                         transform?: string | undefined;
                         rubricPrompt?: string | string[] | {
                             role: string;
                             content: string;
                         }[] | undefined;
-                        provider?: any;
                         threshold?: number | undefined;
                         weight?: number | undefined;
                         metric?: string | undefined;
                     } | {
                         type: "assert-set";
                         assert: {
-                            type: import("./types").AssertionType;
-                            value?: import("./types").AssertionValue | undefined;
+                            type: import("../types").AssertionType;
+                            value?: import("../types").AssertionValue | undefined;
+                            provider?: any;
                             transform?: string | undefined;
                             rubricPrompt?: string | string[] | {
                                 role: string;
                                 content: string;
                             }[] | undefined;
-                            provider?: any;
                             threshold?: number | undefined;
                             weight?: number | undefined;
                             metric?: string | undefined;
@@ -557,49 +558,6 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                     })[] | undefined;
                     threshold?: number | undefined;
                 })[];
-                evaluateOptions: {
-                    maxConcurrency?: number | undefined;
-                    repeat?: number | undefined;
-                    delay?: number | undefined;
-                    cache?: boolean | undefined;
-                    interactiveProviders?: boolean | undefined;
-                    generateSuggestions?: boolean | undefined;
-                    showProgressBar?: boolean | undefined;
-                    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: import("./types").RunEvalOptions) => void) | undefined;
-                    eventSource?: string | undefined;
-                };
-                commandLineOptions: {
-                    prompts?: string[] | undefined;
-                    providers?: string[] | undefined;
-                    output?: string[] | undefined;
-                    maxConcurrency?: string | undefined;
-                    repeat?: string | undefined;
-                    delay?: string | undefined;
-                    vars?: string | undefined;
-                    tests?: string | undefined;
-                    config?: string[] | undefined;
-                    assertions?: string | undefined;
-                    modelOutputs?: string | undefined;
-                    verbose?: boolean | undefined;
-                    grader?: string | undefined;
-                    tableCellMaxLength?: string | undefined;
-                    write?: boolean | undefined;
-                    cache?: boolean | undefined;
-                    table?: boolean | undefined;
-                    share?: boolean | undefined;
-                    progressBar?: boolean | undefined;
-                    watch?: boolean | undefined;
-                    interactiveProviders?: boolean | undefined;
-                    filterFailing?: string | undefined;
-                    filterFirstN?: string | undefined;
-                    filterPattern?: string | undefined;
-                    filterProviders?: string | undefined;
-                    var?: Record<string, string> | undefined;
-                    generateSuggestions?: boolean | undefined;
-                    promptPrefix?: string | undefined;
-                    promptSuffix?: string | undefined;
-                    envFile?: string | undefined;
-                };
                 env?: {
                     ANTHROPIC_API_KEY?: string | undefined;
                     BAM_API_KEY?: string | undefined;
@@ -636,7 +594,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 metadata?: Record<string, any> | undefined;
                 description?: string | undefined;
                 scenarios?: {
-                    tests: {
+                    config: {
                         options?: ({
                             prefix?: string | undefined;
                             suffix?: string | undefined;
@@ -645,11 +603,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             postprocess?: string | undefined;
                             storeOutputAs?: string | undefined;
                         } & {
+                            provider?: any;
                             rubricPrompt?: string | string[] | {
                                 role: string;
                                 content: string;
                             }[] | undefined;
-                            provider?: any;
                             factuality?: {
                                 subset?: number | undefined;
                                 superset?: number | undefined;
@@ -661,12 +619,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             disableVarExpansion?: boolean | undefined;
                             disableConversationVar?: boolean | undefined;
                         }) | undefined;
-                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
                         provider?: string | {
+                            id?: string | undefined;
+                            config?: any;
                             prompts?: string[] | undefined;
                             delay?: number | undefined;
-                            config?: any;
-                            id?: string | undefined;
                             label?: string | undefined;
                             transform?: string | undefined;
                             env?: {
@@ -768,31 +725,33 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                                 classification?: Record<string, number> | undefined;
                             }>) | undefined;
                         } | undefined;
+                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                        metadata?: Record<string, any> | undefined;
                         description?: string | undefined;
                         providerOutput?: string | {} | undefined;
                         assert?: ({
-                            type: import("./types").AssertionType;
-                            value?: import("./types").AssertionValue | undefined;
+                            type: import("../types").AssertionType;
+                            value?: import("../types").AssertionValue | undefined;
+                            provider?: any;
                             transform?: string | undefined;
                             rubricPrompt?: string | string[] | {
                                 role: string;
                                 content: string;
                             }[] | undefined;
-                            provider?: any;
                             threshold?: number | undefined;
                             weight?: number | undefined;
                             metric?: string | undefined;
                         } | {
                             type: "assert-set";
                             assert: {
-                                type: import("./types").AssertionType;
-                                value?: import("./types").AssertionValue | undefined;
+                                type: import("../types").AssertionType;
+                                value?: import("../types").AssertionValue | undefined;
+                                provider?: any;
                                 transform?: string | undefined;
                                 rubricPrompt?: string | string[] | {
                                     role: string;
                                     content: string;
                                 }[] | undefined;
-                                provider?: any;
                                 threshold?: number | undefined;
                                 weight?: number | undefined;
                                 metric?: string | undefined;
@@ -803,7 +762,7 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         })[] | undefined;
                         threshold?: number | undefined;
                     }[];
-                    config: {
+                    tests: {
                         options?: ({
                             prefix?: string | undefined;
                             suffix?: string | undefined;
@@ -812,11 +771,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             postprocess?: string | undefined;
                             storeOutputAs?: string | undefined;
                         } & {
+                            provider?: any;
                             rubricPrompt?: string | string[] | {
                                 role: string;
                                 content: string;
                             }[] | undefined;
-                            provider?: any;
                             factuality?: {
                                 subset?: number | undefined;
                                 superset?: number | undefined;
@@ -828,12 +787,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             disableVarExpansion?: boolean | undefined;
                             disableConversationVar?: boolean | undefined;
                         }) | undefined;
-                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
                         provider?: string | {
+                            id?: string | undefined;
+                            config?: any;
                             prompts?: string[] | undefined;
                             delay?: number | undefined;
-                            config?: any;
-                            id?: string | undefined;
                             label?: string | undefined;
                             transform?: string | undefined;
                             env?: {
@@ -935,31 +893,33 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                                 classification?: Record<string, number> | undefined;
                             }>) | undefined;
                         } | undefined;
+                        vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                        metadata?: Record<string, any> | undefined;
                         description?: string | undefined;
                         providerOutput?: string | {} | undefined;
                         assert?: ({
-                            type: import("./types").AssertionType;
-                            value?: import("./types").AssertionValue | undefined;
+                            type: import("../types").AssertionType;
+                            value?: import("../types").AssertionValue | undefined;
+                            provider?: any;
                             transform?: string | undefined;
                             rubricPrompt?: string | string[] | {
                                 role: string;
                                 content: string;
                             }[] | undefined;
-                            provider?: any;
                             threshold?: number | undefined;
                             weight?: number | undefined;
                             metric?: string | undefined;
                         } | {
                             type: "assert-set";
                             assert: {
-                                type: import("./types").AssertionType;
-                                value?: import("./types").AssertionValue | undefined;
+                                type: import("../types").AssertionType;
+                                value?: import("../types").AssertionValue | undefined;
+                                provider?: any;
                                 transform?: string | undefined;
                                 rubricPrompt?: string | string[] | {
                                     role: string;
                                     content: string;
                                 }[] | undefined;
-                                provider?: any;
                                 threshold?: number | undefined;
                                 weight?: number | undefined;
                                 metric?: string | undefined;
@@ -981,11 +941,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         postprocess?: string | undefined;
                         storeOutputAs?: string | undefined;
                     } & {
+                        provider?: any;
                         rubricPrompt?: string | string[] | {
                             role: string;
                             content: string;
                         }[] | undefined;
-                        provider?: any;
                         factuality?: {
                             subset?: number | undefined;
                             superset?: number | undefined;
@@ -997,12 +957,11 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                         disableVarExpansion?: boolean | undefined;
                         disableConversationVar?: boolean | undefined;
                     }) | undefined;
-                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
                     provider?: string | {
+                        id?: string | undefined;
+                        config?: any;
                         prompts?: string[] | undefined;
                         delay?: number | undefined;
-                        config?: any;
-                        id?: string | undefined;
                         label?: string | undefined;
                         transform?: string | undefined;
                         env?: {
@@ -1104,30 +1063,32 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                             classification?: Record<string, number> | undefined;
                         }>) | undefined;
                     } | undefined;
+                    vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                    metadata?: Record<string, any> | undefined;
                     providerOutput?: string | {} | undefined;
                     assert?: ({
-                        type: import("./types").AssertionType;
-                        value?: import("./types").AssertionValue | undefined;
+                        type: import("../types").AssertionType;
+                        value?: import("../types").AssertionValue | undefined;
+                        provider?: any;
                         transform?: string | undefined;
                         rubricPrompt?: string | string[] | {
                             role: string;
                             content: string;
                         }[] | undefined;
-                        provider?: any;
                         threshold?: number | undefined;
                         weight?: number | undefined;
                         metric?: string | undefined;
                     } | {
                         type: "assert-set";
                         assert: {
-                            type: import("./types").AssertionType;
-                            value?: import("./types").AssertionValue | undefined;
+                            type: import("../types").AssertionType;
+                            value?: import("../types").AssertionValue | undefined;
+                            provider?: any;
                             transform?: string | undefined;
                             rubricPrompt?: string | string[] | {
                                 role: string;
                                 content: string;
                             }[] | undefined;
-                            provider?: any;
                             threshold?: number | undefined;
                             weight?: number | undefined;
                             metric?: string | undefined;
@@ -1140,14 +1101,68 @@ export declare const evals: import("drizzle-orm/sqlite-core").SQLiteTableWithCol
                 } | undefined;
                 nunjucksFilters?: Record<string, string> | undefined;
                 derivedMetrics?: {
-                    value: string | ((args_0: Record<string, number>, args_1: import("./types").RunEvalOptions, ...args_2: unknown[]) => number);
+                    value: string | ((args_0: Record<string, number>, args_1: import("../types").RunEvalOptions, ...args_2: unknown[]) => number);
                     name: string;
                 }[] | undefined;
+                redteam?: {
+                    plugins: any[];
+                    strategies: ({
+                        id: string;
+                    } | {
+                        id: string;
+                    })[];
+                    provider?: string | undefined;
+                    injectVar?: string | undefined;
+                    purpose?: string | undefined;
+                } | undefined;
                 outputPath?: string | string[] | undefined;
                 sharing?: boolean | {
                     apiBaseUrl?: string | undefined;
                     appBaseUrl?: string | undefined;
                 } | undefined;
+                evaluateOptions?: {
+                    repeat?: number | undefined;
+                    cache?: boolean | undefined;
+                    maxConcurrency?: number | undefined;
+                    delay?: number | undefined;
+                    interactiveProviders?: boolean | undefined;
+                    generateSuggestions?: boolean | undefined;
+                    showProgressBar?: boolean | undefined;
+                    progressCallback?: ((args_0: number, args_1: number, args_2: number, args_3: import("../types").RunEvalOptions) => void) | undefined;
+                    eventSource?: string | undefined;
+                } | undefined;
+                commandLineOptions?: {
+                    repeat?: string | undefined;
+                    cache?: boolean | undefined;
+                    config?: string[] | undefined;
+                    envFile?: string | undefined;
+                    output?: string[] | undefined;
+                    write?: boolean | undefined;
+                    prompts?: string[] | undefined;
+                    providers?: string[] | undefined;
+                    maxConcurrency?: string | undefined;
+                    delay?: string | undefined;
+                    vars?: string | undefined;
+                    tests?: string | undefined;
+                    assertions?: string | undefined;
+                    modelOutputs?: string | undefined;
+                    verbose?: boolean | undefined;
+                    grader?: string | undefined;
+                    tableCellMaxLength?: string | undefined;
+                    table?: boolean | undefined;
+                    share?: boolean | undefined;
+                    progressBar?: boolean | undefined;
+                    watch?: boolean | undefined;
+                    interactiveProviders?: boolean | undefined;
+                    filterFailing?: string | undefined;
+                    filterFirstN?: string | undefined;
+                    filterPattern?: string | undefined;
+                    filterProviders?: string | undefined;
+                    var?: Record<string, string> | undefined;
+                    generateSuggestions?: boolean | undefined;
+                    promptPrefix?: string | undefined;
+                    promptSuffix?: string | undefined;
+                } | undefined;
             }>;
             driverParam: string;
             notNull: true;
@@ -1238,11 +1253,11 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
                     postprocess?: string | undefined;
                     storeOutputAs?: string | undefined;
                 } & {
+                    provider?: any;
                     rubricPrompt?: string | string[] | {
                         role: string;
                         content: string;
                     }[] | undefined;
-                    provider?: any;
                     factuality?: {
                         subset?: number | undefined;
                         superset?: number | undefined;
@@ -1254,12 +1269,11 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
                     disableVarExpansion?: boolean | undefined;
                     disableConversationVar?: boolean | undefined;
                 }) | undefined;
-                vars?: Record<string, string | any[] | string[] | {}> | undefined;
                 provider?: string | {
+                    id?: string | undefined;
+                    config?: any;
                     prompts?: string[] | undefined;
                     delay?: number | undefined;
-                    config?: any;
-                    id?: string | undefined;
                     label?: string | undefined;
                     transform?: string | undefined;
                     env?: {
@@ -1361,31 +1375,33 @@ export declare const datasets: import("drizzle-orm/sqlite-core").SQLiteTableWith
                         classification?: Record<string, number> | undefined;
                     }>) | undefined;
                 } | undefined;
+                vars?: Record<string, string | any[] | string[] | {}> | undefined;
+                metadata?: Record<string, any> | undefined;
                 description?: string | undefined;
                 providerOutput?: string | {} | undefined;
                 assert?: ({
-                    type: import("./types").AssertionType;
-                    value?: import("./types").AssertionValue | undefined;
+                    type: import("../types").AssertionType;
+                    value?: import("../types").AssertionValue | undefined;
+                    provider?: any;
                     transform?: string | undefined;
                     rubricPrompt?: string | string[] | {
                         role: string;
                         content: string;
                     }[] | undefined;
-                    provider?: any;
                     threshold?: number | undefined;
                     weight?: number | undefined;
                     metric?: string | undefined;
                 } | {
                     type: "assert-set";
                     assert: {
-                        type: import("./types").AssertionType;
-                        value?: import("./types").AssertionValue | undefined;
+                        type: import("../types").AssertionType;
+                        value?: import("../types").AssertionValue | undefined;
+                        provider?: any;
                         transform?: string | undefined;
                         rubricPrompt?: string | string[] | {
                             role: string;
                             content: string;
                         }[] | undefined;
-                        provider?: any;
                         threshold?: number | undefined;
                         weight?: number | undefined;
                         metric?: string | undefined;
@@ -1479,7 +1495,4 @@ export declare const evalsToDatasetsRelations: import("drizzle-orm").Relations<"
     eval: import("drizzle-orm").One<"evals", true>;
     dataset: import("drizzle-orm").One<"datasets", true>;
 }>;
-export declare function getDbPath(): string;
-export declare function getDbSignalPath(): string;
-export declare function getDb(): import("drizzle-orm/better-sqlite3").BetterSQLite3Database<Record<string, unknown>>;
-//# sourceMappingURL=database.d.ts.map
+//# sourceMappingURL=operations.d.ts.map

package/dist/src/{database.d.ts.map → database/operations.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"database.d.ts","sourceRoot":"","sources":["../../src/database.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,SAAS,CAAC;AAK9D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC;AA+CJ,wBAAgB,SAAS,WAExB;AAED,wBAAgB,eAAe,WAE9B;AAED,wBAAgB,KAAK,wFAMpB"}
+{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,MAAM,UAAU,CAAC;AAI/D,eAAO,MAAM,OAAO;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMlB,CAAC;AAEH,eAAO,MAAM,KAAK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAShB,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe1B,CAAC;AAEF,eAAO,MAAM,gBAAgB;;EAE1B,CAAC;AAIJ,eAAO,MAAM,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAMnB,CAAC;AAEH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAe3B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;EAE3B,CAAC;AAIJ,eAAO,MAAM,cAAc;;;EAGxB,CAAC;AAEJ,eAAO,MAAM,uBAAuB;;;EASjC,CAAC;AAEJ,eAAO,MAAM,wBAAwB;;;EASlC,CAAC"}

package/dist/src/{database.js → database/operations.js}

@@ -1,18 +1,8 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.evalsToDatasetsRelations = exports.evalsToPromptsRelations = exports.evalsRelations = exports.datasetsRelations = exports.evalsToDatasets = exports.datasets = exports.promptsRelations = exports.evalsToPrompts = exports.evals = exports.prompts = void 0;
-exports.getDbPath = getDbPath;
-exports.getDbSignalPath = getDbSignalPath;
-exports.getDb = getDb;
-const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const drizzle_orm_1 = require("drizzle-orm");
-const better_sqlite3_2 = require("drizzle-orm/better-sqlite3");
 const sqlite_core_1 = require("drizzle-orm/sqlite-core");
-const node_path_1 = __importDefault(require("node:path"));
-const util_1 = require("./util");
 // ------------ Prompts ------------
 exports.prompts = (0, sqlite_core_1.sqliteTable)('prompts', {
     id: (0, sqlite_core_1.text)('id').primaryKey(),
@@ -135,18 +125,4 @@ export const llmOutputsRelations = relations(llmOutputs, ({ one }) => ({
   }),
 }));
 */
-let dbInstance = null;
-function getDbPath() {
-    return node_path_1.default.resolve((0, util_1.getConfigDirectoryPath)(true /* createIfNotExists */), 'promptfoo.db');
-}
-function getDbSignalPath() {
-    return node_path_1.default.resolve((0, util_1.getConfigDirectoryPath)(true /* createIfNotExists */), 'evalLastWritten');
-}
-function getDb() {
-    if (!dbInstance) {
-        const sqlite = new better_sqlite3_1.default(getDbPath());
-        dbInstance = (0, better_sqlite3_2.drizzle)(sqlite);
-    }
-    return dbInstance;
-}
-//# sourceMappingURL=database.js.map
+//# sourceMappingURL=operations.js.map

package/dist/src/database/operations.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../../src/database/operations.ts"],"names":[],"mappings":";;;AAAA,6CAA6C;AAC7C,yDAAiF;AAGjF,oCAAoC;AAEvB,QAAA,OAAO,GAAG,IAAA,yBAAW,EAAC,SAAS,EAAE;IAC5C,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,mBAAmB,CAAC;IAClC,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC,CAAC,OAAO,EAAE;CACjC,CAAC,CAAC;AAEU,QAAA,KAAK,GAAG,IAAA,yBAAW,EAAC,OAAO,EAAE;IACxC,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,mBAAmB,CAAC;IAClC,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,CAAC;IACtB,WAAW,EAAE,IAAA,kBAAI,EAAC,aAAa,CAAC;IAChC,OAAO,EAAE,IAAA,kBAAI,EAAC,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAAmB,CAAC,OAAO,EAAE;IAC7E,MAAM,EAAE,IAAA,kBAAI,EAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B,CAAC,OAAO,EAAE;CACnF,CAAC,CAAC;AAEU,QAAA,cAAc,GAAG,IAAA,yBAAW,EACvC,kBAAkB,EAClB;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,yFAAyF;IACzF,uDAAuD;IACvD,QAAQ,EAAE,IAAA,kBAAI,EAAC,WAAW,CAAC;SACxB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;CAChC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;CACpD,CAAC,CACH,CAAC;AAEW,QAAA,gBAAgB,GAAG,IAAA,uBAAS,EAAC,eAAO,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAChE,cAAc,EAAE,IAAI,CAAC,sBAAc,CAAC;CACrC,CAAC,CAAC,CAAC;AAEJ,qCAAqC;AAExB,QAAA,QAAQ,GAAG,IAAA,yBAAW,EAAC,UAAU,EAAE;IAC9C,EAAE,EAAE,IAAA,kBAAI,EAAC,IAAI,CAAC,CAAC,UAAU,EAAE;IAC3B,KAAK,EAAE,IAAA,kBAAI,EAAC,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,EAA0B;IACtE,SAAS,EAAE,IAAA,qBAAO,EAAC,YAAY,CAAC;SAC7B,OAAO,EAAE;SACT,OAAO,CAAC,IAAA,iBAAG,EAAA,mBAAmB,CAAC;CACnC,CAAC,CAAC;AAEU,QAAA,eAAe,GAAG,IAAA,yBAAW,EACxC,mBAAmB,EACnB;IACE,MAAM,EAAE,IAAA,kBAAI,EAAC,SAAS,CAAC;SACpB,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;IAC7B,yFAAyF;IACzF,uDAAuD;IACvD,SAAS,EAAE,IAAA,kBAAI,EAAC,YAAY,CAAC;SAC1B,OAAO,EAAE;SACT,UAAU,CAAC,GAAG,EAAE,CAAC,gBAAQ,CAAC,EAAE,CAAC;CACjC,EACD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACN,EAAE,EAAE,IAAA,wBAAU,EAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;CACrD,CAAC,CACH,CAAC;AAEW,QAAA,iBAAiB,GAAG,IAAA,uBAAS,EAAC,gBAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClE,eAAe,EAAE,IAAI,CAAC,uBAAe,CAAC;CACvC,CAAC,CAAC,CAAC;AAEJ,kCAAkC;AAErB,QAAA,cAAc,GAAG,IAAA,uBAAS,EAAC,aAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC5D,cAAc,EAAE,IAAI,CAAC,sBAAc,CAAC;IACpC,eAAe,EAAE,IAAI,CAAC,uBAAe,CAAC;CACvC,CAAC,CAAC,CAAC;AAES,QAAA,uBAAuB,GAAG,IAAA,uBAAS,EAAC,sBAAc,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC7E,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,sBAAc,CAAC,MAAM,CAAC;QAC/B,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,MAAM,EAAE,GAAG,CAAC,eAAO,EAAE;QACnB,MAAM,EAAE,CAAC,sBAAc,CAAC,QAAQ,CAAC;QACjC,UAAU,EAAE,CAAC,eAAO,CAAC,EAAE,CAAC;KACzB,CAAC;CACH,CAAC,CAAC,CAAC;AAES,QAAA,wBAAwB,GAAG,IAAA,uBAAS,EAAC,uBAAe,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/E,IAAI,EAAE,GAAG,CAAC,aAAK,EAAE;QACf,MAAM,EAAE,CAAC,uBAAe,CAAC,MAAM,CAAC;QAChC,UAAU,EAAE,CAAC,aAAK,CAAC,EAAE,CAAC;KACvB,CAAC;IACF,OAAO,EAAE,GAAG,CAAC,gBAAQ,EAAE;QACrB,MAAM,EAAE,CAAC,uBAAe,CAAC,SAAS,CAAC;QACnC,UAAU,EAAE,CAAC,gBAAQ,CAAC,EAAE,CAAC;KAC1B,CAAC;CACH,CAAC,CAAC,CAAC;AAEJ,oCAAoC;AACpC,wDAAwD;AAExD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAsCE"}

package/dist/src/evaluator.js

@@ -46,7 +46,7 @@ const cliState_1 = __importDefault(require("./cliState"));
 const esm_1 = require("./esm");
 const logger_1 = __importDefault(require("./logger"));
 const azureopenaiUtil_1 = require("./providers/azureopenaiUtil");
-const wrapper_1 = require("./python/wrapper");
+const pythonUtils_1 = require("./python/pythonUtils");
 const suggestions_1 = require("./suggestions");
 const telemetry_1 = __importDefault(require("./telemetry"));
 const util_1 = require("./util");
@@ -156,7 +156,7 @@ async function renderPrompt(prompt, vars, nunjucksFilters, provider) {
                     vars[varName] = javascriptOutput.output;
                     break;
                 case 'py':
-                    const pythonScriptOutput = (await (0, wrapper_1.runPython)(filePath, 'get_var', [
+                    const pythonScriptOutput = (await (0, pythonUtils_1.runPython)(filePath, 'get_var', [
                         varName,
                         basePrompt,
                         vars,