promptfoo 0.46.0 → 0.48.0

package/dist/src/util.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.transformOutput = exports.printBorder = exports.getNunjucksEngine = exports.readFilters = exports.getEvalsWithPredicate = exports.getEvalFromHash = exports.getEvals = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCasesWithPredicate = exports.getTestCases = exports.getPromptsWithPredicate = exports.getPrompts = exports.sha256 = exports.getPromptsForTestCasesHash = exports.getPromptsForTestCases = exports.readLatestResults = exports.updateResult = exports.readResult = exports.dateToFilename = exports.filenameToDate = exports.cleanupOldResults = exports.listPreviousResults = exports.listPreviousResultFilenames = exports.writeLatestResults = exports.getLatestResultsPath = exports.setConfigDirectoryPath = exports.getConfigDirectoryPath = exports.writeOutput = exports.writeMultipleOutputs = exports.readConfigs = exports.readConfig = exports.dereferenceConfig = exports.maybeReadConfig = exports.maybeRecordFirstRun = exports.readGlobalConfig = exports.resetGlobalConfig = void 0;
+exports.transformOutput = exports.printBorder = exports.getNunjucksEngine = exports.readFilters = exports.getEvalsWithPredicate = exports.getEvalFromHash = exports.getEvals = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCasesWithPredicate = exports.getTestCases = exports.getPromptsWithPredicate = exports.getPrompts = exports.sha256 = exports.getPromptsForTestCasesHash = exports.getPromptsForTestCases = exports.readLatestResults = exports.updateResult = exports.readResult_fileSystem = exports.readResult = exports.dateToFilename = exports.filenameToDate = exports.cleanupOldFileResults = exports.migrateResultsFromFileSystemToDatabase = exports.listPreviousResults_fileSystem = exports.listPreviousResultFilenames_fileSystem = exports.listPreviousResults = exports.writeResultsToDatabase = exports.getLatestResultsPath = exports.setConfigDirectoryPath = exports.getConfigDirectoryPath = exports.writeOutput = exports.writeMultipleOutputs = exports.readConfigs = exports.readConfig = exports.dereferenceConfig = exports.maybeReadConfig = exports.maybeRecordFirstRun = exports.readGlobalConfig = exports.resetGlobalConfig = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const os = __importStar(require("os"));
@@ -37,9 +37,12 @@ const nunjucks_1 = __importDefault(require("nunjucks"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
 const sync_1 = require("csv-stringify/sync");
 const glob_1 = require("glob");
+const drizzle_orm_1 = require("drizzle-orm");
 const logger_1 = __importDefault(require("./logger"));
 const esm_1 = require("./esm");
 const testCases_1 = require("./testCases");
+const database_1 = require("./database");
+const migrate_1 = require("./migrate");
 let globalConfigCache = null;
 function resetGlobalConfig() {
     globalConfigCache = null;
@@ -182,6 +185,12 @@ async function readConfig(configPath) {
     }
 }
 exports.readConfig = readConfig;
+/**
+ * Reads multiple configuration files and combines them into a single UnifiedConfig.
+ *
+ * @param {string[]} configPaths - An array of paths to configuration files. Supports glob patterns.
+ * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
+ */
 async function readConfigs(configPaths) {
     const configs = [];
     for (const configPath of configPaths) {
@@ -230,16 +239,18 @@ async function readConfigs(configPaths) {
         }
         return relativePath;
     };
+    const seenPrompts = new Set();
     configs.forEach((config, idx) => {
         if (typeof config.prompts === 'string') {
             (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix string and map-type prompts');
-            config.prompts = makeAbsolute(configPaths[idx], config.prompts);
-            prompts.push(config.prompts);
+            const absolutePrompt = makeAbsolute(configPaths[idx], config.prompts);
+            seenPrompts.add(absolutePrompt);
         }
         else if (Array.isArray(config.prompts)) {
             (0, tiny_invariant_1.default)(Array.isArray(prompts), 'Cannot mix configs with map and array-type prompts');
-            config.prompts = config.prompts.map((prompt) => makeAbsolute(configPaths[idx], prompt));
-            prompts.push(...config.prompts);
+            config.prompts
+                .map((prompt) => makeAbsolute(configPaths[idx], prompt))
+                .forEach((prompt) => seenPrompts.add(prompt));
         }
         else {
             // Object format such as { 'prompts/prompt1.txt': 'foo', 'prompts/prompt2.txt': 'bar' }
@@ -247,6 +258,9 @@ async function readConfigs(configPaths) {
             prompts = { ...prompts, ...config.prompts };
         }
     });
+    if (Array.isArray(prompts)) {
+        prompts.push(...Array.from(seenPrompts));
+    }
     // Combine all configs into a single UnifiedConfig
     const combinedConfig = {
         description: configs.map((config) => config.description).join(', '),
@@ -345,44 +359,114 @@ function setConfigDirectoryPath(newPath) {
     configDirectoryPath = newPath;
 }
 exports.setConfigDirectoryPath = setConfigDirectoryPath;
+/**
+ * TODO(ian): Remove this
+ * @deprecated Use readLatestResults directly instead.
+ */
 function getLatestResultsPath() {
     return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
 }
 exports.getLatestResultsPath = getLatestResultsPath;
-function writeLatestResults(results, config) {
-    const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
-    // Replace hyphens with colons (Windows compatibility).
-    const filename = dateToFilename(new Date());
-    const newResultsPath = path.join(resultsDirectory, filename);
-    const latestResultsPath = getLatestResultsPath();
+async function writeResultsToDatabase(results, config, createdAt) {
+    createdAt = createdAt || new Date();
+    const evalId = `eval-${createdAt.toISOString().slice(0, 19)}`;
+    const db = (0, database_1.getDb)();
+    const promises = [];
+    promises.push(db
+        .insert(database_1.evals)
+        .values({
+        id: evalId,
+        createdAt: createdAt.getTime(),
+        description: config.description,
+        config,
+        results,
+    })
+        .onConflictDoNothing()
+        .run());
+    logger_1.default.debug(`Inserting eval ${evalId}`);
+    // Record prompt relation
+    for (const prompt of results.table.head.prompts) {
+        const promptId = sha256(prompt.display);
+        promises.push(db
+            .insert(database_1.prompts)
+            .values({
+            id: promptId,
+            prompt: prompt.display,
+        })
+            .onConflictDoNothing()
+            .run());
+        promises.push(db
+            .insert(database_1.evalsToPrompts)
+            .values({
+            evalId,
+            promptId,
+        })
+            .onConflictDoNothing()
+            .run());
+        logger_1.default.debug(`Inserting prompt ${promptId}`);
+    }
+    // Record dataset relation
+    const datasetId = sha256(JSON.stringify(config.tests || []));
+    promises.push(db
+        .insert(database_1.datasets)
+        .values({
+        id: datasetId,
+        tests: config.tests,
+    })
+        .onConflictDoNothing()
+        .run());
+    promises.push(db
+        .insert(database_1.evalsToDatasets)
+        .values({
+        evalId,
+        datasetId,
+    })
+        .onConflictDoNothing()
+        .run());
+    logger_1.default.debug(`Inserting dataset ${datasetId}`);
+    logger_1.default.debug(`Awaiting ${promises.length} promises to database...`);
+    await Promise.all(promises);
+    // "touch" db signal path
+    const filePath = (0, database_1.getDbSignalPath)();
     try {
-        fs.mkdirSync(resultsDirectory, { recursive: true });
-        const resultsFileData = {
-            version: 2,
-            createdAt: new Date().toISOString(),
-            config,
-            results,
-        };
-        fs.writeFileSync(newResultsPath, JSON.stringify(resultsFileData, null, 2));
-        // Use copy instead of symlink to avoid issues with Windows permissions.
-        try {
-            // Backwards compatibility: delete old symlink.
-            fs.unlinkSync(latestResultsPath);
-        }
-        catch { }
-        fs.copyFileSync(newResultsPath, latestResultsPath);
-        cleanupOldResults();
-        return filename;
+        const now = new Date();
+        fs.utimesSync(filePath, now, now);
     }
     catch (err) {
-        logger_1.default.error(`Failed to write latest results to ${newResultsPath}:\n${err}`);
-        return null;
+        fs.closeSync(fs.openSync(filePath, 'w'));
     }
+    return evalId;
 }
-exports.writeLatestResults = writeLatestResults;
-const resultsCache = {};
-function listPreviousResultFilenames() {
+exports.writeResultsToDatabase = writeResultsToDatabase;
+/**
+ *
+ * @returns Last 100 evals in descending order.
+ */
+function listPreviousResults() {
+    const db = (0, database_1.getDb)();
+    const results = db
+        .select({
+        name: database_1.evals.id,
+        description: database_1.evals.description,
+    })
+        .from(database_1.evals)
+        .orderBy((0, drizzle_orm_1.desc)(database_1.evals.createdAt))
+        .limit(100)
+        .all();
+    return results.map((result) => ({
+        evalId: result.name,
+        description: result.description,
+    }));
+}
+exports.listPreviousResults = listPreviousResults;
+/**
+ * @deprecated Used only for migration to sqlite
+ */
+function listPreviousResultFilenames_fileSystem() {
     const directory = path.join(getConfigDirectoryPath(), 'output');
+    if (!fs.existsSync(directory)) {
+        return [];
+    }
     const files = fs.readdirSync(directory);
     const resultsFiles = files.filter((file) => file.startsWith('eval-') && file.endsWith('.json'));
     return resultsFiles.sort((a, b) => {
@@ -391,10 +475,17 @@ function listPreviousResultFilenames() {
         return statA.birthtime.getTime() - statB.birthtime.getTime(); // sort in ascending order
     });
 }
-exports.listPreviousResultFilenames = listPreviousResultFilenames;
-function listPreviousResults() {
+exports.listPreviousResultFilenames_fileSystem = listPreviousResultFilenames_fileSystem;
+const resultsCache = {};
+/**
+ * @deprecated Used only for migration to sqlite
+ */
+function listPreviousResults_fileSystem() {
     const directory = path.join(getConfigDirectoryPath(), 'output');
-    const sortedFiles = listPreviousResultFilenames();
+    if (!fs.existsSync(directory)) {
+        return [];
+    }
+    const sortedFiles = listPreviousResultFilenames_fileSystem();
     return sortedFiles.map((fileName) => {
         if (!resultsCache[fileName]) {
             try {
@@ -412,15 +503,71 @@ function listPreviousResults() {
         };
     });
 }
-exports.listPreviousResults = listPreviousResults;
+exports.listPreviousResults_fileSystem = listPreviousResults_fileSystem;
+let attemptedMigration = false;
+async function migrateResultsFromFileSystemToDatabase() {
+    if (attemptedMigration) {
+        // TODO(ian): Record this bit in the database.
+        return;
+    }
+    // First run db migrations
+    logger_1.default.debug('Running db migrations...');
+    await (0, migrate_1.runDbMigrations)();
+    const fileNames = listPreviousResultFilenames_fileSystem();
+    if (fileNames.length === 0) {
+        return;
+    }
+    logger_1.default.info(`🔁 Migrating ${fileNames.length} flat files to local database.`);
+    logger_1.default.info('This is a one-time operation and may take a minute...');
+    attemptedMigration = true;
+    const outputDir = path.join(getConfigDirectoryPath(), 'output');
+    const backupDir = `${outputDir}-backup-${new Date()
+        .toISOString()
+        .slice(0, 10)
+        .replace(/-/g, '')}`;
+    try {
+        fs.cpSync(outputDir, backupDir, { recursive: true });
+        logger_1.default.info(`Backup of output directory created at ${backupDir}`);
+    }
+    catch (backupError) {
+        logger_1.default.error(`Failed to create backup of output directory: ${backupError}`);
+        return;
+    }
+    logger_1.default.info('Moving files into database...');
+    const migrationPromises = fileNames.map(async (fileName) => {
+        const fileData = readResult_fileSystem(fileName);
+        if (fileData) {
+            await writeResultsToDatabase(fileData.result.results, fileData.result.config, filenameToDate(fileName));
+            logger_1.default.debug(`Migrated ${fileName} to database.`);
+            try {
+                fs.unlinkSync(path.join(outputDir, fileName));
+            }
+            catch (err) {
+                logger_1.default.warn(`Failed to delete ${fileName} after migration: ${err}`);
+            }
+        }
+        else {
+            logger_1.default.warn(`Failed to migrate result ${fileName} due to read error.`);
+        }
+    });
+    await Promise.all(migrationPromises);
+    try {
+        fs.unlinkSync(getLatestResultsPath());
+    }
+    catch (err) {
+        logger_1.default.warn(`Failed to delete latest.json: ${err}`);
+    }
+    logger_1.default.info('Migration complete. Please restart your web server if it is running.');
+}
+exports.migrateResultsFromFileSystemToDatabase = migrateResultsFromFileSystemToDatabase;
 const RESULT_HISTORY_LENGTH = parseInt(process.env.RESULT_HISTORY_LENGTH || '', 10) || 100;
-function cleanupOldResults(remaining = RESULT_HISTORY_LENGTH) {
-    const sortedFilenames = listPreviousResultFilenames();
+function cleanupOldFileResults(remaining = RESULT_HISTORY_LENGTH) {
+    const sortedFilenames = listPreviousResultFilenames_fileSystem();
     for (let i = 0; i < sortedFilenames.length - remaining; i++) {
         fs.unlinkSync(path.join(getConfigDirectoryPath(), 'output', sortedFilenames[i]));
     }
 }
-exports.cleanupOldResults = cleanupOldResults;
+exports.cleanupOldFileResults = cleanupOldFileResults;
 function filenameToDate(filename) {
     const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
     // Replace hyphens with colons where necessary (Windows compatibility).
@@ -428,27 +575,67 @@ function filenameToDate(filename) {
     const timePart = dateParts[1].replace(/-/g, ':');
     const formattedDateString = `${dateParts[0]}T${timePart}`;
     const date = new Date(formattedDateString);
+    return date;
+    /*
     return date.toLocaleDateString('en-US', {
-        year: 'numeric',
-        month: 'long',
-        day: 'numeric',
-        hour: '2-digit',
-        minute: '2-digit',
-        second: '2-digit',
-        timeZoneName: 'short',
+      year: 'numeric',
+      month: 'long',
+      day: 'numeric',
+      hour: '2-digit',
+      minute: '2-digit',
+      second: '2-digit',
+      timeZoneName: 'short',
     });
+    */
 }
 exports.filenameToDate = filenameToDate;
 function dateToFilename(date) {
     return `eval-${date.toISOString().replace(/:/g, '-')}.json`;
 }
 exports.dateToFilename = dateToFilename;
-function readResult(name) {
+async function readResult(id) {
+    const db = (0, database_1.getDb)();
+    try {
+        const evalResult = await db
+            .select({
+            id: database_1.evals.id,
+            createdAt: database_1.evals.createdAt,
+            results: database_1.evals.results,
+            config: database_1.evals.config,
+        })
+            .from(database_1.evals)
+            .where((0, drizzle_orm_1.eq)(database_1.evals.id, id))
+            .execute();
+        if (evalResult.length === 0) {
+            return undefined;
+        }
+        const { id: resultId, createdAt, results, config } = evalResult[0];
+        const result = {
+            version: 3,
+            createdAt: new Date(createdAt).toISOString().slice(0, 10),
+            results,
+            config,
+        };
+        return {
+            id: resultId,
+            result,
+            createdAt: new Date(createdAt),
+        };
+    }
+    catch (err) {
+        logger_1.default.error(`Failed to read result with ID ${id} from database:\n${err}`);
+    }
+}
+exports.readResult = readResult;
+/**
+ * @deprecated Used only for migration to sqlite
+ */
+function readResult_fileSystem(name) {
     const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
     const resultsPath = path.join(resultsDirectory, name);
     try {
         const result = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
-        const createdAt = new Date(filenameToDate(name));
+        const createdAt = filenameToDate(name);
         return {
             id: sha256(JSON.stringify(result.config)),
             result,
@@ -459,35 +646,70 @@ function readResult(name) {
         logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
     }
 }
-exports.readResult = readResult;
-function updateResult(filename, newConfig, newTable) {
-    const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
-    const safeFilename = path.basename(filename);
-    const resultsPath = path.join(resultsDirectory, safeFilename);
+exports.readResult_fileSystem = readResult_fileSystem;
+async function updateResult(id, newConfig, newTable) {
+    const db = (0, database_1.getDb)();
     try {
-        const evalData = JSON.parse(fs.readFileSync(resultsPath, 'utf-8'));
+        // Fetch the existing eval data from the database
+        const existingEval = await db
+            .select({
+            config: database_1.evals.config,
+            results: database_1.evals.results,
+        })
+            .from(database_1.evals)
+            .where((0, drizzle_orm_1.eq)(database_1.evals.id, id))
+            .limit(1)
+            .all();
+        if (existingEval.length === 0) {
+            logger_1.default.error(`Eval with ID ${id} not found.`);
+            return;
+        }
+        const evalData = existingEval[0];
         if (newConfig) {
             evalData.config = newConfig;
         }
         if (newTable) {
             evalData.results.table = newTable;
         }
-        resultsCache[safeFilename] = evalData;
-        fs.writeFileSync(resultsPath, JSON.stringify(evalData, null, 2));
-        logger_1.default.info(`Updated eval at ${resultsPath}`);
-        const resultFilenames = listPreviousResultFilenames();
-        if (filename === resultFilenames[resultFilenames.length - 1]) {
-            // Overwite latest.json too
-            fs.copyFileSync(resultsPath, getLatestResultsPath());
-        }
+        await db
+            .update(database_1.evals)
+            .set({
+            description: evalData.config.description,
+            config: evalData.config,
+            results: evalData.results,
+        })
+            .where((0, drizzle_orm_1.eq)(database_1.evals.id, id))
+            .run();
+        logger_1.default.info(`Updated eval with ID ${id}`);
     }
     catch (err) {
-        logger_1.default.error(`Failed to update eval at ${resultsPath}:\n${err}`);
+        logger_1.default.error(`Failed to update eval with ID ${id}:\n${err}`);
     }
 }
 exports.updateResult = updateResult;
-function readLatestResults() {
-    return JSON.parse(fs.readFileSync(getLatestResultsPath(), 'utf-8'));
+async function readLatestResults() {
+    const db = (0, database_1.getDb)();
+    const latestResults = await db
+        .select({
+        id: database_1.evals.id,
+        createdAt: database_1.evals.createdAt,
+        description: database_1.evals.description,
+        results: database_1.evals.results,
+        config: database_1.evals.config,
+    })
+        .from(database_1.evals)
+        .orderBy((0, drizzle_orm_1.desc)(database_1.evals.createdAt))
+        .limit(1);
+    if (!latestResults || latestResults.length === 0) {
+        return undefined;
+    }
+    const latestResult = latestResults[0];
+    return {
+        version: 3,
+        createdAt: new Date(latestResult.createdAt).toISOString(),
+        results: latestResult.results,
+        config: latestResult.config,
+    };
 }
 exports.readLatestResults = readLatestResults;
 function getPromptsForTestCases(testCases) {
@@ -512,26 +734,39 @@ function getPrompts() {
     return getPromptsWithPredicate(() => true);
 }
 exports.getPrompts = getPrompts;
-function getPromptsWithPredicate(predicate) {
-    const resultFilenames = listPreviousResultFilenames();
+async function getPromptsWithPredicate(predicate) {
+    // TODO(ian): Make this use a proper database query
+    const db = (0, database_1.getDb)();
+    const evals_ = await db
+        .select({
+        id: database_1.evals.id,
+        createdAt: database_1.evals.createdAt,
+        results: database_1.evals.results,
+        config: database_1.evals.config,
+    })
+        .from(database_1.evals)
+        .limit(100)
+        .all();
     const groupedPrompts = {};
-    for (const fileName of resultFilenames) {
-        const file = readResult(fileName);
-        if (!file) {
-            continue;
-        }
-        const { result, createdAt } = file;
-        if (result && predicate(result)) {
-            for (const prompt of result.results.table.head.prompts) {
-                const evalId = sha256(JSON.stringify(result.config));
+    for (const eval_ of evals_) {
+        const createdAt = new Date(eval_.createdAt).toISOString();
+        const resultWrapper = {
+            version: 3,
+            createdAt,
+            results: eval_.results,
+            config: eval_.config,
+        };
+        if (predicate(resultWrapper)) {
+            for (const prompt of resultWrapper.results.table.head.prompts) {
                 const promptId = sha256(prompt.raw);
-                const datasetId = result.config.tests ? sha256(JSON.stringify(result.config.tests)) : '-';
+                const datasetId = resultWrapper.config.tests
+                    ? sha256(JSON.stringify(resultWrapper.config.tests))
+                    : '-';
                 if (promptId in groupedPrompts) {
                     groupedPrompts[promptId].recentEvalDate = new Date(Math.max(groupedPrompts[promptId].recentEvalDate.getTime(), new Date(createdAt).getTime()));
                     groupedPrompts[promptId].count += 1;
                     groupedPrompts[promptId].evals.push({
-                        id: evalId,
-                        filePath: fileName,
+                        id: eval_.id,
                         datasetId,
                         metrics: prompt.metrics,
                     });
@@ -542,12 +777,10 @@ function getPromptsWithPredicate(predicate) {
                         id: promptId,
                         prompt,
                         recentEvalDate: new Date(createdAt),
-                        recentEvalId: evalId,
-                        recentEvalFilepath: fileName,
+                        recentEvalId: eval_.id,
                         evals: [
                             {
-                                id: evalId,
-                                filePath: fileName,
+                                id: eval_.id,
                                 datasetId,
                                 metrics: prompt.metrics,
                             },
@@ -560,31 +793,42 @@ function getPromptsWithPredicate(predicate) {
     return Object.values(groupedPrompts);
 }
 exports.getPromptsWithPredicate = getPromptsWithPredicate;
-function getTestCases() {
+async function getTestCases() {
     return getTestCasesWithPredicate(() => true);
 }
 exports.getTestCases = getTestCases;
-function getTestCasesWithPredicate(predicate) {
-    const resultFilenames = listPreviousResultFilenames();
+async function getTestCasesWithPredicate(predicate) {
+    const db = (0, database_1.getDb)();
+    const evals_ = await db
+        .select({
+        id: database_1.evals.id,
+        createdAt: database_1.evals.createdAt,
+        results: database_1.evals.results,
+        config: database_1.evals.config,
+    })
+        .from(database_1.evals)
+        .limit(100)
+        .all();
     const groupedTestCases = {};
-    for (const fileName of resultFilenames) {
-        const file = readResult(fileName);
-        if (!file) {
-            continue;
-        }
-        const { result, createdAt } = file;
-        const testCases = result?.config?.tests;
-        if (testCases && predicate(result)) {
-            const evalId = sha256(JSON.stringify(result.config));
+    for (const eval_ of evals_) {
+        const createdAt = new Date(eval_.createdAt).toISOString();
+        const resultWrapper = {
+            version: 3,
+            createdAt,
+            results: eval_.results,
+            config: eval_.config,
+        };
+        const testCases = resultWrapper.config.tests;
+        if (testCases && predicate(resultWrapper)) {
+            const evalId = eval_.id;
             const datasetId = sha256(JSON.stringify(testCases));
             if (datasetId in groupedTestCases) {
-                groupedTestCases[datasetId].recentEvalDate = new Date(Math.max(groupedTestCases[datasetId].recentEvalDate.getTime(), new Date(createdAt).getTime()));
+                groupedTestCases[datasetId].recentEvalDate = new Date(Math.max(groupedTestCases[datasetId].recentEvalDate.getTime(), eval_.createdAt));
                 groupedTestCases[datasetId].count += 1;
-                const newPrompts = result.results.table.head.prompts.map((prompt) => ({
+                const newPrompts = resultWrapper.results.table.head.prompts.map((prompt) => ({
                     id: sha256(prompt.raw),
                     prompt,
                     evalId,
-                    evalFilepath: fileName,
                 }));
                 const promptsById = {};
                 for (const prompt of groupedTestCases[datasetId].prompts.concat(newPrompts)) {
@@ -595,11 +839,10 @@ function getTestCasesWithPredicate(predicate) {
                 groupedTestCases[datasetId].prompts = Object.values(promptsById);
             }
             else {
-                const newPrompts = result.results.table.head.prompts.map((prompt) => ({
-                    id: (0, crypto_1.createHash)('sha256').update(prompt.raw).digest('hex'),
+                const newPrompts = resultWrapper.results.table.head.prompts.map((prompt) => ({
+                    id: sha256(prompt.raw),
                     prompt,
                     evalId,
-                    evalFilepath: fileName,
                 }));
                 const promptsById = {};
                 for (const prompt of newPrompts) {
@@ -613,7 +856,6 @@ function getTestCasesWithPredicate(predicate) {
                     testCases,
                     recentEvalDate: new Date(createdAt),
                     recentEvalId: evalId,
-                    recentEvalFilepath: fileName,
                     prompts: Object.values(promptsById),
                 };
             }
@@ -622,8 +864,8 @@ function getTestCasesWithPredicate(predicate) {
     return Object.values(groupedTestCases);
 }
 exports.getTestCasesWithPredicate = getTestCasesWithPredicate;
-function getPromptFromHash(hash) {
-    const prompts = getPrompts();
+async function getPromptFromHash(hash) {
+    const prompts = await getPrompts();
     for (const prompt of prompts) {
         if (prompt.id.startsWith(hash)) {
             return prompt;
@@ -632,8 +874,8 @@ function getPromptFromHash(hash) {
     return undefined;
 }
 exports.getPromptFromHash = getPromptFromHash;
-function getDatasetFromHash(hash) {
-    const datasets = getTestCases();
+async function getDatasetFromHash(hash) {
+    const datasets = await getTestCases();
     for (const dataset of datasets) {
         if (dataset.id.startsWith(hash)) {
             return dataset;
@@ -642,13 +884,13 @@ function getDatasetFromHash(hash) {
     return undefined;
 }
 exports.getDatasetFromHash = getDatasetFromHash;
-function getEvals() {
+async function getEvals() {
     return getEvalsWithPredicate(() => true);
 }
 exports.getEvals = getEvals;
-function getEvalFromHash(hash) {
-    const evals = getEvals();
-    for (const eval_ of evals) {
+async function getEvalFromHash(hash) {
+    const evals_ = await getEvals();
+    for (const eval_ of evals_) {
         if (eval_.id.startsWith(hash)) {
             return eval_;
         }
@@ -656,23 +898,34 @@ function getEvalFromHash(hash) {
     return undefined;
 }
 exports.getEvalFromHash = getEvalFromHash;
-function getEvalsWithPredicate(predicate) {
+async function getEvalsWithPredicate(predicate) {
+    const db = (0, database_1.getDb)();
+    const evals_ = await db
+        .select({
+        id: database_1.evals.id,
+        createdAt: database_1.evals.createdAt,
+        results: database_1.evals.results,
+        config: database_1.evals.config,
+    })
+        .from(database_1.evals)
+        .limit(100)
+        .all();
     const ret = [];
-    const resultsFilenames = listPreviousResultFilenames();
-    for (const fileName of resultsFilenames) {
-        const file = readResult(fileName);
-        if (!file) {
-            continue;
-        }
-        const { result, createdAt } = file;
-        if (result && predicate(result)) {
-            const evalId = sha256(fileName + ':' + JSON.stringify(result.config));
+    for (const eval_ of evals_) {
+        const createdAt = new Date(eval_.createdAt).toISOString();
+        const resultWrapper = {
+            version: 3,
+            createdAt: createdAt,
+            results: eval_.results,
+            config: eval_.config,
+        };
+        if (predicate(resultWrapper)) {
+            const evalId = eval_.id;
             ret.push({
                 id: evalId,
-                filePath: fileName,
-                date: createdAt,
-                config: result.config,
-                results: result.results,
+                date: new Date(eval_.createdAt),
+                config: eval_.config,
+                results: eval_.results,
             });
         }
     }