promptfoo 0.68.3 → 0.69.1

package/dist/src/util.js

@@ -26,7 +26,54 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.parsePathOrGlob = exports.extractJsonObjects = exports.renderVarsInObject = exports.safeJsonStringify = exports.resultIsForTestCase = exports.varsMatch = exports.providerToIdentifier = exports.getStandaloneEvals = exports.setupEnv = exports.transformOutput = exports.printBorder = exports.readFilters = exports.deleteEval = exports.getEvalFromId = exports.getEvals = exports.getEvalsWithPredicate = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCases = exports.getPrompts = exports.getTestCasesWithPredicate = exports.getPromptsForTestCases = exports.getPromptsForTestCasesHash = exports.getPromptsWithPredicate = exports.readLatestResults = exports.updateResult = exports.readResult = exports.cleanupOldFileResults = exports.migrateResultsFromFileSystemToDatabase = exports.readResult_fileSystem = exports.dateToFilename = exports.filenameToDate = exports.listPreviousResults_fileSystem = exports.listPreviousResultFilenames_fileSystem = exports.listPreviousResults = exports.writeResultsToDatabase = exports.getLatestResultsPath = exports.setConfigDirectoryPath = exports.getConfigDirectoryPath = exports.readOutput = exports.sha256 = exports.writeMultipleOutputs = exports.writeOutput = exports.getNunjucksEngine = exports.readConfigs = exports.maybeReadConfig = exports.readConfig = exports.dereferenceConfig = void 0;
+exports.dereferenceConfig = dereferenceConfig;
+exports.readConfig = readConfig;
+exports.maybeReadConfig = maybeReadConfig;
+exports.readConfigs = readConfigs;
+exports.getNunjucksEngine = getNunjucksEngine;
+exports.writeOutput = writeOutput;
+exports.writeMultipleOutputs = writeMultipleOutputs;
+exports.sha256 = sha256;
+exports.readOutput = readOutput;
+exports.getConfigDirectoryPath = getConfigDirectoryPath;
+exports.setConfigDirectoryPath = setConfigDirectoryPath;
+exports.getLatestResultsPath = getLatestResultsPath;
+exports.writeResultsToDatabase = writeResultsToDatabase;
+exports.listPreviousResults = listPreviousResults;
+exports.listPreviousResultFilenames_fileSystem = listPreviousResultFilenames_fileSystem;
+exports.listPreviousResults_fileSystem = listPreviousResults_fileSystem;
+exports.filenameToDate = filenameToDate;
+exports.dateToFilename = dateToFilename;
+exports.readResult_fileSystem = readResult_fileSystem;
+exports.migrateResultsFromFileSystemToDatabase = migrateResultsFromFileSystemToDatabase;
+exports.cleanupOldFileResults = cleanupOldFileResults;
+exports.readResult = readResult;
+exports.updateResult = updateResult;
+exports.readLatestResults = readLatestResults;
+exports.getPromptsWithPredicate = getPromptsWithPredicate;
+exports.getPromptsForTestCasesHash = getPromptsForTestCasesHash;
+exports.getPromptsForTestCases = getPromptsForTestCases;
+exports.getTestCasesWithPredicate = getTestCasesWithPredicate;
+exports.getPrompts = getPrompts;
+exports.getTestCases = getTestCases;
+exports.getPromptFromHash = getPromptFromHash;
+exports.getDatasetFromHash = getDatasetFromHash;
+exports.getEvalsWithPredicate = getEvalsWithPredicate;
+exports.getEvals = getEvals;
+exports.getEvalFromId = getEvalFromId;
+exports.deleteEval = deleteEval;
+exports.readFilters = readFilters;
+exports.printBorder = printBorder;
+exports.transformOutput = transformOutput;
+exports.setupEnv = setupEnv;
+exports.getStandaloneEvals = getStandaloneEvals;
+exports.providerToIdentifier = providerToIdentifier;
+exports.varsMatch = varsMatch;
+exports.resultIsForTestCase = resultIsForTestCase;
+exports.safeJsonStringify = safeJsonStringify;
+exports.renderVarsInObject = renderVarsInObject;
+exports.extractJsonObjects = extractJsonObjects;
+exports.parsePathOrGlob = parsePathOrGlob;
 const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
 const crypto_1 = require("crypto");
 const sync_1 = require("csv-stringify/sync");
@@ -40,6 +87,7 @@ const nunjucks_1 = __importDefault(require("nunjucks"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const accounts_1 = require("./accounts");
 const cliState_1 = __importDefault(require("./cliState"));
 const constants_1 = require("./constants");
 const database_1 = require("./database");
@@ -135,7 +183,6 @@ async function dereferenceConfig(rawConfig) {
     }
     return config;
 }
-exports.dereferenceConfig = dereferenceConfig;
 async function readConfig(configPath) {
     const ext = path.parse(configPath).ext;
     switch (ext) {
@@ -152,14 +199,12 @@ async function readConfig(configPath) {
             throw new Error(`Unsupported configuration file format: ${ext}`);
     }
 }
-exports.readConfig = readConfig;
 async function maybeReadConfig(configPath) {
     if (!fs.existsSync(configPath)) {
         return undefined;
     }
     return readConfig(configPath);
 }
-exports.maybeReadConfig = maybeReadConfig;
 /**
  * Reads multiple configuration files and combines them into a single UnifiedConfig.
  *
@@ -290,7 +335,6 @@ async function readConfigs(configPaths) {
     };
     return combinedConfig;
 }
-exports.readConfigs = readConfigs;
 function getNunjucksEngine(filters) {
     if (process.env.PROMPTFOO_DISABLE_TEMPLATING) {
         return {
@@ -307,17 +351,16 @@ function getNunjucksEngine(filters) {
     }
     return env;
 }
-exports.getNunjucksEngine = getNunjucksEngine;
 async function writeOutput(outputPath, evalId, results, config, shareableUrl) {
     const outputExtension = outputPath.split('.').pop()?.toLowerCase();
     const outputToSimpleString = (output) => {
         const passFailText = output.pass ? '[PASS]' : '[FAIL]';
         const namedScoresText = Object.entries(output.namedScores)
-            .map(([name, value]) => `${name}: ${value.toFixed(2)}`)
+            .map(([name, value]) => `${name}: ${value?.toFixed(2)}`)
             .join(', ');
         const scoreText = namedScoresText.length > 0
-            ? `(${output.score.toFixed(2)}, ${namedScoresText})`
-            : `(${output.score.toFixed(2)})`;
+            ? `(${output.score?.toFixed(2)}, ${namedScoresText})`
+            : `(${output.score?.toFixed(2)})`;
         const gradingResultText = output.gradingResult
             ? `${output.pass ? 'Pass' : 'Fail'} Reason: ${output.gradingResult.reason}`
             : '';
@@ -385,15 +428,12 @@ ${gradingResultText}`.trim();
         }
     }
 }
-exports.writeOutput = writeOutput;
 async function writeMultipleOutputs(outputPaths, evalId, results, config, shareableUrl) {
     await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalId, results, config, shareableUrl)));
 }
-exports.writeMultipleOutputs = writeMultipleOutputs;
 function sha256(str) {
     return (0, crypto_1.createHash)('sha256').update(str).digest('hex');
 }
-exports.sha256 = sha256;
 async function readOutput(outputPath) {
     const ext = path.parse(outputPath).ext.slice(1);
     switch (ext) {
@@ -403,7 +443,6 @@ async function readOutput(outputPath) {
             throw new Error(`Unsupported output file format: ${ext} currently only supports json`);
     }
 }
-exports.readOutput = readOutput;
 let configDirectoryPath = process.env.PROMPTFOO_CONFIG_DIR;
 function getConfigDirectoryPath(createIfNotExists = false) {
     const p = configDirectoryPath || path.join(os.homedir(), '.promptfoo');
@@ -412,11 +451,9 @@ function getConfigDirectoryPath(createIfNotExists = false) {
     }
     return p;
 }
-exports.getConfigDirectoryPath = getConfigDirectoryPath;
 function setConfigDirectoryPath(newPath) {
     configDirectoryPath = newPath;
 }
-exports.setConfigDirectoryPath = setConfigDirectoryPath;
 /**
  * TODO(ian): Remove this
  * @deprecated Use readLatestResults directly instead.
@@ -424,7 +461,6 @@ exports.setConfigDirectoryPath = setConfigDirectoryPath;
 function getLatestResultsPath() {
     return path.join(getConfigDirectoryPath(), 'output', 'latest.json');
 }
-exports.getLatestResultsPath = getLatestResultsPath;
 async function writeResultsToDatabase(results, config, createdAt) {
     createdAt = createdAt || (results.timestamp ? new Date(results.timestamp) : new Date());
     const evalId = `eval-${createdAt.toISOString().slice(0, 19)}`;
@@ -435,6 +471,7 @@ async function writeResultsToDatabase(results, config, createdAt) {
         .values({
         id: evalId,
         createdAt: createdAt.getTime(),
+        author: (0, accounts_1.getAuthor)(),
         description: config.description,
         config,
         results,
@@ -496,7 +533,6 @@ async function writeResultsToDatabase(results, config, createdAt) {
     }
     return evalId;
 }
-exports.writeResultsToDatabase = writeResultsToDatabase;
 /**
  *
  * @returns Last n evals in descending order.
@@ -521,7 +557,6 @@ function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription) {
         description: result.description,
     }));
 }
-exports.listPreviousResults = listPreviousResults;
 /**
  * @deprecated Used only for migration to sqlite
  */
@@ -538,7 +573,6 @@ function listPreviousResultFilenames_fileSystem() {
         return statA.birthtime.getTime() - statB.birthtime.getTime(); // sort in ascending order
     });
 }
-exports.listPreviousResultFilenames_fileSystem = listPreviousResultFilenames_fileSystem;
 const resultsCache = {};
 /**
  * @deprecated Used only for migration to sqlite
@@ -566,7 +600,6 @@ function listPreviousResults_fileSystem() {
         };
     });
 }
-exports.listPreviousResults_fileSystem = listPreviousResults_fileSystem;
 function filenameToDate(filename) {
     const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
     // Replace hyphens with colons where necessary (Windows compatibility).
@@ -587,11 +620,9 @@ function filenameToDate(filename) {
     });
     */
 }
-exports.filenameToDate = filenameToDate;
 function dateToFilename(date) {
     return `eval-${date.toISOString().replace(/:/g, '-')}.json`;
 }
-exports.dateToFilename = dateToFilename;
 /**
  * @deprecated Used only for migration to sqlite
  */
@@ -611,7 +642,6 @@ function readResult_fileSystem(name) {
         logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
     }
 }
-exports.readResult_fileSystem = readResult_fileSystem;
 let attemptedMigration = false;
 async function migrateResultsFromFileSystemToDatabase() {
     if (attemptedMigration) {
@@ -667,7 +697,6 @@ async function migrateResultsFromFileSystemToDatabase() {
     }
     logger_1.default.info('Migration complete. Please restart your web server if it is running.');
 }
-exports.migrateResultsFromFileSystemToDatabase = migrateResultsFromFileSystemToDatabase;
 const RESULT_HISTORY_LENGTH = parseInt(process.env.RESULT_HISTORY_LENGTH || '', 10) || DEFAULT_QUERY_LIMIT;
 function cleanupOldFileResults(remaining = RESULT_HISTORY_LENGTH) {
     const sortedFilenames = listPreviousResultFilenames_fileSystem();
@@ -675,7 +704,6 @@ function cleanupOldFileResults(remaining = RESULT_HISTORY_LENGTH) {
         fs.unlinkSync(path.join(getConfigDirectoryPath(), 'output', sortedFilenames[i]));
     }
 }
-exports.cleanupOldFileResults = cleanupOldFileResults;
 async function readResult(id) {
     const db = (0, database_1.getDb)();
     try {
@@ -683,6 +711,7 @@ async function readResult(id) {
             .select({
             id: database_1.evals.id,
             createdAt: database_1.evals.createdAt,
+            author: database_1.evals.author,
             results: database_1.evals.results,
             config: database_1.evals.config,
         })
@@ -692,10 +721,11 @@ async function readResult(id) {
         if (evalResult.length === 0) {
             return undefined;
         }
-        const { id: resultId, createdAt, results, config } = evalResult[0];
+        const { id: resultId, createdAt, results, config, author } = evalResult[0];
         const result = {
             version: 3,
             createdAt: new Date(createdAt).toISOString().slice(0, 10),
+            author,
             results,
             config,
         };
@@ -709,7 +739,6 @@ async function readResult(id) {
         logger_1.default.error(`Failed to read result with ID ${id} from database:\n${err}`);
     }
 }
-exports.readResult = readResult;
 async function updateResult(id, newConfig, newTable) {
     const db = (0, database_1.getDb)();
     try {
@@ -749,13 +778,13 @@ async function updateResult(id, newConfig, newTable) {
         logger_1.default.error(`Failed to update eval with ID ${id}:\n${err}`);
     }
 }
-exports.updateResult = updateResult;
 async function readLatestResults(filterDescription) {
     const db = (0, database_1.getDb)();
     let latestResults = await db
         .select({
         id: database_1.evals.id,
         createdAt: database_1.evals.createdAt,
+        author: database_1.evals.author,
         description: database_1.evals.description,
         results: database_1.evals.results,
         config: database_1.evals.config,
@@ -774,11 +803,11 @@ async function readLatestResults(filterDescription) {
     return {
         version: 3,
         createdAt: new Date(latestResult.createdAt).toISOString(),
+        author: latestResult.author,
         results: latestResult.results,
         config: latestResult.config,
     };
 }
-exports.readLatestResults = readLatestResults;
 async function getPromptsWithPredicate(predicate, limit) {
     // TODO(ian): Make this use a proper database query
     const db = (0, database_1.getDb)();
@@ -786,6 +815,7 @@ async function getPromptsWithPredicate(predicate, limit) {
         .select({
         id: database_1.evals.id,
         createdAt: database_1.evals.createdAt,
+        author: database_1.evals.author,
         results: database_1.evals.results,
         config: database_1.evals.config,
     })
@@ -798,6 +828,7 @@ async function getPromptsWithPredicate(predicate, limit) {
         const resultWrapper = {
             version: 3,
             createdAt,
+            author: eval_.author,
             results: eval_.results,
             config: eval_.config,
         };
@@ -837,7 +868,6 @@ async function getPromptsWithPredicate(predicate, limit) {
     }
     return Object.values(groupedPrompts);
 }
-exports.getPromptsWithPredicate = getPromptsWithPredicate;
 function getPromptsForTestCasesHash(testCasesSha256, limit = DEFAULT_QUERY_LIMIT) {
     return getPromptsWithPredicate((result) => {
         const testsJson = JSON.stringify(result.config.tests);
@@ -845,19 +875,18 @@ function getPromptsForTestCasesHash(testCasesSha256, limit = DEFAULT_QUERY_LIMIT
         return hash === testCasesSha256;
     }, limit);
 }
-exports.getPromptsForTestCasesHash = getPromptsForTestCasesHash;
 function getPromptsForTestCases(testCases) {
     const testCasesJson = JSON.stringify(testCases);
     const testCasesSha256 = sha256(testCasesJson);
     return getPromptsForTestCasesHash(testCasesSha256);
 }
-exports.getPromptsForTestCases = getPromptsForTestCases;
 async function getTestCasesWithPredicate(predicate, limit) {
     const db = (0, database_1.getDb)();
     const evals_ = await db
         .select({
         id: database_1.evals.id,
         createdAt: database_1.evals.createdAt,
+        author: database_1.evals.author,
         results: database_1.evals.results,
         config: database_1.evals.config,
     })
@@ -870,6 +899,7 @@ async function getTestCasesWithPredicate(predicate, limit) {
         const resultWrapper = {
             version: 3,
             createdAt,
+            author: eval_.author,
             results: eval_.results,
             config: eval_.config,
         };
@@ -918,15 +948,12 @@ async function getTestCasesWithPredicate(predicate, limit) {
     }
     return Object.values(groupedTestCases);
 }
-exports.getTestCasesWithPredicate = getTestCasesWithPredicate;
 function getPrompts(limit = DEFAULT_QUERY_LIMIT) {
     return getPromptsWithPredicate(() => true, limit);
 }
-exports.getPrompts = getPrompts;
 async function getTestCases(limit = DEFAULT_QUERY_LIMIT) {
     return getTestCasesWithPredicate(() => true, limit);
 }
-exports.getTestCases = getTestCases;
 async function getPromptFromHash(hash) {
     const prompts = await getPrompts();
     for (const prompt of prompts) {
@@ -936,7 +963,6 @@ async function getPromptFromHash(hash) {
     }
     return undefined;
 }
-exports.getPromptFromHash = getPromptFromHash;
 async function getDatasetFromHash(hash) {
     const datasets = await getTestCases();
     for (const dataset of datasets) {
@@ -946,13 +972,13 @@ async function getDatasetFromHash(hash) {
     }
     return undefined;
 }
-exports.getDatasetFromHash = getDatasetFromHash;
 async function getEvalsWithPredicate(predicate, limit) {
     const db = (0, database_1.getDb)();
     const evals_ = await db
         .select({
         id: database_1.evals.id,
         createdAt: database_1.evals.createdAt,
+        author: database_1.evals.author,
         results: database_1.evals.results,
         config: database_1.evals.config,
         description: database_1.evals.description,
@@ -967,6 +993,7 @@ async function getEvalsWithPredicate(predicate, limit) {
         const resultWrapper = {
             version: 3,
             createdAt: createdAt,
+            author: eval_.author,
             results: eval_.results,
             config: eval_.config,
         };
@@ -983,11 +1010,9 @@ async function getEvalsWithPredicate(predicate, limit) {
     }
     return ret;
 }
-exports.getEvalsWithPredicate = getEvalsWithPredicate;
 async function getEvals(limit = DEFAULT_QUERY_LIMIT) {
     return getEvalsWithPredicate(() => true, limit);
 }
-exports.getEvals = getEvals;
 async function getEvalFromId(hash) {
     const evals_ = await getEvals();
     for (const eval_ of evals_) {
@@ -997,7 +1022,6 @@ async function getEvalFromId(hash) {
     }
     return undefined;
 }
-exports.getEvalFromId = getEvalFromId;
 async function deleteEval(evalId) {
     const db = (0, database_1.getDb)();
     await db.transaction(async () => {
@@ -1011,7 +1035,6 @@ async function deleteEval(evalId) {
         }
     });
 }
-exports.deleteEval = deleteEval;
 async function readFilters(filters) {
     const ret = {};
     const basePath = cliState_1.default.basePath || '';
@@ -1027,12 +1050,10 @@ async function readFilters(filters) {
     }
     return ret;
 }
-exports.readFilters = readFilters;
 function printBorder() {
     const border = '='.repeat(constants_1.TERMINAL_MAX_WIDTH);
     logger_1.default.info(border);
 }
-exports.printBorder = printBorder;
 async function transformOutput(codeOrFilepath, output, context) {
     let postprocessFn;
     if (codeOrFilepath.startsWith('file://')) {
@@ -1069,7 +1090,6 @@ async function transformOutput(codeOrFilepath, output, context) {
     }
     return ret;
 }
-exports.transformOutput = transformOutput;
 function setupEnv(envPath) {
     if (envPath) {
         logger_1.default.info(`Loading environment variables from ${envPath}`);
@@ -1079,7 +1099,6 @@ function setupEnv(envPath) {
         dotenv_1.default.config();
     }
 }
-exports.setupEnv = setupEnv;
 function getStandaloneEvals(limit = DEFAULT_QUERY_LIMIT) {
     const db = (0, database_1.getDb)();
     const results = db
@@ -1111,7 +1130,6 @@ function getStandaloneEvals(limit = DEFAULT_QUERY_LIMIT) {
     });
     return flatResults;
 }
-exports.getStandaloneEvals = getStandaloneEvals;
 function providerToIdentifier(provider) {
     if ((0, types_1.isApiProvider)(provider)) {
         return provider.id();
@@ -1121,18 +1139,15 @@ function providerToIdentifier(provider) {
     }
     return provider;
 }
-exports.providerToIdentifier = providerToIdentifier;
 function varsMatch(vars1, vars2) {
     return (0, fast_deep_equal_1.default)(vars1, vars2);
 }
-exports.varsMatch = varsMatch;
 function resultIsForTestCase(result, testCase) {
     const providersMatch = testCase.provider
         ? providerToIdentifier(testCase.provider) === providerToIdentifier(result.provider)
         : true;
     return varsMatch(testCase.vars, result.vars) && providersMatch;
 }
-exports.resultIsForTestCase = resultIsForTestCase;
 function safeJsonStringify(value, prettyPrint = false) {
     // Prevent circular references
     const cache = new Set();
@@ -1146,7 +1161,6 @@ function safeJsonStringify(value, prettyPrint = false) {
         return val;
     }, space);
 }
-exports.safeJsonStringify = safeJsonStringify;
 function renderVarsInObject(obj, vars) {
     // Renders nunjucks template strings with context variables
     if (!vars || process.env.PROMPTFOO_DISABLE_TEMPLATING) {
@@ -1171,7 +1185,6 @@ function renderVarsInObject(obj, vars) {
     }
     return obj;
 }
-exports.renderVarsInObject = renderVarsInObject;
 function extractJsonObjects(str) {
     // This will extract all json objects from a string
     const jsonObjects = [];
@@ -1201,7 +1214,6 @@ function extractJsonObjects(str) {
     }
     return jsonObjects;
 }
-exports.extractJsonObjects = extractJsonObjects;
 /**
  * Parses a file path or glob pattern to extract function names and file extensions.
  * Function names can be specified in the filename like this:
@@ -1211,10 +1223,10 @@ exports.extractJsonObjects = extractJsonObjects;
  * @returns Parsed details including function name, file extension, and directory status.
  */
 function parsePathOrGlob(basePath, promptPath) {
-    let filePath = path.join(basePath, promptPath);
-    if (filePath.includes('file:')) {
-        filePath = filePath.split('file:')[1];
+    if (promptPath.startsWith('file://')) {
+        promptPath = promptPath.slice(7);
     }
+    const filePath = path.join(basePath, promptPath);
     let stats;
     try {
         stats = fs.statSync(filePath);
@@ -1241,5 +1253,4 @@ function parsePathOrGlob(basePath, promptPath) {
         isPathPattern,
     };
 }
-exports.parsePathOrGlob = parsePathOrGlob;
 //# sourceMappingURL=util.js.map