promptfoo 0.121.4 → 0.121.7

package/dist/src/{util-C8e5uydV.js → util-Bx677_k2.js}

@@ -1,19 +1,21 @@
-import { S as getEnvBool, a as logger, f as sanitizeObject, k as state } from "./logger-B88EkIn6.js";
-import { A as TERMINAL_MAX_WIDTH, I as VERSION, t as fetchWithProxy } from "./fetch-B0Z3Oe4k.js";
-import { t as invariant } from "./invariant-vgHWClmd.js";
-import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-Dh4dOLlt.js";
-import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-DHIZ6_k8.js";
-import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-Bgh5SOn6.js";
-import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-BArZuxsI.js";
-import { r as runPython } from "./pythonUtils-Cldx7huE.js";
-import dotenv from "dotenv";
+import { S as getEnvBool, a as logger, f as sanitizeObject, k as state } from "./logger-Ct2S6Yx-.js";
+import { t as invariant } from "./invariant-Ddh24eXh.js";
+import { Gt as isProviderOptions, Wt as isApiProvider, g as ResultFailureReason, h as OutputFileExtension } from "./types-D6glLbdF.js";
+import { N as TERMINAL_MAX_WIDTH, t as fetchWithProxy } from "./fetch-It34O8Ur.js";
+import { n as VERSION } from "./version-lpHV_53E.js";
+import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-CXRfY3Ss.js";
+import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-BTK1W7lG.js";
+import { n as renderVarsInObject, o as getNunjucksEngine } from "./render-DFfDeYUK.js";
+import { r as runPython } from "./pythonUtils-D0BYebvX.js";
 import * as fs$2 from "fs";
+import fs from "fs";
 import * as path$1 from "path";
 import path from "path";
-import * as os$1 from "os";
 import yaml from "js-yaml";
-import * as fsPromises from "fs/promises";
+import dotenv from "dotenv";
+import * as os$1 from "os";
 import dedent from "dedent";
+import * as fsPromises from "fs/promises";
 import nunjucks from "nunjucks";
 import { parse as parse$1 } from "csv-parse/sync";
 import { globSync, hasMagic } from "glob";
@@ -196,7 +198,14 @@ function maybeLoadConfigFromExternalFile(config, context) {
 		const result = {};
 		for (const key of Object.keys(config)) {
 			const childContext = key === "value" && "type" in config && typeof config.type === "string" && (config.type === "python" || config.type === "javascript") ? "assertion" : key === "vars" ? "vars" : context;
-			result[key] = maybeLoadConfigFromExternalFile(config[key], childContext);
+			const value = maybeLoadConfigFromExternalFile(config[key], childContext);
+			if (key === "__proto__") Object.defineProperty(result, key, {
+				value,
+				enumerable: true,
+				configurable: true,
+				writable: true
+			});
+			else result[key] = value;
 		}
 		return result;
 	}
@@ -383,11 +392,32 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
 	return loaded;
 }
 //#endregion
-//#region src/util/provider.ts
+//#region src/util/providerRef.ts
+const PROVIDER_OPTION_KEYS = new Set([
+	"id",
+	"label",
+	"config",
+	"prompts",
+	"transform",
+	"delay",
+	"env",
+	"inputs"
+]);
+/** Returns true if the value is a non-empty string suitable as a provider identifier. */
+function isValidProviderId(id) {
+	return typeof id === "string" && id !== "";
+}
+function getProviderLabel(provider) {
+	if ((typeof provider === "object" || typeof provider === "function") && provider !== null && "label" in provider && typeof provider.label === "string") return provider.label;
+}
+/**
+* Resolves relative file paths in provider IDs to absolute paths for consistent matching.
+* Handles file://, exec:, python:, golang: prefixes and bare .js/.ts/.mjs paths.
+*/
 function canonicalizeProviderId(id) {
 	if (id.startsWith("file://")) {
 		const filePath = id.slice(7);
-		return path$1.isAbsolute(filePath) ? id : `file://${path$1.resolve(filePath)}`;
+		return path.isAbsolute(filePath) ? id : `file://${path.resolve(filePath)}`;
 	}
 	for (const prefix of [
 		"exec:",
@@ -395,19 +425,121 @@ function canonicalizeProviderId(id) {
 		"golang:"
 	]) if (id.startsWith(prefix)) {
 		const filePath = id.slice(prefix.length);
-		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path$1.resolve(filePath)}`;
+		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.resolve(filePath)}`;
 		return id;
 	}
-	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path$1.resolve(id)}`;
+	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.resolve(id)}`;
 	return id;
 }
-function getProviderLabel(provider) {
-	return provider?.label && typeof provider.label === "string" ? provider.label : void 0;
+/**
+* Returns true for provider refs that should be expanded from YAML/JSON config files.
+*/
+function isProviderConfigFileReference(providerPath) {
+	return providerPath.startsWith("file://") && (providerPath.endsWith(".yaml") || providerPath.endsWith(".yml") || providerPath.endsWith(".json"));
 }
+/**
+* Reads a provider config file and normalizes single-provider and multi-provider files.
+* Returns a `wasArray` flag so callers can detect multi-provider files that require
+* `loadApiProviders` instead of `loadApiProvider`.
+*/
+function readProviderConfigFile(providerPath, basePath) {
+	const relativePath = providerPath.slice(7);
+	const resolvedPath = path.isAbsolute(relativePath) ? relativePath : path.join(basePath || process.cwd(), relativePath);
+	let rawContent;
+	try {
+		rawContent = yaml.load(fs.readFileSync(resolvedPath, "utf8"));
+	} catch (err) {
+		throw new Error(`Failed to load provider config ${relativePath}: ${err instanceof Error ? err.message : err}`);
+	}
+	const fileContent = maybeLoadConfigFromExternalFile(rawContent);
+	invariant(fileContent, `Provider config ${relativePath} is undefined`);
+	return {
+		configs: [fileContent].flat(),
+		relativePath,
+		wasArray: Array.isArray(fileContent)
+	};
+}
+/**
+* Loads provider config objects from a file-backed provider reference.
+*/
+function loadProviderConfigsFromFile(providerPath, basePath) {
+	return readProviderConfigFile(providerPath, basePath).configs;
+}
+/**
+* Pure, synchronous classifier that converts every supported provider reference shape
+* into a discriminated descriptor. Does not read files or instantiate providers.
+*/
+function normalizeProviderRef(provider, options = {}) {
+	const { index } = options;
+	if (typeof provider === "string") {
+		if (!isValidProviderId(provider)) return {
+			kind: "unknown",
+			id: index === void 0 ? "unknown" : `unknown-${index}`
+		};
+		if (isProviderConfigFileReference(provider)) return {
+			kind: "file",
+			id: provider,
+			loadProviderPath: provider
+		};
+		return {
+			kind: "named",
+			id: provider,
+			loadProviderPath: provider
+		};
+	}
+	if (typeof provider === "function") {
+		const label = getProviderLabel(provider);
+		return {
+			kind: "function",
+			id: label ?? (index === void 0 ? "custom-function" : `custom-function-${index}`),
+			label
+		};
+	}
+	if (typeof provider === "object" && provider !== null && !Array.isArray(provider)) {
+		const providerId = provider.id;
+		const label = getProviderLabel(provider);
+		if (isValidProviderId(providerId)) return {
+			kind: "options",
+			id: providerId,
+			label,
+			loadOptions: provider,
+			loadProviderPath: providerId
+		};
+		const keys = Object.keys(provider);
+		if (keys.length === 1 && !PROVIDER_OPTION_KEYS.has(keys[0])) {
+			const originalId = keys[0];
+			const providerObject = provider[originalId];
+			if (typeof providerObject === "object" && providerObject !== null && !Array.isArray(providerObject) && isValidProviderId(originalId)) {
+				const id = isValidProviderId(providerObject.id) ? providerObject.id : originalId;
+				return {
+					kind: "map",
+					id,
+					label: getProviderLabel(providerObject),
+					loadOptions: {
+						...providerObject,
+						id
+					},
+					loadProviderPath: originalId
+				};
+			}
+		}
+		if (isValidProviderId(label)) return {
+			kind: "unknown",
+			id: label,
+			label
+		};
+	}
+	return {
+		kind: "unknown",
+		id: index === void 0 ? "unknown" : `unknown-${index}`
+	};
+}
+//#endregion
+//#region src/util/provider.ts
 function providerToIdentifier(provider) {
 	if (!provider) return;
 	if (typeof provider === "string") return canonicalizeProviderId(provider);
-	const label = getProviderLabel(provider);
+	const { label } = normalizeProviderRef(provider);
 	if (label) return label;
 	if (isApiProvider(provider)) return canonicalizeProviderId(provider.id());
 	if (isProviderOptions(provider)) {
@@ -806,15 +938,6 @@ async function writeCsvToGoogleSheet(rows, url) {
 //#endregion
 //#region src/server/utils/evalTableUtils.ts
 /**
-* Error thrown when a comparison eval is not found.
-*/
-var ComparisonEvalNotFoundError = class extends Error {
-	constructor(evalId) {
-		super(`Comparison eval not found: ${evalId}`);
-		this.name = "ComparisonEvalNotFoundError";
-	}
-};
-/**
 *
 *
 *
@@ -928,122 +1051,6 @@ function tableRowToCsvValues(row, options = {}) {
 	return rowValues;
 }
 /**
-* Generates CSV data from evaluation table data.
-*
-* Column structure per prompt:
-* - Output: Pure LLM output text (no pass/fail prefix)
-* - Status: PASS | FAIL | ERROR
-* - Score: Numeric score (e.g., "1.00")
-* - Named Scores: JSON object with per-assertion scores (e.g., {"clarity": 0.90, "accuracy": 0.85})
-* - Grader Reason: Explanation from the grader
-* - Comment: Additional grader comment
-*
-* This function is the single source of truth for CSV generation,
-* used by both the WebUI export and CLI export.
-*
-* @param table - The evaluation table data
-* @param options - Export options
-* @returns CSV formatted string
-*/
-function evalTableToCsv(table, options = { isRedteam: false }) {
-	const { isRedteam } = options;
-	const hasDescriptions = table.body.some((row) => row.test.description);
-	return stringify([buildCsvHeaders(table.head.vars, table.head.prompts, {
-		hasDescriptions,
-		isRedteam
-	}), ...table.body.map((row) => tableRowToCsvValues(row, {
-		hasDescriptions,
-		isRedteam
-	}))]);
-}
-/**
-* Generate JSON data from evaluation table
-* @param table Evaluation table data
-* @returns JSON object
-*/
-function evalTableToJson(table) {
-	return table;
-}
-/**
-* Merges comparison tables with the main table for side-by-side CSV export.
-*
-* @param mainEvalId - The ID of the main evaluation
-* @param mainTable - The main evaluation table
-* @param comparisonData - Array of comparison eval data (eval ID and table)
-* @returns Merged table with all prompts and outputs combined
-*/
-function mergeComparisonTables(mainEvalId, mainTable, comparisonData) {
-	return {
-		head: {
-			prompts: [...mainTable.head.prompts.map((prompt) => ({
-				...prompt,
-				label: `[${mainEvalId}] ${prompt.label || ""}`
-			})), ...comparisonData.flatMap(({ evalId, table }) => table.head.prompts.map((prompt) => ({
-				...prompt,
-				label: `[${evalId}] ${prompt.label || ""}`
-			})))],
-			vars: mainTable.head.vars
-		},
-		body: mainTable.body.map((row) => {
-			const testIdx = row.testIdx;
-			const matchingRows = comparisonData.map(({ table }) => table.body.find((compRow) => compRow.testIdx === testIdx)).filter((r) => r !== void 0);
-			return {
-				...row,
-				outputs: [...row.outputs, ...matchingRows.flatMap((r) => r.outputs)]
-			};
-		})
-	};
-}
-/**
-* High-level function to generate CSV from an evaluation.
-*
-* Used by WebUI for CSV downloads (with or without comparison evals).
-* For CLI exports, use `streamEvalCsv` which is more memory-efficient
-* for large datasets.
-*
-* Both functions use the same underlying formatting (`evalTableToCsv`,
-* `buildCsvHeaders`, `tableRowToCsvValues`) to ensure consistent output.
-*
-* @param eval_ - The evaluation to export
-* @param options - Export options including filters and comparison eval IDs
-* @returns CSV formatted string
-* @throws ComparisonEvalNotFoundError if a comparison eval ID is not found
-* @throws Error if comparison exports requested without findEvalById callback
-*/
-async function generateEvalCsv(eval_, options = {}) {
-	const UNLIMITED_RESULTS = Number.MAX_SAFE_INTEGER;
-	const mainTable = await eval_.getTablePage({
-		offset: 0,
-		limit: UNLIMITED_RESULTS,
-		filterMode: options.filterMode,
-		searchQuery: options.searchQuery,
-		filters: options.filters
-	});
-	let finalTable = mainTable;
-	if (options.comparisonEvalIds && options.comparisonEvalIds.length > 0) {
-		if (!options.findEvalById) throw new Error("findEvalById callback is required for comparison exports. Pass Eval.findById when calling from server routes.");
-		const indices = mainTable.body.map((row) => row.testIdx);
-		const comparisonData = await Promise.all(options.comparisonEvalIds.map(async (comparisonEvalId) => {
-			const comparisonEval = await options.findEvalById(comparisonEvalId);
-			if (!comparisonEval) throw new ComparisonEvalNotFoundError(comparisonEvalId);
-			const table = await comparisonEval.getTablePage({
-				offset: 0,
-				limit: indices.length,
-				filterMode: "all",
-				testIndices: indices,
-				searchQuery: options.searchQuery,
-				filters: options.filters
-			});
-			return {
-				evalId: comparisonEval.id,
-				table
-			};
-		}));
-		finalTable = mergeComparisonTables(eval_.id, mainTable, comparisonData);
-	}
-	return evalTableToCsv(finalTable, { isRedteam: Boolean(eval_.config.redteam) });
-}
-/**
 * Stream CSV data from an evaluation in batches.
 *
 * This is more memory-efficient for large evaluations as it processes
@@ -1163,7 +1170,7 @@ function createOutputMetadata(evalRecord) {
 		arch: os$1.arch(),
 		exportedAt: (/* @__PURE__ */ new Date()).toISOString(),
 		evaluationCreatedAt,
-		author: evalRecord.author
+		author: evalRecord.author ?? void 0
 	};
 }
 /**
@@ -1298,6 +1305,6 @@ function printBorder() {
 	logger.info(border);
 }
 //#endregion
-export { readFilters as A, getResolvedRelativePath as C, maybeLoadResponseFormatFromExternalFile as D, maybeLoadFromExternalFileWithVars as E, loadFunction as M, parseFileUrl as N, maybeLoadToolsFromExternalFile as O, getNunjucksEngineForFilePath as S, maybeLoadFromExternalFile as T, getProviderDescription as _, evalTableToJson as a, isOpenAiProvider as b, fetchCsvFromGoogleSheet as c, extractRuntimeVars as d, filterRuntimeVars as f, doesProviderRefMatch as g, checkProviderApiKeys as h, ComparisonEvalNotFoundError as i, readOutput as j, parsePathOrGlob as k, setupEnv as l, resultIsForTestCase as m, writeMultipleOutputs as n, generateEvalCsv as o, getTestCaseDeduplicationKey as p, writeOutput as r, mergeComparisonTables as s, printBorder as t, deduplicateTestCases as u, isAnthropicProvider as v, maybeLoadConfigFromExternalFile as w, isProviderAllowed as x, isGoogleProvider as y };
+export { readFilters as A, getResolvedRelativePath as C, maybeLoadResponseFormatFromExternalFile as D, maybeLoadFromExternalFileWithVars as E, loadFunction as M, parseFileUrl as N, maybeLoadToolsFromExternalFile as O, getNunjucksEngineForFilePath as S, maybeLoadFromExternalFile as T, isProviderAllowed as _, setupEnv as a, normalizeProviderRef as b, filterRuntimeVars as c, checkProviderApiKeys as d, doesProviderRefMatch as f, isOpenAiProvider as g, isGoogleProvider as h, fetchCsvFromGoogleSheet as i, readOutput as j, parsePathOrGlob as k, getTestCaseDeduplicationKey as l, isAnthropicProvider as m, writeMultipleOutputs as n, deduplicateTestCases as o, getProviderDescription as p, writeOutput as r, extractRuntimeVars as s, printBorder as t, resultIsForTestCase as u, isProviderConfigFileReference as v, maybeLoadConfigFromExternalFile as w, readProviderConfigFile as x, loadProviderConfigsFromFile as y };
 
-//# sourceMappingURL=util-C8e5uydV.js.map
+//# sourceMappingURL=util-Bx677_k2.js.map