promptfoo 0.120.24 → 0.120.26

package/dist/src/index.cjs

@@ -1,39 +1,40 @@
 Object.defineProperty(exports, '__esModule', { value: true });
-const require_logger = require('./logger-CMp-NS-e.cjs');
+const require_logger = require('./logger-BdZ-IqBc.cjs');
 const require_invariant = require('./invariant-CKcJAQ6M.cjs');
-const require_esm = require('./esm-gWVPXn3r.cjs');
-const require_pythonUtils = require('./pythonUtils-rOCm9w_5.cjs');
+const require_esm = require('./esm-BQkx5roy.cjs');
+const require_pythonUtils = require('./pythonUtils-DZ6EbdY4.cjs');
 const require_fileExtensions = require('./fileExtensions-DkJYkWUy.cjs');
-const require_transform = require('./transform-DshYLyBq.cjs');
-const require_graders = require('./graders-2dabIkXs.cjs');
-const require_types = require('./types-B-XUqfNs.cjs');
-const require_util = require('./util-C7A-PlKK.cjs');
-const require_fetch = require('./fetch-MvWxCS5U.cjs');
-const require_cache = require('./cache-KNsTJLKP.cjs');
-const require_providers = require('./providers-BWoVY_Wz.cjs');
-const require_utils = require('./utils-Dezi3MBH.cjs');
-const require_extractor = require('./extractor-D-RM2m6A.cjs');
-require('./genaiTracer-BQ8AmajD.cjs');
-const require_chat = require('./chat-DOQIJx7O.cjs');
-const require_tokenUsageUtils = require('./tokenUsageUtils-BtZd3sP7.cjs');
-const require_transform$1 = require('./transform-Bw1IstDE.cjs');
-require('./messages-B5HsO_dM.cjs');
-require('./util-CV99ps44.cjs');
-require('./openai-BjpdxIOG.cjs');
-const require_util$2 = require('./util-DTJWKLkl.cjs');
-require('./completion-D6BTVt6e.cjs');
-const require_accounts = require('./accounts-BwVGYllP.cjs');
-const require_server = require('./server-CSbLW-UI.cjs');
-const require_blobs = require('./blobs-DOdc-E8L.cjs');
-const require_tables = require('./tables-Dansasnu.cjs');
-const require_telemetry = require('./telemetry-DFDFKdnr.cjs');
-const require_text = require('./text-PYISqVm1.cjs');
-const require_store = require('./store-BpumNYCl.cjs');
-require('./base-Bso8ccGu.cjs');
-require('./image-DS9hlgSc.cjs');
-const require_providerRegistry = require('./providerRegistry-HGQd2MF6.cjs');
-const require_rubyUtils = require('./rubyUtils-rnCVDgH-.cjs');
-const require_evalResult = require('./evalResult-BU4huold.cjs');
+const require_transform = require('./transform-Kd6u-oNm.cjs');
+const require_graders = require('./graders-DzUUnUjC.cjs');
+const require_types = require('./types-BIfttHrT.cjs');
+const require_util = require('./util-DUYOvxAy.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
+const require_providers = require('./providers-BNKVY53V.cjs');
+const require_utils = require('./utils-DFaZa6Rf.cjs');
+const require_extractor = require('./extractor-DbhlYEeo.cjs');
+require('./genaiTracer-CQlpZkrp.cjs');
+const require_chat = require('./chat-C4zqjObh.cjs');
+const require_tokenUsageUtils = require('./tokenUsageUtils-CXhxVj72.cjs');
+const require_transform$1 = require('./transform-ivxEY4f7.cjs');
+require('./messages-DXV3Qh8_.cjs');
+require('./util-DkFTvieG.cjs');
+require('./responses-zOtKtnY_.cjs');
+require('./openai-Bigwjgo1.cjs');
+const require_util$2 = require('./util-C08Kns6-.cjs');
+require('./completion-DDyL3Cb2.cjs');
+const require_accounts = require('./accounts-BzEY8H3v.cjs');
+const require_server = require('./server-CgUQ25qW.cjs');
+const require_blobs = require('./blobs-673H0jCl.cjs');
+const require_tables = require('./tables-B9E1kRp-.cjs');
+const require_telemetry = require('./telemetry-D0_yFdtU.cjs');
+const require_text = require('./text-DF2hMKdg.cjs');
+const require_store = require('./store-BqwfFEyF.cjs');
+require('./base-f71xxWai.cjs');
+require('./image-C3BjJUAU.cjs');
+const require_providerRegistry = require('./providerRegistry-lc7a7utN.cjs');
+const require_rubyUtils = require('./rubyUtils-DCVaJ3mc.cjs');
+const require_evalResult = require('./evalResult-DlRfu_Rq.cjs');
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -306,7 +307,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry_default.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger_default.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BvmMgacx.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-B6Xo4KZM.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			require_logger.logger_default.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -329,7 +330,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger_default.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-BvmMgacx.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-B6Xo4KZM.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger_default.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -364,7 +365,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger_default.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-CrMinjmN.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-m5KT6Ly7.cjs"));
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -2775,7 +2776,7 @@ const ASSERTION_HANDLERS = {
 	"llm-rubric": handleLlmRubric,
 	meteor: async (params) => {
 		try {
-			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-DRuEfzuQ.cjs"));
+			const { handleMeteorAssertion } = await Promise.resolve().then(() => require("./meteor-odmwVbyG.cjs"));
 			return handleMeteorAssertion(params);
 		} catch (error) {
 			if (error instanceof Error && (error.message.includes("Cannot find module") || error.message.includes("natural\" package is required"))) return {
@@ -2885,9 +2886,9 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 		let filePath = fileRef;
 		let functionName;
 		if (fileRef.includes(":")) {
-			const [pathPart, funcPart] = fileRef.split(":");
-			filePath = pathPart;
-			functionName = funcPart;
+			const colonIndex = fileRef.indexOf(":");
+			filePath = fileRef.slice(0, colonIndex);
+			functionName = fileRef.slice(colonIndex + 1);
 		}
 		filePath = path.default.resolve(basePath, filePath);
 		if (require_fileExtensions.isJavascriptFile(filePath)) {
@@ -2905,7 +2906,7 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 			};
 		}
 		else if (filePath.endsWith(".rb")) try {
-			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-BLd6EE1u.cjs"));
+			const { runRuby } = await Promise.resolve().then(() => require("./rubyUtils-vb8OYFC-.cjs"));
 			valueFromScript = await runRuby(filePath, functionName || "get_assert", [output, context]);
 			require_logger.logger_default.debug(`Ruby script ${filePath} output: ${valueFromScript}`);
 		} catch (error) {
@@ -3913,7 +3914,7 @@ async function runEval({ provider, prompt, test, testSuite, delay, nunjucksFilte
 			promptIdx,
 			testIdx
 		});
-		require_logger.logger_default.error("Provider call failed during eval", logContext);
+		if (!(err instanceof Error && err.name === "AbortError")) require_logger.logger_default.error("Provider call failed during eval", logContext);
 		return [{
 			...setup,
 			error: errorWithStack,
@@ -4094,11 +4095,17 @@ var Evaluator = class {
 		let globalTimeout;
 		let globalAbortController;
 		const processedIndices = /* @__PURE__ */ new Set();
+		let targetUnavailable = false;
+		let targetErrorStatus;
+		const targetErrorAbortController = new AbortController();
 		let ciProgressReporter = null;
 		let progressBarManager = null;
+		let providerAbortSignal = options.abortSignal;
+		let combinedAbortSignal = options.abortSignal ? AbortSignal.any([options.abortSignal, targetErrorAbortController.signal]) : targetErrorAbortController.signal;
 		if (maxEvalTimeMs > 0) {
 			globalAbortController = new AbortController();
-			options.abortSignal = options.abortSignal ? AbortSignal.any([options.abortSignal, globalAbortController.signal]) : globalAbortController.signal;
+			providerAbortSignal = providerAbortSignal ? AbortSignal.any([providerAbortSignal, globalAbortController.signal]) : globalAbortController.signal;
+			combinedAbortSignal = AbortSignal.any([combinedAbortSignal, globalAbortController.signal]);
 			globalTimeout = setTimeout(() => {
 				evalTimedOut = true;
 				globalAbortController?.abort();
@@ -4106,7 +4113,7 @@ var Evaluator = class {
 		}
 		const vars = /* @__PURE__ */ new Set();
 		const checkAbort = () => {
-			if (options.abortSignal?.aborted) throw new Error("Operation cancelled");
+			if (combinedAbortSignal.aborted) throw new Error("Operation cancelled");
 		};
 		if (!options.silent) require_logger.logger_default.info(`Starting evaluation ${this.evalRecord.id}`);
 		checkAbort();
@@ -4267,7 +4274,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (require_types.isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-CiZCpIxz.cjs"));
+					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BMZZmPBJ.cjs"));
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4337,7 +4344,7 @@ var Evaluator = class {
 							registers: this.registers,
 							isRedteam: testSuite.redteam != null,
 							concurrency,
-							abortSignal: options.abortSignal,
+							abortSignal: providerAbortSignal,
 							evalId: this.evalRecord.id,
 							rateLimitRegistry: this.rateLimitRegistry
 						});
@@ -4351,7 +4358,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (require_logger.cliState_default.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-CqFeWKYx.cjs"));
+			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-CxTP-LMm.cjs"));
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: require_logger.cliState_default.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -4404,6 +4411,14 @@ var Evaluator = class {
 					require_logger.logger_default.error(`Error saving result: ${error} ${require_logger.safeJsonStringify(resultSummary)}`);
 				}
 				for (const writer of this.fileWriters) await writer.write(row);
+				const httpStatus = row.response?.metadata?.http?.status;
+				if (typeof httpStatus === "number" && require_cache.isNonTransientHttpStatus(httpStatus)) {
+					targetUnavailable = true;
+					targetErrorStatus = httpStatus;
+					require_logger.logger_default.error(`Target returned HTTP ${httpStatus}. Aborting scan - this error will not resolve on retry.`);
+					targetErrorAbortController.abort();
+					break;
+				}
 				const { promptIdx } = row;
 				const metrics = prompts[promptIdx].metrics;
 				require_invariant.invariant(metrics, "Expected prompt.metrics to be set");
@@ -4567,6 +4582,7 @@ var Evaluator = class {
 		if (this.options.showProgressBar && progressBarManager) await progressBarManager.initialize(runEvalOptions, concurrency, 0);
 		try {
 			if (serialRunEvalOptions.length > 0) for (const evalStep of serialRunEvalOptions) {
+				checkAbort();
 				if (isWebUI) {
 					const provider = evalStep.provider.label || evalStep.provider.id();
 					const vars = formatVarsForDisplay(evalStep.test.vars || {}, 50);
@@ -4584,22 +4600,32 @@ var Evaluator = class {
 				await this.evalRecord.addPrompts(prompts);
 			});
 		} catch (err) {
-			if (options.abortSignal?.aborted) if (evalTimedOut) require_logger.logger_default.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
-			else {
-				require_logger.logger_default.info("Evaluation interrupted, saving progress...");
-				if (globalTimeout) clearTimeout(globalTimeout);
-				if (progressBarManager) progressBarManager.stop();
-				if (ciProgressReporter) ciProgressReporter.finish();
-				this.evalRecord.setVars(Array.from(vars));
-				await this.evalRecord.addPrompts(prompts);
-				updateSignalFile(this.evalRecord.id);
-				return this.evalRecord;
-			}
-			else {
+			if (combinedAbortSignal.aborted) {
+				if (evalTimedOut) require_logger.logger_default.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
+				else if (!targetUnavailable) {
+					require_logger.logger_default.info("Evaluation interrupted, saving progress...");
+					if (globalTimeout) clearTimeout(globalTimeout);
+					if (progressBarManager) progressBarManager.stop();
+					if (ciProgressReporter) ciProgressReporter.finish();
+					this.evalRecord.setVars(Array.from(vars));
+					await this.evalRecord.addPrompts(prompts);
+					updateSignalFile(this.evalRecord.id);
+					return this.evalRecord;
+				}
+			} else {
 				if (ciProgressReporter) ciProgressReporter.error(`Evaluation failed: ${String(err)}`);
 				throw err;
 			}
 		}
+		if (targetUnavailable) {
+			if (globalTimeout) clearTimeout(globalTimeout);
+			if (progressBarManager) progressBarManager.stop();
+			if (ciProgressReporter) ciProgressReporter.error(`Target unavailable (HTTP ${targetErrorStatus})`);
+			this.evalRecord.setVars(Array.from(vars));
+			await this.evalRecord.addPrompts(prompts);
+			updateSignalFile(this.evalRecord.id);
+			return this.evalRecord;
+		}
 		const compareRowsCount = rowsWithSelectBestAssertion.size + rowsWithMaxScoreAssertion.size;
 		if (progressBarManager) {
 			if (compareRowsCount > 0) progressBarManager.updateTotalCount(compareRowsCount);
@@ -5306,10 +5332,17 @@ function convertResultsToTable(eval_) {
 				if (keyToUpdate) result.vars[keyToUpdate] = actualPrompt;
 			}
 		}
-		if (result.metadata?.sessionId && !result.vars?.sessionId) {
-			result.vars = result.vars || {};
-			result.vars.sessionId = result.metadata.sessionId;
-			varsForHeader.add("sessionId");
+		if (!result.vars?.sessionId) {
+			const metadataSessionIds = result.metadata?.sessionIds;
+			if (Array.isArray(metadataSessionIds) && metadataSessionIds.length > 0) {
+				result.vars = result.vars || {};
+				result.vars.sessionId = metadataSessionIds.filter((id) => id != null && id !== "").map(String).join("\n");
+				varsForHeader.add("sessionId");
+			} else if (result.metadata?.sessionId) {
+				result.vars = result.vars || {};
+				result.vars.sessionId = result.metadata.sessionId;
+				varsForHeader.add("sessionId");
+			}
 		}
 		const transformDisplayVars = result.response?.metadata?.transformDisplayVars;
 		if (transformDisplayVars) {
@@ -5365,7 +5398,12 @@ function convertResultsToTable(eval_) {
 				model: result.response.video.model,
 				aspectRatio: result.response.video.aspectRatio,
 				resolution: result.response.video.resolution
-			} : void 0
+			} : void 0,
+			images: result.response?.images?.map((img) => ({
+				data: img.data,
+				blobRef: img.blobRef,
+				mimeType: img.mimeType
+			}))
 		};
 		require_invariant.invariant(result.promptId, "Prompt ID is required");
 		row.testIdx = result.testIdx;
@@ -5431,7 +5469,12 @@ function convertEvalResultToTableCell(result) {
 			model: result.response.video.model,
 			aspectRatio: result.response.video.aspectRatio,
 			resolution: result.response.video.resolution
-		} : void 0
+		} : void 0,
+		images: result.response?.images?.map((img) => ({
+			data: img.data,
+			blobRef: img.blobRef,
+			mimeType: img.mimeType
+		}))
 	};
 }
 function convertTestResultsToTableRow(results, varsForHeader) {
@@ -5440,10 +5483,13 @@ function convertTestResultsToTableRow(results, varsForHeader) {
 		outputs: [],
 		vars: Object.values(varsForHeader).map((varName) => {
 			if (varName === "sessionId") {
-				const sessionId = results[0].testCase.vars?.sessionId;
-				const varValue = sessionId == null || sessionId === "" ? results[0].metadata?.sessionId ?? "" : sessionId;
-				if (typeof varValue === "string") return varValue;
-				return JSON.stringify(varValue);
+				const sessionIdFromVars = results[0].testCase.vars?.sessionId;
+				if (sessionIdFromVars != null && sessionIdFromVars !== "") return typeof sessionIdFromVars === "string" ? sessionIdFromVars : JSON.stringify(sessionIdFromVars);
+				const metadataSessionIds = results[0].metadata?.sessionIds;
+				if (Array.isArray(metadataSessionIds) && metadataSessionIds.length > 0) return metadataSessionIds.filter((id) => id != null && id !== "").map(String).join("\n");
+				const metadataSessionId = results[0].metadata?.sessionId;
+				if (metadataSessionId != null) return typeof metadataSessionId === "string" ? metadataSessionId : JSON.stringify(metadataSessionId);
+				return "";
 			}
 			const varValue = results[0].testCase.vars?.[varName] ?? "";
 			if (typeof varValue === "string") return varValue;
@@ -5723,7 +5769,13 @@ var Eval = class Eval {
 	_resultsLoaded = false;
 	runtimeOptions;
 	_shared = false;
+	/** Total wall-clock duration. For redteam evals: generationDurationMs + evaluationDurationMs.
+	*  For non-redteam evals: equals evaluationDurationMs (generation phase is N/A). */
 	durationMs;
+	/** Time spent generating adversarial test cases (redteam only). */
+	generationDurationMs;
+	/** Time spent running the evaluation phase. */
+	evaluationDurationMs;
 	/**
 	* The shareable URL for this evaluation, if it has been shared.
 	* Set by the evaluate() function when sharing is enabled.
@@ -5742,8 +5794,11 @@ var Eval = class Eval {
 		const eval_ = evalData[0];
 		const datasetId = datasetResults[0]?.datasetId;
 		const resultsObj = eval_.results;
-		const rawDurationMs = resultsObj && "durationMs" in resultsObj ? resultsObj.durationMs : void 0;
-		const durationMs = typeof rawDurationMs === "number" && Number.isFinite(rawDurationMs) && rawDurationMs >= 0 ? rawDurationMs : void 0;
+		const validateDuration = (raw) => typeof raw === "number" && Number.isFinite(raw) && raw >= 0 ? raw : void 0;
+		const rawDurationMs = validateDuration(resultsObj?.["durationMs"]);
+		const generationDurationMs = validateDuration(resultsObj?.["generationDurationMs"]);
+		const evaluationDurationMs = validateDuration(resultsObj?.["evaluationDurationMs"]);
+		const durationMs = rawDurationMs ?? (generationDurationMs != null || evaluationDurationMs != null ? (generationDurationMs ?? 0) + (evaluationDurationMs ?? 0) : void 0);
 		const evalInstance = new Eval(eval_.config, {
 			id: eval_.id,
 			createdAt: new Date(eval_.createdAt),
@@ -5754,7 +5809,9 @@ var Eval = class Eval {
 			persisted: true,
 			vars: eval_.vars || [],
 			runtimeOptions: eval_.runtimeOptions ?? void 0,
-			durationMs
+			durationMs,
+			generationDurationMs,
+			evaluationDurationMs
 		});
 		if (eval_.results && "table" in eval_.results) evalInstance.oldResults = eval_.results;
 		if (!eval_.vars || eval_.vars.length === 0) {
@@ -5879,6 +5936,8 @@ var Eval = class Eval {
 		this.vars = opts?.vars || [];
 		this.runtimeOptions = opts?.runtimeOptions;
 		this.durationMs = opts?.durationMs;
+		this.generationDurationMs = opts?.generationDurationMs;
+		this.evaluationDurationMs = opts?.evaluationDurationMs;
 	}
 	version() {
 		/**
@@ -5909,7 +5968,13 @@ var Eval = class Eval {
 		if (this.useOldResults()) {
 			require_invariant.invariant(this.oldResults, "Old results not found");
 			updateObj.results = this.oldResults;
-		} else if (this.durationMs !== void 0) updateObj.results = { durationMs: this.durationMs };
+		} else if (this.durationMs !== void 0 || this.generationDurationMs !== void 0 || this.evaluationDurationMs !== void 0) {
+			let expr = drizzle_orm.sql`CASE WHEN json_valid(${require_tables.evalsTable.results}) AND json_type(${require_tables.evalsTable.results}) = 'object' THEN ${require_tables.evalsTable.results} ELSE '{}' END`;
+			if (this.durationMs !== void 0) expr = drizzle_orm.sql`json_set(${expr}, '$.durationMs', ${this.durationMs})`;
+			if (this.generationDurationMs !== void 0) expr = drizzle_orm.sql`json_set(${expr}, '$.generationDurationMs', ${this.generationDurationMs})`;
+			if (this.evaluationDurationMs !== void 0) expr = drizzle_orm.sql`json_set(${expr}, '$.evaluationDurationMs', ${this.evaluationDurationMs})`;
+			updateObj.results = expr;
+		}
 		db.update(require_tables.evalsTable).set(updateObj).where((0, drizzle_orm.eq)(require_tables.evalsTable.id, this.id)).run();
 		this.persisted = true;
 	}
@@ -5919,8 +5984,17 @@ var Eval = class Eval {
 	addVar(varName) {
 		this.vars.push(varName);
 	}
+	/** Sets the evaluation phase duration and recomputes the total. Called by the evaluator. */
 	setDurationMs(durationMs) {
-		this.durationMs = durationMs;
+		if (!Number.isFinite(durationMs) || durationMs < 0) return;
+		this.evaluationDurationMs = durationMs;
+		this.durationMs = (this.generationDurationMs ?? 0) + durationMs;
+	}
+	/** Sets the generation phase duration and recomputes the total. Called by doRedteamRun. */
+	setGenerationDurationMs(durationMs) {
+		if (!Number.isFinite(durationMs) || durationMs < 0) return;
+		this.generationDurationMs = durationMs;
+		this.durationMs = durationMs + (this.evaluationDurationMs ?? 0);
 	}
 	getPrompts() {
 		if (this.useOldResults()) {
@@ -5958,6 +6032,27 @@ var Eval = class Eval {
 	async getTotalResultRowCount() {
 		return getTotalResultRowCount(this.id);
 	}
+	/**
+	* Find a non-transient HTTP error status from evaluation results.
+	* Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+	*
+	* For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+	* For non-persisted evals: Falls back to scanning in-memory results.
+	*/
+	async findTargetErrorStatus() {
+		const scanInMemory = () => {
+			for (const result of this.results) {
+				const status = result.response?.metadata?.http?.status;
+				if (typeof status === "number" && require_cache.isNonTransientHttpStatus(status)) return status;
+			}
+		};
+		if (!this.persisted) return scanInMemory();
+		try {
+			return require_tables.getDb().select({ httpStatus: drizzle_orm.sql`CAST(json_extract(${require_tables.evalResultsTable.response}, '$.metadata.http.status') AS INTEGER)` }).from(require_tables.evalResultsTable).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, this.id), drizzle_orm.sql`json_extract(${require_tables.evalResultsTable.response}, '$.metadata.http.status') IN (${drizzle_orm.sql.join(require_cache.NON_TRANSIENT_HTTP_STATUSES.map((s) => drizzle_orm.sql`${s}`), drizzle_orm.sql`, `)})`)).limit(1).get()?.httpStatus ?? void 0;
+		} catch {
+			return scanInMemory();
+		}
+	}
 	async fetchResultsByTestIdx(testIdx) {
 		return await require_evalResult.EvalResult.findManyByEvalId(this.id, { testIdx });
 	}
@@ -6180,7 +6275,12 @@ var Eval = class Eval {
 			};
 		}
 		const allResults = await require_evalResult.EvalResult.findManyByEvalIdAndTestIndices(this.id, testIndices);
-		if (allResults.some((result) => result.metadata?.sessionId && !result.testCase?.vars?.sessionId) && !vars.includes("sessionId")) {
+		if (allResults.some((result) => {
+			const hasSessionIds = Array.isArray(result.metadata?.sessionIds) && result.metadata.sessionIds.length > 0;
+			const hasSessionId = Boolean(result.metadata?.sessionId);
+			const notInVars = !result.testCase?.vars?.sessionId;
+			return (hasSessionIds || hasSessionId) && notInVars;
+		}) && !vars.includes("sessionId")) {
 			vars.push("sessionId");
 			vars.sort();
 		}
@@ -6241,7 +6341,9 @@ var Eval = class Eval {
 			failures: 0,
 			errors: 0,
 			tokenUsage: require_tokenUsageUtils.createEmptyTokenUsage(),
-			durationMs: this.durationMs
+			durationMs: this.durationMs,
+			generationDurationMs: this.generationDurationMs,
+			evaluationDurationMs: this.evaluationDurationMs
 		};
 		for (const prompt of this.prompts) {
 			stats.successes += prompt.metrics?.testPassCount ?? 0;
@@ -7539,6 +7641,24 @@ function validateTestProviderReferences(tests, providers, defaultTest, scenarios
 	});
 }
 
+//#endregion
+//#region src/util/config/extensions.ts
+/**
+* Supported config file extensions, sorted by frequency of use.
+* Order matters: loaders try each in sequence and stop at the first match.
+*/
+const DEFAULT_CONFIG_EXTENSIONS = [
+	"yaml",
+	"yml",
+	"json",
+	"cjs",
+	"cts",
+	"js",
+	"mjs",
+	"mts",
+	"ts"
+];
+
 //#endregion
 //#region src/util/config/load.ts
 /**
@@ -7547,6 +7667,34 @@ function validateTestProviderReferences(tests, providers, defaultTest, scenarios
 function isTestCaseWithVars(test) {
 	return typeof test === "object" && test !== null && "vars" in test;
 }
+/**
+* When --providers is used alongside a config file that has providers defined,
+* maps each CLI provider token to a matching config provider (preserving its config
+* options like num_ctx, temperature). Unmatched tokens are kept as bare strings.
+*
+* Matching priority per token:
+* 1. Exact match on provider id
+* 2. Exact match on provider label
+* 3. Provider-prefix match: config id ends with `:cliProvider` (e.g. CLI `llama3.1:8b`
+*    matches config `ollama:llama3.1:8b`). First match wins if multiple configs share a suffix.
+* 4. No match: keep raw CLI string for fresh provider creation
+*/
+function resolveCliProvidersWithConfig(cliProviders, configProviders) {
+	if (!configProviders || !Array.isArray(configProviders)) return cliProviders;
+	const indexed = configProviders.map((cp, i) => ({
+		provider: cp,
+		...getProviderIdAndLabel(cp, i)
+	}));
+	return cliProviders.map((cliProvider) => {
+		const exactId = indexed.find((entry) => entry.id === cliProvider);
+		if (exactId) return exactId.provider;
+		const exactLabel = indexed.find((entry) => entry.label === cliProvider);
+		if (exactLabel) return exactLabel.provider;
+		const prefixMatch = indexed.find((entry) => entry.id.endsWith(":" + cliProvider));
+		if (prefixMatch) return prefixMatch.provider;
+		return cliProvider;
+	});
+}
 async function dereferenceConfig(rawConfig) {
 	if (require_logger.getEnvBool("PROMPTFOO_DISABLE_REF_PARSER")) return rawConfig;
 	const extractFunctionParameters = (functions) => {
@@ -7683,7 +7831,7 @@ async function combineConfigs(configPaths) {
 	const configs = [];
 	for (const configPath of configPaths) {
 		const globPaths = (0, glob.globSync)(path.resolve(process$1.default.cwd(), configPath), { windowsPathsNoEscape: true });
-		if (globPaths.length === 0) throw new Error(`No configuration file found at ${configPath}. Run "promptfoo init" to create one or pass --config path/to/promptfooconfig.yaml.`);
+		if (globPaths.length === 0) throw new Error(`No configuration file found at ${configPath}. Run "${promptfooCommand("init")}" to create one or pass --config path/to/promptfooconfig.yaml.`);
 		for (const globPath of globPaths) {
 			const config = await readConfig(globPath);
 			configs.push(config);
@@ -7896,7 +8044,7 @@ async function resolveConfigs(cmdObj, _defaultConfig, type) {
 		tags: fileConfig.tags || defaultConfig.tags,
 		description: cmdObj.description || fileConfig.description || defaultConfig.description,
 		prompts: cmdObj.prompts || fileConfig.prompts || defaultConfig.prompts || [],
-		providers: cmdObj.providers || fileConfig.providers || defaultConfig.providers || [],
+		providers: fileConfig.providers || defaultConfig.providers || [],
 		tests: cmdObj.tests || cmdObj.vars || fileConfig.tests || defaultConfig.tests || [],
 		scenarios: fileConfig.scenarios || defaultConfig.scenarios,
 		env: fileConfig.env || defaultConfig.env,
@@ -7911,11 +8059,14 @@ async function resolveConfigs(cmdObj, _defaultConfig, type) {
 		evaluateOptions: fileConfig.evaluateOptions || defaultConfig.evaluateOptions
 	};
 	const hasPrompts = [config.prompts].flat().filter(Boolean).length > 0;
-	const hasProviders = [config.providers].flat().filter(Boolean).length > 0;
+	const hasProviders = cmdObj.providers && cmdObj.providers.length > 0 || [config.providers].flat().filter(Boolean).length > 0;
 	if (!Boolean(configPaths) && !hasPrompts && !hasProviders && !require_logger.isCI()) {
+		const extList = DEFAULT_CONFIG_EXTENSIONS.join(", ");
 		require_logger.logger_default.warn(dedent.default`
       ${chalk.default.yellow.bold("⚠️  No promptfooconfig found")}
 
+      ${chalk.default.white(`Searched in ${chalk.default.bold(process$1.default.cwd())} for promptfooconfig.{${extList}}`)}
+
       ${chalk.default.white("Try running with:")}
 
       ${chalk.default.cyan(`${promptfooCommand("")} eval -c ${chalk.default.bold("path/to/promptfooconfig.yaml")}`)}
@@ -7936,8 +8087,9 @@ async function resolveConfigs(cmdObj, _defaultConfig, type) {
 	}
 	require_invariant.invariant(Array.isArray(config.providers), "providers must be an array");
 	const resolvedProviderConfigs = require_providers.resolveProviderConfigs(config.providers, { basePath });
+	const cliFilteredProviderConfigs = (cmdObj.providers ? resolveCliProvidersWithConfig(cmdObj.providers, resolvedProviderConfigs) : resolvedProviderConfigs) ?? [];
 	const filterOption = cmdObj.filterProviders || cmdObj.filterTargets;
-	const filteredProviderConfigs = filterProviderConfigs(resolvedProviderConfigs, filterOption);
+	const filteredProviderConfigs = filterProviderConfigs(cliFilteredProviderConfigs, filterOption);
 	if (filterOption && Array.isArray(filteredProviderConfigs) && filteredProviderConfigs.length === 0) require_logger.logger_default.warn(`No providers matched the filter "${filterOption}". Check your --filter-providers/--filter-targets value.`);
 	let parsedPrompts = await require_graders.readPrompts(config.prompts, cmdObj.prompts ? void 0 : basePath);
 	if (cmdObj.filterPrompts) {
@@ -7973,7 +8125,7 @@ async function resolveConfigs(cmdObj, _defaultConfig, type) {
 	}
 	const parsedProviderPromptMap = require_graders.readProviderPromptMap({ providers: filteredProviderConfigs }, parsedPrompts);
 	if (parsedPrompts.length === 0) {
-		require_logger.logger_default.error("No prompts found");
+		require_logger.logger_default.error("No prompts found. Add a `prompts:` entry to your config or pass --prompts path/to/prompt.txt.");
 		process$1.default.exit(1);
 	}
 	const defaultTest = {
@@ -9581,6 +9733,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 		seen.add(key);
 		return true;
 	});
+	const needsGoalExtraction = strategies.some((s) => require_providers.Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
 	await require_providers.validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
 	const redteamProvider = await require_providers.redteamProviderManager.getProvider({ provider });
@@ -9772,13 +9925,15 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) require_logger.logger_default.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
 				const testCasesWithMetadata = allPluginTests;
-				require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
-					const promptVar = testCase.vars?.[injectVar];
-					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
-					const policy = getPolicyText(testCase.metadata);
-					const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
-					testCase.metadata.goal = extractedGoal;
+				if (needsGoalExtraction) {
+					require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} tests from ${plugin.id}...`);
+					for (const testCase of testCasesWithMetadata) {
+						const promptVar = testCase.vars?.[injectVar];
+						const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
+						const policy = getPolicyText(testCase.metadata);
+						const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+						testCase.metadata.goal = extractedGoal;
+					}
 				}
 				testCases.push(...testCasesWithMetadata);
 			}
@@ -9813,13 +9968,15 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					...t.metadata || {}
 				}
 			}));
-			require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-			for (const testCase of testCasesWithMetadata) {
-				const promptVar = testCase.vars?.[injectVar];
-				const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
-				const policy = getPolicyText(testCase.metadata);
-				const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
-				testCase.metadata.goal = extractedGoal;
+			if (needsGoalExtraction) {
+				require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
+				for (const testCase of testCasesWithMetadata) {
+					const promptVar = testCase.vars?.[injectVar];
+					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
+					const policy = getPolicyText(testCase.metadata);
+					const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+					testCase.metadata.goal = extractedGoal;
+				}
 			}
 			testCases.push(...testCasesWithMetadata);
 			require_logger.logger_default.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
@@ -10842,17 +10999,7 @@ async function loadDefaultConfig(dir, configName = "promptfooconfig") {
 	if (configCache.has(cacheKey)) return configCache.get(cacheKey);
 	let defaultConfig = {};
 	let defaultConfigPath;
-	for (const ext of [
-		"yaml",
-		"yml",
-		"json",
-		"cjs",
-		"cts",
-		"js",
-		"mjs",
-		"mts",
-		"ts"
-	]) {
+	for (const ext of DEFAULT_CONFIG_EXTENSIONS) {
 		const configPath = path.default.join(dir, `${configName}.${ext}`);
 		const maybeConfig = await maybeReadConfig(configPath);
 		if (maybeConfig) {
@@ -10958,11 +11105,31 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker } = params;
+	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
 	const lines = [];
 	const completionType = isRedteam ? "Red team" : "Eval";
-	const completionMessage = writeToDatabase && shareableUrl ? `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}` : writeToDatabase && activelySharing ? `${chalk.default.green("✓")} ${completionType} complete` : writeToDatabase ? `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})` : `${chalk.default.green("✓")} ${completionType} complete`;
+	const wasAborted = targetErrorStatus != null;
+	let completionMessage;
+	if (wasAborted) {
+		completionMessage = `${chalk.default.red("✗")} ${completionType} aborted`;
+		if (writeToDatabase) completionMessage += ` (ID: ${chalk.default.cyan(evalId)})`;
+	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
+	else if (writeToDatabase) completionMessage = `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})`;
+	else completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
 	lines.push(completionMessage);
+	if (wasAborted && targetErrorStatus != null) {
+		lines.push("");
+		lines.push(chalk.default.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
+		lines.push(chalk.default.red(`  Target returned HTTP ${targetErrorStatus}`));
+		lines.push("");
+		lines.push(chalk.default.yellow("Possible causes:"));
+		lines.push(chalk.default.yellow("  • Invalid API key or authentication (401/403)"));
+		lines.push(chalk.default.yellow("  • Target endpoint does not exist (404)"));
+		lines.push(chalk.default.yellow("  • Server does not support the request (501)"));
+		lines.push("");
+		lines.push(chalk.default.cyan("To fix: Check your target configuration and credentials."));
+	}
 	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
 		lines.push("");
 		lines.push(`» View results: ${chalk.default.green.bold("promptfoo view")}`);
@@ -11205,6 +11372,21 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 	let testSuite = void 0;
 	let _basePath = void 0;
 	let commandLineOptions = void 0;
+	const configArgs = Array.isArray(cmdObj.config) ? cmdObj.config : typeof cmdObj.config === "string" ? [cmdObj.config] : [];
+	const uuidConfigArgs = configArgs.filter((configArg) => require_types.isUuid(configArg));
+	if (configArgs.length > 1 && uuidConfigArgs.length > 0) throw new Error("Cloud config UUID mode supports exactly one -c value. Use: promptfoo eval -c <cloud-config-uuid>");
+	if (configArgs.length === 1 && uuidConfigArgs.length === 1) {
+		const cloudConfigId = uuidConfigArgs[0];
+		if (cmdObj.watch) throw new Error("--watch is not supported when using a cloud config UUID with -c. Use a local config file path for watch mode.");
+		try {
+			defaultConfig = await require_providers.getEvalConfigFromCloud(cloudConfigId);
+		} catch (error) {
+			const reason = error instanceof Error ? error.message : String(error);
+			throw new Error(`Failed to load cloud eval config "${cloudConfigId}". ${reason}. Cloud UUID inputs do not fall back to local file paths. Check authentication and that the UUID exists.`);
+		}
+		cmdObj.config = void 0;
+		defaultConfigPath = void 0;
+	}
 	const runEvaluation = async (initialization) => {
 		const startTime = Date.now();
 		require_telemetry.telemetry_default.record("command_used", {
@@ -11228,7 +11410,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 						...defaultConfig,
 						...dirConfig
 					};
-				} else require_logger.logger_default.warn(`No configuration file found in directory: ${configPath}`);
+				} else require_logger.logger_default.warn(`No configuration file found in directory: ${configPath}. Looked for promptfooconfig.{${DEFAULT_CONFIG_EXTENSIONS.join(",")}}. Run "${promptfooCommand("init")}" or pass --config path/to/promptfooconfig.yaml.`);
 			}
 		}
 		const resumeRaw = cmdObj.resume;
@@ -11508,6 +11690,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const isRedteam = Boolean(config.redteam);
 		const duration = Math.round((Date.now() - startTime) / 1e3);
 		const tracker = require_providers.TokenUsageTracker.getInstance();
+		const targetErrorStatus = await evalRecord.findTargetErrorStatus();
 		const summaryLines = generateEvalSummary({
 			evalId: evalRecord.id,
 			isRedteam,
@@ -11523,7 +11706,8 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			errors,
 			duration,
 			maxConcurrency,
-			tracker
+			tracker,
+			targetErrorStatus
 		});
 		if (cmdObj.write && wantsToShare && !canShareEval) {
 			require_logger.logger_default.info(summaryLines[0]);
@@ -11578,7 +11762,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			if (initialization) {
 				const configPaths = (cmdObj.config || [defaultConfigPath]).filter(Boolean);
 				if (!configPaths.length) {
-					require_logger.logger_default.error("Could not locate config file(s) to watch");
+					require_logger.logger_default.error(`Could not locate config file(s) to watch. Pass --config path/to/promptfooconfig.yaml or run from a directory containing promptfooconfig.{${DEFAULT_CONFIG_EXTENSIONS.join(",")}}.`);
 					process.exitCode = 1;
 					return ret;
 				}
@@ -11736,6 +11920,7 @@ async function doRedteamRun(options) {
 	require_logger.logger_default.info("Generating test cases...");
 	const { maxConcurrency, ...passThroughOptions } = options;
 	let redteamConfig;
+	const generationStartTime = Date.now();
 	try {
 		redteamConfig = await doGenerateRedteam({
 			...passThroughOptions,
@@ -11759,6 +11944,7 @@ async function doRedteamRun(options) {
 		}
 		throw error;
 	}
+	const generationDurationMs = Date.now() - generationStartTime;
 	if (!redteamConfig || !fs.existsSync(redteamPath)) {
 		require_logger.logger_default.info("No test cases generated. Skipping scan.");
 		if (verboseToggleCleanup) verboseToggleCleanup();
@@ -11781,7 +11967,14 @@ async function doRedteamRun(options) {
 		abortSignal: options.abortSignal,
 		progressCallback: options.progressCallback
 	});
-	require_logger.logger_default.info(chalk.default.green("\nRed team scan complete!"));
+	if (evalResult && generationDurationMs >= 0) {
+		evalResult.setGenerationDurationMs(generationDurationMs);
+		if (evalResult.persisted) await evalResult.save();
+		const totalMs = evalResult.durationMs ?? 0;
+		const evalMs = evalResult.evaluationDurationMs ?? 0;
+		require_logger.logger_default.info(chalk.default.gray(`Total scan time: ${formatDuration(totalMs / 1e3)} (generation: ${formatDuration(generationDurationMs / 1e3)}, evaluation: ${formatDuration(evalMs / 1e3)})`));
+	}
+	if (evalResult ? await evalResult.findTargetErrorStatus() != null : false) {} else require_logger.logger_default.info(chalk.default.green("\nRed team scan complete!"));
 	if (!evalResult?.shared) if (options.liveRedteamConfig) require_logger.logger_default.info(chalk.default.blue(`To view the results, click the ${chalk.default.bold("View Report")} button or run ${chalk.default.bold(promptfooCommand("redteam report"))} on the command line.`));
 	else require_logger.logger_default.info(chalk.default.blue(`To view the results, run ${chalk.default.bold(promptfooCommand("redteam report"))}`));
 	require_logger.setLogCallback(null);