promptfoo 0.120.25 → 0.120.26

package/dist/src/graders-spkuVC-E.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+import "./chunk-DHDDz29n.js";
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./telemetry-BXyVqyAg.js";
+import "./types-t52w-XsS.js";
+import "./server-DWmZLfCy.js";
+import "./providers-BlEhY5mi.js";
+import "./cache-mIszOnuz.js";
+import "./util-BSh4a_Q8.js";
+import "./esm-CYhseqj4.js";
+import "./pythonUtils-r1uBuA0n.js";
+import "./transform-CqTFr7KR.js";
+import { n as getGraderById, t as GRADERS } from "./graders-BaMCwIKp.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./genaiTracer-CqNnnXrE.js";
+import "./chat-DIywASPG.js";
+import "./transform-CoP2bJ7P.js";
+import "./messages-D61tPFQo.js";
+import "./util-vjscpUzy.js";
+import "./responses-TsdODUpm.js";
+import "./openai-CT5fwbve.js";
+import "./util-CUEt0Vum.js";
+import "./completion-BBJ6zmG3.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import "./store-D4gdn9ih.js";
+import "./base-BaXmtXYp.js";
+import "./image-mhAGP07h.js";
+import "./providerRegistry-Xf0qdqGQ.js";
+import "./rubyUtils-BtjlqyXt.js";
+
+export { getGraderById };

package/dist/src/{image-Bf8RiJ2g.js → image-BXt_7u0v.js}

@@ -1,7 +1,7 @@
 import "./chunk-FhC4c-0y.js";
 import { b as getEnvString, i as logger_default } from "./logger-w8Ozp0Td.js";
-import { h as REQUEST_TIMEOUT_MS } from "./fetch-zJuFPxN1.js";
-import { a as fetchWithCache } from "./cache-C4A4H4Xb.js";
+import { h as REQUEST_TIMEOUT_MS } from "./fetch-18MuNu9i.js";
+import { a as fetchWithCache } from "./cache-BLLayYEN.js";
 
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
@@ -162,4 +162,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 
 //#endregion
 export { createHyperbolicImageProvider };
-//# sourceMappingURL=image-Bf8RiJ2g.js.map
+//# sourceMappingURL=image-BXt_7u0v.js.map

package/dist/src/{image-LhLO5KCW.js → image-BiEVdpdP.js}

@@ -1,8 +1,8 @@
 import { i as logger_default } from "./logger-w8Ozp0Td.js";
-import { h as REQUEST_TIMEOUT_MS } from "./fetch-zJuFPxN1.js";
-import { a as fetchWithCache } from "./cache-C4A4H4Xb.js";
+import { h as REQUEST_TIMEOUT_MS } from "./fetch-18MuNu9i.js";
+import { a as fetchWithCache } from "./cache-BLLayYEN.js";
 import { t as OpenAiGenericProvider } from "./openai-Dz3surb_.js";
-import { s as formatOpenAiError } from "./util-CfPdWQJX.js";
+import { s as formatOpenAiError } from "./util-Cl0zfT3V.js";
 import { t as ellipsize } from "./text-Dm78AVGG.js";
 
 //#region src/providers/openai/image.ts
@@ -215,4 +215,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 
 //#endregion
 export { callOpenAiImageApi as n, formatOutput as r, OpenAiImageProvider as t };
-//# sourceMappingURL=image-LhLO5KCW.js.map
+//# sourceMappingURL=image-BiEVdpdP.js.map

package/dist/src/{image-CaVWfZub.cjs → image-C3BjJUAU.cjs}

@@ -1,8 +1,8 @@
 const require_logger = require('./logger-BdZ-IqBc.cjs');
-const require_fetch = require('./fetch-Dy2YML9G.cjs');
-const require_cache = require('./cache-S9MAoF9H.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
 const require_openai = require('./openai-Bigwjgo1.cjs');
-const require_util = require('./util-D8z5QOy8.cjs');
+const require_util = require('./util-C08Kns6-.cjs');
 const require_text = require('./text-DF2hMKdg.cjs');
 
 //#region src/providers/openai/image.ts
@@ -232,4 +232,4 @@ Object.defineProperty(exports, 'formatOutput', {
     return formatOutput;
   }
 });
-//# sourceMappingURL=image-CaVWfZub.cjs.map
+//# sourceMappingURL=image-C3BjJUAU.cjs.map

package/dist/src/{image-BnF25Osq.cjs → image-D10zEe1f.cjs}

@@ -1,6 +1,6 @@
 const require_logger = require('./logger-BdZ-IqBc.cjs');
-const require_fetch = require('./fetch-Dy2YML9G.cjs');
-const require_cache = require('./cache-S9MAoF9H.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
 
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
@@ -161,4 +161,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 
 //#endregion
 exports.createHyperbolicImageProvider = createHyperbolicImageProvider;
-//# sourceMappingURL=image-BnF25Osq.cjs.map
+//# sourceMappingURL=image-D10zEe1f.cjs.map

package/dist/src/{image-DIn_AW1Z.js → image-DB4sHxdJ.js}

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 import "./chunk-DHDDz29n.js";
 import { C as getEnvString, o as logger_default } from "./logger-BotXmWKW.js";
-import { d as REQUEST_TIMEOUT_MS } from "./fetch-BuP-gm0E.js";
-import "./cloud-lRosK6Nk.js";
-import { a as fetchWithCache } from "./cache-Bg1Wpvzq.js";
+import { d as REQUEST_TIMEOUT_MS } from "./fetch-SRsE6Ctl.js";
+import "./cloud-BMbRVJFw.js";
+import { a as fetchWithCache } from "./cache-mIszOnuz.js";
 
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
@@ -164,4 +164,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 
 //#endregion
 export { createHyperbolicImageProvider };
-//# sourceMappingURL=image-DIn_AW1Z.js.map
+//# sourceMappingURL=image-DB4sHxdJ.js.map

package/dist/src/{image-DFwp1aAV.js → image-mhAGP07h.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { o as logger_default } from "./logger-BotXmWKW.js";
-import { d as REQUEST_TIMEOUT_MS } from "./fetch-BuP-gm0E.js";
-import { a as fetchWithCache } from "./cache-Bg1Wpvzq.js";
-import { t as OpenAiGenericProvider } from "./openai-jNIREoAW.js";
-import { s as formatOpenAiError } from "./util-CttPoePn.js";
-import { t as ellipsize } from "./text-C1cXk9ij.js";
+import { d as REQUEST_TIMEOUT_MS } from "./fetch-SRsE6Ctl.js";
+import { a as fetchWithCache } from "./cache-mIszOnuz.js";
+import { t as OpenAiGenericProvider } from "./openai-CT5fwbve.js";
+import { s as formatOpenAiError } from "./util-CUEt0Vum.js";
+import { t as ellipsize } from "./text-DgMr_tiM.js";
 
 //#region src/providers/openai/image.ts
 const DALLE2_VALID_SIZES = [
@@ -216,4 +216,4 @@ var OpenAiImageProvider = class extends OpenAiGenericProvider {
 
 //#endregion
 export { callOpenAiImageApi as n, formatOutput as r, OpenAiImageProvider as t };
-//# sourceMappingURL=image-DFwp1aAV.js.map
+//# sourceMappingURL=image-mhAGP07h.js.map

package/dist/src/index.cjs

@@ -5,36 +5,36 @@ const require_esm = require('./esm-BQkx5roy.cjs');
 const require_pythonUtils = require('./pythonUtils-DZ6EbdY4.cjs');
 const require_fileExtensions = require('./fileExtensions-DkJYkWUy.cjs');
 const require_transform = require('./transform-Kd6u-oNm.cjs');
-const require_graders = require('./graders-NBsuYeMx.cjs');
-const require_types = require('./types-Brw5kggg.cjs');
-const require_util = require('./util-Dab5xp2E.cjs');
-const require_fetch = require('./fetch-Dy2YML9G.cjs');
-const require_cache = require('./cache-S9MAoF9H.cjs');
-const require_providers = require('./providers-CpUrPa8s.cjs');
-const require_utils = require('./utils-Db5KwiXp.cjs');
-const require_extractor = require('./extractor-BPBlLyOn.cjs');
+const require_graders = require('./graders-DzUUnUjC.cjs');
+const require_types = require('./types-BIfttHrT.cjs');
+const require_util = require('./util-DUYOvxAy.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
+const require_providers = require('./providers-BNKVY53V.cjs');
+const require_utils = require('./utils-DFaZa6Rf.cjs');
+const require_extractor = require('./extractor-DbhlYEeo.cjs');
 require('./genaiTracer-CQlpZkrp.cjs');
-const require_chat = require('./chat-DTCPbuld.cjs');
+const require_chat = require('./chat-C4zqjObh.cjs');
 const require_tokenUsageUtils = require('./tokenUsageUtils-CXhxVj72.cjs');
-const require_transform$1 = require('./transform-CCSNqTI-.cjs');
-require('./messages-16jb9LP7.cjs');
-require('./util-CBAXTxfy.cjs');
-require('./responses-CXJACENh.cjs');
+const require_transform$1 = require('./transform-ivxEY4f7.cjs');
+require('./messages-DXV3Qh8_.cjs');
+require('./util-DkFTvieG.cjs');
+require('./responses-zOtKtnY_.cjs');
 require('./openai-Bigwjgo1.cjs');
-const require_util$2 = require('./util-D8z5QOy8.cjs');
-require('./completion-DV0Ys99K.cjs');
-const require_accounts = require('./accounts-CwqY22lU.cjs');
-const require_server = require('./server-C_O6-VE4.cjs');
-const require_blobs = require('./blobs-_kNRY6vU.cjs');
-const require_tables = require('./tables-w-r-abWq.cjs');
-const require_telemetry = require('./telemetry-Y1esJM0v.cjs');
+const require_util$2 = require('./util-C08Kns6-.cjs');
+require('./completion-DDyL3Cb2.cjs');
+const require_accounts = require('./accounts-BzEY8H3v.cjs');
+const require_server = require('./server-CgUQ25qW.cjs');
+const require_blobs = require('./blobs-673H0jCl.cjs');
+const require_tables = require('./tables-B9E1kRp-.cjs');
+const require_telemetry = require('./telemetry-D0_yFdtU.cjs');
 const require_text = require('./text-DF2hMKdg.cjs');
-const require_store = require('./store-F-KTiqfb.cjs');
-require('./base-D8VQYhNQ.cjs');
-require('./image-CaVWfZub.cjs');
+const require_store = require('./store-BqwfFEyF.cjs');
+require('./base-f71xxWai.cjs');
+require('./image-C3BjJUAU.cjs');
 const require_providerRegistry = require('./providerRegistry-lc7a7utN.cjs');
 const require_rubyUtils = require('./rubyUtils-DCVaJ3mc.cjs');
-const require_evalResult = require('./evalResult-hI1mhi_g.cjs');
+const require_evalResult = require('./evalResult-DlRfu_Rq.cjs');
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -307,7 +307,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		require_telemetry.telemetry_default.record("feature_used", { feature: "tracing" });
 		try {
 			require_logger.logger_default.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-th2zHwjZ.cjs"));
+			const { startOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-B6Xo4KZM.cjs"));
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			require_logger.logger_default.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -330,7 +330,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		require_logger.logger_default.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-th2zHwjZ.cjs"));
+		const { stopOTLPReceiver } = await Promise.resolve().then(() => require("./otlpReceiver-B6Xo4KZM.cjs"));
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		require_logger.logger_default.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -365,7 +365,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	require_logger.logger_default.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await Promise.resolve().then(() => require("./store-awdDxuiD.cjs"));
+	const { getTraceStore } = await Promise.resolve().then(() => require("./store-m5KT6Ly7.cjs"));
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -2886,9 +2886,9 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 		let filePath = fileRef;
 		let functionName;
 		if (fileRef.includes(":")) {
-			const [pathPart, funcPart] = fileRef.split(":");
-			filePath = pathPart;
-			functionName = funcPart;
+			const colonIndex = fileRef.indexOf(":");
+			filePath = fileRef.slice(0, colonIndex);
+			functionName = fileRef.slice(colonIndex + 1);
 		}
 		filePath = path.default.resolve(basePath, filePath);
 		if (require_fileExtensions.isJavascriptFile(filePath)) {
@@ -3914,7 +3914,7 @@ async function runEval({ provider, prompt, test, testSuite, delay, nunjucksFilte
 			promptIdx,
 			testIdx
 		});
-		require_logger.logger_default.error("Provider call failed during eval", logContext);
+		if (!(err instanceof Error && err.name === "AbortError")) require_logger.logger_default.error("Provider call failed during eval", logContext);
 		return [{
 			...setup,
 			error: errorWithStack,
@@ -4095,11 +4095,17 @@ var Evaluator = class {
 		let globalTimeout;
 		let globalAbortController;
 		const processedIndices = /* @__PURE__ */ new Set();
+		let targetUnavailable = false;
+		let targetErrorStatus;
+		const targetErrorAbortController = new AbortController();
 		let ciProgressReporter = null;
 		let progressBarManager = null;
+		let providerAbortSignal = options.abortSignal;
+		let combinedAbortSignal = options.abortSignal ? AbortSignal.any([options.abortSignal, targetErrorAbortController.signal]) : targetErrorAbortController.signal;
 		if (maxEvalTimeMs > 0) {
 			globalAbortController = new AbortController();
-			options.abortSignal = options.abortSignal ? AbortSignal.any([options.abortSignal, globalAbortController.signal]) : globalAbortController.signal;
+			providerAbortSignal = providerAbortSignal ? AbortSignal.any([providerAbortSignal, globalAbortController.signal]) : globalAbortController.signal;
+			combinedAbortSignal = AbortSignal.any([combinedAbortSignal, globalAbortController.signal]);
 			globalTimeout = setTimeout(() => {
 				evalTimedOut = true;
 				globalAbortController?.abort();
@@ -4107,7 +4113,7 @@ var Evaluator = class {
 		}
 		const vars = /* @__PURE__ */ new Set();
 		const checkAbort = () => {
-			if (options.abortSignal?.aborted) throw new Error("Operation cancelled");
+			if (combinedAbortSignal.aborted) throw new Error("Operation cancelled");
 		};
 		if (!options.silent) require_logger.logger_default.info(`Starting evaluation ${this.evalRecord.id}`);
 		checkAbort();
@@ -4268,7 +4274,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (require_types.isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-wpMPsMZ7.cjs"));
+					const { loadApiProvider } = await Promise.resolve().then(() => require("./providers-BMZZmPBJ.cjs"));
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4338,7 +4344,7 @@ var Evaluator = class {
 							registers: this.registers,
 							isRedteam: testSuite.redteam != null,
 							concurrency,
-							abortSignal: options.abortSignal,
+							abortSignal: providerAbortSignal,
 							evalId: this.evalRecord.id,
 							rateLimitRegistry: this.rateLimitRegistry
 						});
@@ -4352,7 +4358,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (require_logger.cliState_default.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-DzRxYLDH.cjs"));
+			const { default: EvalResult } = await Promise.resolve().then(() => require("./evalResult-CxTP-LMm.cjs"));
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: require_logger.cliState_default.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -4405,6 +4411,14 @@ var Evaluator = class {
 					require_logger.logger_default.error(`Error saving result: ${error} ${require_logger.safeJsonStringify(resultSummary)}`);
 				}
 				for (const writer of this.fileWriters) await writer.write(row);
+				const httpStatus = row.response?.metadata?.http?.status;
+				if (typeof httpStatus === "number" && require_cache.isNonTransientHttpStatus(httpStatus)) {
+					targetUnavailable = true;
+					targetErrorStatus = httpStatus;
+					require_logger.logger_default.error(`Target returned HTTP ${httpStatus}. Aborting scan - this error will not resolve on retry.`);
+					targetErrorAbortController.abort();
+					break;
+				}
 				const { promptIdx } = row;
 				const metrics = prompts[promptIdx].metrics;
 				require_invariant.invariant(metrics, "Expected prompt.metrics to be set");
@@ -4568,6 +4582,7 @@ var Evaluator = class {
 		if (this.options.showProgressBar && progressBarManager) await progressBarManager.initialize(runEvalOptions, concurrency, 0);
 		try {
 			if (serialRunEvalOptions.length > 0) for (const evalStep of serialRunEvalOptions) {
+				checkAbort();
 				if (isWebUI) {
 					const provider = evalStep.provider.label || evalStep.provider.id();
 					const vars = formatVarsForDisplay(evalStep.test.vars || {}, 50);
@@ -4585,22 +4600,32 @@ var Evaluator = class {
 				await this.evalRecord.addPrompts(prompts);
 			});
 		} catch (err) {
-			if (options.abortSignal?.aborted) if (evalTimedOut) require_logger.logger_default.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
-			else {
-				require_logger.logger_default.info("Evaluation interrupted, saving progress...");
-				if (globalTimeout) clearTimeout(globalTimeout);
-				if (progressBarManager) progressBarManager.stop();
-				if (ciProgressReporter) ciProgressReporter.finish();
-				this.evalRecord.setVars(Array.from(vars));
-				await this.evalRecord.addPrompts(prompts);
-				updateSignalFile(this.evalRecord.id);
-				return this.evalRecord;
-			}
-			else {
+			if (combinedAbortSignal.aborted) {
+				if (evalTimedOut) require_logger.logger_default.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
+				else if (!targetUnavailable) {
+					require_logger.logger_default.info("Evaluation interrupted, saving progress...");
+					if (globalTimeout) clearTimeout(globalTimeout);
+					if (progressBarManager) progressBarManager.stop();
+					if (ciProgressReporter) ciProgressReporter.finish();
+					this.evalRecord.setVars(Array.from(vars));
+					await this.evalRecord.addPrompts(prompts);
+					updateSignalFile(this.evalRecord.id);
+					return this.evalRecord;
+				}
+			} else {
 				if (ciProgressReporter) ciProgressReporter.error(`Evaluation failed: ${String(err)}`);
 				throw err;
 			}
 		}
+		if (targetUnavailable) {
+			if (globalTimeout) clearTimeout(globalTimeout);
+			if (progressBarManager) progressBarManager.stop();
+			if (ciProgressReporter) ciProgressReporter.error(`Target unavailable (HTTP ${targetErrorStatus})`);
+			this.evalRecord.setVars(Array.from(vars));
+			await this.evalRecord.addPrompts(prompts);
+			updateSignalFile(this.evalRecord.id);
+			return this.evalRecord;
+		}
 		const compareRowsCount = rowsWithSelectBestAssertion.size + rowsWithMaxScoreAssertion.size;
 		if (progressBarManager) {
 			if (compareRowsCount > 0) progressBarManager.updateTotalCount(compareRowsCount);
@@ -5373,7 +5398,12 @@ function convertResultsToTable(eval_) {
 				model: result.response.video.model,
 				aspectRatio: result.response.video.aspectRatio,
 				resolution: result.response.video.resolution
-			} : void 0
+			} : void 0,
+			images: result.response?.images?.map((img) => ({
+				data: img.data,
+				blobRef: img.blobRef,
+				mimeType: img.mimeType
+			}))
 		};
 		require_invariant.invariant(result.promptId, "Prompt ID is required");
 		row.testIdx = result.testIdx;
@@ -5439,7 +5469,12 @@ function convertEvalResultToTableCell(result) {
 			model: result.response.video.model,
 			aspectRatio: result.response.video.aspectRatio,
 			resolution: result.response.video.resolution
-		} : void 0
+		} : void 0,
+		images: result.response?.images?.map((img) => ({
+			data: img.data,
+			blobRef: img.blobRef,
+			mimeType: img.mimeType
+		}))
 	};
 }
 function convertTestResultsToTableRow(results, varsForHeader) {
@@ -5997,6 +6032,27 @@ var Eval = class Eval {
 	async getTotalResultRowCount() {
 		return getTotalResultRowCount(this.id);
 	}
+	/**
+	* Find a non-transient HTTP error status from evaluation results.
+	* Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+	*
+	* For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+	* For non-persisted evals: Falls back to scanning in-memory results.
+	*/
+	async findTargetErrorStatus() {
+		const scanInMemory = () => {
+			for (const result of this.results) {
+				const status = result.response?.metadata?.http?.status;
+				if (typeof status === "number" && require_cache.isNonTransientHttpStatus(status)) return status;
+			}
+		};
+		if (!this.persisted) return scanInMemory();
+		try {
+			return require_tables.getDb().select({ httpStatus: drizzle_orm.sql`CAST(json_extract(${require_tables.evalResultsTable.response}, '$.metadata.http.status') AS INTEGER)` }).from(require_tables.evalResultsTable).where((0, drizzle_orm.and)((0, drizzle_orm.eq)(require_tables.evalResultsTable.evalId, this.id), drizzle_orm.sql`json_extract(${require_tables.evalResultsTable.response}, '$.metadata.http.status') IN (${drizzle_orm.sql.join(require_cache.NON_TRANSIENT_HTTP_STATUSES.map((s) => drizzle_orm.sql`${s}`), drizzle_orm.sql`, `)})`)).limit(1).get()?.httpStatus ?? void 0;
+		} catch {
+			return scanInMemory();
+		}
+	}
 	async fetchResultsByTestIdx(testIdx) {
 		return await require_evalResult.EvalResult.findManyByEvalId(this.id, { testIdx });
 	}
@@ -9677,6 +9733,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 		seen.add(key);
 		return true;
 	});
+	const needsGoalExtraction = strategies.some((s) => require_providers.Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
 	await require_providers.validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
 	const redteamProvider = await require_providers.redteamProviderManager.getProvider({ provider });
@@ -9868,13 +9925,15 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) require_logger.logger_default.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
 				const testCasesWithMetadata = allPluginTests;
-				require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
-					const promptVar = testCase.vars?.[injectVar];
-					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
-					const policy = getPolicyText(testCase.metadata);
-					const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
-					testCase.metadata.goal = extractedGoal;
+				if (needsGoalExtraction) {
+					require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} tests from ${plugin.id}...`);
+					for (const testCase of testCasesWithMetadata) {
+						const promptVar = testCase.vars?.[injectVar];
+						const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
+						const policy = getPolicyText(testCase.metadata);
+						const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+						testCase.metadata.goal = extractedGoal;
+					}
 				}
 				testCases.push(...testCasesWithMetadata);
 			}
@@ -9909,13 +9968,15 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					...t.metadata || {}
 				}
 			}));
-			require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-			for (const testCase of testCasesWithMetadata) {
-				const promptVar = testCase.vars?.[injectVar];
-				const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
-				const policy = getPolicyText(testCase.metadata);
-				const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
-				testCase.metadata.goal = extractedGoal;
+			if (needsGoalExtraction) {
+				require_logger.logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
+				for (const testCase of testCasesWithMetadata) {
+					const promptVar = testCase.vars?.[injectVar];
+					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
+					const policy = getPolicyText(testCase.metadata);
+					const extractedGoal = await require_providers.extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+					testCase.metadata.goal = extractedGoal;
+				}
 			}
 			testCases.push(...testCasesWithMetadata);
 			require_logger.logger_default.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
@@ -11044,11 +11105,31 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker } = params;
+	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
 	const lines = [];
 	const completionType = isRedteam ? "Red team" : "Eval";
-	const completionMessage = writeToDatabase && shareableUrl ? `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}` : writeToDatabase && activelySharing ? `${chalk.default.green("✓")} ${completionType} complete` : writeToDatabase ? `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})` : `${chalk.default.green("✓")} ${completionType} complete`;
+	const wasAborted = targetErrorStatus != null;
+	let completionMessage;
+	if (wasAborted) {
+		completionMessage = `${chalk.default.red("✗")} ${completionType} aborted`;
+		if (writeToDatabase) completionMessage += ` (ID: ${chalk.default.cyan(evalId)})`;
+	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.default.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
+	else if (writeToDatabase) completionMessage = `${chalk.default.green("✓")} ${completionType} complete (ID: ${chalk.default.cyan(evalId)})`;
+	else completionMessage = `${chalk.default.green("✓")} ${completionType} complete`;
 	lines.push(completionMessage);
+	if (wasAborted && targetErrorStatus != null) {
+		lines.push("");
+		lines.push(chalk.default.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
+		lines.push(chalk.default.red(`  Target returned HTTP ${targetErrorStatus}`));
+		lines.push("");
+		lines.push(chalk.default.yellow("Possible causes:"));
+		lines.push(chalk.default.yellow("  • Invalid API key or authentication (401/403)"));
+		lines.push(chalk.default.yellow("  • Target endpoint does not exist (404)"));
+		lines.push(chalk.default.yellow("  • Server does not support the request (501)"));
+		lines.push("");
+		lines.push(chalk.default.cyan("To fix: Check your target configuration and credentials."));
+	}
 	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
 		lines.push("");
 		lines.push(`» View results: ${chalk.default.green.bold("promptfoo view")}`);
@@ -11609,6 +11690,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const isRedteam = Boolean(config.redteam);
 		const duration = Math.round((Date.now() - startTime) / 1e3);
 		const tracker = require_providers.TokenUsageTracker.getInstance();
+		const targetErrorStatus = await evalRecord.findTargetErrorStatus();
 		const summaryLines = generateEvalSummary({
 			evalId: evalRecord.id,
 			isRedteam,
@@ -11624,7 +11706,8 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			errors,
 			duration,
 			maxConcurrency,
-			tracker
+			tracker,
+			targetErrorStatus
 		});
 		if (cmdObj.write && wantsToShare && !canShareEval) {
 			require_logger.logger_default.info(summaryLines[0]);
@@ -11679,7 +11762,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			if (initialization) {
 				const configPaths = (cmdObj.config || [defaultConfigPath]).filter(Boolean);
 				if (!configPaths.length) {
-					require_logger.logger_default.error("Could not locate config file(s) to watch");
+					require_logger.logger_default.error(`Could not locate config file(s) to watch. Pass --config path/to/promptfooconfig.yaml or run from a directory containing promptfooconfig.{${DEFAULT_CONFIG_EXTENSIONS.join(",")}}.`);
 					process.exitCode = 1;
 					return ret;
 				}
@@ -11891,7 +11974,7 @@ async function doRedteamRun(options) {
 		const evalMs = evalResult.evaluationDurationMs ?? 0;
 		require_logger.logger_default.info(chalk.default.gray(`Total scan time: ${formatDuration(totalMs / 1e3)} (generation: ${formatDuration(generationDurationMs / 1e3)}, evaluation: ${formatDuration(evalMs / 1e3)})`));
 	}
-	require_logger.logger_default.info(chalk.default.green("\nRed team scan complete!"));
+	if (evalResult ? await evalResult.findTargetErrorStatus() != null : false) {} else require_logger.logger_default.info(chalk.default.green("\nRed team scan complete!"));
 	if (!evalResult?.shared) if (options.liveRedteamConfig) require_logger.logger_default.info(chalk.default.blue(`To view the results, click the ${chalk.default.bold("View Report")} button or run ${chalk.default.bold(promptfooCommand("redteam report"))} on the command line.`));
 	else require_logger.logger_default.info(chalk.default.blue(`To view the results, run ${chalk.default.bold(promptfooCommand("redteam report"))}`));
 	require_logger.setLogCallback(null);