promptfoo 0.120.25 → 0.120.26

package/dist/src/{docker-Dx_FN1ag.js → docker-BuButc4D.js}

@@ -1,19 +1,19 @@
 #!/usr/bin/env node
 import "./chunk-DHDDz29n.js";
 import { C as getEnvString, o as logger_default } from "./logger-BotXmWKW.js";
-import "./fetch-BuP-gm0E.js";
-import "./cloud-lRosK6Nk.js";
-import "./types-CMDs7Kzu.js";
-import { a as fetchWithCache } from "./cache-Bg1Wpvzq.js";
-import "./util-BECSJ0T_.js";
-import "./esm-79BpbQzg.js";
-import "./pythonUtils-CUPlf1H0.js";
-import "./genaiTracer-foKtbVa2.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-D_cro0wd.js";
-import "./transform-gpzlPrwG.js";
-import "./openai-jNIREoAW.js";
-import "./util-CttPoePn.js";
-import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-BTbouQDz.js";
+import "./fetch-SRsE6Ctl.js";
+import "./cloud-BMbRVJFw.js";
+import "./types-t52w-XsS.js";
+import { a as fetchWithCache } from "./cache-mIszOnuz.js";
+import "./util-BSh4a_Q8.js";
+import "./esm-CYhseqj4.js";
+import "./pythonUtils-r1uBuA0n.js";
+import "./genaiTracer-CqNnnXrE.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DIywASPG.js";
+import "./transform-CoP2bJ7P.js";
+import "./openai-CT5fwbve.js";
+import "./util-CUEt0Vum.js";
+import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-BBJ6zmG3.js";
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -90,4 +90,4 @@ var DMREmbeddingProvider = class extends OpenAiEmbeddingProvider {
 
 //#endregion
 export { createDockerProvider };
-//# sourceMappingURL=docker-Dx_FN1ag.js.map
+//# sourceMappingURL=docker-BuButc4D.js.map

package/dist/src/{docker-zUAnbYTM.cjs → docker-Ckw-j7Rr.cjs}

@@ -1,16 +1,16 @@
 const require_logger = require('./logger-BdZ-IqBc.cjs');
 require('./esm-BQkx5roy.cjs');
 require('./pythonUtils-DZ6EbdY4.cjs');
-require('./types-Brw5kggg.cjs');
-require('./util-Dab5xp2E.cjs');
-require('./fetch-Dy2YML9G.cjs');
-const require_cache = require('./cache-S9MAoF9H.cjs');
+require('./types-BIfttHrT.cjs');
+require('./util-DUYOvxAy.cjs');
+require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
 require('./genaiTracer-CQlpZkrp.cjs');
-const require_chat = require('./chat-DTCPbuld.cjs');
-require('./transform-CCSNqTI-.cjs');
+const require_chat = require('./chat-C4zqjObh.cjs');
+require('./transform-ivxEY4f7.cjs');
 require('./openai-Bigwjgo1.cjs');
-require('./util-D8z5QOy8.cjs');
-const require_completion = require('./completion-DV0Ys99K.cjs');
+require('./util-C08Kns6-.cjs');
+const require_completion = require('./completion-DDyL3Cb2.cjs');
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -87,4 +87,4 @@ var DMREmbeddingProvider = class extends require_completion.OpenAiEmbeddingProvi
 
 //#endregion
 exports.createDockerProvider = createDockerProvider;
-//# sourceMappingURL=docker-zUAnbYTM.cjs.map
+//# sourceMappingURL=docker-Ckw-j7Rr.cjs.map

package/dist/src/{docker-BAYnlbAW.js → docker-JAAubMw3.js}

@@ -2,16 +2,16 @@ import "./chunk-FhC4c-0y.js";
 import { b as getEnvString, i as logger_default } from "./logger-w8Ozp0Td.js";
 import "./esm-rDtG_2rg.js";
 import "./pythonUtils-vMlk9Qp5.js";
-import "./types-CslP2TyG.js";
-import "./util-CY_XguB9.js";
-import "./fetch-zJuFPxN1.js";
-import { a as fetchWithCache } from "./cache-C4A4H4Xb.js";
+import "./types-DMVjYLpx.js";
+import "./util-DiCePfDu.js";
+import "./fetch-18MuNu9i.js";
+import { a as fetchWithCache } from "./cache-BLLayYEN.js";
 import "./genaiTracer-Ce19n68P.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-noXtEAh5.js";
-import "./transform-DosdxVdh.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-C8Ei4f87.js";
+import "./transform-DMaxQwDx.js";
 import "./openai-Dz3surb_.js";
-import "./util-CfPdWQJX.js";
-import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-HYrX5gEL.js";
+import "./util-Cl0zfT3V.js";
+import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-D9_MDlnd.js";
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -88,4 +88,4 @@ var DMREmbeddingProvider = class extends OpenAiEmbeddingProvider {
 
 //#endregion
 export { createDockerProvider };
-//# sourceMappingURL=docker-BAYnlbAW.js.map
+//# sourceMappingURL=docker-JAAubMw3.js.map

package/dist/src/errors-DnGCbnx8.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+//#region src/util/fetch/errors.ts
+/**
+* Non-transient HTTP status codes that indicate the target is unavailable or misconfigured.
+* These errors will not resolve on retry and should abort the scan immediately.
+*
+* - 401: Unauthorized - authentication required or invalid credentials
+* - 403: Forbidden - valid credentials but access denied
+* - 404: Not Found - target endpoint doesn't exist
+* - 501: Not Implemented - server doesn't support the request method
+*
+* Excluded: 500 (often transient — server crashes, DB timeouts, deployment rollouts,
+* or input-dependent bugs where one prompt triggers it but the next doesn't),
+* 502/503/504 (typically transient gateway issues).
+*/
+const NON_TRANSIENT_HTTP_STATUSES = [
+	401,
+	403,
+	404,
+	501
+];
+function isNonTransientHttpStatus(status) {
+	return NON_TRANSIENT_HTTP_STATUSES.includes(status);
+}
+function isTransientConnectionError(error) {
+	if (!error) return false;
+	const code = error.code;
+	if (code === "ECONNRESET" || code === "EPIPE") return true;
+	const message = (error.message ?? "").toLowerCase();
+	if (message.includes("eproto") && (message.includes("wrong version number") || message.includes("self signed") || message.includes("unable to verify") || message.includes("unknown ca") || message.includes("cert"))) return false;
+	return message.includes("bad record mac") || message.includes("eproto") || message.includes("econnreset") || message.includes("socket hang up");
+}
+
+//#endregion
+export { isNonTransientHttpStatus as n, isTransientConnectionError as r, NON_TRANSIENT_HTTP_STATUSES as t };
+//# sourceMappingURL=errors-DnGCbnx8.js.map

package/dist/src/{esm-79BpbQzg.js → esm-CYhseqj4.js}

@@ -380,4 +380,4 @@ function loadCjsModule(modulePath) {
 
 //#endregion
 export { safeJoin as a, resolvePackageEntryPoint as i, getWrapperDir as n, safeResolve as o, importModule as r, getDirectory as t };
-//# sourceMappingURL=esm-79BpbQzg.js.map
+//# sourceMappingURL=esm-CYhseqj4.js.map

package/dist/src/eval-CKHWqG9f.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./types-t52w-XsS.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import "./store-D4gdn9ih.js";
+import "./signal-Bl32q42d.js";
+import { a as createEvalId, i as combineFilterConditions, n as EvalQueries, o as escapeJsonPathKey, r as buildSafeJsonPath, s as getEvalSummaries, t as Eval } from "./eval-CYrbG57o.js";
+import "./evalResult-6JaUIStC.js";
+
+export { Eval as default };

package/dist/src/{eval-Cxb9alW8.js → eval-CYrbG57o.js}

@@ -1,16 +1,17 @@
 #!/usr/bin/env node
 import { b as getEnvBool, o as logger_default } from "./logger-BotXmWKW.js";
-import { I as HUMAN_ASSERTION_TYPE, T as DEFAULT_QUERY_LIMIT, c as getCurrentTimestamp } from "./fetch-BuP-gm0E.js";
+import { I as HUMAN_ASSERTION_TYPE, T as DEFAULT_QUERY_LIMIT, c as getCurrentTimestamp } from "./fetch-SRsE6Ctl.js";
 import { t as invariant } from "./invariant-DT20jrBd.js";
-import { o as getUserEmail } from "./accounts-Bzx8GnL6.js";
-import { U as riskCategorySeverityMap, ct as PLUGIN_CATEGORIES, s as ResultFailureReason } from "./types-CMDs7Kzu.js";
-import { t as hashPrompt } from "./utils-ICP7kLcY.js";
-import { a as sha256, i as randomSequence } from "./extractor-BFoThUOa.js";
-import { a as createEmptyTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-D2ko2eFQ.js";
-import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, i as datasetsTable, l as evalsToTagsTable, o as evalsTable, p as tagsTable, s as evalsToDatasetsTable } from "./tables-ChjAuVAR.js";
-import { n as getTraceStore } from "./store-rueRdzcm.js";
-import { i as getActualPrompt, r as updateSignalFile } from "./signal-D4VhSJOY.js";
-import { t as EvalResult } from "./evalResult-B8LbgWMA.js";
+import { o as getUserEmail } from "./accounts-DHHiXsy6.js";
+import { U as riskCategorySeverityMap, ct as PLUGIN_CATEGORIES, s as ResultFailureReason } from "./types-t52w-XsS.js";
+import { n as isNonTransientHttpStatus, t as NON_TRANSIENT_HTTP_STATUSES } from "./errors-DnGCbnx8.js";
+import { t as hashPrompt } from "./utils-JaY9veb5.js";
+import { a as sha256, i as randomSequence } from "./extractor-LSYjrhK0.js";
+import { a as createEmptyTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-DmZSD2eU.js";
+import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, i as datasetsTable, l as evalsToTagsTable, o as evalsTable, p as tagsTable, s as evalsToDatasetsTable } from "./tables-C7TT2XVn.js";
+import { n as getTraceStore } from "./store-D4gdn9ih.js";
+import { i as getActualPrompt, r as updateSignalFile } from "./signal-Bl32q42d.js";
+import { t as EvalResult } from "./evalResult-6JaUIStC.js";
 import { and, desc, eq, sql } from "drizzle-orm";
 
 //#region src/redteam/metrics.ts
@@ -407,7 +408,12 @@ function convertResultsToTable(eval_) {
 				model: result.response.video.model,
 				aspectRatio: result.response.video.aspectRatio,
 				resolution: result.response.video.resolution
-			} : void 0
+			} : void 0,
+			images: result.response?.images?.map((img) => ({
+				data: img.data,
+				blobRef: img.blobRef,
+				mimeType: img.mimeType
+			}))
 		};
 		invariant(result.promptId, "Prompt ID is required");
 		row.testIdx = result.testIdx;
@@ -473,7 +479,12 @@ function convertEvalResultToTableCell(result) {
 			model: result.response.video.model,
 			aspectRatio: result.response.video.aspectRatio,
 			resolution: result.response.video.resolution
-		} : void 0
+		} : void 0,
+		images: result.response?.images?.map((img) => ({
+			data: img.data,
+			blobRef: img.blobRef,
+			mimeType: img.mimeType
+		}))
 	};
 }
 function convertTestResultsToTableRow(results, varsForHeader) {
@@ -1031,6 +1042,27 @@ var Eval = class Eval {
 	async getTotalResultRowCount() {
 		return getTotalResultRowCount(this.id);
 	}
+	/**
+	* Find a non-transient HTTP error status from evaluation results.
+	* Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+	*
+	* For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+	* For non-persisted evals: Falls back to scanning in-memory results.
+	*/
+	async findTargetErrorStatus() {
+		const scanInMemory = () => {
+			for (const result of this.results) {
+				const status = result.response?.metadata?.http?.status;
+				if (typeof status === "number" && isNonTransientHttpStatus(status)) return status;
+			}
+		};
+		if (!this.persisted) return scanInMemory();
+		try {
+			return getDb().select({ httpStatus: sql`CAST(json_extract(${evalResultsTable.response}, '$.metadata.http.status') AS INTEGER)` }).from(evalResultsTable).where(and(eq(evalResultsTable.evalId, this.id), sql`json_extract(${evalResultsTable.response}, '$.metadata.http.status') IN (${sql.join(NON_TRANSIENT_HTTP_STATUSES.map((s) => sql`${s}`), sql`, `)})`)).limit(1).get()?.httpStatus ?? void 0;
+		} catch {
+			return scanInMemory();
+		}
+	}
 	async fetchResultsByTestIdx(testIdx) {
 		return await EvalResult.findManyByEvalId(this.id, { testIdx });
 	}
@@ -1594,4 +1626,4 @@ async function getEvalSummaries(datasetId, type, includeProviders = false) {
 
 //#endregion
 export { createEvalId as a, combineFilterConditions as i, EvalQueries as n, escapeJsonPathKey as o, buildSafeJsonPath as r, getEvalSummaries as s, Eval as t };
-//# sourceMappingURL=eval-Cxb9alW8.js.map
+//# sourceMappingURL=eval-CYrbG57o.js.map

package/dist/src/{evalResult-B8LbgWMA.js → evalResult-6JaUIStC.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import { b as getEnvBool, g as safeJsonStringify, o as logger_default } from "./logger-BotXmWKW.js";
-import { c as getCurrentTimestamp } from "./fetch-BuP-gm0E.js";
-import { f as isResultFailureReason, m as isProviderOptions, p as isApiProvider, s as ResultFailureReason } from "./types-CMDs7Kzu.js";
-import { t as hashPrompt } from "./utils-ICP7kLcY.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-BFoThUOa.js";
-import { a as evalResultsTable, g as getDb } from "./tables-ChjAuVAR.js";
+import { c as getCurrentTimestamp } from "./fetch-SRsE6Ctl.js";
+import { f as isResultFailureReason, m as isProviderOptions, p as isApiProvider, s as ResultFailureReason } from "./types-t52w-XsS.js";
+import { t as hashPrompt } from "./utils-JaY9veb5.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-LSYjrhK0.js";
+import { a as evalResultsTable, g as getDb } from "./tables-C7TT2XVn.js";
 import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
 
 //#region src/models/evalResult.ts
@@ -289,4 +289,4 @@ var EvalResult = class EvalResult {
 
 //#endregion
 export { sanitizeProvider as n, EvalResult as t };
-//# sourceMappingURL=evalResult-B8LbgWMA.js.map
+//# sourceMappingURL=evalResult-6JaUIStC.js.map

package/dist/src/{evalResult-CJNSpa3q.js → evalResult-COsVttMA.js}

@@ -1,9 +1,9 @@
 import { _ as getEnvBool, i as logger_default, m as safeJsonStringify } from "./logger-w8Ozp0Td.js";
-import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-CslP2TyG.js";
-import { f as getCurrentTimestamp } from "./fetch-zJuFPxN1.js";
-import { t as hashPrompt } from "./utils-BTeLHwiH.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-CuUWqwrE.js";
-import { i as evalResultsTable, p as getDb } from "./tables-dNaVPyDr.js";
+import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-DMVjYLpx.js";
+import { f as getCurrentTimestamp } from "./fetch-18MuNu9i.js";
+import { t as hashPrompt } from "./utils-CVzb4YiI.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-Hs7la_19.js";
+import { i as evalResultsTable, p as getDb } from "./tables-D-NSwNIb.js";
 import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
 
 //#region src/models/evalResult.ts
@@ -288,4 +288,4 @@ var EvalResult = class EvalResult {
 
 //#endregion
 export { sanitizeProvider as n, EvalResult as t };
-//# sourceMappingURL=evalResult-CJNSpa3q.js.map
+//# sourceMappingURL=evalResult-COsVttMA.js.map

package/dist/src/evalResult-CxTP-LMm.cjs

@@ -0,0 +1,11 @@
+require('./logger-BdZ-IqBc.cjs');
+require('./types-BIfttHrT.cjs');
+require('./fetch-ZMn_oemb.cjs');
+require('./utils-DFaZa6Rf.cjs');
+require('./extractor-DbhlYEeo.cjs');
+require('./accounts-BzEY8H3v.cjs');
+require('./blobs-673H0jCl.cjs');
+require('./tables-B9E1kRp-.cjs');
+const require_evalResult = require('./evalResult-DlRfu_Rq.cjs');
+
+exports.default = require_evalResult.EvalResult;

package/dist/src/evalResult-CzLURDcP.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./types-t52w-XsS.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import { n as sanitizeProvider, t as EvalResult } from "./evalResult-6JaUIStC.js";
+
+export { EvalResult as default };

package/dist/src/{evalResult-hI1mhi_g.cjs → evalResult-DlRfu_Rq.cjs}

@@ -1,9 +1,9 @@
 const require_logger = require('./logger-BdZ-IqBc.cjs');
-const require_types = require('./types-Brw5kggg.cjs');
-const require_fetch = require('./fetch-Dy2YML9G.cjs');
-const require_utils = require('./utils-Db5KwiXp.cjs');
-const require_extractor = require('./extractor-BPBlLyOn.cjs');
-const require_tables = require('./tables-w-r-abWq.cjs');
+const require_types = require('./types-BIfttHrT.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_utils = require('./utils-DFaZa6Rf.cjs');
+const require_extractor = require('./extractor-DbhlYEeo.cjs');
+const require_tables = require('./tables-B9E1kRp-.cjs');
 let drizzle_orm = require("drizzle-orm");
 
 //#region src/models/evalResult.ts
@@ -299,4 +299,4 @@ Object.defineProperty(exports, 'sanitizeProvider', {
     return sanitizeProvider;
   }
 });
-//# sourceMappingURL=evalResult-hI1mhi_g.cjs.map
+//# sourceMappingURL=evalResult-DlRfu_Rq.cjs.map

package/dist/src/evalResult-DyttNQ_G.js

@@ -0,0 +1,11 @@
+import "./logger-w8Ozp0Td.js";
+import "./types-DMVjYLpx.js";
+import "./fetch-18MuNu9i.js";
+import "./utils-CVzb4YiI.js";
+import "./extractor-Hs7la_19.js";
+import "./accounts-R3gfCR_g.js";
+import "./blobs-C9J2mVgo.js";
+import "./tables-D-NSwNIb.js";
+import { n as sanitizeProvider, t as EvalResult } from "./evalResult-COsVttMA.js";
+
+export { EvalResult as default };

package/dist/src/evaluator-0PvfeBYh.js

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+import "./chunk-DHDDz29n.js";
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./telemetry-BXyVqyAg.js";
+import "./types-t52w-XsS.js";
+import "./server-DWmZLfCy.js";
+import "./providers-BlEhY5mi.js";
+import "./cache-mIszOnuz.js";
+import "./util-BSh4a_Q8.js";
+import "./esm-CYhseqj4.js";
+import "./pythonUtils-r1uBuA0n.js";
+import "./transform-CqTFr7KR.js";
+import "./graders-BaMCwIKp.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./genaiTracer-CqNnnXrE.js";
+import "./chat-DIywASPG.js";
+import "./transform-CoP2bJ7P.js";
+import "./messages-D61tPFQo.js";
+import "./util-vjscpUzy.js";
+import "./responses-TsdODUpm.js";
+import "./openai-CT5fwbve.js";
+import "./util-CUEt0Vum.js";
+import "./completion-BBJ6zmG3.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import "./store-D4gdn9ih.js";
+import "./base-BaXmtXYp.js";
+import "./image-mhAGP07h.js";
+import "./providerRegistry-Xf0qdqGQ.js";
+import "./rubyUtils-BtjlqyXt.js";
+import { a as runEval, i as isAllowedPrompt, n as formatVarsForDisplay, r as generateVarCombinations, t as evaluate } from "./evaluator-3EJCMTs0.js";
+import "./signal-Bl32q42d.js";
+
+export { evaluate };