promptfoo 0.120.24 → 0.120.26

package/dist/src/{completion-D6BTVt6e.cjs → completion-DDyL3Cb2.cjs}

@@ -1,8 +1,8 @@
-const require_logger = require('./logger-CMp-NS-e.cjs');
-const require_fetch = require('./fetch-MvWxCS5U.cjs');
-const require_cache = require('./cache-KNsTJLKP.cjs');
-const require_openai = require('./openai-BjpdxIOG.cjs');
-const require_util = require('./util-DTJWKLkl.cjs');
+const require_logger = require('./logger-BdZ-IqBc.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
+const require_openai = require('./openai-Bigwjgo1.cjs');
+const require_util = require('./util-C08Kns6-.cjs');
 
 //#region src/providers/openai/embedding.ts
 var OpenAiEmbeddingProvider = class extends require_openai.OpenAiGenericProvider {
@@ -130,4 +130,4 @@ Object.defineProperty(exports, 'OpenAiEmbeddingProvider', {
     return OpenAiEmbeddingProvider;
   }
 });
-//# sourceMappingURL=completion-D6BTVt6e.cjs.map
+//# sourceMappingURL=completion-DDyL3Cb2.cjs.map

package/dist/src/{docker-CkOzlD1W.js → docker-BuButc4D.js}

@@ -1,19 +1,19 @@
 #!/usr/bin/env node
 import "./chunk-DHDDz29n.js";
-import { C as getEnvString, o as logger_default } from "./logger-Bzi5o47S.js";
-import "./fetch-Mb7SbeM-.js";
-import "./cloud-ByJVRSQI.js";
-import "./types-CgrxBFgm.js";
-import { a as fetchWithCache } from "./cache-DHm8Pc8l.js";
-import "./util-M2MoNCUR.js";
-import "./esm-Lh6czAFT.js";
-import "./pythonUtils-B9JA-gsC.js";
-import "./genaiTracer-foKtbVa2.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CVOyOBYk.js";
-import "./transform-Bn-lgBE2.js";
-import "./openai-Cv9pEKxp.js";
-import "./util-DHBpsbZE.js";
-import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-BLe7SUA4.js";
+import { C as getEnvString, o as logger_default } from "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./cloud-BMbRVJFw.js";
+import "./types-t52w-XsS.js";
+import { a as fetchWithCache } from "./cache-mIszOnuz.js";
+import "./util-BSh4a_Q8.js";
+import "./esm-CYhseqj4.js";
+import "./pythonUtils-r1uBuA0n.js";
+import "./genaiTracer-CqNnnXrE.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-DIywASPG.js";
+import "./transform-CoP2bJ7P.js";
+import "./openai-CT5fwbve.js";
+import "./util-CUEt0Vum.js";
+import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-BBJ6zmG3.js";
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -90,4 +90,4 @@ var DMREmbeddingProvider = class extends OpenAiEmbeddingProvider {
 
 //#endregion
 export { createDockerProvider };
-//# sourceMappingURL=docker-CkOzlD1W.js.map
+//# sourceMappingURL=docker-BuButc4D.js.map

package/dist/src/{docker-BWsSX8Pn.cjs → docker-Ckw-j7Rr.cjs}

@@ -1,16 +1,16 @@
-const require_logger = require('./logger-CMp-NS-e.cjs');
-require('./esm-gWVPXn3r.cjs');
-require('./pythonUtils-rOCm9w_5.cjs');
-require('./types-B-XUqfNs.cjs');
-require('./util-C7A-PlKK.cjs');
-require('./fetch-MvWxCS5U.cjs');
-const require_cache = require('./cache-KNsTJLKP.cjs');
-require('./genaiTracer-BQ8AmajD.cjs');
-const require_chat = require('./chat-DOQIJx7O.cjs');
-require('./transform-Bw1IstDE.cjs');
-require('./openai-BjpdxIOG.cjs');
-require('./util-DTJWKLkl.cjs');
-const require_completion = require('./completion-D6BTVt6e.cjs');
+const require_logger = require('./logger-BdZ-IqBc.cjs');
+require('./esm-BQkx5roy.cjs');
+require('./pythonUtils-DZ6EbdY4.cjs');
+require('./types-BIfttHrT.cjs');
+require('./util-DUYOvxAy.cjs');
+require('./fetch-ZMn_oemb.cjs');
+const require_cache = require('./cache-7xULbvt3.cjs');
+require('./genaiTracer-CQlpZkrp.cjs');
+const require_chat = require('./chat-C4zqjObh.cjs');
+require('./transform-ivxEY4f7.cjs');
+require('./openai-Bigwjgo1.cjs');
+require('./util-C08Kns6-.cjs');
+const require_completion = require('./completion-DDyL3Cb2.cjs');
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -87,4 +87,4 @@ var DMREmbeddingProvider = class extends require_completion.OpenAiEmbeddingProvi
 
 //#endregion
 exports.createDockerProvider = createDockerProvider;
-//# sourceMappingURL=docker-BWsSX8Pn.cjs.map
+//# sourceMappingURL=docker-Ckw-j7Rr.cjs.map

package/dist/src/{docker-CMf5SGLq.js → docker-JAAubMw3.js}

@@ -1,17 +1,17 @@
 import "./chunk-FhC4c-0y.js";
-import { b as getEnvString, i as logger_default } from "./logger-q5I8CByj.js";
-import "./esm-D40XsWlR.js";
-import "./pythonUtils-eNq6Wsfr.js";
-import "./types-BsU_PxR3.js";
-import "./util-WYC3rB_p.js";
-import "./fetch-BIfcW-Fd.js";
-import { a as fetchWithCache } from "./cache-BMl3AIo5.js";
+import { b as getEnvString, i as logger_default } from "./logger-w8Ozp0Td.js";
+import "./esm-rDtG_2rg.js";
+import "./pythonUtils-vMlk9Qp5.js";
+import "./types-DMVjYLpx.js";
+import "./util-DiCePfDu.js";
+import "./fetch-18MuNu9i.js";
+import { a as fetchWithCache } from "./cache-BLLayYEN.js";
 import "./genaiTracer-Ce19n68P.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-DDGU7Tik.js";
-import "./transform-BUrxadlA.js";
-import "./openai-CitF-gEN.js";
-import "./util-Bnw6EyZN.js";
-import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-1RAiLcwi.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-C8Ei4f87.js";
+import "./transform-DMaxQwDx.js";
+import "./openai-Dz3surb_.js";
+import "./util-Cl0zfT3V.js";
+import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-D9_MDlnd.js";
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -88,4 +88,4 @@ var DMREmbeddingProvider = class extends OpenAiEmbeddingProvider {
 
 //#endregion
 export { createDockerProvider };
-//# sourceMappingURL=docker-CMf5SGLq.js.map
+//# sourceMappingURL=docker-JAAubMw3.js.map

package/dist/src/{docker-C0AzMsuf.js → docker-vnOg96gi.js}

@@ -1,4 +1,4 @@
-import { A as logger_default, P as getEnvString, c as OpenAiChatCompletionProvider, g as fetchWithCache, n as OpenAiCompletionProvider, s as OpenAiEmbeddingProvider } from "./server/index.js";
+import { I as getEnvString, _ as fetchWithCache, c as OpenAiEmbeddingProvider, j as logger_default, l as OpenAiChatCompletionProvider, n as OpenAiCompletionProvider } from "./server/index.js";
 
 //#region src/providers/docker.ts
 async function fetchLocalModels(apiBaseUrl) {
@@ -75,4 +75,4 @@ var DMREmbeddingProvider = class extends OpenAiEmbeddingProvider {
 
 //#endregion
 export { createDockerProvider };
-//# sourceMappingURL=docker-C0AzMsuf.js.map
+//# sourceMappingURL=docker-vnOg96gi.js.map

package/dist/src/errors-DnGCbnx8.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+//#region src/util/fetch/errors.ts
+/**
+* Non-transient HTTP status codes that indicate the target is unavailable or misconfigured.
+* These errors will not resolve on retry and should abort the scan immediately.
+*
+* - 401: Unauthorized - authentication required or invalid credentials
+* - 403: Forbidden - valid credentials but access denied
+* - 404: Not Found - target endpoint doesn't exist
+* - 501: Not Implemented - server doesn't support the request method
+*
+* Excluded: 500 (often transient — server crashes, DB timeouts, deployment rollouts,
+* or input-dependent bugs where one prompt triggers it but the next doesn't),
+* 502/503/504 (typically transient gateway issues).
+*/
+const NON_TRANSIENT_HTTP_STATUSES = [
+	401,
+	403,
+	404,
+	501
+];
+function isNonTransientHttpStatus(status) {
+	return NON_TRANSIENT_HTTP_STATUSES.includes(status);
+}
+function isTransientConnectionError(error) {
+	if (!error) return false;
+	const code = error.code;
+	if (code === "ECONNRESET" || code === "EPIPE") return true;
+	const message = (error.message ?? "").toLowerCase();
+	if (message.includes("eproto") && (message.includes("wrong version number") || message.includes("self signed") || message.includes("unable to verify") || message.includes("unknown ca") || message.includes("cert"))) return false;
+	return message.includes("bad record mac") || message.includes("eproto") || message.includes("econnreset") || message.includes("socket hang up");
+}
+
+//#endregion
+export { isNonTransientHttpStatus as n, isTransientConnectionError as r, NON_TRANSIENT_HTTP_STATUSES as t };
+//# sourceMappingURL=errors-DnGCbnx8.js.map

package/dist/src/{esm-gWVPXn3r.cjs → esm-BQkx5roy.cjs}

@@ -1,4 +1,4 @@
-const require_logger = require('./logger-CMp-NS-e.cjs');
+const require_logger = require('./logger-BdZ-IqBc.cjs');
 let path = require("path");
 path = require_logger.__toESM(path);
 let node_fs = require("node:fs");
@@ -408,4 +408,4 @@ Object.defineProperty(exports, 'safeResolve', {
     return safeResolve;
   }
 });
-//# sourceMappingURL=esm-gWVPXn3r.cjs.map
+//# sourceMappingURL=esm-BQkx5roy.cjs.map

package/dist/src/{esm-Lh6czAFT.js → esm-CYhseqj4.js}

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { o as logger_default } from "./logger-Bzi5o47S.js";
+import { o as logger_default } from "./logger-BotXmWKW.js";
 import { createRequire } from "node:module";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import fs from "node:fs";
@@ -380,4 +380,4 @@ function loadCjsModule(modulePath) {
 
 //#endregion
 export { safeJoin as a, resolvePackageEntryPoint as i, getWrapperDir as n, safeResolve as o, importModule as r, getDirectory as t };
-//# sourceMappingURL=esm-Lh6czAFT.js.map
+//# sourceMappingURL=esm-CYhseqj4.js.map

package/dist/src/{esm-D40XsWlR.js → esm-rDtG_2rg.js}

@@ -1,4 +1,4 @@
-import { i as logger_default } from "./logger-q5I8CByj.js";
+import { i as logger_default } from "./logger-w8Ozp0Td.js";
 import { createRequire } from "node:module";
 import path from "path";
 import path$1 from "node:path";
@@ -379,4 +379,4 @@ function loadCjsModule(modulePath) {
 
 //#endregion
 export { safeJoin as a, resolvePackageEntryPoint as i, getWrapperDir as n, safeResolve as o, importModule as r, getDirectory as t };
-//# sourceMappingURL=esm-D40XsWlR.js.map
+//# sourceMappingURL=esm-rDtG_2rg.js.map

package/dist/src/eval-CKHWqG9f.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./types-t52w-XsS.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import "./store-D4gdn9ih.js";
+import "./signal-Bl32q42d.js";
+import { a as createEvalId, i as combineFilterConditions, n as EvalQueries, o as escapeJsonPathKey, r as buildSafeJsonPath, s as getEvalSummaries, t as Eval } from "./eval-CYrbG57o.js";
+import "./evalResult-6JaUIStC.js";
+
+export { Eval as default };

package/dist/src/{eval-WaSg64w3.js → eval-CYrbG57o.js}

@@ -1,16 +1,17 @@
 #!/usr/bin/env node
-import { b as getEnvBool, o as logger_default } from "./logger-Bzi5o47S.js";
-import { I as HUMAN_ASSERTION_TYPE, T as DEFAULT_QUERY_LIMIT, c as getCurrentTimestamp } from "./fetch-Mb7SbeM-.js";
+import { b as getEnvBool, o as logger_default } from "./logger-BotXmWKW.js";
+import { I as HUMAN_ASSERTION_TYPE, T as DEFAULT_QUERY_LIMIT, c as getCurrentTimestamp } from "./fetch-SRsE6Ctl.js";
 import { t as invariant } from "./invariant-DT20jrBd.js";
-import { o as getUserEmail } from "./accounts-BOlMHrvn.js";
-import { U as riskCategorySeverityMap, ct as PLUGIN_CATEGORIES, s as ResultFailureReason } from "./types-CgrxBFgm.js";
-import { t as hashPrompt } from "./utils-GiyI2K4P.js";
-import { a as sha256, i as randomSequence } from "./extractor-BA90EtLq.js";
-import { a as createEmptyTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-DoinwgKF.js";
-import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, i as datasetsTable, l as evalsToTagsTable, o as evalsTable, p as tagsTable, s as evalsToDatasetsTable } from "./tables-BjaApSAB.js";
-import { n as getTraceStore } from "./store-BFLqwuc_.js";
-import { i as getActualPrompt, r as updateSignalFile } from "./signal-DM_SPNmi.js";
-import { t as EvalResult } from "./evalResult-CGrGU_cG.js";
+import { o as getUserEmail } from "./accounts-DHHiXsy6.js";
+import { U as riskCategorySeverityMap, ct as PLUGIN_CATEGORIES, s as ResultFailureReason } from "./types-t52w-XsS.js";
+import { n as isNonTransientHttpStatus, t as NON_TRANSIENT_HTTP_STATUSES } from "./errors-DnGCbnx8.js";
+import { t as hashPrompt } from "./utils-JaY9veb5.js";
+import { a as sha256, i as randomSequence } from "./extractor-LSYjrhK0.js";
+import { a as createEmptyTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-DmZSD2eU.js";
+import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, i as datasetsTable, l as evalsToTagsTable, o as evalsTable, p as tagsTable, s as evalsToDatasetsTable } from "./tables-C7TT2XVn.js";
+import { n as getTraceStore } from "./store-D4gdn9ih.js";
+import { i as getActualPrompt, r as updateSignalFile } from "./signal-Bl32q42d.js";
+import { t as EvalResult } from "./evalResult-6JaUIStC.js";
 import { and, desc, eq, sql } from "drizzle-orm";
 
 //#region src/redteam/metrics.ts
@@ -341,10 +342,17 @@ function convertResultsToTable(eval_) {
 				if (keyToUpdate) result.vars[keyToUpdate] = actualPrompt;
 			}
 		}
-		if (result.metadata?.sessionId && !result.vars?.sessionId) {
-			result.vars = result.vars || {};
-			result.vars.sessionId = result.metadata.sessionId;
-			varsForHeader.add("sessionId");
+		if (!result.vars?.sessionId) {
+			const metadataSessionIds = result.metadata?.sessionIds;
+			if (Array.isArray(metadataSessionIds) && metadataSessionIds.length > 0) {
+				result.vars = result.vars || {};
+				result.vars.sessionId = metadataSessionIds.filter((id) => id != null && id !== "").map(String).join("\n");
+				varsForHeader.add("sessionId");
+			} else if (result.metadata?.sessionId) {
+				result.vars = result.vars || {};
+				result.vars.sessionId = result.metadata.sessionId;
+				varsForHeader.add("sessionId");
+			}
 		}
 		const transformDisplayVars = result.response?.metadata?.transformDisplayVars;
 		if (transformDisplayVars) {
@@ -400,7 +408,12 @@ function convertResultsToTable(eval_) {
 				model: result.response.video.model,
 				aspectRatio: result.response.video.aspectRatio,
 				resolution: result.response.video.resolution
-			} : void 0
+			} : void 0,
+			images: result.response?.images?.map((img) => ({
+				data: img.data,
+				blobRef: img.blobRef,
+				mimeType: img.mimeType
+			}))
 		};
 		invariant(result.promptId, "Prompt ID is required");
 		row.testIdx = result.testIdx;
@@ -466,7 +479,12 @@ function convertEvalResultToTableCell(result) {
 			model: result.response.video.model,
 			aspectRatio: result.response.video.aspectRatio,
 			resolution: result.response.video.resolution
-		} : void 0
+		} : void 0,
+		images: result.response?.images?.map((img) => ({
+			data: img.data,
+			blobRef: img.blobRef,
+			mimeType: img.mimeType
+		}))
 	};
 }
 function convertTestResultsToTableRow(results, varsForHeader) {
@@ -475,10 +493,13 @@ function convertTestResultsToTableRow(results, varsForHeader) {
 		outputs: [],
 		vars: Object.values(varsForHeader).map((varName) => {
 			if (varName === "sessionId") {
-				const sessionId = results[0].testCase.vars?.sessionId;
-				const varValue = sessionId == null || sessionId === "" ? results[0].metadata?.sessionId ?? "" : sessionId;
-				if (typeof varValue === "string") return varValue;
-				return JSON.stringify(varValue);
+				const sessionIdFromVars = results[0].testCase.vars?.sessionId;
+				if (sessionIdFromVars != null && sessionIdFromVars !== "") return typeof sessionIdFromVars === "string" ? sessionIdFromVars : JSON.stringify(sessionIdFromVars);
+				const metadataSessionIds = results[0].metadata?.sessionIds;
+				if (Array.isArray(metadataSessionIds) && metadataSessionIds.length > 0) return metadataSessionIds.filter((id) => id != null && id !== "").map(String).join("\n");
+				const metadataSessionId = results[0].metadata?.sessionId;
+				if (metadataSessionId != null) return typeof metadataSessionId === "string" ? metadataSessionId : JSON.stringify(metadataSessionId);
+				return "";
 			}
 			const varValue = results[0].testCase.vars?.[varName] ?? "";
 			if (typeof varValue === "string") return varValue;
@@ -758,7 +779,13 @@ var Eval = class Eval {
 	_resultsLoaded = false;
 	runtimeOptions;
 	_shared = false;
+	/** Total wall-clock duration. For redteam evals: generationDurationMs + evaluationDurationMs.
+	*  For non-redteam evals: equals evaluationDurationMs (generation phase is N/A). */
 	durationMs;
+	/** Time spent generating adversarial test cases (redteam only). */
+	generationDurationMs;
+	/** Time spent running the evaluation phase. */
+	evaluationDurationMs;
 	/**
 	* The shareable URL for this evaluation, if it has been shared.
 	* Set by the evaluate() function when sharing is enabled.
@@ -777,8 +804,11 @@ var Eval = class Eval {
 		const eval_ = evalData[0];
 		const datasetId = datasetResults[0]?.datasetId;
 		const resultsObj = eval_.results;
-		const rawDurationMs = resultsObj && "durationMs" in resultsObj ? resultsObj.durationMs : void 0;
-		const durationMs = typeof rawDurationMs === "number" && Number.isFinite(rawDurationMs) && rawDurationMs >= 0 ? rawDurationMs : void 0;
+		const validateDuration = (raw) => typeof raw === "number" && Number.isFinite(raw) && raw >= 0 ? raw : void 0;
+		const rawDurationMs = validateDuration(resultsObj?.["durationMs"]);
+		const generationDurationMs = validateDuration(resultsObj?.["generationDurationMs"]);
+		const evaluationDurationMs = validateDuration(resultsObj?.["evaluationDurationMs"]);
+		const durationMs = rawDurationMs ?? (generationDurationMs != null || evaluationDurationMs != null ? (generationDurationMs ?? 0) + (evaluationDurationMs ?? 0) : void 0);
 		const evalInstance = new Eval(eval_.config, {
 			id: eval_.id,
 			createdAt: new Date(eval_.createdAt),
@@ -789,7 +819,9 @@ var Eval = class Eval {
 			persisted: true,
 			vars: eval_.vars || [],
 			runtimeOptions: eval_.runtimeOptions ?? void 0,
-			durationMs
+			durationMs,
+			generationDurationMs,
+			evaluationDurationMs
 		});
 		if (eval_.results && "table" in eval_.results) evalInstance.oldResults = eval_.results;
 		if (!eval_.vars || eval_.vars.length === 0) {
@@ -914,6 +946,8 @@ var Eval = class Eval {
 		this.vars = opts?.vars || [];
 		this.runtimeOptions = opts?.runtimeOptions;
 		this.durationMs = opts?.durationMs;
+		this.generationDurationMs = opts?.generationDurationMs;
+		this.evaluationDurationMs = opts?.evaluationDurationMs;
 	}
 	version() {
 		/**
@@ -944,7 +978,13 @@ var Eval = class Eval {
 		if (this.useOldResults()) {
 			invariant(this.oldResults, "Old results not found");
 			updateObj.results = this.oldResults;
-		} else if (this.durationMs !== void 0) updateObj.results = { durationMs: this.durationMs };
+		} else if (this.durationMs !== void 0 || this.generationDurationMs !== void 0 || this.evaluationDurationMs !== void 0) {
+			let expr = sql`CASE WHEN json_valid(${evalsTable.results}) AND json_type(${evalsTable.results}) = 'object' THEN ${evalsTable.results} ELSE '{}' END`;
+			if (this.durationMs !== void 0) expr = sql`json_set(${expr}, '$.durationMs', ${this.durationMs})`;
+			if (this.generationDurationMs !== void 0) expr = sql`json_set(${expr}, '$.generationDurationMs', ${this.generationDurationMs})`;
+			if (this.evaluationDurationMs !== void 0) expr = sql`json_set(${expr}, '$.evaluationDurationMs', ${this.evaluationDurationMs})`;
+			updateObj.results = expr;
+		}
 		db.update(evalsTable).set(updateObj).where(eq(evalsTable.id, this.id)).run();
 		this.persisted = true;
 	}
@@ -954,8 +994,17 @@ var Eval = class Eval {
 	addVar(varName) {
 		this.vars.push(varName);
 	}
+	/** Sets the evaluation phase duration and recomputes the total. Called by the evaluator. */
 	setDurationMs(durationMs) {
-		this.durationMs = durationMs;
+		if (!Number.isFinite(durationMs) || durationMs < 0) return;
+		this.evaluationDurationMs = durationMs;
+		this.durationMs = (this.generationDurationMs ?? 0) + durationMs;
+	}
+	/** Sets the generation phase duration and recomputes the total. Called by doRedteamRun. */
+	setGenerationDurationMs(durationMs) {
+		if (!Number.isFinite(durationMs) || durationMs < 0) return;
+		this.generationDurationMs = durationMs;
+		this.durationMs = durationMs + (this.evaluationDurationMs ?? 0);
 	}
 	getPrompts() {
 		if (this.useOldResults()) {
@@ -993,6 +1042,27 @@ var Eval = class Eval {
 	async getTotalResultRowCount() {
 		return getTotalResultRowCount(this.id);
 	}
+	/**
+	* Find a non-transient HTTP error status from evaluation results.
+	* Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+	*
+	* For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+	* For non-persisted evals: Falls back to scanning in-memory results.
+	*/
+	async findTargetErrorStatus() {
+		const scanInMemory = () => {
+			for (const result of this.results) {
+				const status = result.response?.metadata?.http?.status;
+				if (typeof status === "number" && isNonTransientHttpStatus(status)) return status;
+			}
+		};
+		if (!this.persisted) return scanInMemory();
+		try {
+			return getDb().select({ httpStatus: sql`CAST(json_extract(${evalResultsTable.response}, '$.metadata.http.status') AS INTEGER)` }).from(evalResultsTable).where(and(eq(evalResultsTable.evalId, this.id), sql`json_extract(${evalResultsTable.response}, '$.metadata.http.status') IN (${sql.join(NON_TRANSIENT_HTTP_STATUSES.map((s) => sql`${s}`), sql`, `)})`)).limit(1).get()?.httpStatus ?? void 0;
+		} catch {
+			return scanInMemory();
+		}
+	}
 	async fetchResultsByTestIdx(testIdx) {
 		return await EvalResult.findManyByEvalId(this.id, { testIdx });
 	}
@@ -1215,7 +1285,12 @@ var Eval = class Eval {
 			};
 		}
 		const allResults = await EvalResult.findManyByEvalIdAndTestIndices(this.id, testIndices);
-		if (allResults.some((result) => result.metadata?.sessionId && !result.testCase?.vars?.sessionId) && !vars.includes("sessionId")) {
+		if (allResults.some((result) => {
+			const hasSessionIds = Array.isArray(result.metadata?.sessionIds) && result.metadata.sessionIds.length > 0;
+			const hasSessionId = Boolean(result.metadata?.sessionId);
+			const notInVars = !result.testCase?.vars?.sessionId;
+			return (hasSessionIds || hasSessionId) && notInVars;
+		}) && !vars.includes("sessionId")) {
 			vars.push("sessionId");
 			vars.sort();
 		}
@@ -1276,7 +1351,9 @@ var Eval = class Eval {
 			failures: 0,
 			errors: 0,
 			tokenUsage: createEmptyTokenUsage(),
-			durationMs: this.durationMs
+			durationMs: this.durationMs,
+			generationDurationMs: this.generationDurationMs,
+			evaluationDurationMs: this.evaluationDurationMs
 		};
 		for (const prompt of this.prompts) {
 			stats.successes += prompt.metrics?.testPassCount ?? 0;
@@ -1549,4 +1626,4 @@ async function getEvalSummaries(datasetId, type, includeProviders = false) {
 
 //#endregion
 export { createEvalId as a, combineFilterConditions as i, EvalQueries as n, escapeJsonPathKey as o, buildSafeJsonPath as r, getEvalSummaries as s, Eval as t };
-//# sourceMappingURL=eval-WaSg64w3.js.map
+//# sourceMappingURL=eval-CYrbG57o.js.map

package/dist/src/{evalResult-CGrGU_cG.js → evalResult-6JaUIStC.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
-import { b as getEnvBool, g as safeJsonStringify, o as logger_default } from "./logger-Bzi5o47S.js";
-import { c as getCurrentTimestamp } from "./fetch-Mb7SbeM-.js";
-import { f as isResultFailureReason, m as isProviderOptions, p as isApiProvider, s as ResultFailureReason } from "./types-CgrxBFgm.js";
-import { t as hashPrompt } from "./utils-GiyI2K4P.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-BA90EtLq.js";
-import { a as evalResultsTable, g as getDb } from "./tables-BjaApSAB.js";
+import { b as getEnvBool, g as safeJsonStringify, o as logger_default } from "./logger-BotXmWKW.js";
+import { c as getCurrentTimestamp } from "./fetch-SRsE6Ctl.js";
+import { f as isResultFailureReason, m as isProviderOptions, p as isApiProvider, s as ResultFailureReason } from "./types-t52w-XsS.js";
+import { t as hashPrompt } from "./utils-JaY9veb5.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-LSYjrhK0.js";
+import { a as evalResultsTable, g as getDb } from "./tables-C7TT2XVn.js";
 import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
 
 //#region src/models/evalResult.ts
@@ -289,4 +289,4 @@ var EvalResult = class EvalResult {
 
 //#endregion
 export { sanitizeProvider as n, EvalResult as t };
-//# sourceMappingURL=evalResult-CGrGU_cG.js.map
+//# sourceMappingURL=evalResult-6JaUIStC.js.map

package/dist/src/{evalResult-C_UM63nn.js → evalResult-COsVttMA.js}

@@ -1,9 +1,9 @@
-import { _ as getEnvBool, i as logger_default, m as safeJsonStringify } from "./logger-q5I8CByj.js";
-import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-BsU_PxR3.js";
-import { f as getCurrentTimestamp } from "./fetch-BIfcW-Fd.js";
-import { t as hashPrompt } from "./utils-BryuD3vq.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-Da8IxCeS.js";
-import { i as evalResultsTable, p as getDb } from "./tables-Dwexr5Z6.js";
+import { _ as getEnvBool, i as logger_default, m as safeJsonStringify } from "./logger-w8Ozp0Td.js";
+import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-DMVjYLpx.js";
+import { f as getCurrentTimestamp } from "./fetch-18MuNu9i.js";
+import { t as hashPrompt } from "./utils-CVzb4YiI.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-Hs7la_19.js";
+import { i as evalResultsTable, p as getDb } from "./tables-D-NSwNIb.js";
 import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
 
 //#region src/models/evalResult.ts
@@ -288,4 +288,4 @@ var EvalResult = class EvalResult {
 
 //#endregion
 export { sanitizeProvider as n, EvalResult as t };
-//# sourceMappingURL=evalResult-C_UM63nn.js.map
+//# sourceMappingURL=evalResult-COsVttMA.js.map

package/dist/src/evalResult-CxTP-LMm.cjs

@@ -0,0 +1,11 @@
+require('./logger-BdZ-IqBc.cjs');
+require('./types-BIfttHrT.cjs');
+require('./fetch-ZMn_oemb.cjs');
+require('./utils-DFaZa6Rf.cjs');
+require('./extractor-DbhlYEeo.cjs');
+require('./accounts-BzEY8H3v.cjs');
+require('./blobs-673H0jCl.cjs');
+require('./tables-B9E1kRp-.cjs');
+const require_evalResult = require('./evalResult-DlRfu_Rq.cjs');
+
+exports.default = require_evalResult.EvalResult;

package/dist/src/evalResult-CzLURDcP.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./types-t52w-XsS.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import { n as sanitizeProvider, t as EvalResult } from "./evalResult-6JaUIStC.js";
+
+export { EvalResult as default };

package/dist/src/{evalResult-BU4huold.cjs → evalResult-DlRfu_Rq.cjs}

@@ -1,9 +1,9 @@
-const require_logger = require('./logger-CMp-NS-e.cjs');
-const require_types = require('./types-B-XUqfNs.cjs');
-const require_fetch = require('./fetch-MvWxCS5U.cjs');
-const require_utils = require('./utils-Dezi3MBH.cjs');
-const require_extractor = require('./extractor-D-RM2m6A.cjs');
-const require_tables = require('./tables-Dansasnu.cjs');
+const require_logger = require('./logger-BdZ-IqBc.cjs');
+const require_types = require('./types-BIfttHrT.cjs');
+const require_fetch = require('./fetch-ZMn_oemb.cjs');
+const require_utils = require('./utils-DFaZa6Rf.cjs');
+const require_extractor = require('./extractor-DbhlYEeo.cjs');
+const require_tables = require('./tables-B9E1kRp-.cjs');
 let drizzle_orm = require("drizzle-orm");
 
 //#region src/models/evalResult.ts
@@ -299,4 +299,4 @@ Object.defineProperty(exports, 'sanitizeProvider', {
     return sanitizeProvider;
   }
 });
-//# sourceMappingURL=evalResult-BU4huold.cjs.map
+//# sourceMappingURL=evalResult-DlRfu_Rq.cjs.map

package/dist/src/evalResult-DyttNQ_G.js

@@ -0,0 +1,11 @@
+import "./logger-w8Ozp0Td.js";
+import "./types-DMVjYLpx.js";
+import "./fetch-18MuNu9i.js";
+import "./utils-CVzb4YiI.js";
+import "./extractor-Hs7la_19.js";
+import "./accounts-R3gfCR_g.js";
+import "./blobs-C9J2mVgo.js";
+import "./tables-D-NSwNIb.js";
+import { n as sanitizeProvider, t as EvalResult } from "./evalResult-COsVttMA.js";
+
+export { EvalResult as default };

package/dist/src/evaluator-0PvfeBYh.js

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+import "./chunk-DHDDz29n.js";
+import "./logger-BotXmWKW.js";
+import "./fetch-SRsE6Ctl.js";
+import "./accounts-DHHiXsy6.js";
+import "./cloud-BMbRVJFw.js";
+import "./telemetry-BXyVqyAg.js";
+import "./types-t52w-XsS.js";
+import "./server-DWmZLfCy.js";
+import "./providers-BlEhY5mi.js";
+import "./cache-mIszOnuz.js";
+import "./util-BSh4a_Q8.js";
+import "./esm-CYhseqj4.js";
+import "./pythonUtils-r1uBuA0n.js";
+import "./transform-CqTFr7KR.js";
+import "./graders-BaMCwIKp.js";
+import "./utils-JaY9veb5.js";
+import "./extractor-LSYjrhK0.js";
+import "./genaiTracer-CqNnnXrE.js";
+import "./chat-DIywASPG.js";
+import "./transform-CoP2bJ7P.js";
+import "./messages-D61tPFQo.js";
+import "./util-vjscpUzy.js";
+import "./responses-TsdODUpm.js";
+import "./openai-CT5fwbve.js";
+import "./util-CUEt0Vum.js";
+import "./completion-BBJ6zmG3.js";
+import "./blobs-kt8v3UyH.js";
+import "./tables-C7TT2XVn.js";
+import "./store-D4gdn9ih.js";
+import "./base-BaXmtXYp.js";
+import "./image-mhAGP07h.js";
+import "./providerRegistry-Xf0qdqGQ.js";
+import "./rubyUtils-BtjlqyXt.js";
+import { a as runEval, i as isAllowedPrompt, n as formatVarsForDisplay, r as generateVarCombinations, t as evaluate } from "./evaluator-3EJCMTs0.js";
+import "./signal-Bl32q42d.js";
+
+export { evaluate };