promptfoo 0.121.3 → 0.121.5

package/dist/src/{eval-BTqTn7lb.js → eval-DJ_4A-tr.js}

@@ -1,17 +1,17 @@
 #!/usr/bin/env node
-import { C as getEnvBool, s as logger } from "./logger-BcJBzSSA.js";
-import { F as HUMAN_ASSERTION_TYPE, c as getCurrentTimestamp } from "./fetch-DoVRJZhJ.js";
-import { t as invariant } from "./invariant-BtWWVVhl.js";
-import { o as getUserEmail } from "./accounts-CFLK3mnD.js";
-import { H as riskCategorySeverityMap, s as ResultFailureReason, st as PLUGIN_CATEGORIES } from "./types-CIhFeUC4.js";
-import { n as isNonTransientHttpStatus, t as NON_TRANSIENT_HTTP_STATUSES } from "./errors-P6ll7XSJ.js";
-import { t as hashPrompt } from "./utils-DEuL4VNB.js";
-import { n as sha256, t as randomSequence } from "./createHash-Da8fMwqB.js";
-import { a as createEmptyTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-DflFMjS0.js";
-import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, i as datasetsTable, l as evalsToTagsTable, o as evalsTable, p as tagsTable, s as evalsToDatasetsTable } from "./tables-DmzvLbeZ.js";
-import { n as getTraceStore } from "./store-P8OKm19S.js";
-import { i as getActualPrompt, r as updateSignalFile } from "./signal-C3ZTsUgi.js";
-import { t as EvalResult } from "./evalResult-DUDShQrm.js";
+import { T as getEnvBool, s as logger } from "./logger-BbY6ypFL.js";
+import { F as HUMAN_ASSERTION_TYPE, c as getCurrentTimestamp } from "./fetch-B6ch2nU2.js";
+import { t as invariant } from "./invariant-B2Rf6avk.js";
+import { o as getUserEmail } from "./accounts-CLJHCDDb.js";
+import { H as riskCategorySeverityMap, ct as PLUGIN_CATEGORIES, s as ResultFailureReason } from "./types-BVH9hjgW.js";
+import { n as isNonTransientHttpStatus, t as NON_TRANSIENT_HTTP_STATUSES } from "./errors-DFHe4L-n.js";
+import { a as createEmptyTokenUsage, r as accumulateTokenUsage } from "./tokenUsageUtils-2wIvAhB3.js";
+import { a as evalResultsTable, c as evalsToPromptsTable, d as promptsTable, g as getDb, i as datasetsTable, l as evalsToTagsTable, o as evalsTable, p as tagsTable, s as evalsToDatasetsTable } from "./tables-NlvH23ky.js";
+import { n as getTraceStore } from "./store-CPh25336.js";
+import { n as sha256, t as randomSequence } from "./createHash-DPpsZgFF.js";
+import { t as hashPrompt } from "./utils-kt7lv30R.js";
+import { i as getActualPrompt, r as updateSignalFile } from "./signal-DTtUuU3l.js";
+import { t as EvalResult } from "./evalResult-D6P5I5il.js";
 import { and, desc, eq, sql } from "drizzle-orm";
 //#region src/redteam/metrics.ts
 /**
@@ -170,6 +170,7 @@ async function calculateWithOptimizedQuery(opts) {
 			},
 			namedScores: {},
 			namedScoresCount: {},
+			namedScoreWeights: {},
 			assertPassCount: 0,
 			assertFailCount: 0
 		};
@@ -196,15 +197,37 @@ async function aggregateNamedScores(metrics, whereSql) {
 	const query = sql`
     SELECT
       prompt_idx,
-      json_each.key as metric_name,
-      SUM(CAST(json_each.value AS REAL)) as metric_sum,
-      COUNT(*) as metric_count
-    FROM eval_results,
-      json_each(eval_results.named_scores)
+      score_entries.key as metric_name,
+      SUM(
+        CASE
+          WHEN weight_entries.value IS NOT NULL THEN
+            CAST(score_entries.value AS REAL) * CAST(weight_entries.value AS REAL)
+          ELSE CAST(score_entries.value AS REAL)
+        END
+      ) as metric_sum,
+      COUNT(*) as metric_count,
+      SUM(
+        CASE
+          WHEN weight_entries.value IS NOT NULL THEN CAST(weight_entries.value AS REAL)
+          ELSE 1
+        END
+      ) as metric_weight_total
+    FROM eval_results
+    JOIN json_each(eval_results.named_scores) as score_entries
+    LEFT JOIN json_each(
+      CASE
+        WHEN grading_result IS NOT NULL
+          AND json_valid(grading_result)
+          AND json_type(json_extract(grading_result, '$.namedScoreWeights')) = 'object'
+        THEN json_extract(grading_result, '$.namedScoreWeights')
+        ELSE json('{}')
+      END
+    ) as weight_entries
+      ON weight_entries.key = score_entries.key
     WHERE ${whereSql}
       AND named_scores IS NOT NULL
       AND json_valid(named_scores)
-    GROUP BY prompt_idx, json_each.key
+    GROUP BY prompt_idx, score_entries.key
   `;
 	const results = await db.all(query);
 	for (const row of results) {
@@ -212,6 +235,8 @@ async function aggregateNamedScores(metrics, whereSql) {
 		if (idx >= 0 && idx < metrics.length && metrics[idx]) {
 			metrics[idx].namedScores[row.metric_name] = row.metric_sum;
 			metrics[idx].namedScoresCount[row.metric_name] = row.metric_count;
+			metrics[idx].namedScoreWeights ||= {};
+			metrics[idx].namedScoreWeights[row.metric_name] = row.metric_weight_total;
 		}
 	}
 }
@@ -294,6 +319,7 @@ function createEmptyMetricsArray(numPrompts) {
 		},
 		namedScores: {},
 		namedScoresCount: {},
+		namedScoreWeights: {},
 		cost: 0
 	}));
 }
@@ -1312,7 +1338,10 @@ var Eval = class Eval {
 	}
 	async addPrompts(prompts) {
 		this.prompts = prompts;
-		if (this.persisted) getDb().update(evalsTable).set({ prompts }).where(eq(evalsTable.id, this.id)).run();
+		if (this.persisted) {
+			getDb().update(evalsTable).set({ prompts }).where(eq(evalsTable.id, this.id)).run();
+			updateSignalFile(this.id);
+		}
 	}
 	async setResults(results) {
 		this.results = results;
@@ -1620,4 +1649,4 @@ async function getEvalSummaries(datasetId, type, includeProviders = false) {
 //#endregion
 export { createEvalId as a, combineFilterConditions as i, EvalQueries as n, escapeJsonPathKey as o, buildSafeJsonPath as r, getEvalSummaries as s, Eval as t };
 
-//# sourceMappingURL=eval-BTqTn7lb.js.map
+//# sourceMappingURL=eval-DJ_4A-tr.js.map

package/dist/src/evalResult-BBJAHAtw.cjs

@@ -0,0 +1,2 @@
+const require_evalResult = require("./evalResult-Cx-8OWkb.cjs");
+exports.default = require_evalResult.EvalResult;

package/dist/src/evalResult-BBK58h2B.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { t as EvalResult } from "./evalResult-D6P5I5il.js";
+export { EvalResult as default };

package/dist/src/{evalResult-DpARzUCb.cjs → evalResult-Cx-8OWkb.cjs}

@@ -1,9 +1,9 @@
-const require_logger = require("./logger-D5iKBpu_.cjs");
-const require_types = require("./types-D8cGDZbL.cjs");
-const require_fetch = require("./fetch-BnR9wSnm.cjs");
-const require_utils = require("./utils-DKw8mrgr.cjs");
-const require_tables = require("./tables-BdZQEpRz.cjs");
-const require_extractor = require("./extractor-DG3sSfXE.cjs");
+const require_logger = require("./logger-COuQb2xB.cjs");
+const require_fetch = require("./fetch-NuqXW1Xb.cjs");
+const require_types = require("./types-CgG2rKiW.cjs");
+const require_tables = require("./tables-CXbaZ9y1.cjs");
+const require_extractor = require("./extractor-Dxr2J_wK.cjs");
+const require_utils = require("./utils-DkVeShIB.cjs");
 let drizzle_orm = require("drizzle-orm");
 //#region src/models/evalResult.ts
 function sanitizeProvider(provider) {
@@ -55,6 +55,24 @@ function sanitizeForDb(obj) {
 		return Array.isArray(obj) ? [] : null;
 	}
 }
+/**
+* Sanitize a per-test-case field for persistence: strips circular refs,
+* collapses class instances (e.g. live SDK clients that leaked in via
+* `defaultTest.options.provider`), and redacts credential fields (`apiKey`,
+* `token`, etc.) at any depth. Use this for any slot that can carry a provider
+* config — notably `testCase.options.provider` and `prompt.config.provider`,
+* where the resolved runtime provider (with its Anthropic / Bedrock SDK
+* client) flows in from the evaluator. Without this, credentials configured on
+* the judge provider end up in the Eval results both in the DB and in the
+* polling response served by `/api/eval/job/:id`.
+*/
+function sanitizeForDbWithSecrets(obj) {
+	if (obj === null || obj === void 0) return obj;
+	return require_logger.sanitizeObject(obj, {
+		context: "evalResult field",
+		maxDepth: Number.POSITIVE_INFINITY
+	});
+}
 var EvalResult = class EvalResult {
 	static async createFromEvaluateResult(evalId, result, opts) {
 		const persist = opts?.persist == null ? true : opts.persist;
@@ -71,10 +89,10 @@ var EvalResult = class EvalResult {
 		const args = {
 			id: crypto.randomUUID(),
 			evalId,
-			testCase: sanitizeForDb(preSanitizeTestCase),
+			testCase: sanitizeForDbWithSecrets(preSanitizeTestCase),
 			promptIdx: result.promptIdx,
 			testIdx: result.testIdx,
-			prompt: sanitizeForDb(prompt),
+			prompt: sanitizeForDbWithSecrets(prompt),
 			promptId: require_utils.hashPrompt(prompt),
 			error: error?.toString(),
 			success,
@@ -113,8 +131,8 @@ var EvalResult = class EvalResult {
 			for (const result of processedResults) {
 				const sanitizedResult = {
 					...result,
-					testCase: sanitizeForDb(result.testCase),
-					prompt: sanitizeForDb(result.prompt),
+					testCase: sanitizeForDbWithSecrets(result.testCase),
+					prompt: sanitizeForDbWithSecrets(result.prompt),
 					response: sanitizeForDb(result.response),
 					gradingResult: sanitizeForDb(result.gradingResult),
 					namedScores: sanitizeForDb(result.namedScores),
@@ -298,4 +316,4 @@ Object.defineProperty(exports, "sanitizeProvider", {
 	}
 });
 
-//# sourceMappingURL=evalResult-DpARzUCb.cjs.map
+//# sourceMappingURL=evalResult-Cx-8OWkb.cjs.map

package/dist/src/{evalResult-DUDShQrm.js → evalResult-D6P5I5il.js}

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
-import { C as getEnvBool, s as logger, y as safeJsonStringify } from "./logger-BcJBzSSA.js";
-import { c as getCurrentTimestamp } from "./fetch-DoVRJZhJ.js";
-import { f as isResultFailureReason, m as isProviderOptions, p as isApiProvider, s as ResultFailureReason } from "./types-CIhFeUC4.js";
-import { t as hashPrompt } from "./utils-DEuL4VNB.js";
-import { a as evalResultsTable, g as getDb } from "./tables-DmzvLbeZ.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-CAZ2G3Kh.js";
+import { T as getEnvBool, m as sanitizeObject, s as logger, x as safeJsonStringify } from "./logger-BbY6ypFL.js";
+import { c as getCurrentTimestamp } from "./fetch-B6ch2nU2.js";
+import { f as isResultFailureReason, m as isProviderOptions, p as isApiProvider, s as ResultFailureReason } from "./types-BVH9hjgW.js";
+import { a as evalResultsTable, g as getDb } from "./tables-NlvH23ky.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-DxyiFhPk.js";
+import { t as hashPrompt } from "./utils-kt7lv30R.js";
 import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
 //#region src/models/evalResult.ts
 function sanitizeProvider(provider) {
@@ -56,6 +56,24 @@ function sanitizeForDb(obj) {
 		return Array.isArray(obj) ? [] : null;
 	}
 }
+/**
+* Sanitize a per-test-case field for persistence: strips circular refs,
+* collapses class instances (e.g. live SDK clients that leaked in via
+* `defaultTest.options.provider`), and redacts credential fields (`apiKey`,
+* `token`, etc.) at any depth. Use this for any slot that can carry a provider
+* config — notably `testCase.options.provider` and `prompt.config.provider`,
+* where the resolved runtime provider (with its Anthropic / Bedrock SDK
+* client) flows in from the evaluator. Without this, credentials configured on
+* the judge provider end up in the Eval results both in the DB and in the
+* polling response served by `/api/eval/job/:id`.
+*/
+function sanitizeForDbWithSecrets(obj) {
+	if (obj === null || obj === void 0) return obj;
+	return sanitizeObject(obj, {
+		context: "evalResult field",
+		maxDepth: Number.POSITIVE_INFINITY
+	});
+}
 var EvalResult = class EvalResult {
 	static async createFromEvaluateResult(evalId, result, opts) {
 		const persist = opts?.persist == null ? true : opts.persist;
@@ -72,10 +90,10 @@ var EvalResult = class EvalResult {
 		const args = {
 			id: crypto.randomUUID(),
 			evalId,
-			testCase: sanitizeForDb(preSanitizeTestCase),
+			testCase: sanitizeForDbWithSecrets(preSanitizeTestCase),
 			promptIdx: result.promptIdx,
 			testIdx: result.testIdx,
-			prompt: sanitizeForDb(prompt),
+			prompt: sanitizeForDbWithSecrets(prompt),
 			promptId: hashPrompt(prompt),
 			error: error?.toString(),
 			success,
@@ -114,8 +132,8 @@ var EvalResult = class EvalResult {
 			for (const result of processedResults) {
 				const sanitizedResult = {
 					...result,
-					testCase: sanitizeForDb(result.testCase),
-					prompt: sanitizeForDb(result.prompt),
+					testCase: sanitizeForDbWithSecrets(result.testCase),
+					prompt: sanitizeForDbWithSecrets(result.prompt),
 					response: sanitizeForDb(result.response),
 					gradingResult: sanitizeForDb(result.gradingResult),
 					namedScores: sanitizeForDb(result.namedScores),
@@ -288,4 +306,4 @@ var EvalResult = class EvalResult {
 //#endregion
 export { sanitizeProvider as n, EvalResult as t };
 
-//# sourceMappingURL=evalResult-DUDShQrm.js.map
+//# sourceMappingURL=evalResult-D6P5I5il.js.map

package/dist/src/{evalResult-BkIhRdTe.js → evalResult-pSvGWFMo.js}

@@ -1,10 +1,10 @@
 import { t as __exportAll } from "./chunk-DEq-mXcV.js";
-import { _ as safeJsonStringify, a as logger, b as getEnvBool } from "./logger-DO8_zM18.js";
-import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-q8GXGF65.js";
-import { u as getCurrentTimestamp } from "./fetch-CVAtKnI3.js";
-import { t as hashPrompt } from "./utils-CFxO9KGo.js";
-import { i as evalResultsTable, p as getDb } from "./tables-kC7R5kiK.js";
-import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-D_wd8jxt.js";
+import { S as getEnvBool, a as logger, f as sanitizeObject, y as safeJsonStringify } from "./logger-Ct2S6Yx-.js";
+import { a as getCurrentTimestamp } from "./fetch-D9xxyC1p.js";
+import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-DNRZVOue.js";
+import { i as evalResultsTable, p as getDb } from "./tables-WgdUZ8Ck.js";
+import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-BM3jRERL.js";
+import { t as hashPrompt } from "./utils-o8S5huU2.js";
 import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
 //#region src/models/evalResult.ts
 var evalResult_exports = /* @__PURE__ */ __exportAll({
@@ -60,6 +60,24 @@ function sanitizeForDb(obj) {
 		return Array.isArray(obj) ? [] : null;
 	}
 }
+/**
+* Sanitize a per-test-case field for persistence: strips circular refs,
+* collapses class instances (e.g. live SDK clients that leaked in via
+* `defaultTest.options.provider`), and redacts credential fields (`apiKey`,
+* `token`, etc.) at any depth. Use this for any slot that can carry a provider
+* config — notably `testCase.options.provider` and `prompt.config.provider`,
+* where the resolved runtime provider (with its Anthropic / Bedrock SDK
+* client) flows in from the evaluator. Without this, credentials configured on
+* the judge provider end up in the Eval results both in the DB and in the
+* polling response served by `/api/eval/job/:id`.
+*/
+function sanitizeForDbWithSecrets(obj) {
+	if (obj === null || obj === void 0) return obj;
+	return sanitizeObject(obj, {
+		context: "evalResult field",
+		maxDepth: Number.POSITIVE_INFINITY
+	});
+}
 var EvalResult = class EvalResult {
 	static async createFromEvaluateResult(evalId, result, opts) {
 		const persist = opts?.persist == null ? true : opts.persist;
@@ -76,10 +94,10 @@ var EvalResult = class EvalResult {
 		const args = {
 			id: crypto.randomUUID(),
 			evalId,
-			testCase: sanitizeForDb(preSanitizeTestCase),
+			testCase: sanitizeForDbWithSecrets(preSanitizeTestCase),
 			promptIdx: result.promptIdx,
 			testIdx: result.testIdx,
-			prompt: sanitizeForDb(prompt),
+			prompt: sanitizeForDbWithSecrets(prompt),
 			promptId: hashPrompt(prompt),
 			error: error?.toString(),
 			success,
@@ -118,8 +136,8 @@ var EvalResult = class EvalResult {
 			for (const result of processedResults) {
 				const sanitizedResult = {
 					...result,
-					testCase: sanitizeForDb(result.testCase),
-					prompt: sanitizeForDb(result.prompt),
+					testCase: sanitizeForDbWithSecrets(result.testCase),
+					prompt: sanitizeForDbWithSecrets(result.prompt),
 					response: sanitizeForDb(result.response),
 					gradingResult: sanitizeForDb(result.gradingResult),
 					namedScores: sanitizeForDb(result.namedScores),
@@ -292,4 +310,4 @@ var EvalResult = class EvalResult {
 //#endregion
 export { evalResult_exports as n, EvalResult as t };
 
-//# sourceMappingURL=evalResult-BkIhRdTe.js.map
+//# sourceMappingURL=evalResult-pSvGWFMo.js.map

package/dist/src/evalResult-spPqh1G_.js

@@ -0,0 +1,2 @@
+import { t as EvalResult } from "./server/index.js";
+export { EvalResult as default };