promptfoo 0.121.4 → 0.121.7

package/dist/src/{store-DAAyxcy6.cjs → store-B2NDDooM.cjs}

@@ -1,7 +1,12 @@
-const require_logger = require("./logger-COuQb2xB.cjs");
-const require_tables = require("./tables-DUfh1F7Z.cjs");
+const require_rolldown_runtime = require("./rolldown-runtime-D_mwlA32.cjs");
+const require_logger = require("./logger-cfNpzI4o.cjs");
+const require_tables = require("./tables-DBIJU0WE.cjs");
 let drizzle_orm = require("drizzle-orm");
 //#region src/tracing/store.ts
+var store_exports = /* @__PURE__ */ require_rolldown_runtime.__exportAll({
+	TraceStore: () => TraceStore,
+	getTraceStore: () => getTraceStore
+});
 const SENSITIVE_ATTRIBUTE_KEYS = [
 	"authorization",
 	"cookie",
@@ -13,6 +18,27 @@ const SENSITIVE_ATTRIBUTE_KEYS = [
 	"password",
 	"passphrase"
 ];
+const NORMALIZED_SENSITIVE_ATTRIBUTE_KEYS = SENSITIVE_ATTRIBUTE_KEYS.map((key) => key.replace(/[^a-z0-9]/g, ""));
+const SAFE_TOKEN_ATTRIBUTE_KEYS = new Set([
+	"gen_ai.request.max_tokens",
+	"gen_ai.usage.input_tokens",
+	"gen_ai.usage.output_tokens",
+	"gen_ai.usage.total_tokens",
+	"gen_ai.usage.cached_tokens",
+	"gen_ai.usage.reasoning_tokens",
+	"gen_ai.usage.accepted_prediction_tokens",
+	"gen_ai.usage.rejected_prediction_tokens",
+	"gen_ai.usage.cache_read_input_tokens",
+	"gen_ai.usage.cache_creation_input_tokens"
+]);
+function isSensitiveAttributeKey(key) {
+	const lowerKey = key.toLowerCase();
+	if (SAFE_TOKEN_ATTRIBUTE_KEYS.has(lowerKey)) return false;
+	const normalizedKey = lowerKey.replace(/[^a-z0-9]/g, "");
+	return SENSITIVE_ATTRIBUTE_KEYS.some((sensitiveKey, index) => {
+		return lowerKey.includes(sensitiveKey) || normalizedKey.includes(NORMALIZED_SENSITIVE_ATTRIBUTE_KEYS[index]);
+	});
+}
 function sanitizeAttributes(attributes) {
 	if (!attributes) return {};
 	const sanitizeValue = (value) => {
@@ -23,8 +49,7 @@ function sanitizeAttributes(attributes) {
 	};
 	const sanitized = {};
 	for (const [key, value] of Object.entries(attributes)) {
-		const lowerKey = key.toLowerCase();
-		if (SENSITIVE_ATTRIBUTE_KEYS.some((sensitiveKey) => lowerKey.includes(sensitiveKey))) {
+		if (isSensitiveAttributeKey(key)) {
 			sanitized[key] = "<redacted>";
 			continue;
 		}
@@ -32,6 +57,19 @@ function sanitizeAttributes(attributes) {
 	}
 	return sanitized;
 }
+function serializeSpan(span, shouldSanitizeAttributes = true) {
+	const rawAttributes = span.attributes ?? void 0;
+	return {
+		spanId: span.spanId,
+		parentSpanId: span.parentSpanId ?? void 0,
+		name: span.name,
+		startTime: span.startTime,
+		endTime: span.endTime ?? void 0,
+		attributes: rawAttributes ? shouldSanitizeAttributes ? sanitizeAttributes(rawAttributes) : rawAttributes : void 0,
+		statusCode: span.statusCode ?? void 0,
+		statusMessage: span.statusMessage ?? void 0
+	};
+}
 function computeDepth(span, spanMap, depthCache) {
 	if (depthCache.has(span.spanId)) return depthCache.get(span.spanId);
 	if (!span.parentSpanId || !spanMap.has(span.parentSpanId)) {
@@ -81,7 +119,9 @@ var TraceStore = class {
 			else {
 				require_logger.logger.debug(`[TraceStore] Verifying trace ${traceId} exists`);
 				if ((await db.select().from(require_tables.tracesTable).where((0, drizzle_orm.eq)(require_tables.tracesTable.traceId, traceId)).limit(1)).length === 0) {
-					require_logger.logger.warn(`[TraceStore] Trace ${traceId} not found, skipping ${spans.length} spans. This may indicate spans arrived before trace was created.`);
+					const message = `[TraceStore] Trace ${traceId} not found, skipping ${spans.length} spans. This may indicate spans arrived before trace was created.`;
+					if (options?.warnIfMissingTrace === false) require_logger.logger.debug(message);
+					else require_logger.logger.warn(message);
 					return {
 						stored: false,
 						reason: `Trace ${traceId} not found`
@@ -112,7 +152,8 @@ var TraceStore = class {
 			throw error;
 		}
 	}
-	async getTracesByEvaluation(evaluationId) {
+	async getTracesByEvaluation(evaluationId, options = {}) {
+		const { sanitizeAttributes: shouldSanitize = true } = options;
 		try {
 			require_logger.logger.debug(`[TraceStore] Fetching traces for evaluation ${evaluationId}`);
 			const db = this.getDatabase();
@@ -123,8 +164,11 @@ var TraceStore = class {
 				const spans = await db.select().from(require_tables.spansTable).where((0, drizzle_orm.eq)(require_tables.spansTable.traceId, trace.traceId));
 				require_logger.logger.debug(`[TraceStore] Found ${spans.length} spans for trace ${trace.traceId}`);
 				return {
-					...trace,
-					spans
+					traceId: trace.traceId,
+					evaluationId: trace.evaluationId,
+					testCaseId: trace.testCaseId,
+					metadata: trace.metadata ?? void 0,
+					spans: spans.map((span) => serializeSpan(span, shouldSanitize))
 				};
 			}));
 			require_logger.logger.debug(`[TraceStore] Returning ${tracesWithSpans.length} traces with spans`);
@@ -134,7 +178,8 @@ var TraceStore = class {
 			throw error;
 		}
 	}
-	async getTrace(traceId) {
+	async getTrace(traceId, options = {}) {
+		const { sanitizeAttributes: shouldSanitize = true } = options;
 		try {
 			require_logger.logger.debug(`[TraceStore] Fetching trace ${traceId}`);
 			const db = this.getDatabase();
@@ -152,16 +197,7 @@ var TraceStore = class {
 				evaluationId: trace.evaluationId,
 				testCaseId: trace.testCaseId,
 				metadata: trace.metadata ?? void 0,
-				spans: spans.map((span) => ({
-					spanId: span.spanId,
-					parentSpanId: span.parentSpanId ?? void 0,
-					name: span.name,
-					startTime: span.startTime,
-					endTime: span.endTime ?? void 0,
-					attributes: span.attributes ?? void 0,
-					statusCode: span.statusCode ?? void 0,
-					statusMessage: span.statusMessage ?? void 0
-				}))
+				spans: spans.map((span) => serializeSpan(span, shouldSanitize))
 			};
 		} catch (error) {
 			require_logger.logger.error(`[TraceStore] Failed to get trace: ${error}`);
@@ -230,17 +266,17 @@ function getTraceStore() {
 	return traceStore;
 }
 //#endregion
-Object.defineProperty(exports, "TraceStore", {
+Object.defineProperty(exports, "getTraceStore", {
 	enumerable: true,
 	get: function() {
-		return TraceStore;
+		return getTraceStore;
 	}
 });
-Object.defineProperty(exports, "getTraceStore", {
+Object.defineProperty(exports, "store_exports", {
 	enumerable: true,
 	get: function() {
-		return getTraceStore;
+		return store_exports;
 	}
 });
 
-//# sourceMappingURL=store-DAAyxcy6.cjs.map
+//# sourceMappingURL=store-B2NDDooM.cjs.map

package/dist/src/{store-CYEy5J2D.js → store-DKd5592Q.js}

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import { s as logger } from "./logger-DksKw1Qc.js";
-import { f as spansTable, g as getDb, m as tracesTable } from "./tables-C4CH3zRr.js";
+import { s as logger } from "./logger-BbY6ypFL.js";
+import { f as spansTable, g as getDb, m as tracesTable } from "./tables-DafUHOeh.js";
 import { asc, eq, lt } from "drizzle-orm";
 //#region src/tracing/store.ts
 const SENSITIVE_ATTRIBUTE_KEYS = [
@@ -14,6 +14,27 @@ const SENSITIVE_ATTRIBUTE_KEYS = [
 	"password",
 	"passphrase"
 ];
+const NORMALIZED_SENSITIVE_ATTRIBUTE_KEYS = SENSITIVE_ATTRIBUTE_KEYS.map((key) => key.replace(/[^a-z0-9]/g, ""));
+const SAFE_TOKEN_ATTRIBUTE_KEYS = new Set([
+	"gen_ai.request.max_tokens",
+	"gen_ai.usage.input_tokens",
+	"gen_ai.usage.output_tokens",
+	"gen_ai.usage.total_tokens",
+	"gen_ai.usage.cached_tokens",
+	"gen_ai.usage.reasoning_tokens",
+	"gen_ai.usage.accepted_prediction_tokens",
+	"gen_ai.usage.rejected_prediction_tokens",
+	"gen_ai.usage.cache_read_input_tokens",
+	"gen_ai.usage.cache_creation_input_tokens"
+]);
+function isSensitiveAttributeKey(key) {
+	const lowerKey = key.toLowerCase();
+	if (SAFE_TOKEN_ATTRIBUTE_KEYS.has(lowerKey)) return false;
+	const normalizedKey = lowerKey.replace(/[^a-z0-9]/g, "");
+	return SENSITIVE_ATTRIBUTE_KEYS.some((sensitiveKey, index) => {
+		return lowerKey.includes(sensitiveKey) || normalizedKey.includes(NORMALIZED_SENSITIVE_ATTRIBUTE_KEYS[index]);
+	});
+}
 function sanitizeAttributes(attributes) {
 	if (!attributes) return {};
 	const sanitizeValue = (value) => {
@@ -24,8 +45,7 @@ function sanitizeAttributes(attributes) {
 	};
 	const sanitized = {};
 	for (const [key, value] of Object.entries(attributes)) {
-		const lowerKey = key.toLowerCase();
-		if (SENSITIVE_ATTRIBUTE_KEYS.some((sensitiveKey) => lowerKey.includes(sensitiveKey))) {
+		if (isSensitiveAttributeKey(key)) {
 			sanitized[key] = "<redacted>";
 			continue;
 		}
@@ -33,6 +53,19 @@ function sanitizeAttributes(attributes) {
 	}
 	return sanitized;
 }
+function serializeSpan(span, shouldSanitizeAttributes = true) {
+	const rawAttributes = span.attributes ?? void 0;
+	return {
+		spanId: span.spanId,
+		parentSpanId: span.parentSpanId ?? void 0,
+		name: span.name,
+		startTime: span.startTime,
+		endTime: span.endTime ?? void 0,
+		attributes: rawAttributes ? shouldSanitizeAttributes ? sanitizeAttributes(rawAttributes) : rawAttributes : void 0,
+		statusCode: span.statusCode ?? void 0,
+		statusMessage: span.statusMessage ?? void 0
+	};
+}
 function computeDepth(span, spanMap, depthCache) {
 	if (depthCache.has(span.spanId)) return depthCache.get(span.spanId);
 	if (!span.parentSpanId || !spanMap.has(span.parentSpanId)) {
@@ -82,7 +115,9 @@ var TraceStore = class {
 			else {
 				logger.debug(`[TraceStore] Verifying trace ${traceId} exists`);
 				if ((await db.select().from(tracesTable).where(eq(tracesTable.traceId, traceId)).limit(1)).length === 0) {
-					logger.warn(`[TraceStore] Trace ${traceId} not found, skipping ${spans.length} spans. This may indicate spans arrived before trace was created.`);
+					const message = `[TraceStore] Trace ${traceId} not found, skipping ${spans.length} spans. This may indicate spans arrived before trace was created.`;
+					if (options?.warnIfMissingTrace === false) logger.debug(message);
+					else logger.warn(message);
 					return {
 						stored: false,
 						reason: `Trace ${traceId} not found`
@@ -113,7 +148,8 @@ var TraceStore = class {
 			throw error;
 		}
 	}
-	async getTracesByEvaluation(evaluationId) {
+	async getTracesByEvaluation(evaluationId, options = {}) {
+		const { sanitizeAttributes: shouldSanitize = true } = options;
 		try {
 			logger.debug(`[TraceStore] Fetching traces for evaluation ${evaluationId}`);
 			const db = this.getDatabase();
@@ -124,8 +160,11 @@ var TraceStore = class {
 				const spans = await db.select().from(spansTable).where(eq(spansTable.traceId, trace.traceId));
 				logger.debug(`[TraceStore] Found ${spans.length} spans for trace ${trace.traceId}`);
 				return {
-					...trace,
-					spans
+					traceId: trace.traceId,
+					evaluationId: trace.evaluationId,
+					testCaseId: trace.testCaseId,
+					metadata: trace.metadata ?? void 0,
+					spans: spans.map((span) => serializeSpan(span, shouldSanitize))
 				};
 			}));
 			logger.debug(`[TraceStore] Returning ${tracesWithSpans.length} traces with spans`);
@@ -135,7 +174,8 @@ var TraceStore = class {
 			throw error;
 		}
 	}
-	async getTrace(traceId) {
+	async getTrace(traceId, options = {}) {
+		const { sanitizeAttributes: shouldSanitize = true } = options;
 		try {
 			logger.debug(`[TraceStore] Fetching trace ${traceId}`);
 			const db = this.getDatabase();
@@ -153,16 +193,7 @@ var TraceStore = class {
 				evaluationId: trace.evaluationId,
 				testCaseId: trace.testCaseId,
 				metadata: trace.metadata ?? void 0,
-				spans: spans.map((span) => ({
-					spanId: span.spanId,
-					parentSpanId: span.parentSpanId ?? void 0,
-					name: span.name,
-					startTime: span.startTime,
-					endTime: span.endTime ?? void 0,
-					attributes: span.attributes ?? void 0,
-					statusCode: span.statusCode ?? void 0,
-					statusMessage: span.statusMessage ?? void 0
-				}))
+				spans: spans.map((span) => serializeSpan(span, shouldSanitize))
 			};
 		} catch (error) {
 			logger.error(`[TraceStore] Failed to get trace: ${error}`);
@@ -233,4 +264,4 @@ function getTraceStore() {
 //#endregion
 export { getTraceStore as n, TraceStore as t };
 
-//# sourceMappingURL=store-CYEy5J2D.js.map
+//# sourceMappingURL=store-DKd5592Q.js.map

package/dist/src/{store-M0b1WfYb.js → store-HpopRVzl.js}

@@ -1,6 +1,6 @@
 import { t as __exportAll } from "./chunk-DEq-mXcV.js";
 import { a as logger } from "./logger-Ct2S6Yx-.js";
-import { f as tracesTable, p as getDb, u as spansTable } from "./tables-DQ4WU5tX.js";
+import { f as tracesTable, p as getDb, u as spansTable } from "./tables-CRSXQ2Ke.js";
 import { asc, eq, lt } from "drizzle-orm";
 //#region src/tracing/store.ts
 var store_exports = /* @__PURE__ */ __exportAll({
@@ -18,6 +18,27 @@ const SENSITIVE_ATTRIBUTE_KEYS = [
 	"password",
 	"passphrase"
 ];
+const NORMALIZED_SENSITIVE_ATTRIBUTE_KEYS = SENSITIVE_ATTRIBUTE_KEYS.map((key) => key.replace(/[^a-z0-9]/g, ""));
+const SAFE_TOKEN_ATTRIBUTE_KEYS = new Set([
+	"gen_ai.request.max_tokens",
+	"gen_ai.usage.input_tokens",
+	"gen_ai.usage.output_tokens",
+	"gen_ai.usage.total_tokens",
+	"gen_ai.usage.cached_tokens",
+	"gen_ai.usage.reasoning_tokens",
+	"gen_ai.usage.accepted_prediction_tokens",
+	"gen_ai.usage.rejected_prediction_tokens",
+	"gen_ai.usage.cache_read_input_tokens",
+	"gen_ai.usage.cache_creation_input_tokens"
+]);
+function isSensitiveAttributeKey(key) {
+	const lowerKey = key.toLowerCase();
+	if (SAFE_TOKEN_ATTRIBUTE_KEYS.has(lowerKey)) return false;
+	const normalizedKey = lowerKey.replace(/[^a-z0-9]/g, "");
+	return SENSITIVE_ATTRIBUTE_KEYS.some((sensitiveKey, index) => {
+		return lowerKey.includes(sensitiveKey) || normalizedKey.includes(NORMALIZED_SENSITIVE_ATTRIBUTE_KEYS[index]);
+	});
+}
 function sanitizeAttributes(attributes) {
 	if (!attributes) return {};
 	const sanitizeValue = (value) => {
@@ -28,8 +49,7 @@ function sanitizeAttributes(attributes) {
 	};
 	const sanitized = {};
 	for (const [key, value] of Object.entries(attributes)) {
-		const lowerKey = key.toLowerCase();
-		if (SENSITIVE_ATTRIBUTE_KEYS.some((sensitiveKey) => lowerKey.includes(sensitiveKey))) {
+		if (isSensitiveAttributeKey(key)) {
 			sanitized[key] = "<redacted>";
 			continue;
 		}
@@ -37,6 +57,19 @@ function sanitizeAttributes(attributes) {
 	}
 	return sanitized;
 }
+function serializeSpan(span, shouldSanitizeAttributes = true) {
+	const rawAttributes = span.attributes ?? void 0;
+	return {
+		spanId: span.spanId,
+		parentSpanId: span.parentSpanId ?? void 0,
+		name: span.name,
+		startTime: span.startTime,
+		endTime: span.endTime ?? void 0,
+		attributes: rawAttributes ? shouldSanitizeAttributes ? sanitizeAttributes(rawAttributes) : rawAttributes : void 0,
+		statusCode: span.statusCode ?? void 0,
+		statusMessage: span.statusMessage ?? void 0
+	};
+}
 function computeDepth(span, spanMap, depthCache) {
 	if (depthCache.has(span.spanId)) return depthCache.get(span.spanId);
 	if (!span.parentSpanId || !spanMap.has(span.parentSpanId)) {
@@ -86,7 +119,9 @@ var TraceStore = class {
 			else {
 				logger.debug(`[TraceStore] Verifying trace ${traceId} exists`);
 				if ((await db.select().from(tracesTable).where(eq(tracesTable.traceId, traceId)).limit(1)).length === 0) {
-					logger.warn(`[TraceStore] Trace ${traceId} not found, skipping ${spans.length} spans. This may indicate spans arrived before trace was created.`);
+					const message = `[TraceStore] Trace ${traceId} not found, skipping ${spans.length} spans. This may indicate spans arrived before trace was created.`;
+					if (options?.warnIfMissingTrace === false) logger.debug(message);
+					else logger.warn(message);
 					return {
 						stored: false,
 						reason: `Trace ${traceId} not found`
@@ -117,7 +152,8 @@ var TraceStore = class {
 			throw error;
 		}
 	}
-	async getTracesByEvaluation(evaluationId) {
+	async getTracesByEvaluation(evaluationId, options = {}) {
+		const { sanitizeAttributes: shouldSanitize = true } = options;
 		try {
 			logger.debug(`[TraceStore] Fetching traces for evaluation ${evaluationId}`);
 			const db = this.getDatabase();
@@ -128,8 +164,11 @@ var TraceStore = class {
 				const spans = await db.select().from(spansTable).where(eq(spansTable.traceId, trace.traceId));
 				logger.debug(`[TraceStore] Found ${spans.length} spans for trace ${trace.traceId}`);
 				return {
-					...trace,
-					spans
+					traceId: trace.traceId,
+					evaluationId: trace.evaluationId,
+					testCaseId: trace.testCaseId,
+					metadata: trace.metadata ?? void 0,
+					spans: spans.map((span) => serializeSpan(span, shouldSanitize))
 				};
 			}));
 			logger.debug(`[TraceStore] Returning ${tracesWithSpans.length} traces with spans`);
@@ -139,7 +178,8 @@ var TraceStore = class {
 			throw error;
 		}
 	}
-	async getTrace(traceId) {
+	async getTrace(traceId, options = {}) {
+		const { sanitizeAttributes: shouldSanitize = true } = options;
 		try {
 			logger.debug(`[TraceStore] Fetching trace ${traceId}`);
 			const db = this.getDatabase();
@@ -157,16 +197,7 @@ var TraceStore = class {
 				evaluationId: trace.evaluationId,
 				testCaseId: trace.testCaseId,
 				metadata: trace.metadata ?? void 0,
-				spans: spans.map((span) => ({
-					spanId: span.spanId,
-					parentSpanId: span.parentSpanId ?? void 0,
-					name: span.name,
-					startTime: span.startTime,
-					endTime: span.endTime ?? void 0,
-					attributes: span.attributes ?? void 0,
-					statusCode: span.statusCode ?? void 0,
-					statusMessage: span.statusMessage ?? void 0
-				}))
+				spans: spans.map((span) => serializeSpan(span, shouldSanitize))
 			};
 		} catch (error) {
 			logger.error(`[TraceStore] Failed to get trace: ${error}`);
@@ -237,4 +268,4 @@ function getTraceStore() {
 //#endregion
 export { store_exports as n, getTraceStore as t };
 
-//# sourceMappingURL=store-M0b1WfYb.js.map
+//# sourceMappingURL=store-HpopRVzl.js.map

package/dist/src/store-IbiRIF3k.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { n as getTraceStore } from "./store-DKd5592Q.js";
+export { getTraceStore };