promptfoo 0.120.27 → 0.121.1

package/dist/src/util-ZZH-3QZz.js

@@ -0,0 +1,293 @@
+import { m as calculateCost } from "./fetch-4M3YRaqL.js";
+import { x as parseDataUrl } from "./transform-DGxXocjk.js";
+//#region src/providers/anthropic/util.ts
+const ANTHROPIC_MODELS = [
+	...["claude-sonnet-4-6", "claude-sonnet-4-6-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 3 / 1e6,
+			output: 15 / 1e6
+		}
+	})),
+	...["claude-opus-4-6", "claude-opus-4-6-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 5 / 1e6,
+			output: 25 / 1e6
+		}
+	})),
+	...["claude-opus-4-5-20251101", "claude-opus-4-5-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 5 / 1e6,
+			output: 25 / 1e6
+		}
+	})),
+	...[
+		"claude-opus-4-1-20250805",
+		"claude-opus-4-20250514",
+		"claude-opus-4-0",
+		"claude-opus-4-latest"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 15 / 1e6,
+			output: 75 / 1e6
+		}
+	})),
+	...[
+		"claude-sonnet-4-5-20250929",
+		"claude-sonnet-4-5-latest",
+		"claude-sonnet-4-20250514",
+		"claude-sonnet-4-0",
+		"claude-sonnet-4-latest"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 3 / 1e6,
+			output: 15 / 1e6
+		}
+	})),
+	...["claude-haiku-4-5-20251001", "claude-haiku-4-5-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1 / 1e6,
+			output: 5 / 1e6
+		}
+	})),
+	...["claude-2.0"].map((model) => ({
+		id: model,
+		cost: {
+			input: .008 / 1e3,
+			output: .024 / 1e3
+		}
+	})),
+	...["claude-2.1"].map((model) => ({
+		id: model,
+		cost: {
+			input: .008 / 1e3,
+			output: .024 / 1e3
+		}
+	})),
+	...["claude-3-haiku-20240307", "claude-3-haiku-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 25e-5 / 1e3,
+			output: .00125 / 1e3
+		}
+	})),
+	...["claude-3-opus-20240229", "claude-3-opus-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: .015 / 1e3,
+			output: .075 / 1e3
+		}
+	})),
+	...["claude-3-5-haiku-20241022", "claude-3-5-haiku-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: .8 / 1e6,
+			output: 4 / 1e6
+		}
+	})),
+	...[
+		"claude-3-5-sonnet-20240620",
+		"claude-3-5-sonnet-20241022",
+		"claude-3-5-sonnet-latest",
+		"claude-3-7-sonnet-20250219",
+		"claude-3-7-sonnet-latest"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 3 / 1e6,
+			output: 15 / 1e6
+		}
+	}))
+];
+function outputFromMessage(message, showThinking) {
+	const hasToolUse = message.content.some((block) => block.type === "tool_use");
+	const hasThinking = message.content.some((block) => block.type === "thinking" || block.type === "redacted_thinking");
+	if (hasToolUse || hasThinking) return message.content.map((block) => {
+		if (block.type === "text") return block.text;
+		else if (block.type === "thinking" && showThinking) return `Thinking: ${block.thinking}\nSignature: ${block.signature}`;
+		else if (block.type === "redacted_thinking" && showThinking) return `Redacted Thinking: ${block.data}`;
+		else if (block.type !== "thinking" && block.type !== "redacted_thinking") return JSON.stringify(block);
+		return "";
+	}).filter((text) => text !== "").join("\n\n");
+	return message.content.map((block) => {
+		return block.text;
+	}).join("\n\n");
+}
+/**
+* Automatically extracts base64 data from data URLs for Anthropic image content.
+* This ensures compatibility with our universal data URL generation without requiring
+* users to modify their prompt templates with Nunjucks filters.
+*/
+function processAnthropicImageContent(content) {
+	return content.map((item) => {
+		if (item.type === "image" && item.source && item.source.type === "base64") {
+			const parsed = parseDataUrl(item.source.data);
+			if (parsed) return {
+				...item,
+				source: {
+					...item.source,
+					media_type: item.source.media_type || parsed.mimeType,
+					data: parsed.base64Data
+				}
+			};
+		}
+		return item;
+	});
+}
+function parseMessages(messages) {
+	try {
+		const parsed = JSON.parse(messages);
+		if (Array.isArray(parsed)) {
+			const systemMessage = parsed.find((msg) => msg.role === "system");
+			const thinking = parsed.find((msg) => msg.thinking)?.thinking;
+			return {
+				extractedMessages: parsed.filter((msg) => msg.role && msg.role !== "system").map((msg) => ({
+					role: msg.role,
+					content: Array.isArray(msg.content) ? processAnthropicImageContent(msg.content) : [{
+						type: "text",
+						text: msg.content
+					}]
+				})),
+				system: systemMessage ? Array.isArray(systemMessage.content) ? systemMessage.content : [{
+					type: "text",
+					text: systemMessage.content
+				}] : void 0,
+				thinking
+			};
+		}
+	} catch {}
+	const lines = messages.split("\n").map((line) => line.trim()).filter((line) => line);
+	let system;
+	let thinking;
+	const extractedMessages = [];
+	let currentRole = null;
+	let currentContent = [];
+	const pushMessage = () => {
+		if (currentRole && currentContent.length > 0) {
+			extractedMessages.push({
+				role: currentRole,
+				content: [{
+					type: "text",
+					text: currentContent.join("\n")
+				}]
+			});
+			currentContent = [];
+		}
+	};
+	for (const line of lines) if (line.startsWith("system:")) system = [{
+		type: "text",
+		text: line.slice(7).trim()
+	}];
+	else if (line.startsWith("thinking:")) try {
+		thinking = JSON.parse(line.slice(9).trim());
+	} catch {}
+	else if (line.startsWith("user:") || line.startsWith("assistant:")) {
+		pushMessage();
+		currentRole = line.startsWith("user:") ? "user" : "assistant";
+		currentContent.push(line.slice(line.indexOf(":") + 1).trim());
+	} else if (currentRole) currentContent.push(line);
+	else {
+		currentRole = "user";
+		currentContent.push(line);
+	}
+	pushMessage();
+	if (extractedMessages.length === 0 && !system) extractedMessages.push({
+		role: "user",
+		content: [{
+			type: "text",
+			text: messages.trim()
+		}]
+	});
+	return {
+		system,
+		extractedMessages,
+		thinking
+	};
+}
+function calculateAnthropicCost(modelName, config, promptTokens, completionTokens) {
+	if ([
+		"claude-sonnet-4-5-20250929",
+		"claude-sonnet-4-5-latest",
+		"claude-sonnet-4-6",
+		"claude-sonnet-4-6-latest"
+	].includes(modelName) && Number.isFinite(promptTokens) && Number.isFinite(completionTokens) && typeof promptTokens !== "undefined" && typeof completionTokens !== "undefined") {
+		const inputCost = config.cost ?? (promptTokens > 2e5 ? 6 / 1e6 : 3 / 1e6);
+		const outputCost = config.cost ?? (promptTokens > 2e5 ? 22.5 / 1e6 : 15 / 1e6);
+		return inputCost * promptTokens + outputCost * completionTokens;
+	}
+	return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
+}
+function getTokenUsage(data, cached) {
+	if (data.usage) {
+		const total_tokens = data.usage.input_tokens + data.usage.output_tokens;
+		if (cached) return {
+			cached: total_tokens,
+			total: total_tokens
+		};
+		else return {
+			total: total_tokens,
+			prompt: data.usage.input_tokens || 0,
+			completion: data.usage.output_tokens || 0
+		};
+	}
+	return {};
+}
+/**
+* Processes tools configuration to handle web fetch and web search tools
+*/
+function processAnthropicTools(tools = []) {
+	const processedTools = [];
+	const requiredBetaFeatures = [];
+	for (const tool of tools) if ("type" in tool) if (tool.type === "web_fetch_20250910") {
+		processedTools.push(transformWebFetchTool(tool));
+		if (!requiredBetaFeatures.includes("web-fetch-2025-09-10")) requiredBetaFeatures.push("web-fetch-2025-09-10");
+	} else if (tool.type === "web_search_20250305") processedTools.push(transformWebSearchTool(tool));
+	else processedTools.push(tool);
+	else {
+		processedTools.push(tool);
+		if ("strict" in tool && tool.strict === true) {
+			if (!requiredBetaFeatures.includes("structured-outputs-2025-11-13")) requiredBetaFeatures.push("structured-outputs-2025-11-13");
+		}
+	}
+	return {
+		processedTools,
+		requiredBetaFeatures
+	};
+}
+/**
+* Transform web fetch tool config to Anthropic beta tool format
+*/
+function transformWebFetchTool(config) {
+	const tool = {
+		type: "web_fetch_20250910",
+		name: "web_fetch"
+	};
+	if (config.max_uses !== void 0) tool.max_uses = config.max_uses;
+	if (config.allowed_domains) tool.allowed_domains = config.allowed_domains;
+	if (config.blocked_domains) tool.blocked_domains = config.blocked_domains;
+	if (config.citations) tool.citations = config.citations;
+	if (config.max_content_tokens !== void 0) tool.max_content_tokens = config.max_content_tokens;
+	if (config.cache_control) tool.cache_control = config.cache_control;
+	return tool;
+}
+/**
+* Transform web search tool config to Anthropic beta tool format
+*/
+function transformWebSearchTool(config) {
+	const tool = {
+		type: "web_search_20250305",
+		name: "web_search"
+	};
+	if (config.max_uses !== void 0) tool.max_uses = config.max_uses;
+	if (config.cache_control) tool.cache_control = config.cache_control;
+	return tool;
+}
+//#endregion
+export { parseMessages as a, outputFromMessage as i, calculateAnthropicCost as n, processAnthropicTools as o, getTokenUsage as r, ANTHROPIC_MODELS as t };
+
+//# sourceMappingURL=util-ZZH-3QZz.js.map

package/dist/src/{utils-BjLy-Q72.cjs → utils-Cz9qXqII.cjs}

@@ -1,6 +1,5 @@
-const require_logger = require('./logger-B7sBeGa0.cjs');
-const require_createHash = require('./createHash-CYQy4YeL.cjs');
-
+const require_logger = require("./logger-Cp1GPUjj.cjs");
+const require_createHash = require("./createHash-CfZSc0b4.cjs");
 //#region src/prompts/constants.ts
 const PROMPT_DELIMITER = require_logger.getEnvString("PROMPTFOO_PROMPT_SEPARATOR") || "---";
 const VALID_FILE_EXTENSIONS = [
@@ -19,7 +18,6 @@ const VALID_FILE_EXTENSIONS = [
 	".yml",
 	".yaml"
 ];
-
 //#endregion
 //#region src/models/prompt.ts
 /**
@@ -38,7 +36,6 @@ function generateIdFromPrompt(prompt) {
 	if (prompt.id) return require_createHash.sha256(prompt.id);
 	return require_createHash.sha256(typeof prompt.raw === "object" ? JSON.stringify(prompt.raw) : prompt.raw);
 }
-
 //#endregion
 //#region src/prompts/utils.ts
 /**
@@ -91,36 +88,36 @@ function normalizeInput(promptPathOrGlobs) {
 function hashPrompt(prompt) {
 	return generateIdFromPrompt(prompt);
 }
-
 //#endregion
-Object.defineProperty(exports, 'PROMPT_DELIMITER', {
-  enumerable: true,
-  get: function () {
-    return PROMPT_DELIMITER;
-  }
+Object.defineProperty(exports, "PROMPT_DELIMITER", {
+	enumerable: true,
+	get: function() {
+		return PROMPT_DELIMITER;
+	}
 });
-Object.defineProperty(exports, 'generateIdFromPrompt', {
-  enumerable: true,
-  get: function () {
-    return generateIdFromPrompt;
-  }
+Object.defineProperty(exports, "generateIdFromPrompt", {
+	enumerable: true,
+	get: function() {
+		return generateIdFromPrompt;
+	}
 });
-Object.defineProperty(exports, 'hashPrompt', {
-  enumerable: true,
-  get: function () {
-    return hashPrompt;
-  }
+Object.defineProperty(exports, "hashPrompt", {
+	enumerable: true,
+	get: function() {
+		return hashPrompt;
+	}
 });
-Object.defineProperty(exports, 'maybeFilePath', {
-  enumerable: true,
-  get: function () {
-    return maybeFilePath;
-  }
+Object.defineProperty(exports, "maybeFilePath", {
+	enumerable: true,
+	get: function() {
+		return maybeFilePath;
+	}
 });
-Object.defineProperty(exports, 'normalizeInput', {
-  enumerable: true,
-  get: function () {
-    return normalizeInput;
-  }
+Object.defineProperty(exports, "normalizeInput", {
+	enumerable: true,
+	get: function() {
+		return normalizeInput;
+	}
 });
-//# sourceMappingURL=utils-BjLy-Q72.cjs.map
+
+//# sourceMappingURL=utils-Cz9qXqII.cjs.map

package/dist/src/{utils-CFMn2yHW.js → utils-XiOAgly5.js}

@@ -1,6 +1,5 @@
-import { b as getEnvString } from "./logger-CG1uZPbQ.js";
-import { n as sha256 } from "./createHash-Dw_iLu31.js";
-
+import { b as getEnvString } from "./logger-CT3IKMKA.js";
+import { n as sha256 } from "./createHash-DmPQkvBh.js";
 //#region src/prompts/constants.ts
 const PROMPT_DELIMITER = getEnvString("PROMPTFOO_PROMPT_SEPARATOR") || "---";
 const VALID_FILE_EXTENSIONS = [
@@ -19,7 +18,6 @@ const VALID_FILE_EXTENSIONS = [
 	".yml",
 	".yaml"
 ];
-
 //#endregion
 //#region src/models/prompt.ts
 /**
@@ -38,7 +36,6 @@ function generateIdFromPrompt(prompt) {
 	if (prompt.id) return sha256(prompt.id);
 	return sha256(typeof prompt.raw === "object" ? JSON.stringify(prompt.raw) : prompt.raw);
 }
-
 //#endregion
 //#region src/prompts/utils.ts
 /**
@@ -91,7 +88,7 @@ function normalizeInput(promptPathOrGlobs) {
 function hashPrompt(prompt) {
 	return generateIdFromPrompt(prompt);
 }
-
 //#endregion
 export { PROMPT_DELIMITER as a, generateIdFromPrompt as i, maybeFilePath as n, normalizeInput as r, hashPrompt as t };
-//# sourceMappingURL=utils-CFMn2yHW.js.map
+
+//# sourceMappingURL=utils-XiOAgly5.js.map

package/dist/src/utils-dLokC-eR.js

@@ -0,0 +1,94 @@
+import { b as getEnvString } from "./logger-DLcq4dWf.js";
+import { n as sha256 } from "./createHash-CTQmL3G2.js";
+//#region src/models/prompt.ts
+/**
+* Generates a unique identifier for a prompt based on its properties.
+*
+* Priority order:
+* 1. If label is truthy, hash the label
+* 2. If id is truthy, hash the id
+* 3. Otherwise, hash the raw content (stringified if object)
+*
+* @param prompt - The prompt object to generate an ID for
+* @returns A SHA-256 hash string
+*/
+function generateIdFromPrompt(prompt) {
+	if (prompt.label) return sha256(prompt.label);
+	if (prompt.id) return sha256(prompt.id);
+	return sha256(typeof prompt.raw === "object" ? JSON.stringify(prompt.raw) : prompt.raw);
+}
+//#endregion
+//#region src/prompts/constants.ts
+const PROMPT_DELIMITER = getEnvString("PROMPTFOO_PROMPT_SEPARATOR") || "---";
+const VALID_FILE_EXTENSIONS = [
+	".cjs",
+	".cts",
+	".j2",
+	".js",
+	".json",
+	".jsonl",
+	".md",
+	".mjs",
+	".mts",
+	".py",
+	".ts",
+	".txt",
+	".yml",
+	".yaml"
+];
+//#endregion
+//#region src/prompts/utils.ts
+/**
+* Determines if a string is a valid file path.
+* @param str - The string to check.
+* @returns True if the string is a valid file path, false otherwise.
+*/
+function maybeFilePath(str) {
+	if (typeof str !== "string") throw new Error(`Invalid input: ${JSON.stringify(str)}`);
+	if ([
+		"\n",
+		"portkey://",
+		"langfuse://",
+		"helicone://"
+	].some((substring) => str.includes(substring))) return false;
+	return str.startsWith("file://") || VALID_FILE_EXTENSIONS.some((ext) => {
+		const tokens = str.split(":");
+		return tokens.pop()?.endsWith(ext) || tokens.pop()?.endsWith(ext);
+	}) || str.charAt(str.length - 3) === "." || str.charAt(str.length - 4) === "." || str.includes("*") || str.includes("/") || str.includes("\\");
+}
+/**
+* Normalizes the input prompt to an array of prompts, rejecting invalid and empty inputs.
+* @param promptPathOrGlobs - The input prompt.
+* @returns The normalized prompts.
+* @throws If the input is invalid or empty.
+*/
+function normalizeInput(promptPathOrGlobs) {
+	if (!promptPathOrGlobs || (typeof promptPathOrGlobs === "string" || Array.isArray(promptPathOrGlobs)) && promptPathOrGlobs.length === 0) throw new Error(`Invalid input prompt: ${JSON.stringify(promptPathOrGlobs)}`);
+	if (typeof promptPathOrGlobs === "string") return [{ raw: promptPathOrGlobs }];
+	if (Array.isArray(promptPathOrGlobs)) return promptPathOrGlobs.map((promptPathOrGlob, _index) => {
+		if (typeof promptPathOrGlob === "string") return { raw: promptPathOrGlob };
+		return {
+			raw: promptPathOrGlob.raw || promptPathOrGlob.id,
+			...promptPathOrGlob
+		};
+	});
+	if (typeof promptPathOrGlobs === "object" && Object.keys(promptPathOrGlobs).length) return Object.entries(promptPathOrGlobs).map(([raw, key]) => ({
+		label: key,
+		raw
+	}));
+	throw new Error(`Invalid input prompt: ${JSON.stringify(promptPathOrGlobs)}`);
+}
+/**
+* Generates a hash identifier for a prompt.
+* This is an alias for generateIdFromPrompt for backward compatibility.
+*
+* @param prompt - The prompt to hash
+* @returns A SHA-256 hash string
+*/
+function hashPrompt(prompt) {
+	return generateIdFromPrompt(prompt);
+}
+//#endregion
+export { generateIdFromPrompt as a, PROMPT_DELIMITER as i, maybeFilePath as n, normalizeInput as r, hashPrompt as t };
+
+//# sourceMappingURL=utils-dLokC-eR.js.map

package/dist/src/{utils-DvWMzuMx.js → utils-f2-Moju7.js}

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
-import { C as getEnvString } from "./logger-LSBxlt7a.js";
-import { n as sha256 } from "./createHash-CJcfskIZ.js";
-
+import { C as getEnvString } from "./logger-KkObSCzq.js";
+import { n as sha256 } from "./createHash-Da8fMwqB.js";
 //#region src/prompts/constants.ts
 const PROMPT_DELIMITER = getEnvString("PROMPTFOO_PROMPT_SEPARATOR") || "---";
 const VALID_FILE_EXTENSIONS = [
@@ -20,7 +19,6 @@ const VALID_FILE_EXTENSIONS = [
 	".yml",
 	".yaml"
 ];
-
 //#endregion
 //#region src/models/prompt.ts
 /**
@@ -39,7 +37,6 @@ function generateIdFromPrompt(prompt) {
 	if (prompt.id) return sha256(prompt.id);
 	return sha256(typeof prompt.raw === "object" ? JSON.stringify(prompt.raw) : prompt.raw);
 }
-
 //#endregion
 //#region src/prompts/utils.ts
 /**
@@ -92,7 +89,7 @@ function normalizeInput(promptPathOrGlobs) {
 function hashPrompt(prompt) {
 	return generateIdFromPrompt(prompt);
 }
-
 //#endregion
 export { PROMPT_DELIMITER as a, generateIdFromPrompt as i, maybeFilePath as n, normalizeInput as r, hashPrompt as t };
-//# sourceMappingURL=utils-DvWMzuMx.js.map
+
+//# sourceMappingURL=utils-f2-Moju7.js.map