promptfoo 0.120.27 → 0.121.1

package/dist/src/util-6-GqIvzS.js

@@ -0,0 +1,599 @@
+import { d as getAjv, m as safeJsonStringify } from "./logger-DLcq4dWf.js";
+import { m as calculateCost } from "./fetch-4M3YRaqL.js";
+import { E as maybeLoadFromExternalFileWithVars } from "./util-5cB-L7U3.js";
+import OpenAI from "openai";
+//#region src/providers/openai/util.ts
+const ajv = getAjv();
+const OPENAI_CHAT_MODELS = [
+	{
+		id: "gpt-4o-mini-tts",
+		cost: {
+			input: .6 / 1e6,
+			output: 0 / 1e6,
+			audioOutput: 12 / 1e6
+		}
+	},
+	...["gpt-4o-search-preview", "gpt-4o-search-preview-2025-03-11"].map((model) => ({
+		id: model,
+		cost: {
+			input: 2.5 / 1e6,
+			output: 10 / 1e6
+		}
+	})),
+	...["gpt-4o-mini-search-preview", "gpt-4o-mini-search-preview-2025-03-11"].map((model) => ({
+		id: model,
+		cost: {
+			input: .15 / 1e6,
+			output: .6 / 1e6
+		}
+	})),
+	...["computer-use-preview", "computer-use-preview-2025-03-11"].map((model) => ({
+		id: model,
+		cost: {
+			input: 3 / 1e6,
+			output: 12 / 1e6
+		}
+	})),
+	...["chatgpt-4o-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 5 / 1e6,
+			output: 15 / 1e6
+		}
+	})),
+	...["gpt-4.1", "gpt-4.1-2025-04-14"].map((model) => ({
+		id: model,
+		cost: {
+			input: 2 / 1e6,
+			output: 8 / 1e6
+		}
+	})),
+	...["gpt-4.1-mini", "gpt-4.1-mini-2025-04-14"].map((model) => ({
+		id: model,
+		cost: {
+			input: .4 / 1e6,
+			output: 1.6 / 1e6
+		}
+	})),
+	...["gpt-4.1-nano", "gpt-4.1-nano-2025-04-14"].map((model) => ({
+		id: model,
+		cost: {
+			input: .1 / 1e6,
+			output: .4 / 1e6
+		}
+	})),
+	...["o1-pro", "o1-pro-2025-03-19"].map((model) => ({
+		id: model,
+		cost: {
+			input: 150 / 1e6,
+			output: 600 / 1e6
+		}
+	})),
+	...[
+		"o1",
+		"o1-2024-12-17",
+		"o1-preview",
+		"o1-preview-2024-09-12"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 15 / 1e6,
+			output: 60 / 1e6
+		}
+	})),
+	...["o1-mini", "o1-mini-2024-09-12"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.1 / 1e6,
+			output: 4.4 / 1e6
+		}
+	})),
+	...["o3", "o3-2025-04-16"].map((model) => ({
+		id: model,
+		cost: {
+			input: 2 / 1e6,
+			output: 8 / 1e6
+		}
+	})),
+	...["o3-pro", "o3-pro-2025-06-10"].map((model) => ({
+		id: model,
+		cost: {
+			input: 20 / 1e6,
+			output: 80 / 1e6
+		}
+	})),
+	...["o3-mini", "o3-mini-2025-01-31"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.1 / 1e6,
+			output: 4.4 / 1e6
+		}
+	})),
+	...[
+		"gpt-4o-audio-preview",
+		"gpt-4o-audio-preview-2024-12-17",
+		"gpt-4o-audio-preview-2024-10-01",
+		"gpt-4o-audio-preview-2025-06-03"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 2.5 / 1e6,
+			output: 10 / 1e6,
+			audioInput: 40 / 1e6,
+			audioOutput: 80 / 1e6
+		}
+	})),
+	...["gpt-4o-mini-audio-preview", "gpt-4o-mini-audio-preview-2024-12-17"].map((model) => ({
+		id: model,
+		cost: {
+			input: .15 / 1e6,
+			output: .6 / 1e6,
+			audioInput: 10 / 1e6,
+			audioOutput: 20 / 1e6
+		}
+	})),
+	...[
+		"gpt-4o",
+		"gpt-4o-2024-11-20",
+		"gpt-4o-2024-08-06"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 2.5 / 1e6,
+			output: 10 / 1e6
+		}
+	})),
+	...["gpt-4o-2024-05-13"].map((model) => ({
+		id: model,
+		cost: {
+			input: 5 / 1e6,
+			output: 15 / 1e6
+		}
+	})),
+	...["gpt-4o-mini", "gpt-4o-mini-2024-07-18"].map((model) => ({
+		id: model,
+		cost: {
+			input: .15 / 1e6,
+			output: .6 / 1e6
+		}
+	})),
+	...[
+		"gpt-4",
+		"gpt-4-0613",
+		"gpt-4-0314"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 30 / 1e6,
+			output: 60 / 1e6
+		}
+	})),
+	...[
+		"gpt-4-32k",
+		"gpt-4-32k-0314",
+		"gpt-4-32k-0613"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 60 / 1e6,
+			output: 120 / 1e6
+		}
+	})),
+	...[
+		"gpt-4-turbo",
+		"gpt-4-turbo-2024-04-09",
+		"gpt-4-turbo-preview",
+		"gpt-4-0125-preview",
+		"gpt-4-1106-preview",
+		"gpt-4-1106-vision-preview",
+		"gpt-4-vision-preview"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 10 / 1e6,
+			output: 30 / 1e6
+		}
+	})),
+	{
+		id: "gpt-3.5-turbo",
+		cost: {
+			input: .5 / 1e6,
+			output: 1.5 / 1e6
+		}
+	},
+	{
+		id: "gpt-3.5-turbo-0125",
+		cost: {
+			input: .5 / 1e6,
+			output: 1.5 / 1e6
+		}
+	},
+	{
+		id: "gpt-3.5-turbo-1106",
+		cost: {
+			input: 1 / 1e6,
+			output: 2 / 1e6
+		}
+	},
+	...["gpt-3.5-turbo-0301", "gpt-3.5-turbo-0613"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.5 / 1e6,
+			output: 2 / 1e6
+		}
+	})),
+	...["gpt-3.5-turbo-16k", "gpt-3.5-turbo-16k-0613"].map((model) => ({
+		id: model,
+		cost: {
+			input: 3 / 1e6,
+			output: 4 / 1e6
+		}
+	})),
+	...["gpt-3.5-turbo-instruct"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.5 / 1e6,
+			output: 2 / 1e6
+		}
+	})),
+	...["o4-mini", "o4-mini-2025-04-16"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.1 / 1e6,
+			output: 4.4 / 1e6
+		}
+	})),
+	...[
+		"gpt-5",
+		"gpt-5-2025-08-07",
+		"gpt-5-chat",
+		"gpt-5-chat-latest"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.25 / 1e6,
+			output: 10 / 1e6
+		}
+	})),
+	...["gpt-5-nano", "gpt-5-nano-2025-08-07"].map((model) => ({
+		id: model,
+		cost: {
+			input: .05 / 1e6,
+			output: .4 / 1e6
+		}
+	})),
+	...["gpt-5-mini", "gpt-5-mini-2025-08-07"].map((model) => ({
+		id: model,
+		cost: {
+			input: .25 / 1e6,
+			output: 2 / 1e6
+		}
+	})),
+	...["codex-mini-latest"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.5 / 1e6,
+			output: 6 / 1e6
+		}
+	})),
+	...["gpt-5-codex"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.25 / 1e6,
+			output: 10 / 1e6
+		}
+	})),
+	...["gpt-5-pro", "gpt-5-pro-2025-10-06"].map((model) => ({
+		id: model,
+		cost: {
+			input: 15 / 1e6,
+			output: 120 / 1e6
+		}
+	})),
+	...[
+		"gpt-5.1",
+		"gpt-5.1-2025-11-13",
+		"gpt-5.1-chat-latest"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.25 / 1e6,
+			output: 10 / 1e6
+		}
+	})),
+	...["gpt-5.1-nano"].map((model) => ({
+		id: model,
+		cost: {
+			input: .05 / 1e6,
+			output: .4 / 1e6
+		}
+	})),
+	...["gpt-5.1-mini"].map((model) => ({
+		id: model,
+		cost: {
+			input: .25 / 1e6,
+			output: 2 / 1e6
+		}
+	})),
+	...["gpt-5.1-codex", "gpt-5.1-codex-max"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.25 / 1e6,
+			output: 10 / 1e6
+		}
+	})),
+	...[
+		"gpt-5.2",
+		"gpt-5.2-2025-12-11",
+		"gpt-5.2-chat-latest",
+		"gpt-5.2-codex"
+	].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.75 / 1e6,
+			output: 14 / 1e6
+		}
+	})),
+	...["gpt-5.2-pro", "gpt-5.2-pro-2025-12-11"].map((model) => ({
+		id: model,
+		cost: {
+			input: 15 / 1e6,
+			output: 120 / 1e6
+		}
+	})),
+	...["gpt-5.3-chat-latest", "gpt-5.3-codex"].map((model) => ({
+		id: model,
+		cost: {
+			input: 1.75 / 1e6,
+			output: 14 / 1e6
+		}
+	})),
+	...["gpt-5.3-codex-spark"].map((model) => ({
+		id: model,
+		cost: {
+			input: .5 / 1e6,
+			output: 4 / 1e6
+		}
+	})),
+	...["gpt-5.4", "gpt-5.4-2026-03-05"].map((model) => ({
+		id: model,
+		cost: {
+			input: 2.5 / 1e6,
+			output: 15 / 1e6
+		}
+	})),
+	...["gpt-5.4-pro", "gpt-5.4-pro-2026-03-05"].map((model) => ({
+		id: model,
+		cost: {
+			input: 30 / 1e6,
+			output: 180 / 1e6
+		}
+	})),
+	...["gpt-audio", "gpt-audio-2025-08-28"].map((model) => ({
+		id: model,
+		cost: {
+			input: 2.5 / 1e6,
+			output: 10 / 1e6,
+			audioInput: 40 / 1e6,
+			audioOutput: 80 / 1e6
+		}
+	})),
+	...["gpt-audio-mini", "gpt-audio-mini-2025-10-06"].map((model) => ({
+		id: model,
+		cost: {
+			input: .6 / 1e6,
+			output: 2.4 / 1e6,
+			audioInput: 10 / 1e6,
+			audioOutput: 20 / 1e6
+		}
+	}))
+];
+const OPENAI_DEEP_RESEARCH_MODELS = [...["o3-deep-research", "o3-deep-research-2025-06-26"].map((model) => ({
+	id: model,
+	cost: {
+		input: 10 / 1e6,
+		output: 40 / 1e6
+	}
+})), ...["o4-mini-deep-research", "o4-mini-deep-research-2025-06-26"].map((model) => ({
+	id: model,
+	cost: {
+		input: 2 / 1e6,
+		output: 8 / 1e6
+	}
+}))];
+const OPENAI_COMPLETION_MODELS = [
+	{
+		id: "gpt-3.5-turbo-instruct",
+		cost: {
+			input: 1.5 / 1e6,
+			output: 2 / 1e6
+		}
+	},
+	{ id: "text-davinci-002" },
+	{ id: "text-babbage-002" }
+];
+const OPENAI_REALTIME_MODELS = [
+	{
+		id: "gpt-realtime",
+		type: "chat",
+		cost: {
+			input: 32 / 1e6,
+			output: 64 / 1e6,
+			audioInput: 32 / 1e6,
+			audioOutput: 64 / 1e6
+		}
+	},
+	{
+		id: "gpt-realtime",
+		type: "chat",
+		cost: {
+			input: 4 / 1e6,
+			output: 16 / 1e6,
+			audioInput: 40 / 1e6,
+			audioOutput: 80 / 1e6
+		}
+	},
+	{
+		id: "gpt-4o-realtime-preview",
+		type: "chat",
+		cost: {
+			input: 5 / 1e6,
+			output: 20 / 1e6,
+			audioInput: 40 / 1e6,
+			audioOutput: 80 / 1e6
+		}
+	},
+	{
+		id: "gpt-4o-realtime-preview-2024-12-17",
+		type: "chat",
+		cost: {
+			input: 5 / 1e6,
+			output: 20 / 1e6,
+			audioInput: 40 / 1e6,
+			audioOutput: 80 / 1e6
+		}
+	},
+	{
+		id: "gpt-4o-realtime-preview-2024-10-01",
+		type: "chat",
+		cost: {
+			input: 5 / 1e6,
+			output: 20 / 1e6,
+			audioInput: 100 / 1e6,
+			audioOutput: 200 / 1e6
+		}
+	},
+	{
+		id: "gpt-4o-mini-realtime-preview",
+		type: "chat",
+		cost: {
+			input: .6 / 1e6,
+			output: 2.4 / 1e6,
+			audioInput: 10 / 1e6,
+			audioOutput: 20 / 1e6
+		}
+	},
+	{
+		id: "gpt-4o-mini-realtime-preview-2024-12-17",
+		type: "chat",
+		cost: {
+			input: .6 / 1e6,
+			output: 2.4 / 1e6,
+			audioInput: 10 / 1e6,
+			audioOutput: 20 / 1e6
+		}
+	},
+	{
+		id: "gpt-realtime-mini",
+		type: "chat",
+		cost: {
+			input: .6 / 1e6,
+			output: 2.4 / 1e6,
+			audioInput: 10 / 1e6,
+			audioOutput: 20 / 1e6
+		}
+	},
+	{
+		id: "gpt-realtime-mini-2025-10-06",
+		type: "chat",
+		cost: {
+			input: .6 / 1e6,
+			output: 2.4 / 1e6,
+			audioInput: 10 / 1e6,
+			audioOutput: 20 / 1e6
+		}
+	}
+];
+const OPENAI_TRANSCRIPTION_MODELS = [
+	{
+		id: "gpt-4o-transcribe",
+		cost: { perMinute: .006 }
+	},
+	{
+		id: "gpt-4o-mini-transcribe",
+		cost: { perMinute: .003 }
+	},
+	{
+		id: "gpt-4o-transcribe-diarize",
+		cost: { perMinute: .006 }
+	},
+	{
+		id: "whisper-1",
+		cost: { perMinute: .006 }
+	}
+];
+function calculateOpenAICost(modelName, config, promptTokens, completionTokens, audioPromptTokens, audioCompletionTokens) {
+	if (!audioPromptTokens && !audioCompletionTokens) return calculateCost(modelName, config, promptTokens, completionTokens, [
+		...OPENAI_CHAT_MODELS,
+		...OPENAI_COMPLETION_MODELS,
+		...OPENAI_REALTIME_MODELS,
+		...OPENAI_DEEP_RESEARCH_MODELS
+	]);
+	if (!Number.isFinite(promptTokens) || !Number.isFinite(completionTokens) || !Number.isFinite(audioPromptTokens) || !Number.isFinite(audioCompletionTokens) || typeof promptTokens === "undefined" || typeof completionTokens === "undefined" || typeof audioPromptTokens === "undefined" || typeof audioCompletionTokens === "undefined") return;
+	const model = [
+		...OPENAI_CHAT_MODELS,
+		...OPENAI_COMPLETION_MODELS,
+		...OPENAI_REALTIME_MODELS,
+		...OPENAI_DEEP_RESEARCH_MODELS
+	].find((m) => m.id === modelName);
+	if (!model || !model.cost) return;
+	let totalCost = 0;
+	const inputCost = config.cost ?? model.cost.input;
+	const outputCost = config.cost ?? model.cost.output;
+	totalCost += inputCost * promptTokens + outputCost * completionTokens;
+	if ("audioInput" in model.cost || "audioOutput" in model.cost) {
+		const audioInputCost = config.audioCost ?? model.cost.audioInput ?? 0;
+		const audioOutputCost = config.audioCost ?? model.cost.audioOutput ?? 0;
+		totalCost += audioInputCost * audioPromptTokens + audioOutputCost * audioCompletionTokens;
+	}
+	return totalCost;
+}
+function failApiCall(err) {
+	if (err instanceof OpenAI.APIError) {
+		const errorType = err.error?.type || err.type || "unknown";
+		const errorMessage = err.error?.message || err.message || "Unknown error";
+		return { error: `API error: ${errorType}${err.status ? ` ${err.status}` : ""} ${errorMessage}` };
+	}
+	return { error: `API error: ${String(err)}` };
+}
+function getTokenUsage(data, cached) {
+	if (data.usage) if (cached) return {
+		cached: data.usage.total_tokens,
+		total: data.usage.total_tokens
+	};
+	else return {
+		total: data.usage.total_tokens,
+		prompt: data.usage.prompt_tokens || 0,
+		completion: data.usage.completion_tokens || 0,
+		numRequests: 1,
+		...data.usage.completion_tokens_details ? { completionDetails: {
+			reasoning: data.usage.completion_tokens_details.reasoning_tokens,
+			acceptedPrediction: data.usage.completion_tokens_details.accepted_prediction_tokens,
+			rejectedPrediction: data.usage.completion_tokens_details.rejected_prediction_tokens
+		} } : {}
+	};
+	return {};
+}
+function validateFunctionCall(output, functions, vars) {
+	if (typeof output === "object" && "function_call" in output) output = output.function_call;
+	const functionCall = output;
+	if (typeof functionCall !== "object" || typeof functionCall.name !== "string" || typeof functionCall.arguments !== "string") throw new Error(`OpenAI did not return a valid-looking function call: ${JSON.stringify(functionCall)}`);
+	const interpolatedFunctions = maybeLoadFromExternalFileWithVars(functions, vars);
+	const functionArgs = JSON.parse(functionCall.arguments);
+	const functionName = functionCall.name;
+	const functionSchema = interpolatedFunctions?.find((f) => f.name === functionName)?.parameters;
+	if (!functionSchema) throw new Error(`Called "${functionName}", but there is no function with that name`);
+	const validate = ajv.compile(functionSchema);
+	if (!validate(functionArgs)) throw new Error(`Call to "${functionName}" does not match schema: ${JSON.stringify(validate.errors)}`);
+}
+function formatOpenAiError(data) {
+	let errorMessage = `API error: ${data.error.message}`;
+	if (data.error.type) errorMessage += `, Type: ${data.error.type}`;
+	if (data.error.code) errorMessage += `, Code: ${data.error.code}`;
+	errorMessage += "\n\n" + safeJsonStringify(data, true);
+	return errorMessage;
+}
+//#endregion
+export { calculateOpenAICost as a, getTokenUsage as c, OPENAI_TRANSCRIPTION_MODELS as i, validateFunctionCall as l, OPENAI_COMPLETION_MODELS as n, failApiCall as o, OPENAI_REALTIME_MODELS as r, formatOpenAiError as s, OPENAI_CHAT_MODELS as t };
+
+//# sourceMappingURL=util-6-GqIvzS.js.map

package/dist/src/{util-CoQjmE3u.js → util-B7T3SiBS.js}

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
-import { p as calculateCost } from "./fetch-CXZI9RRr.js";
-import { x as parseDataUrl } from "./transform-DeGlxb0D.js";
-
+import { p as calculateCost } from "./fetch-BMv0O527.js";
+import { x as parseDataUrl } from "./transform-Cgi24fJ7.js";
 //#region src/providers/anthropic/util.ts
 const ANTHROPIC_MODELS = [
 	...["claude-sonnet-4-6", "claude-sonnet-4-6-latest"].map((model) => ({
@@ -289,7 +288,7 @@ function transformWebSearchTool(config) {
 	if (config.cache_control) tool.cache_control = config.cache_control;
 	return tool;
 }
-
 //#endregion
 export { parseMessages as a, outputFromMessage as i, calculateAnthropicCost as n, processAnthropicTools as o, getTokenUsage as r, ANTHROPIC_MODELS as t };
-//# sourceMappingURL=util-CoQjmE3u.js.map
+
+//# sourceMappingURL=util-B7T3SiBS.js.map

package/dist/src/{util-D9eLdGfa.js → util-Betm42rL.js}

@@ -1,8 +1,7 @@
-import { d as getAjv, m as safeJsonStringify } from "./logger-CG1uZPbQ.js";
-import { l as maybeLoadFromExternalFileWithVars } from "./util-DQ984syk.js";
-import { g as calculateCost } from "./fetch-D3OHf-lV.js";
+import { d as getAjv, m as safeJsonStringify } from "./logger-CT3IKMKA.js";
+import { l as maybeLoadFromExternalFileWithVars } from "./util-Dlz_Wvgm.js";
+import { m as calculateCost } from "./fetch-60Gzydls.js";
 import OpenAI from "openai";
-
 //#region src/providers/openai/util.ts
 const ajv = getAjv();
 const OPENAI_CHAT_MODELS = [
@@ -594,7 +593,7 @@ function formatOpenAiError(data) {
 	errorMessage += "\n\n" + safeJsonStringify(data, true);
 	return errorMessage;
 }
-
 //#endregion
 export { calculateOpenAICost as a, getTokenUsage as c, OPENAI_TRANSCRIPTION_MODELS as i, validateFunctionCall as l, OPENAI_COMPLETION_MODELS as n, failApiCall as o, OPENAI_REALTIME_MODELS as r, formatOpenAiError as s, OPENAI_CHAT_MODELS as t };
-//# sourceMappingURL=util-D9eLdGfa.js.map
+
+//# sourceMappingURL=util-Betm42rL.js.map

package/dist/src/{util-Bm_-UMD_.js → util-C-PPYSMq.js}

@@ -1,9 +1,8 @@
 #!/usr/bin/env node
-import { g as safeJsonStringify, p as getAjv } from "./logger-LSBxlt7a.js";
-import { p as calculateCost } from "./fetch-CXZI9RRr.js";
-import { m as maybeLoadFromExternalFileWithVars } from "./util-_h4pVqrz.js";
+import { g as safeJsonStringify, p as getAjv } from "./logger-KkObSCzq.js";
+import { p as calculateCost } from "./fetch-BMv0O527.js";
+import { m as maybeLoadFromExternalFileWithVars } from "./util-YT5HPZaS.js";
 import OpenAI from "openai";
-
 //#region src/providers/openai/util.ts
 const ajv = getAjv();
 const OPENAI_CHAT_MODELS = [
@@ -595,7 +594,7 @@ function formatOpenAiError(data) {
 	errorMessage += "\n\n" + safeJsonStringify(data, true);
 	return errorMessage;
 }
-
 //#endregion
 export { calculateOpenAICost as a, getTokenUsage as c, OPENAI_TRANSCRIPTION_MODELS as i, validateFunctionCall as l, OPENAI_COMPLETION_MODELS as n, failApiCall as o, OPENAI_REALTIME_MODELS as r, formatOpenAiError as s, OPENAI_CHAT_MODELS as t };
-//# sourceMappingURL=util-Bm_-UMD_.js.map
+
+//# sourceMappingURL=util-C-PPYSMq.js.map