promptfoo 0.121.5 → 0.121.7

package/dist/src/image-B4oBtu6J.js

@@ -0,0 +1,443 @@
+#!/usr/bin/env node
+import { s as logger } from "./logger-BbY6ypFL.js";
+import { _ as REQUEST_TIMEOUT_MS } from "./fetch-DXUnXkVU.js";
+import { a as fetchWithCache } from "./cache-DpPWrkTE.js";
+import { t as OpenAiGenericProvider } from "./openai-CJrsh9n4.js";
+import { s as formatOpenAiError } from "./util-5WnCSb0h.js";
+import { t as ellipsize } from "./text-Db-Wt2u2.js";
+//#region src/providers/openai/image.ts
+const DALLE2_VALID_SIZES = [
+	"256x256",
+	"512x512",
+	"1024x1024"
+];
+const DALLE3_VALID_SIZES = [
+	"1024x1024",
+	"1792x1024",
+	"1024x1792"
+];
+const GPT_IMAGE1_VALID_SIZES = [
+	"1024x1024",
+	"1024x1536",
+	"1536x1024"
+];
+const GPT_IMAGE2_MAX_EDGE = 3840;
+const GPT_IMAGE2_MIN_PIXELS = 655360;
+const GPT_IMAGE2_MAX_PIXELS = 8294400;
+const DATED_GPT_IMAGE2_MODEL_PATTERN = /^gpt-image-2-\d{4}-\d{2}-\d{2}$/;
+const DEFAULT_SIZE = "1024x1024";
+const DALLE2_COSTS = {
+	"256x256": .016,
+	"512x512": .018,
+	"1024x1024": .02
+};
+const DALLE3_COSTS = {
+	standard_1024x1024: .04,
+	standard_1024x1792: .08,
+	standard_1792x1024: .08,
+	hd_1024x1024: .08,
+	hd_1024x1792: .12,
+	hd_1792x1024: .12
+};
+const GPT_IMAGE1_COSTS = {
+	low_1024x1024: .011,
+	low_1024x1536: .016,
+	low_1536x1024: .016,
+	medium_1024x1024: .042,
+	medium_1024x1536: .063,
+	medium_1536x1024: .063,
+	high_1024x1024: .167,
+	high_1024x1536: .25,
+	high_1536x1024: .25
+};
+const GPT_IMAGE1_MINI_COSTS = {
+	low_1024x1024: .005,
+	low_1024x1536: .006,
+	low_1536x1024: .006,
+	medium_1024x1024: .011,
+	medium_1024x1536: .015,
+	medium_1536x1024: .015,
+	high_1024x1024: .036,
+	high_1024x1536: .052,
+	high_1536x1024: .052
+};
+const GPT_IMAGE2_COSTS = {
+	low_1024x1024: .006,
+	low_1024x1536: .005,
+	low_1536x1024: .005,
+	medium_1024x1024: .053,
+	medium_1024x1536: .041,
+	medium_1536x1024: .041,
+	high_1024x1024: .211,
+	high_1024x1536: .165,
+	high_1536x1024: .165
+};
+const GPT_IMAGE1_5_COSTS = {
+	low_1024x1024: .064,
+	low_1024x1536: .096,
+	low_1536x1024: .096,
+	medium_1024x1024: .128,
+	medium_1024x1536: .192,
+	medium_1536x1024: .192,
+	high_1024x1024: .192,
+	high_1024x1536: .288,
+	high_1536x1024: .288
+};
+const GPT_IMAGE_QUALITIES = [
+	"low",
+	"medium",
+	"high",
+	"auto"
+];
+const GPT_IMAGE1_BACKGROUNDS = [
+	"transparent",
+	"opaque",
+	"auto"
+];
+const GPT_IMAGE2_BACKGROUNDS = ["opaque", "auto"];
+const GPT_IMAGE_OUTPUT_FORMATS = [
+	"png",
+	"jpeg",
+	"webp"
+];
+const GPT_IMAGE_MODERATION_VALUES = ["auto", "low"];
+function isGptImage2(model) {
+	return model === "gpt-image-2" || DATED_GPT_IMAGE2_MODEL_PATTERN.test(model);
+}
+function isGptImage1(model) {
+	return model === "gpt-image-1" || model.startsWith("gpt-image-1-2025");
+}
+function isGptImage1Mini(model) {
+	return model === "gpt-image-1-mini" || model.startsWith("gpt-image-1-mini-2025");
+}
+function isGptImage15(model) {
+	return model === "gpt-image-1.5" || model.startsWith("gpt-image-1.5-2025");
+}
+function isGptImageModel(model) {
+	return isGptImage2(model) || isGptImage1(model) || isGptImage1Mini(model) || isGptImage15(model);
+}
+function getGptImageModelDisplayName(model) {
+	if (isGptImage2(model)) return "GPT Image 2";
+	if (isGptImage15(model)) return "GPT Image 1.5";
+	if (isGptImage1Mini(model)) return "GPT Image 1 Mini";
+	return "GPT Image 1";
+}
+function validateGptImage2Size(size) {
+	if (size === "auto") return { valid: true };
+	const constraints = "Valid sizes are auto or WIDTHxHEIGHT where both dimensions are multiples of 16, the maximum edge is 3840px, the long edge to short edge ratio is at most 3:1, and total pixels are between 655,360 and 8,294,400.";
+	const sizeMatch = /^(\d+)x(\d+)$/.exec(size);
+	if (!sizeMatch) return {
+		valid: false,
+		message: `Invalid size "${size}" for GPT Image 2. ${constraints}`
+	};
+	const width = Number(sizeMatch[1]);
+	const height = Number(sizeMatch[2]);
+	const longEdge = Math.max(width, height);
+	const shortEdge = Math.min(width, height);
+	const totalPixels = width * height;
+	if (width <= 0 || height <= 0 || longEdge > GPT_IMAGE2_MAX_EDGE || width % 16 !== 0 || height % 16 !== 0 || longEdge / shortEdge > 3 || totalPixels < GPT_IMAGE2_MIN_PIXELS || totalPixels > GPT_IMAGE2_MAX_PIXELS) return {
+		valid: false,
+		message: `Invalid size "${size}" for GPT Image 2. ${constraints}`
+	};
+	return { valid: true };
+}
+function validateSizeForModel(size, model) {
+	if (model === "dall-e-3" && !DALLE3_VALID_SIZES.includes(size)) return {
+		valid: false,
+		message: `Invalid size "${size}" for DALL-E 3. Valid sizes are: ${DALLE3_VALID_SIZES.join(", ")}`
+	};
+	if (model === "dall-e-2" && !DALLE2_VALID_SIZES.includes(size)) return {
+		valid: false,
+		message: `Invalid size "${size}" for DALL-E 2. Valid sizes are: ${DALLE2_VALID_SIZES.join(", ")}`
+	};
+	if (isGptImage2(model)) return validateGptImage2Size(size);
+	if (isGptImageModel(model) && size !== "auto" && !GPT_IMAGE1_VALID_SIZES.includes(size)) return {
+		valid: false,
+		message: `Invalid size "${size}" for ${getGptImageModelDisplayName(model)}. Valid sizes are: ${GPT_IMAGE1_VALID_SIZES.join(", ")}, auto`
+	};
+	return { valid: true };
+}
+function validateNForModel(n, model) {
+	if (n === void 0) return { valid: true };
+	if (typeof n !== "number" || !Number.isInteger(n) || n < 1) return {
+		valid: false,
+		message: "n must be a positive integer."
+	};
+	if (model === "dall-e-3" && n !== 1) return {
+		valid: false,
+		message: "n must be 1 for DALL-E 3."
+	};
+	if (n > 10) return {
+		valid: false,
+		message: "n must be between 1 and 10."
+	};
+	return { valid: true };
+}
+function validateGptImageQualityForModel(quality, model) {
+	if (!isGptImageModel(model) || quality === void 0) return { valid: true };
+	if (typeof quality !== "string" || !GPT_IMAGE_QUALITIES.includes(quality)) return {
+		valid: false,
+		message: `Invalid quality "${String(quality)}" for ${getGptImageModelDisplayName(model)}. Valid qualities are: ${GPT_IMAGE_QUALITIES.join(", ")}.`
+	};
+	return { valid: true };
+}
+function validateBackgroundForModel(background, outputFormat, model) {
+	if (!isGptImageModel(model) || background === void 0) return { valid: true };
+	if (typeof background !== "string") return {
+		valid: false,
+		message: `Invalid background "${String(background)}" for ${getGptImageModelDisplayName(model)}.`
+	};
+	if (isGptImage2(model) && background === "transparent") return {
+		valid: false,
+		message: "background: \"transparent\" is not supported for GPT Image 2. Use \"opaque\" or \"auto\"."
+	};
+	if (isGptImage2(model) && !GPT_IMAGE2_BACKGROUNDS.includes(background)) return {
+		valid: false,
+		message: `Invalid background "${background}" for GPT Image 2. Valid backgrounds are: ${GPT_IMAGE2_BACKGROUNDS.join(", ")}.`
+	};
+	if (!isGptImage2(model) && !GPT_IMAGE1_BACKGROUNDS.includes(background)) return {
+		valid: false,
+		message: `Invalid background "${background}" for ${getGptImageModelDisplayName(model)}. Valid backgrounds are: ${GPT_IMAGE1_BACKGROUNDS.join(", ")}.`
+	};
+	if (background === "transparent" && outputFormat === "jpeg") return {
+		valid: false,
+		message: "background: \"transparent\" is not supported with output_format: \"jpeg\". Use \"png\" or \"webp\", or choose \"opaque\" or \"auto\" background."
+	};
+	return { valid: true };
+}
+function validateOutputFormatForModel(outputFormat, model) {
+	if (!isGptImageModel(model) || outputFormat === void 0) return { valid: true };
+	if (typeof outputFormat !== "string" || !GPT_IMAGE_OUTPUT_FORMATS.includes(outputFormat)) return {
+		valid: false,
+		message: `Invalid output_format "${String(outputFormat)}" for ${getGptImageModelDisplayName(model)}. Valid output formats are: ${GPT_IMAGE_OUTPUT_FORMATS.join(", ")}.`
+	};
+	return { valid: true };
+}
+function validateOutputCompressionForModel(outputCompression, outputFormat, model) {
+	if (!isGptImageModel(model) || outputCompression === void 0) return { valid: true };
+	if (typeof outputCompression !== "number" || !Number.isFinite(outputCompression) || outputCompression < 0 || outputCompression > 100) return {
+		valid: false,
+		message: "output_compression must be a number between 0 and 100."
+	};
+	if (outputFormat !== "jpeg" && outputFormat !== "webp") return {
+		valid: false,
+		message: "output_compression is only supported when output_format is \"jpeg\" or \"webp\". Set output_format to \"jpeg\" or \"webp\", or remove output_compression."
+	};
+	return { valid: true };
+}
+function validateModerationForModel(moderation, model) {
+	if (!isGptImageModel(model) || moderation === void 0) return { valid: true };
+	if (typeof moderation !== "string" || !GPT_IMAGE_MODERATION_VALUES.includes(moderation)) return {
+		valid: false,
+		message: `Invalid moderation "${String(moderation)}" for ${getGptImageModelDisplayName(model)}. Valid moderation values are: ${GPT_IMAGE_MODERATION_VALUES.join(", ")}.`
+	};
+	return { valid: true };
+}
+function validateUnsupportedImageOptions(config) {
+	if (config.stream === true) return {
+		valid: false,
+		message: "Streaming image generation is not supported by the openai:image provider yet. Remove stream, or use a provider that supports streaming image events."
+	};
+	if (config.partial_images !== void 0) return {
+		valid: false,
+		message: "partial_images is only supported for streaming image generation, which the openai:image provider does not support yet."
+	};
+	if (config.image !== void 0 || config.mask !== void 0 || config.input_fidelity !== void 0) return {
+		valid: false,
+		message: "Image edit/reference inputs are not implemented in the openai:image provider yet; only text-to-image generation is supported."
+	};
+	return { valid: true };
+}
+function validateImageRequestConfig(config, model, size) {
+	return [
+		validateUnsupportedImageOptions(config),
+		validateNForModel(config.n, model),
+		validateSizeForModel(size, model),
+		validateGptImageQualityForModel("quality" in config ? config.quality : void 0, model),
+		validateOutputFormatForModel("output_format" in config ? config.output_format : void 0, model),
+		validateBackgroundForModel("background" in config ? config.background : void 0, "output_format" in config ? config.output_format : void 0, model),
+		validateOutputCompressionForModel("output_compression" in config ? config.output_compression : void 0, "output_format" in config ? config.output_format : void 0, model),
+		validateModerationForModel("moderation" in config ? config.moderation : void 0, model)
+	].find((validation) => !validation.valid) || { valid: true };
+}
+function getMimeTypeForOutputFormat(outputFormat) {
+	switch (outputFormat) {
+		case "jpeg": return "image/jpeg";
+		case "webp": return "image/webp";
+		default: return "image/png";
+	}
+}
+function inferMimeTypeFromUrl(url) {
+	try {
+		const pathname = new URL(url).pathname.toLowerCase();
+		if (pathname.endsWith(".jpg") || pathname.endsWith(".jpeg")) return "image/jpeg";
+		if (pathname.endsWith(".webp")) return "image/webp";
+		if (pathname.endsWith(".gif")) return "image/gif";
+		if (pathname.endsWith(".svg")) return "image/svg+xml";
+		if (pathname.endsWith(".png")) return "image/png";
+	} catch {}
+}
+function buildStructuredImageOutputs(data, outputFormat) {
+	if (!Array.isArray(data.data) || data.data.length === 0) return;
+	return data.data.map((item) => {
+		if (item.b64_json) {
+			const mimeType = getMimeTypeForOutputFormat(outputFormat);
+			return {
+				data: `data:${mimeType};base64,${item.b64_json}`,
+				mimeType
+			};
+		}
+		if (item.url) {
+			const mimeType = inferMimeTypeFromUrl(item.url);
+			return mimeType ? {
+				data: item.url,
+				mimeType
+			} : { data: item.url };
+		}
+		return null;
+	}).filter((item) => item !== null);
+}
+function formatOutput(data, prompt, responseFormat, outputFormat) {
+	if (responseFormat === "b64_json") {
+		const b64Json = data.data[0].b64_json;
+		if (!b64Json) return { error: `No base64 image data found in response: ${JSON.stringify(data)}` };
+		return `data:${getMimeTypeForOutputFormat(outputFormat)};base64,${b64Json}`;
+	} else {
+		const url = data.data[0].url;
+		if (!url) return { error: `No image URL found in response: ${JSON.stringify(data)}` };
+		return `![${ellipsize(prompt.replace(/\r?\n|\r/g, " ").replace(/\[/g, "(").replace(/\]/g, ")"), 50)}](${url})`;
+	}
+}
+function prepareRequestBody(model, prompt, size, responseFormat, config) {
+	const body = {
+		model,
+		prompt,
+		n: config.n ?? 1,
+		size
+	};
+	if ("user" in config && config.user) body.user = config.user;
+	if (!isGptImageModel(model)) body.response_format = responseFormat;
+	if (model === "dall-e-3") {
+		if ("quality" in config && config.quality) body.quality = config.quality;
+		if ("style" in config && config.style) body.style = config.style;
+	}
+	if (isGptImageModel(model)) {
+		if ("quality" in config && config.quality) body.quality = config.quality;
+		if ("background" in config && config.background) body.background = config.background;
+		if ("output_format" in config && config.output_format) body.output_format = config.output_format;
+		if ("output_compression" in config && config.output_compression !== void 0) body.output_compression = config.output_compression;
+		if ("moderation" in config && config.moderation) body.moderation = config.moderation;
+	}
+	return body;
+}
+function calculateImageCost(model, size, quality, n = 1) {
+	const imageQuality = quality || "standard";
+	const gptImageQuality = quality === "medium" || quality === "high" || quality === "low" ? quality : "low";
+	if (model === "dall-e-3") return (DALLE3_COSTS[`${imageQuality}_${size}`] || DALLE3_COSTS["standard_1024x1024"]) * n;
+	else if (model === "dall-e-2") return (DALLE2_COSTS[size] || DALLE2_COSTS["1024x1024"]) * n;
+	else if (isGptImage2(model)) {
+		if (quality !== "medium" && quality !== "high" && quality !== "low") return;
+		const costPerImage = GPT_IMAGE2_COSTS[`${gptImageQuality}_${size}`];
+		if (costPerImage === void 0) return;
+		return costPerImage * n;
+	} else if (isGptImage1(model)) return (GPT_IMAGE1_COSTS[`${gptImageQuality}_${size}`] || GPT_IMAGE1_COSTS["low_1024x1024"]) * n;
+	else if (isGptImage1Mini(model)) return (GPT_IMAGE1_MINI_COSTS[`${gptImageQuality}_${size}`] || GPT_IMAGE1_MINI_COSTS["low_1024x1024"]) * n;
+	else if (isGptImage15(model)) return (GPT_IMAGE1_5_COSTS[`${gptImageQuality}_${size}`] || GPT_IMAGE1_5_COSTS["low_1024x1024"]) * n;
+	return .04 * n;
+}
+function getImageTokenUsage(data, cached) {
+	if (!data.usage) return;
+	const prompt = data.usage.prompt_tokens ?? data.usage.input_tokens ?? 0;
+	const completion = data.usage.completion_tokens ?? data.usage.output_tokens ?? 0;
+	const total = data.usage.total_tokens ?? prompt + completion;
+	if (cached) return {
+		cached: total,
+		total
+	};
+	return {
+		prompt,
+		completion,
+		total,
+		numRequests: 1
+	};
+}
+async function callOpenAiImageApi(url, body, headers, timeout) {
+	return await fetchWithCache(url, {
+		method: "POST",
+		headers,
+		body: JSON.stringify(body)
+	}, timeout);
+}
+async function processApiResponse(data, prompt, responseFormat, cached, model, size, latencyMs, quality, n = 1, outputFormat) {
+	if (data.error) {
+		await data?.deleteFromCache?.();
+		return { error: formatOpenAiError(data) };
+	}
+	try {
+		const formattedOutput = formatOutput(data, prompt, responseFormat, outputFormat);
+		if (typeof formattedOutput === "object") return formattedOutput;
+		const cost = cached ? 0 : calculateImageCost(model, size, quality, n);
+		const tokenUsage = getImageTokenUsage(data, cached);
+		return {
+			output: formattedOutput,
+			images: buildStructuredImageOutputs(data, outputFormat),
+			cached,
+			latencyMs,
+			...cost === void 0 ? {} : { cost },
+			...tokenUsage ? { tokenUsage } : {},
+			...data.usage ? { metadata: { usage: data.usage } } : {},
+			...responseFormat === "b64_json" ? {
+				isBase64: true,
+				format: "json"
+			} : {}
+		};
+	} catch (err) {
+		await data?.deleteFromCache?.();
+		return { error: `API error: ${String(err)}: ${JSON.stringify(data)}` };
+	}
+}
+var OpenAiImageProvider = class extends OpenAiGenericProvider {
+	config;
+	constructor(modelName, options = {}) {
+		super(modelName, options);
+		this.config = options.config || {};
+	}
+	async callApi(prompt, context, _callApiOptions) {
+		if (this.requiresApiKey() && !this.getApiKey()) throw new Error(this.getMissingApiKeyErrorMessage());
+		const config = {
+			...this.config,
+			...context?.prompt?.config
+		};
+		const model = config.model || this.modelName;
+		const operation = "operation" in config && config.operation || "generation";
+		const responseFormat = isGptImageModel(model) ? "b64_json" : config.response_format || "url";
+		if (operation !== "generation") return { error: `Only 'generation' operations are currently supported. '${operation}' operations are not implemented.` };
+		const endpoint = "/images/generations";
+		const size = config.size || DEFAULT_SIZE;
+		const requestValidation = validateImageRequestConfig(config, model, size);
+		if (!requestValidation.valid) return { error: requestValidation.message };
+		const body = prepareRequestBody(model, prompt, size, responseFormat, config);
+		const headers = {
+			"Content-Type": "application/json",
+			...this.getApiKey() ? { Authorization: `Bearer ${this.getApiKey()}` } : {},
+			...this.getOrganization() ? { "OpenAI-Organization": this.getOrganization() } : {},
+			...config.headers
+		};
+		let data, status, statusText;
+		let cached = false;
+		let latencyMs;
+		try {
+			({data, cached, status, statusText, latencyMs} = await callOpenAiImageApi(`${this.getApiUrl()}${endpoint}`, body, headers, REQUEST_TIMEOUT_MS));
+			if (status < 200 || status >= 300) return { error: `API error: ${status} ${statusText}\n${typeof data === "string" ? data : JSON.stringify(data)}` };
+		} catch (err) {
+			logger.error(`API call error: ${String(err)}`);
+			await data?.deleteFromCache?.();
+			return { error: `API call error: ${String(err)}` };
+		}
+		return processApiResponse(data, prompt, responseFormat, cached, model, size, latencyMs, config.quality, config.n ?? 1, "output_format" in config ? config.output_format : void 0);
+	}
+};
+//#endregion
+export { formatOutput as i, buildStructuredImageOutputs as n, callOpenAiImageApi as r, OpenAiImageProvider as t };
+
+//# sourceMappingURL=image-B4oBtu6J.js.map

package/dist/src/{image-Dr_3I3nK.js → image-BN-hjLL9.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 import { O as getEnvString, s as logger } from "./logger-BbY6ypFL.js";
-import { m as REQUEST_TIMEOUT_MS } from "./fetch-B6ch2nU2.js";
-import { a as fetchWithCache } from "./cache-BI5BY7ey.js";
+import { _ as REQUEST_TIMEOUT_MS } from "./fetch-DXUnXkVU.js";
+import { a as fetchWithCache } from "./cache-DpPWrkTE.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -161,4 +161,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-Dr_3I3nK.js.map
+//# sourceMappingURL=image-BN-hjLL9.js.map

package/dist/src/{image-BeWaInPF.js → image-B_fPIwdg.js}

@@ -1,6 +1,6 @@
 import { T as getEnvString, a as logger } from "./logger-Ct2S6Yx-.js";
-import { u as REQUEST_TIMEOUT_MS } from "./fetch-D9xxyC1p.js";
-import { r as fetchWithCache } from "./cache-Bzttsk0X.js";
+import { p as REQUEST_TIMEOUT_MS } from "./fetch-It34O8Ur.js";
+import { r as fetchWithCache } from "./cache-DIXbtkNO.js";
 //#region src/providers/hyperbolic/image.ts
 const HYPERBOLIC_IMAGE_MODELS = [
 	{
@@ -160,4 +160,4 @@ function createHyperbolicImageProvider(providerPath, options = {}) {
 //#endregion
 export { createHyperbolicImageProvider };
 
-//# sourceMappingURL=image-BeWaInPF.js.map
+//# sourceMappingURL=image-B_fPIwdg.js.map