promptfoo 0.121.1 → 0.121.3

package/README.md

@@ -20,6 +20,8 @@
   <a href="https://discord.gg/promptfoo">Discord</a>
 </p>
 
+> Update (March 16, 2026): Promptfoo has joined OpenAI. Promptfoo remains open source and MIT licensed. Read the [company update](https://www.promptfoo.dev/blog/promptfoo-joining-openai/).
+
 ## Quick Start
 
 ```sh

package/dist/src/{accounts-xrUGFA6n.js → accounts-B2XmGjty.js}

@@ -1,5 +1,5 @@
-import { C as isCI, b as getEnvString, i as logger } from "./logger-CT3IKMKA.js";
-import { O as TERMINAL_MAX_WIDTH, a as CloudConfig, c as writeGlobalConfig, l as writeGlobalConfigPartial, r as fetchWithTimeout, s as readGlobalConfig } from "./fetch-60Gzydls.js";
+import { C as getEnvString, E as isCI, a as logger } from "./logger-DO8_zM18.js";
+import { O as TERMINAL_MAX_WIDTH, a as CloudConfig, c as writeGlobalConfig, l as writeGlobalConfigPartial, r as fetchWithTimeout, s as readGlobalConfig } from "./fetch-CVAtKnI3.js";
 import chalk from "chalk";
 import { z } from "zod";
 import input from "@inquirer/input";
@@ -76,7 +76,7 @@ function getAuthMethod() {
 * Used by both CLI and server routes
 */
 async function checkEmailStatus(options) {
-	const { default: telemetry } = await import("./telemetry-C15ziL8u.js").then((n) => n.n);
+	const { default: telemetry } = await import("./telemetry-BugWqKiu.js").then((n) => n.n);
 	const ciMode = isCI();
 	const userEmail = ciMode ? CI_PLACEHOLDER_EMAIL : getUserEmail();
 	if (!userEmail) return {
@@ -125,7 +125,7 @@ async function checkEmailStatus(options) {
 	}
 }
 async function promptForEmailUnverified() {
-	const { default: telemetry } = await import("./telemetry-C15ziL8u.js").then((n) => n.n);
+	const { default: telemetry } = await import("./telemetry-BugWqKiu.js").then((n) => n.n);
 	const ciMode = isCI();
 	const existingEmail = getUserEmail();
 	let email = ciMode ? CI_PLACEHOLDER_EMAIL : existingEmail;
@@ -198,4 +198,4 @@ async function checkEmailStatusAndMaybeExit(options) {
 //#endregion
 export { getUserId as a, setUserEmail as c, getUserEmail as i, getAuthMethod as n, isLoggedIntoCloud as o, getAuthor as r, promptForEmailUnverified as s, checkEmailStatusAndMaybeExit as t };
 
-//# sourceMappingURL=accounts-xrUGFA6n.js.map
+//# sourceMappingURL=accounts-B2XmGjty.js.map

package/dist/src/{accounts-Bx-x3bmW.cjs → accounts-BPyfpSeU.cjs}

@@ -1,5 +1,5 @@
-const require_logger = require("./logger-Cp1GPUjj.cjs");
-const require_fetch = require("./fetch-BxUk8odA.cjs");
+const require_logger = require("./logger-D5iKBpu_.cjs");
+const require_fetch = require("./fetch-BnR9wSnm.cjs");
 let chalk = require("chalk");
 chalk = require_logger.__toESM(chalk);
 let zod = require("zod");
@@ -78,7 +78,7 @@ function getAuthMethod() {
 * Used by both CLI and server routes
 */
 async function checkEmailStatus(options) {
-	const { default: telemetry } = await Promise.resolve().then(() => require("./telemetry-5BCRNBbe.cjs"));
+	const { default: telemetry } = await Promise.resolve().then(() => require("./telemetry-p8Pwqm1i.cjs"));
 	const ciMode = require_logger.isCI();
 	const userEmail = ciMode ? CI_PLACEHOLDER_EMAIL : getUserEmail();
 	if (!userEmail) return {
@@ -127,7 +127,7 @@ async function checkEmailStatus(options) {
 	}
 }
 async function promptForEmailUnverified() {
-	const { default: telemetry } = await Promise.resolve().then(() => require("./telemetry-5BCRNBbe.cjs"));
+	const { default: telemetry } = await Promise.resolve().then(() => require("./telemetry-p8Pwqm1i.cjs"));
 	const ciMode = require_logger.isCI();
 	const existingEmail = getUserEmail();
 	let email = ciMode ? CI_PLACEHOLDER_EMAIL : existingEmail;
@@ -247,4 +247,4 @@ Object.defineProperty(exports, "setUserEmail", {
 	}
 });
 
-//# sourceMappingURL=accounts-Bx-x3bmW.cjs.map
+//# sourceMappingURL=accounts-BPyfpSeU.cjs.map

package/dist/src/{accounts-CMqkzrVf.js → accounts-CFLK3mnD.js}

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
-import { C as getEnvString, E as isCI, o as logger } from "./logger-KkObSCzq.js";
-import { D as TERMINAL_MAX_WIDTH, i as fetchWithTimeout } from "./fetch-BMv0O527.js";
-import { c as writeGlobalConfigPartial, o as readGlobalConfig, r as CloudConfig, s as writeGlobalConfig } from "./cloud-Bc9526yV.js";
+import { E as getEnvString, k as isCI, s as logger } from "./logger-BcJBzSSA.js";
+import { D as TERMINAL_MAX_WIDTH, i as fetchWithTimeout } from "./fetch-DoVRJZhJ.js";
+import { c as writeGlobalConfigPartial, o as readGlobalConfig, r as CloudConfig, s as writeGlobalConfig } from "./cloud-z8KZpUoa.js";
 import { z } from "zod";
 import input from "@inquirer/input";
 import chalk from "chalk";
@@ -83,7 +83,7 @@ function getAuthMethod() {
 * Used by both CLI and server routes
 */
 async function checkEmailStatus(options) {
-	const { default: telemetry } = await import("./telemetry-D4W5hboe.js");
+	const { default: telemetry } = await import("./telemetry-Yig0Tino.js");
 	const ciMode = isCI();
 	const userEmail = ciMode ? CI_PLACEHOLDER_EMAIL : getUserEmail();
 	if (!userEmail) return {
@@ -132,7 +132,7 @@ async function checkEmailStatus(options) {
 	}
 }
 async function promptForEmailUnverified() {
-	const { default: telemetry } = await import("./telemetry-D4W5hboe.js");
+	const { default: telemetry } = await import("./telemetry-Yig0Tino.js");
 	const ciMode = isCI();
 	const existingEmail = getUserEmail();
 	let email = ciMode ? CI_PLACEHOLDER_EMAIL : existingEmail;
@@ -205,4 +205,4 @@ async function checkEmailStatusAndMaybeExit(options) {
 //#endregion
 export { getAuthor as a, isLoggedIntoCloud as c, getAuthMethod as i, promptForEmailUnverified as l, checkEmailStatusAndMaybeExit as n, getUserEmail as o, clearUserEmail as r, getUserId as s, checkEmailStatus as t, setUserEmail as u };
 
-//# sourceMappingURL=accounts-CMqkzrVf.js.map
+//# sourceMappingURL=accounts-CFLK3mnD.js.map

package/dist/src/{accounts-BgNJDBE6.js → accounts-Xatc0RYb.js}

@@ -1,5 +1,5 @@
-import { C as isCI, b as getEnvString, i as logger } from "./logger-DLcq4dWf.js";
-import { O as TERMINAL_MAX_WIDTH, a as CloudConfig, c as writeGlobalConfig, l as writeGlobalConfigPartial, r as fetchWithTimeout, s as readGlobalConfig } from "./fetch-4M3YRaqL.js";
+import { C as getEnvString, E as isCI, a as logger } from "./logger-BnkjG2jt.js";
+import { O as TERMINAL_MAX_WIDTH, a as CloudConfig, c as writeGlobalConfig, l as writeGlobalConfigPartial, r as fetchWithTimeout, s as readGlobalConfig } from "./fetch-BiYv2BZc.js";
 import chalk from "chalk";
 import input from "@inquirer/input";
 import { z } from "zod";
@@ -81,7 +81,7 @@ function getAuthMethod() {
 * Used by both CLI and server routes
 */
 async function checkEmailStatus(options) {
-	const { default: telemetry } = await import("./telemetry-DMb2Mpfm.js").then((n) => n.r);
+	const { default: telemetry } = await import("./telemetry-DPXLd7UE.js").then((n) => n.r);
 	const ciMode = isCI();
 	const userEmail = ciMode ? CI_PLACEHOLDER_EMAIL : getUserEmail();
 	if (!userEmail) return {
@@ -130,7 +130,7 @@ async function checkEmailStatus(options) {
 	}
 }
 async function promptForEmailUnverified() {
-	const { default: telemetry } = await import("./telemetry-DMb2Mpfm.js").then((n) => n.r);
+	const { default: telemetry } = await import("./telemetry-DPXLd7UE.js").then((n) => n.r);
 	const ciMode = isCI();
 	const existingEmail = getUserEmail();
 	let email = ciMode ? CI_PLACEHOLDER_EMAIL : existingEmail;
@@ -203,4 +203,4 @@ async function checkEmailStatusAndMaybeExit(options) {
 //#endregion
 export { getAuthor as a, isLoggedIntoCloud as c, getAuthMethod as i, promptForEmailUnverified as l, checkEmailStatusAndMaybeExit as n, getUserEmail as o, clearUserEmail as r, getUserId as s, checkEmailStatus as t, setUserEmail as u };
 
-//# sourceMappingURL=accounts-BgNJDBE6.js.map
+//# sourceMappingURL=accounts-Xatc0RYb.js.map

package/dist/src/{agentic-utils-BKIN5PKu.js → agentic-utils-36epdqwB.js}

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
-import { o as logger } from "./logger-KkObSCzq.js";
-import { o as getCache, s as isCacheEnabled } from "./cache-CG0SlR1d.js";
+import { s as logger } from "./logger-BcJBzSSA.js";
+import { o as getCache, s as isCacheEnabled } from "./cache-DSqR6ezl.js";
 import fs from "fs";
 import path from "path";
 import dedent from "dedent";
@@ -151,4 +151,4 @@ async function cacheResponse(cacheResult, response, debugContext) {
 //#endregion
 export { initializeAgenticCache as i, generateCacheKey as n, getCachedResponse as r, cacheResponse as t };
 
-//# sourceMappingURL=agentic-utils-BKIN5PKu.js.map
+//# sourceMappingURL=agentic-utils-36epdqwB.js.map

package/dist/src/{cometapi-DkXrKi5z.js → agentic-utils-D8yXo5Lm.js}

@@ -1,17 +1,5 @@
-import { b as getEnvString, i as logger } from "./logger-CT3IKMKA.js";
-import "./esm-Cd1AjG1D.js";
-import "./pythonUtils-D5nxkQ0P.js";
-import "./types-CLKiCBW3.js";
-import "./util-Dlz_Wvgm.js";
-import "./fetch-60Gzydls.js";
-import { a as isCacheEnabled, i as getCache } from "./cache-8XhNqPKW.js";
-import "./genaiTracer-D3fD9dNV.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-CznLWr_D.js";
-import "./transform-DGLazrMm.js";
-import "./openai-DElQ-fPX.js";
-import "./util-Betm42rL.js";
-import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-C_P3ypkJ.js";
-import { t as OpenAiImageProvider } from "./image-BmEZqVmk.js";
+import { a as logger } from "./logger-DO8_zM18.js";
+import { a as isCacheEnabled, i as getCache } from "./cache-CaT5tPgo.js";
 import fs from "fs";
 import path from "path";
 import dedent from "dedent";
@@ -160,51 +148,6 @@ async function cacheResponse(cacheResult, response, debugContext) {
 	}
 }
 //#endregion
-//#region src/providers/cometapi.ts
-/**
-* CometAPI Image Provider - extends OpenAI Image Provider for CometAPI's image generation models
-*/
-var CometApiImageProvider = class extends OpenAiImageProvider {
-	constructor(modelName, options = {}) {
-		super(modelName, {
-			...options,
-			config: {
-				...options.config,
-				apiKeyEnvar: "COMETAPI_KEY",
-				apiBaseUrl: "https://api.cometapi.com/v1"
-			}
-		});
-	}
-	getApiKey() {
-		if (this.config?.apiKey) return this.config.apiKey;
-		return getEnvString("COMETAPI_KEY");
-	}
-	getApiUrlDefault() {
-		return "https://api.cometapi.com/v1";
-	}
-};
-/**
-* Factory for creating CometAPI providers using OpenAI-compatible endpoints.
-*/
-function createCometApiProvider(providerPath, options = {}) {
-	const splits = providerPath.split(":");
-	const type = splits[1];
-	const modelName = splits.slice(2).join(":");
-	const openaiOptions = {
-		...options,
-		config: {
-			...options.config || {},
-			apiBaseUrl: "https://api.cometapi.com/v1",
-			apiKeyEnvar: "COMETAPI_KEY"
-		}
-	};
-	if (type === "chat") return new OpenAiChatCompletionProvider(modelName, openaiOptions);
-	else if (type === "completion") return new OpenAiCompletionProvider(modelName, openaiOptions);
-	else if (type === "embedding" || type === "embeddings") return new OpenAiEmbeddingProvider(modelName, openaiOptions);
-	else if (type === "image") return new CometApiImageProvider(modelName, openaiOptions);
-	return new OpenAiChatCompletionProvider(splits.slice(1).join(":"), openaiOptions);
-}
-//#endregion
-export { createCometApiProvider, initializeAgenticCache as i, generateCacheKey as n, getCachedResponse as r, cacheResponse as t };
+export { initializeAgenticCache as i, generateCacheKey as n, getCachedResponse as r, cacheResponse as t };
 
-//# sourceMappingURL=cometapi-DkXrKi5z.js.map
+//# sourceMappingURL=agentic-utils-D8yXo5Lm.js.map

package/dist/src/{cometapi-vY6aDZgo.cjs → agentic-utils-DAVsChuB.cjs}

@@ -1,17 +1,5 @@
-const require_logger = require("./logger-Cp1GPUjj.cjs");
-require("./esm-CnNt7sI4.cjs");
-require("./pythonUtils-CTU3Y3lw.cjs");
-require("./types-LJ0r3wbR.cjs");
-require("./util-Yz-1aEhW.cjs");
-require("./fetch-BxUk8odA.cjs");
-const require_cache = require("./cache-COish3-W.cjs");
-require("./genaiTracer-DN4dQywX.cjs");
-const require_chat = require("./chat-DaqekjFr.cjs");
-require("./transform-DOcQeLld.cjs");
-require("./openai-Cuif0GEt.cjs");
-require("./util-Db0a0AFH.cjs");
-const require_completion = require("./completion-CDOouNzq.cjs");
-const require_image = require("./image-DTedmQPg.cjs");
+const require_logger = require("./logger-D5iKBpu_.cjs");
+const require_cache = require("./cache-C5yFZ4gC.cjs");
 let fs = require("fs");
 fs = require_logger.__toESM(fs);
 let path = require("path");
@@ -164,55 +152,29 @@ async function cacheResponse(cacheResult, response, debugContext) {
 	}
 }
 //#endregion
-//#region src/providers/cometapi.ts
-/**
-* CometAPI Image Provider - extends OpenAI Image Provider for CometAPI's image generation models
-*/
-var CometApiImageProvider = class extends require_image.OpenAiImageProvider {
-	constructor(modelName, options = {}) {
-		super(modelName, {
-			...options,
-			config: {
-				...options.config,
-				apiKeyEnvar: "COMETAPI_KEY",
-				apiBaseUrl: "https://api.cometapi.com/v1"
-			}
-		});
+Object.defineProperty(exports, "cacheResponse", {
+	enumerable: true,
+	get: function() {
+		return cacheResponse;
 	}
-	getApiKey() {
-		if (this.config?.apiKey) return this.config.apiKey;
-		return require_logger.getEnvString("COMETAPI_KEY");
+});
+Object.defineProperty(exports, "generateCacheKey", {
+	enumerable: true,
+	get: function() {
+		return generateCacheKey;
 	}
-	getApiUrlDefault() {
-		return "https://api.cometapi.com/v1";
+});
+Object.defineProperty(exports, "getCachedResponse", {
+	enumerable: true,
+	get: function() {
+		return getCachedResponse;
 	}
-};
-/**
-* Factory for creating CometAPI providers using OpenAI-compatible endpoints.
-*/
-function createCometApiProvider(providerPath, options = {}) {
-	const splits = providerPath.split(":");
-	const type = splits[1];
-	const modelName = splits.slice(2).join(":");
-	const openaiOptions = {
-		...options,
-		config: {
-			...options.config || {},
-			apiBaseUrl: "https://api.cometapi.com/v1",
-			apiKeyEnvar: "COMETAPI_KEY"
-		}
-	};
-	if (type === "chat") return new require_chat.OpenAiChatCompletionProvider(modelName, openaiOptions);
-	else if (type === "completion") return new require_completion.OpenAiCompletionProvider(modelName, openaiOptions);
-	else if (type === "embedding" || type === "embeddings") return new require_completion.OpenAiEmbeddingProvider(modelName, openaiOptions);
-	else if (type === "image") return new CometApiImageProvider(modelName, openaiOptions);
-	return new require_chat.OpenAiChatCompletionProvider(splits.slice(1).join(":"), openaiOptions);
-}
-//#endregion
-exports.cacheResponse = cacheResponse;
-exports.createCometApiProvider = createCometApiProvider;
-exports.generateCacheKey = generateCacheKey;
-exports.getCachedResponse = getCachedResponse;
-exports.initializeAgenticCache = initializeAgenticCache;
+});
+Object.defineProperty(exports, "initializeAgenticCache", {
+	enumerable: true,
+	get: function() {
+		return initializeAgenticCache;
+	}
+});
 
-//# sourceMappingURL=cometapi-vY6aDZgo.cjs.map
+//# sourceMappingURL=agentic-utils-DAVsChuB.cjs.map

package/dist/src/agentic-utils-DIYAAYE7.js

@@ -0,0 +1,153 @@
+import { a as logger } from "./logger-BnkjG2jt.js";
+import { a as isCacheEnabled, i as getCache } from "./cache-HP0NP4k3.js";
+import fs from "fs";
+import path from "path";
+import dedent from "dedent";
+import crypto from "crypto";
+//#region src/providers/agentic-utils.ts
+/**
+* Shared utilities for agentic providers (Claude Agent SDK, OpenCode SDK, etc.)
+*
+* These utilities handle common functionality needed by coding agent providers:
+* - Working directory fingerprinting for cache key generation
+* - Response caching with fingerprint support
+*/
+/**
+* Timeout for working directory fingerprint generation (ms)
+* Prevents hanging on extremely large directories
+*/
+const FINGERPRINT_TIMEOUT_MS = 2e3;
+/**
+* Get a fingerprint for a working directory to use as a cache key.
+* Checks directory mtime and descendant file mtimes recursively.
+*
+* This allows for caching prompts that use the same working directory
+* when the files haven't changed.
+*
+* @param workingDir - Absolute path to the working directory
+* @returns SHA-256 hash fingerprint of the directory state
+* @throws Error if fingerprinting times out or directory is inaccessible
+*/
+async function getWorkingDirFingerprint(workingDir) {
+	const dirMtime = fs.statSync(workingDir).mtimeMs;
+	const startTime = Date.now();
+	const getAllFiles = (dir, files = []) => {
+		if (Date.now() - startTime > FINGERPRINT_TIMEOUT_MS) throw new Error("Working directory fingerprint timed out");
+		const entries = fs.readdirSync(dir, { withFileTypes: true });
+		for (const entry of entries) {
+			const fullPath = path.join(dir, entry.name);
+			if (entry.isDirectory()) getAllFiles(fullPath, files);
+			else if (entry.isFile()) files.push(fullPath);
+		}
+		return files;
+	};
+	const fingerprintData = `dir:${dirMtime};files:${getAllFiles(workingDir).map((file) => {
+		const stat = fs.statSync(file);
+		return `${path.relative(workingDir, file)}:${stat.mtimeMs}`;
+	}).sort().join(",")}`;
+	return crypto.createHash("sha256").update(fingerprintData).digest("hex");
+}
+/**
+* Generate a cache key from arbitrary data using SHA-256 hash
+*
+* @param prefix - Cache key prefix (provider identifier)
+* @param data - Data to hash for the cache key
+* @returns Prefixed SHA-256 hash cache key
+*/
+function generateCacheKey(prefix, data) {
+	const stringified = JSON.stringify(data);
+	return `${prefix}:${crypto.createHash("sha256").update(stringified).digest("hex")}`;
+}
+/**
+* Initialize cache and check for cached response
+*
+* This handles the common caching pattern used by agentic providers:
+* 1. Check if caching is enabled
+* 2. Generate working directory fingerprint if needed
+* 3. Generate cache key
+* 4. Return cache configuration for use by the provider
+*
+* @param options - Cache options including prefix and working directory
+* @param cacheKeyData - Data to include in the cache key
+* @returns Cache configuration and optional cached response
+*/
+async function initializeAgenticCache(options, cacheKeyData) {
+	if (!isCacheEnabled()) return {
+		shouldCache: false,
+		shouldReadCache: false,
+		shouldWriteCache: false
+	};
+	if (options.mcp && !options.cacheMcp) return {
+		shouldCache: false,
+		shouldReadCache: false,
+		shouldWriteCache: false
+	};
+	let workingDirFingerprint = null;
+	if (options.workingDir) try {
+		workingDirFingerprint = await getWorkingDirFingerprint(options.workingDir);
+	} catch (error) {
+		logger.error(dedent`Error getting working directory fingerprint for cache key - ${options.workingDir}: ${String(error)}
+
+        Caching is disabled.`);
+		return {
+			shouldCache: false,
+			shouldReadCache: false,
+			shouldWriteCache: false
+		};
+	}
+	const cache = await getCache();
+	const cacheKey = generateCacheKey(options.cacheKeyPrefix, {
+		...cacheKeyData,
+		workingDirFingerprint,
+		...options.mcp ? { mcp: options.mcp } : {}
+	});
+	return {
+		shouldCache: true,
+		shouldReadCache: !options.bustCache,
+		shouldWriteCache: true,
+		cache,
+		cacheKey,
+		workingDirFingerprint
+	};
+}
+/**
+* Try to get a cached response
+*
+* @param cacheResult - Result from initializeAgenticCache
+* @param debugContext - Context for debug logging (e.g., prompt preview)
+* @returns Cached ProviderResponse if found, undefined otherwise
+*/
+async function getCachedResponse(cacheResult, debugContext) {
+	if (!cacheResult.shouldReadCache || !cacheResult.cache || !cacheResult.cacheKey) return;
+	try {
+		const cachedResponse = await cacheResult.cache.get(cacheResult.cacheKey);
+		if (cachedResponse) {
+			logger.debug(`Returning cached response${debugContext ? ` for ${debugContext}` : ""} (cache key: ${cacheResult.cacheKey})`);
+			return {
+				...JSON.parse(cachedResponse),
+				cached: true
+			};
+		}
+	} catch (error) {
+		logger.error(`Error getting cached response: ${String(error)}`);
+	}
+}
+/**
+* Cache a provider response
+*
+* @param cacheResult - Result from initializeAgenticCache
+* @param response - The ProviderResponse to cache
+* @param debugContext - Context for debug logging
+*/
+async function cacheResponse(cacheResult, response, debugContext) {
+	if (!cacheResult.shouldWriteCache || !cacheResult.cache || !cacheResult.cacheKey) return;
+	try {
+		await cacheResult.cache.set(cacheResult.cacheKey, JSON.stringify(response));
+	} catch (error) {
+		logger.error(`Error caching response${debugContext ? ` for ${debugContext}` : ""}: ${String(error)}`);
+	}
+}
+//#endregion
+export { initializeAgenticCache as i, generateCacheKey as n, getCachedResponse as r, cacheResponse as t };
+
+//# sourceMappingURL=agentic-utils-DIYAAYE7.js.map