promptfoo 0.121.4 → 0.121.5

package/dist/src/{messages-zWbkLLHz.js → messages-B9dSjrNf.js}

@@ -1,24 +1,231 @@
 #!/usr/bin/env node
-import { D as getEnvInt, E as getEnvFloat, O as getEnvString, s as logger } from "./logger-DksKw1Qc.js";
-import { C as transformToolChoice, w as transformTools } from "./fetch-BEWnXrrG.js";
-import { n as withGenAISpan } from "./genaiTracer-DWdZ28hY.js";
-import { o as getCache, s as isCacheEnabled } from "./cache-mb7c8hbp.js";
-import { g as maybeLoadToolsFromExternalFile, h as maybeLoadResponseFormatFromExternalFile } from "./util-DxWpWjhc.js";
-import { i as normalizeFinishReason, n as MCPClient } from "./chat-Dr3DUQ0D.js";
-import { a as createEmptyTokenUsage } from "./tokenUsageUtils-CXrvO-wA.js";
-import { n as transformMCPToolsToAnthropic } from "./transform-zDhMmzwX.js";
-import { a as outputFromMessage, i as getTokenUsage, n as calculateAnthropicCost, o as parseMessages, r as getRefusalDetails, s as processAnthropicTools, t as ANTHROPIC_MODELS } from "./util-DDs-7g6-.js";
+import { D as getEnvInt, E as getEnvFloat, O as getEnvString, s as logger } from "./logger-BbY6ypFL.js";
+import { C as transformToolChoice, w as transformTools } from "./fetch-B6ch2nU2.js";
+import { n as withGenAISpan } from "./genaiTracer-C3ZPQU60.js";
+import { o as getCache, s as isCacheEnabled } from "./cache-BI5BY7ey.js";
+import { M as maybeLoadResponseFormatFromExternalFile, N as maybeLoadToolsFromExternalFile } from "./util-BQOCAHQC.js";
+import { a as createEmptyTokenUsage } from "./tokenUsageUtils-2wIvAhB3.js";
+import { i as normalizeFinishReason, n as MCPClient } from "./chat-BLOdH60v.js";
+import { n as transformMCPToolsToAnthropic } from "./transform-B-b6Cq-q.js";
+import { a as outputFromMessage, i as getTokenUsage, n as calculateAnthropicCost, o as parseMessages, r as getRefusalDetails, s as processAnthropicTools, t as ANTHROPIC_MODELS } from "./util-DN0-b81k.js";
+import path from "node:path";
+import fs from "node:fs";
 import Anthropic, { APIError } from "@anthropic-ai/sdk";
+import { execFileSync } from "node:child_process";
+import os from "node:os";
+//#region src/providers/anthropic/claudeCodeAuth.ts
+/**
+* Claude Code prepends this identity block as the first `system` message when
+* calling the Anthropic Messages API with an OAuth token. As of 2025-Q4 the
+* API returns HTTP 400 `invalid_request_error` when an OAuth-authenticated
+* request's first system block is not this exact string, so any caller
+* reusing a Claude Code OAuth token must inject this block before the
+* user-provided system prompt. Re-verify if Anthropic ships an OAuth-native
+* `/v1/messages` path that drops the requirement.
+*/
+const CLAUDE_CODE_IDENTITY_PROMPT = "You are Claude Code, Anthropic's official CLI for Claude.";
+/**
+* Anthropic beta headers required when authenticating the Messages API with a
+* Claude Code OAuth token (e.g. a Claude Max subscription access token, or a
+* long-lived token minted by `claude setup-token`).
+*
+* - `oauth-2025-04-20` enables OAuth bearer tokens on `/v1/messages`.
+* - `claude-code-20250219` enables the Claude Code tool-use surface area the
+*   OAuth token is scoped to.
+*
+* Revisit if Anthropic ships an OAuth-native Messages path that no longer
+* requires these betas.
+*/
+const CLAUDE_CODE_OAUTH_BETA_FEATURES = Object.freeze(["claude-code-20250219", "oauth-2025-04-20"]);
+/**
+* The `user-agent` value Claude Code CLI sends on OAuth-authenticated
+* Messages requests. Anthropic's API gates OAuth tokens to the Claude Code
+* app identity, so requests that use a different user-agent fail with 401.
+*
+* The `1.0.0` suffix is cosmetic — Anthropic gates on the `claude-cli`
+* product identifier, not the semver. Revisit if OAuth requests start
+* returning 401 after SDK or CLI updates.
+*/
+const CLAUDE_CODE_USER_AGENT = "claude-cli/1.0.0 (external, promptfoo)";
+const CLAUDE_CODE_KEYCHAIN_SERVICE = "Claude Code-credentials";
+const CLAUDE_CODE_CREDENTIALS_FILE = path.join(".claude", ".credentials.json");
+function resolveCredentialsPath() {
+	return path.join(os.homedir(), CLAUDE_CODE_CREDENTIALS_FILE);
+}
+/**
+* `security` exits with status 44 when the requested generic-password entry
+* does not exist. Any other non-zero exit (or throw) indicates a real failure
+* — ACL denial, corrupted entry, missing binary, timeout — and is worth a
+* `warn` log rather than silently falling through.
+*/
+const SECURITY_ENTRY_NOT_FOUND = 44;
+function parseCredential(raw) {
+	if (!raw || typeof raw !== "object") return {
+		ok: false,
+		reason: "credential JSON is not an object"
+	};
+	const claudeAiOauth = raw.claudeAiOauth;
+	if (!claudeAiOauth || typeof claudeAiOauth !== "object") return {
+		ok: false,
+		reason: "missing `claudeAiOauth` object"
+	};
+	const inner = claudeAiOauth;
+	const accessToken = inner.accessToken;
+	if (typeof accessToken !== "string" || !accessToken) return {
+		ok: false,
+		reason: "missing `claudeAiOauth.accessToken` string"
+	};
+	const credential = { accessToken };
+	if (typeof inner.refreshToken === "string" && inner.refreshToken) credential.refreshToken = inner.refreshToken;
+	if (typeof inner.expiresAt === "number" && Number.isFinite(inner.expiresAt)) credential.expiresAt = inner.expiresAt;
+	if (typeof inner.subscriptionType === "string") credential.subscriptionType = inner.subscriptionType;
+	return {
+		ok: true,
+		credential
+	};
+}
+function parseJsonBlob(blob) {
+	let parsed;
+	try {
+		parsed = JSON.parse(blob);
+	} catch (err) {
+		return {
+			ok: false,
+			reason: `invalid JSON: ${err instanceof Error ? err.message : String(err)}`
+		};
+	}
+	return parseCredential(parsed);
+}
+function readFromMacosKeychain() {
+	if (process.platform !== "darwin") return null;
+	let out;
+	try {
+		out = execFileSync("security", [
+			"find-generic-password",
+			"-s",
+			CLAUDE_CODE_KEYCHAIN_SERVICE,
+			"-w"
+		], {
+			encoding: "utf-8",
+			stdio: [
+				"ignore",
+				"pipe",
+				"ignore"
+			],
+			timeout: 5e3
+		});
+	} catch (err) {
+		if (err?.status === SECURITY_ENTRY_NOT_FOUND) {
+			logger.debug("[anthropic] No Claude Code credential entry in the macOS keychain; trying file fallback.");
+			return null;
+		}
+		logger.warn("[anthropic] Failed to read Claude Code credential from macOS keychain; falling back to file lookup.", { error: err instanceof Error ? err.message : String(err) });
+		return null;
+	}
+	const trimmed = out.trim();
+	if (!trimmed) {
+		logger.warn("[anthropic] Claude Code macOS keychain entry is empty. Run `claude /login` to refresh it.");
+		return null;
+	}
+	const result = parseJsonBlob(trimmed);
+	if (!result.ok) {
+		logger.warn(`[anthropic] Claude Code macOS keychain entry is malformed (${result.reason}). Run \`claude /login\` to refresh it.`);
+		return null;
+	}
+	return result.credential;
+}
+function readFromFile() {
+	const filePath = resolveCredentialsPath();
+	if (!fs.existsSync(filePath)) return null;
+	let raw;
+	try {
+		raw = fs.readFileSync(filePath, "utf-8");
+	} catch (err) {
+		logger.warn(`[anthropic] Claude Code credentials file at ${filePath} exists but could not be read. Run \`claude /login\` to refresh it.`, { error: err instanceof Error ? err.message : String(err) });
+		return null;
+	}
+	const result = parseJsonBlob(raw);
+	if (!result.ok) {
+		logger.warn(`[anthropic] Claude Code credentials file at ${filePath} is malformed (${result.reason}). Run \`claude /login\` to refresh it.`);
+		return null;
+	}
+	return result.credential;
+}
+/**
+* Loads a Claude Code OAuth credential from the local environment.
+*
+* Resolution order:
+* 1. macOS keychain (`security find-generic-password -s "Claude Code-credentials"`)
+*    — only attempted on darwin.
+* 2. `$HOME/.claude/.credentials.json` — the Linux/Windows default used by
+*    Claude Code, and a fallback on macOS when the keychain entry is missing.
+*    On Windows this resolves to `%USERPROFILE%\.claude\.credentials.json`.
+*
+* Returns `null` when no credential is available. Callers should check
+* {@link isCredentialExpired} on a non-null return before using it — this
+* function does not filter out expired credentials so callers can decide
+* whether to warn, refuse, or surface a 401 to the user.
+*
+* Never throws. Missing credentials fall through silently; corrupted or
+* unreadable credentials are logged at `warn` level with a reason so broken
+* setups are diagnosable without enabling debug logging.
+*/
+function loadClaudeCodeCredential() {
+	return readFromMacosKeychain() ?? readFromFile();
+}
+/**
+* Returns `true` when the credential's `expiresAt` (ms since epoch) is in the
+* past. Credentials without an `expiresAt` are treated as non-expired — the
+* Anthropic API will reject them with a 401 if the token is actually invalid.
+*/
+function isCredentialExpired(credential) {
+	if (credential.expiresAt == null) return false;
+	return Date.now() >= credential.expiresAt;
+}
+//#endregion
 //#region src/providers/anthropic/generic.ts
 /**
-* Generic provider class for Anthropic APIs
-* Serves as a base class with shared functionality for all Anthropic providers
+* Generic provider class for Anthropic APIs.
+*
+* Serves as a base class with shared functionality for all Anthropic
+* subclass providers. Handles API key resolution and, for subclasses that
+* opt in via {@link AnthropicGenericProvider.SUPPORTS_CLAUDE_CODE_OAUTH},
+* falls back to an OAuth token loaded from a local Claude Code session so
+* Claude.ai subscribers can run evals without a separate console API key.
+*
+* The OAuth fallback is opt-in so it only affects
+* `AnthropicMessagesProvider`. Claude Code OAuth tokens are gated to the
+* Messages API (`/v1/messages`) — forwarding them to the legacy
+* text-completion endpoint would fail at request time, which would cause
+* `anthropic:completion:*` configs to bypass promptfoo's upfront preflight
+* check and then produce a less useful error for every test case.
 */
 var AnthropicGenericProvider = class {
+	/**
+	* Subclasses that can authenticate via a Claude Code OAuth session
+	* should override this to `true`. The base class's constructor reads the
+	* flag from `this.constructor` so only OAuth-capable subclasses attempt
+	* the credential lookup (and only they honor `apiKeyRequired: false`).
+	*/
+	static SUPPORTS_CLAUDE_CODE_OAUTH = false;
 	modelName;
 	config;
 	env;
 	apiKey;
+	/**
+	* `true` when this provider is authenticating via a Claude Code OAuth token
+	* rather than a standard `ANTHROPIC_API_KEY`. Subclasses use this to inject
+	* the Claude Code identity system block that Anthropic's API requires for
+	* OAuth-authenticated Messages requests.
+	*/
+	usingClaudeCodeOAuth;
+	/**
+	* The Claude Code credential used to authenticate, when
+	* `usingClaudeCodeOAuth` is `true`. Kept so subclasses can re-check expiry
+	* at request time and surface a "run `claude /login`" error instead of a
+	* raw 401 from the SDK.
+	*/
+	claudeCodeCredential;
 	anthropic;
 	constructor(modelName, options = {}) {
 		const { config, id, env } = options;
@@ -26,9 +233,28 @@ var AnthropicGenericProvider = class {
 		this.modelName = modelName;
 		this.config = config || {};
 		this.apiKey = this.getApiKey();
+		this.usingClaudeCodeOAuth = false;
+		let authToken;
+		const defaultHeaders = {};
+		const subclass = this.constructor;
+		if (!this.apiKey && this.config.apiKeyRequired === false && subclass.SUPPORTS_CLAUDE_CODE_OAUTH) {
+			const credential = loadClaudeCodeCredential();
+			if (credential) {
+				if (isCredentialExpired(credential)) logger.warn("[anthropic] Claude Code OAuth credential is expired. Run `claude /login` to refresh it.");
+				else logger.debug("[anthropic] Authenticating with Claude Code OAuth credential from local session.");
+				authToken = credential.accessToken;
+				this.usingClaudeCodeOAuth = true;
+				this.claudeCodeCredential = credential;
+				defaultHeaders["anthropic-beta"] = CLAUDE_CODE_OAUTH_BETA_FEATURES.join(",");
+				defaultHeaders["user-agent"] = CLAUDE_CODE_USER_AGENT;
+				defaultHeaders["x-app"] = "cli";
+			} else logger.warn("[anthropic] apiKeyRequired is false but no Claude Code credential was found. Run `claude /login` to create one, or set ANTHROPIC_API_KEY.");
+		}
 		this.anthropic = new Anthropic({
-			apiKey: this.apiKey,
-			baseURL: this.getApiBaseUrl()
+			apiKey: this.apiKey ?? null,
+			authToken: authToken ?? null,
+			baseURL: this.getApiBaseUrl(),
+			...Object.keys(defaultHeaders).length > 0 ? { defaultHeaders } : {}
 		});
 		this.id = id ? () => id : this.id;
 	}
@@ -38,7 +264,16 @@ var AnthropicGenericProvider = class {
 	toString() {
 		return `[Anthropic Provider ${this.modelName}]`;
 	}
+	/**
+	* Whether promptfoo's preflight check should require an API key before
+	* dispatching requests to this provider. Returns `false` only for
+	* OAuth-capable subclasses (see {@link SUPPORTS_CLAUDE_CODE_OAUTH}) when
+	* the user has opted in via `apiKeyRequired: false`. All other subclasses
+	* keep the preflight check so missing-credential errors surface upfront
+	* rather than at per-request call time.
+	*/
 	requiresApiKey() {
+		if (this.constructor.SUPPORTS_CLAUDE_CODE_OAUTH && this.config.apiKeyRequired === false) return false;
 		return true;
 	}
 	getApiKey() {
@@ -70,6 +305,7 @@ function isThinkingEnabled(thinking) {
 var AnthropicMessagesProvider = class AnthropicMessagesProvider extends AnthropicGenericProvider {
 	mcpClient = null;
 	initializationPromise = null;
+	static SUPPORTS_CLAUDE_CODE_OAUTH = true;
 	static ANTHROPIC_MODELS = ANTHROPIC_MODELS;
 	static ANTHROPIC_MODELS_NAMES = ANTHROPIC_MODELS.map((model) => model.id);
 	constructor(modelName, options = {}) {
@@ -96,7 +332,8 @@ var AnthropicMessagesProvider = class AnthropicMessagesProvider extends Anthropi
 	}
 	async callApi(prompt, context) {
 		if (this.initializationPromise != null) await this.initializationPromise;
-		if (!this.apiKey) throw new Error("Anthropic API key is not set. Set the ANTHROPIC_API_KEY environment variable or add `apiKey` to the provider config.");
+		if (!this.apiKey && !this.usingClaudeCodeOAuth) throw new Error("Anthropic API key is not set. Set the ANTHROPIC_API_KEY environment variable or add `apiKey` to the provider config. Alternatively, if you have an active Claude Code session, set `apiKeyRequired: false` in the provider config to authenticate via Claude Code.");
+		if (this.usingClaudeCodeOAuth && this.claudeCodeCredential && isCredentialExpired(this.claudeCodeCredential)) throw new Error("Claude Code OAuth credential is expired. Run `claude /login` to refresh it, then re-run the eval.");
 		if (!this.modelName) throw new Error("Anthropic model name is not set. Please provide a valid model name.");
 		const spanContext = {
 			system: "anthropic",
@@ -159,10 +396,14 @@ var AnthropicMessagesProvider = class AnthropicMessagesProvider extends Anthropi
 		if (this.modelName.startsWith("claude-opus-4-6") && extractedMessages.length > 0) {
 			if (extractedMessages[extractedMessages.length - 1].role === "assistant") logger.warn("Assistant message prefilling is not supported on Claude Opus 4.6 and will cause a 400 error. Remove the trailing assistant message from your prompt.");
 		}
+		const resolvedSystem = this.usingClaudeCodeOAuth ? [{
+			type: "text",
+			text: CLAUDE_CODE_IDENTITY_PROMPT
+		}, ...system ?? []] : system;
 		const shouldStream = config.stream ?? false;
 		const params = {
 			model: this.modelName,
-			...system ? { system } : {},
+			...resolvedSystem && resolvedSystem.length > 0 ? { system: resolvedSystem } : {},
 			max_tokens: config.max_tokens ?? getEnvInt("ANTHROPIC_MAX_TOKENS", thinkingEnabled ? 2048 : 1024),
 			messages: extractedMessages,
 			stream: shouldStream,
@@ -185,9 +426,16 @@ var AnthropicMessagesProvider = class AnthropicMessagesProvider extends Anthropi
 		logger.debug("Calling Anthropic Messages API", { params });
 		const headers = { ...config.headers || {} };
 		let allBetaFeatures = [...config.beta || [], ...requiredBetaFeatures];
+		const userBetaHeader = config.headers?.["anthropic-beta"];
+		if (typeof userBetaHeader === "string" && userBetaHeader.length > 0) allBetaFeatures.push(...userBetaHeader.split(",").map((entry) => entry.trim()).filter((entry) => entry.length > 0));
 		if (processedOutputFormat && !allBetaFeatures.includes("structured-outputs-2025-11-13")) allBetaFeatures.push("structured-outputs-2025-11-13");
+		if (this.usingClaudeCodeOAuth) allBetaFeatures.push(...CLAUDE_CODE_OAUTH_BETA_FEATURES);
 		allBetaFeatures = [...new Set(allBetaFeatures)];
 		if (allBetaFeatures.length > 0) headers["anthropic-beta"] = allBetaFeatures.join(",");
+		if (this.usingClaudeCodeOAuth) {
+			headers["user-agent"] = CLAUDE_CODE_USER_AGENT;
+			headers["x-app"] = "cli";
+		}
 		const cache = await getCache();
 		const { metadata: _metadata, ...cacheKeyParams } = params;
 		const cacheKey = `anthropic:${JSON.stringify(cacheKeyParams)}`;
@@ -293,4 +541,4 @@ var AnthropicMessagesProvider = class AnthropicMessagesProvider extends Anthropi
 //#endregion
 export { AnthropicGenericProvider as n, AnthropicMessagesProvider as t };
 
-//# sourceMappingURL=messages-zWbkLLHz.js.map
+//# sourceMappingURL=messages-B9dSjrNf.js.map