promptfoo 0.121.5 → 0.121.8

package/dist/src/{claude-agent-sdk-DYv_AJ8u.js → claude-agent-sdk-BGUac_kS.js}

@@ -1,14 +1,77 @@
 #!/usr/bin/env node
 import { N as state, O as getEnvString, s as logger } from "./logger-BbY6ypFL.js";
-import { i as resolvePackageEntryPoint, o as safeResolve, r as importModule } from "./esm-BX8fwlAO.js";
-import { t as transformMCPConfigToClaudeCode } from "./transform-B-b6Cq-q.js";
-import { t as ANTHROPIC_MODELS } from "./util-DN0-b81k.js";
-import { i as initializeAgenticCache, r as getCachedResponse, t as cacheResponse } from "./agentic-utils-B5krlibj.js";
+import { i as resolvePackageEntryPoint, o as safeResolve, r as importModule } from "./esm-Bexx2PFc.js";
+import { i as withGenAISpan, n as getTraceparent, r as sanitizeBody, t as getGenAITracer } from "./genaiTracer-BPVvltoW.js";
+import { t as transformMCPConfigToClaudeCode } from "./transform-DtTfiGoh.js";
+import { t as ANTHROPIC_MODELS } from "./util-X4KQgyVD.js";
+import { n as PROMPTFOO_RESOURCE_ATTR_TRACE_ID, t as PROMPTFOO_RESOURCE_ATTR_PARENT_SPAN_ID } from "./resourceAttributes-D1jP3kL5.js";
+import { i as initializeAgenticCache, r as getCachedResponse, t as cacheResponse } from "./agentic-utils-BJKAkz2e.js";
 import dedent from "dedent";
 import path from "node:path";
 import fs from "node:fs";
+import { SpanStatusCode, trace } from "@opentelemetry/api";
 import os from "node:os";
 //#region src/providers/claude-agent-sdk.ts
+/** Hard cap for attribute body length on synthesized tool spans. */
+const TOOL_SPAN_BODY_LIMIT = 4096;
+/**
+* Append promptfoo-specific resource-attribute kvs to a W3C-style
+* `OTEL_RESOURCE_ATTRIBUTES` string, removing trailing whitespace/commas from
+* the existing value and stripping any previous occurrence of our keys so the
+* producer can't double-up. Returns the new string. Exported for tests.
+*/
+function appendPromptfooResourceAttrs(existing, traceId, parentSpanId) {
+	const incoming = `${PROMPTFOO_RESOURCE_ATTR_TRACE_ID}=${traceId},${PROMPTFOO_RESOURCE_ATTR_PARENT_SPAN_ID}=${parentSpanId}`;
+	if (!existing) return incoming;
+	const cleaned = existing.split(",").map((pair) => pair.trim()).filter((pair) => pair.length > 0 && !pair.startsWith(`promptfoo.trace_id=`) && !pair.startsWith(`promptfoo.parent_span_id=`)).join(",");
+	return cleaned.length > 0 ? `${cleaned},${incoming}` : incoming;
+}
+function stringifyForSpan(value) {
+	if (value === void 0 || value === null) return;
+	let raw;
+	if (typeof value === "string") raw = value;
+	else try {
+		raw = JSON.stringify(value);
+	} catch {
+		return "<unserializable>";
+	}
+	if (raw === void 0) return;
+	const sanitized = sanitizeBody(raw);
+	return sanitized.length > TOOL_SPAN_BODY_LIMIT ? `${sanitized.slice(0, TOOL_SPAN_BODY_LIMIT - 15)}... [truncated]` : sanitized;
+}
+/**
+* Emit a child span for a single tool call. Parents to the currently active
+* span (the provider's GenAI wrapper) so the UI shows an agent → tool hierarchy
+* similar to the OpenAI Agents SDK.
+*
+* When `incomplete` is true the call never produced a matching `tool_result`
+* (aborted run, stop hook, or the stream ended mid-tool). Such spans are
+* flagged via `tool.incomplete` and marked ERROR so traces surface the gap
+* instead of silently dropping the tool.
+*/
+function emitToolSpan(entry, startTimeMs, endTimeMs, isError, incomplete = false) {
+	try {
+		const tracer = getGenAITracer();
+		const attributes = {
+			"tool.name": entry.name,
+			"tool.is_error": isError
+		};
+		if (incomplete) attributes["tool.incomplete"] = true;
+		const input = stringifyForSpan(entry.input);
+		if (input !== void 0) attributes["tool.input"] = input;
+		const output = stringifyForSpan(entry.output);
+		if (output !== void 0) attributes["tool.output"] = output;
+		if (entry.parentToolUseId) attributes["tool.parent_id"] = entry.parentToolUseId;
+		const span = tracer.startSpan(`tool ${entry.name}`, {
+			startTime: startTimeMs,
+			attributes
+		});
+		span.setStatus({ code: isError || incomplete ? SpanStatusCode.ERROR : SpanStatusCode.OK });
+		span.end(endTimeMs);
+	} catch (err) {
+		logger.warn(`[ClaudeAgentSDK] Failed to emit tool span for ${entry.name}: ${err}`);
+	}
+}
 function deriveSkillCalls(toolCalls) {
 	return toolCalls.filter((toolCall) => toolCall.name === "Skill").flatMap((toolCall) => {
 		const skillName = toolCall.input && typeof toolCall.input === "object" && typeof toolCall.input.skill === "string" ? toolCall.input.skill.trim() : "";
@@ -56,12 +119,17 @@ const CLAUDE_CODE_MODEL_ALIASES = [
 * Uses resolvePackageEntryPoint to handle ESM-only packages with restrictive exports
 */
 async function loadClaudeCodeSDK() {
-	const claudeCodePath = resolvePackageEntryPoint("@anthropic-ai/claude-agent-sdk", state.basePath && path.isAbsolute(state.basePath) ? state.basePath : process.cwd());
-	if (!claudeCodePath) throw new Error(dedent`The @anthropic-ai/claude-agent-sdk package is required but not installed.
+	const basePath = state.basePath && path.isAbsolute(state.basePath) ? state.basePath : process.cwd();
+	const claudeCodePath = resolvePackageEntryPoint("@anthropic-ai/claude-agent-sdk", basePath);
+	if (!claudeCodePath) throw new Error(dedent`The @anthropic-ai/claude-agent-sdk package could not be resolved from ${basePath}.
 
       To use the Claude Agent SDK provider, install it with:
         npm install @anthropic-ai/claude-agent-sdk
 
+      If the package is already installed elsewhere, run promptfoo from the
+      project root (or point the config at that root) so node_modules is on
+      the resolution path.
+
       For more information, see: https://www.promptfoo.dev/docs/providers/claude-agent-sdk/`);
 	try {
 		return importModule(claudeCodePath);
@@ -147,6 +215,10 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 		};
 		const env = {};
 		for (const key of Object.keys(process.env).sort()) if (process.env[key] !== void 0) env[key] = process.env[key];
+		if (config.env) for (const key of Object.keys(config.env).sort()) {
+			const value = config.env[key];
+			if (value !== void 0) env[key] = value;
+		}
 		if (this.env) for (const key of Object.keys(this.env).sort()) {
 			const value = this.env[key];
 			if (value !== void 0) env[key] = value;
@@ -158,6 +230,7 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 		if (config.allow_all_tools && ("custom_allowed_tools" in config || "append_allowed_tools" in config)) throw new Error("Cannot specify both allow_all_tools and custom_allowed_tools or append_allowed_tools");
 		if ("custom_allowed_tools" in config && "append_allowed_tools" in config) throw new Error("Cannot specify both custom_allowed_tools and append_allowed_tools");
 		if (config.permission_mode === "bypassPermissions" && !config.allow_dangerously_skip_permissions) throw new Error("permission_mode 'bypassPermissions' requires allow_dangerously_skip_permissions: true as a safety measure");
+		if (config.title && (config.resume || config.continue)) logger.warn("[ClaudeAgentSDK] `title` is ignored when `resume` or `continue` is set; the persisted session title is used instead.");
 		const defaultAllowedTools = config.working_dir ? FS_READONLY_ALLOWED_TOOLS : [];
 		let allowedTools = config.allow_all_tools ? void 0 : defaultAllowedTools;
 		if ("custom_allowed_tools" in config) allowedTools = Array.from(new Set(config.custom_allowed_tools ?? [])).sort();
@@ -179,7 +252,8 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 			systemPrompt: config.custom_system_prompt ? config.custom_system_prompt : {
 				type: "preset",
 				preset: "claude_code",
-				append: config.append_system_prompt
+				append: config.append_system_prompt,
+				...config.exclude_dynamic_sections ? { excludeDynamicSections: true } : {}
 			},
 			maxThinkingTokens: config.max_thinking_tokens,
 			allowedTools,
@@ -263,7 +337,8 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 			stderr: config.stderr,
 			spawnClaudeCodeProcess: config.spawn_claude_code_process,
 			canUseTool,
-			onElicitation: config.on_elicitation
+			onElicitation: config.on_elicitation,
+			title: config.title
 		};
 		const queryParams = {
 			prompt,
@@ -278,44 +353,103 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 			}
 		})}`);
 		try {
-			if (!this.claudeCodeModule) this.claudeCodeModule = await loadClaudeCodeSDK();
-			const res = await this.claudeCodeModule.query(queryParams);
-			const toolCallsMap = /* @__PURE__ */ new Map();
-			for await (const msg of res) if (msg.type === "assistant") {
-				for (const block of msg.message.content) if (block.type === "tool_use") toolCallsMap.set(block.id, {
-					id: block.id,
-					name: block.name,
-					input: block.input,
-					output: void 0,
-					is_error: false,
-					parentToolUseId: msg.parent_tool_use_id
-				});
-			} else if (msg.type === "user") {
-				const content = msg.message?.content;
-				if (Array.isArray(content)) {
-					for (const block of content) if (block.type === "tool_result") {
-						const entry = toolCallsMap.get(block.tool_use_id);
-						if (entry) {
-							entry.output = block.content;
-							entry.is_error = block.is_error ?? false;
-						}
+			return await withGenAISpan({
+				system: "anthropic",
+				operationName: "chat",
+				model: config.model || "default",
+				providerId: this.providerId,
+				traceparent: context?.traceparent,
+				maxTokens: config.max_thinking_tokens,
+				requestBody: prompt
+			}, async () => {
+				const ZERO_TRACE_ID = "00000000000000000000000000000000";
+				const active = getTraceparent();
+				const traceparent = (active && !active.includes(ZERO_TRACE_ID) ? active : void 0) ?? context?.traceparent;
+				if (traceparent && !env.TRACEPARENT) env.TRACEPARENT = traceparent;
+				const [, tpTraceId, tpSpanId] = traceparent ? traceparent.split("-") : [];
+				if (tpTraceId && tpSpanId) env.OTEL_RESOURCE_ATTRIBUTES = appendPromptfooResourceAttrs(env.OTEL_RESOURCE_ATTRIBUTES, tpTraceId, tpSpanId);
+				if (!this.claudeCodeModule) this.claudeCodeModule = await loadClaudeCodeSDK();
+				const res = await this.claudeCodeModule.query(queryParams);
+				const toolCallsMap = /* @__PURE__ */ new Map();
+				const toolStartTimes = /* @__PURE__ */ new Map();
+				const drainOrphans = () => {
+					if (toolStartTimes.size === 0) return;
+					const endedAt = Date.now();
+					for (const [toolUseId, startMs] of toolStartTimes) {
+						const entry = toolCallsMap.get(toolUseId);
+						if (entry) emitToolSpan(entry, startMs, endedAt, false, true);
 					}
-				}
-			} else if (msg.type === "result") {
-				const raw = JSON.stringify(msg);
-				const tokenUsage = {
-					prompt: msg.usage?.input_tokens,
-					completion: msg.usage?.output_tokens,
-					total: msg.usage?.input_tokens && msg.usage?.output_tokens ? msg.usage?.input_tokens + msg.usage?.output_tokens : void 0
+					toolStartTimes.clear();
 				};
-				const cost = msg.total_cost_usd ?? 0;
-				const sessionId = msg.session_id;
-				const toolCallsArray = Array.from(toolCallsMap.values());
-				const skillCalls = deriveSkillCalls(toolCallsArray);
-				if (msg.subtype === "success") {
-					logger.debug(`Claude Agent SDK response: ${raw}`);
-					const response = {
-						output: msg.structured_output === void 0 ? msg.result : msg.structured_output,
+				for await (const msg of res) if (msg.type === "assistant") {
+					for (const block of msg.message.content) if (block.type === "tool_use") {
+						toolCallsMap.set(block.id, {
+							id: block.id,
+							name: block.name,
+							input: block.input,
+							output: void 0,
+							is_error: false,
+							parentToolUseId: msg.parent_tool_use_id
+						});
+						toolStartTimes.set(block.id, Date.now());
+					}
+				} else if (msg.type === "user") {
+					const content = msg.message?.content;
+					if (Array.isArray(content)) {
+						for (const block of content) if (block.type === "tool_result") {
+							const entry = toolCallsMap.get(block.tool_use_id);
+							if (entry) {
+								entry.output = block.content;
+								entry.is_error = block.is_error ?? false;
+								const startMs = toolStartTimes.get(block.tool_use_id);
+								if (startMs !== void 0) {
+									emitToolSpan(entry, startMs, Date.now(), entry.is_error);
+									toolStartTimes.delete(block.tool_use_id);
+								}
+							}
+						}
+					}
+				} else if (msg.type === "result") {
+					const raw = JSON.stringify(msg);
+					const tokenUsage = {
+						prompt: msg.usage?.input_tokens,
+						completion: msg.usage?.output_tokens,
+						total: msg.usage?.input_tokens && msg.usage?.output_tokens ? msg.usage?.input_tokens + msg.usage?.output_tokens : void 0
+					};
+					const cost = msg.total_cost_usd ?? 0;
+					const sessionId = msg.session_id;
+					const toolCallsArray = Array.from(toolCallsMap.values());
+					const skillCalls = deriveSkillCalls(toolCallsArray);
+					drainOrphans();
+					const abortedTerminalReason = typeof msg.terminal_reason === "string" && (msg.terminal_reason.startsWith("aborted_") || msg.terminal_reason === "hook_stopped") ? msg.terminal_reason : void 0;
+					if (abortedTerminalReason) trace.getActiveSpan()?.setStatus({
+						code: SpanStatusCode.ERROR,
+						message: `aborted: ${abortedTerminalReason}`
+					});
+					if (msg.subtype === "success") {
+						logger.debug(`Claude Agent SDK response: ${raw}`);
+						const response = {
+							output: msg.structured_output === void 0 ? msg.result : msg.structured_output,
+							tokenUsage,
+							cost,
+							raw,
+							sessionId,
+							metadata: {
+								skillCalls,
+								toolCalls: toolCallsArray,
+								numTurns: msg.num_turns,
+								durationMs: msg.duration_ms,
+								durationApiMs: msg.duration_api_ms,
+								modelUsage: msg.modelUsage,
+								permissionDenials: msg.permission_denials,
+								...msg.terminal_reason === void 0 ? {} : { terminalReason: msg.terminal_reason },
+								...msg.structured_output === void 0 ? {} : { structuredOutput: msg.structured_output }
+							}
+						};
+						await cacheResponse(cacheResult, response, "Claude Agent SDK");
+						return response;
+					} else return {
+						error: `Claude Agent SDK call failed: ${msg.subtype}`,
 						tokenUsage,
 						cost,
 						raw,
@@ -328,31 +462,44 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 							durationApiMs: msg.duration_api_ms,
 							modelUsage: msg.modelUsage,
 							permissionDenials: msg.permission_denials,
-							...msg.terminal_reason === void 0 ? {} : { terminalReason: msg.terminal_reason },
-							...msg.structured_output === void 0 ? {} : { structuredOutput: msg.structured_output }
+							...msg.terminal_reason === void 0 ? {} : { terminalReason: msg.terminal_reason }
 						}
 					};
-					await cacheResponse(cacheResult, response, "Claude Agent SDK");
-					return response;
-				} else return {
-					error: `Claude Agent SDK call failed: ${msg.subtype}`,
-					tokenUsage,
-					cost,
-					raw,
-					sessionId,
-					metadata: {
-						skillCalls,
-						toolCalls: toolCallsArray,
-						numTurns: msg.num_turns,
-						durationMs: msg.duration_ms,
-						durationApiMs: msg.duration_api_ms,
-						modelUsage: msg.modelUsage,
-						permissionDenials: msg.permission_denials,
-						...msg.terminal_reason === void 0 ? {} : { terminalReason: msg.terminal_reason }
+				}
+				drainOrphans();
+				return { error: "Claude Agent SDK call didn't return a result" };
+			}, (response) => {
+				const metadata = response.metadata ?? {};
+				const additional = {};
+				if (typeof metadata.numTurns === "number") additional["gen_ai.agent.num_turns"] = metadata.numTurns;
+				if (typeof metadata.durationApiMs === "number") additional["gen_ai.agent.duration_api_ms"] = metadata.durationApiMs;
+				if (typeof response.cost === "number" && response.cost > 0) additional["gen_ai.agent.cost_usd"] = response.cost;
+				const toolCalls = metadata.toolCalls;
+				if (Array.isArray(toolCalls) && toolCalls.length > 0) additional["gen_ai.agent.tool_call_count"] = toolCalls.length;
+				let responseModel;
+				const modelUsage = metadata.modelUsage;
+				if (modelUsage && typeof modelUsage === "object" && !Array.isArray(modelUsage)) {
+					let topUsage = -1;
+					for (const [key, usage] of Object.entries(modelUsage)) {
+						if (typeof key !== "string" || key.length === 0 || key === "undefined") continue;
+						const total = (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0);
+						if (total > topUsage) {
+							topUsage = total;
+							responseModel = key;
+						}
 					}
+				}
+				const finishReasons = typeof metadata.terminalReason === "string" ? [metadata.terminalReason] : void 0;
+				return {
+					tokenUsage: response.tokenUsage,
+					responseModel,
+					responseId: response.sessionId,
+					finishReasons,
+					cacheHit: response.cached,
+					responseBody: typeof response.output === "string" ? response.output : response.output === void 0 ? void 0 : JSON.stringify(response.output),
+					additionalAttributes: Object.keys(additional).length > 0 ? additional : void 0
 				};
-			}
-			return { error: "Claude Agent SDK call didn't return a result" };
+			});
 		} catch (error) {
 			if (error?.name === "AbortError" || callOptions?.abortSignal?.aborted) {
 				logger.warn("Claude Agent SDK call aborted");
@@ -387,4 +534,4 @@ var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
 //#endregion
 export { ClaudeCodeSDKProvider };
 
-//# sourceMappingURL=claude-agent-sdk-DYv_AJ8u.js.map
+//# sourceMappingURL=claude-agent-sdk-BGUac_kS.js.map