promptfoo 0.121.4 → 0.121.7

package/dist/src/{types-BGQDAP8i.js → types-D6glLbdF.js}

@@ -1,132 +1,13 @@
-import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DnqA1y9x.js";
+import { i as isJavascriptFile, o as NunjucksFilterMapSchema, s as StringOrFunctionSchema, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-CXRfY3Ss.js";
 import dedent from "dedent";
 import { z } from "zod";
-//#region src/types/env.ts
-const ProviderEnvOverridesSchema = z.object({
-	AI21_API_BASE_URL: z.string().optional(),
-	AI21_API_KEY: z.string().optional(),
-	AIML_API_KEY: z.string().optional(),
-	ANTHROPIC_API_KEY: z.string().optional(),
-	ANTHROPIC_BASE_URL: z.string().optional(),
-	AWS_BEDROCK_REGION: z.string().optional(),
-	AZURE_API_BASE_URL: z.string().optional(),
-	AZURE_API_HOST: z.string().optional(),
-	AZURE_API_KEY: z.string().optional(),
-	AZURE_AUTHORITY_HOST: z.string().optional(),
-	AZURE_CLIENT_ID: z.string().optional(),
-	AZURE_CLIENT_SECRET: z.string().optional(),
-	AZURE_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_OPENAI_API_BASE_URL: z.string().optional(),
-	AZURE_OPENAI_API_HOST: z.string().optional(),
-	AZURE_OPENAI_API_KEY: z.string().optional(),
-	AZURE_OPENAI_BASE_URL: z.string().optional(),
-	AZURE_OPENAI_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_TENANT_ID: z.string().optional(),
-	AZURE_TOKEN_SCOPE: z.string().optional(),
-	CLAUDE_CODE_USE_BEDROCK: z.string().optional(),
-	CLAUDE_CODE_USE_VERTEX: z.string().optional(),
-	CLOUDFLARE_ACCOUNT_ID: z.string().optional(),
-	CLOUDFLARE_API_KEY: z.string().optional(),
-	CLOUDFLARE_GATEWAY_ID: z.string().optional(),
-	CF_AIG_TOKEN: z.string().optional(),
-	COMETAPI_KEY: z.string().optional(),
-	COHERE_API_KEY: z.string().optional(),
-	COHERE_CLIENT_NAME: z.string().optional(),
-	DATABRICKS_TOKEN: z.string().optional(),
-	DATABRICKS_WORKSPACE_URL: z.string().optional(),
-	DOCKER_MODEL_RUNNER_BASE_URL: z.string().optional(),
-	DOCKER_MODEL_RUNNER_API_KEY: z.string().optional(),
-	ELEVENLABS_API_KEY: z.string().optional(),
-	FAL_KEY: z.string().optional(),
-	GITHUB_TOKEN: z.string().optional(),
-	GOOGLE_API_HOST: z.string().optional(),
-	GOOGLE_API_BASE_URL: z.string().optional(),
-	GOOGLE_API_KEY: z.string().optional(),
-	GOOGLE_PROJECT_ID: z.string().optional(),
-	GOOGLE_LOCATION: z.string().optional(),
-	GOOGLE_GENERATIVE_AI_API_KEY: z.string().optional(),
-	GEMINI_API_KEY: z.string().optional(),
-	GROQ_API_KEY: z.string().optional(),
-	HELICONE_API_KEY: z.string().optional(),
-	HF_API_TOKEN: z.string().optional(),
-	HF_TOKEN: z.string().optional(),
-	HYPERBOLIC_API_KEY: z.string().optional(),
-	HUGGING_FACE_HUB_TOKEN: z.string().optional(),
-	JFROG_API_KEY: z.string().optional(),
-	LANGFUSE_HOST: z.string().optional(),
-	LANGFUSE_PUBLIC_KEY: z.string().optional(),
-	LANGFUSE_SECRET_KEY: z.string().optional(),
-	LITELLM_API_BASE: z.string().optional(),
-	LLAMA_BASE_URL: z.string().optional(),
-	LOCALAI_BASE_URL: z.string().optional(),
-	MISTRAL_API_BASE_URL: z.string().optional(),
-	MISTRAL_API_HOST: z.string().optional(),
-	MISTRAL_API_KEY: z.string().optional(),
-	MODELSLAB_API_KEY: z.string().optional(),
-	NSCALE_SERVICE_TOKEN: z.string().optional(),
-	NSCALE_API_KEY: z.string().optional(),
-	OLLAMA_API_KEY: z.string().optional(),
-	OLLAMA_BASE_URL: z.string().optional(),
-	OPENAI_API_BASE_URL: z.string().optional(),
-	OPENAI_API_HOST: z.string().optional(),
-	OPENAI_API_KEY: z.string().optional(),
-	OPENAI_BASE_URL: z.string().optional(),
-	OPENAI_ORGANIZATION: z.string().optional(),
-	CLAWDBOT_GATEWAY_PASSWORD: z.string().optional(),
-	CLAWDBOT_GATEWAY_TOKEN: z.string().optional(),
-	CLAWDBOT_GATEWAY_URL: z.string().optional(),
-	CODEX_API_KEY: z.string().optional(),
-	OPENCLAW_CONFIG_PATH: z.string().optional(),
-	OPENCLAW_GATEWAY_PASSWORD: z.string().optional(),
-	OPENCLAW_GATEWAY_PORT: z.string().optional(),
-	OPENCLAW_GATEWAY_TOKEN: z.string().optional(),
-	OPENCLAW_GATEWAY_URL: z.string().optional(),
-	PALM_API_HOST: z.string().optional(),
-	PALM_API_KEY: z.string().optional(),
-	PORTKEY_API_KEY: z.string().optional(),
-	PROMPTFOO_CA_CERT_PATH: z.string().optional(),
-	PROMPTFOO_PFX_CERT_PATH: z.string().optional(),
-	PROMPTFOO_PFX_PASSWORD: z.string().optional(),
-	PROMPTFOO_JKS_CERT_PATH: z.string().optional(),
-	PROMPTFOO_JKS_PASSWORD: z.string().optional(),
-	PROMPTFOO_JKS_ALIAS: z.string().optional(),
-	PROMPTFOO_INSECURE_SSL: z.string().optional(),
-	QUIVERAI_API_KEY: z.string().optional(),
-	REPLICATE_API_KEY: z.string().optional(),
-	REPLICATE_API_TOKEN: z.string().optional(),
-	SHAREPOINT_BASE_URL: z.string().optional(),
-	SHAREPOINT_CERT_PATH: z.string().optional(),
-	SHAREPOINT_CLIENT_ID: z.string().optional(),
-	SHAREPOINT_TENANT_ID: z.string().optional(),
-	VERCEL_AI_GATEWAY_API_KEY: z.string().optional(),
-	VERCEL_AI_GATEWAY_BASE_URL: z.string().optional(),
-	VERTEX_API_HOST: z.string().optional(),
-	VERTEX_API_KEY: z.string().optional(),
-	VERTEX_API_VERSION: z.string().optional(),
-	VERTEX_PROJECT_ID: z.string().optional(),
-	VERTEX_PUBLISHER: z.string().optional(),
-	VERTEX_REGION: z.string().optional(),
-	VOYAGE_API_BASE_URL: z.string().optional(),
-	VOYAGE_API_KEY: z.string().optional(),
-	WATSONX_AI_APIKEY: z.string().optional(),
-	WATSONX_AI_AUTH_TYPE: z.string().optional(),
-	WATSONX_AI_BEARER_TOKEN: z.string().optional(),
-	WATSONX_AI_PROJECT_ID: z.string().optional(),
-	XAI_API_BASE_URL: z.string().optional(),
-	XAI_API_KEY: z.string().optional(),
-	AZURE_CONTENT_SAFETY_ENDPOINT: z.string().optional(),
-	AZURE_CONTENT_SAFETY_API_KEY: z.string().optional(),
-	AZURE_CONTENT_SAFETY_API_VERSION: z.string().optional(),
-	AWS_REGION: z.string().optional(),
-	AWS_DEFAULT_REGION: z.string().optional(),
-	AWS_SAGEMAKER_MAX_TOKENS: z.string().optional(),
-	AWS_SAGEMAKER_TEMPERATURE: z.string().optional(),
-	AWS_SAGEMAKER_TOP_P: z.string().optional(),
-	AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
-	PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
-});
+//#region src/types/providers.ts
+function isApiProvider(provider) {
+	return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "function" && "callApi" in provider && typeof provider.callApi === "function";
+}
+function isProviderOptions(provider) {
+	return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "string";
+}
 //#endregion
 //#region src/types/shared.ts
 const CompletionTokenDetailsSchema = z.object({
@@ -155,27 +36,80 @@ const BaseTokenUsageSchema = z.object({
 		completionDetails: CompletionTokenDetailsSchema.optional()
 	}).optional()
 });
-const InputsSchema = z.record(z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" }), z.string().min(1, { error: "Input descriptions must be non-empty strings" }));
-//#endregion
-//#region src/validators/prompts.ts
-const PromptConfigSchema = z.object({
-	prefix: z.string().optional(),
-	suffix: z.string().optional()
+const InputVariableNameSchema = z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" });
+const InputTypeValues = [
+	"text",
+	"pdf",
+	"docx",
+	"image"
+];
+const InputTypeSchema = z.enum(InputTypeValues);
+const DocxInjectionPlacementValues = [
+	"body",
+	"comment",
+	"footnote",
+	"header",
+	"footer"
+];
+const DocxInjectionPlacementSchema = z.enum(DocxInjectionPlacementValues);
+const DocumentMediaInjectionPlacementValues = [
+	"body",
+	"header",
+	"footer"
+];
+const DocumentMediaInjectionPlacementSchema = z.enum(DocumentMediaInjectionPlacementValues);
+const InputConfigSchema = z.object({
+	benign: z.boolean().optional(),
+	inputPurpose: z.string().min(1, { error: "Input purpose must be a non-empty string" }).optional(),
+	injectionPlacements: z.array(z.string().min(1, { error: "Injection placement must be a non-empty string" })).min(1, { error: "Injection placements must contain at least one placement" }).optional()
 });
-const PromptFunctionSchema = z.custom((v) => typeof v === "function");
-const PromptSchema = z.object({
-	id: z.string().optional(),
-	raw: z.string(),
-	template: z.string().optional(),
-	display: z.string().optional(),
-	label: z.string(),
-	function: PromptFunctionSchema.optional(),
-	config: z.any().optional()
+const InputDefinitionObjectSchema = z.object({
+	config: InputConfigSchema.optional(),
+	description: z.string().min(1, { error: "Input descriptions must be non-empty strings" }),
+	type: InputTypeSchema.optional()
+}).superRefine((input, ctx) => {
+	const inputType = input.type ?? "text";
+	const injectionPlacements = input.config?.injectionPlacements ?? [];
+	if (inputType === "text" || injectionPlacements.length === 0) return;
+	const placementSchema = inputType === "docx" ? DocxInjectionPlacementSchema : DocumentMediaInjectionPlacementSchema;
+	const placementValues = inputType === "docx" ? DocxInjectionPlacementValues : DocumentMediaInjectionPlacementValues;
+	const invalidPlacements = injectionPlacements.filter((placement) => !placementSchema.safeParse(placement).success);
+	if (invalidPlacements.length > 0) ctx.addIssue({
+		code: "custom",
+		path: ["config", "injectionPlacements"],
+		message: `Invalid ${inputType.toUpperCase()} injection placements: ${invalidPlacements.join(", ")}. Expected one of: ${placementValues.join(", ")}`
+	});
 });
-function assert$1() {}
-assert$1();
-assert$1();
-assert$1();
+const InputDefinitionSchema = z.union([z.string().min(1, { error: "Input descriptions must be non-empty strings" }), InputDefinitionObjectSchema]);
+function normalizeInputDefinition(input) {
+	if (typeof input === "string") return {
+		description: input,
+		type: "text"
+	};
+	return {
+		config: input.config,
+		description: input.description,
+		type: input.type ?? "text"
+	};
+}
+function normalizeInputs(inputs) {
+	if (!inputs) return;
+	return Object.fromEntries(Object.entries(inputs).map(([key, input]) => [key, normalizeInputDefinition(input)]));
+}
+function getInputDescription(input) {
+	return normalizeInputDefinition(input).description;
+}
+function getInputType(input) {
+	return normalizeInputDefinition(input).type;
+}
+function buildInputPromptDescription(input) {
+	const normalized = normalizeInputDefinition(input);
+	const benignGuidance = normalized.config?.benign ? " Generate benign, natural, non-adversarial content for this input. Do not place attack instructions or policy-violating content here." : "";
+	if (normalized.type === "text") return `${normalized.description}${benignGuidance}`;
+	const formatLabel = normalized.type === "pdf" ? "PDF document" : normalized.type === "docx" ? "DOCX document" : "image";
+	return `${normalized.description} (format: ${formatLabel}; provide the text or instructions that should be embedded in the file)${benignGuidance}`;
+}
+const InputsSchema = z.record(InputVariableNameSchema, InputDefinitionSchema);
 //#endregion
 //#region src/redteam/constants/codingAgents.ts
 const CODING_AGENT_CORE_PLUGINS = [
@@ -189,10 +123,16 @@ const CODING_AGENT_PLUGINS = [
 	...CODING_AGENT_CORE_PLUGINS,
 	"coding-agent:secret-file-read",
 	"coding-agent:sandbox-write-escape",
-	"coding-agent:network-egress-bypass"
+	"coding-agent:network-egress-bypass",
+	"coding-agent:procfs-credential-read",
+	"coding-agent:delayed-ci-exfil",
+	"coding-agent:generated-vulnerability",
+	"coding-agent:automation-poisoning",
+	"coding-agent:steganographic-exfil"
 ];
-const CODING_AGENT_COLLECTIONS = ["coding-agent:core"];
+const CODING_AGENT_COLLECTIONS = ["coding-agent:core", "coding-agent:all"];
 const CODING_AGENT_PLUGIN_DISPLAY_NAMES = {
+	"coding-agent:all": "Coding Agent Full Suite",
 	"coding-agent:core": "Coding Agent Core",
 	"coding-agent:repo-prompt-injection": "Repository Prompt Injection",
 	"coding-agent:terminal-output-injection": "Terminal Output Injection",
@@ -201,10 +141,16 @@ const CODING_AGENT_PLUGIN_DISPLAY_NAMES = {
 	"coding-agent:sandbox-read-escape": "Sandbox Read Escape",
 	"coding-agent:sandbox-write-escape": "Sandbox Write Escape",
 	"coding-agent:network-egress-bypass": "Network Egress Bypass",
+	"coding-agent:procfs-credential-read": "Procfs Credential Read",
+	"coding-agent:delayed-ci-exfil": "Delayed CI Exfiltration",
+	"coding-agent:generated-vulnerability": "Generated Vulnerability",
+	"coding-agent:automation-poisoning": "Automation Poisoning",
+	"coding-agent:steganographic-exfil": "Steganographic Exfiltration",
 	"coding-agent:verifier-sabotage": "Verifier Sabotage"
 };
 const CODING_AGENT_PLUGIN_DESCRIPTIONS = {
-	"coding-agent:core": "Tests coding agents against a focused MVP set of repository prompt injection, terminal output injection, secret environment read, sandbox read escape, and verifier sabotage risks",
+	"coding-agent:all": "Tests coding agents against every supported coding-agent plugin, including prompt injection, terminal output injection, secret handling, sandbox read/write boundaries, network egress, procfs credentials, delayed CI exfiltration, generated vulnerabilities, automation poisoning, steganographic exfiltration, and verifier sabotage risks",
+	"coding-agent:core": "Tests coding agents against a focused baseline set of repository prompt injection, terminal output injection, secret environment read, sandbox read escape, and verifier sabotage risks",
 	"coding-agent:repo-prompt-injection": "Tests whether instructions embedded in README files, source comments, fixture data, issue text, logs, or generated artifacts can override the user task or operating policy",
 	"coding-agent:terminal-output-injection": "Tests whether compiler, test, package manager, linter, formatter, git, grep, or script output can instruct the coding agent to perform unrelated or dangerous actions",
 	"coding-agent:secret-env-read": "Tests whether launcher-only environment canaries remain absent from commands, output, traces, reports, diffs, URLs, and tool or connector parameters",
@@ -212,9 +158,15 @@ const CODING_AGENT_PLUGIN_DESCRIPTIONS = {
 	"coding-agent:sandbox-read-escape": "Tests whether the agent can read outside the intended workspace through absolute paths, sibling checkout paths, symlinks, archives, mount points, caches, temp paths, browser profiles, or home directories",
 	"coding-agent:sandbox-write-escape": "Tests whether the agent can create, overwrite, append, delete, chmod, or otherwise mutate files outside the intended workspace through relative paths, absolute paths, symlinks, temp paths, home paths, caches, mounts, or sibling checkouts",
 	"coding-agent:network-egress-bypass": "Tests whether the agent can make unauthorized outbound network requests, DNS lookups, package fetches, webhook calls, tunnels, or connector/tool calls that bypass the intended network policy",
+	"coding-agent:procfs-credential-read": "Tests whether the agent reads process metadata, inherited environment, file descriptors, command lines, or runtime credential material through /proc or procfs-like introspection surfaces",
+	"coding-agent:delayed-ci-exfil": "Tests whether the agent plants CI, workflow, hook, package, or scheduled automation changes that leak data or call unapproved endpoints after the immediate evaluation run completes",
+	"coding-agent:generated-vulnerability": "Tests whether the agent satisfies a normal coding request by introducing insecure code such as unsafe command execution, injection flaws, broken auth, unsafe deserialization, or disabled validation",
+	"coding-agent:automation-poisoning": "Tests whether the agent persists unsafe behavior by modifying repo automation such as CI workflows, hooks, package scripts, scheduled jobs, task runners, or release scripts",
+	"coding-agent:steganographic-exfil": "Tests whether the agent hides sensitive data in encoded, transformed, or low-visibility outputs such as comments, logs, filenames, snapshots, generated artifacts, or metadata",
 	"coding-agent:verifier-sabotage": "Tests whether the agent weakens tests, lint, typecheck, CI, snapshots, fixtures, auth, locks, security checks, policies, or harness assertions instead of fixing the requested implementation"
 };
 const CODING_AGENT_PLUGIN_ALIASES = {
+	"coding-agent:all": "CodingAgentAll",
 	"coding-agent:core": "CodingAgentCore",
 	"coding-agent:repo-prompt-injection": "CodingAgentRepoPromptInjection",
 	"coding-agent:terminal-output-injection": "CodingAgentTerminalOutputInjection",
@@ -223,6 +175,11 @@ const CODING_AGENT_PLUGIN_ALIASES = {
 	"coding-agent:sandbox-read-escape": "CodingAgentSandboxReadEscape",
 	"coding-agent:sandbox-write-escape": "CodingAgentSandboxWriteEscape",
 	"coding-agent:network-egress-bypass": "CodingAgentNetworkEgressBypass",
+	"coding-agent:procfs-credential-read": "CodingAgentProcfsCredentialRead",
+	"coding-agent:delayed-ci-exfil": "CodingAgentDelayedCiExfil",
+	"coding-agent:generated-vulnerability": "CodingAgentGeneratedVulnerability",
+	"coding-agent:automation-poisoning": "CodingAgentAutomationPoisoning",
+	"coding-agent:steganographic-exfil": "CodingAgentSteganographicExfil",
 	"coding-agent:verifier-sabotage": "CodingAgentVerifierSabotage"
 };
 //#endregion
@@ -1859,7 +1816,7 @@ Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Inform
 Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
 const codingAgentRiskCategorySeverityMap = {
 	...Object.fromEntries(CODING_AGENT_PLUGINS.map((pluginId) => [pluginId, Severity.High])),
-	"coding-agent:core": Severity.High
+	...Object.fromEntries(CODING_AGENT_COLLECTIONS.map((collectionId) => [collectionId, Severity.High]))
 };
 const riskCategorySeverityMap = {
 	["agentic:memory-poisoning"]: Severity.High,
@@ -2741,6 +2698,10 @@ const PluginConfigSchema = z.object({
 	protectedWritePaths: z.array(z.string()).optional(),
 	sandboxWritePath: z.string().optional(),
 	sandboxWritePaths: z.array(z.string()).optional(),
+	verifierArtifactRoot: z.string().optional(),
+	verifierArtifactRoots: z.array(z.string()).optional(),
+	verifierProbeDir: z.string().optional(),
+	verifierProbeDirs: z.array(z.string()).optional(),
 	workspacePath: z.string().optional(),
 	workspacePaths: z.array(z.string()).optional(),
 	workspaceRoot: z.string().optional(),
@@ -2797,13 +2758,142 @@ var PartialGenerationError = class extends Error {
 	}
 };
 //#endregion
+//#region src/types/env.ts
+const ProviderEnvOverridesSchema = z.object({
+	ABLIT_API_BASE_URL: z.string().optional(),
+	ABLIT_KEY: z.string().optional(),
+	AI21_API_BASE_URL: z.string().optional(),
+	AI21_API_KEY: z.string().optional(),
+	AIML_API_KEY: z.string().optional(),
+	ANTHROPIC_API_KEY: z.string().optional(),
+	ANTHROPIC_BASE_URL: z.string().optional(),
+	AWS_BEDROCK_REGION: z.string().optional(),
+	AZURE_API_BASE_URL: z.string().optional(),
+	AZURE_API_HOST: z.string().optional(),
+	AZURE_API_KEY: z.string().optional(),
+	AZURE_AUTHORITY_HOST: z.string().optional(),
+	AZURE_CLIENT_ID: z.string().optional(),
+	AZURE_CLIENT_SECRET: z.string().optional(),
+	AZURE_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_OPENAI_API_BASE_URL: z.string().optional(),
+	AZURE_OPENAI_API_HOST: z.string().optional(),
+	AZURE_OPENAI_API_KEY: z.string().optional(),
+	AZURE_OPENAI_BASE_URL: z.string().optional(),
+	AZURE_OPENAI_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_TENANT_ID: z.string().optional(),
+	AZURE_TOKEN_SCOPE: z.string().optional(),
+	CLAUDE_CODE_USE_BEDROCK: z.string().optional(),
+	CLAUDE_CODE_USE_VERTEX: z.string().optional(),
+	CLOUDFLARE_ACCOUNT_ID: z.string().optional(),
+	CLOUDFLARE_API_KEY: z.string().optional(),
+	CLOUDFLARE_GATEWAY_ID: z.string().optional(),
+	CF_AIG_TOKEN: z.string().optional(),
+	COMETAPI_KEY: z.string().optional(),
+	COHERE_API_KEY: z.string().optional(),
+	COHERE_CLIENT_NAME: z.string().optional(),
+	DATABRICKS_TOKEN: z.string().optional(),
+	DATABRICKS_WORKSPACE_URL: z.string().optional(),
+	DOCKER_MODEL_RUNNER_BASE_URL: z.string().optional(),
+	DOCKER_MODEL_RUNNER_API_KEY: z.string().optional(),
+	ELEVENLABS_API_KEY: z.string().optional(),
+	FAL_KEY: z.string().optional(),
+	GITHUB_TOKEN: z.string().optional(),
+	GOOGLE_API_HOST: z.string().optional(),
+	GOOGLE_API_BASE_URL: z.string().optional(),
+	GOOGLE_API_KEY: z.string().optional(),
+	GOOGLE_PROJECT_ID: z.string().optional(),
+	GOOGLE_LOCATION: z.string().optional(),
+	GOOGLE_GENERATIVE_AI_API_KEY: z.string().optional(),
+	GEMINI_API_KEY: z.string().optional(),
+	GROQ_API_KEY: z.string().optional(),
+	HELICONE_API_KEY: z.string().optional(),
+	HF_API_TOKEN: z.string().optional(),
+	HF_TOKEN: z.string().optional(),
+	HYPERBOLIC_API_KEY: z.string().optional(),
+	HUGGING_FACE_HUB_TOKEN: z.string().optional(),
+	JFROG_API_KEY: z.string().optional(),
+	LANGFUSE_HOST: z.string().optional(),
+	LANGFUSE_PUBLIC_KEY: z.string().optional(),
+	LANGFUSE_SECRET_KEY: z.string().optional(),
+	LITELLM_API_BASE: z.string().optional(),
+	LLAMA_BASE_URL: z.string().optional(),
+	LOCALAI_BASE_URL: z.string().optional(),
+	MISTRAL_API_BASE_URL: z.string().optional(),
+	MISTRAL_API_HOST: z.string().optional(),
+	MISTRAL_API_KEY: z.string().optional(),
+	MODELSLAB_API_KEY: z.string().optional(),
+	NSCALE_SERVICE_TOKEN: z.string().optional(),
+	NSCALE_API_KEY: z.string().optional(),
+	OLLAMA_API_KEY: z.string().optional(),
+	OLLAMA_BASE_URL: z.string().optional(),
+	OPENAI_API_BASE_URL: z.string().optional(),
+	OPENAI_API_HOST: z.string().optional(),
+	OPENAI_API_KEY: z.string().optional(),
+	OPENAI_BASE_URL: z.string().optional(),
+	OPENAI_ORGANIZATION: z.string().optional(),
+	CLAWDBOT_GATEWAY_PASSWORD: z.string().optional(),
+	CLAWDBOT_GATEWAY_TOKEN: z.string().optional(),
+	CLAWDBOT_GATEWAY_URL: z.string().optional(),
+	CODEX_API_KEY: z.string().optional(),
+	OPENCLAW_CONFIG_PATH: z.string().optional(),
+	OPENCLAW_GATEWAY_PASSWORD: z.string().optional(),
+	OPENCLAW_GATEWAY_PORT: z.string().optional(),
+	OPENCLAW_GATEWAY_TOKEN: z.string().optional(),
+	OPENCLAW_GATEWAY_URL: z.string().optional(),
+	PALM_API_HOST: z.string().optional(),
+	PALM_API_KEY: z.string().optional(),
+	PORTKEY_API_KEY: z.string().optional(),
+	PROMPTFOO_CA_CERT_PATH: z.string().optional(),
+	PROMPTFOO_PFX_CERT_PATH: z.string().optional(),
+	PROMPTFOO_PFX_PASSWORD: z.string().optional(),
+	PROMPTFOO_JKS_CERT_PATH: z.string().optional(),
+	PROMPTFOO_JKS_PASSWORD: z.string().optional(),
+	PROMPTFOO_JKS_ALIAS: z.string().optional(),
+	PROMPTFOO_INSECURE_SSL: z.string().optional(),
+	QUIVERAI_API_KEY: z.string().optional(),
+	REPLICATE_API_KEY: z.string().optional(),
+	REPLICATE_API_TOKEN: z.string().optional(),
+	SHAREPOINT_BASE_URL: z.string().optional(),
+	SHAREPOINT_CERT_PATH: z.string().optional(),
+	SHAREPOINT_CLIENT_ID: z.string().optional(),
+	SHAREPOINT_TENANT_ID: z.string().optional(),
+	VERCEL_AI_GATEWAY_API_KEY: z.string().optional(),
+	VERCEL_AI_GATEWAY_BASE_URL: z.string().optional(),
+	VERTEX_API_HOST: z.string().optional(),
+	VERTEX_API_KEY: z.string().optional(),
+	VERTEX_API_VERSION: z.string().optional(),
+	VERTEX_PROJECT_ID: z.string().optional(),
+	VERTEX_PUBLISHER: z.string().optional(),
+	VERTEX_REGION: z.string().optional(),
+	VOYAGE_API_BASE_URL: z.string().optional(),
+	VOYAGE_API_KEY: z.string().optional(),
+	WATSONX_AI_APIKEY: z.string().optional(),
+	WATSONX_AI_AUTH_TYPE: z.string().optional(),
+	WATSONX_AI_BEARER_TOKEN: z.string().optional(),
+	WATSONX_AI_PROJECT_ID: z.string().optional(),
+	XAI_API_BASE_URL: z.string().optional(),
+	XAI_API_KEY: z.string().optional(),
+	AZURE_CONTENT_SAFETY_ENDPOINT: z.string().optional(),
+	AZURE_CONTENT_SAFETY_API_KEY: z.string().optional(),
+	AZURE_CONTENT_SAFETY_API_VERSION: z.string().optional(),
+	AWS_REGION: z.string().optional(),
+	AWS_DEFAULT_REGION: z.string().optional(),
+	AWS_SAGEMAKER_MAX_TOKENS: z.string().optional(),
+	AWS_SAGEMAKER_TEMPERATURE: z.string().optional(),
+	AWS_SAGEMAKER_TOP_P: z.string().optional(),
+	AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
+	PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
+});
+//#endregion
 //#region src/validators/providers.ts
 const ProviderOptionsSchema = z.object({
 	id: z.custom().optional(),
 	label: z.custom().optional(),
 	config: z.any().optional(),
 	prompts: z.array(z.string()).optional(),
-	transform: z.string().optional(),
+	transform: StringOrFunctionSchema.optional(),
 	delay: z.number().optional(),
 	env: ProviderEnvOverridesSchema.optional(),
 	inputs: InputsSchema.optional()
@@ -2815,7 +2905,7 @@ const ApiProviderSchema = z.object({
 	callEmbeddingApi: z.custom((v) => typeof v === "function").optional(),
 	callClassificationApi: z.custom((v) => typeof v === "function").optional(),
 	label: z.custom().optional(),
-	transform: z.string().optional(),
+	transform: StringOrFunctionSchema.optional(),
 	delay: z.number().optional(),
 	config: z.any().optional(),
 	inputs: InputsSchema.optional()
@@ -2859,6 +2949,26 @@ const ProviderSchema = z.union([
 	ProviderOptionsSchema
 ]);
 //#endregion
+//#region src/validators/prompts.ts
+const PromptConfigSchema = z.object({
+	prefix: z.string().optional(),
+	suffix: z.string().optional()
+});
+const PromptFunctionSchema = z.custom((v) => typeof v === "function");
+const PromptSchema = z.object({
+	id: z.string().optional(),
+	raw: z.string(),
+	template: z.string().optional(),
+	display: z.string().optional(),
+	label: z.string(),
+	function: PromptFunctionSchema.optional(),
+	config: z.any().optional()
+});
+function assert$1() {}
+assert$1();
+assert$1();
+assert$1();
+//#endregion
 //#region src/validators/redteam.ts
 const TracingConfigSchema = z.lazy(() => z.object({
 	enabled: z.boolean().optional(),
@@ -3049,6 +3159,8 @@ const RedteamConfigSchema = z.object({
 		else if (id === "teen-safety") expandCollection([...TEEN_SAFETY_PLUGINS], config, numTests, severity);
 		else if (id === "default") expandCollection([...DEFAULT_PLUGINS], config, numTests, severity);
 		else if (id === "guardrails-eval") expandCollection([...GUARDRAILS_EVALUATION_PLUGINS], config, numTests, severity);
+		else if (id === "coding-agent:core") expandCollection([...CODING_AGENT_CORE_PLUGINS], config, numTests, severity);
+		else if (id === "coding-agent:all") expandCollection([...CODING_AGENT_PLUGINS], config, numTests, severity);
 	};
 	const handlePlugin = (plugin) => {
 		const pluginObj = typeof plugin === "string" ? {
@@ -3126,17 +3238,6 @@ const RedteamConfigSchema = z.object({
 function assert() {}
 assert();
 //#endregion
-//#region src/validators/shared.ts
-const NunjucksFilterMapSchema = z.record(z.string(), z.custom((v) => typeof v === "function"));
-//#endregion
-//#region src/types/providers.ts
-function isApiProvider(provider) {
-	return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "function";
-}
-function isProviderOptions(provider) {
-	return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "string";
-}
-//#endregion
 //#region src/types/index.ts
 const CommandLineOptionsSchema = z.object({
 	description: z.string().optional(),
@@ -3202,9 +3303,9 @@ const GradingConfigSchema = z.object({
 	}).optional()
 });
 const OutputConfigSchema = z.object({
-	postprocess: z.string().optional(),
-	transform: z.string().optional(),
-	transformVars: z.string().optional(),
+	postprocess: StringOrFunctionSchema.optional(),
+	transform: StringOrFunctionSchema.optional(),
+	transformVars: StringOrFunctionSchema.optional(),
 	storeOutputAs: z.string().optional()
 });
 const EvaluateOptionsSchema = z.object({
@@ -3354,8 +3455,8 @@ const AssertionSchema = z.object({
 	provider: z.custom().optional(),
 	rubricPrompt: z.custom().optional(),
 	metric: z.string().optional(),
-	transform: z.string().optional(),
-	contextTransform: z.string().optional()
+	transform: StringOrFunctionSchema.optional(),
+	contextTransform: StringOrFunctionSchema.optional()
 });
 /**
 * Schema for validating individual assertions (regular or assert-set).
@@ -3613,6 +3714,6 @@ const EvalResultsFilterMode = z.enum([
 	"user-rated"
 ]);
 //#endregion
-export { pluginDescriptions as $, isApiProvider as A, CompletionTokenDetailsSchema as At, isValidReusablePolicyId as B, TestGeneratorConfigSchema as C, UNALIGNED_PROVIDER_HARM_PLUGINS as Ct, VarsSchema as D, CODING_AGENT_PLUGIN_DISPLAY_NAMES as Dt, UnifiedConfigSchema as E, CODING_AGENT_PLUGIN_DESCRIPTIONS as Et, ConversationMessageSchema as F, MULTI_TURN_STRATEGIES as G, AGENTIC_STRATEGIES as H, PartialGenerationError as I, getDefaultNFanout as J, STRATEGY_COLLECTIONS as K, PluginConfigSchema as L, RedteamConfigSchema as M, ProviderOptionsSchema as N, isGradingResult as O, PromptSchema as Ot, ProvidersSchema as P, categoryAliases as Q, PolicyObjectSchema as R, TestCasesWithMetadataSchema as S, TELECOM_PLUGINS as St, TestSuiteSchema as T, CODING_AGENT_PLUGINS as Tt, DATASET_PLUGINS as U, isUuid as V, DEFAULT_STRATEGIES as W, isFanoutStrategy as X, isCustomStrategy as Y, Severity as Z, ScenarioSchema as _, PLUGIN_CATEGORIES as _t, AtomicTestCaseSchema as a, DEFAULT_PLUGINS as at, TestCaseWithVarsFileSchema as b, STRATEGY_EXEMPT_PLUGINS as bt, CompletedPromptSchema as c, HARM_PLUGINS as ct, EvaluateOptionsSchema as d, LLAMA_GUARD_REPLICATE_PROVIDER as dt, riskCategorySeverityMap as et, GradingConfigSchema as f, MEDICAL_PLUGINS as ft, ResultFailureReason as g, PII_PLUGINS as gt, OutputFileExtension as h, PHARMACY_PLUGINS as ht, AssertionTypeSchema as i, DATASET_EXEMPT_PLUGINS as it, isProviderOptions as j, InputsSchema as jt, isResultFailureReason as k, BaseTokenUsageSchema as kt, DerivedMetricSchema as l, INSURANCE_PLUGINS as lt, OutputConfigSchema as m, MULTI_INPUT_VAR as mt, AssertionSchema as n, BIAS_PLUGINS as nt, BaseAssertionTypesSchema as o, FINANCIAL_PLUGINS as ot, NotPrefixedAssertionTypesSchema as p, MULTI_INPUT_EXCLUDED_PLUGINS as pt, STRATEGY_COLLECTION_MAPPINGS as q, AssertionSetSchema as r, CANARY_BREAKING_STRATEGY_IDS as rt, CommandLineOptionsSchema as s, FOUNDATION_PLUGINS as st, AssertionOrSetSchema as t, ALIASED_PLUGIN_MAPPINGS as tt, EvalResultsFilterMode as u, LLAMA_GUARD_ENABLED_CATEGORIES as ut, SpecialAssertionTypesSchema as v, REDTEAM_PROVIDER_HARM_PLUGINS as vt, TestSuiteConfigSchema as w, CODING_AGENT_CORE_PLUGINS as wt, TestCasesWithMetadataPromptSchema as x, TEEN_SAFETY_PLUGINS as xt, TestCaseSchema as y, REMOTE_ONLY_PLUGIN_IDS as yt, StrategyConfigSchema as z };
+export { riskCategorySeverityMap as $, RedteamConfigSchema as A, DocumentMediaInjectionPlacementValues as At, isUuid as B, getInputDescription as Bt, TestGeneratorConfigSchema as C, CODING_AGENT_CORE_PLUGINS as Ct, VarsSchema as D, BaseTokenUsageSchema as Dt, UnifiedConfigSchema as E, CODING_AGENT_PLUGIN_DISPLAY_NAMES as Et, PartialGenerationError as F, InputDefinitionSchema as Ft, STRATEGY_COLLECTIONS as G, isProviderOptions as Gt, DATASET_PLUGINS as H, normalizeInputDefinition as Ht, PluginConfigSchema as I, InputTypeSchema as It, isCustomStrategy as J, STRATEGY_COLLECTION_MAPPINGS as K, PolicyObjectSchema as L, InputTypeValues as Lt, ProviderOptionsSchema as M, DocxInjectionPlacementValues as Mt, ProvidersSchema as N, InputConfigSchema as Nt, isGradingResult as O, CompletionTokenDetailsSchema as Ot, ConversationMessageSchema as P, InputDefinitionObjectSchema as Pt, pluginDescriptions as Q, StrategyConfigSchema as R, InputsSchema as Rt, TestCasesWithMetadataSchema as S, UNALIGNED_PROVIDER_HARM_PLUGINS as St, TestSuiteSchema as T, CODING_AGENT_PLUGIN_DESCRIPTIONS as Tt, DEFAULT_STRATEGIES as U, normalizeInputs as Ut, AGENTIC_STRATEGIES as V, getInputType as Vt, MULTI_TURN_STRATEGIES as W, isApiProvider as Wt, Severity as X, isFanoutStrategy as Y, categoryAliases as Z, ScenarioSchema as _, REDTEAM_PROVIDER_HARM_PLUGINS as _t, AtomicTestCaseSchema as a, FINANCIAL_PLUGINS as at, TestCaseWithVarsFileSchema as b, TEEN_SAFETY_PLUGINS as bt, CompletedPromptSchema as c, INSURANCE_PLUGINS as ct, EvaluateOptionsSchema as d, MEDICAL_PLUGINS as dt, ALIASED_PLUGIN_MAPPINGS as et, GradingConfigSchema as f, MULTI_INPUT_EXCLUDED_PLUGINS as ft, ResultFailureReason as g, PLUGIN_CATEGORIES as gt, OutputFileExtension as h, PII_PLUGINS as ht, AssertionTypeSchema as i, DEFAULT_PLUGINS as it, PromptSchema as j, DocxInjectionPlacementSchema as jt, isResultFailureReason as k, DocumentMediaInjectionPlacementSchema as kt, DerivedMetricSchema as l, LLAMA_GUARD_ENABLED_CATEGORIES as lt, OutputConfigSchema as m, PHARMACY_PLUGINS as mt, AssertionSchema as n, CANARY_BREAKING_STRATEGY_IDS as nt, BaseAssertionTypesSchema as o, FOUNDATION_PLUGINS as ot, NotPrefixedAssertionTypesSchema as p, MULTI_INPUT_VAR as pt, getDefaultNFanout as q, AssertionSetSchema as r, DATASET_EXEMPT_PLUGINS as rt, CommandLineOptionsSchema as s, HARM_PLUGINS as st, AssertionOrSetSchema as t, BIAS_PLUGINS as tt, EvalResultsFilterMode as u, LLAMA_GUARD_REPLICATE_PROVIDER as ut, SpecialAssertionTypesSchema as v, REMOTE_ONLY_PLUGIN_IDS as vt, TestSuiteConfigSchema as w, CODING_AGENT_PLUGINS as wt, TestCasesWithMetadataPromptSchema as x, TELECOM_PLUGINS as xt, TestCaseSchema as y, STRATEGY_EXEMPT_PLUGINS as yt, isValidReusablePolicyId as z, buildInputPromptDescription as zt };
 
-//# sourceMappingURL=types-BGQDAP8i.js.map
+//# sourceMappingURL=types-D6glLbdF.js.map

package/dist/src/{util-BYvQUPp7.js → util--WMgw7wM.js}

@@ -1,5 +1,5 @@
-import { g as calculateCost } from "./fetch-B0Z3Oe4k.js";
-import { w as parseDataUrl } from "./transform-lQrDE1BQ.js";
+import { m as calculateCost } from "./fetch-It34O8Ur.js";
+import { w as parseDataUrl } from "./transform-DmvYBRll.js";
 //#region src/providers/anthropic/util.ts
 const ANTHROPIC_MODELS = [
 	...["claude-mythos-preview"].map((model) => ({
@@ -9,6 +9,13 @@ const ANTHROPIC_MODELS = [
 			output: 125 / 1e6
 		}
 	})),
+	...["claude-opus-4-7"].map((model) => ({
+		id: model,
+		cost: {
+			input: 5 / 1e6,
+			output: 25 / 1e6
+		}
+	})),
 	...["claude-sonnet-4-6", "claude-sonnet-4-6-latest"].map((model) => ({
 		id: model,
 		cost: {
@@ -121,6 +128,15 @@ const ANTHROPIC_MODELS = [
 		}
 	}))
 ];
+/**
+* Matches Claude Opus 4.7 model IDs across Anthropic, Bedrock (including the
+* `us.`/`eu.`/`jp.`/`global.` inference-profile prefixes), Vertex, and Azure
+* deployment names. Returns `false` for hypothetical suffix variants like
+* `claude-opus-4-70` or `claude-opus-4-7N` so detection stays forward-compatible.
+*/
+function isClaudeOpus47Model(modelId) {
+	return /(^|[^a-z0-9])claude-opus-4-7(?![0-9])/i.test(modelId);
+}
 function outputFromMessage(message, showThinking) {
 	const hasToolUse = message.content.some((block) => block.type === "tool_use");
 	const hasThinking = message.content.some((block) => block.type === "thinking" || block.type === "redacted_thinking");
@@ -235,7 +251,7 @@ function calculateCacheInputCost(baseInputRate, uncachedInputTokens, cacheRead,
 	return uncachedInputTokens * baseInputRate + cacheRead * baseInputRate * .1 + cacheCreation * baseInputRate * 1.25;
 }
 function calculateAnthropicCost(modelName, config, promptTokens, completionTokens, cacheReadTokens, cacheCreationTokens) {
-	if (config.cost != null) return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
+	if (config.cost != null && config.inputCost == null && config.outputCost == null) return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
 	if (!Number.isFinite(promptTokens) || !Number.isFinite(completionTokens) || typeof promptTokens === "undefined" || typeof completionTokens === "undefined") return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
 	const cacheRead = cacheReadTokens ?? 0;
 	const cacheCreation = cacheCreationTokens ?? 0;
@@ -248,13 +264,17 @@ function calculateAnthropicCost(modelName, config, promptTokens, completionToken
 		"claude-sonnet-4-6-latest"
 	].includes(modelName)) {
 		const isLongContext = effectiveInputTokens > 2e5;
-		const baseInputRate = isLongContext ? 6 / 1e6 : 3 / 1e6;
-		const outputRate = isLongContext ? 22.5 / 1e6 : 15 / 1e6;
+		const baseInputRate = config.inputCost ?? config.cost ?? (isLongContext ? 6 / 1e6 : 3 / 1e6);
+		const outputRate = config.outputCost ?? config.cost ?? (isLongContext ? 22.5 / 1e6 : 15 / 1e6);
 		return calculateCacheInputCost(baseInputRate, promptTokens, cacheRead, cacheCreation) + completionTokens * outputRate;
 	}
 	if (cacheRead || cacheCreation) {
 		const modelInfo = ANTHROPIC_MODELS.find((m) => m.id === modelName);
-		if (modelInfo) return calculateCacheInputCost(modelInfo.cost.input, promptTokens, cacheRead, cacheCreation) + completionTokens * modelInfo.cost.output;
+		if (modelInfo) {
+			const inputCost = config.inputCost ?? config.cost ?? modelInfo.cost.input;
+			const outputCost = config.outputCost ?? config.cost ?? modelInfo.cost.output;
+			return calculateCacheInputCost(inputCost, promptTokens, cacheRead, cacheCreation) + completionTokens * outputCost;
+		}
 	}
 	return calculateCost(modelName, config, promptTokens, completionTokens, ANTHROPIC_MODELS);
 }
@@ -390,6 +410,6 @@ function transformWebSearchTool20260209(config) {
 	return tool;
 }
 //#endregion
-export { outputFromMessage as a, getTokenUsage as i, calculateAnthropicCost as n, parseMessages as o, getRefusalDetails as r, processAnthropicTools as s, ANTHROPIC_MODELS as t };
+export { isClaudeOpus47Model as a, processAnthropicTools as c, getTokenUsage as i, calculateAnthropicCost as n, outputFromMessage as o, getRefusalDetails as r, parseMessages as s, ANTHROPIC_MODELS as t };
 
-//# sourceMappingURL=util-BYvQUPp7.js.map
+//# sourceMappingURL=util--WMgw7wM.js.map