promptfoo 0.122.1 → 0.122.2

package/dist/src/codex-security-DSYUnVdR.js

@@ -0,0 +1,383 @@
+#!/usr/bin/env node
+import { U as state, c as logger } from "./logger-YzVIwMVq.js";
+import { i as resolvePackageEntryPoint, r as importModule, t as getDirectory } from "./esm-BK0NNGWz.js";
+import { t as providerRegistry } from "./providerRegistry-icQCXwgg.js";
+import { o as resolveAgenticWorkingDir } from "./agentic-utils-DOMq4mqJ.js";
+import { n as renderVarsInObject } from "./render-BRIaR8VG.js";
+import "./codex-runtime-BjmQ9rWv.js";
+import { z as z$1 } from "zod";
+import path from "path";
+import dedent from "dedent";
+import fs from "fs/promises";
+import semverSatisfies from "semver/functions/satisfies.js";
+//#region src/providers/openai/codex-security.ts
+const CODEX_SECURITY_OPERATIONS = [
+	"security-scan",
+	"deep-security-scan",
+	"security-diff-scan",
+	"validation"
+];
+const MINIMUM_CODEX_SECURITY_SDK_VERSION = "0.1.18";
+const ReasoningEffortSchema = z$1.enum([
+	"minimal",
+	"low",
+	"medium",
+	"high",
+	"xhigh",
+	"max",
+	"ultra"
+]);
+const CodexSecurityConfigSchema = z$1.object({
+	basePath: z$1.string().optional(),
+	prefix: z$1.string().optional(),
+	suffix: z$1.string().optional(),
+	provider: z$1.unknown().optional(),
+	linkedTargetId: z$1.string().optional(),
+	maxRetries: z$1.number().int().nonnegative().optional(),
+	operation: z$1.enum(CODEX_SECURITY_OPERATIONS).optional(),
+	model: z$1.string().min(1).optional(),
+	model_provider: z$1.string().min(1).optional(),
+	model_reasoning_effort: ReasoningEffortSchema.optional(),
+	reasoning_effort: ReasoningEffortSchema.optional(),
+	repository: z$1.string().min(1).optional(),
+	working_dir: z$1.string().min(1).optional(),
+	paths: z$1.array(z$1.string().min(1)).min(1).optional(),
+	base_ref: z$1.string().min(1).optional(),
+	head_ref: z$1.string().min(1).optional(),
+	working_tree: z$1.boolean().optional(),
+	output_dir: z$1.string().min(1).optional(),
+	archive_existing: z$1.boolean().optional(),
+	knowledge_base_paths: z$1.array(z$1.string().min(1)).optional(),
+	scan_prompt: z$1.string().optional(),
+	validation_prompt: z$1.string().optional(),
+	post_scan_prompt: z$1.string().optional(),
+	expected_plugin_version: z$1.string().optional(),
+	failure_severity: z$1.enum([
+		"low",
+		"medium",
+		"high",
+		"critical"
+	]).optional(),
+	max_cost_usd: z$1.number().positive().optional(),
+	workers: z$1.number().int().positive().optional(),
+	subagents: z$1.number().int().nonnegative().optional(),
+	stop_after_no_new: z$1.number().int().nonnegative().optional(),
+	max_discovery_runs: z$1.number().int().positive().optional(),
+	max_time_hours: z$1.number().positive().optional(),
+	auth: z$1.enum([
+		"auto",
+		"chatgpt",
+		"api-key"
+	]).optional(),
+	plugin_path: z$1.string().min(1).optional(),
+	python_path: z$1.string().min(1).optional(),
+	codex_overrides: z$1.record(z$1.string(), z$1.unknown()).optional(),
+	finding: z$1.union([z$1.string(), z$1.record(z$1.string(), z$1.unknown())]).optional(),
+	finding_file: z$1.string().min(1).optional()
+}).strict().superRefine((config, context) => {
+	if (config.model_reasoning_effort && config.reasoning_effort && config.model_reasoning_effort !== config.reasoning_effort) context.addIssue({
+		code: "custom",
+		path: ["reasoning_effort"],
+		message: "reasoning_effort and model_reasoning_effort must match when both are set"
+	});
+	if (config.working_tree && config.head_ref) context.addIssue({
+		code: "custom",
+		path: ["head_ref"],
+		message: "head_ref cannot be combined with working_tree"
+	});
+	if (config.paths && (config.base_ref || config.head_ref || config.working_tree)) context.addIssue({
+		code: "custom",
+		path: ["paths"],
+		message: "paths cannot be combined with a diff target"
+	});
+});
+const CodexSecurityMergedPromptConfigSchema = CodexSecurityConfigSchema.strip();
+function configError(error) {
+	const details = error.issues.map((issue) => `${issue.path.join(".") || "config"}: ${issue.message}`).join("; ");
+	return /* @__PURE__ */ new Error(`Invalid OpenAI Codex Security provider configuration: ${details}`);
+}
+function parseConfig(config = {}, options = {}) {
+	const parsed = (options.stripUnknownKeys ? CodexSecurityMergedPromptConfigSchema : CodexSecurityConfigSchema).safeParse(config);
+	if (!parsed.success) throw configError(parsed.error);
+	return parsed.data;
+}
+async function loadCodexSecurity() {
+	const basePaths = [path.resolve(getDirectory(), ".."), path.resolve(getDirectory(), "../..")];
+	const visitedEntryPoints = /* @__PURE__ */ new Set();
+	const incompatibleVersions = /* @__PURE__ */ new Set();
+	let importFailed = false;
+	for (const basePath of new Set(basePaths)) {
+		const entryPoint = resolvePackageEntryPoint("@openai/codex-security", basePath);
+		if (!entryPoint || visitedEntryPoints.has(entryPoint)) continue;
+		visitedEntryPoints.add(entryPoint);
+		try {
+			const module = await importModule(entryPoint);
+			const version = typeof module.VERSION === "string" ? module.VERSION : "unknown";
+			if (!semverSatisfies(version, `>=${MINIMUM_CODEX_SECURITY_SDK_VERSION}`)) {
+				incompatibleVersions.add(version);
+				logger.warn(`[CodexSecurity] Ignoring @openai/codex-security ${version}; version ${MINIMUM_CODEX_SECURITY_SDK_VERSION} or newer is required for complete security operations and deep-scan usage accounting.`);
+				continue;
+			}
+			return module;
+		} catch (error) {
+			logger.debug("[CodexSecurity] Failed to load SDK", { error });
+			importFailed = true;
+		}
+	}
+	if (importFailed) throw new Error(dedent`Failed to load @openai/codex-security.
+
+      Promptfoo and the SDK require a supported even-numbered Node.js release: ^22.22.0, ^24.0.0, or ^26.0.0.
+      Reinstall them together with:
+        npm install promptfoo @openai/codex-security
+
+      See https://www.promptfoo.dev/docs/providers/openai-codex-security/`);
+	if (incompatibleVersions.size > 0) throw new Error(dedent`The installed @openai/codex-security package is incompatible (${Array.from(incompatibleVersions).join(", ")}).
+
+      Version ${MINIMUM_CODEX_SECURITY_SDK_VERSION} or newer is required for finding validation and accurate deep-worker cost tracking.
+      Install the compatible SDK alongside Promptfoo with:
+        npm install promptfoo @openai/codex-security@^${MINIMUM_CODEX_SECURITY_SDK_VERSION}
+
+      See https://www.promptfoo.dev/docs/providers/openai-codex-security/`);
+	throw new Error(dedent`The @openai/codex-security package is required but not installed.
+
+    Install it alongside Promptfoo with:
+      npm install promptfoo @openai/codex-security
+
+    Requires Node.js ^22.22.0, ^24.0.0, or ^26.0.0.
+    See https://www.promptfoo.dev/docs/providers/openai-codex-security/`);
+}
+function resolveConfigPath(value, configBasePath) {
+	return resolveAgenticWorkingDir(value, configBasePath ?? state.basePath);
+}
+function getTokenUsage(result, observedCost) {
+	const usage = result.turnResult.usage;
+	const values = usage && typeof usage === "object" ? usage : {};
+	const cost = result.cost ?? observedCost;
+	const inputTokens = cost?.inputTokens ?? (typeof values.input_tokens === "number" ? values.input_tokens : void 0);
+	const outputTokens = cost?.outputTokens ?? (typeof values.output_tokens === "number" ? values.output_tokens : void 0);
+	const cachedTokens = cost?.cachedInputTokens ?? (typeof values.cached_input_tokens === "number" ? values.cached_input_tokens : void 0);
+	const cacheWriteTokens = cost?.cacheWriteInputTokens ?? (typeof values.cache_write_input_tokens === "number" ? values.cache_write_input_tokens : void 0);
+	const reasoningTokens = typeof values.reasoning_output_tokens === "number" ? values.reasoning_output_tokens : void 0;
+	if (inputTokens === void 0 && outputTokens === void 0) return;
+	const completionDetails = {
+		...reasoningTokens === void 0 ? {} : { reasoning: reasoningTokens },
+		...cachedTokens === void 0 ? {} : { cacheReadInputTokens: cachedTokens },
+		...cacheWriteTokens === void 0 ? {} : { cacheCreationInputTokens: cacheWriteTokens }
+	};
+	return {
+		...inputTokens === void 0 ? {} : { prompt: inputTokens },
+		...outputTokens === void 0 ? {} : { completion: outputTokens },
+		...cachedTokens === void 0 ? {} : { cached: cachedTokens },
+		...inputTokens === void 0 || outputTokens === void 0 ? {} : { total: inputTokens + outputTokens },
+		...Object.keys(completionDetails).length > 0 ? { completionDetails } : {}
+	};
+}
+var OpenAICodexSecurityProvider = class {
+	config;
+	env;
+	providerId;
+	activeClients = /* @__PURE__ */ new Set();
+	constructor(options = {}) {
+		this.config = parseConfig(options.config);
+		this.env = options.env;
+		this.providerId = options.id ?? "openai:codex-security";
+		providerRegistry.register(this);
+	}
+	id() {
+		return this.providerId;
+	}
+	requiresApiKey() {
+		return false;
+	}
+	toString() {
+		return "[OpenAI Codex Security Provider]";
+	}
+	async cleanup() {
+		const clients = Array.from(this.activeClients);
+		this.activeClients.clear();
+		const results = await Promise.allSettled(clients.map((client) => client.close()));
+		for (const result of results) if (result.status === "rejected") logger.warn("[CodexSecurity] Error while closing SDK client", { error: result.reason });
+	}
+	async shutdown() {
+		try {
+			await this.cleanup();
+		} finally {
+			providerRegistry.unregister(this);
+		}
+	}
+	async callApi(prompt, context, callOptions) {
+		try {
+			const mergedConfig = {
+				...this.config,
+				...context?.prompt?.config
+			};
+			delete mergedConfig.provider;
+			const config = parseConfig(renderVarsInObject(mergedConfig, context?.vars), { stripUnknownKeys: true });
+			const operation = config.operation ?? "security-scan";
+			const repositoryVariable = context?.vars?.repository;
+			const repository = resolveConfigPath(config.repository ?? config.working_dir ?? (typeof repositoryVariable === "string" ? repositoryVariable : void 0), config.basePath) ?? process.cwd();
+			if (callOptions?.abortSignal?.aborted) return { error: "Codex Security operation was aborted before it started." };
+			for (const key of ["OPENAI_API_KEY", "CODEX_API_KEY"]) if (this.env?.[key] && this.env[key] !== process.env[key]) return { error: `Codex Security does not support provider-scoped ${key}. Set ${key} in the Promptfoo process environment before running the evaluation.` };
+			const module = await loadCodexSecurity();
+			const effort = config.model_reasoning_effort ?? config.reasoning_effort;
+			const codexOverrides = {
+				...config.codex_overrides,
+				...config.model ? { model: config.model } : {},
+				...config.model_provider ? { model_provider: config.model_provider } : {},
+				...effort ? { model_reasoning_effort: effort } : {}
+			};
+			const client = new module.CodexSecurity({
+				...config.plugin_path ? { pluginPath: resolveConfigPath(config.plugin_path, config.basePath) } : {},
+				...config.python_path ? { pythonPath: resolveConfigPath(config.python_path, config.basePath) } : {},
+				...Object.keys(codexOverrides).length > 0 ? { codexOverrides } : {}
+			});
+			this.activeClients.add(client);
+			try {
+				if (operation === "validation") return await this.runValidation(client, prompt, repository, config, context, callOptions);
+				return await this.runScan(client, module, prompt, repository, operation, config, callOptions);
+			} finally {
+				this.activeClients.delete(client);
+				try {
+					await client.close();
+				} catch (error) {
+					logger.warn("[CodexSecurity] Error while closing SDK client", { error });
+				}
+			}
+		} catch (error) {
+			return { error: `Codex Security operation failed: ${error instanceof Error ? error.message : String(error)}` };
+		}
+	}
+	async runScan(client, module, prompt, repository, operation, config, callOptions) {
+		const mode = operation === "deep-security-scan" ? "deep" : "standard";
+		const targetResult = this.getScanTarget(module, operation, config);
+		if ("error" in targetResult) return targetResult;
+		const observers = { warnings: [] };
+		const options = this.buildScanOptions(prompt, mode, targetResult.target, config, observers, callOptions);
+		const result = await client.run(repository, options);
+		return this.buildScanResponse(result, module, repository, operation, mode, config, observers);
+	}
+	getScanTarget(module, operation, config) {
+		if (operation === "security-diff-scan") {
+			if (config.working_tree) return { target: module.DiffTarget.workingTree(config.base_ref ? { base: config.base_ref } : void 0) };
+			if (config.base_ref) return { target: module.DiffTarget.refs({
+				base: config.base_ref,
+				...config.head_ref ? { head: config.head_ref } : {}
+			}) };
+			return { error: "Codex Security security-diff-scan requires base_ref or working_tree: true." };
+		}
+		if (config.base_ref || config.head_ref || config.working_tree) return { error: "Git diff target options require operation: security-diff-scan." };
+		return { target: config.paths ?? "repository" };
+	}
+	buildScanOptions(prompt, mode, target, config, observers, callOptions) {
+		const scanPrompt = [config.scan_prompt, prompt].filter(Boolean).join("\n\n");
+		return {
+			mode,
+			target,
+			...scanPrompt ? { scanPrompt } : {},
+			...config.auth ? { auth: config.auth } : {},
+			...config.output_dir ? { outputDir: resolveConfigPath(config.output_dir, config.basePath) } : {},
+			...config.archive_existing === void 0 ? {} : { archiveExisting: config.archive_existing },
+			...config.knowledge_base_paths ? { knowledgeBasePaths: config.knowledge_base_paths.map((knowledgePath) => resolveConfigPath(knowledgePath, config.basePath)) } : {},
+			...config.validation_prompt ? { validationPrompt: config.validation_prompt } : {},
+			...config.post_scan_prompt ? { postScanPrompt: config.post_scan_prompt } : {},
+			...config.expected_plugin_version ? { expectedPluginVersion: config.expected_plugin_version } : {},
+			...config.failure_severity ? { failureSeverity: config.failure_severity } : {},
+			...config.max_cost_usd === void 0 ? {} : { maxCostUsd: config.max_cost_usd },
+			...this.buildDeepScanOptions(mode, config),
+			...callOptions?.abortSignal ? { signal: callOptions.abortSignal } : {},
+			onCost: (cost) => {
+				observers.cost = { ...cost };
+			},
+			onProgress: (progress) => {
+				observers.progress = progress;
+			},
+			onWarning: (warning) => {
+				observers.warnings.push(warning);
+			}
+		};
+	}
+	buildDeepScanOptions(mode, config) {
+		if (mode !== "deep") return {};
+		return {
+			...config.workers === void 0 ? {} : { workers: config.workers },
+			...config.subagents === void 0 ? {} : { subagents: config.subagents },
+			...config.stop_after_no_new === void 0 ? {} : { stopAfterNoNew: config.stop_after_no_new },
+			...config.max_discovery_runs === void 0 ? {} : { maxDiscoveryRuns: config.max_discovery_runs },
+			...config.max_time_hours === void 0 ? {} : { maxTimeHours: config.max_time_hours }
+		};
+	}
+	buildScanResponse(result, module, repository, operation, mode, config, observers) {
+		const cost = result.cost ?? observers.cost;
+		const tokenUsage = getTokenUsage(result, observers.cost);
+		const findings = Array.isArray(result.findings?.findings) ? result.findings.findings : [];
+		const model = result.turnResult.model ?? cost?.model ?? config.model;
+		return {
+			output: JSON.stringify(result.toJSON()),
+			format: "json",
+			raw: result.toJSON(),
+			cached: false,
+			sessionId: result.threadId,
+			...cost ? { cost: cost.estimatedUsd } : {},
+			...tokenUsage ? { tokenUsage } : {},
+			metadata: {
+				operation,
+				mode,
+				repository,
+				...model ? { model } : {},
+				...config.model_reasoning_effort || config.reasoning_effort ? { reasoningEffort: config.model_reasoning_effort ?? config.reasoning_effort } : {},
+				findingsCount: findings.length,
+				coverage: result.coverage,
+				scanDir: result.scanDir,
+				reportPath: result.reportPath,
+				findingsPath: result.findingsPath,
+				coveragePath: result.coveragePath,
+				...result.sarifPath ? { sarifPath: result.sarifPath } : {},
+				pluginVersion: result.pluginVersion,
+				sdkVersion: module.VERSION,
+				...observers.progress ? { progress: observers.progress } : {},
+				...observers.warnings.length > 0 ? { warnings: observers.warnings } : {},
+				skillCalls: [{ name: operation }]
+			}
+		};
+	}
+	async runValidation(client, prompt, repository, config, context, callOptions) {
+		const findingVariable = context?.vars?.finding;
+		const contextualFinding = typeof findingVariable === "string" || findingVariable !== null && typeof findingVariable === "object" && !Array.isArray(findingVariable) ? findingVariable : void 0;
+		let finding = config.finding ?? contextualFinding ?? prompt;
+		if (config.finding_file) {
+			const findingContents = await fs.readFile(resolveConfigPath(config.finding_file, config.basePath), "utf8");
+			try {
+				finding = JSON.parse(findingContents);
+			} catch {
+				finding = findingContents;
+			}
+		}
+		const options = {
+			repositoryPath: repository,
+			finding,
+			...config.auth ? { auth: config.auth } : {},
+			...config.output_dir ? { outputDir: resolveConfigPath(config.output_dir, config.basePath) } : {},
+			...callOptions?.abortSignal ? { signal: callOptions.abortSignal } : {}
+		};
+		const result = await client.validate(options);
+		return {
+			output: JSON.stringify(result),
+			format: "json",
+			raw: result,
+			cached: false,
+			...result.threadId ? { sessionId: result.threadId } : {},
+			metadata: {
+				operation: "validation",
+				repository,
+				disposition: result.disposition,
+				outputDir: result.outputDir,
+				...config.model ? { model: config.model } : {},
+				skillCalls: [{ name: "validation" }]
+			}
+		};
+	}
+};
+//#endregion
+export { OpenAICodexSecurityProvider };
+
+//# sourceMappingURL=codex-security-DSYUnVdR.js.map