promptfoo 0.125.0 → 0.125.1

package/dist/src/remoteGeneration-g3KvW-sP.js

@@ -0,0 +1,371 @@
+#!/usr/bin/env node
+import { J as getEnvBool, Q as getEnvString, c as logger, lt as state } from "./logger-pCNnmVuz.js";
+import { l as isLoggedIntoCloud } from "./accounts-BOxNYxVN.js";
+import { n as CloudConfig } from "./cloud-CVSUEmAy.js";
+import { t as getPackageVersion } from "./packageVersion-CVhzH1IZ.js";
+import { n as resolveProviderApiKey } from "./credentials-DPd2q2su.js";
+import { t as providerRegistry } from "./providerRegistry-wn1xUT6S.js";
+import { n as OpenAICodexSDKProvider, r as resolveCodexSdkPackage } from "./codex-sdk-C2ZLxa6r.js";
+import fs from "fs";
+import path from "path";
+import os from "os";
+import { context, propagation } from "@opentelemetry/api";
+import { createHmac, randomBytes } from "node:crypto";
+import semverSatisfies from "semver/functions/satisfies.js";
+//#region src/providers/openai/codexDefaults.ts
+const CODEX_AUTH_FILENAME = "auth.json";
+const CODEX_DEFAULT_PROVIDERS_CACHE_EVICTION_GRACE_MS = 6e4;
+const CODEX_DEFAULT_PROVIDERS_CACHE_MAX_ENTRIES = 32;
+const CODEX_DEFAULT_PROVIDERS_CACHE_HMAC_CONTEXT = "promptfoo:codex-default-provider-cache-key";
+const CODEX_DEFAULT_PROVIDERS_CACHE_HMAC_KEY = randomBytes(32);
+let codexDefaultWorkingDir;
+const CODEX_GRADING_OUTPUT_SCHEMA = {
+	type: "object",
+	properties: {
+		pass: { type: "boolean" },
+		score: { type: "number" },
+		reason: { type: "string" }
+	},
+	required: [
+		"pass",
+		"score",
+		"reason"
+	],
+	additionalProperties: false
+};
+const codexDefaultProvidersByCacheKey = /* @__PURE__ */ new Map();
+const evictedCodexDefaultProviderBundles = /* @__PURE__ */ new Set();
+const codexSdkAvailabilityByEntryPoint = /* @__PURE__ */ new Map();
+function getCodexEnvString(env, key) {
+	return env?.[key] ?? getEnvString(key);
+}
+function getCodexHome(env) {
+	const homeDir = os.homedir?.();
+	const defaultHome = typeof homeDir === "string" && homeDir ? path.join(homeDir, ".codex") : void 0;
+	const codexHome = getCodexEnvString(env, "CODEX_HOME") || defaultHome || ".codex";
+	return path.resolve(codexHome);
+}
+function getTempDirectory() {
+	const tempDir = os.tmpdir?.();
+	return typeof tempDir === "string" && tempDir ? tempDir : process.cwd();
+}
+function getCodexDefaultWorkingDir() {
+	if (!codexDefaultWorkingDir) codexDefaultWorkingDir = fs.mkdtempSync(path.join(getTempDirectory(), "promptfoo-codex-default-"));
+	return codexDefaultWorkingDir;
+}
+function hasCodexAuthFile(env) {
+	const authPath = path.join(getCodexHome(env), CODEX_AUTH_FILENAME);
+	try {
+		const stats = fs.statSync(authPath);
+		return stats.isFile() && stats.size > 0;
+	} catch (err) {
+		const code = err?.code;
+		if (code !== "ENOENT" && code !== "ENOTDIR") logger.debug("[CodexDefaults] Could not stat Codex auth file", {
+			authPath,
+			code,
+			err
+		});
+		return false;
+	}
+}
+function canLoadCodexSdkPackage() {
+	const entryPoint = resolveCodexSdkPackage();
+	if (!entryPoint) return false;
+	const cached = codexSdkAvailabilityByEntryPoint.get(entryPoint);
+	if (cached !== void 0) return cached;
+	let compatible = false;
+	try {
+		const version = getPackageVersion("@openai/codex-sdk", entryPoint);
+		compatible = Boolean(version && semverSatisfies(version, "^0.156.1"));
+	} catch (error) {
+		logger.debug("[CodexDefaults] Could not read SDK metadata", { error });
+	}
+	codexSdkAvailabilityByEntryPoint.set(entryPoint, compatible);
+	return compatible;
+}
+function hasCodexDefaultCredentials(env) {
+	return (Boolean(resolveProviderApiKey(void 0, env, ["OPENAI_API_KEY", "CODEX_API_KEY"])) || hasCodexAuthFile(env)) && canLoadCodexSdkPackage();
+}
+function getCodexDefaultProviderConfig(env, config, defaultWorkingDir = getCodexDefaultWorkingDir()) {
+	const codexHome = getCodexEnvString(env, "CODEX_HOME");
+	const workingDir = config?.working_dir || defaultWorkingDir;
+	fs.mkdirSync(workingDir, { recursive: true });
+	const cliEnv = {
+		...codexHome ? { CODEX_HOME: path.resolve(codexHome) } : {},
+		...config?.cli_env
+	};
+	return {
+		approval_policy: "never",
+		sandbox_mode: "read-only",
+		skip_git_repo_check: true,
+		working_dir: workingDir,
+		...config,
+		...Object.keys(cliEnv).length > 0 ? { cli_env: cliEnv } : {}
+	};
+}
+function getSecretCacheFingerprint(value) {
+	return value ? createHmac("sha256", CODEX_DEFAULT_PROVIDERS_CACHE_HMAC_KEY).update(CODEX_DEFAULT_PROVIDERS_CACHE_HMAC_CONTEXT).update("\0").update(value).digest("hex") : void 0;
+}
+function getCodexDefaultProvidersCacheKey(env) {
+	const resolvedApiKey = resolveProviderApiKey(void 0, env, ["OPENAI_API_KEY", "CODEX_API_KEY"]);
+	return JSON.stringify({
+		CODEX_HOME: getCodexEnvString(env, "CODEX_HOME"),
+		hasResolvedApiKey: Boolean(resolvedApiKey),
+		resolvedApiKeyFingerprint: getSecretCacheFingerprint(resolvedApiKey)
+	});
+}
+function getUniqueCodexDefaultProviders(providers) {
+	return Array.from(/* @__PURE__ */ new Set([
+		providers.gradingJsonProvider,
+		providers.gradingProvider,
+		providers.llmRubricProvider,
+		providers.suggestionsProvider,
+		providers.synthesizeProvider,
+		providers.webSearchProvider
+	]));
+}
+function canActOnShutdown(providers) {
+	return !providers.cancelled && providers.shutdownRequested && providers.activeCalls === 0 && !providers.shutdownPromise;
+}
+function shutdownCodexDefaultProviderBundle(providers) {
+	if (!canActOnShutdown(providers)) return;
+	providers.shutdownPromise = Promise.all(getUniqueCodexDefaultProviders(providers).map((provider) => provider.shutdown().catch((error) => {
+		logger.warn("[CodexDefaults] Error shutting down evicted provider", { error });
+	}))).then(() => void 0).finally(() => {
+		evictedCodexDefaultProviderBundles.delete(providers);
+	});
+}
+function clearCodexDefaultProviderShutdownTimer(providers) {
+	if (providers.shutdownTimer) {
+		clearTimeout(providers.shutdownTimer);
+		providers.shutdownTimer = void 0;
+	}
+}
+function scheduleCodexDefaultProviderBundleShutdown(providers) {
+	if (!canActOnShutdown(providers) || providers.shutdownTimer) return;
+	providers.shutdownTimer = setTimeout(() => {
+		providers.shutdownTimer = void 0;
+		shutdownCodexDefaultProviderBundle(providers);
+	}, CODEX_DEFAULT_PROVIDERS_CACHE_EVICTION_GRACE_MS);
+	providers.shutdownTimer.unref?.();
+}
+function requestCodexDefaultProviderBundleShutdown(providers) {
+	if (providers.cancelled) return;
+	providers.shutdownRequested = true;
+	evictedCodexDefaultProviderBundles.add(providers);
+	scheduleCodexDefaultProviderBundleShutdown(providers);
+}
+async function resurrectCodexDefaultProviderBundle(providers) {
+	await providers.shutdownPromise;
+	if (providers.cancelled) return;
+	for (const provider of getUniqueCodexDefaultProviders(providers)) providerRegistry.register(provider);
+	providers.shutdownPromise = void 0;
+	evictedCodexDefaultProviderBundles.add(providers);
+}
+function trackCodexDefaultProviderUsage(providers) {
+	for (const provider of getUniqueCodexDefaultProviders(providers)) {
+		const callApi = provider.callApi.bind(provider);
+		provider.callApi = async (...args) => {
+			if (providers.shutdownPromise !== void 0) await resurrectCodexDefaultProviderBundle(providers);
+			clearCodexDefaultProviderShutdownTimer(providers);
+			providers.activeCalls += 1;
+			try {
+				return await callApi(...args);
+			} finally {
+				providers.activeCalls -= 1;
+				scheduleCodexDefaultProviderBundleShutdown(providers);
+			}
+		};
+	}
+}
+function cacheCodexDefaultProviders(cacheKey, providers) {
+	codexDefaultProvidersByCacheKey.set(cacheKey, providers);
+	while (codexDefaultProvidersByCacheKey.size > CODEX_DEFAULT_PROVIDERS_CACHE_MAX_ENTRIES) {
+		const oldestEntry = codexDefaultProvidersByCacheKey.entries().next().value;
+		if (!oldestEntry) break;
+		const [oldestCacheKey, oldestProviders] = oldestEntry;
+		codexDefaultProvidersByCacheKey.delete(oldestCacheKey);
+		requestCodexDefaultProviderBundleShutdown(oldestProviders);
+	}
+	return providers;
+}
+function getCodexDefaultProviders(env) {
+	const cacheKey = getCodexDefaultProvidersCacheKey(env);
+	const cachedProviders = codexDefaultProvidersByCacheKey.get(cacheKey);
+	if (cachedProviders) {
+		codexDefaultProvidersByCacheKey.delete(cacheKey);
+		codexDefaultProvidersByCacheKey.set(cacheKey, cachedProviders);
+		return cachedProviders;
+	}
+	const defaultWorkingDir = getCodexDefaultWorkingDir();
+	const gradingProvider = new OpenAICodexSDKProvider({
+		config: getCodexDefaultProviderConfig(env, void 0, defaultWorkingDir),
+		env
+	});
+	const gradingJsonProvider = new OpenAICodexSDKProvider({
+		config: getCodexDefaultProviderConfig(env, { output_schema: CODEX_GRADING_OUTPUT_SCHEMA }, defaultWorkingDir),
+		env
+	});
+	const providers = {
+		activeCalls: 0,
+		cancelled: false,
+		gradingJsonProvider,
+		gradingProvider,
+		llmRubricProvider: gradingJsonProvider,
+		shutdownRequested: false,
+		suggestionsProvider: gradingProvider,
+		synthesizeProvider: gradingProvider,
+		webSearchProvider: new OpenAICodexSDKProvider({
+			config: getCodexDefaultProviderConfig(env, {
+				network_access_enabled: true,
+				output_schema: CODEX_GRADING_OUTPUT_SCHEMA,
+				web_search_mode: "live"
+			}, defaultWorkingDir),
+			env
+		})
+	};
+	trackCodexDefaultProviderUsage(providers);
+	return cacheCodexDefaultProviders(cacheKey, providers);
+}
+//#endregion
+//#region src/redteam/remoteGenerationContext.ts
+function isRecord(value) {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+function remoteGenerationContextPayload(contextOrCloudTargetId) {
+	const cloudTargetId = typeof contextOrCloudTargetId === "string" ? contextOrCloudTargetId : isRecord(contextOrCloudTargetId) && typeof contextOrCloudTargetId.cloudTargetId === "string" ? contextOrCloudTargetId.cloudTargetId : void 0;
+	return cloudTargetId ? { targetId: cloudTargetId } : {};
+}
+function getCloudTargetIdFromProviderId(providerId) {
+	return typeof providerId === "string" && providerId.startsWith("promptfoo://provider/") ? providerId.slice(21) || void 0 : void 0;
+}
+function getCloudTargetIdFromTargetIds(targetIds) {
+	for (const targetId of targetIds) {
+		const cloudTargetId = getCloudTargetIdFromProviderId(targetId);
+		if (cloudTargetId) return cloudTargetId;
+	}
+}
+function resolveRedteamGenerationContext({ cloudTargetDatabaseId, redteamGenerationContext, targetIds }) {
+	const providerTargetIds = redteamGenerationContext?.providerTargetIds ?? targetIds ?? [];
+	return {
+		providerTargetIds,
+		cloudTargetId: redteamGenerationContext?.cloudTargetId ?? cloudTargetDatabaseId ?? getCloudTargetIdFromTargetIds(providerTargetIds)
+	};
+}
+//#endregion
+//#region src/redteam/remoteGeneration.ts
+function providerRemoteGenerationContextPayload(contextOrCloudTargetId) {
+	return remoteGenerationContextPayload(contextOrCloudTargetId);
+}
+function resolveRemoteEndpoint(envName, path) {
+	const envUrl = getEnvString(envName);
+	if (envUrl) return envUrl;
+	const cloudConfig = new CloudConfig();
+	if (cloudConfig.isEnabled()) return cloudConfig.getApiHost() + path;
+	return "https://api.promptfoo.app" + path;
+}
+/**
+* Gets the remote generation API endpoint URL.
+* Prioritizes: env var > cloud config > default endpoint.
+* @returns The remote generation URL
+*/
+function getRemoteGenerationUrl() {
+	return resolveRemoteEndpoint("PROMPTFOO_REMOTE_GENERATION_URL", "/api/v1/task");
+}
+/**
+* Builds headers for a remote-generation request.
+*
+* Authentication is injected centrally at the fetch layer (monkeyPatchFetch, mirrored
+* in cache.ts for cache-key parity) and only when the request URL's origin matches the
+* configured Promptfoo Cloud host (cloudConfig.getApiHost(), incl. on-prem). Keeping
+* this helper auth-free prevents cloud credentials from leaking to custom
+* remote-generation endpoints.
+*/
+function getRemoteGenerationHeaders(extraHeaders) {
+	const propagatedHeaders = {};
+	propagation.inject(context.active(), propagatedHeaders);
+	return {
+		"Content-Type": "application/json",
+		...propagatedHeaders.traceparent ? { traceparent: propagatedHeaders.traceparent } : {},
+		...propagatedHeaders.tracestate ? { tracestate: propagatedHeaders.tracestate } : {},
+		...extraHeaders
+	};
+}
+/**
+* Check if remote generation should never be used.
+* Respects both the general and redteam-specific disable flags.
+* @returns true if remote generation is disabled
+*/
+function neverGenerateRemote() {
+	if (getEnvBool("PROMPTFOO_DISABLE_REMOTE_GENERATION")) return true;
+	return getEnvBool("PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION");
+}
+/**
+* Check if remote generation should never be used for non-redteam features.
+* This allows granular control: disable redteam remote generation while allowing
+* regular SimulatedUser to use remote generation.
+* @returns true if ALL remote generation is disabled
+*/
+function neverGenerateRemoteForRegularEvals() {
+	return getEnvBool("PROMPTFOO_DISABLE_REMOTE_GENERATION");
+}
+/**
+* Builds a remote URL with a substituted pathname, honoring env vars / cloud config.
+*/
+function buildRemoteUrl(pathname, fallback) {
+	if (neverGenerateRemote()) return null;
+	const envUrl = getEnvString("PROMPTFOO_REMOTE_GENERATION_URL");
+	if (envUrl) try {
+		const url = new URL(envUrl);
+		url.pathname = pathname;
+		return url.toString();
+	} catch {
+		return fallback;
+	}
+	const cloudConfig = new CloudConfig();
+	if (cloudConfig.isEnabled()) return `${cloudConfig.getApiHost()}${pathname}`;
+	return fallback;
+}
+/**
+* Gets the URL for checking remote API health based on configuration.
+* @returns The health check URL, or null if remote generation is disabled.
+*/
+function getRemoteHealthUrl() {
+	return buildRemoteUrl("/health", "https://api.promptfoo.app/health");
+}
+/**
+* Gets the URL for checking remote API version based on configuration.
+* @returns The version check URL, or null if remote generation is disabled.
+*/
+function getRemoteVersionUrl() {
+	return buildRemoteUrl("/version", "https://api.promptfoo.app/version");
+}
+function getRemoteGenerationDisabledError(strategyName) {
+	return `${strategyName} requires remote generation, which is currently disabled for this configuration. To enable it, run with --remote, set PROMPTFOO_REMOTE_GENERATION_URL to a self-hosted endpoint, or log into Promptfoo Cloud with \`promptfoo auth login\`.`;
+}
+function getRemoteGenerationExplicitlyDisabledError(strategyName) {
+	const activeFlags = ["PROMPTFOO_DISABLE_REMOTE_GENERATION", "PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION"].filter((flag) => getEnvBool(flag));
+	return `${strategyName} requires remote generation, which has been explicitly disabled. To enable it, ${activeFlags.length > 0 ? `unset ${activeFlags.join(" and ")}` : "unset PROMPTFOO_DISABLE_REMOTE_GENERATION or PROMPTFOO_DISABLE_REDTEAM_REMOTE_GENERATION (whichever is set)"}. Once re-enabled, you can point at a self-hosted endpoint with PROMPTFOO_REMOTE_GENERATION_URL or use Promptfoo Cloud via \`promptfoo auth login\`.`;
+}
+/**
+* Determines if remote generation should be used based on configuration.
+* @returns true if remote generation should be used
+*/
+function shouldGenerateRemote(options) {
+	if (neverGenerateRemote()) return false;
+	if (getEnvString("PROMPTFOO_REMOTE_GENERATION_URL")) return true;
+	if (isLoggedIntoCloud()) return true;
+	return !(Boolean(getEnvString("OPENAI_API_KEY")) || options?.canUseCodexDefaultProvider && !options?.requireEmbeddingProvider && hasCodexDefaultCredentials()) || (state.remote ?? false);
+}
+/**
+* Gets the URL for unaligned model inference (harmful content generation).
+* Prioritizes: env var > cloud config > default endpoint.
+* @returns The unaligned inference URL
+*/
+function getRemoteGenerationUrlForUnaligned() {
+	return resolveRemoteEndpoint("PROMPTFOO_UNALIGNED_INFERENCE_ENDPOINT", "/api/v1/task/harmful");
+}
+//#endregion
+export { getCodexDefaultProviders as _, getRemoteGenerationUrlForUnaligned as a, neverGenerateRemote as c, shouldGenerateRemote as d, getCloudTargetIdFromProviderId as f, resolveRedteamGenerationContext as g, remoteGenerationContextPayload as h, getRemoteGenerationUrl as i, neverGenerateRemoteForRegularEvals as l, isRecord as m, getRemoteGenerationExplicitlyDisabledError as n, getRemoteHealthUrl as o, getCloudTargetIdFromTargetIds as p, getRemoteGenerationHeaders as r, getRemoteVersionUrl as s, getRemoteGenerationDisabledError as t, providerRemoteGenerationContextPayload as u, hasCodexDefaultCredentials as v };
+
+//# sourceMappingURL=remoteGeneration-g3KvW-sP.js.map