promptfoo 0.121.4 → 0.121.5

package/dist/src/{types-BGQDAP8i.js → types-DNRZVOue.js}

@@ -1,133 +1,6 @@
-import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-DnqA1y9x.js";
+import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-8CjoL7vB.js";
 import dedent from "dedent";
 import { z } from "zod";
-//#region src/types/env.ts
-const ProviderEnvOverridesSchema = z.object({
-	AI21_API_BASE_URL: z.string().optional(),
-	AI21_API_KEY: z.string().optional(),
-	AIML_API_KEY: z.string().optional(),
-	ANTHROPIC_API_KEY: z.string().optional(),
-	ANTHROPIC_BASE_URL: z.string().optional(),
-	AWS_BEDROCK_REGION: z.string().optional(),
-	AZURE_API_BASE_URL: z.string().optional(),
-	AZURE_API_HOST: z.string().optional(),
-	AZURE_API_KEY: z.string().optional(),
-	AZURE_AUTHORITY_HOST: z.string().optional(),
-	AZURE_CLIENT_ID: z.string().optional(),
-	AZURE_CLIENT_SECRET: z.string().optional(),
-	AZURE_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_OPENAI_API_BASE_URL: z.string().optional(),
-	AZURE_OPENAI_API_HOST: z.string().optional(),
-	AZURE_OPENAI_API_KEY: z.string().optional(),
-	AZURE_OPENAI_BASE_URL: z.string().optional(),
-	AZURE_OPENAI_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
-	AZURE_TENANT_ID: z.string().optional(),
-	AZURE_TOKEN_SCOPE: z.string().optional(),
-	CLAUDE_CODE_USE_BEDROCK: z.string().optional(),
-	CLAUDE_CODE_USE_VERTEX: z.string().optional(),
-	CLOUDFLARE_ACCOUNT_ID: z.string().optional(),
-	CLOUDFLARE_API_KEY: z.string().optional(),
-	CLOUDFLARE_GATEWAY_ID: z.string().optional(),
-	CF_AIG_TOKEN: z.string().optional(),
-	COMETAPI_KEY: z.string().optional(),
-	COHERE_API_KEY: z.string().optional(),
-	COHERE_CLIENT_NAME: z.string().optional(),
-	DATABRICKS_TOKEN: z.string().optional(),
-	DATABRICKS_WORKSPACE_URL: z.string().optional(),
-	DOCKER_MODEL_RUNNER_BASE_URL: z.string().optional(),
-	DOCKER_MODEL_RUNNER_API_KEY: z.string().optional(),
-	ELEVENLABS_API_KEY: z.string().optional(),
-	FAL_KEY: z.string().optional(),
-	GITHUB_TOKEN: z.string().optional(),
-	GOOGLE_API_HOST: z.string().optional(),
-	GOOGLE_API_BASE_URL: z.string().optional(),
-	GOOGLE_API_KEY: z.string().optional(),
-	GOOGLE_PROJECT_ID: z.string().optional(),
-	GOOGLE_LOCATION: z.string().optional(),
-	GOOGLE_GENERATIVE_AI_API_KEY: z.string().optional(),
-	GEMINI_API_KEY: z.string().optional(),
-	GROQ_API_KEY: z.string().optional(),
-	HELICONE_API_KEY: z.string().optional(),
-	HF_API_TOKEN: z.string().optional(),
-	HF_TOKEN: z.string().optional(),
-	HYPERBOLIC_API_KEY: z.string().optional(),
-	HUGGING_FACE_HUB_TOKEN: z.string().optional(),
-	JFROG_API_KEY: z.string().optional(),
-	LANGFUSE_HOST: z.string().optional(),
-	LANGFUSE_PUBLIC_KEY: z.string().optional(),
-	LANGFUSE_SECRET_KEY: z.string().optional(),
-	LITELLM_API_BASE: z.string().optional(),
-	LLAMA_BASE_URL: z.string().optional(),
-	LOCALAI_BASE_URL: z.string().optional(),
-	MISTRAL_API_BASE_URL: z.string().optional(),
-	MISTRAL_API_HOST: z.string().optional(),
-	MISTRAL_API_KEY: z.string().optional(),
-	MODELSLAB_API_KEY: z.string().optional(),
-	NSCALE_SERVICE_TOKEN: z.string().optional(),
-	NSCALE_API_KEY: z.string().optional(),
-	OLLAMA_API_KEY: z.string().optional(),
-	OLLAMA_BASE_URL: z.string().optional(),
-	OPENAI_API_BASE_URL: z.string().optional(),
-	OPENAI_API_HOST: z.string().optional(),
-	OPENAI_API_KEY: z.string().optional(),
-	OPENAI_BASE_URL: z.string().optional(),
-	OPENAI_ORGANIZATION: z.string().optional(),
-	CLAWDBOT_GATEWAY_PASSWORD: z.string().optional(),
-	CLAWDBOT_GATEWAY_TOKEN: z.string().optional(),
-	CLAWDBOT_GATEWAY_URL: z.string().optional(),
-	CODEX_API_KEY: z.string().optional(),
-	OPENCLAW_CONFIG_PATH: z.string().optional(),
-	OPENCLAW_GATEWAY_PASSWORD: z.string().optional(),
-	OPENCLAW_GATEWAY_PORT: z.string().optional(),
-	OPENCLAW_GATEWAY_TOKEN: z.string().optional(),
-	OPENCLAW_GATEWAY_URL: z.string().optional(),
-	PALM_API_HOST: z.string().optional(),
-	PALM_API_KEY: z.string().optional(),
-	PORTKEY_API_KEY: z.string().optional(),
-	PROMPTFOO_CA_CERT_PATH: z.string().optional(),
-	PROMPTFOO_PFX_CERT_PATH: z.string().optional(),
-	PROMPTFOO_PFX_PASSWORD: z.string().optional(),
-	PROMPTFOO_JKS_CERT_PATH: z.string().optional(),
-	PROMPTFOO_JKS_PASSWORD: z.string().optional(),
-	PROMPTFOO_JKS_ALIAS: z.string().optional(),
-	PROMPTFOO_INSECURE_SSL: z.string().optional(),
-	QUIVERAI_API_KEY: z.string().optional(),
-	REPLICATE_API_KEY: z.string().optional(),
-	REPLICATE_API_TOKEN: z.string().optional(),
-	SHAREPOINT_BASE_URL: z.string().optional(),
-	SHAREPOINT_CERT_PATH: z.string().optional(),
-	SHAREPOINT_CLIENT_ID: z.string().optional(),
-	SHAREPOINT_TENANT_ID: z.string().optional(),
-	VERCEL_AI_GATEWAY_API_KEY: z.string().optional(),
-	VERCEL_AI_GATEWAY_BASE_URL: z.string().optional(),
-	VERTEX_API_HOST: z.string().optional(),
-	VERTEX_API_KEY: z.string().optional(),
-	VERTEX_API_VERSION: z.string().optional(),
-	VERTEX_PROJECT_ID: z.string().optional(),
-	VERTEX_PUBLISHER: z.string().optional(),
-	VERTEX_REGION: z.string().optional(),
-	VOYAGE_API_BASE_URL: z.string().optional(),
-	VOYAGE_API_KEY: z.string().optional(),
-	WATSONX_AI_APIKEY: z.string().optional(),
-	WATSONX_AI_AUTH_TYPE: z.string().optional(),
-	WATSONX_AI_BEARER_TOKEN: z.string().optional(),
-	WATSONX_AI_PROJECT_ID: z.string().optional(),
-	XAI_API_BASE_URL: z.string().optional(),
-	XAI_API_KEY: z.string().optional(),
-	AZURE_CONTENT_SAFETY_ENDPOINT: z.string().optional(),
-	AZURE_CONTENT_SAFETY_API_KEY: z.string().optional(),
-	AZURE_CONTENT_SAFETY_API_VERSION: z.string().optional(),
-	AWS_REGION: z.string().optional(),
-	AWS_DEFAULT_REGION: z.string().optional(),
-	AWS_SAGEMAKER_MAX_TOKENS: z.string().optional(),
-	AWS_SAGEMAKER_TEMPERATURE: z.string().optional(),
-	AWS_SAGEMAKER_TOP_P: z.string().optional(),
-	AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
-	PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
-});
-//#endregion
 //#region src/types/shared.ts
 const CompletionTokenDetailsSchema = z.object({
 	reasoning: z.number().optional(),
@@ -157,26 +30,6 @@ const BaseTokenUsageSchema = z.object({
 });
 const InputsSchema = z.record(z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" }), z.string().min(1, { error: "Input descriptions must be non-empty strings" }));
 //#endregion
-//#region src/validators/prompts.ts
-const PromptConfigSchema = z.object({
-	prefix: z.string().optional(),
-	suffix: z.string().optional()
-});
-const PromptFunctionSchema = z.custom((v) => typeof v === "function");
-const PromptSchema = z.object({
-	id: z.string().optional(),
-	raw: z.string(),
-	template: z.string().optional(),
-	display: z.string().optional(),
-	label: z.string(),
-	function: PromptFunctionSchema.optional(),
-	config: z.any().optional()
-});
-function assert$1() {}
-assert$1();
-assert$1();
-assert$1();
-//#endregion
 //#region src/redteam/constants/codingAgents.ts
 const CODING_AGENT_CORE_PLUGINS = [
 	"coding-agent:repo-prompt-injection",
@@ -2797,6 +2650,135 @@ var PartialGenerationError = class extends Error {
 	}
 };
 //#endregion
+//#region src/types/env.ts
+const ProviderEnvOverridesSchema = z.object({
+	ABLIT_API_BASE_URL: z.string().optional(),
+	ABLIT_KEY: z.string().optional(),
+	AI21_API_BASE_URL: z.string().optional(),
+	AI21_API_KEY: z.string().optional(),
+	AIML_API_KEY: z.string().optional(),
+	ANTHROPIC_API_KEY: z.string().optional(),
+	ANTHROPIC_BASE_URL: z.string().optional(),
+	AWS_BEDROCK_REGION: z.string().optional(),
+	AZURE_API_BASE_URL: z.string().optional(),
+	AZURE_API_HOST: z.string().optional(),
+	AZURE_API_KEY: z.string().optional(),
+	AZURE_AUTHORITY_HOST: z.string().optional(),
+	AZURE_CLIENT_ID: z.string().optional(),
+	AZURE_CLIENT_SECRET: z.string().optional(),
+	AZURE_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_OPENAI_API_BASE_URL: z.string().optional(),
+	AZURE_OPENAI_API_HOST: z.string().optional(),
+	AZURE_OPENAI_API_KEY: z.string().optional(),
+	AZURE_OPENAI_BASE_URL: z.string().optional(),
+	AZURE_OPENAI_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: z.string().optional(),
+	AZURE_TENANT_ID: z.string().optional(),
+	AZURE_TOKEN_SCOPE: z.string().optional(),
+	CLAUDE_CODE_USE_BEDROCK: z.string().optional(),
+	CLAUDE_CODE_USE_VERTEX: z.string().optional(),
+	CLOUDFLARE_ACCOUNT_ID: z.string().optional(),
+	CLOUDFLARE_API_KEY: z.string().optional(),
+	CLOUDFLARE_GATEWAY_ID: z.string().optional(),
+	CF_AIG_TOKEN: z.string().optional(),
+	COMETAPI_KEY: z.string().optional(),
+	COHERE_API_KEY: z.string().optional(),
+	COHERE_CLIENT_NAME: z.string().optional(),
+	DATABRICKS_TOKEN: z.string().optional(),
+	DATABRICKS_WORKSPACE_URL: z.string().optional(),
+	DOCKER_MODEL_RUNNER_BASE_URL: z.string().optional(),
+	DOCKER_MODEL_RUNNER_API_KEY: z.string().optional(),
+	ELEVENLABS_API_KEY: z.string().optional(),
+	FAL_KEY: z.string().optional(),
+	GITHUB_TOKEN: z.string().optional(),
+	GOOGLE_API_HOST: z.string().optional(),
+	GOOGLE_API_BASE_URL: z.string().optional(),
+	GOOGLE_API_KEY: z.string().optional(),
+	GOOGLE_PROJECT_ID: z.string().optional(),
+	GOOGLE_LOCATION: z.string().optional(),
+	GOOGLE_GENERATIVE_AI_API_KEY: z.string().optional(),
+	GEMINI_API_KEY: z.string().optional(),
+	GROQ_API_KEY: z.string().optional(),
+	HELICONE_API_KEY: z.string().optional(),
+	HF_API_TOKEN: z.string().optional(),
+	HF_TOKEN: z.string().optional(),
+	HYPERBOLIC_API_KEY: z.string().optional(),
+	HUGGING_FACE_HUB_TOKEN: z.string().optional(),
+	JFROG_API_KEY: z.string().optional(),
+	LANGFUSE_HOST: z.string().optional(),
+	LANGFUSE_PUBLIC_KEY: z.string().optional(),
+	LANGFUSE_SECRET_KEY: z.string().optional(),
+	LITELLM_API_BASE: z.string().optional(),
+	LLAMA_BASE_URL: z.string().optional(),
+	LOCALAI_BASE_URL: z.string().optional(),
+	MISTRAL_API_BASE_URL: z.string().optional(),
+	MISTRAL_API_HOST: z.string().optional(),
+	MISTRAL_API_KEY: z.string().optional(),
+	MODELSLAB_API_KEY: z.string().optional(),
+	NSCALE_SERVICE_TOKEN: z.string().optional(),
+	NSCALE_API_KEY: z.string().optional(),
+	OLLAMA_API_KEY: z.string().optional(),
+	OLLAMA_BASE_URL: z.string().optional(),
+	OPENAI_API_BASE_URL: z.string().optional(),
+	OPENAI_API_HOST: z.string().optional(),
+	OPENAI_API_KEY: z.string().optional(),
+	OPENAI_BASE_URL: z.string().optional(),
+	OPENAI_ORGANIZATION: z.string().optional(),
+	CLAWDBOT_GATEWAY_PASSWORD: z.string().optional(),
+	CLAWDBOT_GATEWAY_TOKEN: z.string().optional(),
+	CLAWDBOT_GATEWAY_URL: z.string().optional(),
+	CODEX_API_KEY: z.string().optional(),
+	OPENCLAW_CONFIG_PATH: z.string().optional(),
+	OPENCLAW_GATEWAY_PASSWORD: z.string().optional(),
+	OPENCLAW_GATEWAY_PORT: z.string().optional(),
+	OPENCLAW_GATEWAY_TOKEN: z.string().optional(),
+	OPENCLAW_GATEWAY_URL: z.string().optional(),
+	PALM_API_HOST: z.string().optional(),
+	PALM_API_KEY: z.string().optional(),
+	PORTKEY_API_KEY: z.string().optional(),
+	PROMPTFOO_CA_CERT_PATH: z.string().optional(),
+	PROMPTFOO_PFX_CERT_PATH: z.string().optional(),
+	PROMPTFOO_PFX_PASSWORD: z.string().optional(),
+	PROMPTFOO_JKS_CERT_PATH: z.string().optional(),
+	PROMPTFOO_JKS_PASSWORD: z.string().optional(),
+	PROMPTFOO_JKS_ALIAS: z.string().optional(),
+	PROMPTFOO_INSECURE_SSL: z.string().optional(),
+	QUIVERAI_API_KEY: z.string().optional(),
+	REPLICATE_API_KEY: z.string().optional(),
+	REPLICATE_API_TOKEN: z.string().optional(),
+	SHAREPOINT_BASE_URL: z.string().optional(),
+	SHAREPOINT_CERT_PATH: z.string().optional(),
+	SHAREPOINT_CLIENT_ID: z.string().optional(),
+	SHAREPOINT_TENANT_ID: z.string().optional(),
+	VERCEL_AI_GATEWAY_API_KEY: z.string().optional(),
+	VERCEL_AI_GATEWAY_BASE_URL: z.string().optional(),
+	VERTEX_API_HOST: z.string().optional(),
+	VERTEX_API_KEY: z.string().optional(),
+	VERTEX_API_VERSION: z.string().optional(),
+	VERTEX_PROJECT_ID: z.string().optional(),
+	VERTEX_PUBLISHER: z.string().optional(),
+	VERTEX_REGION: z.string().optional(),
+	VOYAGE_API_BASE_URL: z.string().optional(),
+	VOYAGE_API_KEY: z.string().optional(),
+	WATSONX_AI_APIKEY: z.string().optional(),
+	WATSONX_AI_AUTH_TYPE: z.string().optional(),
+	WATSONX_AI_BEARER_TOKEN: z.string().optional(),
+	WATSONX_AI_PROJECT_ID: z.string().optional(),
+	XAI_API_BASE_URL: z.string().optional(),
+	XAI_API_KEY: z.string().optional(),
+	AZURE_CONTENT_SAFETY_ENDPOINT: z.string().optional(),
+	AZURE_CONTENT_SAFETY_API_KEY: z.string().optional(),
+	AZURE_CONTENT_SAFETY_API_VERSION: z.string().optional(),
+	AWS_REGION: z.string().optional(),
+	AWS_DEFAULT_REGION: z.string().optional(),
+	AWS_SAGEMAKER_MAX_TOKENS: z.string().optional(),
+	AWS_SAGEMAKER_TEMPERATURE: z.string().optional(),
+	AWS_SAGEMAKER_TOP_P: z.string().optional(),
+	AWS_SAGEMAKER_MAX_RETRIES: z.string().optional(),
+	PROMPTFOO_EVAL_TIMEOUT_MS: z.string().optional()
+});
+//#endregion
 //#region src/validators/providers.ts
 const ProviderOptionsSchema = z.object({
 	id: z.custom().optional(),
@@ -2859,6 +2841,26 @@ const ProviderSchema = z.union([
 	ProviderOptionsSchema
 ]);
 //#endregion
+//#region src/validators/prompts.ts
+const PromptConfigSchema = z.object({
+	prefix: z.string().optional(),
+	suffix: z.string().optional()
+});
+const PromptFunctionSchema = z.custom((v) => typeof v === "function");
+const PromptSchema = z.object({
+	id: z.string().optional(),
+	raw: z.string(),
+	template: z.string().optional(),
+	display: z.string().optional(),
+	label: z.string(),
+	function: PromptFunctionSchema.optional(),
+	config: z.any().optional()
+});
+function assert$1() {}
+assert$1();
+assert$1();
+assert$1();
+//#endregion
 //#region src/validators/redteam.ts
 const TracingConfigSchema = z.lazy(() => z.object({
 	enabled: z.boolean().optional(),
@@ -3613,6 +3615,6 @@ const EvalResultsFilterMode = z.enum([
 	"user-rated"
 ]);
 //#endregion
-export { pluginDescriptions as $, isApiProvider as A, CompletionTokenDetailsSchema as At, isValidReusablePolicyId as B, TestGeneratorConfigSchema as C, UNALIGNED_PROVIDER_HARM_PLUGINS as Ct, VarsSchema as D, CODING_AGENT_PLUGIN_DISPLAY_NAMES as Dt, UnifiedConfigSchema as E, CODING_AGENT_PLUGIN_DESCRIPTIONS as Et, ConversationMessageSchema as F, MULTI_TURN_STRATEGIES as G, AGENTIC_STRATEGIES as H, PartialGenerationError as I, getDefaultNFanout as J, STRATEGY_COLLECTIONS as K, PluginConfigSchema as L, RedteamConfigSchema as M, ProviderOptionsSchema as N, isGradingResult as O, PromptSchema as Ot, ProvidersSchema as P, categoryAliases as Q, PolicyObjectSchema as R, TestCasesWithMetadataSchema as S, TELECOM_PLUGINS as St, TestSuiteSchema as T, CODING_AGENT_PLUGINS as Tt, DATASET_PLUGINS as U, isUuid as V, DEFAULT_STRATEGIES as W, isFanoutStrategy as X, isCustomStrategy as Y, Severity as Z, ScenarioSchema as _, PLUGIN_CATEGORIES as _t, AtomicTestCaseSchema as a, DEFAULT_PLUGINS as at, TestCaseWithVarsFileSchema as b, STRATEGY_EXEMPT_PLUGINS as bt, CompletedPromptSchema as c, HARM_PLUGINS as ct, EvaluateOptionsSchema as d, LLAMA_GUARD_REPLICATE_PROVIDER as dt, riskCategorySeverityMap as et, GradingConfigSchema as f, MEDICAL_PLUGINS as ft, ResultFailureReason as g, PII_PLUGINS as gt, OutputFileExtension as h, PHARMACY_PLUGINS as ht, AssertionTypeSchema as i, DATASET_EXEMPT_PLUGINS as it, isProviderOptions as j, InputsSchema as jt, isResultFailureReason as k, BaseTokenUsageSchema as kt, DerivedMetricSchema as l, INSURANCE_PLUGINS as lt, OutputConfigSchema as m, MULTI_INPUT_VAR as mt, AssertionSchema as n, BIAS_PLUGINS as nt, BaseAssertionTypesSchema as o, FINANCIAL_PLUGINS as ot, NotPrefixedAssertionTypesSchema as p, MULTI_INPUT_EXCLUDED_PLUGINS as pt, STRATEGY_COLLECTION_MAPPINGS as q, AssertionSetSchema as r, CANARY_BREAKING_STRATEGY_IDS as rt, CommandLineOptionsSchema as s, FOUNDATION_PLUGINS as st, AssertionOrSetSchema as t, ALIASED_PLUGIN_MAPPINGS as tt, EvalResultsFilterMode as u, LLAMA_GUARD_ENABLED_CATEGORIES as ut, SpecialAssertionTypesSchema as v, REDTEAM_PROVIDER_HARM_PLUGINS as vt, TestSuiteConfigSchema as w, CODING_AGENT_CORE_PLUGINS as wt, TestCasesWithMetadataPromptSchema as x, TEEN_SAFETY_PLUGINS as xt, TestCaseSchema as y, REMOTE_ONLY_PLUGIN_IDS as yt, StrategyConfigSchema as z };
+export { categoryAliases as $, isApiProvider as A, CompletionTokenDetailsSchema as At, StrategyConfigSchema as B, TestGeneratorConfigSchema as C, TELECOM_PLUGINS as Ct, VarsSchema as D, CODING_AGENT_PLUGIN_DESCRIPTIONS as Dt, UnifiedConfigSchema as E, CODING_AGENT_PLUGINS as Et, ProvidersSchema as F, DEFAULT_STRATEGIES as G, isUuid as H, ConversationMessageSchema as I, STRATEGY_COLLECTION_MAPPINGS as J, MULTI_TURN_STRATEGIES as K, PartialGenerationError as L, RedteamConfigSchema as M, PromptSchema as N, isGradingResult as O, CODING_AGENT_PLUGIN_DISPLAY_NAMES as Ot, ProviderOptionsSchema as P, Severity as Q, PluginConfigSchema as R, TestCasesWithMetadataSchema as S, TEEN_SAFETY_PLUGINS as St, TestSuiteSchema as T, CODING_AGENT_CORE_PLUGINS as Tt, AGENTIC_STRATEGIES as U, isValidReusablePolicyId as V, DATASET_PLUGINS as W, isCustomStrategy as X, getDefaultNFanout as Y, isFanoutStrategy as Z, ScenarioSchema as _, PII_PLUGINS as _t, AtomicTestCaseSchema as a, DATASET_EXEMPT_PLUGINS as at, TestCaseWithVarsFileSchema as b, REMOTE_ONLY_PLUGIN_IDS as bt, CompletedPromptSchema as c, FOUNDATION_PLUGINS as ct, EvaluateOptionsSchema as d, LLAMA_GUARD_ENABLED_CATEGORIES as dt, pluginDescriptions as et, GradingConfigSchema as f, LLAMA_GUARD_REPLICATE_PROVIDER as ft, ResultFailureReason as g, PHARMACY_PLUGINS as gt, OutputFileExtension as h, MULTI_INPUT_VAR as ht, AssertionTypeSchema as i, CANARY_BREAKING_STRATEGY_IDS as it, isProviderOptions as j, InputsSchema as jt, isResultFailureReason as k, BaseTokenUsageSchema as kt, DerivedMetricSchema as l, HARM_PLUGINS as lt, OutputConfigSchema as m, MULTI_INPUT_EXCLUDED_PLUGINS as mt, AssertionSchema as n, ALIASED_PLUGIN_MAPPINGS as nt, BaseAssertionTypesSchema as o, DEFAULT_PLUGINS as ot, NotPrefixedAssertionTypesSchema as p, MEDICAL_PLUGINS as pt, STRATEGY_COLLECTIONS as q, AssertionSetSchema as r, BIAS_PLUGINS as rt, CommandLineOptionsSchema as s, FINANCIAL_PLUGINS as st, AssertionOrSetSchema as t, riskCategorySeverityMap as tt, EvalResultsFilterMode as u, INSURANCE_PLUGINS as ut, SpecialAssertionTypesSchema as v, PLUGIN_CATEGORIES as vt, TestSuiteConfigSchema as w, UNALIGNED_PROVIDER_HARM_PLUGINS as wt, TestCasesWithMetadataPromptSchema as x, STRATEGY_EXEMPT_PLUGINS as xt, TestCaseSchema as y, REDTEAM_PROVIDER_HARM_PLUGINS as yt, PolicyObjectSchema as z };
 
-//# sourceMappingURL=types-BGQDAP8i.js.map
+//# sourceMappingURL=types-DNRZVOue.js.map

package/dist/src/{util-C8e5uydV.js → util-3pBZZb_H.js}

@@ -1,13 +1,15 @@
-import { S as getEnvBool, a as logger, f as sanitizeObject, k as state } from "./logger-B88EkIn6.js";
-import { A as TERMINAL_MAX_WIDTH, I as VERSION, t as fetchWithProxy } from "./fetch-B0Z3Oe4k.js";
-import { t as invariant } from "./invariant-vgHWClmd.js";
-import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-Dh4dOLlt.js";
-import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-DHIZ6_k8.js";
-import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-Bgh5SOn6.js";
-import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-BArZuxsI.js";
-import { r as runPython } from "./pythonUtils-Cldx7huE.js";
+import { S as getEnvBool, a as logger, f as sanitizeObject, k as state } from "./logger-KD8JjCRJ.js";
+import { A as TERMINAL_MAX_WIDTH, t as fetchWithProxy } from "./fetch-Y5qX_kST.js";
+import { n as VERSION } from "./version-0frU0UTr.js";
+import { t as invariant } from "./invariant-DIYf9sP1.js";
+import { o as safeResolve, r as importModule, t as getDirectory } from "./esm-B6whoAcf.js";
+import { a as getNunjucksEngine, n as renderVarsInObject } from "./render-7uNJ2V14.js";
+import { m as isProviderOptions, o as OutputFileExtension, p as isApiProvider, s as ResultFailureReason } from "./types-BDjGOq4E.js";
+import { i as isJavascriptFile, t as JAVASCRIPT_EXTENSIONS } from "./fileExtensions-BGh-W-HT.js";
+import { r as runPython } from "./pythonUtils-C4tltmIn.js";
 import dotenv from "dotenv";
 import * as fs$2 from "fs";
+import fs from "fs";
 import * as path$1 from "path";
 import path from "path";
 import * as os$1 from "os";
@@ -383,11 +385,32 @@ async function maybeLoadToolsFromExternalFile(tools, vars) {
 	return loaded;
 }
 //#endregion
-//#region src/util/provider.ts
+//#region src/util/providerRef.ts
+const PROVIDER_OPTION_KEYS = new Set([
+	"id",
+	"label",
+	"config",
+	"prompts",
+	"transform",
+	"delay",
+	"env",
+	"inputs"
+]);
+/** Returns true if the value is a non-empty string suitable as a provider identifier. */
+function isValidProviderId(id) {
+	return typeof id === "string" && id !== "";
+}
+function getProviderLabel(provider) {
+	if ((typeof provider === "object" || typeof provider === "function") && provider !== null && "label" in provider && typeof provider.label === "string") return provider.label;
+}
+/**
+* Resolves relative file paths in provider IDs to absolute paths for consistent matching.
+* Handles file://, exec:, python:, golang: prefixes and bare .js/.ts/.mjs paths.
+*/
 function canonicalizeProviderId(id) {
 	if (id.startsWith("file://")) {
 		const filePath = id.slice(7);
-		return path$1.isAbsolute(filePath) ? id : `file://${path$1.resolve(filePath)}`;
+		return path.isAbsolute(filePath) ? id : `file://${path.resolve(filePath)}`;
 	}
 	for (const prefix of [
 		"exec:",
@@ -395,19 +418,121 @@ function canonicalizeProviderId(id) {
 		"golang:"
 	]) if (id.startsWith(prefix)) {
 		const filePath = id.slice(prefix.length);
-		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path$1.resolve(filePath)}`;
+		if (filePath.includes("/") || filePath.includes("\\")) return `${prefix}${path.resolve(filePath)}`;
 		return id;
 	}
-	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path$1.resolve(id)}`;
+	if ((id.endsWith(".js") || id.endsWith(".ts") || id.endsWith(".mjs")) && (id.includes("/") || id.includes("\\"))) return `file://${path.resolve(id)}`;
 	return id;
 }
-function getProviderLabel(provider) {
-	return provider?.label && typeof provider.label === "string" ? provider.label : void 0;
+/**
+* Returns true for provider refs that should be expanded from YAML/JSON config files.
+*/
+function isProviderConfigFileReference(providerPath) {
+	return providerPath.startsWith("file://") && (providerPath.endsWith(".yaml") || providerPath.endsWith(".yml") || providerPath.endsWith(".json"));
+}
+/**
+* Reads a provider config file and normalizes single-provider and multi-provider files.
+* Returns a `wasArray` flag so callers can detect multi-provider files that require
+* `loadApiProviders` instead of `loadApiProvider`.
+*/
+function readProviderConfigFile(providerPath, basePath) {
+	const relativePath = providerPath.slice(7);
+	const resolvedPath = path.isAbsolute(relativePath) ? relativePath : path.join(basePath || process.cwd(), relativePath);
+	let rawContent;
+	try {
+		rawContent = yaml.load(fs.readFileSync(resolvedPath, "utf8"));
+	} catch (err) {
+		throw new Error(`Failed to load provider config ${relativePath}: ${err instanceof Error ? err.message : err}`);
+	}
+	const fileContent = maybeLoadConfigFromExternalFile(rawContent);
+	invariant(fileContent, `Provider config ${relativePath} is undefined`);
+	return {
+		configs: [fileContent].flat(),
+		relativePath,
+		wasArray: Array.isArray(fileContent)
+	};
+}
+/**
+* Loads provider config objects from a file-backed provider reference.
+*/
+function loadProviderConfigsFromFile(providerPath, basePath) {
+	return readProviderConfigFile(providerPath, basePath).configs;
 }
+/**
+* Pure, synchronous classifier that converts every supported provider reference shape
+* into a discriminated descriptor. Does not read files or instantiate providers.
+*/
+function normalizeProviderRef(provider, options = {}) {
+	const { index } = options;
+	if (typeof provider === "string") {
+		if (!isValidProviderId(provider)) return {
+			kind: "unknown",
+			id: index === void 0 ? "unknown" : `unknown-${index}`
+		};
+		if (isProviderConfigFileReference(provider)) return {
+			kind: "file",
+			id: provider,
+			loadProviderPath: provider
+		};
+		return {
+			kind: "named",
+			id: provider,
+			loadProviderPath: provider
+		};
+	}
+	if (typeof provider === "function") {
+		const label = getProviderLabel(provider);
+		return {
+			kind: "function",
+			id: label ?? (index === void 0 ? "custom-function" : `custom-function-${index}`),
+			label
+		};
+	}
+	if (typeof provider === "object" && provider !== null && !Array.isArray(provider)) {
+		const providerId = provider.id;
+		const label = getProviderLabel(provider);
+		if (isValidProviderId(providerId)) return {
+			kind: "options",
+			id: providerId,
+			label,
+			loadOptions: provider,
+			loadProviderPath: providerId
+		};
+		const keys = Object.keys(provider);
+		if (keys.length === 1 && !PROVIDER_OPTION_KEYS.has(keys[0])) {
+			const originalId = keys[0];
+			const providerObject = provider[originalId];
+			if (typeof providerObject === "object" && providerObject !== null && !Array.isArray(providerObject) && isValidProviderId(originalId)) {
+				const id = isValidProviderId(providerObject.id) ? providerObject.id : originalId;
+				return {
+					kind: "map",
+					id,
+					label: getProviderLabel(providerObject),
+					loadOptions: {
+						...providerObject,
+						id
+					},
+					loadProviderPath: originalId
+				};
+			}
+		}
+		if (isValidProviderId(label)) return {
+			kind: "unknown",
+			id: label,
+			label
+		};
+	}
+	return {
+		kind: "unknown",
+		id: index === void 0 ? "unknown" : `unknown-${index}`
+	};
+}
+//#endregion
+//#region src/util/provider.ts
 function providerToIdentifier(provider) {
 	if (!provider) return;
 	if (typeof provider === "string") return canonicalizeProviderId(provider);
-	const label = getProviderLabel(provider);
+	const { label } = normalizeProviderRef(provider);
 	if (label) return label;
 	if (isApiProvider(provider)) return canonicalizeProviderId(provider.id());
 	if (isProviderOptions(provider)) {
@@ -1298,6 +1423,6 @@ function printBorder() {
 	logger.info(border);
 }
 //#endregion
-export { readFilters as A, getResolvedRelativePath as C, maybeLoadResponseFormatFromExternalFile as D, maybeLoadFromExternalFileWithVars as E, loadFunction as M, parseFileUrl as N, maybeLoadToolsFromExternalFile as O, getNunjucksEngineForFilePath as S, maybeLoadFromExternalFile as T, getProviderDescription as _, evalTableToJson as a, isOpenAiProvider as b, fetchCsvFromGoogleSheet as c, extractRuntimeVars as d, filterRuntimeVars as f, doesProviderRefMatch as g, checkProviderApiKeys as h, ComparisonEvalNotFoundError as i, readOutput as j, parsePathOrGlob as k, setupEnv as l, resultIsForTestCase as m, writeMultipleOutputs as n, generateEvalCsv as o, getTestCaseDeduplicationKey as p, writeOutput as r, mergeComparisonTables as s, printBorder as t, deduplicateTestCases as u, isAnthropicProvider as v, maybeLoadConfigFromExternalFile as w, isProviderAllowed as x, isGoogleProvider as y };
+export { maybeLoadFromExternalFileWithVars as A, loadProviderConfigsFromFile as C, getResolvedRelativePath as D, getNunjucksEngineForFilePath as E, readOutput as F, loadFunction as I, parseFileUrl as L, maybeLoadToolsFromExternalFile as M, parsePathOrGlob as N, maybeLoadConfigFromExternalFile as O, readFilters as P, isProviderConfigFileReference as S, readProviderConfigFile as T, getProviderDescription as _, evalTableToJson as a, isOpenAiProvider as b, fetchCsvFromGoogleSheet as c, extractRuntimeVars as d, filterRuntimeVars as f, doesProviderRefMatch as g, checkProviderApiKeys as h, ComparisonEvalNotFoundError as i, maybeLoadResponseFormatFromExternalFile as j, maybeLoadFromExternalFile as k, setupEnv as l, resultIsForTestCase as m, writeMultipleOutputs as n, generateEvalCsv as o, getTestCaseDeduplicationKey as p, writeOutput as r, mergeComparisonTables as s, printBorder as t, deduplicateTestCases as u, isAnthropicProvider as v, normalizeProviderRef as w, isProviderAllowed as x, isGoogleProvider as y };
 
-//# sourceMappingURL=util-C8e5uydV.js.map
+//# sourceMappingURL=util-3pBZZb_H.js.map