promptfoo 0.120.25 → 0.120.26

package/dist/src/index.d.cts

@@ -364,7 +364,7 @@ interface ProviderResponse {
     http?: {
       status: number;
       statusText: string;
-      headers: Record<string, string>;
+      headers?: Record<string, string>;
       requestHeaders?: Record<string, string>;
     };
     [key: string]: any;
@@ -418,6 +418,12 @@ interface ProviderResponse {
     aspectRatio?: string;
     resolution?: string;
   };
+  images?: ImageOutput[];
+}
+interface ImageOutput {
+  data?: string;
+  blobRef?: BlobRef;
+  mimeType?: string;
 }
 interface ProviderEmbeddingResponse {
   cached?: boolean;
@@ -778,7 +784,7 @@ type PIIPlugin = (typeof PII_PLUGINS)[number];
 type BiasPlugin = (typeof BIAS_PLUGINS)[number];
 declare const BASE_PLUGINS: readonly ["contracts", "excessive-agency", "hallucination", "hijacking", "politics"];
 type BasePlugin = (typeof BASE_PLUGINS)[number];
-declare const ADDITIONAL_PLUGINS: readonly ["aegis", "ascii-smuggling", "beavertails", "bfla", "bola", "cca", "competitors", "coppa", "cross-session-leak", "cyberseceval", "data-exfil", "debug-access", "divergent-repetition", "donotanswer", "ferpa", "harmbench", "toxic-chat", "imitation", "indirect-prompt-injection", "mcp", "medical:anchoring-bias", "medical:hallucination", "medical:incorrect-knowledge", "medical:off-label-use", "medical:prioritization-error", "medical:sycophancy", "financial:calculation-error", "financial:compliance-violation", "financial:confidential-disclosure", "financial:counterfactual", "financial:data-leakage", "financial:defamation", "financial:hallucination", "financial:impartiality", "financial:misconduct", "financial:sycophancy", "ecommerce:compliance-bypass", "ecommerce:order-fraud", "ecommerce:pci-dss", "ecommerce:price-manipulation", "goal-misalignment", "insurance:coverage-discrimination", "insurance:network-misinformation", "insurance:phi-disclosure", "off-topic", "overreliance", "pharmacy:controlled-substance-compliance", "pharmacy:dosage-calculation", "pharmacy:drug-interaction", "telecom:cpni-disclosure", "telecom:location-disclosure", "telecom:account-takeover", "telecom:e911-misinformation", "telecom:tcpa-violation", "telecom:unauthorized-changes", "telecom:fraud-enablement", "telecom:porting-misinformation", "telecom:billing-misinformation", "telecom:coverage-misinformation", "telecom:law-enforcement-request-handling", "telecom:accessibility-violation", "realestate:fair-housing-discrimination", "realestate:steering", "realestate:discriminatory-listings", "realestate:lending-discrimination", "realestate:valuation-bias", "realestate:accessibility-discrimination", "realestate:advertising-discrimination", "realestate:source-of-income", "pliny", "prompt-extraction", "rag-document-exfiltration", "rag-poisoning", "rag-source-attribution", "rbac", "reasoning-dos", "religion", "shell-injection", "special-token-injection", "sql-injection", "ssrf", "system-prompt-override", "tool-discovery", "unsafebench", "unverifiable-claims", "vlguard", "vlsu", "wordplay", "xstest"];
+declare const ADDITIONAL_PLUGINS: readonly ["aegis", "ascii-smuggling", "beavertails", "bfla", "bola", "cca", "competitors", "coppa", "cross-session-leak", "cyberseceval", "data-exfil", "debug-access", "divergent-repetition", "donotanswer", "ferpa", "harmbench", "toxic-chat", "imitation", "indirect-prompt-injection", "mcp", "model-identification", "medical:anchoring-bias", "medical:hallucination", "medical:incorrect-knowledge", "medical:off-label-use", "medical:prioritization-error", "medical:sycophancy", "financial:calculation-error", "financial:compliance-violation", "financial:confidential-disclosure", "financial:counterfactual", "financial:data-leakage", "financial:defamation", "financial:hallucination", "financial:impartiality", "financial:misconduct", "financial:sox-compliance", "financial:sycophancy", "ecommerce:compliance-bypass", "ecommerce:order-fraud", "ecommerce:pci-dss", "ecommerce:price-manipulation", "goal-misalignment", "insurance:coverage-discrimination", "insurance:network-misinformation", "insurance:phi-disclosure", "off-topic", "overreliance", "pharmacy:controlled-substance-compliance", "pharmacy:dosage-calculation", "pharmacy:drug-interaction", "telecom:cpni-disclosure", "telecom:location-disclosure", "telecom:account-takeover", "telecom:e911-misinformation", "telecom:tcpa-violation", "telecom:unauthorized-changes", "telecom:fraud-enablement", "telecom:porting-misinformation", "telecom:billing-misinformation", "telecom:coverage-misinformation", "telecom:law-enforcement-request-handling", "telecom:accessibility-violation", "realestate:fair-housing-discrimination", "realestate:steering", "realestate:discriminatory-listings", "realestate:lending-discrimination", "realestate:valuation-bias", "realestate:accessibility-discrimination", "realestate:advertising-discrimination", "realestate:source-of-income", "pliny", "prompt-extraction", "rag-document-exfiltration", "rag-poisoning", "rag-source-attribution", "rbac", "reasoning-dos", "religion", "shell-injection", "special-token-injection", "sql-injection", "ssrf", "system-prompt-override", "tool-discovery", "unsafebench", "unverifiable-claims", "vlguard", "vlsu", "wordplay", "xstest"];
 type AdditionalPlugin = (typeof ADDITIONAL_PLUGINS)[number];
 declare const CONFIG_REQUIRED_PLUGINS: readonly ["intent", "policy"];
 type ConfigRequiredPlugin = (typeof CONFIG_REQUIRED_PLUGINS)[number];
@@ -1502,6 +1508,7 @@ interface EvaluateTableOutput {
     aspectRatio?: string;
     resolution?: string;
   };
+  images?: ImageOutput[];
 }
 interface EvaluateTableRow {
   description?: string;
@@ -17991,6 +17998,8 @@ interface PluginFactory {
 interface Strategy {
   id: string;
   action: (testCases: TestCaseWithPlugin[], injectVar: string, config: Record<string, any>, strategyId?: string) => Promise<TestCase[]>;
+  /** Whether this strategy requires pre-extracted intent/goal metadata on test cases. */
+  requiresGoalExtraction?: boolean;
 }
 //#endregion
 //#region src/external/matchers/deepeval.d.ts
@@ -18355,6 +18364,14 @@ declare class Eval {
    * This may be higher than getResultsCount() when there are multiple prompts/providers.
    */
   getTotalResultRowCount(): Promise<number>;
+  /**
+   * Find a non-transient HTTP error status from evaluation results.
+   * Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+   *
+   * For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+   * For non-persisted evals: Falls back to scanning in-memory results.
+   */
+  findTargetErrorStatus(): Promise<number | undefined>;
   fetchResultsByTestIdx(testIdx: number): Promise<EvalResult[]>;
   /**
    * CRITICAL: Builds the WHERE SQL clause for filtering results.
@@ -18713,5 +18730,5 @@ declare const _default$1: {
   };
 };
 //#endregion
-export { type AfterAllExtensionHookContext, type AfterEachExtensionHookContext, AgentCancelledPayload, AgentErrorPayload, ApiClassificationProvider, ApiEmbeddingProvider, ApiModerationProvider, ApiProvider, ApiSimilarityProvider, Assertion, AssertionOrSet, AssertionOrSetSchema, AssertionParams, AssertionSchema, AssertionSet, AssertionSetSchema, AssertionType, AssertionTypeSchema, AssertionValue, AssertionValueFunction, AssertionValueFunctionContext, AssertionValueFunctionResult, AtomicTestCase, AtomicTestCaseSchema, AudioGradingConfig, BaseAssertionTypes, BaseAssertionTypesSchema, BaseRedteamMetadata, BaseTokenUsage, BaseTokenUsageSchema, type BeforeAllExtensionHookContext, type BeforeEachExtensionHookContext, CallApiContextParams, CallApiFunction, CallApiOptionsParams, ChatMessage, CommandLineOptions, CommandLineOptionsSchema, CompletedPrompt, CompletedPromptSchema, CompletionTokenDetails, CompletionTokenDetailsSchema, ConversationMessage, ConversationMessageSchema, CsvRow, DefaultProviders, DerivedMetric, DerivedMetricSchema, type EnvOverrides, EvalConversations, EvalRegisters, EvalResultsFilterMode, EvalSummary, EvalTableDTO, EvalWithMetadata, EvaluateOptions, EvaluateOptionsSchema, EvaluateResult, EvaluateStats, EvaluateSummaryV2, EvaluateSummaryV3, EvaluateTable, EvaluateTableOutput, EvaluateTableRow, EvaluateTestSuite, EvaluateTestSuiteWithEvaluateOptions, type ExtensionHookContextMap, FailedPluginInfo, FilePath, GradingConfig, GradingConfigSchema, GradingResult, GuardrailResponse, InferSchema, type Inputs, InputsSchema, Intent, Job, LoadApiProviderContext, ModerationFlag, Modifier, NotPrefixedAssertionTypesSchema, NunjucksFilterMap, OutputConfig, OutputConfigSchema, OutputFile, OutputFileExtension, OutputMetadata, PartialGenerationError, PluginActionParams, PluginConfig, PluginConfigSchema, PoliciesById, Policy, PolicyObject, PolicyObjectSchema, Prompt, PromptConfig, PromptContent, PromptFunction, PromptFunctionContext, PromptFunctionResult, PromptMetrics, PromptWithMetadata, ProviderClassificationResponse, ProviderEmbeddingResponse, ProviderFunction, ProviderId, ProviderLabel, ProviderModerationResponse, ProviderOptions, ProviderOptionsMap, ProviderResponse, ProviderSimilarityResponse, ProviderTestResponse, ProviderType, ProviderTypeMap, ProvidersSchema, RateLimitRegistryRef, RedteamAssertionTypes, RedteamCliGenerateOptions, RedteamContext, RedteamFileConfig, RedteamGenerateOptions, RedteamHistoryEntry, RedteamMediaData, RedteamObjectConfig, RedteamPlugin, RedteamPluginObject, RedteamRunOptions, RedteamStrategy, RedteamStrategyObject, ResultFailureReason, ResultLightweight, ResultLightweightWithLabel, ResultSuggestion, ResultsFile, RunEvalOptions, SavedRedteamConfig, Scenario, ScenarioSchema, ScoringFunction, ServerPromptWithMetadata, SharedResults, SpecialAssertionTypes, SpecialAssertionTypesSchema, StrategyConfig, StrategyConfigSchema, SynthesizeOptions, TestCase, TestCaseSchema, TestCaseWithPlugin, TestCaseWithVarsFile, TestCaseWithVarsFileSchema, TestCasesWithMetadata, TestCasesWithMetadataPrompt, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfig, TestGeneratorConfigSchema, TestSuite, TestSuiteConfig, TestSuiteConfigSchema, TestSuiteSchema, type TokenUsage, TraceData, TraceSpan, TracingConfig, UnifiedConfig, UnifiedConfigSchema, VarMapping, type VarValue, Vars, VarsSchema, ZodLikeSchema, _default as assertions, cache_d_exports as cache, _default$1 as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
+export { type AfterAllExtensionHookContext, type AfterEachExtensionHookContext, AgentCancelledPayload, AgentErrorPayload, ApiClassificationProvider, ApiEmbeddingProvider, ApiModerationProvider, ApiProvider, ApiSimilarityProvider, Assertion, AssertionOrSet, AssertionOrSetSchema, AssertionParams, AssertionSchema, AssertionSet, AssertionSetSchema, AssertionType, AssertionTypeSchema, AssertionValue, AssertionValueFunction, AssertionValueFunctionContext, AssertionValueFunctionResult, AtomicTestCase, AtomicTestCaseSchema, AudioGradingConfig, BaseAssertionTypes, BaseAssertionTypesSchema, BaseRedteamMetadata, BaseTokenUsage, BaseTokenUsageSchema, type BeforeAllExtensionHookContext, type BeforeEachExtensionHookContext, CallApiContextParams, CallApiFunction, CallApiOptionsParams, ChatMessage, CommandLineOptions, CommandLineOptionsSchema, CompletedPrompt, CompletedPromptSchema, CompletionTokenDetails, CompletionTokenDetailsSchema, ConversationMessage, ConversationMessageSchema, CsvRow, DefaultProviders, DerivedMetric, DerivedMetricSchema, type EnvOverrides, EvalConversations, EvalRegisters, EvalResultsFilterMode, EvalSummary, EvalTableDTO, EvalWithMetadata, EvaluateOptions, EvaluateOptionsSchema, EvaluateResult, EvaluateStats, EvaluateSummaryV2, EvaluateSummaryV3, EvaluateTable, EvaluateTableOutput, EvaluateTableRow, EvaluateTestSuite, EvaluateTestSuiteWithEvaluateOptions, type ExtensionHookContextMap, FailedPluginInfo, FilePath, GradingConfig, GradingConfigSchema, GradingResult, GuardrailResponse, ImageOutput, InferSchema, type Inputs, InputsSchema, Intent, Job, LoadApiProviderContext, ModerationFlag, Modifier, NotPrefixedAssertionTypesSchema, NunjucksFilterMap, OutputConfig, OutputConfigSchema, OutputFile, OutputFileExtension, OutputMetadata, PartialGenerationError, PluginActionParams, PluginConfig, PluginConfigSchema, PoliciesById, Policy, PolicyObject, PolicyObjectSchema, Prompt, PromptConfig, PromptContent, PromptFunction, PromptFunctionContext, PromptFunctionResult, PromptMetrics, PromptWithMetadata, ProviderClassificationResponse, ProviderEmbeddingResponse, ProviderFunction, ProviderId, ProviderLabel, ProviderModerationResponse, ProviderOptions, ProviderOptionsMap, ProviderResponse, ProviderSimilarityResponse, ProviderTestResponse, ProviderType, ProviderTypeMap, ProvidersSchema, RateLimitRegistryRef, RedteamAssertionTypes, RedteamCliGenerateOptions, RedteamContext, RedteamFileConfig, RedteamGenerateOptions, RedteamHistoryEntry, RedteamMediaData, RedteamObjectConfig, RedteamPlugin, RedteamPluginObject, RedteamRunOptions, RedteamStrategy, RedteamStrategyObject, ResultFailureReason, ResultLightweight, ResultLightweightWithLabel, ResultSuggestion, ResultsFile, RunEvalOptions, SavedRedteamConfig, Scenario, ScenarioSchema, ScoringFunction, ServerPromptWithMetadata, SharedResults, SpecialAssertionTypes, SpecialAssertionTypesSchema, StrategyConfig, StrategyConfigSchema, SynthesizeOptions, TestCase, TestCaseSchema, TestCaseWithPlugin, TestCaseWithVarsFile, TestCaseWithVarsFileSchema, TestCasesWithMetadata, TestCasesWithMetadataPrompt, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfig, TestGeneratorConfigSchema, TestSuite, TestSuiteConfig, TestSuiteConfigSchema, TestSuiteSchema, type TokenUsage, TraceData, TraceSpan, TracingConfig, UnifiedConfig, UnifiedConfigSchema, VarMapping, type VarValue, Vars, VarsSchema, ZodLikeSchema, _default as assertions, cache_d_exports as cache, _default$1 as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
 //# sourceMappingURL=index.d.cts.map

package/dist/src/index.d.ts

@@ -361,7 +361,7 @@ interface ProviderResponse {
     http?: {
       status: number;
       statusText: string;
-      headers: Record<string, string>;
+      headers?: Record<string, string>;
       requestHeaders?: Record<string, string>;
     };
     [key: string]: any;
@@ -415,6 +415,12 @@ interface ProviderResponse {
     aspectRatio?: string;
     resolution?: string;
   };
+  images?: ImageOutput[];
+}
+interface ImageOutput {
+  data?: string;
+  blobRef?: BlobRef;
+  mimeType?: string;
 }
 interface ProviderEmbeddingResponse {
   cached?: boolean;
@@ -775,7 +781,7 @@ type PIIPlugin = (typeof PII_PLUGINS)[number];
 type BiasPlugin = (typeof BIAS_PLUGINS)[number];
 declare const BASE_PLUGINS: readonly ["contracts", "excessive-agency", "hallucination", "hijacking", "politics"];
 type BasePlugin = (typeof BASE_PLUGINS)[number];
-declare const ADDITIONAL_PLUGINS: readonly ["aegis", "ascii-smuggling", "beavertails", "bfla", "bola", "cca", "competitors", "coppa", "cross-session-leak", "cyberseceval", "data-exfil", "debug-access", "divergent-repetition", "donotanswer", "ferpa", "harmbench", "toxic-chat", "imitation", "indirect-prompt-injection", "mcp", "medical:anchoring-bias", "medical:hallucination", "medical:incorrect-knowledge", "medical:off-label-use", "medical:prioritization-error", "medical:sycophancy", "financial:calculation-error", "financial:compliance-violation", "financial:confidential-disclosure", "financial:counterfactual", "financial:data-leakage", "financial:defamation", "financial:hallucination", "financial:impartiality", "financial:misconduct", "financial:sycophancy", "ecommerce:compliance-bypass", "ecommerce:order-fraud", "ecommerce:pci-dss", "ecommerce:price-manipulation", "goal-misalignment", "insurance:coverage-discrimination", "insurance:network-misinformation", "insurance:phi-disclosure", "off-topic", "overreliance", "pharmacy:controlled-substance-compliance", "pharmacy:dosage-calculation", "pharmacy:drug-interaction", "telecom:cpni-disclosure", "telecom:location-disclosure", "telecom:account-takeover", "telecom:e911-misinformation", "telecom:tcpa-violation", "telecom:unauthorized-changes", "telecom:fraud-enablement", "telecom:porting-misinformation", "telecom:billing-misinformation", "telecom:coverage-misinformation", "telecom:law-enforcement-request-handling", "telecom:accessibility-violation", "realestate:fair-housing-discrimination", "realestate:steering", "realestate:discriminatory-listings", "realestate:lending-discrimination", "realestate:valuation-bias", "realestate:accessibility-discrimination", "realestate:advertising-discrimination", "realestate:source-of-income", "pliny", "prompt-extraction", "rag-document-exfiltration", "rag-poisoning", "rag-source-attribution", "rbac", "reasoning-dos", "religion", "shell-injection", "special-token-injection", "sql-injection", "ssrf", "system-prompt-override", "tool-discovery", "unsafebench", "unverifiable-claims", "vlguard", "vlsu", "wordplay", "xstest"];
+declare const ADDITIONAL_PLUGINS: readonly ["aegis", "ascii-smuggling", "beavertails", "bfla", "bola", "cca", "competitors", "coppa", "cross-session-leak", "cyberseceval", "data-exfil", "debug-access", "divergent-repetition", "donotanswer", "ferpa", "harmbench", "toxic-chat", "imitation", "indirect-prompt-injection", "mcp", "model-identification", "medical:anchoring-bias", "medical:hallucination", "medical:incorrect-knowledge", "medical:off-label-use", "medical:prioritization-error", "medical:sycophancy", "financial:calculation-error", "financial:compliance-violation", "financial:confidential-disclosure", "financial:counterfactual", "financial:data-leakage", "financial:defamation", "financial:hallucination", "financial:impartiality", "financial:misconduct", "financial:sox-compliance", "financial:sycophancy", "ecommerce:compliance-bypass", "ecommerce:order-fraud", "ecommerce:pci-dss", "ecommerce:price-manipulation", "goal-misalignment", "insurance:coverage-discrimination", "insurance:network-misinformation", "insurance:phi-disclosure", "off-topic", "overreliance", "pharmacy:controlled-substance-compliance", "pharmacy:dosage-calculation", "pharmacy:drug-interaction", "telecom:cpni-disclosure", "telecom:location-disclosure", "telecom:account-takeover", "telecom:e911-misinformation", "telecom:tcpa-violation", "telecom:unauthorized-changes", "telecom:fraud-enablement", "telecom:porting-misinformation", "telecom:billing-misinformation", "telecom:coverage-misinformation", "telecom:law-enforcement-request-handling", "telecom:accessibility-violation", "realestate:fair-housing-discrimination", "realestate:steering", "realestate:discriminatory-listings", "realestate:lending-discrimination", "realestate:valuation-bias", "realestate:accessibility-discrimination", "realestate:advertising-discrimination", "realestate:source-of-income", "pliny", "prompt-extraction", "rag-document-exfiltration", "rag-poisoning", "rag-source-attribution", "rbac", "reasoning-dos", "religion", "shell-injection", "special-token-injection", "sql-injection", "ssrf", "system-prompt-override", "tool-discovery", "unsafebench", "unverifiable-claims", "vlguard", "vlsu", "wordplay", "xstest"];
 type AdditionalPlugin = (typeof ADDITIONAL_PLUGINS)[number];
 declare const CONFIG_REQUIRED_PLUGINS: readonly ["intent", "policy"];
 type ConfigRequiredPlugin = (typeof CONFIG_REQUIRED_PLUGINS)[number];
@@ -1499,6 +1505,7 @@ interface EvaluateTableOutput {
     aspectRatio?: string;
     resolution?: string;
   };
+  images?: ImageOutput[];
 }
 interface EvaluateTableRow {
   description?: string;
@@ -17988,6 +17995,8 @@ interface PluginFactory {
 interface Strategy {
   id: string;
   action: (testCases: TestCaseWithPlugin[], injectVar: string, config: Record<string, any>, strategyId?: string) => Promise<TestCase[]>;
+  /** Whether this strategy requires pre-extracted intent/goal metadata on test cases. */
+  requiresGoalExtraction?: boolean;
 }
 //#endregion
 //#region src/external/matchers/deepeval.d.ts
@@ -18352,6 +18361,14 @@ declare class Eval {
    * This may be higher than getResultsCount() when there are multiple prompts/providers.
    */
   getTotalResultRowCount(): Promise<number>;
+  /**
+   * Find a non-transient HTTP error status from evaluation results.
+   * Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+   *
+   * For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+   * For non-persisted evals: Falls back to scanning in-memory results.
+   */
+  findTargetErrorStatus(): Promise<number | undefined>;
   fetchResultsByTestIdx(testIdx: number): Promise<EvalResult[]>;
   /**
    * CRITICAL: Builds the WHERE SQL clause for filtering results.
@@ -18710,5 +18727,5 @@ declare const _default$1: {
   };
 };
 //#endregion
-export { type AfterAllExtensionHookContext, type AfterEachExtensionHookContext, AgentCancelledPayload, AgentErrorPayload, ApiClassificationProvider, ApiEmbeddingProvider, ApiModerationProvider, ApiProvider, ApiSimilarityProvider, Assertion, AssertionOrSet, AssertionOrSetSchema, AssertionParams, AssertionSchema, AssertionSet, AssertionSetSchema, AssertionType, AssertionTypeSchema, AssertionValue, AssertionValueFunction, AssertionValueFunctionContext, AssertionValueFunctionResult, AtomicTestCase, AtomicTestCaseSchema, AudioGradingConfig, BaseAssertionTypes, BaseAssertionTypesSchema, BaseRedteamMetadata, BaseTokenUsage, BaseTokenUsageSchema, type BeforeAllExtensionHookContext, type BeforeEachExtensionHookContext, CallApiContextParams, CallApiFunction, CallApiOptionsParams, ChatMessage, CommandLineOptions, CommandLineOptionsSchema, CompletedPrompt, CompletedPromptSchema, CompletionTokenDetails, CompletionTokenDetailsSchema, ConversationMessage, ConversationMessageSchema, CsvRow, DefaultProviders, DerivedMetric, DerivedMetricSchema, type EnvOverrides, EvalConversations, EvalRegisters, EvalResultsFilterMode, EvalSummary, EvalTableDTO, EvalWithMetadata, EvaluateOptions, EvaluateOptionsSchema, EvaluateResult, EvaluateStats, EvaluateSummaryV2, EvaluateSummaryV3, EvaluateTable, EvaluateTableOutput, EvaluateTableRow, EvaluateTestSuite, EvaluateTestSuiteWithEvaluateOptions, type ExtensionHookContextMap, FailedPluginInfo, FilePath, GradingConfig, GradingConfigSchema, GradingResult, GuardrailResponse, InferSchema, type Inputs, InputsSchema, Intent, Job, LoadApiProviderContext, ModerationFlag, Modifier, NotPrefixedAssertionTypesSchema, NunjucksFilterMap, OutputConfig, OutputConfigSchema, OutputFile, OutputFileExtension, OutputMetadata, PartialGenerationError, PluginActionParams, PluginConfig, PluginConfigSchema, PoliciesById, Policy, PolicyObject, PolicyObjectSchema, Prompt, PromptConfig, PromptContent, PromptFunction, PromptFunctionContext, PromptFunctionResult, PromptMetrics, PromptWithMetadata, ProviderClassificationResponse, ProviderEmbeddingResponse, ProviderFunction, ProviderId, ProviderLabel, ProviderModerationResponse, ProviderOptions, ProviderOptionsMap, ProviderResponse, ProviderSimilarityResponse, ProviderTestResponse, ProviderType, ProviderTypeMap, ProvidersSchema, RateLimitRegistryRef, RedteamAssertionTypes, RedteamCliGenerateOptions, RedteamContext, RedteamFileConfig, RedteamGenerateOptions, RedteamHistoryEntry, RedteamMediaData, RedteamObjectConfig, RedteamPlugin, RedteamPluginObject, RedteamRunOptions, RedteamStrategy, RedteamStrategyObject, ResultFailureReason, ResultLightweight, ResultLightweightWithLabel, ResultSuggestion, ResultsFile, RunEvalOptions, SavedRedteamConfig, Scenario, ScenarioSchema, ScoringFunction, ServerPromptWithMetadata, SharedResults, SpecialAssertionTypes, SpecialAssertionTypesSchema, StrategyConfig, StrategyConfigSchema, SynthesizeOptions, TestCase, TestCaseSchema, TestCaseWithPlugin, TestCaseWithVarsFile, TestCaseWithVarsFileSchema, TestCasesWithMetadata, TestCasesWithMetadataPrompt, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfig, TestGeneratorConfigSchema, TestSuite, TestSuiteConfig, TestSuiteConfigSchema, TestSuiteSchema, type TokenUsage, TraceData, TraceSpan, TracingConfig, UnifiedConfig, UnifiedConfigSchema, VarMapping, type VarValue, Vars, VarsSchema, ZodLikeSchema, _default as assertions, cache_d_exports as cache, _default$1 as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
+export { type AfterAllExtensionHookContext, type AfterEachExtensionHookContext, AgentCancelledPayload, AgentErrorPayload, ApiClassificationProvider, ApiEmbeddingProvider, ApiModerationProvider, ApiProvider, ApiSimilarityProvider, Assertion, AssertionOrSet, AssertionOrSetSchema, AssertionParams, AssertionSchema, AssertionSet, AssertionSetSchema, AssertionType, AssertionTypeSchema, AssertionValue, AssertionValueFunction, AssertionValueFunctionContext, AssertionValueFunctionResult, AtomicTestCase, AtomicTestCaseSchema, AudioGradingConfig, BaseAssertionTypes, BaseAssertionTypesSchema, BaseRedteamMetadata, BaseTokenUsage, BaseTokenUsageSchema, type BeforeAllExtensionHookContext, type BeforeEachExtensionHookContext, CallApiContextParams, CallApiFunction, CallApiOptionsParams, ChatMessage, CommandLineOptions, CommandLineOptionsSchema, CompletedPrompt, CompletedPromptSchema, CompletionTokenDetails, CompletionTokenDetailsSchema, ConversationMessage, ConversationMessageSchema, CsvRow, DefaultProviders, DerivedMetric, DerivedMetricSchema, type EnvOverrides, EvalConversations, EvalRegisters, EvalResultsFilterMode, EvalSummary, EvalTableDTO, EvalWithMetadata, EvaluateOptions, EvaluateOptionsSchema, EvaluateResult, EvaluateStats, EvaluateSummaryV2, EvaluateSummaryV3, EvaluateTable, EvaluateTableOutput, EvaluateTableRow, EvaluateTestSuite, EvaluateTestSuiteWithEvaluateOptions, type ExtensionHookContextMap, FailedPluginInfo, FilePath, GradingConfig, GradingConfigSchema, GradingResult, GuardrailResponse, ImageOutput, InferSchema, type Inputs, InputsSchema, Intent, Job, LoadApiProviderContext, ModerationFlag, Modifier, NotPrefixedAssertionTypesSchema, NunjucksFilterMap, OutputConfig, OutputConfigSchema, OutputFile, OutputFileExtension, OutputMetadata, PartialGenerationError, PluginActionParams, PluginConfig, PluginConfigSchema, PoliciesById, Policy, PolicyObject, PolicyObjectSchema, Prompt, PromptConfig, PromptContent, PromptFunction, PromptFunctionContext, PromptFunctionResult, PromptMetrics, PromptWithMetadata, ProviderClassificationResponse, ProviderEmbeddingResponse, ProviderFunction, ProviderId, ProviderLabel, ProviderModerationResponse, ProviderOptions, ProviderOptionsMap, ProviderResponse, ProviderSimilarityResponse, ProviderTestResponse, ProviderType, ProviderTypeMap, ProvidersSchema, RateLimitRegistryRef, RedteamAssertionTypes, RedteamCliGenerateOptions, RedteamContext, RedteamFileConfig, RedteamGenerateOptions, RedteamHistoryEntry, RedteamMediaData, RedteamObjectConfig, RedteamPlugin, RedteamPluginObject, RedteamRunOptions, RedteamStrategy, RedteamStrategyObject, ResultFailureReason, ResultLightweight, ResultLightweightWithLabel, ResultSuggestion, ResultsFile, RunEvalOptions, SavedRedteamConfig, Scenario, ScenarioSchema, ScoringFunction, ServerPromptWithMetadata, SharedResults, SpecialAssertionTypes, SpecialAssertionTypesSchema, StrategyConfig, StrategyConfigSchema, SynthesizeOptions, TestCase, TestCaseSchema, TestCaseWithPlugin, TestCaseWithVarsFile, TestCaseWithVarsFileSchema, TestCasesWithMetadata, TestCasesWithMetadataPrompt, TestCasesWithMetadataPromptSchema, TestCasesWithMetadataSchema, TestGeneratorConfig, TestGeneratorConfigSchema, TestSuite, TestSuiteConfig, TestSuiteConfigSchema, TestSuiteSchema, type TokenUsage, TraceData, TraceSpan, TracingConfig, UnifiedConfig, UnifiedConfigSchema, VarMapping, type VarValue, Vars, VarsSchema, ZodLikeSchema, _default as assertions, cache_d_exports as cache, _default$1 as default, evaluate, generateTable, guardrails, isApiProvider, isGradingResult, isProviderOptions, isResultFailureReason, loadApiProvider, redteam };
 //# sourceMappingURL=index.d.ts.map

package/dist/src/index.js

@@ -5,36 +5,36 @@ import { r as importModule, t as getDirectory } from "./esm-rDtG_2rg.js";
 import { r as runPython } from "./pythonUtils-vMlk9Qp5.js";
 import { i as isJavascriptFile } from "./fileExtensions-ePDqouxn.js";
 import { n as transform, r as getProcessShim, t as TransformInputType } from "./transform-D8dILpfZ.js";
-import { $ as matchesSearchRubric, A as BeavertailsPlugin, B as getAndCheckProvider, C as HarmbenchPlugin, D as DebugAccessPlugin, E as DivergentRepetitionPlugin, F as retryWithDeduplication, G as matchesContextFaithfulness, H as matchesAnswerRelevance, I as sampleArray, J as matchesFactuality, K as matchesContextRecall, L as fetchHuggingFaceDataset, M as RedteamGraderBase, N as RedteamPluginBase, O as CrossSessionLeakPlugin, P as getCustomPolicies, Q as matchesPiScore, R as callProviderWithContext, S as ImitationPlugin, T as ExcessiveAgencyPlugin, U as matchesClassification, V as loadRubricPrompt, W as matchesClosedQa, X as matchesLlmRubric, Y as matchesGEval, Z as matchesModeration, _ as makeInlinePolicyIdSync, a as UnverifiableClaimsPlugin, at as processPrompts, b as OverreliancePlugin, c as ToolDiscoveryPlugin, ct as SUGGEST_PROMPTS_SYSTEM_MESSAGE, d as RbacPlugin, dt as loadFromJavaScriptFile, et as matchesSelectBest, f as PromptExtractionPlugin, ft as processFileReference, g as isValidPolicyObject, h as determinePolicyTypeFromId, i as VLGuardPlugin, it as DefaultSuggestionsProvider, j as AegisPlugin, k as ContractPlugin, l as SqlInjectionPlugin, lt as coerceString, m as PolicyPlugin, n as getGraderById, nt as selectMaxScore, o as UnsafeBenchPlugin, ot as readPrompts, p as PoliticsPlugin, pt as resolveContext, q as matchesContextRelevance, r as VLSUPlugin, rt as getDefaultProviders, s as ToxicChatPlugin, st as readProviderPromptMap, t as GRADERS, tt as matchesSimilarity, u as ShellInjectionPlugin, ut as getFinalTest, v as PlinyPlugin, w as HallucinationPlugin, x as IntentPlugin, y as getPiiLeakTestsForCategory, z as fail } from "./graders-CtvuaC8D.js";
-import { A as isApiProvider, C as TestGeneratorConfigSchema, Ct as BaseTokenUsageSchema, D as VarsSchema, E as UnifiedConfigSchema, F as ConversationMessageSchema, I as PartialGenerationError, J as getDefaultNFanout, K as STRATEGY_COLLECTIONS, L as PluginConfigSchema, M as RedteamConfigSchema, O as isGradingResult, P as ProvidersSchema, Q as categoryAliases, R as PolicyObjectSchema, S as TestCasesWithMetadataSchema, St as PromptSchema, T as TestSuiteSchema, Tt as InputsSchema, V as isUuid, W as DEFAULT_STRATEGIES, X as isFanoutStrategy, Z as Severity, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TELECOM_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as riskCategorySeverityMap, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as isProviderOptions, k as isResultFailureReason, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as BIAS_PLUGINS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as STRATEGY_COLLECTION_MAPPINGS, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as ALIASED_PLUGIN_MAPPINGS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CompletionTokenDetailsSchema, x as TestCasesWithMetadataPromptSchema, xt as UNALIGNED_PROVIDER_HARM_PLUGINS, y as TestCaseSchema, z as StrategyConfigSchema } from "./types-CslP2TyG.js";
-import { A as isAnthropicProvider, C as deduplicateTestCases, D as resultIsForTestCase, E as getTestCaseDeduplicationKey, M as isOpenAiProvider, N as isProviderAllowed, O as doesProviderRefMatch, S as setupEnv, T as filterRuntimeVars, b as loadFunction, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, h as renderEnvOnlyInObject, i as fetchCsvFromGoogleSheet, j as isGoogleProvider, k as getProviderDescription, m as readOutput, n as writeMultipleOutputs, p as readFilters, r as writeOutput, s as maybeLoadConfigFromExternalFile, t as printBorder, v as extractVariablesFromTemplates, w as extractRuntimeVars, x as parseFileUrl, y as getNunjucksEngine } from "./util-CY_XguB9.js";
-import { F as getShareViewBaseUrl, L as VERSION, M as TERMINAL_MAX_WIDTH, N as getDefaultShareViewBaseUrl, O as DEFAULT_MAX_CONCURRENCY, P as getShareApiBaseUrl, R as FILE_METADATA_KEY, S as parseChatPrompt, c as cloudConfig, f as getCurrentTimestamp, h as REQUEST_TIMEOUT_MS, k as DEFAULT_QUERY_LIMIT, n as fetchWithRetries, p as sleep, r as fetchWithTimeout, s as CloudConfig, t as fetchWithProxy, y as isPromptfooSampleTarget, z as HUMAN_ASSERTION_TYPE } from "./fetch-zJuFPxN1.js";
-import { a as fetchWithCache, o as getCache, r as disableCache, t as cache_exports } from "./cache-C4A4H4Xb.js";
-import { A as createRateLimitRegistry, B as isCloudProvider, C as collectFileMetadata, D as loadFromPackage, E as isPackagePath, F as getCloudDatabaseId, I as getEvalConfigFromCloud, J as AIStudioChatProvider, L as getOrgContext, M as PromptfooHarmfulCompletionProvider, O as redteamProviderManager, P as checkCloudPermissions, R as getPluginSeverityOverridesFromCloud, T as runExtensionHook, V as resolveTeamId, _ as extractVariablesFromJson, a as resolveProviderConfigs, b as isBasicRefusal, c as Strategies, d as pluginMatchesStrategyTargets, f as checkExfilTracking, g as extractPromptFromTags, i as resolveProvider, j as createProviderRateLimitOptions, k as TokenUsageTracker, l as loadStrategy, m as extractGoalFromPrompt, n as loadApiProvider, o as MCPProvider, q as VertexChatProvider, r as loadApiProviders, s as GoogleLiveProvider, t as getProviderIds, u as validateStrategies, v as getSessionId, w as renderPrompt, y as getShortPluginId } from "./providers-D3lk_R7Q.js";
-import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-BTeLHwiH.js";
-import { a as sha256, i as randomSequence, n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-CuUWqwrE.js";
+import { $ as matchesSearchRubric, A as BeavertailsPlugin, B as getAndCheckProvider, C as HarmbenchPlugin, D as DebugAccessPlugin, E as DivergentRepetitionPlugin, F as retryWithDeduplication, G as matchesContextFaithfulness, H as matchesAnswerRelevance, I as sampleArray, J as matchesFactuality, K as matchesContextRecall, L as fetchHuggingFaceDataset, M as RedteamGraderBase, N as RedteamPluginBase, O as CrossSessionLeakPlugin, P as getCustomPolicies, Q as matchesPiScore, R as callProviderWithContext, S as ImitationPlugin, T as ExcessiveAgencyPlugin, U as matchesClassification, V as loadRubricPrompt, W as matchesClosedQa, X as matchesLlmRubric, Y as matchesGEval, Z as matchesModeration, _ as makeInlinePolicyIdSync, a as UnverifiableClaimsPlugin, at as processPrompts, b as OverreliancePlugin, c as ToolDiscoveryPlugin, ct as SUGGEST_PROMPTS_SYSTEM_MESSAGE, d as RbacPlugin, dt as loadFromJavaScriptFile, et as matchesSelectBest, f as PromptExtractionPlugin, ft as processFileReference, g as isValidPolicyObject, h as determinePolicyTypeFromId, i as VLGuardPlugin, it as DefaultSuggestionsProvider, j as AegisPlugin, k as ContractPlugin, l as SqlInjectionPlugin, lt as coerceString, m as PolicyPlugin, n as getGraderById, nt as selectMaxScore, o as UnsafeBenchPlugin, ot as readPrompts, p as PoliticsPlugin, pt as resolveContext, q as matchesContextRelevance, r as VLSUPlugin, rt as getDefaultProviders, s as ToxicChatPlugin, st as readProviderPromptMap, t as GRADERS, tt as matchesSimilarity, u as ShellInjectionPlugin, ut as getFinalTest, v as PlinyPlugin, w as HallucinationPlugin, x as IntentPlugin, y as getPiiLeakTestsForCategory, z as fail } from "./graders-QsALpIdy.js";
+import { A as isApiProvider, C as TestGeneratorConfigSchema, Ct as BaseTokenUsageSchema, D as VarsSchema, E as UnifiedConfigSchema, F as ConversationMessageSchema, I as PartialGenerationError, J as getDefaultNFanout, K as STRATEGY_COLLECTIONS, L as PluginConfigSchema, M as RedteamConfigSchema, O as isGradingResult, P as ProvidersSchema, Q as categoryAliases, R as PolicyObjectSchema, S as TestCasesWithMetadataSchema, St as PromptSchema, T as TestSuiteSchema, Tt as InputsSchema, V as isUuid, W as DEFAULT_STRATEGIES, X as isFanoutStrategy, Z as Severity, _ as ScenarioSchema, _t as REDTEAM_PROVIDER_HARM_PLUGINS, a as AtomicTestCaseSchema, at as FINANCIAL_PLUGINS, b as TestCaseWithVarsFileSchema, bt as TELECOM_PLUGINS, c as CompletedPromptSchema, ct as INSURANCE_PLUGINS, d as EvaluateOptionsSchema, dt as MEDICAL_PLUGINS, et as riskCategorySeverityMap, f as GradingConfigSchema, ft as MULTI_INPUT_EXCLUDED_PLUGINS, g as ResultFailureReason, gt as PLUGIN_CATEGORIES, h as OutputFileExtension, ht as PII_PLUGINS, i as AssertionTypeSchema, it as DEFAULT_PLUGINS, j as isProviderOptions, k as isResultFailureReason, l as DerivedMetricSchema, lt as LLAMA_GUARD_ENABLED_CATEGORIES, m as OutputConfigSchema, mt as PHARMACY_PLUGINS, n as AssertionSchema, nt as BIAS_PLUGINS, o as BaseAssertionTypesSchema, ot as FOUNDATION_PLUGINS, p as NotPrefixedAssertionTypesSchema, pt as MULTI_INPUT_VAR, q as STRATEGY_COLLECTION_MAPPINGS, r as AssertionSetSchema, rt as DATASET_EXEMPT_PLUGINS, s as CommandLineOptionsSchema, st as HARM_PLUGINS, t as AssertionOrSetSchema, tt as ALIASED_PLUGIN_MAPPINGS, u as EvalResultsFilterMode, ut as LLAMA_GUARD_REPLICATE_PROVIDER, v as SpecialAssertionTypesSchema, vt as REMOTE_ONLY_PLUGIN_IDS, w as TestSuiteConfigSchema, wt as CompletionTokenDetailsSchema, x as TestCasesWithMetadataPromptSchema, xt as UNALIGNED_PROVIDER_HARM_PLUGINS, y as TestCaseSchema, z as StrategyConfigSchema } from "./types-DMVjYLpx.js";
+import { A as isAnthropicProvider, C as deduplicateTestCases, D as resultIsForTestCase, E as getTestCaseDeduplicationKey, M as isOpenAiProvider, N as isProviderAllowed, O as doesProviderRefMatch, S as setupEnv, T as filterRuntimeVars, b as loadFunction, c as maybeLoadFromExternalFile, d as maybeLoadToolsFromExternalFile, h as renderEnvOnlyInObject, i as fetchCsvFromGoogleSheet, j as isGoogleProvider, k as getProviderDescription, m as readOutput, n as writeMultipleOutputs, p as readFilters, r as writeOutput, s as maybeLoadConfigFromExternalFile, t as printBorder, v as extractVariablesFromTemplates, w as extractRuntimeVars, x as parseFileUrl, y as getNunjucksEngine } from "./util-DiCePfDu.js";
+import { F as getShareViewBaseUrl, L as VERSION, M as TERMINAL_MAX_WIDTH, N as getDefaultShareViewBaseUrl, O as DEFAULT_MAX_CONCURRENCY, P as getShareApiBaseUrl, R as FILE_METADATA_KEY, S as parseChatPrompt, c as cloudConfig, f as getCurrentTimestamp, h as REQUEST_TIMEOUT_MS, k as DEFAULT_QUERY_LIMIT, n as fetchWithRetries, p as sleep, r as fetchWithTimeout, s as CloudConfig, t as fetchWithProxy, y as isPromptfooSampleTarget, z as HUMAN_ASSERTION_TYPE } from "./fetch-18MuNu9i.js";
+import { a as fetchWithCache, c as NON_TRANSIENT_HTTP_STATUSES, l as isNonTransientHttpStatus, o as getCache, r as disableCache, t as cache_exports } from "./cache-BLLayYEN.js";
+import { A as createRateLimitRegistry, B as isCloudProvider, C as collectFileMetadata, D as loadFromPackage, E as isPackagePath, F as getCloudDatabaseId, I as getEvalConfigFromCloud, J as AIStudioChatProvider, L as getOrgContext, M as PromptfooHarmfulCompletionProvider, O as redteamProviderManager, P as checkCloudPermissions, R as getPluginSeverityOverridesFromCloud, T as runExtensionHook, V as resolveTeamId, _ as extractVariablesFromJson, a as resolveProviderConfigs, b as isBasicRefusal, c as Strategies, d as pluginMatchesStrategyTargets, f as checkExfilTracking, g as extractPromptFromTags, i as resolveProvider, j as createProviderRateLimitOptions, k as TokenUsageTracker, l as loadStrategy, m as extractGoalFromPrompt, n as loadApiProvider, o as MCPProvider, q as VertexChatProvider, r as loadApiProviders, s as GoogleLiveProvider, t as getProviderIds, u as validateStrategies, v as getSessionId, w as renderPrompt, y as getShortPluginId } from "./providers-BiNq_Iyc.js";
+import { i as generateIdFromPrompt, t as hashPrompt } from "./utils-CVzb4YiI.js";
+import { a as sha256, i as randomSequence, n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-Hs7la_19.js";
 import "./genaiTracer-Ce19n68P.js";
-import { t as OpenAiChatCompletionProvider } from "./chat-noXtEAh5.js";
+import { t as OpenAiChatCompletionProvider } from "./chat-C8Ei4f87.js";
 import { a as createEmptyTokenUsage, i as createEmptyAssertions, n as accumulateResponseTokenUsage, o as normalizeTokenUsage, r as accumulateTokenUsage, t as accumulateAssertionTokenUsage } from "./tokenUsageUtils-FZd5O_4A.js";
-import { p as validateFunctionCall } from "./transform-DosdxVdh.js";
-import "./messages-Bg2ekFPc.js";
-import "./util-Dft1RxsL.js";
-import "./responses-BtU_i-R0.js";
+import { m as validateFunctionCall } from "./transform-DMaxQwDx.js";
+import "./messages-CDZYGNlS.js";
+import "./util-mJ58qbbw.js";
+import "./responses-DLLjADw5.js";
 import "./openai-Dz3surb_.js";
-import { l as validateFunctionCall$1 } from "./util-CfPdWQJX.js";
-import "./completion-HYrX5gEL.js";
-import { c as setUserEmail, i as getUserEmail, l as EMAIL_OK_STATUS, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-BSwMxhY3.js";
-import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-BPC37wlX.js";
-import { t as getBlobByHash } from "./blobs-Dnt2rxpc.js";
-import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-dNaVPyDr.js";
-import { n as telemetry_default } from "./telemetry-Dai7PdlR.js";
+import { l as validateFunctionCall$1 } from "./util-Cl0zfT3V.js";
+import "./completion-D9_MDlnd.js";
+import { c as setUserEmail, i as getUserEmail, l as EMAIL_OK_STATUS, r as getAuthor, s as promptForEmailUnverified, t as checkEmailStatusAndMaybeExit } from "./accounts-R3gfCR_g.js";
+import { i as getRemoteGenerationUrl, l as shouldGenerateRemote, o as getRemoteHealthUrl, r as promptYesNo, s as neverGenerateRemote } from "./server-CbMTRQkg.js";
+import { t as getBlobByHash } from "./blobs-C9J2mVgo.js";
+import { a as evalsTable, c as evalsToTagsTable, d as tagsTable, i as evalResultsTable, l as promptsTable, m as getDbSignalPath, o as evalsToDatasetsTable, p as getDb, r as datasetsTable, s as evalsToPromptsTable } from "./tables-D-NSwNIb.js";
+import { n as telemetry_default } from "./telemetry-DZ_7PaVq.js";
 import { t as ellipsize } from "./text-Dm78AVGG.js";
-import { n as getTraceStore } from "./store-BTdkdmK9.js";
-import "./base-CwCV0t8r.js";
-import "./image-LhLO5KCW.js";
+import { n as getTraceStore } from "./store-2ocbYY9D.js";
+import "./base-Dtp8b4_N.js";
+import "./image-BiEVdpdP.js";
 import { t as providerRegistry } from "./providerRegistry-wCWd7sKQ.js";
 import { n as runRuby } from "./rubyUtils-Cs35SDYa.js";
-import { t as EvalResult } from "./evalResult-CJNSpa3q.js";
+import { t as EvalResult } from "./evalResult-COsVttMA.js";
 import * as fs$1 from "fs";
 import fs, { createWriteStream } from "fs";
 import * as path$2 from "path";
@@ -292,7 +292,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 		telemetry_default.record("feature_used", { feature: "tracing" });
 		try {
 			logger_default.debug("[EvaluatorTracing] Tracing configuration detected, starting OTLP receiver");
-			const { startOTLPReceiver } = await import("./otlpReceiver-uytrRKdo.js");
+			const { startOTLPReceiver } = await import("./otlpReceiver-BO0rbDzh.js");
 			const port = testSuite.tracing.otlp.http.port || 4318;
 			const host = testSuite.tracing.otlp.http.host || "127.0.0.1";
 			logger_default.debug(`[EvaluatorTracing] Starting OTLP receiver on ${host}:${port}`);
@@ -315,7 +315,7 @@ async function startOtlpReceiverIfNeeded(testSuite) {
 async function stopOtlpReceiverIfNeeded() {
 	if (otlpReceiverStarted) try {
 		logger_default.debug("[EvaluatorTracing] Stopping OTLP receiver");
-		const { stopOTLPReceiver } = await import("./otlpReceiver-uytrRKdo.js");
+		const { stopOTLPReceiver } = await import("./otlpReceiver-BO0rbDzh.js");
 		await stopOTLPReceiver();
 		otlpReceiverStarted = false;
 		logger_default.info("[EvaluatorTracing] OTLP receiver stopped successfully");
@@ -350,7 +350,7 @@ async function generateTraceContextIfNeeded(test, evaluateOptions, testIdx, prom
 	}
 	if (!tracingEnabled) return null;
 	logger_default.debug("[EvaluatorTracing] Importing trace store");
-	const { getTraceStore } = await import("./store-C7U17qcu.js");
+	const { getTraceStore } = await import("./store-D_lq_8oQ.js");
 	const traceStore = getTraceStore();
 	const traceId = generateTraceId();
 	const spanId = generateSpanId();
@@ -2871,9 +2871,9 @@ async function runAssertion({ prompt, provider, assertion, test, vars, latencyMs
 		let filePath = fileRef;
 		let functionName;
 		if (fileRef.includes(":")) {
-			const [pathPart, funcPart] = fileRef.split(":");
-			filePath = pathPart;
-			functionName = funcPart;
+			const colonIndex = fileRef.indexOf(":");
+			filePath = fileRef.slice(0, colonIndex);
+			functionName = fileRef.slice(colonIndex + 1);
 		}
 		filePath = path.resolve(basePath, filePath);
 		if (isJavascriptFile(filePath)) {
@@ -3899,7 +3899,7 @@ async function runEval({ provider, prompt, test, testSuite, delay, nunjucksFilte
 			promptIdx,
 			testIdx
 		});
-		logger_default.error("Provider call failed during eval", logContext);
+		if (!(err instanceof Error && err.name === "AbortError")) logger_default.error("Provider call failed during eval", logContext);
 		return [{
 			...setup,
 			error: errorWithStack,
@@ -4080,11 +4080,17 @@ var Evaluator = class {
 		let globalTimeout;
 		let globalAbortController;
 		const processedIndices = /* @__PURE__ */ new Set();
+		let targetUnavailable = false;
+		let targetErrorStatus;
+		const targetErrorAbortController = new AbortController();
 		let ciProgressReporter = null;
 		let progressBarManager = null;
+		let providerAbortSignal = options.abortSignal;
+		let combinedAbortSignal = options.abortSignal ? AbortSignal.any([options.abortSignal, targetErrorAbortController.signal]) : targetErrorAbortController.signal;
 		if (maxEvalTimeMs > 0) {
 			globalAbortController = new AbortController();
-			options.abortSignal = options.abortSignal ? AbortSignal.any([options.abortSignal, globalAbortController.signal]) : globalAbortController.signal;
+			providerAbortSignal = providerAbortSignal ? AbortSignal.any([providerAbortSignal, globalAbortController.signal]) : globalAbortController.signal;
+			combinedAbortSignal = AbortSignal.any([combinedAbortSignal, globalAbortController.signal]);
 			globalTimeout = setTimeout(() => {
 				evalTimedOut = true;
 				globalAbortController?.abort();
@@ -4092,7 +4098,7 @@ var Evaluator = class {
 		}
 		const vars = /* @__PURE__ */ new Set();
 		const checkAbort = () => {
-			if (options.abortSignal?.aborted) throw new Error("Operation cancelled");
+			if (combinedAbortSignal.aborted) throw new Error("Operation cancelled");
 		};
 		if (!options.silent) logger_default.info(`Starting evaluation ${this.evalRecord.id}`);
 		checkAbort();
@@ -4253,7 +4259,7 @@ var Evaluator = class {
 				const defaultProvider = testSuite.defaultTest.provider;
 				if (isApiProvider(defaultProvider)) testCase.provider = defaultProvider;
 				else if (typeof defaultProvider === "object" && defaultProvider.id) {
-					const { loadApiProvider } = await import("./providers-D_tJatIk.js");
+					const { loadApiProvider } = await import("./providers-CQQrNaJk.js");
 					testCase.provider = await loadApiProvider(typeof defaultProvider.id === "function" ? defaultProvider.id() : defaultProvider.id, { options: defaultProvider });
 				} else testCase.provider = defaultProvider;
 			}
@@ -4323,7 +4329,7 @@ var Evaluator = class {
 							registers: this.registers,
 							isRedteam: testSuite.redteam != null,
 							concurrency,
-							abortSignal: options.abortSignal,
+							abortSignal: providerAbortSignal,
 							evalId: this.evalRecord.id,
 							rateLimitRegistry: this.rateLimitRegistry
 						});
@@ -4337,7 +4343,7 @@ var Evaluator = class {
 			if (evalOption.test.assert?.some((a) => a.type === "max-score")) rowsWithMaxScoreAssertion.add(evalOption.testIdx);
 		}
 		if (cliState_default.resume && this.evalRecord.persisted) try {
-			const { default: EvalResult } = await import("./evalResult-DGx1-Ela.js");
+			const { default: EvalResult } = await import("./evalResult-DyttNQ_G.js");
 			const completedPairs = await EvalResult.getCompletedIndexPairs(this.evalRecord.id, { excludeErrors: cliState_default.retryMode });
 			const originalCount = runEvalOptions.length;
 			for (let i = runEvalOptions.length - 1; i >= 0; i--) {
@@ -4390,6 +4396,14 @@ var Evaluator = class {
 					logger_default.error(`Error saving result: ${error} ${safeJsonStringify(resultSummary)}`);
 				}
 				for (const writer of this.fileWriters) await writer.write(row);
+				const httpStatus = row.response?.metadata?.http?.status;
+				if (typeof httpStatus === "number" && isNonTransientHttpStatus(httpStatus)) {
+					targetUnavailable = true;
+					targetErrorStatus = httpStatus;
+					logger_default.error(`Target returned HTTP ${httpStatus}. Aborting scan - this error will not resolve on retry.`);
+					targetErrorAbortController.abort();
+					break;
+				}
 				const { promptIdx } = row;
 				const metrics = prompts[promptIdx].metrics;
 				invariant(metrics, "Expected prompt.metrics to be set");
@@ -4553,6 +4567,7 @@ var Evaluator = class {
 		if (this.options.showProgressBar && progressBarManager) await progressBarManager.initialize(runEvalOptions, concurrency, 0);
 		try {
 			if (serialRunEvalOptions.length > 0) for (const evalStep of serialRunEvalOptions) {
+				checkAbort();
 				if (isWebUI) {
 					const provider = evalStep.provider.label || evalStep.provider.id();
 					const vars = formatVarsForDisplay(evalStep.test.vars || {}, 50);
@@ -4570,22 +4585,32 @@ var Evaluator = class {
 				await this.evalRecord.addPrompts(prompts);
 			});
 		} catch (err) {
-			if (options.abortSignal?.aborted) if (evalTimedOut) logger_default.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
-			else {
-				logger_default.info("Evaluation interrupted, saving progress...");
-				if (globalTimeout) clearTimeout(globalTimeout);
-				if (progressBarManager) progressBarManager.stop();
-				if (ciProgressReporter) ciProgressReporter.finish();
-				this.evalRecord.setVars(Array.from(vars));
-				await this.evalRecord.addPrompts(prompts);
-				updateSignalFile(this.evalRecord.id);
-				return this.evalRecord;
-			}
-			else {
+			if (combinedAbortSignal.aborted) {
+				if (evalTimedOut) logger_default.warn(`Evaluation stopped after reaching max duration (${maxEvalTimeMs}ms)`);
+				else if (!targetUnavailable) {
+					logger_default.info("Evaluation interrupted, saving progress...");
+					if (globalTimeout) clearTimeout(globalTimeout);
+					if (progressBarManager) progressBarManager.stop();
+					if (ciProgressReporter) ciProgressReporter.finish();
+					this.evalRecord.setVars(Array.from(vars));
+					await this.evalRecord.addPrompts(prompts);
+					updateSignalFile(this.evalRecord.id);
+					return this.evalRecord;
+				}
+			} else {
 				if (ciProgressReporter) ciProgressReporter.error(`Evaluation failed: ${String(err)}`);
 				throw err;
 			}
 		}
+		if (targetUnavailable) {
+			if (globalTimeout) clearTimeout(globalTimeout);
+			if (progressBarManager) progressBarManager.stop();
+			if (ciProgressReporter) ciProgressReporter.error(`Target unavailable (HTTP ${targetErrorStatus})`);
+			this.evalRecord.setVars(Array.from(vars));
+			await this.evalRecord.addPrompts(prompts);
+			updateSignalFile(this.evalRecord.id);
+			return this.evalRecord;
+		}
 		const compareRowsCount = rowsWithSelectBestAssertion.size + rowsWithMaxScoreAssertion.size;
 		if (progressBarManager) {
 			if (compareRowsCount > 0) progressBarManager.updateTotalCount(compareRowsCount);
@@ -5362,7 +5387,12 @@ function convertResultsToTable(eval_) {
 				model: result.response.video.model,
 				aspectRatio: result.response.video.aspectRatio,
 				resolution: result.response.video.resolution
-			} : void 0
+			} : void 0,
+			images: result.response?.images?.map((img) => ({
+				data: img.data,
+				blobRef: img.blobRef,
+				mimeType: img.mimeType
+			}))
 		};
 		invariant(result.promptId, "Prompt ID is required");
 		row.testIdx = result.testIdx;
@@ -5428,7 +5458,12 @@ function convertEvalResultToTableCell(result) {
 			model: result.response.video.model,
 			aspectRatio: result.response.video.aspectRatio,
 			resolution: result.response.video.resolution
-		} : void 0
+		} : void 0,
+		images: result.response?.images?.map((img) => ({
+			data: img.data,
+			blobRef: img.blobRef,
+			mimeType: img.mimeType
+		}))
 	};
 }
 function convertTestResultsToTableRow(results, varsForHeader) {
@@ -5986,6 +6021,27 @@ var Eval = class Eval {
 	async getTotalResultRowCount() {
 		return getTotalResultRowCount(this.id);
 	}
+	/**
+	* Find a non-transient HTTP error status from evaluation results.
+	* Returns the first non-transient status (401, 403, 404, 500, 501) found, or undefined.
+	*
+	* For persisted evals: Uses efficient O(1) database query with LIMIT 1.
+	* For non-persisted evals: Falls back to scanning in-memory results.
+	*/
+	async findTargetErrorStatus() {
+		const scanInMemory = () => {
+			for (const result of this.results) {
+				const status = result.response?.metadata?.http?.status;
+				if (typeof status === "number" && isNonTransientHttpStatus(status)) return status;
+			}
+		};
+		if (!this.persisted) return scanInMemory();
+		try {
+			return getDb().select({ httpStatus: sql`CAST(json_extract(${evalResultsTable.response}, '$.metadata.http.status') AS INTEGER)` }).from(evalResultsTable).where(and(eq(evalResultsTable.evalId, this.id), sql`json_extract(${evalResultsTable.response}, '$.metadata.http.status') IN (${sql.join(NON_TRANSIENT_HTTP_STATUSES.map((s) => sql`${s}`), sql`, `)})`)).limit(1).get()?.httpStatus ?? void 0;
+		} catch {
+			return scanInMemory();
+		}
+	}
 	async fetchResultsByTestIdx(testIdx) {
 		return await EvalResult.findManyByEvalId(this.id, { testIdx });
 	}
@@ -9666,6 +9722,7 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 		seen.add(key);
 		return true;
 	});
+	const needsGoalExtraction = strategies.some((s) => Strategies.find((def) => def.id === s.id)?.requiresGoalExtraction);
 	await validateStrategies(strategies);
 	await validateSharpDependency(strategies, plugins);
 	const redteamProvider = await redteamProviderManager.getProvider({ provider });
@@ -9857,13 +9914,15 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 			if (!Array.isArray(allPluginTests) || allPluginTests.length === 0) logger_default.warn(`Failed to generate tests for ${plugin.id}`);
 			else {
 				const testCasesWithMetadata = allPluginTests;
-				logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} tests from ${plugin.id}...`);
-				for (const testCase of testCasesWithMetadata) {
-					const promptVar = testCase.vars?.[injectVar];
-					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
-					const policy = getPolicyText(testCase.metadata);
-					const extractedGoal = await extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
-					testCase.metadata.goal = extractedGoal;
+				if (needsGoalExtraction) {
+					logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} tests from ${plugin.id}...`);
+					for (const testCase of testCasesWithMetadata) {
+						const promptVar = testCase.vars?.[injectVar];
+						const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
+						const policy = getPolicyText(testCase.metadata);
+						const extractedGoal = await extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+						testCase.metadata.goal = extractedGoal;
+					}
 				}
 				testCases.push(...testCasesWithMetadata);
 			}
@@ -9898,13 +9957,15 @@ async function synthesize({ abortSignal, delay, entities: entitiesOverride, inje
 					...t.metadata || {}
 				}
 			}));
-			logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
-			for (const testCase of testCasesWithMetadata) {
-				const promptVar = testCase.vars?.[injectVar];
-				const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
-				const policy = getPolicyText(testCase.metadata);
-				const extractedGoal = await extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
-				testCase.metadata.goal = extractedGoal;
+			if (needsGoalExtraction) {
+				logger_default.debug(`Extracting goal for ${testCasesWithMetadata.length} custom tests from ${plugin.id}...`);
+				for (const testCase of testCasesWithMetadata) {
+					const promptVar = testCase.vars?.[injectVar];
+					const prompt = Array.isArray(promptVar) ? promptVar[0] : String(promptVar);
+					const policy = getPolicyText(testCase.metadata);
+					const extractedGoal = await extractGoalFromPrompt(prompt, purpose, plugin.id, policy);
+					testCase.metadata.goal = extractedGoal;
+				}
 			}
 			testCases.push(...testCasesWithMetadata);
 			logger_default.debug(`Added ${customTests.length} custom test cases from ${plugin.id}`);
@@ -11033,11 +11094,31 @@ function formatDuration(seconds) {
 * ```
 */
 function generateEvalSummary(params) {
-	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker } = params;
+	const { evalId, isRedteam, writeToDatabase, shareableUrl, wantsToShare, hasExplicitDisable, cloudEnabled, activelySharing = false, tokenUsage, successes, failures, errors, duration, maxConcurrency, tracker, targetErrorStatus } = params;
 	const lines = [];
 	const completionType = isRedteam ? "Red team" : "Eval";
-	const completionMessage = writeToDatabase && shareableUrl ? `${chalk.green("✓")} ${completionType} complete: ${shareableUrl}` : writeToDatabase && activelySharing ? `${chalk.green("✓")} ${completionType} complete` : writeToDatabase ? `${chalk.green("✓")} ${completionType} complete (ID: ${chalk.cyan(evalId)})` : `${chalk.green("✓")} ${completionType} complete`;
+	const wasAborted = targetErrorStatus != null;
+	let completionMessage;
+	if (wasAborted) {
+		completionMessage = `${chalk.red("✗")} ${completionType} aborted`;
+		if (writeToDatabase) completionMessage += ` (ID: ${chalk.cyan(evalId)})`;
+	} else if (writeToDatabase && shareableUrl) completionMessage = `${chalk.green("✓")} ${completionType} complete: ${shareableUrl}`;
+	else if (writeToDatabase && activelySharing) completionMessage = `${chalk.green("✓")} ${completionType} complete`;
+	else if (writeToDatabase) completionMessage = `${chalk.green("✓")} ${completionType} complete (ID: ${chalk.cyan(evalId)})`;
+	else completionMessage = `${chalk.green("✓")} ${completionType} complete`;
 	lines.push(completionMessage);
+	if (wasAborted && targetErrorStatus != null) {
+		lines.push("");
+		lines.push(chalk.red.bold("Scan stopped: Target is unavailable and will not recover on retry."));
+		lines.push(chalk.red(`  Target returned HTTP ${targetErrorStatus}`));
+		lines.push("");
+		lines.push(chalk.yellow("Possible causes:"));
+		lines.push(chalk.yellow("  • Invalid API key or authentication (401/403)"));
+		lines.push(chalk.yellow("  • Target endpoint does not exist (404)"));
+		lines.push(chalk.yellow("  • Server does not support the request (501)"));
+		lines.push("");
+		lines.push(chalk.cyan("To fix: Check your target configuration and credentials."));
+	}
 	if (writeToDatabase && !shareableUrl && !wantsToShare && !activelySharing) {
 		lines.push("");
 		lines.push(`» View results: ${chalk.green.bold("promptfoo view")}`);
@@ -11598,6 +11679,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 		const isRedteam = Boolean(config.redteam);
 		const duration = Math.round((Date.now() - startTime) / 1e3);
 		const tracker = TokenUsageTracker.getInstance();
+		const targetErrorStatus = await evalRecord.findTargetErrorStatus();
 		const summaryLines = generateEvalSummary({
 			evalId: evalRecord.id,
 			isRedteam,
@@ -11613,7 +11695,8 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			errors,
 			duration,
 			maxConcurrency,
-			tracker
+			tracker,
+			targetErrorStatus
 		});
 		if (cmdObj.write && wantsToShare && !canShareEval) {
 			logger_default.info(summaryLines[0]);
@@ -11668,7 +11751,7 @@ async function doEval(cmdObj, defaultConfig, defaultConfigPath, evaluateOptions)
 			if (initialization) {
 				const configPaths = (cmdObj.config || [defaultConfigPath]).filter(Boolean);
 				if (!configPaths.length) {
-					logger_default.error("Could not locate config file(s) to watch");
+					logger_default.error(`Could not locate config file(s) to watch. Pass --config path/to/promptfooconfig.yaml or run from a directory containing promptfooconfig.{${DEFAULT_CONFIG_EXTENSIONS.join(",")}}.`);
 					process.exitCode = 1;
 					return ret;
 				}
@@ -11880,7 +11963,7 @@ async function doRedteamRun(options) {
 		const evalMs = evalResult.evaluationDurationMs ?? 0;
 		logger_default.info(chalk.gray(`Total scan time: ${formatDuration(totalMs / 1e3)} (generation: ${formatDuration(generationDurationMs / 1e3)}, evaluation: ${formatDuration(evalMs / 1e3)})`));
 	}
-	logger_default.info(chalk.green("\nRed team scan complete!"));
+	if (evalResult ? await evalResult.findTargetErrorStatus() != null : false) {} else logger_default.info(chalk.green("\nRed team scan complete!"));
 	if (!evalResult?.shared) if (options.liveRedteamConfig) logger_default.info(chalk.blue(`To view the results, click the ${chalk.bold("View Report")} button or run ${chalk.bold(promptfooCommand("redteam report"))} on the command line.`));
 	else logger_default.info(chalk.blue(`To view the results, run ${chalk.bold(promptfooCommand("redteam report"))}`));
 	setLogCallback(null);

package/dist/src/{interactiveCheck-Dmm9vjpx.js → interactiveCheck-DU-MAhp5.js}

@@ -57,4 +57,4 @@ function shouldUseInkUI() {
 
 //#endregion
 export { shouldUseInkUI as n, canUseInteractiveUI as t };
-//# sourceMappingURL=interactiveCheck-Dmm9vjpx.js.map
+//# sourceMappingURL=interactiveCheck-DU-MAhp5.js.map