promptfoo 0.119.13 → 0.119.14

package/dist/src/providers/webSearchUtils.d.ts

@@ -0,0 +1,17 @@
+import type { ApiProvider } from '../types/index';
+/**
+ * Check if a provider has web search capabilities
+ * @param provider The provider to check
+ * @returns true if the provider supports web search
+ */
+export declare function hasWebSearchCapability(provider: ApiProvider | null | undefined): boolean;
+/**
+ * Load a provider with web search capabilities.
+ * Tries multiple providers in order of preference until one succeeds.
+ * Uses the latest and most capable models from each provider with specific checkpoint IDs.
+ *
+ * @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+ * @returns A provider with web search capabilities or null
+ */
+export declare function loadWebSearchProvider(preferAnthropic?: boolean): Promise<ApiProvider | null>;
+//# sourceMappingURL=webSearchUtils.d.ts.map

package/dist/src/providers/webSearchUtils.js

@@ -0,0 +1,169 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.hasWebSearchCapability = hasWebSearchCapability;
+exports.loadWebSearchProvider = loadWebSearchProvider;
+const logger_1 = __importDefault(require("../logger"));
+const index_1 = require("../providers/index");
+/**
+ * Check if a provider has web search capabilities
+ * @param provider The provider to check
+ * @returns true if the provider supports web search
+ */
+function hasWebSearchCapability(provider) {
+    if (!provider) {
+        return false;
+    }
+    const id = provider.id();
+    // Perplexity has built-in web search
+    if (id.includes('perplexity')) {
+        return true;
+    }
+    // Check for Google/Gemini with search tools
+    if ((id.includes('google') || id.includes('gemini') || id.includes('vertex')) &&
+        provider.config?.tools?.some((t) => t.googleSearch !== undefined)) {
+        return true;
+    }
+    // Check for xAI with search parameters
+    if (id.includes('xai') && provider.config?.search_parameters?.mode === 'on') {
+        return true;
+    }
+    // Check for OpenAI responses API with web_search_preview tool
+    if (id.includes('openai:responses') &&
+        provider.config?.tools?.some((t) => t.type === 'web_search_preview')) {
+        return true;
+    }
+    // Check for Anthropic with web_search tool
+    if (id.includes('anthropic') &&
+        provider.config?.tools?.some((t) => t.type === 'web_search_20250305')) {
+        return true;
+    }
+    return false;
+}
+/**
+ * Load a provider with web search capabilities.
+ * Tries multiple providers in order of preference until one succeeds.
+ * Uses the latest and most capable models from each provider with specific checkpoint IDs.
+ *
+ * @param preferAnthropic Whether to try Anthropic first (true) or OpenAI first (false)
+ * @returns A provider with web search capabilities or null
+ */
+async function loadWebSearchProvider(preferAnthropic = false) {
+    // Anthropic Claude 4.5 Opus (November 2024 checkpoint) with web search tool
+    const loadAnthropicWebSearch = async () => {
+        try {
+            return await (0, index_1.loadApiProvider)('anthropic:messages:claude-opus-4-5-20251101', {
+                options: {
+                    config: {
+                        tools: [
+                            {
+                                type: 'web_search_20250305',
+                                name: 'web_search',
+                                max_uses: 5,
+                            },
+                        ],
+                    },
+                },
+            });
+        }
+        catch (err) {
+            logger_1.default.debug(`Failed to load Anthropic web search provider: ${err}`);
+            return null;
+        }
+    };
+    // OpenAI GPT-5.1 with web search tool (via responses API)
+    const loadOpenAIWebSearch = async () => {
+        try {
+            return await (0, index_1.loadApiProvider)('openai:responses:gpt-5.1', {
+                options: {
+                    config: { tools: [{ type: 'web_search_preview' }] },
+                },
+            });
+        }
+        catch (err) {
+            logger_1.default.debug(`Failed to load OpenAI web search provider: ${err}`);
+            return null;
+        }
+    };
+    // Perplexity Sonar Pro (built-in web search)
+    const loadPerplexity = async () => {
+        try {
+            return await (0, index_1.loadApiProvider)('perplexity:sonar-pro');
+        }
+        catch (err) {
+            logger_1.default.debug(`Failed to load Perplexity provider: ${err}`);
+            return null;
+        }
+    };
+    // Google Gemini 3 Pro Preview with googleSearch tool
+    const loadGoogleWebSearch = async () => {
+        try {
+            return await (0, index_1.loadApiProvider)('google:gemini-3-pro-preview', {
+                options: {
+                    config: { tools: [{ googleSearch: {} }] },
+                },
+            });
+        }
+        catch (err) {
+            logger_1.default.debug(`Failed to load Google web search provider: ${err}`);
+            return null;
+        }
+    };
+    // Vertex AI Gemini 3 Pro Preview with googleSearch tool
+    const loadVertexWebSearch = async () => {
+        try {
+            return await (0, index_1.loadApiProvider)('vertex:gemini-3-pro-preview', {
+                options: {
+                    config: { tools: [{ googleSearch: {} }] },
+                },
+            });
+        }
+        catch (err) {
+            logger_1.default.debug(`Failed to load Vertex web search provider: ${err}`);
+            return null;
+        }
+    };
+    // xAI Grok 4.1 Fast Reasoning with live web search
+    const loadXaiWebSearch = async () => {
+        try {
+            return await (0, index_1.loadApiProvider)('xai:grok-4-1-fast-reasoning', {
+                options: {
+                    config: { search_parameters: { mode: 'on' } },
+                },
+            });
+        }
+        catch (err) {
+            logger_1.default.debug(`Failed to load xAI web search provider: ${err}`);
+            return null;
+        }
+    };
+    // Order providers based on preference
+    const providers = preferAnthropic
+        ? [
+            loadAnthropicWebSearch,
+            loadOpenAIWebSearch,
+            loadPerplexity,
+            loadGoogleWebSearch,
+            loadVertexWebSearch,
+            loadXaiWebSearch,
+        ]
+        : [
+            loadOpenAIWebSearch,
+            loadAnthropicWebSearch,
+            loadPerplexity,
+            loadGoogleWebSearch,
+            loadVertexWebSearch,
+            loadXaiWebSearch,
+        ];
+    for (const getProvider of providers) {
+        const provider = await getProvider();
+        if (provider) {
+            logger_1.default.info(`Using ${provider.id()} as web search provider`);
+            return provider;
+        }
+    }
+    return null;
+}
+//# sourceMappingURL=webSearchUtils.js.map

package/dist/src/providers/xai/chat.d.ts

@@ -1,9 +1,70 @@
 import type { ApiProvider, ProviderOptions } from '../../types/index';
 import type { OpenAiCompletionOptions } from '../openai/types';
+/**
+ * xAI Agent Tools API - Server-side tool types
+ *
+ * TODO: These interfaces are defined for future use when the xAI Responses API
+ * is supported. Currently, Agent Tools (web_search, x_search, code_execution,
+ * collections_search, mcp) require the Responses API endpoint which is different
+ * from the Chat Completions API we currently use.
+ *
+ * For now, use `search_parameters` (Live Search) for real-time web/X search,
+ * which works with the Chat Completions API.
+ *
+ * See: https://docs.x.ai/docs/tools/overview
+ */
+export interface XAIWebSearchTool {
+    type: 'web_search';
+    filters?: {
+        /** Only search within these domains (max 5). Cannot be used with excluded_domains. */
+        allowed_domains?: string[];
+        /** Exclude these domains from search (max 5). Cannot be used with allowed_domains. */
+        excluded_domains?: string[];
+    };
+    /** Enable the model to view and analyze images encountered during search */
+    enable_image_understanding?: boolean;
+}
+export interface XAIXSearchTool {
+    type: 'x_search';
+    /** Only consider posts from these X handles (max 10). Cannot be used with excluded_x_handles. */
+    allowed_x_handles?: string[];
+    /** Exclude posts from these X handles (max 10). Cannot be used with allowed_x_handles. */
+    excluded_x_handles?: string[];
+    /** Start date for search results (ISO8601 format: YYYY-MM-DD) */
+    from_date?: string;
+    /** End date for search results (ISO8601 format: YYYY-MM-DD) */
+    to_date?: string;
+    /** Enable the model to view and analyze images in X posts */
+    enable_image_understanding?: boolean;
+    /** Enable the model to view and analyze videos in X posts */
+    enable_video_understanding?: boolean;
+}
+export interface XAICodeExecutionTool {
+    type: 'code_execution' | 'code_interpreter';
+}
+export interface XAICollectionsSearchTool {
+    type: 'collections_search' | 'file_search';
+    /** Collection IDs to search within */
+    collection_ids?: string[];
+}
+export interface XAIMCPTool {
+    type: 'mcp';
+    /** URL of the MCP server */
+    server_url: string;
+    /** Optional label for the MCP server */
+    server_label?: string;
+    /** Headers to send with MCP requests */
+    headers?: Record<string, string>;
+    /** Allowed tools from this MCP server */
+    allowed_tools?: string[];
+}
+export type XAIAgentTool = XAIWebSearchTool | XAIXSearchTool | XAICodeExecutionTool | XAICollectionsSearchTool | XAIMCPTool;
 type XAIConfig = {
     region?: string;
     reasoning_effort?: 'low' | 'high';
     search_parameters?: Record<string, any>;
+    /** xAI Agent Tools - server-side tools for agentic workflows */
+    agent_tools?: XAIAgentTool[];
 } & OpenAiCompletionOptions;
 type XAIProviderOptions = Omit<ProviderOptions, 'config'> & {
     config?: {

package/dist/src/providers/xai/chat.js

@@ -11,6 +11,24 @@ const index_1 = require("../../util/index");
 const invariant_1 = __importDefault(require("../../util/invariant"));
 const chat_1 = require("../openai/chat");
 exports.XAI_CHAT_MODELS = [
+    // Grok 4.1 Fast Models (2M context window)
+    {
+        id: 'grok-4-1-fast-reasoning',
+        cost: {
+            input: 0.2 / 1e6,
+            output: 0.5 / 1e6,
+            cache_read: 0.05 / 1e6,
+        },
+        aliases: ['grok-4-1-fast', 'grok-4-1-fast-latest'],
+    },
+    {
+        id: 'grok-4-1-fast-non-reasoning',
+        cost: {
+            input: 0.2 / 1e6,
+            output: 0.5 / 1e6,
+            cache_read: 0.05 / 1e6,
+        },
+    },
     // Grok Code Fast Models
     {
         id: 'grok-code-fast-1',
@@ -29,6 +47,24 @@ exports.XAI_CHAT_MODELS = [
             cache_read: 0.02 / 1e6,
         },
     },
+    // Grok-4 Fast Models (2M context window)
+    {
+        id: 'grok-4-fast-reasoning',
+        cost: {
+            input: 0.2 / 1e6,
+            output: 0.5 / 1e6,
+            cache_read: 0.05 / 1e6,
+        },
+        aliases: ['grok-4-fast', 'grok-4-fast-latest'],
+    },
+    {
+        id: 'grok-4-fast-non-reasoning',
+        cost: {
+            input: 0.2 / 1e6,
+            output: 0.5 / 1e6,
+            cache_read: 0.05 / 1e6,
+        },
+    },
     // Grok-4 Models
     {
         id: 'grok-4-0709',
@@ -123,12 +159,23 @@ exports.GROK_REASONING_EFFORT_MODELS = [
 ];
 // All reasoning models (including Grok-4 which doesn't support reasoning_effort)
 exports.GROK_REASONING_MODELS = [
+    // Grok 4.1 Fast reasoning
+    'grok-4-1-fast-reasoning',
+    'grok-4-1-fast',
+    'grok-4-1-fast-latest',
+    // Grok Code Fast
     'grok-code-fast-1',
     'grok-code-fast',
     'grok-code-fast-1-0825',
+    // Grok 4 Fast reasoning
+    'grok-4-fast-reasoning',
+    'grok-4-fast',
+    'grok-4-fast-latest',
+    // Grok 4
     'grok-4-0709',
     'grok-4',
     'grok-4-latest',
+    // Grok 3 mini
     'grok-3-mini-beta',
     'grok-3-mini',
     'grok-3-mini-latest',
@@ -136,8 +183,23 @@ exports.GROK_REASONING_MODELS = [
     'grok-3-mini-fast',
     'grok-3-mini-fast-latest',
 ];
-// Grok-4 models that have specific parameter restrictions
-exports.GROK_4_MODELS = ['grok-4-0709', 'grok-4', 'grok-4-latest'];
+// Grok-4+ models that have specific parameter restrictions (no presence_penalty, frequency_penalty, stop, reasoning_effort)
+exports.GROK_4_MODELS = [
+    // Grok 4.1 Fast
+    'grok-4-1-fast-reasoning',
+    'grok-4-1-fast',
+    'grok-4-1-fast-latest',
+    'grok-4-1-fast-non-reasoning',
+    // Grok 4 Fast
+    'grok-4-fast-reasoning',
+    'grok-4-fast',
+    'grok-4-fast-latest',
+    'grok-4-fast-non-reasoning',
+    // Grok 4
+    'grok-4-0709',
+    'grok-4',
+    'grok-4-latest',
+];
 /**
  * Calculate xAI Grok cost based on model name and token usage
  */
@@ -190,11 +252,14 @@ class XAIProvider extends chat_1.OpenAiChatCompletionProvider {
         if (!this.supportsReasoningEffort() && result.body.reasoning_effort) {
             delete result.body.reasoning_effort;
         }
-        // Handle search parameters
+        // Handle search parameters (Live Search)
         const searchParams = this.originalConfig?.search_parameters;
         if (searchParams) {
             result.body.search_parameters = (0, index_1.renderVarsInObject)(searchParams, context?.vars);
         }
+        // Note: xAI Agent Tools (web_search, x_search, code_execution, etc.) require
+        // the Responses API endpoint which is not yet implemented. For now, use
+        // search_parameters for live search functionality.
         return result;
     }
     constructor(modelName, providerOptions) {

package/dist/src/providers/xai/responses.d.ts

@@ -0,0 +1,189 @@
+import type { ApiProvider, CallApiContextParams, CallApiOptionsParams, ProviderOptions, ProviderResponse } from '../../types/index';
+import type { EnvOverrides } from '../../types/env';
+/**
+ * xAI Agent Tools - Server-side tools for autonomous agent workflows
+ * These tools run on xAI's infrastructure and enable the model to:
+ * - Search the web (web_search) and X (x_search) in real-time
+ * - Execute Python code (code_interpreter)
+ * - Search uploaded collections (collections_search)
+ * - Connect to MCP servers (mcp)
+ */
+export interface XAIWebSearchTool {
+    type: 'web_search';
+    /** Domain filters for web search */
+    filters?: {
+        allowed_domains?: string[];
+        excluded_domains?: string[];
+    };
+    /** Enable image understanding during web browsing */
+    enable_image_understanding?: boolean;
+}
+export interface XAIXSearchTool {
+    type: 'x_search';
+    /** X handle filters */
+    allowed_x_handles?: string[];
+    excluded_x_handles?: string[];
+    /** Date range for search (ISO8601 format: YYYY-MM-DD) */
+    from_date?: string;
+    to_date?: string;
+    /** Enable multimodal understanding */
+    enable_image_understanding?: boolean;
+    enable_video_understanding?: boolean;
+}
+export interface XAICodeInterpreterTool {
+    type: 'code_interpreter';
+    /** Container configuration */
+    container?: {
+        /** Pre-installed pip packages */
+        pip_packages?: string[];
+    };
+}
+export interface XAICollectionsSearchTool {
+    type: 'collections_search';
+    /** Collection IDs to search */
+    collection_ids?: string[];
+}
+export interface XAIMCPTool {
+    type: 'mcp';
+    /** MCP server URL */
+    server_url: string;
+    /** Optional server label */
+    server_label?: string;
+    /** Headers for MCP requests */
+    headers?: Record<string, string>;
+    /** Allowed tools from this server */
+    allowed_tools?: string[];
+}
+export type XAIAgentTool = XAIWebSearchTool | XAIXSearchTool | XAICodeInterpreterTool | XAICollectionsSearchTool | XAIMCPTool;
+export interface XAIResponsesConfig {
+    /** API key (defaults to XAI_API_KEY env var) */
+    apiKey?: string;
+    /** API base URL (defaults to https://api.x.ai/v1) */
+    apiBaseUrl?: string;
+    /** Region for regional endpoints (e.g., 'us-west-1') */
+    region?: string;
+    /** Temperature (0-2) */
+    temperature?: number;
+    /** Top P sampling */
+    top_p?: number;
+    /** Maximum output tokens */
+    max_output_tokens?: number;
+    /** System instructions */
+    instructions?: string;
+    /** Response format (json_object, json_schema, or text) */
+    response_format?: any;
+    /** xAI Agent Tools (server-side tools) */
+    tools?: (XAIAgentTool | any)[];
+    /** Tool choice configuration */
+    tool_choice?: 'auto' | 'required' | 'none' | {
+        type: 'function';
+        function: {
+            name: string;
+        };
+    };
+    /** Enable parallel tool calls */
+    parallel_tool_calls?: boolean;
+    /** Store response for later retrieval */
+    store?: boolean;
+    /** Previous response ID for multi-turn conversations */
+    previous_response_id?: string;
+    /** User identifier */
+    user?: string;
+    /** Custom headers */
+    headers?: Record<string, string>;
+    /** Maximum retries */
+    maxRetries?: number;
+    /** Function callbacks for client-side function tools */
+    functionToolCallbacks?: Record<string, (args: string) => Promise<string>>;
+    /** Passthrough parameters */
+    passthrough?: Record<string, any>;
+}
+/**
+ * xAI Responses API Provider
+ *
+ * Supports xAI's Responses API with Agent Tools for autonomous agent workflows.
+ * This enables Grok models to autonomously search the web, X, execute code,
+ * and interact with MCP servers.
+ *
+ * Usage:
+ *   xai:responses:grok-4-1-fast-reasoning
+ *   xai:responses:grok-4-fast
+ *   xai:responses:grok-4
+ */
+export declare class XAIResponsesProvider implements ApiProvider {
+    modelName: string;
+    config: XAIResponsesConfig;
+    env?: EnvOverrides;
+    private functionCallbackHandler;
+    private processor;
+    constructor(modelName: string, options?: {
+        config?: XAIResponsesConfig;
+        id?: string;
+        env?: EnvOverrides;
+    });
+    id(): string;
+    toString(): string;
+    toJSON(): {
+        provider: string;
+        model: string;
+        config: {
+            apiKey: undefined;
+            /** API base URL (defaults to https://api.x.ai/v1) */
+            apiBaseUrl?: string;
+            /** Region for regional endpoints (e.g., 'us-west-1') */
+            region?: string;
+            /** Temperature (0-2) */
+            temperature?: number;
+            /** Top P sampling */
+            top_p?: number;
+            /** Maximum output tokens */
+            max_output_tokens?: number;
+            /** System instructions */
+            instructions?: string;
+            /** Response format (json_object, json_schema, or text) */
+            response_format?: any;
+            /** xAI Agent Tools (server-side tools) */
+            tools?: (XAIAgentTool | any)[];
+            /** Tool choice configuration */
+            tool_choice?: "auto" | "required" | "none" | {
+                type: "function";
+                function: {
+                    name: string;
+                };
+            };
+            /** Enable parallel tool calls */
+            parallel_tool_calls?: boolean;
+            /** Store response for later retrieval */
+            store?: boolean;
+            /** Previous response ID for multi-turn conversations */
+            previous_response_id?: string;
+            /** User identifier */
+            user?: string;
+            /** Custom headers */
+            headers?: Record<string, string>;
+            /** Maximum retries */
+            maxRetries?: number;
+            /** Function callbacks for client-side function tools */
+            functionToolCallbacks?: Record<string, (args: string) => Promise<string>>;
+            /** Passthrough parameters */
+            passthrough?: Record<string, any>;
+        };
+    };
+    protected getApiKey(): string | undefined;
+    protected getApiUrl(): string;
+    getRequestBody(prompt: string, context?: CallApiContextParams, _callApiOptions?: CallApiOptionsParams): Promise<{
+        body: Record<string, any>;
+        config: any;
+    }>;
+    callApi(prompt: string, context?: CallApiContextParams, callApiOptions?: CallApiOptionsParams): Promise<ProviderResponse>;
+    private getTokenUsage;
+}
+/**
+ * Create an xAI Responses provider
+ *
+ * @param providerPath - Provider path in format xai:responses:<model>
+ * @param options - Provider options
+ * @returns XAIResponsesProvider instance
+ */
+export declare function createXAIResponsesProvider(providerPath: string, options?: ProviderOptions): ApiProvider;
+//# sourceMappingURL=responses.d.ts.map