promptfoo 0.119.13 → 0.119.14

package/dist/src/envars.d.ts

@@ -107,6 +107,22 @@ type EnvVars = {
     REQUEST_TIMEOUT_MS?: number;
     RESULT_HISTORY_LENGTH?: number;
     WEBHOOK_TIMEOUT?: number;
+    /**
+     * Default timeout in milliseconds for MCP tool calls.
+     * This overrides the MCP SDK's default 60-second timeout.
+     * Can be overridden per-provider via config.mcp.timeout.
+     */
+    MCP_REQUEST_TIMEOUT_MS?: number;
+    /**
+     * Enable debug logging for MCP connections.
+     * Can be overridden per-provider via config.mcp.debug.
+     */
+    MCP_DEBUG?: boolean;
+    /**
+     * Enable verbose output for MCP connections.
+     * Can be overridden per-provider via config.mcp.verbose.
+     */
+    MCP_VERBOSE?: boolean;
     PROMPTFOO_POSTHOG_KEY?: string;
     PROMPTFOO_POSTHOG_HOST?: string;
     /**
@@ -208,6 +224,7 @@ type EnvVars = {
     OPENAI_STOP?: string;
     OPENAI_TEMPERATURE?: number;
     OPENAI_TOP_P?: number;
+    CODEX_API_KEY?: string;
     OPENROUTER_API_KEY?: string;
     PORTKEY_API_BASE_URL?: string;
     PORTKEY_API_KEY?: string;

package/dist/src/generated/constants.js

@@ -2,6 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.POSTHOG_KEY = void 0;
 // This file is auto-generated during build. Do not edit manually.
-// Generated at: 2025-11-25T16:43:03.753Z
+// Generated at: 2025-12-01T18:11:57.753Z
 exports.POSTHOG_KEY = 'phc_E5n5uHnDo2eREJL1uqX1cIlbkoRby4yFWt3V94HqRRg';
 //# sourceMappingURL=constants.js.map

package/dist/src/logger.d.ts

@@ -73,5 +73,10 @@ export declare function logRequestResponse(options: {
     response?: Response | null;
     error?: boolean;
 }): Promise<void>;
+/**
+ * Close all file transports and cleanup logger resources
+ * Should be called during graceful shutdown to prevent event loop hanging
+ */
+export declare function closeLogger(): void;
 export default logger;
 //# sourceMappingURL=logger.d.ts.map

package/dist/src/logger.js

@@ -47,6 +47,7 @@ exports.isDebugEnabled = isDebugEnabled;
 exports.initializeRunLogging = initializeRunLogging;
 exports.setLogger = setLogger;
 exports.logRequestResponse = logRequestResponse;
+exports.closeLogger = closeLogger;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const chalk_1 = __importDefault(require("chalk"));
@@ -389,6 +390,33 @@ async function logRequestResponse(options) {
         logMethod(`Api Request`, logObject);
     }
 }
+/**
+ * Close all file transports and cleanup logger resources
+ * Should be called during graceful shutdown to prevent event loop hanging
+ */
+function closeLogger() {
+    try {
+        // Close all file transports
+        const fileTransports = exports.winstonLogger.transports.filter((transport) => transport instanceof winston_1.default.transports.File);
+        for (const transport of fileTransports) {
+            const filename = transport.filename;
+            if (filename) {
+                logger.debug(`Closing log file: ${filename}`);
+            }
+            if (typeof transport.close === 'function') {
+                transport.close();
+            }
+            exports.winstonLogger.remove(transport);
+        }
+        if (fileTransports.length > 0) {
+            logger.debug('Logger cleanup complete');
+        }
+    }
+    catch (error) {
+        // Can't use logger here since we're shutting it down
+        console.error(`Error closing logger: ${error}`);
+    }
+}
 // Initialize source maps if debug is enabled at startup
 if ((0, envars_1.getEnvString)('LOG_LEVEL', 'info') === 'debug') {
     initializeSourceMapSupport();

package/dist/src/main.js

@@ -39,12 +39,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.addCommonOptionsRecursively = addCommonOptionsRecursively;
 const commander_1 = require("commander");
+const undici_1 = require("undici");
 const package_json_1 = require("../package.json");
 const checkNodeVersion_1 = require("./checkNodeVersion");
 const cliState_1 = __importDefault(require("./cliState"));
+const index_1 = require("./codeScan/index");
 const auth_1 = require("./commands/auth");
 const cache_1 = require("./commands/cache");
-const index_1 = require("./codeScan/index");
 const config_1 = require("./commands/config");
 const debug_1 = require("./commands/debug");
 const delete_1 = require("./commands/delete");
@@ -63,6 +64,7 @@ const share_1 = require("./commands/share");
 const show_1 = require("./commands/show");
 const validate_1 = require("./commands/validate");
 const view_1 = require("./commands/view");
+const index_3 = require("./database/index");
 const logger_1 = __importStar(require("./logger"));
 const migrate_1 = require("./migrate");
 const discover_1 = require("./redteam/commands/discover");
@@ -76,8 +78,8 @@ const simba_1 = require("./redteam/commands/simba");
 const telemetry_1 = __importDefault(require("./telemetry"));
 const updates_1 = require("./updates");
 const default_1 = require("./util/config/default");
-const index_3 = require("./util/errors/index");
-const index_4 = require("./util/index");
+const index_4 = require("./util/errors/index");
+const index_5 = require("./util/index");
 /**
  * Adds verbose and env-file options to all commands recursively
  */
@@ -97,7 +99,7 @@ function addCommonOptionsRecursively(command) {
         }
         const envPath = thisCommand.opts().envFile || thisCommand.opts().envPath;
         if (envPath) {
-            (0, index_4.setupEnv)(envPath);
+            (0, index_5.setupEnv)(envPath);
             logger_1.default.debug(`Loading environment from ${envPath}`);
         }
     });
@@ -159,12 +161,12 @@ async function main() {
     // Add common options to all commands recursively
     addCommonOptionsRecursively(program);
     program.hook('postAction', async () => {
-        (0, index_3.printErrorInformation)(cliState_1.default.errorLogFile, cliState_1.default.debugLogFile);
+        (0, index_4.printErrorInformation)(cliState_1.default.errorLogFile, cliState_1.default.debugLogFile);
         if (cliState_1.default.postActionCallback) {
             await cliState_1.default.postActionCallback();
         }
     });
-    program.parse();
+    await program.parseAsync();
 }
 if (require.main === module) {
     (0, checkNodeVersion_1.checkNodeVersion)();
@@ -172,6 +174,15 @@ if (require.main === module) {
         logger_1.default.debug('Shutting down gracefully...');
         await telemetry_1.default.shutdown();
         logger_1.default.debug('Shutdown complete');
+        (0, logger_1.closeLogger)();
+        (0, index_3.closeDbIfOpen)();
+        try {
+            const dispatcher = (0, undici_1.getGlobalDispatcher)();
+            await dispatcher.destroy();
+        }
+        catch {
+            // Silently handle dispatcher destroy errors
+        }
     });
 }
 //# sourceMappingURL=main.js.map

package/dist/src/matchers.d.ts

@@ -35,6 +35,7 @@ interface ModerationMatchOptions {
     assistantResponse: string;
     categories?: string[];
 }
+export declare function matchesSearchRubric(rubric: string, llmOutput: string, grading?: GradingConfig, vars?: Record<string, string | object>, assertion?: Assertion, _provider?: ApiProvider): Promise<GradingResult>;
 export declare function matchesModeration({ userPrompt, assistantResponse, categories }: ModerationMatchOptions, grading?: GradingConfig): Promise<{
     pass: boolean;
     score: number;

package/dist/src/matchers.js

@@ -20,6 +20,7 @@ exports.matchesContextRelevance = matchesContextRelevance;
 exports.matchesContextFaithfulness = matchesContextFaithfulness;
 exports.matchesSelectBest = matchesSelectBest;
 exports.selectMaxScore = selectMaxScore;
+exports.matchesSearchRubric = matchesSearchRubric;
 exports.matchesModeration = matchesModeration;
 const path_1 = __importDefault(require("path"));
 const utils_1 = require("./assertions/utils");
@@ -29,6 +30,8 @@ const logger_1 = __importDefault(require("./logger"));
 const index_1 = require("./prompts/index");
 const index_2 = require("./providers/index");
 const defaults_1 = require("./providers/defaults");
+const webSearchUtils_1 = require("./providers/webSearchUtils");
+const grading_1 = require("./prompts/grading");
 const constants_1 = require("./redteam/constants");
 const remoteGeneration_1 = require("./redteam/remoteGeneration");
 const remoteGrading_1 = require("./remoteGrading");
@@ -1224,6 +1227,83 @@ async function selectMaxScore(outputs, resultsWithGradingResults, assertion) {
         };
     });
 }
+async function matchesSearchRubric(rubric, llmOutput, grading, vars, assertion, _provider) {
+    if (!grading) {
+        throw new Error('Cannot grade output without grading config. Specify --grader option or grading config.');
+    }
+    // Search rubric assertion is like llm-rubric but with web search capabilities
+    const defaultProviders = await (0, defaults_1.getDefaultProviders)();
+    // Get a provider with web search capabilities
+    let searchProvider = grading.provider ||
+        defaultProviders.webSearchProvider ||
+        defaultProviders.llmRubricProvider ||
+        defaultProviders.gradingProvider;
+    // Check if current provider has web search, if not try to load one
+    if (!(0, webSearchUtils_1.hasWebSearchCapability)(searchProvider)) {
+        // Try to load a provider with web search capabilities
+        // For search-rubric assertion, prefer Anthropic first (pass true)
+        const webSearchProvider = await (0, webSearchUtils_1.loadWebSearchProvider)(true);
+        if (webSearchProvider) {
+            searchProvider = webSearchProvider;
+        }
+    }
+    // Ensure we have a provider with web search capabilities
+    if (!searchProvider || !(0, webSearchUtils_1.hasWebSearchCapability)(searchProvider)) {
+        throw new Error('search-rubric assertion requires a grading provider with web search capabilities. ' +
+            'Use --grader with a web search provider (e.g., anthropic:messages:claude-sonnet-4, openai:responses:o4-mini with tools configured, perplexity:sonar) or configure one in defaultTest.options.provider');
+    }
+    // Load the web search rubric prompt
+    const rubricPrompt = await loadRubricPrompt(grading?.rubricPrompt, grading_1.DEFAULT_WEB_SEARCH_PROMPT);
+    const prompt = await renderLlmRubricPrompt(rubricPrompt, {
+        output: tryParse(llmOutput),
+        rubric,
+        ...(vars || {}),
+    });
+    // Get the evaluation from the search provider
+    const resp = await searchProvider.callApi(prompt);
+    if (resp.error || !resp.output) {
+        return {
+            pass: false,
+            score: 0,
+            reason: `Search rubric evaluation failed: ${resp.error || 'No output'}`,
+            tokensUsed: resp.tokenUsage,
+            assertion,
+        };
+    }
+    // Parse the response
+    try {
+        const result = (0, json_1.extractFirstJsonObject)(String(resp.output));
+        // Apply threshold if specified
+        let pass = result.pass ?? false;
+        const score = typeof result.score === 'number' ? result.score : pass ? 1 : 0;
+        if (assertion?.threshold !== undefined) {
+            pass = pass && score >= assertion.threshold;
+        }
+        return {
+            pass,
+            score,
+            reason: result.reason || 'No reason provided',
+            tokensUsed: resp.tokenUsage,
+            assertion,
+            metadata: {
+                searchResults: result.searchResults || [],
+                searchProvider: searchProvider.id(),
+            },
+        };
+    }
+    catch {
+        // Try to parse as a simple pass/fail
+        const outputLower = String(resp.output).toLowerCase();
+        const pass = outputLower.includes('"pass":true') || outputLower.includes('"pass": true');
+        return {
+            pass,
+            score: pass ? 1 : 0,
+            reason: resp.output,
+            tokensUsed: resp.tokenUsage,
+            assertion,
+        };
+    }
+}
 async function matchesModeration({ userPrompt, assistantResponse, categories = [] }, grading) {
     if (!assistantResponse) {
         return {

package/dist/src/models/eval.d.ts

@@ -1,6 +1,6 @@
 import { type CompletedPrompt, type EvalSummary, type EvaluateResult, type EvaluateStats, type EvaluateSummaryV2, type EvaluateSummaryV3, type EvaluateTable, type EvaluateTableRow, type Prompt, type ResultsFile, type UnifiedConfig } from '../types/index';
 import EvalResult from './evalResult';
-import type { EvalResultsFilterMode } from '../types/index';
+import type { EvalResultsFilterMode, TraceData } from '../types/index';
 export declare function createEvalId(createdAt?: Date): string;
 /** Result from queries extracting variable keys with eval IDs */
 export interface VarKeyWithEvalIdResult {
@@ -181,6 +181,7 @@ export default class Eval {
     clearResults(): void;
     getStats(): EvaluateStats;
     toEvaluateSummary(): Promise<EvaluateSummaryV3 | EvaluateSummaryV2>;
+    getTraces(): Promise<TraceData[]>;
     toResultsFile(): Promise<ResultsFile>;
     delete(): Promise<void>;
     /**

package/dist/src/models/eval.js

@@ -17,8 +17,11 @@ const accounts_1 = require("../globalConfig/accounts");
 const logger_1 = __importDefault(require("../logger"));
 const utils_1 = require("../prompts/utils");
 const constants_2 = require("../redteam/constants");
+const metrics_1 = require("../redteam/metrics");
 const sharedFrontend_1 = require("../redteam/sharedFrontend");
+const store_1 = require("../tracing/store");
 const index_2 = require("../types/index");
+const calculateFilteredMetrics_1 = require("../util/calculateFilteredMetrics");
 const convertEvalResultsToTable_1 = require("../util/convertEvalResultsToTable");
 const createHash_1 = require("../util/createHash");
 const index_3 = require("../util/exportToFile/index");
@@ -27,8 +30,6 @@ const time_1 = require("../util/time");
 const tokenUsageUtils_1 = require("../util/tokenUsageUtils");
 const evalPerformance_1 = require("./evalPerformance");
 const evalResult_1 = __importDefault(require("./evalResult"));
-const calculateFilteredMetrics_1 = require("../util/calculateFilteredMetrics");
-const metrics_1 = require("../redteam/metrics");
 /**
  * Sanitizes runtime options to ensure only JSON-serializable data is persisted.
  * Removes non-serializable fields like AbortSignal, functions, and symbols.
@@ -813,7 +814,47 @@ class Eval {
             stats,
         };
     }
+    async getTraces() {
+        try {
+            const traceStore = (0, store_1.getTraceStore)();
+            const tracesData = await traceStore.getTracesByEvaluation(this.id);
+            // Transform trace data to match the expected schema
+            return tracesData.map((trace) => ({
+                traceId: trace.traceId,
+                evaluationId: trace.evaluationId,
+                testCaseId: trace.testCaseId,
+                metadata: trace.metadata,
+                spans: (trace.spans || []).map((span) => {
+                    // Calculate duration
+                    const durationMs = span.endTime && span.startTime ? (span.endTime - span.startTime) / 1000000 : undefined;
+                    // Map status code
+                    const statusCode = span.statusCode === 1 ? 'ok' : span.statusCode === 2 ? 'error' : 'unset';
+                    return {
+                        spanId: span.spanId,
+                        parentSpanId: span.parentSpanId,
+                        name: span.name,
+                        kind: span.kind || 'unspecified',
+                        startTime: span.startTime,
+                        endTime: span.endTime,
+                        durationMs,
+                        attributes: span.attributes || {},
+                        status: {
+                            code: statusCode,
+                            message: span.statusMessage,
+                        },
+                        depth: 0, // Will be calculated on the server side when storing
+                        events: span.events || [],
+                    };
+                }),
+            }));
+        }
+        catch (error) {
+            logger_1.default.debug(`Failed to fetch traces for eval ${this.id}: ${error}`);
+            return [];
+        }
+    }
     async toResultsFile() {
+        const traces = await this.getTraces();
         const results = {
             version: this.version(),
             createdAt: new Date(this.createdAt).toISOString(),
@@ -822,6 +863,7 @@ class Eval {
             author: this.author || null,
             prompts: this.getPrompts(),
             datasetId: this.datasetId || null,
+            ...(traces.length > 0 && { traces }),
         };
         return results;
     }

package/dist/src/prompts/grading.d.ts

@@ -13,4 +13,5 @@ export declare const SUGGEST_PROMPTS_SYSTEM_MESSAGE: {
     content: string;
 };
 export declare const SELECT_BEST_PROMPT: string;
+export declare const DEFAULT_WEB_SEARCH_PROMPT: string;
 //# sourceMappingURL=grading.d.ts.map

package/dist/src/prompts/grading.js

@@ -17,7 +17,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SELECT_BEST_PROMPT = exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = exports.OPENAI_CLOSED_QA_PROMPT = exports.PROMPTFOO_FACTUALITY_PROMPT = exports.DEFAULT_GRADING_PROMPT = void 0;
+exports.DEFAULT_WEB_SEARCH_PROMPT = exports.SELECT_BEST_PROMPT = exports.SUGGEST_PROMPTS_SYSTEM_MESSAGE = exports.OPENAI_CLOSED_QA_PROMPT = exports.PROMPTFOO_FACTUALITY_PROMPT = exports.DEFAULT_GRADING_PROMPT = void 0;
 const dedent_1 = __importDefault(require("dedent"));
 /**
  * This module exports various prompts used for model graded evaluation.
@@ -135,4 +135,29 @@ Here are the pieces of text:
 Output the index of the text that best fits the criteria. You must output a single integer.`,
     },
 ]);
+exports.DEFAULT_WEB_SEARCH_PROMPT = JSON.stringify([
+    {
+        role: 'system',
+        content: (0, dedent_1.default) `You are grading output according to a user-specified rubric, with the ability to search the web for current information. If the statement in the rubric is true, then the output passes the test. You respond with a JSON object with this structure: {reason: string, pass: boolean, score: number}
+
+    You MUST search the web when:
+    - The rubric asks about current information (prices, weather, news, etc.)
+    - Facts need to be verified against recent data
+    - The rubric references time-sensitive information
+
+    Examples:
+
+    <Output>The current CEO of Microsoft is Satya Nadella</Output>
+    <Rubric>Contains accurate information about Microsoft's leadership</Rubric>
+    {"reason": "I searched and confirmed Satya Nadella is indeed the current CEO of Microsoft", "pass": true, "score": 1.0}
+
+    <Output>Bitcoin is trading at $45,000</Output>
+    <Rubric>Provides current Bitcoin price within 10% accuracy</Rubric>
+    {"reason": "Web search shows Bitcoin is currently trading at $98,000, not $45,000. The output is off by more than 50%", "pass": false, "score": 0.0}`,
+    },
+    {
+        role: 'user',
+        content: '<Output>\n{{ output }}\n</Output>\n<Rubric>\n{{ rubric }}\n</Rubric>',
+    },
+]);
 //# sourceMappingURL=grading.js.map

package/dist/src/prompts/index.d.ts

@@ -1,5 +1,6 @@
 import type { EvaluateTestSuite, Prompt, TestSuite, UnifiedConfig } from '../types/index';
 export * from './grading';
+export { DEFAULT_WEB_SEARCH_PROMPT } from './grading';
 /**
  * Reads and maps provider prompts based on the configuration and parsed prompts.
  * @param config - The configuration object.

package/dist/src/prompts/index.js

@@ -17,7 +17,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.GEVAL_PROMPT_EVALUATE = exports.GEVAL_PROMPT_STEPS = void 0;
+exports.GEVAL_PROMPT_EVALUATE = exports.GEVAL_PROMPT_STEPS = exports.DEFAULT_WEB_SEARCH_PROMPT = void 0;
 exports.readProviderPromptMap = readProviderPromptMap;
 exports.readPrompts = readPrompts;
 exports.processPrompts = processPrompts;
@@ -41,6 +41,8 @@ const text_1 = require("./processors/text");
 const yaml_1 = require("./processors/yaml");
 const utils_1 = require("./utils");
 __exportStar(require("./grading"), exports);
+var grading_1 = require("./grading");
+Object.defineProperty(exports, "DEFAULT_WEB_SEARCH_PROMPT", { enumerable: true, get: function () { return grading_1.DEFAULT_WEB_SEARCH_PROMPT; } });
 /**
  * Reads and maps provider prompts based on the configuration and parsed prompts.
  * @param config - The configuration object.
@@ -213,6 +215,7 @@ async function processPrompts(prompts) {
         }
     }))).flat();
 }
+// G-Eval prompts
 exports.GEVAL_PROMPT_STEPS = `
 Given an evaluation criteria which outlines how you should judge some text, generate 3-4 concise evaluation steps for any text based on the criteria below.
 

package/dist/src/providers/adaline.gateway.js

@@ -25,7 +25,7 @@ const embedding_1 = require("./azure/embedding");
 const util_2 = require("./azure/util");
 const ai_studio_1 = require("./google/ai.studio");
 const vertex_2 = require("./google/vertex");
-const groq_2 = require("./groq");
+const index_2 = require("./groq/index");
 const chat_2 = require("./openai/chat");
 const embedding_2 = require("./openai/embedding");
 const util_3 = require("./openai/util");
@@ -396,7 +396,7 @@ class AdalineGatewayChatProvider extends AdalineGatewayGenericProvider {
             }
             else if (this.providerName === 'groq') {
                 const provider = new groq_1.Groq();
-                const parentClass = new groq_2.GroqProvider(this.modelName, this.providerOptions);
+                const parentClass = new index_2.GroqProvider(this.modelName, this.providerOptions);
                 const apiKey = parentClass.getApiKey();
                 if (!apiKey) {
                     throw new Error('Groq API key is not set. Set the GROQ_API_KEY environment variable or add `apiKey` to the provider config.');

package/dist/src/providers/anthropic/defaults.d.ts

@@ -14,5 +14,5 @@ export declare class AnthropicLlmRubricProvider extends AnthropicMessagesProvide
  * @param env - Optional environment overrides
  * @returns Anthropic provider implementations for various functions
  */
-export declare function getAnthropicProviders(env?: EnvOverrides): Pick<DefaultProviders, 'gradingJsonProvider' | 'gradingProvider' | 'llmRubricProvider' | 'suggestionsProvider' | 'synthesizeProvider'>;
+export declare function getAnthropicProviders(env?: EnvOverrides): Pick<DefaultProviders, 'gradingJsonProvider' | 'gradingProvider' | 'llmRubricProvider' | 'suggestionsProvider' | 'synthesizeProvider' | 'webSearchProvider'>;
 //# sourceMappingURL=defaults.d.ts.map

package/dist/src/providers/anthropic/defaults.js

@@ -84,6 +84,19 @@ exports.AnthropicLlmRubricProvider = AnthropicLlmRubricProvider;
 // Private provider factories with lazy loading
 const gradingProviderFactory = createLazyProvider((env) => new messages_1.AnthropicMessagesProvider(exports.DEFAULT_ANTHROPIC_MODEL, { env }));
 const llmRubricProviderFactory = createLazyProvider((env) => new AnthropicLlmRubricProvider(exports.DEFAULT_ANTHROPIC_MODEL, { env }));
+// Web Search Provider with web_search tool
+const webSearchProviderFactory = createLazyProvider((env) => new messages_1.AnthropicMessagesProvider(exports.DEFAULT_ANTHROPIC_MODEL, {
+    env,
+    config: {
+        tools: [
+            {
+                type: 'web_search_20250305',
+                name: 'web_search',
+                max_uses: 5,
+            },
+        ],
+    },
+}));
 /**
  * Gets all default Anthropic providers with the given environment overrides
  * @param env - Optional environment overrides
@@ -93,12 +106,14 @@ function getAnthropicProviders(env) {
     // Get providers with the provided environment variables
     const gradingProvider = gradingProviderFactory.getInstance(env);
     const llmRubricProvider = llmRubricProviderFactory.getInstance(env);
+    const webSearchProvider = webSearchProviderFactory.getInstance(env);
     return {
         gradingJsonProvider: gradingProvider,
         gradingProvider,
         llmRubricProvider,
         suggestionsProvider: gradingProvider,
         synthesizeProvider: gradingProvider,
+        webSearchProvider,
     };
 }
 //# sourceMappingURL=defaults.js.map

package/dist/src/providers/azure/chat.d.ts

@@ -7,7 +7,9 @@ export declare class AzureChatCompletionProvider extends AzureGenericProvider {
     private initializeMCP;
     cleanup(): Promise<void>;
     /**
-     * Check if the current deployment is configured as a reasoning model
+     * Check if the current deployment is configured as a reasoning model.
+     * Reasoning models use max_completion_tokens instead of max_tokens,
+     * don't support temperature, and accept reasoning_effort parameter.
      */
     protected isReasoningModel(): boolean;
     getOpenAiBody(prompt: string, context?: CallApiContextParams, callApiOptions?: CallApiOptionsParams): Promise<Record<string, any>>;

package/dist/src/providers/azure/chat.js

@@ -43,7 +43,9 @@ class AzureChatCompletionProvider extends generic_1.AzureGenericProvider {
         }
     }
     /**
-     * Check if the current deployment is configured as a reasoning model
+     * Check if the current deployment is configured as a reasoning model.
+     * Reasoning models use max_completion_tokens instead of max_tokens,
+     * don't support temperature, and accept reasoning_effort parameter.
      */
     isReasoningModel() {
         // Check explicit config flags first
@@ -53,14 +55,25 @@ class AzureChatCompletionProvider extends generic_1.AzureGenericProvider {
         // Auto-detect reasoning models by deployment name (case-insensitive)
         // Supports both direct names (o1-preview) and prefixed names (prod-o1-mini)
         const lowerName = this.deploymentName.toLowerCase();
-        return (lowerName.startsWith('o1') ||
+        return (
+        // OpenAI reasoning models
+        lowerName.startsWith('o1') ||
             lowerName.includes('-o1') ||
             lowerName.startsWith('o3') ||
             lowerName.includes('-o3') ||
             lowerName.startsWith('o4') ||
             lowerName.includes('-o4') ||
+            // GPT-5 series (reasoning by default)
             lowerName.startsWith('gpt-5') ||
-            lowerName.includes('-gpt-5'));
+            lowerName.includes('-gpt-5') ||
+            // DeepSeek reasoning models
+            lowerName.includes('deepseek-r1') ||
+            lowerName.includes('deepseek_r1') ||
+            // Microsoft Phi reasoning models
+            lowerName.includes('phi-4-reasoning') ||
+            lowerName.includes('phi-4-mini-reasoning') ||
+            // xAI Grok reasoning models
+            (lowerName.includes('grok') && lowerName.includes('reasoning')));
     }
     async getOpenAiBody(prompt, context, callApiOptions) {
         const config = {