promptfoo 0.119.13 → 0.119.14

package/dist/src/providers/azure/responses.d.ts

@@ -5,6 +5,11 @@ export declare class AzureResponsesProvider extends AzureGenericProvider {
     private processor;
     constructor(...args: ConstructorParameters<typeof AzureGenericProvider>);
     private initializeMCP;
+    /**
+     * Check if the current deployment is a reasoning model.
+     * Reasoning models use max_completion_tokens instead of max_tokens,
+     * don't support temperature, and accept reasoning_effort parameter.
+     */
     isReasoningModel(): boolean;
     supportsTemperature(): boolean;
     getAzureResponsesBody(prompt: string, context?: CallApiContextParams, _callApiOptions?: CallApiOptionsParams): Promise<Record<string, any>>;

package/dist/src/providers/azure/responses.js

@@ -35,11 +35,36 @@ class AzureResponsesProvider extends generic_1.AzureGenericProvider {
     async initializeMCP() {
         // TODO: Initialize MCP if needed
     }
+    /**
+     * Check if the current deployment is a reasoning model.
+     * Reasoning models use max_completion_tokens instead of max_tokens,
+     * don't support temperature, and accept reasoning_effort parameter.
+     */
     isReasoningModel() {
-        return (this.deploymentName.startsWith('o1') ||
-            this.deploymentName.startsWith('o3') ||
-            this.deploymentName.startsWith('o4') ||
-            this.deploymentName.startsWith('gpt-5'));
+        // Check explicit config flags first (match chat.ts behavior)
+        if (this.config.isReasoningModel || this.config.o1) {
+            return true;
+        }
+        const lowerName = this.deploymentName.toLowerCase();
+        return (
+        // OpenAI reasoning models
+        lowerName.startsWith('o1') ||
+            lowerName.includes('-o1') ||
+            lowerName.startsWith('o3') ||
+            lowerName.includes('-o3') ||
+            lowerName.startsWith('o4') ||
+            lowerName.includes('-o4') ||
+            // GPT-5 series (reasoning by default)
+            lowerName.startsWith('gpt-5') ||
+            lowerName.includes('-gpt-5') ||
+            // DeepSeek reasoning models
+            lowerName.includes('deepseek-r1') ||
+            lowerName.includes('deepseek_r1') ||
+            // Microsoft Phi reasoning models
+            lowerName.includes('phi-4-reasoning') ||
+            lowerName.includes('phi-4-mini-reasoning') ||
+            // xAI Grok reasoning models
+            (lowerName.includes('grok') && lowerName.includes('reasoning')));
     }
     supportsTemperature() {
         return !this.isReasoningModel();
@@ -107,6 +132,10 @@ class AzureResponsesProvider extends generic_1.AzureGenericProvider {
         else {
             textFormat = { format: { type: 'text' } };
         }
+        // Add verbosity for reasoning models if configured
+        if (isReasoningModel && config.verbosity) {
+            textFormat = { ...textFormat, verbosity: config.verbosity };
+        }
         // Azure Responses API uses 'model' field for deployment name
         const body = {
             model: this.deploymentName,

package/dist/src/providers/azure/types.d.ts

@@ -85,6 +85,10 @@ export interface AzureCompletionOptions {
     stop?: string[];
     seed?: number;
     reasoning_effort?: 'low' | 'medium' | 'high';
+    /**
+     * Controls the verbosity of the model's responses. Only used for reasoning models (GPT-5, o1, o3, etc.).
+     */
+    verbosity?: 'low' | 'medium' | 'high';
     /**
      * If set, automatically call these functions when the model calls them.
      * Keys are function names, values are either functions or strings (JavaScript code).

package/dist/src/providers/bedrock/agents.d.ts

@@ -1,4 +1,4 @@
-import { AwsBedrockGenericProvider } from './index';
+import { AwsBedrockGenericProvider } from './base';
 import type { BedrockAgentRuntimeClient } from '@aws-sdk/client-bedrock-agent-runtime';
 import type { EnvOverrides } from '../../types/env';
 import type { ApiProvider, ProviderResponse } from '../../types/providers';

package/dist/src/providers/bedrock/agents.js

@@ -41,7 +41,7 @@ const cache_1 = require("../../cache");
 const envars_1 = require("../../envars");
 const logger_1 = __importDefault(require("../../logger"));
 const telemetry_1 = __importDefault(require("../../telemetry"));
-const index_1 = require("./index");
+const base_1 = require("./base");
 /**
  * AWS Bedrock Agents provider for invoking deployed AI agents.
  * Supports all Bedrock Agents features including memory, knowledge bases, action groups,
@@ -81,7 +81,7 @@ const index_1 = require("./index");
  *         maximumLength: 2048
  * ```
  */
-class AwsBedrockAgentsProvider extends index_1.AwsBedrockGenericProvider {
+class AwsBedrockAgentsProvider extends base_1.AwsBedrockGenericProvider {
     constructor(agentId, options = {}) {
         super(agentId, options);
         // Validate required fields

package/dist/src/providers/bedrock/base.d.ts

@@ -0,0 +1,40 @@
+/**
+ * AWS Bedrock Base Provider
+ *
+ * Contains the abstract base class for all Bedrock providers.
+ * This is extracted to avoid circular dependency issues.
+ */
+import type { BedrockRuntime, Trace } from '@aws-sdk/client-bedrock-runtime';
+import type { AwsCredentialIdentity, AwsCredentialIdentityProvider } from '@aws-sdk/types';
+import type { EnvOverrides } from '../../types/env';
+export interface BedrockOptions {
+    accessKeyId?: string;
+    apiKey?: string;
+    profile?: string;
+    region?: string;
+    secretAccessKey?: string;
+    sessionToken?: string;
+    guardrailIdentifier?: string;
+    guardrailVersion?: string;
+    trace?: Trace;
+    showThinking?: boolean;
+    endpoint?: string;
+}
+export declare abstract class AwsBedrockGenericProvider {
+    modelName: string;
+    env?: EnvOverrides;
+    bedrock?: BedrockRuntime;
+    config: BedrockOptions;
+    constructor(modelName: string, options?: {
+        config?: BedrockOptions;
+        id?: string;
+        env?: EnvOverrides;
+    });
+    id(): string;
+    toString(): string;
+    protected getApiKey(): string | undefined;
+    getCredentials(): Promise<AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined>;
+    getBedrockInstance(): Promise<BedrockRuntime>;
+    getRegion(): string;
+}
+//# sourceMappingURL=base.d.ts.map

package/dist/src/providers/bedrock/base.js

@@ -0,0 +1,171 @@
+"use strict";
+/**
+ * AWS Bedrock Base Provider
+ *
+ * Contains the abstract base class for all Bedrock providers.
+ * This is extracted to avoid circular dependency issues.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AwsBedrockGenericProvider = void 0;
+const envars_1 = require("../../envars");
+const logger_1 = __importDefault(require("../../logger"));
+const telemetry_1 = __importDefault(require("../../telemetry"));
+class AwsBedrockGenericProvider {
+    constructor(modelName, options = {}) {
+        const { config, id, env } = options;
+        this.env = env;
+        this.modelName = modelName;
+        this.config = config || {};
+        this.id = id ? () => id : this.id;
+        if (this.config.guardrailIdentifier) {
+            telemetry_1.default.record('feature_used', {
+                feature: 'guardrail',
+                provider: 'bedrock',
+            });
+        }
+    }
+    id() {
+        return `bedrock:${this.modelName}`;
+    }
+    toString() {
+        return `[Amazon Bedrock Provider ${this.modelName}]`;
+    }
+    getApiKey() {
+        return this.config.apiKey || (0, envars_1.getEnvString)('AWS_BEARER_TOKEN_BEDROCK');
+    }
+    async getCredentials() {
+        // 1. Explicit credentials have ABSOLUTE highest priority (as documented)
+        if (this.config.accessKeyId && this.config.secretAccessKey) {
+            logger_1.default.debug(`Using credentials from config file`);
+            return {
+                accessKeyId: this.config.accessKeyId,
+                secretAccessKey: this.config.secretAccessKey,
+                sessionToken: this.config.sessionToken,
+            };
+        }
+        // 2. API key authentication as second priority
+        const apiKey = this.getApiKey();
+        if (apiKey) {
+            logger_1.default.debug(`Using Bedrock API key authentication`);
+            // For Bedrock API keys, we don't need traditional AWS credentials
+            // The API key will be handled in the request headers
+            return undefined;
+        }
+        // 3. SSO profile as third priority
+        if (this.config.profile) {
+            logger_1.default.debug(`Using SSO profile: ${this.config.profile}`);
+            try {
+                const { fromSSO } = await Promise.resolve().then(() => __importStar(require('@aws-sdk/credential-provider-sso')));
+                return fromSSO({ profile: this.config.profile });
+            }
+            catch (err) {
+                logger_1.default.error(`Error loading @aws-sdk/credential-provider-sso: ${err}`);
+                throw new Error('The @aws-sdk/credential-provider-sso package is required for SSO profiles. Please install it: npm install @aws-sdk/credential-provider-sso');
+            }
+        }
+        // 4. AWS default credential chain (lowest priority)
+        logger_1.default.debug(`No explicit credentials in config, falling back to AWS default chain`);
+        return undefined;
+    }
+    async getBedrockInstance() {
+        if (!this.bedrock) {
+            let handler;
+            const apiKey = this.getApiKey();
+            // Create request handler for proxy or API key scenarios
+            if ((0, envars_1.getEnvString)('HTTP_PROXY') || (0, envars_1.getEnvString)('HTTPS_PROXY') || apiKey) {
+                try {
+                    const { NodeHttpHandler } = await Promise.resolve().then(() => __importStar(require('@smithy/node-http-handler')));
+                    const { ProxyAgent } = await Promise.resolve().then(() => __importStar(require('proxy-agent')));
+                    // Create handler with proxy support if needed
+                    const proxyAgent = (0, envars_1.getEnvString)('HTTP_PROXY') || (0, envars_1.getEnvString)('HTTPS_PROXY')
+                        ? new ProxyAgent()
+                        : undefined;
+                    handler = new NodeHttpHandler({
+                        ...(proxyAgent ? { httpsAgent: proxyAgent } : {}),
+                        requestTimeout: 300000, // 5 minutes
+                    });
+                    // Add Bearer token middleware for API key authentication
+                    if (apiKey) {
+                        const originalHandle = handler.handle.bind(handler);
+                        handler.handle = async (request, options) => {
+                            // Add Authorization header with Bearer token
+                            request.headers = {
+                                ...request.headers,
+                                Authorization: `Bearer ${apiKey}`,
+                            };
+                            return originalHandle(request, options);
+                        };
+                    }
+                }
+                catch {
+                    const reason = apiKey
+                        ? 'API key authentication requires the @smithy/node-http-handler package'
+                        : 'Proxy configuration requires the @smithy/node-http-handler package';
+                    throw new Error(`${reason}. Please install it in your project or globally.`);
+                }
+            }
+            try {
+                const { BedrockRuntime } = await Promise.resolve().then(() => __importStar(require('@aws-sdk/client-bedrock-runtime')));
+                const credentials = await this.getCredentials();
+                const bedrock = new BedrockRuntime({
+                    region: this.getRegion(),
+                    maxAttempts: (0, envars_1.getEnvInt)('AWS_BEDROCK_MAX_RETRIES', 10),
+                    retryMode: 'adaptive',
+                    ...(credentials ? { credentials } : {}),
+                    ...(handler ? { requestHandler: handler } : {}),
+                    ...(this.config.endpoint ? { endpoint: this.config.endpoint } : {}),
+                });
+                this.bedrock = bedrock;
+            }
+            catch (err) {
+                logger_1.default.error(`Error creating BedrockRuntime: ${err}`);
+                throw new Error('The @aws-sdk/client-bedrock-runtime package is required as a peer dependency. Please install it in your project or globally.');
+            }
+        }
+        return this.bedrock;
+    }
+    getRegion() {
+        return (this.config?.region ||
+            this.env?.AWS_BEDROCK_REGION ||
+            (0, envars_1.getEnvString)('AWS_BEDROCK_REGION') ||
+            'us-east-1');
+    }
+}
+exports.AwsBedrockGenericProvider = AwsBedrockGenericProvider;
+//# sourceMappingURL=base.js.map

package/dist/src/providers/bedrock/converse.d.ts

@@ -0,0 +1,146 @@
+/**
+ * AWS Bedrock Converse API Provider
+ *
+ * This provider implements the AWS Bedrock Converse API, which provides a unified
+ * interface for all Bedrock models. It supports:
+ * - Extended thinking (reasoning/ultrathink) for Claude models
+ * - Tool calling with standardized format
+ * - Streaming responses via ConverseStream
+ * - Performance configuration (latency optimization, service tiers)
+ * - Guardrails integration
+ * - Cache token tracking
+ */
+import { AwsBedrockGenericProvider, type BedrockOptions } from './base';
+import type { Message, SystemContentBlock } from '@aws-sdk/client-bedrock-runtime';
+import type { DocumentType } from '@smithy/types';
+import type { EnvOverrides } from '../../types/env';
+import type { ApiProvider, CallApiContextParams, ProviderResponse } from '../../types/providers';
+/**
+ * Configuration options for the Bedrock Converse API provider
+ * Extends base BedrockOptions with Converse-specific parameters
+ */
+export interface BedrockConverseOptions extends BedrockOptions {
+    maxTokens?: number;
+    max_tokens?: number;
+    temperature?: number;
+    topP?: number;
+    top_p?: number;
+    stopSequences?: string[];
+    stop?: string[];
+    thinking?: {
+        type: 'enabled';
+        budget_tokens: number;
+    };
+    performanceConfig?: {
+        latency: 'standard' | 'optimized';
+    };
+    serviceTier?: {
+        type: 'priority' | 'default' | 'flex';
+    };
+    tools?: BedrockConverseToolConfig[];
+    toolChoice?: 'auto' | 'any' | {
+        tool: {
+            name: string;
+        };
+    };
+    functionToolCallbacks?: Record<string, string | Function>;
+    additionalModelRequestFields?: Record<string, unknown>;
+    additionalModelResponseFieldPaths?: string[];
+    streaming?: boolean;
+}
+/**
+ * Tool configuration for Converse API
+ */
+export interface BedrockConverseToolConfig {
+    toolSpec?: {
+        name: string;
+        description?: string;
+        inputSchema?: {
+            json: DocumentType;
+        };
+    };
+    type?: 'function';
+    function?: {
+        name: string;
+        description?: string;
+        parameters?: Record<string, unknown>;
+    };
+    name?: string;
+    description?: string;
+    input_schema?: Record<string, unknown>;
+}
+/**
+ * Parse prompt into Converse API message format
+ */
+export declare function parseConverseMessages(prompt: string): {
+    messages: Message[];
+    system?: SystemContentBlock[];
+};
+/**
+ * AWS Bedrock Converse API Provider
+ */
+export declare class AwsBedrockConverseProvider extends AwsBedrockGenericProvider implements ApiProvider {
+    config: BedrockConverseOptions;
+    private loadedFunctionCallbacks;
+    constructor(modelName: string, options?: {
+        config?: BedrockConverseOptions;
+        id?: string;
+        env?: EnvOverrides;
+    });
+    id(): string;
+    toString(): string;
+    /**
+     * Loads a function from an external file
+     * @param fileRef The file reference in the format 'file://path/to/file:functionName'
+     * @returns The loaded function
+     */
+    private loadExternalFunction;
+    /**
+     * Executes a function callback with proper error handling
+     */
+    private executeFunctionCallback;
+    /**
+     * Build the inference configuration from options
+     */
+    private buildInferenceConfig;
+    /**
+     * Build the tool configuration from options
+     */
+    private buildToolConfig;
+    /**
+     * Build the guardrail configuration
+     */
+    private buildGuardrailConfig;
+    /**
+     * Build additional model request fields (including thinking config)
+     */
+    private buildAdditionalModelRequestFields;
+    /**
+     * Build performance configuration
+     */
+    private buildPerformanceConfig;
+    /**
+     * Build service tier configuration
+     */
+    private buildServiceTier;
+    /**
+     * Main API call using Converse API
+     */
+    callApi(prompt: string, context?: CallApiContextParams): Promise<ProviderResponse>;
+    /**
+     * Parse the Converse API response into ProviderResponse format
+     */
+    private parseResponse;
+    /**
+     * Streaming API call using ConverseStream
+     *
+     * Note: functionToolCallbacks are not executed in streaming mode.
+     * Tool use blocks are captured and returned in the output, but callbacks
+     * are not automatically invoked. Use non-streaming mode if you need
+     * automatic tool callback execution.
+     */
+    callApiStreaming(prompt: string, context?: CallApiContextParams): Promise<ProviderResponse & {
+        stream?: AsyncIterable<string>;
+    }>;
+}
+//# sourceMappingURL=converse.d.ts.map