promptfoo 0.119.13 → 0.119.14

package/dist/src/providers/xai/responses.js

@@ -0,0 +1,268 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.XAIResponsesProvider = void 0;
+exports.createXAIResponsesProvider = createXAIResponsesProvider;
+const cache_1 = require("../../cache");
+const envars_1 = require("../../envars");
+const logger_1 = __importDefault(require("../../logger"));
+const index_1 = require("../../util/index");
+const file_1 = require("../../util/file");
+const functionCallbackUtils_1 = require("../functionCallbackUtils");
+const shared_1 = require("../shared");
+const index_2 = require("../responses/index");
+const chat_1 = require("./chat");
+/**
+ * xAI Responses API Provider
+ *
+ * Supports xAI's Responses API with Agent Tools for autonomous agent workflows.
+ * This enables Grok models to autonomously search the web, X, execute code,
+ * and interact with MCP servers.
+ *
+ * Usage:
+ *   xai:responses:grok-4-1-fast-reasoning
+ *   xai:responses:grok-4-fast
+ *   xai:responses:grok-4
+ */
+class XAIResponsesProvider {
+    constructor(modelName, options = {}) {
+        this.functionCallbackHandler = new functionCallbackUtils_1.FunctionCallbackHandler();
+        this.modelName = modelName;
+        this.config = options.config || {};
+        this.env = options.env;
+        // Initialize the shared response processor
+        this.processor = new index_2.ResponsesProcessor({
+            modelName: this.modelName,
+            providerType: 'xai',
+            functionCallbackHandler: this.functionCallbackHandler,
+            costCalculator: (modelName, usage, config) => (0, chat_1.calculateXAICost)(modelName, config || {}, usage?.input_tokens || usage?.prompt_tokens, usage?.output_tokens || usage?.completion_tokens) ?? 0,
+        });
+    }
+    id() {
+        return `xai:responses:${this.modelName}`;
+    }
+    toString() {
+        return `[xAI Responses Provider ${this.modelName}]`;
+    }
+    toJSON() {
+        return {
+            provider: 'xai:responses',
+            model: this.modelName,
+            config: {
+                ...this.config,
+                apiKey: undefined, // Don't expose API key
+            },
+        };
+    }
+    getApiKey() {
+        return this.config.apiKey || (0, envars_1.getEnvString)('XAI_API_KEY');
+    }
+    getApiUrl() {
+        if (this.config.apiBaseUrl) {
+            return this.config.apiBaseUrl;
+        }
+        if (this.config.region) {
+            return `https://${this.config.region}.api.x.ai/v1`;
+        }
+        return 'https://api.x.ai/v1';
+    }
+    async getRequestBody(prompt, context, _callApiOptions) {
+        const config = {
+            ...this.config,
+            ...context?.prompt?.config,
+        };
+        // Parse input - can be string or array of messages
+        let input;
+        try {
+            const parsedJson = JSON.parse(prompt);
+            if (Array.isArray(parsedJson)) {
+                input = parsedJson;
+            }
+            else {
+                input = prompt;
+            }
+        }
+        catch {
+            input = prompt;
+        }
+        // Handle max_output_tokens
+        const maxOutputTokens = config.max_output_tokens ?? 4096;
+        // Handle temperature
+        const temperature = config.temperature ?? 0.7;
+        // Load response_format from external file if needed
+        const responseFormat = config.response_format
+            ? (0, file_1.maybeLoadFromExternalFile)((0, index_1.renderVarsInObject)(config.response_format, context?.vars))
+            : undefined;
+        // Build text format configuration
+        let textFormat;
+        if (responseFormat) {
+            if (responseFormat.type === 'json_object') {
+                textFormat = { format: { type: 'json_object' } };
+            }
+            else if (responseFormat.type === 'json_schema') {
+                const schema = (0, file_1.maybeLoadFromExternalFile)((0, index_1.renderVarsInObject)(responseFormat.schema || responseFormat.json_schema?.schema, context?.vars));
+                const schemaName = responseFormat.json_schema?.name || responseFormat.name || 'response_schema';
+                textFormat = {
+                    format: {
+                        type: 'json_schema',
+                        name: schemaName,
+                        schema,
+                        strict: true,
+                    },
+                };
+            }
+            else {
+                textFormat = { format: { type: 'text' } };
+            }
+        }
+        // Load tools from external file if needed
+        const loadedTools = config.tools
+            ? await (0, index_1.maybeLoadToolsFromExternalFile)(config.tools, context?.vars)
+            : undefined;
+        // Build request body
+        const body = {
+            model: this.modelName,
+            input,
+            ...(maxOutputTokens ? { max_output_tokens: maxOutputTokens } : {}),
+            ...(temperature !== undefined ? { temperature } : {}),
+            ...(config.instructions ? { instructions: config.instructions } : {}),
+            ...(config.top_p !== undefined ? { top_p: config.top_p } : {}),
+            ...(loadedTools && loadedTools.length > 0 ? { tools: loadedTools } : {}),
+            ...(config.tool_choice ? { tool_choice: config.tool_choice } : {}),
+            ...(config.previous_response_id ? { previous_response_id: config.previous_response_id } : {}),
+            ...(textFormat ? { text: textFormat } : {}),
+            ...('parallel_tool_calls' in config
+                ? { parallel_tool_calls: Boolean(config.parallel_tool_calls) }
+                : {}),
+            ...('store' in config ? { store: Boolean(config.store) } : {}),
+            ...(config.user ? { user: config.user } : {}),
+            ...(config.passthrough || {}),
+        };
+        // Filter unsupported parameters for Grok-4 models
+        if (chat_1.GROK_4_MODELS.includes(this.modelName)) {
+            delete body.presence_penalty;
+            delete body.frequency_penalty;
+            delete body.stop;
+        }
+        return {
+            body,
+            config: {
+                ...config,
+                tools: loadedTools,
+                response_format: responseFormat,
+            },
+        };
+    }
+    async callApi(prompt, context, callApiOptions) {
+        const apiKey = this.getApiKey();
+        if (!apiKey) {
+            return {
+                error: 'xAI API key is not set. Set the XAI_API_KEY environment variable or add `apiKey` to the provider config.',
+            };
+        }
+        const { body, config } = await this.getRequestBody(prompt, context, callApiOptions);
+        logger_1.default.debug(`[xAI Responses] Calling ${this.getApiUrl()}/responses`, {
+            model: this.modelName,
+            hasTools: !!body.tools?.length,
+            toolTypes: body.tools?.map((t) => t.type),
+        });
+        let data;
+        let cached = false;
+        let status;
+        let statusText;
+        try {
+            const response = await (0, cache_1.fetchWithCache)(`${this.getApiUrl()}/responses`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${apiKey}`,
+                    ...config.headers,
+                },
+                body: JSON.stringify(body),
+            }, shared_1.REQUEST_TIMEOUT_MS, 'json', context?.bustCache ?? context?.debug, this.config.maxRetries);
+            data = response.data;
+            cached = response.cached;
+            status = response.status;
+            statusText = response.statusText;
+            if (status < 200 || status >= 300) {
+                const errorMessage = `xAI API error: ${status} ${statusText}\n${typeof data === 'string' ? data : JSON.stringify(data)}`;
+                // Check for specific error types
+                if (data?.error?.code === 'invalid_prompt') {
+                    return {
+                        output: errorMessage,
+                        tokenUsage: this.getTokenUsage(data, cached),
+                        isRefusal: true,
+                    };
+                }
+                return { error: errorMessage };
+            }
+        }
+        catch (err) {
+            const errorMessage = err instanceof Error ? err.message : String(err);
+            // Handle common xAI errors
+            if (errorMessage.includes('502') || errorMessage.includes('Bad Gateway')) {
+                return {
+                    error: `xAI API error: 502 Bad Gateway - This often indicates an invalid API key.\n\nTip: Ensure your XAI_API_KEY environment variable is set correctly. You can get an API key from https://x.ai/`,
+                };
+            }
+            return {
+                error: `xAI API error: ${errorMessage}\n\nIf this persists, verify your API key at https://x.ai/`,
+            };
+        }
+        if (data.error) {
+            return {
+                error: `xAI API error: ${typeof data.error === 'string' ? data.error : JSON.stringify(data.error)}`,
+            };
+        }
+        // Use shared processor for consistent response handling
+        return this.processor.processResponseOutput(data, config, cached);
+    }
+    getTokenUsage(data, cached) {
+        if (!data.usage) {
+            return {};
+        }
+        if (cached) {
+            const totalTokens = data.usage.total_tokens || (data.usage.input_tokens || 0) + (data.usage.output_tokens || 0);
+            return { cached: totalTokens, total: totalTokens };
+        }
+        const promptTokens = data.usage.prompt_tokens || data.usage.input_tokens || 0;
+        const completionTokens = data.usage.completion_tokens || data.usage.output_tokens || 0;
+        const totalTokens = data.usage.total_tokens || promptTokens + completionTokens;
+        return {
+            total: totalTokens,
+            prompt: promptTokens,
+            completion: completionTokens,
+            ...(data.usage.completion_tokens_details
+                ? {
+                    completionDetails: {
+                        reasoning: data.usage.completion_tokens_details.reasoning_tokens,
+                    },
+                }
+                : {}),
+        };
+    }
+}
+exports.XAIResponsesProvider = XAIResponsesProvider;
+/**
+ * Create an xAI Responses provider
+ *
+ * @param providerPath - Provider path in format xai:responses:<model>
+ * @param options - Provider options
+ * @returns XAIResponsesProvider instance
+ */
+function createXAIResponsesProvider(providerPath, options = {}) {
+    // Parse model name from path: xai:responses:<model>
+    const parts = providerPath.split(':');
+    const modelName = parts.slice(2).join(':');
+    if (!modelName) {
+        throw new Error('Model name is required for xAI Responses provider. Use format: xai:responses:<model>');
+    }
+    return new XAIResponsesProvider(modelName, {
+        config: options.config,
+        id: options.id,
+        env: options.env,
+    });
+}
+//# sourceMappingURL=responses.js.map

package/dist/src/redteam/constants/plugins.d.ts

@@ -3,7 +3,7 @@ export declare const REDTEAM_DEFAULTS: {
     readonly MAX_CONCURRENCY: 4;
     readonly NUM_TESTS: 10;
 };
-export declare const REDTEAM_MODEL = "openai:chat:gpt-4.1-2025-04-14";
+export declare const REDTEAM_MODEL = "openai:chat:gpt-5-2025-08-07";
 export declare const LLAMA_GUARD_REPLICATE_PROVIDER = "replicate:moderation:meta/llama-guard-4-12b";
 export declare const LLAMA_GUARD_ENABLED_CATEGORIES: string[];
 export declare const FOUNDATION_PLUGINS: readonly ["ascii-smuggling", "beavertails", "bias:age", "bias:disability", "bias:gender", "bias:race", "contracts", "cyberseceval", "donotanswer", "divergent-repetition", "excessive-agency", "hallucination", "harmful:chemical-biological-weapons", "harmful:child-exploitation", "harmful:copyright-violations", "harmful:cybercrime", "harmful:cybercrime:malicious-code", "harmful:graphic-content", "harmful:harassment-bullying", "harmful:hate", "harmful:illegal-activities", "harmful:illegal-drugs", "harmful:illegal-drugs:meth", "harmful:indiscriminate-weapons", "harmful:insults", "harmful:intellectual-property", "harmful:misinformation-disinformation", "harmful:non-violent-crime", "harmful:profanity", "harmful:radicalization", "harmful:self-harm", "harmful:sex-crime", "harmful:sexual-content", "harmful:specialized-advice", "harmful:unsafe-practices", "harmful:violent-crime", "harmful:weapons:ied", "hijacking", "imitation", "overreliance", "pii:direct", "pliny", "politics", "religion"];

package/dist/src/redteam/constants/plugins.js

@@ -7,7 +7,7 @@ exports.REDTEAM_DEFAULTS = {
     MAX_CONCURRENCY: 4,
     NUM_TESTS: 10,
 };
-exports.REDTEAM_MODEL = 'openai:chat:gpt-4.1-2025-04-14';
+exports.REDTEAM_MODEL = 'openai:chat:gpt-5-2025-08-07';
 // LlamaGuard 4 is the default on Replicate (supports S14: Code Interpreter Abuse)
 exports.LLAMA_GUARD_REPLICATE_PROVIDER = 'replicate:moderation:meta/llama-guard-4-12b';
 // For LlamaGuard 3 compatibility:

package/dist/src/redteam/constants/strategies.d.ts

@@ -12,7 +12,7 @@ export declare const MULTI_MODAL_STRATEGIES: readonly ["audio", "image", "video"
 export type MultiModalStrategy = (typeof MULTI_MODAL_STRATEGIES)[number];
 export declare const AGENTIC_STRATEGIES: readonly ["crescendo", "goat", "custom", "jailbreak", "jailbreak:hydra", "jailbreak:meta", "jailbreak:tree", "mischievous-user", "simba"];
 export type AgenticStrategy = (typeof AGENTIC_STRATEGIES)[number];
-export declare const DATASET_PLUGINS: readonly ["beavertails", "cyberseceval", "donotanswer", "harmbench", "toxic-chat", "aegis", "pliny", "unsafebench", "xstest"];
+export declare const DATASET_PLUGINS: readonly ["beavertails", "cyberseceval", "donotanswer", "harmbench", "toxic-chat", "aegis", "pliny", "unsafebench", "vlguard", "xstest"];
 export type DatasetPlugin = (typeof DATASET_PLUGINS)[number];
 export declare const ADDITIONAL_STRATEGIES: readonly ["audio", "authoritative-markup-injection", "base64", "best-of-n", "camelcase", "citation", "crescendo", "custom", "emoji", "gcg", "goat", "hex", "homoglyph", "image", "jailbreak:hydra", "jailbreak", "jailbreak:likert", "jailbreak:meta", "jailbreak:tree", "layer", "leetspeak", "math-prompt", "mischievous-user", "morse", "multilingual", "piglatin", "prompt-injection", "retry", "rot13", "simba", "video"];
 export type AdditionalStrategy = (typeof ADDITIONAL_STRATEGIES)[number];

package/dist/src/redteam/constants/strategies.js

@@ -57,6 +57,7 @@ exports.DATASET_PLUGINS = [
     'aegis',
     'pliny',
     'unsafebench',
+    'vlguard',
     'xstest',
 ];
 exports.ADDITIONAL_STRATEGIES = [

package/dist/src/redteam/plugins/vlguard.d.ts

@@ -1,7 +1,16 @@
 import { ImageDatasetGraderBase, ImageDatasetPluginBase, type ImageDatasetPluginConfig } from './imageDatasetPluginBase';
 import { ImageDatasetManager } from './imageDatasetUtils';
-export declare const VALID_CATEGORIES: readonly ["deception", "risky behavior", "privacy", "discrimination"];
-export declare const VALID_SUBCATEGORIES: readonly ["disinformation", "violence", "professional advice", "political", "sexually explicit", "personal data", "sex", "other"];
+declare const SPLIT_INFO: {
+    readonly test: {
+        readonly totalRecords: 1000;
+    };
+    readonly train: {
+        readonly totalRecords: 1999;
+    };
+};
+export type VLGuardSplit = keyof typeof SPLIT_INFO | 'both';
+export declare const VALID_CATEGORIES: readonly ["Privacy", "Risky Behavior", "Deception", "Hateful Speech", "privacy", "risky behavior", "deception", "discrimination"];
+export declare const VALID_SUBCATEGORIES: readonly ["Personal data", "Professional advice", "Political", "Sexually explicit", "Violence", "Disinformation", "Discrimination by sex", "Discrimination by race", "personal data", "professional advice", "political", "sexually explicit", "violence", "disinformation", "sex", "other"];
 export type VLGuardCategory = (typeof VALID_CATEGORIES)[number];
 export type VLGuardSubcategory = (typeof VALID_SUBCATEGORIES)[number];
 interface VLGuardInput {
@@ -9,13 +18,18 @@ interface VLGuardInput {
     category: string;
     subcategory: string;
     question: string;
+    safe: boolean;
 }
 interface VLGuardPluginConfig extends ImageDatasetPluginConfig {
     categories?: VLGuardCategory[];
     subcategories?: VLGuardSubcategory[];
+    includeUnsafe?: boolean;
+    includeSafe?: boolean;
+    split?: VLGuardSplit;
 }
 /**
  * DatasetManager to handle VLGuard dataset caching and filtering
+ * Fetches metadata from {split}.json and images from HuggingFace
  * @internal - exported for testing purposes only
  */
 export declare class VLGuardDatasetManager extends ImageDatasetManager<VLGuardInput> {
@@ -23,19 +37,54 @@ export declare class VLGuardDatasetManager extends ImageDatasetManager<VLGuardIn
     protected pluginId: string;
     protected datasetPath: string;
     protected fetchLimit: number;
+    private metadataCache;
+    private splitCache;
+    private currentSplit;
     private constructor();
     /**
      * Get singleton instance
      */
     static getInstance(): VLGuardDatasetManager;
+    /**
+     * Set the split to use for fetching records
+     */
+    setSplit(split: VLGuardSplit): void;
+    /**
+     * Get the current split
+     */
+    getSplit(): VLGuardSplit;
     /**
      * Clear the cache - useful for testing
      */
     static clearCache(): void;
     /**
-     * Process raw records from Hugging Face into VLGuardInput format
+     * Required by base class but not used since we override ensureDatasetLoaded
+     */
+    protected processRecords(_records: any[]): Promise<VLGuardInput[]>;
+    /**
+     * Fetch metadata from a specific split's JSON file
+     */
+    private fetchMetadataForSplit;
+    /**
+     * Process a single metadata record with its corresponding image URL
+     */
+    private processSingleRecord;
+    /**
+     * Fetch image URLs from the datasets-server API for a specific split (handles pagination)
+     */
+    private fetchImageUrlsForSplit;
+    /**
+     * Process metadata records with URLs and bounded concurrency to avoid OOM
+     */
+    private processMetadataRecordsWithUrls;
+    /**
+     * Load data for a single split and return indexed records with their image map
+     */
+    private loadSplitData;
+    /**
+     * Override ensureDatasetLoaded to use our custom metadata fetching
      */
-    protected processRecords(records: any[]): Promise<VLGuardInput[]>;
+    protected ensureDatasetLoaded(): Promise<void>;
     /**
      * Get records filtered by category, fetching dataset if needed
      */