promptfoo 0.119.13 → 0.119.14

package/dist/src/providers/openai/chatkit-types.d.ts

@@ -0,0 +1,73 @@
+import type { OpenAiSharedOptions } from './types';
+/**
+ * Configuration options for the OpenAI ChatKit Provider
+ */
+export interface OpenAiChatKitOptions extends OpenAiSharedOptions {
+    /**
+     * The ChatKit workflow ID from Agent Builder (e.g., wf_xxxx)
+     */
+    workflowId?: string;
+    /**
+     * The version of the workflow to use
+     */
+    version?: string;
+    /**
+     * User identifier for the ChatKit session.
+     * If not set, a unique ID with timestamp is generated.
+     * @default 'promptfoo-eval-<timestamp>'
+     */
+    userId?: string;
+    /**
+     * Timeout in milliseconds for waiting for a response
+     * @default 120000 (2 minutes)
+     */
+    timeout?: number;
+    /**
+     * Run the browser in headless mode
+     * @default true
+     */
+    headless?: boolean;
+    /**
+     * Port for the local HTTP server serving the ChatKit HTML
+     * @default 0 (random available port)
+     */
+    serverPort?: number;
+    /**
+     * Use a shared browser pool for better concurrency support.
+     * When enabled, a single browser with multiple contexts is used
+     * instead of spawning separate browsers per test.
+     * @default true
+     */
+    usePool?: boolean;
+    /**
+     * Maximum number of concurrent browser contexts when using pool mode.
+     * Only applies when usePool is true.
+     * If not specified, defaults to the value of --max-concurrency (or 4).
+     */
+    poolSize?: number;
+    /**
+     * How to handle workflow approval steps.
+     * - 'auto-approve': Automatically click approve button when detected
+     * - 'auto-reject': Automatically click reject button when detected
+     * - 'skip': Don't interact with approvals, capture the approval prompt as output
+     * @default 'auto-approve'
+     */
+    approvalHandling?: 'auto-approve' | 'auto-reject' | 'skip';
+    /**
+     * Maximum number of approval steps to process per message.
+     * Prevents infinite loops if workflow has chained approvals.
+     * @default 5
+     */
+    maxApprovals?: number;
+    /**
+     * Enable stateful/multi-turn conversation mode.
+     * When enabled:
+     * - The browser page is kept alive between calls
+     * - First message uses newThread: true, subsequent messages use newThread: false
+     * - Requires --max-concurrency 1 for reliable behavior
+     * - Useful for workflows that ask follow-up questions
+     * @default false
+     */
+    stateful?: boolean;
+}
+//# sourceMappingURL=chatkit-types.d.ts.map

package/dist/src/providers/openai/chatkit-types.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=chatkit-types.js.map

package/dist/src/providers/openai/chatkit.d.ts

@@ -0,0 +1,76 @@
+/**
+ * OpenAI ChatKit Provider
+ *
+ * Evaluates ChatKit workflows deployed via Agent Builder using Playwright
+ * to interact with the ChatKit web component.
+ *
+ * ChatKit workflows created in OpenAI's Agent Builder don't expose a direct
+ * REST API for sending messages. Instead, they require interaction through
+ * the ChatKit web component, which this provider automates using Playwright.
+ *
+ * Prerequisites:
+ *   - Playwright installed: npm install playwright && npx playwright install chromium
+ *   - OPENAI_API_KEY environment variable set
+ *
+ * Usage:
+ *   providers:
+ *     - id: openai:chatkit:wf_68ffb83dbfc88190a38103c2bb9f421003f913035dbdb131
+ *       config:
+ *         version: '3'           # Optional: workflow version
+ *         timeout: 120000        # Optional: response timeout in ms (default: 120000)
+ *         headless: true         # Optional: run browser headless (default: true)
+ *
+ * Performance Notes:
+ *   - Each evaluation spawns a browser instance, so it's slower than REST APIs
+ *   - For reliable results, use --max-concurrency 1 to avoid resource contention
+ *   - First test may be slower due to browser launch and ChatKit initialization
+ *
+ * Troubleshooting:
+ *   - "Playwright not found": Run `npx playwright install chromium`
+ *   - Timeout errors: Increase timeout config or use --max-concurrency 1
+ *   - Empty responses: The workflow may not generate text for some inputs
+ */
+import { OpenAiGenericProvider } from './index';
+import type { OpenAiChatKitOptions } from './chatkit-types';
+import type { CallApiContextParams, CallApiOptionsParams, ProviderResponse } from '../../types/index';
+import type { EnvOverrides } from '../../types/env';
+export declare class OpenAiChatKitProvider extends OpenAiGenericProvider {
+    private chatKitConfig;
+    private browser;
+    private context;
+    private page;
+    private server;
+    private serverPort;
+    private initialized;
+    private static defaultUserId;
+    private static getDefaultUserId;
+    constructor(workflowId: string, options?: {
+        config?: OpenAiChatKitOptions;
+        id?: string;
+        env?: EnvOverrides;
+    });
+    id(): string;
+    toString(): string;
+    /**
+     * Initialize the browser and ChatKit page
+     */
+    private initialize;
+    /**
+     * Shutdown method for providerRegistry cleanup
+     */
+    shutdown(): Promise<void>;
+    /**
+     * Clean up browser resources
+     */
+    cleanup(): Promise<void>;
+    /**
+     * Call the ChatKit workflow with the given prompt
+     */
+    callApi(prompt: string, _context?: CallApiContextParams, _callApiOptions?: CallApiOptionsParams): Promise<ProviderResponse>;
+    /**
+     * Pool-based callApi for better concurrency support.
+     * Uses a shared browser with multiple contexts instead of separate browsers.
+     */
+    private callApiWithPool;
+}
+//# sourceMappingURL=chatkit.d.ts.map