whale-code 6.4.0

package/dist/server/lib/batch-client.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Batch API Client — queues non-streaming LLM requests and submits them as batch jobs.
+ *
+ * Supports:
+ * - Anthropic Message Batches API (POST /v1/messages/batches)
+ * - OpenAI Batch API (POST /v1/batches via file upload)
+ *
+ * Batch API typically saves ~50% cost for non-latency-sensitive operations
+ * like workflow steps, bulk classification, and background processing.
+ */
+export interface BatchRequest {
+    id: string;
+    provider: "anthropic" | "openai";
+    model: string;
+    messages: unknown[];
+    system?: unknown;
+    tools?: unknown[];
+    max_tokens: number;
+    temperature?: number;
+}
+export interface BatchResult {
+    id: string;
+    success: boolean;
+    content?: unknown;
+    text?: string;
+    error?: string;
+    usage?: {
+        input_tokens: number;
+        output_tokens: number;
+    };
+}
+export declare class BatchClient {
+    private queue;
+    /**
+     * Add a request to the queue. Call submitBatch() to send them all.
+     */
+    queueRequest(id: string, provider: "anthropic" | "openai", model: string, messages: unknown[], config?: {
+        system?: unknown;
+        tools?: unknown[];
+        max_tokens?: number;
+        temperature?: number;
+    }): void;
+    /**
+     * Submit all queued requests as a batch job. Returns the batch ID.
+     * Clears the queue after submission.
+     */
+    submitBatch(): Promise<{
+        batchId: string;
+        provider: "anthropic" | "openai";
+        count: number;
+    }>;
+    /**
+     * Poll a batch until it completes. Returns a map of custom_id → result.
+     */
+    pollBatch(batchId: string, provider: "anthropic" | "openai"): Promise<Map<string, BatchResult>>;
+    /**
+     * Convenience: queue a single request, submit it as a batch, poll until done, return result.
+     * Best for workflow steps where you want batch pricing but only have one request.
+     */
+    processSingle(id: string, provider: "anthropic" | "openai", model: string, messages: unknown[], config?: {
+        system?: unknown;
+        tools?: unknown[];
+        max_tokens?: number;
+        temperature?: number;
+    }): Promise<BatchResult>;
+    /** Return the current queue size (for diagnostics). */
+    get queueSize(): number;
+    private submitAnthropicBatch;
+    private pollAnthropicBatch;
+    private fetchAnthropicResults;
+    private submitOpenAIBatch;
+    private pollOpenAIBatch;
+    private fetchOpenAIResults;
+    /**
+     * Convert Anthropic-format messages to OpenAI chat format for batch requests.
+     * Simplified conversion — handles the common patterns (text, system, tool_use/tool_result).
+     */
+    private buildOpenAIMessages;
+}
+export declare const batchClient: BatchClient;

package/dist/server/lib/batch-client.js

@@ -0,0 +1,467 @@
+/**
+ * Batch API Client — queues non-streaming LLM requests and submits them as batch jobs.
+ *
+ * Supports:
+ * - Anthropic Message Batches API (POST /v1/messages/batches)
+ * - OpenAI Batch API (POST /v1/batches via file upload)
+ *
+ * Batch API typically saves ~50% cost for non-latency-sensitive operations
+ * like workflow steps, bulk classification, and background processing.
+ */
+import { createLogger } from "./logger.js";
+const log = createLogger("batch-client");
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+const ANTHROPIC_BASE_URL = "https://api.anthropic.com";
+const OPENAI_BASE_URL = "https://api.openai.com";
+const POLL_INTERVAL_MS = 5_000;
+const MAX_POLL_DURATION_MS = 10 * 60 * 1000; // 10 minutes
+const ANTHROPIC_API_VERSION = "2023-06-01";
+const ANTHROPIC_BATCH_BETA = "message-batches-2024-09-24";
+// ============================================================================
+// BATCH CLIENT
+// ============================================================================
+export class BatchClient {
+    queue = [];
+    /**
+     * Add a request to the queue. Call submitBatch() to send them all.
+     */
+    queueRequest(id, provider, model, messages, config = {}) {
+        this.queue.push({
+            id,
+            provider,
+            model,
+            messages,
+            system: config.system,
+            tools: config.tools,
+            max_tokens: config.max_tokens ?? 4096,
+            temperature: config.temperature,
+        });
+        log.info({ id, provider, model, queueSize: this.queue.length }, "queued batch request");
+    }
+    /**
+     * Submit all queued requests as a batch job. Returns the batch ID.
+     * Clears the queue after submission.
+     */
+    async submitBatch() {
+        if (this.queue.length === 0) {
+            throw new Error("No requests queued for batch submission");
+        }
+        // All requests in a batch must use the same provider
+        const provider = this.queue[0].provider;
+        const mixedProvider = this.queue.find(r => r.provider !== provider);
+        if (mixedProvider) {
+            throw new Error(`Mixed providers in batch: ${provider} and ${mixedProvider.provider}. Submit separate batches per provider.`);
+        }
+        const requests = [...this.queue];
+        this.queue = [];
+        if (provider === "anthropic") {
+            return this.submitAnthropicBatch(requests);
+        }
+        else {
+            return this.submitOpenAIBatch(requests);
+        }
+    }
+    /**
+     * Poll a batch until it completes. Returns a map of custom_id → result.
+     */
+    async pollBatch(batchId, provider) {
+        if (provider === "anthropic") {
+            return this.pollAnthropicBatch(batchId);
+        }
+        else {
+            return this.pollOpenAIBatch(batchId);
+        }
+    }
+    /**
+     * Convenience: queue a single request, submit it as a batch, poll until done, return result.
+     * Best for workflow steps where you want batch pricing but only have one request.
+     */
+    async processSingle(id, provider, model, messages, config = {}) {
+        this.queueRequest(id, provider, model, messages, config);
+        const { batchId } = await this.submitBatch();
+        const results = await this.pollBatch(batchId, provider);
+        return results.get(id) || { id, success: false, error: "Result not found in batch response" };
+    }
+    /** Return the current queue size (for diagnostics). */
+    get queueSize() {
+        return this.queue.length;
+    }
+    // ==========================================================================
+    // ANTHROPIC BATCH API
+    // ==========================================================================
+    async submitAnthropicBatch(requests) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey)
+            throw new Error("ANTHROPIC_API_KEY not set");
+        const batchRequests = requests.map(r => {
+            const params = {
+                model: r.model,
+                messages: r.messages,
+                max_tokens: r.max_tokens,
+            };
+            if (r.system)
+                params.system = r.system;
+            if (r.tools?.length) {
+                params.tools = r.tools.map((t) => ({
+                    name: t.name,
+                    description: typeof t.description === "string" ? t.description.slice(0, 4096) : "",
+                    input_schema: t.input_schema,
+                }));
+            }
+            if (r.temperature !== undefined)
+                params.temperature = r.temperature;
+            return { custom_id: r.id, params };
+        });
+        const resp = await fetch(`${ANTHROPIC_BASE_URL}/v1/messages/batches`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+                "anthropic-version": ANTHROPIC_API_VERSION,
+                "anthropic-beta": ANTHROPIC_BATCH_BETA,
+            },
+            body: JSON.stringify({ requests: batchRequests }),
+        });
+        if (!resp.ok) {
+            const errText = await resp.text();
+            throw new Error(`Anthropic batch submit failed (${resp.status}): ${errText.substring(0, 500)}`);
+        }
+        const data = await resp.json();
+        log.info({ batchId: data.id, count: requests.length }, "anthropic batch submitted");
+        return { batchId: data.id, provider: "anthropic", count: requests.length };
+    }
+    async pollAnthropicBatch(batchId) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey)
+            throw new Error("ANTHROPIC_API_KEY not set");
+        const startTime = Date.now();
+        while (Date.now() - startTime < MAX_POLL_DURATION_MS) {
+            const resp = await fetch(`${ANTHROPIC_BASE_URL}/v1/messages/batches/${batchId}`, {
+                headers: {
+                    "x-api-key": apiKey,
+                    "anthropic-version": ANTHROPIC_API_VERSION,
+                    "anthropic-beta": ANTHROPIC_BATCH_BETA,
+                },
+            });
+            if (!resp.ok) {
+                const errText = await resp.text();
+                throw new Error(`Anthropic batch poll failed (${resp.status}): ${errText.substring(0, 500)}`);
+            }
+            const batch = await resp.json();
+            log.info({
+                batchId,
+                status: batch.processing_status,
+                counts: batch.request_counts,
+                elapsed: Date.now() - startTime,
+            }, "anthropic batch poll");
+            if (batch.processing_status === "ended") {
+                return this.fetchAnthropicResults(batchId);
+            }
+            if (batch.processing_status === "canceling" || batch.processing_status === "canceled") {
+                throw new Error(`Anthropic batch ${batchId} was canceled`);
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        throw new Error(`Anthropic batch ${batchId} timed out after ${MAX_POLL_DURATION_MS / 1000}s`);
+    }
+    async fetchAnthropicResults(batchId) {
+        const apiKey = process.env.ANTHROPIC_API_KEY;
+        if (!apiKey)
+            throw new Error("ANTHROPIC_API_KEY not set");
+        const resp = await fetch(`${ANTHROPIC_BASE_URL}/v1/messages/batches/${batchId}/results`, {
+            headers: {
+                "x-api-key": apiKey,
+                "anthropic-version": ANTHROPIC_API_VERSION,
+                "anthropic-beta": ANTHROPIC_BATCH_BETA,
+            },
+        });
+        if (!resp.ok) {
+            const errText = await resp.text();
+            throw new Error(`Anthropic batch results fetch failed (${resp.status}): ${errText.substring(0, 500)}`);
+        }
+        // Results come as JSONL (one JSON object per line)
+        const text = await resp.text();
+        const results = new Map();
+        for (const line of text.split("\n")) {
+            if (!line.trim())
+                continue;
+            try {
+                const item = JSON.parse(line);
+                if (item.result.type === "succeeded" && item.result.message) {
+                    const textBlocks = item.result.message.content
+                        .filter((b) => b.type === "text" && b.text)
+                        .map((b) => b.text);
+                    results.set(item.custom_id, {
+                        id: item.custom_id,
+                        success: true,
+                        content: item.result.message.content,
+                        text: textBlocks.join("\n"),
+                        usage: item.result.message.usage,
+                    });
+                }
+                else {
+                    results.set(item.custom_id, {
+                        id: item.custom_id,
+                        success: false,
+                        error: item.result.error?.message || `Batch item ${item.result.type}`,
+                    });
+                }
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+        log.info({ batchId, resultCount: results.size }, "anthropic batch results fetched");
+        return results;
+    }
+    // ==========================================================================
+    // OPENAI BATCH API
+    // ==========================================================================
+    async submitOpenAIBatch(requests) {
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey)
+            throw new Error("OPENAI_API_KEY not set");
+        // Step 1: Build JSONL content
+        const lines = requests.map(r => {
+            const body = {
+                model: r.model,
+                messages: this.buildOpenAIMessages(r),
+                max_tokens: r.max_tokens,
+            };
+            if (r.tools?.length) {
+                body.tools = r.tools.map((t) => ({
+                    type: "function",
+                    function: {
+                        name: t.name,
+                        description: typeof t.description === "string" ? t.description.slice(0, 4096) : "",
+                        parameters: t.input_schema || {},
+                    },
+                }));
+            }
+            if (r.temperature !== undefined)
+                body.temperature = r.temperature;
+            const item = {
+                custom_id: r.id,
+                method: "POST",
+                url: "/v1/chat/completions",
+                body,
+            };
+            return JSON.stringify(item);
+        });
+        const jsonlContent = lines.join("\n");
+        // Step 2: Upload JSONL as a file
+        const formData = new FormData();
+        const blob = new Blob([jsonlContent], { type: "application/jsonl" });
+        formData.append("file", blob, "batch_requests.jsonl");
+        formData.append("purpose", "batch");
+        const uploadResp = await fetch(`${OPENAI_BASE_URL}/v1/files`, {
+            method: "POST",
+            headers: { Authorization: `Bearer ${apiKey}` },
+            body: formData,
+        });
+        if (!uploadResp.ok) {
+            const errText = await uploadResp.text();
+            throw new Error(`OpenAI file upload failed (${uploadResp.status}): ${errText.substring(0, 500)}`);
+        }
+        const fileData = await uploadResp.json();
+        // Step 3: Create batch
+        const batchResp = await fetch(`${OPENAI_BASE_URL}/v1/batches`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${apiKey}`,
+            },
+            body: JSON.stringify({
+                input_file_id: fileData.id,
+                endpoint: "/v1/chat/completions",
+                completion_window: "24h",
+            }),
+        });
+        if (!batchResp.ok) {
+            const errText = await batchResp.text();
+            throw new Error(`OpenAI batch create failed (${batchResp.status}): ${errText.substring(0, 500)}`);
+        }
+        const batch = await batchResp.json();
+        log.info({ batchId: batch.id, fileId: fileData.id, count: requests.length }, "openai batch submitted");
+        return { batchId: batch.id, provider: "openai", count: requests.length };
+    }
+    async pollOpenAIBatch(batchId) {
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey)
+            throw new Error("OPENAI_API_KEY not set");
+        const startTime = Date.now();
+        while (Date.now() - startTime < MAX_POLL_DURATION_MS) {
+            const resp = await fetch(`${OPENAI_BASE_URL}/v1/batches/${batchId}`, {
+                headers: { Authorization: `Bearer ${apiKey}` },
+            });
+            if (!resp.ok) {
+                const errText = await resp.text();
+                throw new Error(`OpenAI batch poll failed (${resp.status}): ${errText.substring(0, 500)}`);
+            }
+            const batch = await resp.json();
+            log.info({
+                batchId,
+                status: batch.status,
+                counts: batch.request_counts,
+                elapsed: Date.now() - startTime,
+            }, "openai batch poll");
+            if (batch.status === "completed" && batch.output_file_id) {
+                return this.fetchOpenAIResults(batch.output_file_id);
+            }
+            if (batch.status === "failed" || batch.status === "cancelled" || batch.status === "expired") {
+                throw new Error(`OpenAI batch ${batchId} ended with status: ${batch.status}`);
+            }
+            await sleep(POLL_INTERVAL_MS);
+        }
+        throw new Error(`OpenAI batch ${batchId} timed out after ${MAX_POLL_DURATION_MS / 1000}s`);
+    }
+    async fetchOpenAIResults(outputFileId) {
+        const apiKey = process.env.OPENAI_API_KEY;
+        if (!apiKey)
+            throw new Error("OPENAI_API_KEY not set");
+        const resp = await fetch(`${OPENAI_BASE_URL}/v1/files/${outputFileId}/content`, {
+            headers: { Authorization: `Bearer ${apiKey}` },
+        });
+        if (!resp.ok) {
+            const errText = await resp.text();
+            throw new Error(`OpenAI results fetch failed (${resp.status}): ${errText.substring(0, 500)}`);
+        }
+        const text = await resp.text();
+        const results = new Map();
+        for (const line of text.split("\n")) {
+            if (!line.trim())
+                continue;
+            try {
+                const item = JSON.parse(line);
+                if (item.response && item.response.status_code === 200) {
+                    const choice = item.response.body.choices[0];
+                    results.set(item.custom_id, {
+                        id: item.custom_id,
+                        success: true,
+                        content: item.response.body.choices,
+                        text: choice?.message?.content || "",
+                        usage: {
+                            input_tokens: item.response.body.usage.prompt_tokens,
+                            output_tokens: item.response.body.usage.completion_tokens,
+                        },
+                    });
+                }
+                else {
+                    results.set(item.custom_id, {
+                        id: item.custom_id,
+                        success: false,
+                        error: item.error?.message || `HTTP ${item.response?.status_code || "unknown"}`,
+                    });
+                }
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+        log.info({ outputFileId, resultCount: results.size }, "openai batch results fetched");
+        return results;
+    }
+    /**
+     * Convert Anthropic-format messages to OpenAI chat format for batch requests.
+     * Simplified conversion — handles the common patterns (text, system, tool_use/tool_result).
+     */
+    buildOpenAIMessages(r) {
+        const out = [];
+        // System prompt → system message
+        if (r.system) {
+            if (typeof r.system === "string") {
+                out.push({ role: "system", content: r.system });
+            }
+            else if (Array.isArray(r.system)) {
+                const systemText = r.system
+                    .map(s => s.text || "")
+                    .filter(Boolean)
+                    .join("\n");
+                if (systemText)
+                    out.push({ role: "system", content: systemText });
+            }
+        }
+        // Convert messages
+        for (const msg of r.messages) {
+            if (msg.role === "user") {
+                if (typeof msg.content === "string") {
+                    out.push({ role: "user", content: msg.content });
+                }
+                else if (Array.isArray(msg.content)) {
+                    const parts = [];
+                    for (const block of msg.content) {
+                        if (block.type === "text" && block.text) {
+                            parts.push({ type: "text", text: block.text });
+                        }
+                        else if (block.type === "tool_result") {
+                            // OpenAI tool results are separate messages
+                            let output = "";
+                            if (typeof block.content === "string")
+                                output = block.content;
+                            else if (Array.isArray(block.content)) {
+                                output = block.content
+                                    .filter(c => c.type === "text")
+                                    .map(c => c.text)
+                                    .join("\n");
+                            }
+                            out.push({
+                                role: "tool",
+                                tool_call_id: block.tool_use_id,
+                                content: output || "(no output)",
+                            });
+                        }
+                    }
+                    if (parts.length === 1 && parts[0].type === "text") {
+                        out.push({ role: "user", content: parts[0].text });
+                    }
+                    else if (parts.length > 0) {
+                        out.push({ role: "user", content: parts });
+                    }
+                }
+            }
+            else if (msg.role === "assistant") {
+                if (typeof msg.content === "string") {
+                    out.push({ role: "assistant", content: msg.content });
+                }
+                else if (Array.isArray(msg.content)) {
+                    let textContent = "";
+                    const toolCalls = [];
+                    for (const block of msg.content) {
+                        if (block.type === "text" && block.text) {
+                            textContent += block.text;
+                        }
+                        else if (block.type === "tool_use") {
+                            toolCalls.push({
+                                id: block.id,
+                                type: "function",
+                                function: {
+                                    name: block.name,
+                                    arguments: JSON.stringify(block.input || {}),
+                                },
+                            });
+                        }
+                    }
+                    const assistantMsg = { role: "assistant" };
+                    if (textContent)
+                        assistantMsg.content = textContent;
+                    if (toolCalls.length)
+                        assistantMsg.tool_calls = toolCalls;
+                    out.push(assistantMsg);
+                }
+            }
+        }
+        return out;
+    }
+}
+// ============================================================================
+// HELPERS
+// ============================================================================
+function sleep(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+}
+// ============================================================================
+// SINGLETON EXPORT
+// ============================================================================
+export const batchClient = new BatchClient();

package/dist/server/lib/code-worker-pool.d.ts

@@ -0,0 +1,31 @@
+interface ExecuteRequest {
+    code: string;
+    context: {
+        steps: Record<string, unknown>;
+        trigger: Record<string, unknown>;
+        input?: unknown;
+    };
+    timeoutMs: number;
+}
+interface ExecuteResult {
+    success: boolean;
+    output?: unknown;
+    error?: string;
+    logs?: string[];
+}
+/** Initialize the worker pool. Safe to call multiple times. */
+export declare function initWorkerPool(): void;
+/**
+ * Execute code using the worker pool. Falls back to direct fork if pool is unavailable.
+ */
+export declare function executeWithPool(req: ExecuteRequest): Promise<ExecuteResult>;
+/** Gracefully shutdown all workers */
+export declare function shutdownPool(): void;
+/** Pool stats for monitoring */
+export declare function getPoolStats(): {
+    total: number;
+    busy: number;
+    idle: number;
+    queue: number;
+};
+export {};