whale-code 6.4.0 → 6.5.0

package/dist/server/lib/server-agent-loop.js

@@ -13,6 +13,7 @@ import { processStreamWithCallbacks } from "../../shared/sse-parser.js";
 import { MODELS } from "../../shared/constants.js";
 import { dispatchTools, buildAssistantContent } from "../../shared/tool-dispatch.js";
 import { getCachedToolDefs, getFullToolSchemas } from "../tool-router.js";
+import { queueSpan, auditRowToSpan } from "./clickhouse-buffer.js";
 import { DELEGATE_TASK_TOOL_DEF, runServerSubagent, } from "./server-subagent.js";
 import { handleTranscribe } from "../handlers/transcription.js";
 import { preCompact } from "./compaction-service.js";
@@ -49,7 +50,7 @@ function mapToolChoiceForAnthropic(tc) {
 // UNIFIED AGENT LOOP
 // ============================================================================
 export async function runServerAgentLoop(opts) {
-    const { anthropic, model, systemPrompt, messages, tools: inputTools, maxTurns, temperature, enableDelegation = true, enablePromptCaching = true, enableStreaming = true, maxConcurrentTools = DEFAULT_MAX_CONCURRENT_TOOLS, maxCostUsd = DEFAULT_SESSION_COST_BUDGET_USD, onText, onToolStart, onCitation, documents, clientDisconnected = { value: false }, startedAt = Date.now(), maxDurationMs = 5 * 60 * 1000, } = opts;
+    const { anthropic, model, systemPrompt, messages, tools: inputTools, maxTurns, temperature, enableDelegation = true, enablePromptCaching = true, enableStreaming = true, maxConcurrentTools = DEFAULT_MAX_CONCURRENT_TOOLS, maxCostUsd = DEFAULT_SESSION_COST_BUDGET_USD, onText, onToolStart, onCitation, documents, clientDisconnected = { value: false }, startedAt = Date.now(), maxDurationMs = 15 * 60 * 1000, } = opts;
     // Auto-inject delegate_task for all models (subagents always use Claude Haiku/Sonnet)
     // activeTools is mutable — discover_tools adds to it during the session
     const activeTools = [...inputTools];
@@ -92,10 +93,12 @@ export async function runServerAgentLoop(opts) {
     let sessionCostUsd = 0;
     let compactionCount = 0;
     let finalResponse = "";
+    let lastStopReason = "end_turn";
     const allTextResponses = [];
     const allToolNames = [];
     const allCitations = [];
     const turnMetrics = [];
+    const costWarningsEmitted = new Set();
     while (turnCount < maxTurns) {
         // Abort checks
         if (clientDisconnected.value) {
@@ -164,13 +167,17 @@ export async function runServerAgentLoop(opts) {
             ];
         // Resolve tool_choice for this turn
         const recentToolUses = turnMetrics.slice(-3).flatMap(t => t.toolsUsed);
-        const resolvedToolChoice = resolveToolChoice({
+        let resolvedToolChoice = resolveToolChoice({
             toolChoice: opts.toolChoice,
             turnCount,
             recentToolUses,
             availableToolNames: tools.map(t => t.name),
             userMessage: firstUserText,
         });
+        // Anthropic API: forced tool_choice ("any" or specific tool) is incompatible with thinking — downgrade to "auto"
+        if (thinkingCfg.thinking.type !== "disabled" && resolvedToolChoice !== "auto" && resolvedToolChoice !== "none") {
+            resolvedToolChoice = "auto";
+        }
         const { toolChoice: anthropicToolChoice, omitTools } = mapToolChoiceForAnthropic(resolvedToolChoice);
         if (omitTools) {
             log.info({ turn: turnCount, resolvedToolChoice }, "tool_choice=none — omitting tools");
@@ -254,6 +261,15 @@ export async function runServerAgentLoop(opts) {
             cacheReadTokens += turnCacheRead;
             // Update cost (include cache tokens for accurate pricing)
             sessionCostUsd = estimateCostUsd(totalIn, totalOut, model, 0, cacheReadTokens, cacheCreationTokens);
+            // Graduated cost warnings — give the LLM visibility into spend
+            if (isFinite(maxCostUsd)) {
+                for (const pct of [25, 50, 75]) {
+                    if (!costWarningsEmitted.has(pct) && sessionCostUsd >= maxCostUsd * (pct / 100)) {
+                        costWarningsEmitted.add(pct);
+                        onText?.(`\n[Cost warning: ${pct}% of budget used ($${sessionCostUsd.toFixed(2)}/$${maxCostUsd.toFixed(2)}). ${pct >= 75 ? "Wrap up soon." : ""}]`);
+                    }
+                }
+            }
             // Record per-turn metrics for observability
             const turnToolNames = toolUseBlocks.map(b => b.name);
             turnMetrics.push({
@@ -271,6 +287,7 @@ export async function runServerAgentLoop(opts) {
             // Compaction handling — API paused after generating summary.
             // Preserve last 2 messages (1 user + 1 assistant turn) for continuity,
             // then resume. This is NOT a new turn — just context compression.
+            lastStopReason = streamResult.stopReason || "end_turn";
             if (streamResult.stopReason === "compaction" && compactionContent) {
                 compactionCount++;
                 log.info({ compactionCount }, "compaction — preserving last 2 messages, resuming");
@@ -331,6 +348,15 @@ export async function runServerAgentLoop(opts) {
             totalIn += subagentTokens.input;
             totalOut += subagentTokens.output;
             sessionCostUsd = estimateCostUsd(totalIn, totalOut, model, 0, cacheReadTokens, cacheCreationTokens) + subagentTokens.costUsd;
+            // Cost warnings after subagent aggregation (subagents can be expensive)
+            if (isFinite(maxCostUsd)) {
+                for (const pct of [25, 50, 75]) {
+                    if (!costWarningsEmitted.has(pct) && sessionCostUsd >= maxCostUsd * (pct / 100)) {
+                        costWarningsEmitted.add(pct);
+                        onText?.(`\n[Cost warning: ${pct}% of budget used ($${sessionCostUsd.toFixed(2)}/$${maxCostUsd.toFixed(2)}). ${pct >= 75 ? "Wrap up soon." : ""}]`);
+                    }
+                }
+            }
             const assistantContent = buildAssistantContent({ text: currentText, toolUseBlocks, compactionContent });
             messages.push({ role: "assistant", content: assistantContent });
             messages.push({ role: "user", content: toolResults });
@@ -426,6 +452,15 @@ export async function runServerAgentLoop(opts) {
                 }
             }
             sessionCostUsd = estimateCostUsd(totalIn, totalOut, model, 0, cacheReadTokens, cacheCreationTokens);
+            // Graduated cost warnings (non-streaming path)
+            if (isFinite(maxCostUsd)) {
+                for (const pct of [25, 50, 75]) {
+                    if (!costWarningsEmitted.has(pct) && sessionCostUsd >= maxCostUsd * (pct / 100)) {
+                        costWarningsEmitted.add(pct);
+                        onText?.(`\n[Cost warning: ${pct}% of budget used ($${sessionCostUsd.toFixed(2)}/$${maxCostUsd.toFixed(2)}). ${pct >= 75 ? "Wrap up soon." : ""}]`);
+                    }
+                }
+            }
             // Record per-turn metrics (non-streaming)
             const nsTurnToolNames = toolUseBlocks.map(b => b.name);
             turnMetrics.push({
@@ -440,6 +475,7 @@ export async function runServerAgentLoop(opts) {
             });
             if (currentText)
                 allTextResponses.push(currentText);
+            lastStopReason = response.stop_reason || "end_turn";
             // Compaction handling (non-streaming) — same logic as streaming path
             if (response.stop_reason === "compaction" && nsCompactionContent !== null) {
                 compactionCount++;
@@ -488,6 +524,15 @@ export async function runServerAgentLoop(opts) {
             totalIn += nonStreamSubTokens.input;
             totalOut += nonStreamSubTokens.output;
             sessionCostUsd = estimateCostUsd(totalIn, totalOut, model, 0, cacheReadTokens, cacheCreationTokens) + nonStreamSubTokens.costUsd;
+            // Cost warnings after subagent aggregation (non-streaming)
+            if (isFinite(maxCostUsd)) {
+                for (const pct of [25, 50, 75]) {
+                    if (!costWarningsEmitted.has(pct) && sessionCostUsd >= maxCostUsd * (pct / 100)) {
+                        costWarningsEmitted.add(pct);
+                        onText?.(`\n[Cost warning: ${pct}% of budget used ($${sessionCostUsd.toFixed(2)}/$${maxCostUsd.toFixed(2)}). ${pct >= 75 ? "Wrap up soon." : ""}]`);
+                    }
+                }
+            }
             const assistantContent = buildAssistantContent({ text: currentText, toolUseBlocks });
             messages.push({ role: "assistant", content: assistantContent });
             messages.push({ role: "user", content: toolResults });
@@ -514,21 +559,22 @@ export async function runServerAgentLoop(opts) {
         loopDetectorStats: loopDetector.getSessionStats(),
         turns: turnMetrics,
         citations: allCitations,
+        stopReason: lastStopReason,
     };
 }
 // ============================================================================
 // TOOL EXECUTOR FACTORY — creates executor for dispatchTools with delegation
 // ============================================================================
 function makeToolExecutor(opts, tools, allToolNames, subagentTokens, discoveredToolNames) {
-    const { anthropic, supabase, storeId, traceId, userId, userEmail, conversationId, agentId, executeTool, onToolResult, onToolProgress, onSubagentProgress, clientDisconnected = { value: false }, startedAt = Date.now(), maxDurationMs = 5 * 60 * 1000, } = opts;
+    const { anthropic, supabase, storeId, traceId, userId, userEmail, conversationId, agentId, executeTool, onToolResult, onToolProgress, onSubagentProgress, clientDisconnected = { value: false }, startedAt = Date.now(), maxDurationMs = 15 * 60 * 1000, } = opts;
     return async (name, input) => {
         allToolNames.push(name);
-        // Subagent delegation
+        // Subagent delegation — demote models to control cost (sub-agents should never run Opus)
         if (name === "delegate_task") {
             const subPrompt = String(input.prompt || "");
             const subModelInput = String(input.model || "haiku");
-            const subModel = (subModelInput === "opus" ? "opus" :
-                subModelInput === "sonnet" ? "sonnet" : "haiku");
+            const subModel = (subModelInput === "opus" ? "sonnet" :
+                subModelInput === "sonnet" ? "haiku" : "haiku");
             const subMaxTurns = Math.min(Math.max(1, Number(input.max_turns) || 6), 12);
             const subTools = tools.filter((t) => t.name !== "delegate_task");
             const subId = `sub-${Date.now().toString(36)}`;
@@ -547,37 +593,37 @@ function makeToolExecutor(opts, tools, allToolNames, subagentTokens, discoveredT
                 : subModel === "sonnet" ? MODELS.SONNET : MODELS.HAIKU;
             try {
                 const subEndTime = Date.now();
-                const subBytes = new Uint8Array(8);
-                crypto.getRandomValues(subBytes);
-                const subSpanId = Array.from(subBytes).map(b => b.toString(16).padStart(2, "0")).join("");
-                await supabase.from("audit_logs").insert({
+                queueSpan(auditRowToSpan({
                     action: "chat.subagent_complete", severity: "info",
                     store_id: storeId || null, resource_type: "chat_subagent",
                     resource_id: agentId || null, request_id: traceId || null,
                     conversation_id: conversationId || null, source: "server_subagent",
-                    user_id: userId || null, user_email: userEmail || null,
+                    user_id: userId || null,
+                    user_email: userEmail || null,
                     input_tokens: subResult.tokensUsed.input, output_tokens: subResult.tokensUsed.output,
                     total_cost: subResult.costUsd, model: subModelId, duration_ms: subDurationMs,
-                    // OTEL fields
                     trace_id: traceId || null,
-                    span_id: subSpanId,
                     span_kind: "INTERNAL",
                     service_name: "agent-server",
                     status_code: subResult.success ? "OK" : "ERROR",
                     start_time: new Date(subEndTime - subDurationMs).toISOString(),
                     end_time: new Date(subEndTime).toISOString(),
+                    stop_reason: subResult.stopReason || undefined,
+                    turn_number: subResult.turnCount || 1,
+                    parent_conversation_id: conversationId || undefined,
                     details: {
                         subagent_model: subModel, turn_count: subResult.turnCount,
                         tool_calls: subResult.toolsUsed.length, tool_names: subResult.toolsUsed,
                         cost_usd: subResult.costUsd, success: subResult.success,
                         prompt_preview: subPrompt.substring(0, 200),
-                        // gen_ai fields for SwiftUI cost display
                         "gen_ai.request.model": subModelId,
                         "gen_ai.usage.input_tokens": subResult.tokensUsed.input,
                         "gen_ai.usage.output_tokens": subResult.tokensUsed.output,
+                        "gen_ai.usage.cache_read_tokens": subResult.tokensUsed.cacheRead || 0,
+                        "gen_ai.usage.cache_creation_tokens": subResult.tokensUsed.cacheCreation || 0,
                         "gen_ai.usage.cost": subResult.costUsd,
                     },
-                });
+                }));
             }
             catch (err) {
                 log.error({ err: err.message }, "failed to log subagent delegation audit");

package/dist/server/lib/server-subagent.d.ts

@@ -12,10 +12,13 @@ export interface SubagentResult {
     tokensUsed: {
         input: number;
         output: number;
+        cacheRead: number;
+        cacheCreation: number;
     };
     costUsd: number;
     toolsUsed: string[];
     turnCount: number;
+    stopReason: string;
 }
 export interface SubagentProgressEvent {
     subagentId: string;

package/dist/server/lib/server-subagent.js

@@ -5,7 +5,7 @@
  * from index.ts (avoids circular deps) and is fully testable in isolation.
  */
 import { randomUUID } from "node:crypto";
-import { LoopDetector, estimateCostUsd, sanitizeError, isRetryableError, } from "../../shared/agent-core.js";
+import { LoopDetector, estimateCostUsd, sanitizeError, isRetryableError, addPromptCaching, } from "../../shared/agent-core.js";
 import { buildAPIRequest } from "../../shared/api-client.js";
 import { MODELS, MODEL_MAP } from "../../shared/constants.js";
 import { dispatchTools } from "../../shared/tool-dispatch.js";
@@ -90,6 +90,8 @@ export async function runServerSubagent(opts) {
         turn++;
         loopDetector.resetTurn();
         onProgress?.({ subagentId, event: "turn", turn, maxTurns });
+        // Apply prompt caching to tools and messages for cache hits on repeated context
+        const { tools: cachedTools, messages: cachedMessages } = addPromptCaching(tools, messages);
         // Non-streaming API call with retry
         let response;
         try {
@@ -99,8 +101,8 @@ export async function runServerSubagent(opts) {
                     max_tokens: apiConfig.maxTokens,
                     temperature: shouldThink ? 1 : 0.3, // Anthropic requires temp=1 with thinking
                     system,
-                    tools: tools,
-                    messages: messages,
+                    tools: cachedTools,
+                    messages: cachedMessages,
                     betas: apiConfig.betas,
                     context_management: apiConfig.contextManagement,
                     ...(apiConfig.thinking ? { thinking: apiConfig.thinking } : {}),
@@ -178,10 +180,11 @@ function makeResult(success, output, inputTokens, outputTokens, modelId, toolsUs
     return {
         success,
         output,
-        tokensUsed: { input: inputTokens, output: outputTokens },
+        tokensUsed: { input: inputTokens, output: outputTokens, cacheRead: cacheReadTokens, cacheCreation: cacheCreationTokens },
         costUsd: estimateCostUsd(inputTokens, outputTokens, modelId, 0, cacheReadTokens, cacheCreationTokens),
         toolsUsed: [...new Set(toolsUsed)],
         turnCount,
+        stopReason: "end_turn", // subagent always runs to completion or error
     };
 }
 async function withSubagentRetry(fn, maxRetries = 2) {

package/dist/server/lib/supabase-client.js

@@ -1,13 +1,53 @@
-// lib/supabase-client.ts — Resilient Supabase client with retry logic
+// lib/supabase-client.ts — Resilient Supabase client with circuit breaker
 // Fixes intermittent 520 errors from Cloudflare by retrying failed requests
 // and reusing client instances instead of creating new ones per request.
+// Circuit breaker prevents retry storms when Supabase is down.
 import { createClient } from "@supabase/supabase-js";
 const MAX_RETRIES = 3;
 const INITIAL_BACKOFF_MS = 500;
 const MAX_BACKOFF_MS = 5_000;
-/** Custom fetch with retry for 5xx errors (Cloudflare 520/522/524) */
+// ── Circuit breaker ──
+// When Supabase is consistently failing, stop retrying to prevent:
+// 1. Retry storms that make Supabase worse
+// 2. Event loop blocking that causes health check failures
+// 3. SSE stream stalls that freeze the CLI
+const CIRCUIT_FAILURE_THRESHOLD = 10; // Open circuit after 10 consecutive failures
+const CIRCUIT_RESET_MS = 30_000; // Try again after 30s
+let circuitFailures = 0;
+let circuitOpenUntil = 0;
+/** Check if circuit breaker allows the request */
+function isCircuitOpen() {
+    if (circuitFailures < CIRCUIT_FAILURE_THRESHOLD)
+        return false;
+    if (Date.now() >= circuitOpenUntil) {
+        // Half-open: allow one probe request
+        return false;
+    }
+    return true;
+}
+function recordCircuitFailure() {
+    circuitFailures++;
+    if (circuitFailures >= CIRCUIT_FAILURE_THRESHOLD) {
+        circuitOpenUntil = Date.now() + CIRCUIT_RESET_MS;
+        if (circuitFailures === CIRCUIT_FAILURE_THRESHOLD) {
+            console.warn(`[supabase] Circuit breaker OPEN — ${circuitFailures} consecutive failures, pausing for ${CIRCUIT_RESET_MS / 1000}s`);
+        }
+    }
+}
+function recordCircuitSuccess() {
+    if (circuitFailures > 0) {
+        console.info(`[supabase] Circuit breaker reset — connection recovered`);
+    }
+    circuitFailures = 0;
+    circuitOpenUntil = 0;
+}
+/** Custom fetch with retry for 5xx errors + circuit breaker */
 function createRetryFetch(maxRetries = MAX_RETRIES) {
     return async (input, init) => {
+        // Circuit breaker: fail fast when Supabase is known to be down
+        if (isCircuitOpen()) {
+            throw new Error("Supabase circuit breaker open — skipping request");
+        }
         let lastError = null;
         for (let attempt = 0; attempt <= maxRetries; attempt++) {
             try {
@@ -18,17 +58,25 @@ function createRetryFetch(maxRetries = MAX_RETRIES) {
                 });
                 // Retry on 5xx errors (Cloudflare 520 = origin error, 522 = timeout, 524 = timeout)
                 if (res.status >= 500 && attempt < maxRetries) {
+                    recordCircuitFailure();
+                    if (isCircuitOpen()) {
+                        return res; // Don't retry if circuit just opened
+                    }
                     const backoff = Math.min(INITIAL_BACKOFF_MS * Math.pow(2, attempt), MAX_BACKOFF_MS);
                     console.warn(`[supabase] ${res.status} on ${typeof input === 'string' ? input.split('?')[0] : 'request'}, retry ${attempt + 1}/${maxRetries} in ${backoff}ms`);
                     await new Promise(r => setTimeout(r, backoff));
                     continue;
                 }
+                if (res.status < 500) {
+                    recordCircuitSuccess();
+                }
                 return res;
             }
             catch (err) {
                 lastError = err;
+                recordCircuitFailure();
                 // Retry on network errors (ECONNRESET, ETIMEDOUT, etc.)
-                if (attempt < maxRetries) {
+                if (attempt < maxRetries && !isCircuitOpen()) {
                     const backoff = Math.min(INITIAL_BACKOFF_MS * Math.pow(2, attempt), MAX_BACKOFF_MS);
                     console.warn(`[supabase] Network error: ${lastError.message}, retry ${attempt + 1}/${maxRetries} in ${backoff}ms`);
                     await new Promise(r => setTimeout(r, backoff));

package/dist/server/lib/template-resolver.js

@@ -159,14 +159,24 @@ export function evaluateCondition(expression, ctx) {
         [/\scontains\s/, "contains"],
         [/\s!contains\s/, "!contains"],
     ];
+    // Identify quoted regions to avoid matching operators inside them
+    const quoteRegions = [];
+    const quoteRe = /(['"])(?:(?!\1).)*\1/g;
+    let qm;
+    while ((qm = quoteRe.exec(expr)) !== null) {
+        quoteRegions.push([qm.index, qm.index + qm[0].length]);
+    }
     for (const [pattern, op] of operatorPatterns) {
-        // Use the LAST match: resolved template data (left side) may contain
-        // operator-like strings, but the actual operator is the rightmost one.
+        // Use the LAST match that is NOT inside a quoted string.
         const globalRe = new RegExp(pattern.source, "g");
         let lastMatch = null;
         let m;
-        while ((m = globalRe.exec(expr)) !== null)
-            lastMatch = m;
+        while ((m = globalRe.exec(expr)) !== null) {
+            const idx = m.index;
+            const insideQuotes = quoteRegions.some(([s, e]) => idx > s && idx < e);
+            if (!insideQuotes)
+                lastMatch = m;
+        }
         if (!lastMatch || lastMatch.index === undefined)
             continue;
         let left = expr.substring(0, lastMatch.index).trim();

package/dist/server/lib/utils.js

@@ -114,6 +114,21 @@ export function summarizeResult(toolName, action, data) {
                 if (action === "browse")
                     return { categories: d.data ? d.data.categories?.length || 0 : 0, status_summary: d.data ? d.data.product_status_summary : undefined };
                 return { action, count: Array.isArray(d) ? d.length : (d.count ?? (d.products ? d.products.length : (d.id ? 1 : 0))) };
+            case "media": {
+                if (action === "list" || action === "search")
+                    return { count: d.count, total: d.total };
+                if (action === "analytics")
+                    return { total_items: d.total_items, orphan_count: d.orphan_count, storage_mb: d.total_storage_mb };
+                if (action === "upload")
+                    return { media_id: d.media_id, file_name: d.file_name };
+                if (action === "bulk_upload")
+                    return { uploaded_count: d.uploaded_count, failed_count: d.failed_count };
+                if (action === "bulk_update")
+                    return { updated_count: d.updated_count, failed_count: d.failed_count };
+                if (action === "usage")
+                    return { reference_count: d.reference_count };
+                return { action };
+            }
             case "audit_trail":
                 return { count: d.count, days: d.days, actions: d.summary ? Object.keys(d.summary).length : 0 };
             case "telemetry":

package/dist/server/local-agent-gateway.d.ts

@@ -16,6 +16,7 @@ export interface LocalAgent {
     ws: WebSocket;
     storeId: string;
     userId: string | null;
+    nodeId: string | null;
     capabilities: string[];
     connectedAt: Date;
     lastPong: number;
@@ -50,6 +51,7 @@ export declare function getAgentInfo(storeId: string): Array<{
     capabilities: string[];
     connected_at: string;
     uptime_seconds: number;
+    node_id: string | null;
 }>;
 /**
  * Execute a command on the user's local machine via their connected agent.
@@ -79,4 +81,46 @@ export declare function getGatewayStats(): {
     pending_requests: number;
     agents_by_store: Record<string, number>;
 };
+/**
+ * Get active remote desktop sessions.
+ */
+export declare function getRemoteDesktopSessions(): Array<{
+    session_id: string;
+    store_id: string;
+    user_id: string;
+    agent_id: string;
+    started_at: string;
+    frames_relayed: number;
+    bytes_relayed: number;
+}>;
+/**
+ * Get active portal sessions.
+ */
+export declare function getPortalSessions(): Array<{
+    session_id: string;
+    store_id: string;
+    initiator_agent_id: string;
+    target_agent_id: string;
+    capabilities: string[];
+    started_at: string;
+    bytes_relayed: number;
+}>;
+export interface ClusterCommandArgs {
+    action: string;
+    task?: string;
+    max_cells?: number;
+    working_directory?: string;
+    cluster_id?: string;
+    channel_id?: string;
+    conversation_id?: string;
+}
+/**
+ * Execute a cluster command on the connected whale-node.
+ * Sends via WebSocket and waits for the result.
+ */
+export declare function executeClusterCommand(storeId: string, args: ClusterCommandArgs, options?: {
+    timeout?: number;
+    agent_id?: string;
+    node_id?: string;
+}): Promise<AgentResult>;
 export declare function shutdownGateway(): void;