whale-code 6.4.0 → 6.5.1

package/dist/server/index.js

@@ -16,13 +16,14 @@ import { handleTranscribe } from "./handlers/transcription.js";
 import { handleBillingRoutes, incrementUsage, checkPlanLimits } from "./handlers/billing.js";
 import { generateCompaction } from "./lib/compaction-service.js";
 import { initLocalAgentGateway, shutdownGateway as shutdownAgentGateway, getGatewayStats } from "./local-agent-gateway.js";
-import { initSupabase, getServiceClient, createUserClient } from "./lib/supabase-client.js";
+import { initSupabase, getServiceClient } from "./lib/supabase-client.js";
 import { loadCheckpoint, markOrphaned } from "./lib/session-checkpoint.js";
 import { rateLimiter } from "./lib/rate-limiter.js";
 import { sanitizeAndLog } from "./lib/prompt-sanitizer.js";
 import { processWorkflowSteps, processWaitingSteps, handleWebhookIngestion, executeInlineChain, setToolExecutor, setAgentExecutor, setTokenBroadcaster, setStepErrorBroadcaster, verifyGuestApprovalSignature, initWorkerPool, getPoolStats, shutdownPool, processScheduleTriggers, enforceWorkflowTimeouts, processEventTriggers, cleanupOrphanedSteps, processDlqRetries } from "./handlers/workflows.js";
 import { runServerAgentLoop } from "./lib/server-agent-loop.js";
-import { loadTools, loadUserTools, getToolsForAgent, executeTool, loadAgentConfig, setExtendedToolsCache, getExtendedToolsIndex, flushAuditLogs, } from "./tool-router.js";
+import { loadTools, loadUserTools, getToolsForAgent, executeTool, loadAgentConfig, setExtendedToolsCache, getExtendedToolsIndex, flushSpans, } from "./tool-router.js";
+import { queueSpan, auditRowToSpan, classifyErrorType } from "./lib/clickhouse-buffer.js";
 import pg from "pg";
 // ============================================================================
 // PROCESS ERROR HANDLERS
@@ -44,6 +45,9 @@ const SERVICE_ROLE_JWT = process.env.SERVICE_ROLE_JWT || "";
 const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
 const ALLOWED_ORIGINS = (process.env.ALLOWED_ORIGINS || "http://localhost:3000,http://127.0.0.1:3000").split(",").map(s => s.trim());
 const FLY_INTERNAL_SECRET = process.env.FLY_INTERNAL_SECRET || "";
+if (!FLY_INTERNAL_SECRET) {
+    console.warn("[SECURITY] FLY_INTERNAL_SECRET is not set — internal endpoints are unprotected");
+}
 // ============================================================================
 // READINESS STATE
 // ============================================================================
@@ -114,6 +118,76 @@ function safeCompare(a, b) {
 }
 // Tool registry, user tools, executor, and agent loader are in ./tool-router.ts
 // ============================================================================
+// STORE CONTEXT — Server-derived store resolution (Apple-style)
+// The server NEVER blindly trusts a client-supplied storeId. Instead it:
+//   1. Resolves the user's stores from the `users` table via auth.uid()
+//   2. Validates the client hint against the resolved set
+//   3. Falls back to the user's first store if no hint provided
+//   4. Returns null if no store can be resolved (caller must fail closed)
+// ============================================================================
+async function resolveAndValidateStoreId(supabase, clientStoreId, user, isServiceRole, _token, requestUserId) {
+    // Service-role callers: trusted ONLY for internal server-to-server calls
+    // (workflows, cron jobs) that have NO associated user. When a userId is
+    // present (e.g. MCP CLI using env-var service role key), we MUST still
+    // validate store membership to prevent cross-tenant access.
+    if (isServiceRole) {
+        const srUserId = user?.id || requestUserId;
+        if (!srUserId)
+            return clientStoreId || null; // true internal call — pass through
+        if (!clientStoreId)
+            return null; // user-context call without store hint
+        // Validate the user actually belongs to the requested store
+        const { data: membership } = await supabase
+            .from("store_members")
+            .select("id")
+            .eq("store_id", clientStoreId)
+            .eq("user_id", srUserId)
+            .limit(1)
+            .maybeSingle();
+        if (membership)
+            return clientStoreId;
+        // Fallback: check users table (legacy single-store pattern)
+        const { data: userRow } = await supabase
+            .from("users")
+            .select("store_id")
+            .eq("auth_user_id", srUserId)
+            .eq("store_id", clientStoreId)
+            .maybeSingle();
+        if (userRow)
+            return clientStoreId;
+        log.warn({ userId: srUserId, clientStoreId }, "resolveStoreId: service-role caller userId not authorized for store");
+        return null;
+    }
+    if (!user?.id)
+        return null;
+    // Resolve user's actual stores from the `users` table (auth_user_id = auth.uid())
+    const { data: userStores, error } = await supabase
+        .from("users")
+        .select("store_id")
+        .eq("auth_user_id", user.id)
+        .not("store_id", "is", null);
+    if (error || !userStores?.length) {
+        log.warn({ userId: user.id, error }, "resolveStoreId: user has no stores");
+        return null;
+    }
+    const storeIds = userStores.map((r) => r.store_id);
+    // If client provided a hint, validate it's in the user's set
+    if (clientStoreId) {
+        if (storeIds.includes(clientStoreId)) {
+            return clientStoreId;
+        }
+        log.warn({ userId: user.id, clientStoreId, userStores: storeIds }, "resolveStoreId: client storeId not in user's stores");
+        return null; // Reject — don't silently fall back
+    }
+    // No client hint — use first store (single-store users), or require explicit selection
+    if (storeIds.length === 1) {
+        return storeIds[0];
+    }
+    // Multi-store user without a hint — can't guess
+    log.warn({ userId: user.id, storeCount: storeIds.length }, "resolveStoreId: multi-store user must specify storeId");
+    return null;
+}
+// ============================================================================
 // CORS
 // ============================================================================
 function getCorsHeaders(origin) {
@@ -299,7 +373,7 @@ async function setupPgListen() {
 // ============================================================================
 function getAnthropicClient(agent) {
     const key = agent.api_key || ANTHROPIC_API_KEY;
-    return new Anthropic({ apiKey: key, timeout: 5 * 60 * 1000 }); // 5 min for tool-heavy requests
+    return new Anthropic({ apiKey: key, timeout: 15 * 60 * 1000 }); // 15 min for tool-heavy requests
 }
 function sendSSE(res, event) {
     try {
@@ -313,9 +387,85 @@ function jsonResponse(res, status, data, corsHeaders) {
     res.writeHead(status, { "Content-Type": "application/json", ...corsHeaders });
     res.end(JSON.stringify(data));
 }
+/**
+ * Strip large base64 data fields from a JSON string up to `keepFrom` offset.
+ * Uses a linear indexOf scan — safe on strings of any size (no regex stack overflow).
+ * Handles both `"data":"<base64>"` (Anthropic image blocks) and
+ * `"__IMAGE__<mime>__<base64>"` text values (Read tool marker format).
+ */
+function stripLargeBase64Fields(raw, keepFrom) {
+    const MIN_DATA_LEN = 8_000;
+    // ── Pass 1: Strip "data":"<large base64>" fields (Anthropic image source blocks) ──
+    let result = raw;
+    {
+        const DATA_MARKER = '"data":"';
+        const parts = [];
+        let pos = 0;
+        while (pos < keepFrom) {
+            const idx = result.indexOf(DATA_MARKER, pos);
+            if (idx === -1 || idx >= keepFrom) {
+                parts.push(result.slice(pos, keepFrom));
+                pos = keepFrom;
+                break;
+            }
+            parts.push(result.slice(pos, idx + DATA_MARKER.length));
+            pos = idx + DATA_MARKER.length;
+            // Find closing quote (base64 has no backslashes, so simple scan is safe)
+            let end = pos;
+            while (end < result.length && result[end] !== '"')
+                end++;
+            if (end - pos >= MIN_DATA_LEN && end <= keepFrom) {
+                // Large data field in the prune zone — replace with empty
+                parts.push('"');
+                pos = end + 1; // skip past the original closing quote (already replaced)
+            }
+            // else: small field or near keepFrom boundary — leave intact
+        }
+        parts.push(result.slice(keepFrom)); // always keep tail intact
+        result = parts.join("");
+    }
+    // ── Pass 2: Strip __IMAGE__<mime>__<large base64> text markers ──
+    // If the client-side tool-dispatch regex failed to convert these to image
+    // content blocks, they stay as huge text strings in tool_result content.
+    // They appear in JSON as: "__IMAGE__image/png__iVBOR...very long..."
+    {
+        const IMG_MARKER = "__IMAGE__";
+        const adjustedKeepFrom = Math.min(keepFrom, result.length);
+        const parts = [];
+        let pos = 0;
+        while (pos < adjustedKeepFrom) {
+            const idx = result.indexOf(IMG_MARKER, pos);
+            if (idx === -1 || idx >= adjustedKeepFrom) {
+                parts.push(result.slice(pos, adjustedKeepFrom));
+                pos = adjustedKeepFrom;
+                break;
+            }
+            parts.push(result.slice(pos, idx));
+            // Find the end of this text value (closing quote of the JSON string)
+            let end = idx;
+            while (end < result.length && result[end] !== '"')
+                end++;
+            if (end - idx >= MIN_DATA_LEN && end <= adjustedKeepFrom) {
+                // Large __IMAGE__ marker in the prune zone — replace with placeholder
+                parts.push("[image pruned]");
+                pos = end; // land on closing quote; next iteration emits it
+            }
+            else {
+                // Small or near boundary — keep intact
+                parts.push(result.slice(idx, idx + IMG_MARKER.length));
+                pos = idx + IMG_MARKER.length;
+            }
+        }
+        parts.push(result.slice(adjustedKeepFrom));
+        result = parts.join("");
+    }
+    return result;
+}
 async function readBody(req) {
-    // 50MB limit — proxy requests include full conversation history with base64 images
-    const MAX_BODY = 52_428_800;
+    // 500MB hard limit — conversation history with many large images can be very large.
+    // For bodies over 10MB we prune old base64 image data from the raw string before
+    // JSON.parse, so subsequent requests in the same conversation stay bounded.
+    const MAX_BODY = 524_288_000; // 500MB
     return new Promise((resolve, reject) => {
         const chunks = [];
         let size = 0;
@@ -324,16 +474,36 @@ async function readBody(req) {
             size += chunk.length;
             if (size > MAX_BODY && !rejected) {
                 rejected = true;
-                // Drain remaining data instead of destroying socket (avoids Fly proxy 502)
-                req.resume();
-                reject(new Error("Request body too large (max 50MB)"));
+                req.resume(); // drain to avoid Fly proxy 502
+                reject(new Error("Request body too large (max 500MB)"));
                 return;
             }
             if (!rejected)
                 chunks.push(chunk);
         });
-        req.on("end", () => { if (!rejected)
-            resolve(Buffer.concat(chunks).toString("utf8")); });
+        req.on("end", () => {
+            if (rejected)
+                return;
+            // Detect body truncation: if Content-Length was sent and we received less
+            const declaredLen = parseInt(req.headers["content-length"] || "0", 10);
+            if (declaredLen > 0 && size < declaredLen) {
+                log.error({ declaredLen, receivedLen: size }, "Request body truncated during transmission");
+                reject(new Error(`Body truncated: expected ${declaredLen} bytes, got ${size}`));
+                return;
+            }
+            let raw = Buffer.concat(chunks).toString("utf8");
+            // Strip old base64 data from history when body is large.
+            // Threshold lowered to 1MB — image-heavy conversations (e.g. reading
+            // product photos) easily exceed 10MB with just a few images.
+            // Keep the last 1MB intact (current user message with fresh images).
+            // Uses a linear scan to avoid String.replace() stack overflow on huge strings.
+            if (raw.length > 1_000_000) {
+                const keepFrom = Math.max(0, raw.length - 1_048_576); // keep last 1MB
+                raw = stripLargeBase64Fields(raw, keepFrom);
+                log.info({ originalBytes: size, prunedBytes: Buffer.byteLength(raw) }, "Pruned old base64 data from request body");
+            }
+            resolve(raw);
+        });
         req.on("error", reject);
     });
 }
@@ -517,9 +687,9 @@ async function persistAgentTurn(supabase, agent, opts) {
     catch (err) {
         log.error({ err: err.message }, "conversation update failed");
     }
-    // ── Audit log: user message ──
+    // ── Telemetry: user message → ClickHouse ──
     try {
-        await supabase.from("audit_logs").insert({
+        queueSpan(auditRowToSpan({
             action: "chat.user_message",
             severity: "info",
             store_id: storeId || null,
@@ -530,6 +700,13 @@ async function persistAgentTurn(supabase, agent, opts) {
             user_id: userId || null,
             user_email: userEmail || null,
             source,
+            service_name: "agent-server",
+            span_kind: "INTERNAL",
+            status_code: "OK",
+            start_time: new Date().toISOString(),
+            end_time: new Date().toISOString(),
+            duration_ms: 0,
+            input_bytes: typeof message === "string" ? message.length : 0,
             details: {
                 message_preview: message.substring(0, 200),
                 agent_id: agentId,
@@ -541,17 +718,14 @@ async function persistAgentTurn(supabase, agent, opts) {
                     customer_id: senderContext.customerId || null,
                 } : {}),
             },
-        });
+        }));
     }
     catch (err) {
         log.error({ err: err.message }, "audit user_message failed");
     }
-    // ── Audit log: assistant response (OTEL-enriched) ──
+    // ── Telemetry: assistant response → ClickHouse ──
     try {
-        const spanBytes = new Uint8Array(8);
-        crypto.getRandomValues(spanBytes);
-        const spanId = Array.from(spanBytes).map(b => b.toString(16).padStart(2, "0")).join("");
-        await supabase.from("audit_logs").insert({
+        queueSpan(auditRowToSpan({
             action: "chat.assistant_response",
             severity: "info",
             store_id: storeId || null,
@@ -568,12 +742,13 @@ async function persistAgentTurn(supabase, agent, opts) {
             total_cost: result.costUsd,
             model: agentModel,
             trace_id: traceId,
-            span_id: spanId,
             span_kind: "INTERNAL",
             service_name: "agent-server",
             status_code: "OK",
             start_time: new Date(chatStartTime).toISOString(),
             end_time: new Date(chatEndTime).toISOString(),
+            stop_reason: result.stopReason || undefined,
+            turn_number: result.turnCount || 1,
             details: {
                 response_preview: (result.finalText || "").substring(0, 500),
                 agent_id: agentId,
@@ -591,7 +766,6 @@ async function persistAgentTurn(supabase, agent, opts) {
                 session_cost_usd: result.costUsd,
                 cache_creation_tokens: result.tokens.cacheCreation || 0,
                 cache_read_tokens: result.tokens.cacheRead || 0,
-                // Cache efficiency metrics
                 cache_hit_rate: result.tokens.input > 0
                     ? Math.round((result.tokens.cacheRead || 0) / ((result.tokens.cacheRead || 0) + result.tokens.input) * 10000) / 100
                     : 0,
@@ -599,9 +773,7 @@ async function persistAgentTurn(supabase, agent, opts) {
                     ? Math.round((result.tokens.cacheRead || 0) * 0.9 / ((result.tokens.cacheRead || 0) + result.tokens.input) * 10000) / 100
                     : 0,
                 loop_detector_stats: result.loopDetectorStats || null,
-                // Per-turn token breakdowns for cost attribution
                 turns: result.turns || [],
-                // Channel-specific telemetry — fully dynamic
                 ...(senderContext ? {
                     channel_type: senderContext.channelType || null,
                     channel_id: senderContext.channelId || null,
@@ -611,7 +783,7 @@ async function persistAgentTurn(supabase, agent, opts) {
                     customer_name: senderContext.customerName || null,
                 } : {}),
             },
-        });
+        }));
     }
     catch (err) {
         log.error({ err: err.message }, "audit assistant_response failed");
@@ -629,12 +801,17 @@ async function persistAgentTurn(supabase, agent, opts) {
             }
             catch (err2) {
                 log.error({ err: err2.message }, "memory extract failed after retry");
-                await supabase.from("audit_logs").insert({
+                queueSpan(auditRowToSpan({
                     action: "memory.extraction_failed", severity: "warning",
                     store_id: storeId || null, resource_type: "agent_memory",
                     resource_id: agentId, conversation_id: conversationId,
+                    user_id: userId || null, user_email: userEmail || null,
+                    error_type: classifyErrorType(err2.message) || undefined,
+                    service_name: "agent-server", span_kind: "INTERNAL", status_code: "ERROR",
+                    start_time: new Date().toISOString(), end_time: new Date().toISOString(),
+                    error_message: err2.message,
                     details: { error: err2.message, user_message_preview: message.substring(0, 100) },
-                }).then(() => { });
+                }));
             }
         }
     }
@@ -672,51 +849,29 @@ async function handleAgentChat(req, res, supabase, body, user, isServiceRole, to
         jsonResponse(res, 400, { error: "Message too long (max 100K characters)" }, corsHeaders);
         return;
     }
-    // Fallback: resolve user's store when storeId not provided in request
-    if (!storeId && user?.id && !isServiceRole) {
-        try {
-            const { data: userStores } = await supabase
-                .from("user_stores")
-                .select("store_id")
-                .eq("user_id", user.id)
-                .limit(1);
-            if (userStores?.length) {
-                storeId = userStores[0].store_id;
-                log.info({ userId: user.id, storeId }, "resolved user store");
-            }
-        }
-        catch (err) {
-            log.error({ err }, "store resolution error");
-        }
+    // Server-derived store resolution — never trust client blindly
+    const resolvedStoreId = await resolveAndValidateStoreId(supabase, storeId, user, isServiceRole, token, body.userId);
+    // For service-role callers (workflows, internal), also try body.userId fallback
+    if (!resolvedStoreId && isServiceRole && body.userId) {
+        const { data: srStores } = await supabase
+            .from("users")
+            .select("store_id")
+            .eq("auth_user_id", body.userId)
+            .not("store_id", "is", null)
+            .limit(1);
+        if (srStores?.length) {
+            storeId = srStores[0].store_id;
+        }
+    }
+    else {
+        storeId = resolvedStoreId || undefined;
+    }
+    // Fail closed: non-service-role requests MUST have a resolved store
+    if (!storeId && !isServiceRole) {
+        jsonResponse(res, 400, { error: "storeId required for agent chat" }, corsHeaders);
+        return;
     }
     log.info({ storeId: storeId || "NONE", source: body.source || "unknown", isServiceRole, userId: user?.id || body.userId || "NONE" }, "agent-chat request");
-    // Fallback: resolve store from body.userId for service-role requests (e.g. WhaleChat app)
-    if (!storeId && !user?.id && body.userId && isServiceRole) {
-        try {
-            const { data: userStores } = await supabase
-                .from("user_stores")
-                .select("store_id")
-                .eq("user_id", body.userId)
-                .limit(1);
-            if (userStores?.length) {
-                storeId = userStores[0].store_id;
-                log.info({ userId: body.userId, storeId }, "resolved userId store");
-            }
-        }
-        catch (err) {
-            log.error({ err }, "store resolution error");
-        }
-    }
-    // Verify store access (skip for service_role)
-    if (storeId && !isServiceRole) {
-        const userClient = createUserClient(SUPABASE_URL, process.env.SUPABASE_ANON_KEY || "", token);
-        const { data: storeAccess, error: storeErr } = await userClient
-            .from("stores").select("id").eq("id", storeId).limit(1);
-        if (storeErr || !storeAccess?.length) {
-            jsonResponse(res, 403, { error: "Access denied to store" }, corsHeaders);
-            return;
-        }
-    }
     // Agent chat rate limiting — per-store + concurrent cap
     const rateLimitStoreId = storeId || agentId; // fallback to agentId if no store
     const rateCheck = checkAgentChatRateLimit(rateLimitStoreId);
@@ -729,7 +884,7 @@ async function handleAgentChat(req, res, supabase, body, user, isServiceRole, to
     try {
         const userId = user?.id || body.userId || "";
         const userEmail = user?.email || body.userEmail || null;
-        const agent = await loadAgentConfig(supabase, agentId, storeId || undefined);
+        const agent = await loadAgentConfig(supabase, agentId, storeId);
         if (!agent) {
             jsonResponse(res, 404, { error: "Agent not found" }, corsHeaders);
             return;
@@ -823,7 +978,7 @@ async function handleAgentChat(req, res, supabase, body, user, isServiceRole, to
         // Client disconnect detection
         let clientDisconnected = false;
         req.on("close", () => { clientDisconnected = true; });
-        const maxDurationMs = 5 * 60 * 1000;
+        const maxDurationMs = 15 * 60 * 1000;
         const startedAt = Date.now();
         const chatStartTime = Date.now();
         try {
@@ -974,7 +1129,7 @@ const server = http.createServer(async (req, res) => {
         const agentStats = getGatewayStats();
         jsonResponse(res, status, {
             status: ready ? "ok" : "starting",
-            version: process.env.npm_package_version || "6.0.0",
+            version: process.env.npm_package_version || "6.4.0",
             uptime: Math.floor(process.uptime()),
             pg_listen: pgListenReady,
             worker_pool: workerPoolReady,
@@ -1899,7 +2054,7 @@ const server = http.createServer(async (req, res) => {
             rawBody = await readBody(req);
         }
         catch {
-            jsonResponse(res, 413, { error: "Request body too large (max 50MB)" }, corsHeaders);
+            jsonResponse(res, 413, { error: "Request body too large (max 500MB)" }, corsHeaders);
             return;
         }
         const token = authHeader.substring(7);
@@ -1961,8 +2116,16 @@ const server = http.createServer(async (req, res) => {
         try {
             body = JSON.parse(rawBody);
         }
-        catch {
-            jsonResponse(res, 400, { error: "Invalid JSON in request body" }, corsHeaders);
+        catch (parseErr) {
+            const errMsg = parseErr instanceof Error ? parseErr.message : String(parseErr);
+            const bodyLen = rawBody.length;
+            const tail = bodyLen > 200 ? rawBody.slice(bodyLen - 200) : rawBody;
+            log.error({ bodyLen, errMsg, tail }, "JSON parse failed on request body");
+            jsonResponse(res, 400, {
+                error: "Invalid JSON in request body",
+                detail: errMsg,
+                body_length: bodyLen,
+            }, corsHeaders);
             return;
         }
         // Anthropic API proxy mode
@@ -2015,6 +2178,13 @@ const server = http.createServer(async (req, res) => {
                 jsonResponse(res, 400, { error: "tool_name required" }, corsHeaders);
                 return;
             }
+            // Resolve and validate store_id (server-derived, not blindly trusted)
+            const resolvedToolStoreId = await resolveAndValidateStoreId(supabase, store_id || body.storeId, user, isServiceRole, token, body.userId);
+            if (!resolvedToolStoreId && !isServiceRole) {
+                jsonResponse(res, 400, { error: "store_id required for tool execution" }, corsHeaders);
+                return;
+            }
+            const validToolStoreId = resolvedToolStoreId || store_id;
             // Phase 7.2: Per-tool rate limiting
             const toolUserId = user?.id || body.userId || "anon";
             const toolLimit = rateLimiter.checkToolLimit(toolUserId, tool_name);
@@ -2031,11 +2201,11 @@ const server = http.createServer(async (req, res) => {
             }
             // Load user tools if this is a user_tool__ prefixed call
             let utRows;
-            if (tool_name.startsWith("user_tool__") && store_id) {
-                const { rows } = await loadUserTools(supabase, store_id);
+            if (tool_name.startsWith("user_tool__") && validToolStoreId) {
+                const { rows } = await loadUserTools(supabase, validToolStoreId);
                 utRows = rows;
             }
-            const result = await executeTool(supabase, tool_name, (args || {}), store_id || undefined, trace_id || undefined, user?.id || body.userId || null, user?.email || body.userEmail || null, body.source || "whale-code", conversation_id || undefined, utRows);
+            const result = await executeTool(supabase, tool_name, (args || {}), validToolStoreId || undefined, trace_id || undefined, user?.id || body.userId || null, user?.email || body.userEmail || null, body.source || "whale-code", conversation_id || undefined, utRows);
             // Always 200 for tool results — success/failure is in the JSON body.
             // HTTP 500 causes MCP clients to throw before reading the error message.
             jsonResponse(res, 200, result, corsHeaders);
@@ -2048,6 +2218,13 @@ const server = http.createServer(async (req, res) => {
                 jsonResponse(res, 400, { error: "tool_name required" }, corsHeaders);
                 return;
             }
+            // Resolve and validate store_id
+            const resolvedStreamStoreId = await resolveAndValidateStoreId(supabase, store_id || body.storeId, user, isServiceRole, token, body.userId);
+            if (!resolvedStreamStoreId && !isServiceRole) {
+                jsonResponse(res, 400, { error: "store_id required for tool execution" }, corsHeaders);
+                return;
+            }
+            const validStreamStoreId = resolvedStreamStoreId || store_id;
             // Phase 7.2: Per-tool rate limiting (stream mode)
             const streamToolUserId = user?.id || body.userId || "anon";
             const streamToolLimit = rateLimiter.checkToolLimit(streamToolUserId, tool_name);
@@ -2069,12 +2246,19 @@ const server = http.createServer(async (req, res) => {
             });
             const onToolProgress = (_name, progress) => {
                 try {
-                    res.write(JSON.stringify({ type: "progress", progress }) + "\n");
+                    // Structured status events from kali — relay as type "status" for CLI ToolProgress handling
+                    const p = progress;
+                    if (p && p.type === "status" && p.progress) {
+                        res.write(JSON.stringify({ type: "status", progress: p.progress }) + "\n");
+                    }
+                    else {
+                        res.write(JSON.stringify({ type: "progress", progress }) + "\n");
+                    }
                 }
                 catch { /* client disconnected */ }
             };
             try {
-                const result = await executeTool(supabase, tool_name, (args || {}), store_id || undefined, body.trace_id || undefined, user?.id || body.userId || null, user?.email || body.userEmail || null, body.source || "whale-code-stream", body.conversation_id || undefined, undefined, undefined, onToolProgress);
+                const result = await executeTool(supabase, tool_name, (args || {}), validStreamStoreId || undefined, body.trace_id || undefined, user?.id || body.userId || null, user?.email || body.userEmail || null, body.source || "whale-code-stream", body.conversation_id || undefined, undefined, undefined, onToolProgress);
                 res.write(JSON.stringify({ type: "result", ...result }) + "\n");
             }
             catch (err) {
@@ -2083,6 +2267,55 @@ const server = http.createServer(async (req, res) => {
             res.end();
             return;
         }
+        // CLI telemetry ingest — batch of spans from Whale Code CLI
+        if (body.mode === "telemetry_ingest") {
+            const spans = body.spans;
+            if (!Array.isArray(spans) || spans.length === 0) {
+                jsonResponse(res, 400, { error: "spans array required" }, corsHeaders);
+                return;
+            }
+            // Cap at 500 spans per request to prevent abuse
+            const batch = spans.slice(0, 500);
+            let queued = 0;
+            for (const raw of batch) {
+                try {
+                    // Enforce store_id from auth context, not client-provided
+                    const resolvedStoreId = await resolveAndValidateStoreId(supabase, raw.store_id || body.store_id, user, isServiceRole, token, body.userId);
+                    raw.store_id = resolvedStoreId || raw.store_id || null;
+                    raw.user_id = raw.user_id || user?.id || body.userId || null;
+                    raw.user_email = raw.user_email || user?.email || body.userEmail || null;
+                    queueSpan(auditRowToSpan(raw));
+                    queued++;
+                }
+                catch {
+                    // Skip malformed spans
+                }
+            }
+            // Also upsert ai_conversations row if conversation_id provided
+            if (body.conversation_id && body.store_id) {
+                try {
+                    const convStoreId = await resolveAndValidateStoreId(supabase, body.store_id, user, isServiceRole, token, body.userId);
+                    if (convStoreId) {
+                        await supabase.from("ai_conversations").upsert({
+                            id: body.conversation_id,
+                            store_id: convStoreId,
+                            user_id: user?.id || body.userId || null,
+                            title: body.conversation_title || "CLI Session",
+                            metadata: {
+                                source: body.source || "whale_cli",
+                                hostname: body.hostname,
+                                version: body.version,
+                            },
+                        }, { onConflict: "id" });
+                    }
+                }
+                catch {
+                    // Non-critical — spans still ingested
+                }
+            }
+            jsonResponse(res, 200, { success: true, queued }, corsHeaders);
+            return;
+        }
         // Agent chat mode (SSE)
         await handleAgentChat(req, res, supabase, body, user, isServiceRole, token, corsHeaders);
     }
@@ -2255,6 +2488,19 @@ async function invokeAgentForChannel(supabase, agentId, message, storeId, conver
 setNodeAgentInvoker(invokeAgentForChannel);
 webchatAgentInvoker = invokeAgentForChannel;
 // ============================================================================
+// NODE OFFLINE DETECTION
+// ============================================================================
+async function enforceNodeOfflineStatus(supabase) {
+    const threshold = new Date(Date.now() - 3 * 60_000).toISOString(); // 3 missed heartbeats (60s interval)
+    const { data } = await supabase
+        .from("nodes")
+        .update({ status: "offline" })
+        .eq("status", "online")
+        .lt("last_heartbeat", threshold)
+        .select("id");
+    return data?.length || 0;
+}
+// ============================================================================
 // PERSISTENT WORKFLOW WORKER LOOP (5-second interval)
 // ============================================================================
 // Phase 3.1: Increased from 5s to 15s — NOTIFY-driven execution handles the fast path
@@ -2275,16 +2521,17 @@ async function workflowWorkerLoop() {
             processWaitingSteps(supabase),
             Promise.resolve(supabase.rpc("expire_pending_waitpoints")).then(() => { }).catch(e => log.warn({ err: e.message }, "expire_pending_waitpoints failed")), // Non-fatal
         ]);
-        // Schedule triggers + timeout enforcement + event triggers + orphan cleanup + DLQ retries
-        const [scheduled, timedOut, eventsProcessed, orphansCleaned, dlqRetried] = await Promise.all([
+        // Schedule triggers + timeout enforcement + event triggers + orphan cleanup + DLQ retries + node offline detection
+        const [scheduled, timedOut, eventsProcessed, orphansCleaned, dlqRetried, nodesOfflined] = await Promise.all([
             processScheduleTriggers(supabase).catch(e => { log.warn({ err: e.message }, "processScheduleTriggers failed"); return 0; }),
             enforceWorkflowTimeouts(supabase).catch(e => { log.warn({ err: e.message }, "enforceWorkflowTimeouts failed"); return 0; }),
             processEventTriggers(supabase).catch(e => { log.warn({ err: e.message }, "processEventTriggers failed"); return 0; }),
             cleanupOrphanedSteps(supabase).catch(e => { log.warn({ err: e.message }, "cleanupOrphanedSteps failed"); return 0; }),
             processDlqRetries(supabase).catch(e => { log.warn({ err: e.message }, "processDlqRetries failed"); return 0; }),
+            enforceNodeOfflineStatus(supabase).catch(e => { log.warn({ err: e.message }, "enforceNodeOfflineStatus failed"); return 0; }),
         ]);
-        if (stepResult.processed > 0 || waitingResolved > 0 || scheduled > 0 || timedOut > 0 || eventsProcessed > 0 || stepResult.reclaimed > 0 || orphansCleaned > 0 || dlqRetried > 0) {
-            log.info({ processed: stepResult.processed, errors: stepResult.errors, reclaimed: stepResult.reclaimed || 0, waiting: waitingResolved, scheduled, timedOut, events: eventsProcessed, orphans: orphansCleaned, dlqRetries: dlqRetried }, "worker tick");
+        if (stepResult.processed > 0 || waitingResolved > 0 || scheduled > 0 || timedOut > 0 || eventsProcessed > 0 || stepResult.reclaimed > 0 || orphansCleaned > 0 || dlqRetried > 0 || nodesOfflined > 0) {
+            log.info({ processed: stepResult.processed, errors: stepResult.errors, reclaimed: stepResult.reclaimed || 0, waiting: waitingResolved, scheduled, timedOut, events: eventsProcessed, orphans: orphansCleaned, dlqRetries: dlqRetried, nodesOfflined }, "worker tick");
         }
         // Reset backoff on success
         if (consecutiveErrors > 0) {
@@ -2375,14 +2622,13 @@ async function gracefulShutdown(signal) {
         clients.clear();
     }
     sseClients.clear();
-    // 3b. P1 FIX: Flush audit log buffer before shutdown (prevents data loss on crash)
+    // 3b. Flush ClickHouse span buffer before shutdown (prevents data loss on crash)
     try {
-        const sb = getServiceClient();
-        await flushAuditLogs(sb);
-        log.info("audit log buffer flushed");
+        await flushSpans();
+        log.info("span buffer flushed");
     }
     catch (err) {
-        log.error({ err: err.message }, "audit log flush error");
+        log.error({ err: err.message }, "span buffer flush error");
     }
     // 4. Shut down code worker pool
     try {