whale-code 6.5.6 → 6.5.8

package/dist/server/index.js

@@ -5,27 +5,36 @@
 // Shares agent-core with CLI via src/shared/agent-core.ts
 
 import http from "node:http";
-import { randomUUID, timingSafeEqual, createHash } from "node:crypto";
+import { randomUUID } from "node:crypto";
 import Anthropic from "@anthropic-ai/sdk";
 import { createLogger } from "./lib/logger.js";
 const log = createLogger("server");
-import { sanitizeError, resolveAgentLoopConfig } from "../shared/agent-core.js";
-import { MODELS } from "../shared/constants.js";
+import { sanitizeError } from "../shared/agent-core.js";
 import { handleProxy } from "./proxy-handlers.js";
 import { handleNodeRoutes, setNodeAgentInvoker } from "./handlers/nodes.js";
 import { handleTranscribe } from "./handlers/transcription.js";
 import { handleBillingRoutes, incrementUsage, checkPlanLimits } from "./handlers/billing.js";
+import { handleSessionRoutes } from "./handlers/sessions-handlers.js";
 import { generateCompaction } from "./lib/compaction-service.js";
 import { initLocalAgentGateway, shutdownGateway as shutdownAgentGateway, getGatewayStats } from "./local-agent-gateway.js";
 import { initSupabase, getServiceClient } from "./lib/supabase-client.js";
 import { loadCheckpoint, markOrphaned } from "./lib/session-checkpoint.js";
 import { rateLimiter } from "./lib/rate-limiter.js";
 import { sanitizeAndLog } from "./lib/prompt-sanitizer.js";
-import { processWorkflowSteps, processWaitingSteps, handleWebhookIngestion, executeInlineChain, setToolExecutor, setAgentExecutor, setTokenBroadcaster, setStepErrorBroadcaster, verifyGuestApprovalSignature, initWorkerPool, getPoolStats, shutdownPool, processScheduleTriggers, enforceWorkflowTimeouts, processEventTriggers, cleanupOrphanedSteps, processDlqRetries } from "./handlers/workflows.js";
-import { runServerAgentLoop } from "./lib/server-agent-loop.js";
-import { loadTools, loadUserTools, getToolsForAgent, executeTool, loadAgentConfig, setExtendedToolsCache, getExtendedToolsIndex, flushSpans } from "./tool-router.js";
-import { queueSpan, auditRowToSpan, classifyErrorType } from "./lib/clickhouse-buffer.js";
-import pg from "pg";
+import { generateOpenApiSpec } from "./handlers/api-docs.js";
+import { processWorkflowSteps, handleWebhookIngestion, executeInlineChain, initWorkerPool, getPoolStats, shutdownPool, verifyGuestApprovalSignature } from "./handlers/workflows.js";
+import { loadUserTools, executeTool, flushSpans } from "./tool-router.js";
+import { queueSpan, auditRowToSpan } from "./lib/clickhouse-buffer.js";
+
+// Extracted modules — single source of truth (no inline duplicates)
+import { safeCompare, getCorsHeaders, jsonResponse, readBody } from "./server-helpers.js";
+import { sendIpRateLimit } from "./server-rate-limit.js";
+import { resolveAndValidateStoreId } from "./server-store.js";
+import { getSseClients, safeSseWrite, sendWorkflowSSE, getTotalSseClients, sseCleanupInterval, MAX_SSE_CLIENTS_PER_RUN, MAX_SSE_TOTAL_CLIENTS, setupPgListen, closePgClient, pgListenReady } from "./server-sse.js";
+import { handleAgentChat } from "./server-chat.js";
+import { wireToolExecutor, wireAgentExecutor, wireBroadcasters, invokeAgentForChannel } from "./server-executors.js";
+import { startWorkerLoop, stopWorkerLoop } from "./server-worker.js";
+import { getCircuitBreakerStats } from "./server-store-circuit-breaker.js";
 
 // ============================================================================
 // PROCESS ERROR HANDLERS
@@ -52,7 +61,6 @@ const SUPABASE_URL = process.env.SUPABASE_URL;
 const SUPABASE_SERVICE_ROLE_KEY = process.env.SUPABASE_SERVICE_ROLE_KEY;
 const SERVICE_ROLE_JWT = process.env.SERVICE_ROLE_JWT || "";
 const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY;
-const ALLOWED_ORIGINS = (process.env.ALLOWED_ORIGINS || "http://localhost:3000,http://127.0.0.1:3000").split(",").map(s => s.trim());
 const FLY_INTERNAL_SECRET = process.env.FLY_INTERNAL_SECRET || "";
 if (!FLY_INTERNAL_SECRET) {
   console.warn("[SECURITY] FLY_INTERNAL_SECRET is not set — internal endpoints are unprotected");
@@ -62,7 +70,6 @@ if (!FLY_INTERNAL_SECRET) {
 // READINESS STATE
 // ============================================================================
 
-let pgListenReady = false;
 let workerPoolReady = false;
 function isReady() {
   return workerPoolReady; // PG listen is optional (SSE only)
@@ -71,1311 +78,6 @@ function isReady() {
 // Webchat agent invoker — set later to avoid circular deps (same as node agent invoker)
 let webchatAgentInvoker = null;
 
-// ============================================================================
-// RATE LIMITING — Phase 7.2 token-bucket (see lib/rate-limiter.ts)
-// ============================================================================
-
-/** Check IP rate limit and send 429 with proper headers if exceeded */
-function sendIpRateLimit(res, ip, headers) {
-  const result = rateLimiter.checkRequest(`ip:${ip}`, "unauthenticated");
-  if (result.allowed) return false; // not rate-limited
-  res.writeHead(429, {
-    "Retry-After": String(Math.ceil(result.retryAfterMs / 1000) || 1),
-    "X-RateLimit-Remaining": "0",
-    "Content-Type": "application/json",
-    ...headers
-  });
-  res.end(JSON.stringify({
-    error: "Too many requests"
-  }));
-  return true; // was rate-limited
-}
-
-// Agent chat rate limiting — per-store + global concurrent cap
-const agentChatLimiter = new Map();
-const AGENT_CHAT_MAX_PER_MIN = 60;
-const AGENT_CHAT_MAX_CONCURRENT = 10;
-let agentChatConcurrent = 0;
-
-// Periodically clean up stale entries from agentChatLimiter to prevent unbounded Map growth
-setInterval(() => {
-  const now = Date.now();
-  for (const [key, entry] of agentChatLimiter) {
-    if (now - entry.windowStart > 120_000) {
-      agentChatLimiter.delete(key);
-    }
-  }
-}, 60_000);
-function checkAgentChatRateLimit(storeId) {
-  // Concurrent check
-  if (agentChatConcurrent >= AGENT_CHAT_MAX_CONCURRENT) {
-    return {
-      allowed: false,
-      error: "Too many concurrent agent sessions. Please wait."
-    };
-  }
-
-  // Per-store per-minute check
-  const now = Date.now();
-  let entry = agentChatLimiter.get(storeId);
-  if (!entry || now - entry.windowStart > 60_000) {
-    entry = {
-      count: 0,
-      windowStart: now
-    };
-    agentChatLimiter.set(storeId, entry);
-  }
-  entry.count++;
-  if (entry.count > AGENT_CHAT_MAX_PER_MIN) {
-    return {
-      allowed: false,
-      error: `Rate limit exceeded: ${AGENT_CHAT_MAX_PER_MIN}/min for agent chat`
-    };
-  }
-  return {
-    allowed: true
-  };
-}
-
-// Timing-safe secret comparison to prevent timing attacks
-// Hash both values to fixed length before comparing — avoids leaking secret length
-function safeCompare(a, b) {
-  if (!a || !b) return false;
-  const hashA = createHash("sha256").update(a).digest();
-  const hashB = createHash("sha256").update(b).digest();
-  return timingSafeEqual(hashA, hashB);
-}
-
-// ============================================================================
-// TYPES
-// ============================================================================
-
-// Tool registry, user tools, executor, and agent loader are in ./tool-router.ts
-
-// ============================================================================
-// STORE CONTEXT — Server-derived store resolution (Apple-style)
-// The server NEVER blindly trusts a client-supplied storeId. Instead it:
-//   1. Resolves the user's stores from the `users` table via auth.uid()
-//   2. Validates the client hint against the resolved set
-//   3. Falls back to the user's first store if no hint provided
-//   4. Returns null if no store can be resolved (caller must fail closed)
-// ============================================================================
-
-async function resolveAndValidateStoreId(supabase, clientStoreId, user, isServiceRole, _token, requestUserId) {
-  // Service-role callers: trusted ONLY for internal server-to-server calls
-  // (workflows, cron jobs) that have NO associated user. When a userId is
-  // present (e.g. MCP CLI using env-var service role key), we MUST still
-  // validate store membership to prevent cross-tenant access.
-  if (isServiceRole) {
-    const srUserId = user?.id || requestUserId;
-    if (!srUserId) return clientStoreId || null; // true internal call — pass through
-    if (!clientStoreId) return null; // user-context call without store hint
-
-    // Validate the user actually belongs to the requested store
-    const {
-      data: membership
-    } = await supabase.from("store_members").select("id").eq("store_id", clientStoreId).eq("user_id", srUserId).limit(1).maybeSingle();
-    if (membership) return clientStoreId;
-
-    // Fallback: check users table (legacy single-store pattern)
-    const {
-      data: userRow
-    } = await supabase.from("users").select("store_id").eq("auth_user_id", srUserId).eq("store_id", clientStoreId).maybeSingle();
-    if (userRow) return clientStoreId;
-    log.warn({
-      userId: srUserId,
-      clientStoreId
-    }, "resolveStoreId: service-role caller userId not authorized for store");
-    return null;
-  }
-  if (!user?.id) return null;
-
-  // Resolve user's actual stores from the `users` table (auth_user_id = auth.uid())
-  const {
-    data: userStores,
-    error
-  } = await supabase.from("users").select("store_id").eq("auth_user_id", user.id).not("store_id", "is", null);
-  if (error || !userStores?.length) {
-    log.warn({
-      userId: user.id,
-      error
-    }, "resolveStoreId: user has no stores");
-    return null;
-  }
-  const storeIds = userStores.map(r => r.store_id);
-
-  // If client provided a hint, validate it's in the user's set
-  if (clientStoreId) {
-    if (storeIds.includes(clientStoreId)) {
-      return clientStoreId;
-    }
-    log.warn({
-      userId: user.id,
-      clientStoreId,
-      userStores: storeIds
-    }, "resolveStoreId: client storeId not in user's stores");
-    return null; // Reject — don't silently fall back
-  }
-
-  // No client hint — use first store (single-store users), or require explicit selection
-  if (storeIds.length === 1) {
-    return storeIds[0];
-  }
-
-  // Multi-store user without a hint — can't guess
-  log.warn({
-    userId: user.id,
-    storeCount: storeIds.length
-  }, "resolveStoreId: multi-store user must specify storeId");
-  return null;
-}
-
-// ============================================================================
-// CORS
-// ============================================================================
-
-function getCorsHeaders(origin) {
-  const headers = {
-    "X-Content-Type-Options": "nosniff",
-    "X-Frame-Options": "DENY",
-    "X-XSS-Protection": "0",
-    "Referrer-Policy": "strict-origin-when-cross-origin"
-  };
-  if (ALLOWED_ORIGINS.includes("*")) {
-    headers["Access-Control-Allow-Origin"] = "*";
-  } else if (origin && ALLOWED_ORIGINS.includes(origin)) {
-    headers["Access-Control-Allow-Origin"] = origin;
-    headers["Vary"] = "Origin";
-  }
-  // If origin doesn't match, no CORS header = browser blocks the request
-
-  headers["Access-Control-Allow-Methods"] = "GET, POST, OPTIONS";
-  headers["Access-Control-Allow-Headers"] = "Content-Type, Authorization, X-Store-Id";
-  return headers;
-}
-
-// ============================================================================
-// PHASE 3: SSE STREAMING — real-time workflow run progress
-// ============================================================================
-
-// Map<runId, Set<ServerResponse>> for multiplexing SSE clients
-const sseClients = new Map();
-const MAX_SSE_CLIENTS_PER_RUN = 10;
-const MAX_SSE_TOTAL_CLIENTS = 100;
-
-/** Safe SSE write — returns false and calls cleanup if the connection is dead */
-function safeSseWrite(res, data, cleanup) {
-  try {
-    if (res.destroyed || res.writableEnded) {
-      cleanup();
-      return false;
-    }
-    res.write(data);
-    return true;
-  } catch {
-    cleanup();
-    return false;
-  }
-}
-function sendWorkflowSSE(res, data) {
-  try {
-    if (res.destroyed || res.writableEnded) return;
-    res.write(`data: ${JSON.stringify(data)}\n\n`);
-  } catch {/* client disconnected — benign */}
-}
-function broadcastToRun(runId, data) {
-  const clients = sseClients.get(runId);
-  if (!clients?.size) return;
-  // H6 FIX: Prune dead connections during broadcast
-  for (const res of clients) {
-    if (res.destroyed || res.writableEnded) {
-      clients.delete(res);
-      continue;
-    }
-    sendWorkflowSSE(res, data);
-  }
-  if (clients.size === 0) sseClients.delete(runId);
-}
-function getTotalSseClients() {
-  let total = 0;
-  for (const clients of sseClients.values()) total += clients.size;
-  return total;
-}
-
-// H6 FIX: Periodic stale connection cleanup (every 60s)
-const sseCleanupInterval = setInterval(() => {
-  for (const [rid, clients] of sseClients) {
-    for (const res of clients) {
-      if (res.destroyed || res.writableEnded) {
-        clients.delete(res);
-      }
-    }
-    if (clients.size === 0) sseClients.delete(rid);
-  }
-}, 60_000);
-
-// pg LISTEN for real-time notifications
-const DATABASE_URL = process.env.DATABASE_URL || "";
-let pgClient = null;
-let pgReconnectAttempts = 0;
-const MAX_PG_RECONNECT_DELAY = 10_000; // 10s max — keep SSE reconnect snappy
-
-async function setupPgListen() {
-  if (!DATABASE_URL) {
-    log.info("DATABASE_URL not set — SSE streaming disabled, using worker-only mode");
-    return;
-  }
-  try {
-    // Strip sslmode from URL (pg v8 treats sslmode=require as verify-full) and set ssl manually
-    const cleanUrl = DATABASE_URL.replace(/[?&]sslmode=[^&]*/g, "").replace(/\?$/, "");
-    pgClient = new pg.Client({
-      connectionString: cleanUrl,
-      ssl: {
-        rejectUnauthorized: false
-      }
-    });
-    await pgClient.connect();
-    await pgClient.query("LISTEN workflow_step_event");
-    await pgClient.query("LISTEN workflow_run_event");
-    await pgClient.query("LISTEN workflow_step_pending");
-    await pgClient.query("LISTEN workflow_event");
-    await pgClient.query("LISTEN automation_event");
-
-    // Reset reconnect counter on successful connection
-    pgReconnectAttempts = 0;
-    pgListenReady = true;
-
-    // Debounced event trigger processing — fires at most once per 100ms
-    let eventTriggerTimer = null;
-    function debouncedEventProcess() {
-      if (eventTriggerTimer) return;
-      eventTriggerTimer = setTimeout(async () => {
-        eventTriggerTimer = null;
-        try {
-          const sb = getServiceClient();
-          const count = await processEventTriggers(sb);
-          if (count > 0) log.info({
-            count
-          }, "instant event processing");
-        } catch (err) {
-          log.error({
-            err: err.message
-          }, "event trigger processing error");
-        }
-      }, 100);
-    }
-    pgClient.on("notification", msg => {
-      if (!msg.payload) return;
-      try {
-        const data = JSON.parse(msg.payload);
-
-        // Automation event — trigger immediate processing
-        if (msg.channel === "automation_event") {
-          debouncedEventProcess();
-          return;
-        }
-        const runId = data.run_id;
-        if (!runId) return;
-        if (msg.channel === "workflow_step_event") {
-          broadcastToRun(runId, {
-            type: "step_update",
-            ...data
-          });
-        } else if (msg.channel === "workflow_run_event") {
-          broadcastToRun(runId, {
-            type: "run_update",
-            ...data
-          });
-        } else if (msg.channel === "workflow_event") {
-          broadcastToRun(runId, {
-            type: "event",
-            event_type: data.event_type,
-            ...data
-          });
-        } else if (msg.channel === "workflow_step_pending") {
-          // Phase 3.1: NOTIFY-driven step execution — immediate pickup (~50ms vs 5s polling)
-          const sb = getServiceClient();
-          processWorkflowSteps(sb, 1).catch(err => {
-            log.error({
-              err: err.message,
-              runId
-            }, "NOTIFY-driven step processing failed");
-          });
-        }
-      } catch (err) {
-        log.error({
-          err: err.message
-        }, "failed to parse pg notification");
-      }
-    });
-    pgClient.on("error", err => {
-      log.error({
-        err: err.message
-      }, "pg-listen connection error");
-      pgClient = null;
-      // Reconnect with exponential backoff
-      pgReconnectAttempts++;
-      const delay = Math.min(1000 * Math.pow(2, pgReconnectAttempts - 1), MAX_PG_RECONNECT_DELAY);
-      log.warn({
-        delayMs: delay,
-        attempt: pgReconnectAttempts
-      }, "pg-listen reconnecting");
-      setTimeout(() => setupPgListen(), delay);
-    });
-    log.info("pg-listen active on workflow_step_event, workflow_run_event, workflow_step_pending, workflow_event, automation_event");
-  } catch (err) {
-    log.error({
-      err: err.message
-    }, "pg-listen failed to connect");
-    pgClient = null;
-    // Reconnect with exponential backoff on initial connection failure too
-    pgReconnectAttempts++;
-    const delay = Math.min(1000 * Math.pow(2, pgReconnectAttempts - 1), MAX_PG_RECONNECT_DELAY);
-    log.warn({
-      delayMs: delay,
-      attempt: pgReconnectAttempts
-    }, "pg-listen reconnecting");
-    setTimeout(() => setupPgListen(), delay);
-  }
-}
-
-// ============================================================================
-// HELPERS
-// ============================================================================
-
-function getAnthropicClient(agent) {
-  const key = agent.api_key || ANTHROPIC_API_KEY;
-  return new Anthropic({
-    apiKey: key,
-    timeout: 15 * 60 * 1000
-  }); // 15 min for tool-heavy requests
-}
-function sendSSE(res, event) {
-  try {
-    if (res.destroyed || res.writableEnded) return;
-    res.write(`data: ${JSON.stringify(event)}\n\n`);
-  } catch {/* client disconnected — benign */}
-}
-function jsonResponse(res, status, data, corsHeaders) {
-  res.writeHead(status, {
-    "Content-Type": "application/json",
-    ...corsHeaders
-  });
-  res.end(JSON.stringify(data));
-}
-
-/**
- * Strip large base64 data fields from a JSON string up to `keepFrom` offset.
- * Uses a linear indexOf scan — safe on strings of any size (no regex stack overflow).
- * Handles both `"data":"<base64>"` (Anthropic image blocks) and
- * `"__IMAGE__<mime>__<base64>"` text values (Read tool marker format).
- */
-function stripLargeBase64Fields(raw, keepFrom) {
-  const MIN_DATA_LEN = 8_000;
-
-  // ── Pass 1: Strip "data":"<large base64>" fields (Anthropic image source blocks) ──
-  let result = raw;
-  {
-    const DATA_MARKER = '"data":"';
-    const parts = [];
-    let pos = 0;
-    while (pos < keepFrom) {
-      const idx = result.indexOf(DATA_MARKER, pos);
-      if (idx === -1 || idx >= keepFrom) {
-        parts.push(result.slice(pos, keepFrom));
-        pos = keepFrom;
-        break;
-      }
-      parts.push(result.slice(pos, idx + DATA_MARKER.length));
-      pos = idx + DATA_MARKER.length;
-
-      // Find closing quote (base64 has no backslashes, so simple scan is safe)
-      let end = pos;
-      while (end < result.length && result[end] !== '"') end++;
-      if (end - pos >= MIN_DATA_LEN && end <= keepFrom) {
-        // Large data field in the prune zone — replace with empty
-        parts.push('"');
-        pos = end + 1; // skip past the original closing quote (already replaced)
-      }
-      // else: small field or near keepFrom boundary — leave intact
-    }
-    parts.push(result.slice(keepFrom)); // always keep tail intact
-    result = parts.join("");
-  }
-
-  // ── Pass 2: Strip __IMAGE__<mime>__<large base64> text markers ──
-  // If the client-side tool-dispatch regex failed to convert these to image
-  // content blocks, they stay as huge text strings in tool_result content.
-  // They appear in JSON as: "__IMAGE__image/png__iVBOR...very long..."
-  {
-    const IMG_MARKER = "__IMAGE__";
-    const adjustedKeepFrom = Math.min(keepFrom, result.length);
-    const parts = [];
-    let pos = 0;
-    while (pos < adjustedKeepFrom) {
-      const idx = result.indexOf(IMG_MARKER, pos);
-      if (idx === -1 || idx >= adjustedKeepFrom) {
-        parts.push(result.slice(pos, adjustedKeepFrom));
-        pos = adjustedKeepFrom;
-        break;
-      }
-      parts.push(result.slice(pos, idx));
-
-      // Find the end of this text value (closing quote of the JSON string)
-      let end = idx;
-      while (end < result.length && result[end] !== '"') end++;
-      if (end - idx >= MIN_DATA_LEN && end <= adjustedKeepFrom) {
-        // Large __IMAGE__ marker in the prune zone — replace with placeholder
-        parts.push("[image pruned]");
-        pos = end; // land on closing quote; next iteration emits it
-      } else {
-        // Small or near boundary — keep intact
-        parts.push(result.slice(idx, idx + IMG_MARKER.length));
-        pos = idx + IMG_MARKER.length;
-      }
-    }
-    parts.push(result.slice(adjustedKeepFrom));
-    result = parts.join("");
-  }
-  return result;
-}
-async function readBody(req) {
-  // 500MB hard limit — conversation history with many large images can be very large.
-  // For bodies over 10MB we prune old base64 image data from the raw string before
-  // JSON.parse, so subsequent requests in the same conversation stay bounded.
-  const MAX_BODY = 524_288_000; // 500MB
-  return new Promise((resolve, reject) => {
-    const chunks = [];
-    let size = 0;
-    let rejected = false;
-    req.on("data", chunk => {
-      size += chunk.length;
-      if (size > MAX_BODY && !rejected) {
-        rejected = true;
-        req.resume(); // drain to avoid Fly proxy 502
-        reject(new Error("Request body too large (max 500MB)"));
-        return;
-      }
-      if (!rejected) chunks.push(chunk);
-    });
-    req.on("end", () => {
-      if (rejected) return;
-
-      // Detect body truncation: if Content-Length was sent and we received less
-      const declaredLen = parseInt(req.headers["content-length"] || "0", 10);
-      if (declaredLen > 0 && size < declaredLen) {
-        log.error({
-          declaredLen,
-          receivedLen: size
-        }, "Request body truncated during transmission");
-        reject(new Error(`Body truncated: expected ${declaredLen} bytes, got ${size}`));
-        return;
-      }
-      let raw = Buffer.concat(chunks).toString("utf8");
-
-      // Strip old base64 data from history when body is large.
-      // Threshold lowered to 1MB — image-heavy conversations (e.g. reading
-      // product photos) easily exceed 10MB with just a few images.
-      // Keep the last 1MB intact (current user message with fresh images).
-      // Uses a linear scan to avoid String.replace() stack overflow on huge strings.
-      if (raw.length > 1_000_000) {
-        const keepFrom = Math.max(0, raw.length - 1_048_576); // keep last 1MB
-        raw = stripLargeBase64Fields(raw, keepFrom);
-        log.info({
-          originalBytes: size,
-          prunedBytes: Buffer.byteLength(raw)
-        }, "Pruned old base64 data from request body");
-      }
-      resolve(raw);
-    });
-    req.on("error", reject);
-  });
-}
-
-// ============================================================================
-// HISTORY COMPACTION
-// ============================================================================
-
-/**
- * Compact conversation history to fit within a total character budget.
- *
- * Does NOT truncate individual messages or tool results — Anthropic's
- * context_management API handles clearing old tool uses (clear_tool_uses_20250919)
- * and compacting context (compact_20260112) when the context window grows.
- *
- * This function only enforces a total budget by walking newest→oldest and
- * dropping the oldest messages that don't fit.
- */
-function compactHistory(history, maxHistoryChars) {
-  if (!history?.length) return [];
-  let totalChars = 0;
-  const compacted = [];
-  for (let i = history.length - 1; i >= 0; i--) {
-    const msg = history[i];
-    const msgChars = JSON.stringify(msg.content).length;
-    if (totalChars + msgChars > maxHistoryChars) break;
-    totalChars += msgChars;
-    compacted.unshift(msg);
-  }
-  // Ensure starts with user message
-  while (compacted.length > 0 && compacted[0].role !== "user") compacted.shift();
-  return compacted;
-}
-
-// ============================================================================
-// SHARED AGENT HELPERS — used by both SSE chat and channel agent paths
-// ============================================================================
-
-/** Build the full system prompt for an agent. Both SSE chat and channel paths
- *  call this so the prompt logic is never duplicated. */
-async function buildAgentSystemPrompt(supabase, agent, storeId, message, tools, opts) {
-  // --- STATIC portion (cache-friendly, same across conversations) ---
-  // Sanitize the DB-stored agent system prompt to prevent injection attacks
-  const rawAgentPrompt = agent.system_prompt || "You are a helpful assistant.";
-  let systemPrompt = sanitizeAndLog(rawAgentPrompt, "buildAgentSystemPrompt", {
-    agentId: agent.id
-  });
-  if (storeId) systemPrompt += `\n\nYou are operating for store_id: ${storeId}. Always include this in tool calls that require it.`;
-  if (!agent.can_modify) systemPrompt += "\n\nIMPORTANT: You have read-only access. Do not attempt to modify any data.";
-  if (agent.tone && agent.tone !== "professional") systemPrompt += `\n\nTone: Respond in a ${agent.tone} tone.`;
-  if (agent.verbosity === "concise") systemPrompt += "\n\nBe concise — short answers, minimal explanation.";else if (agent.verbosity === "verbose") systemPrompt += "\n\nBe thorough — provide detailed answers with full context.";
-  if (agent.context_config) {
-    const ctx = agent.context_config;
-    if (ctx.includeLocations && ctx.locationIds?.length) systemPrompt += `\n\nFocus on these locations: ${ctx.locationIds.join(", ")}`;
-    if (ctx.includeCustomers && ctx.customerSegments?.length) systemPrompt += `\n\nFocus on these customer segments: ${ctx.customerSegments.join(", ")}`;
-  }
-
-  // Tool manifest — core tools (full schemas already loaded)
-  const toolNames = tools.map(t => t.name);
-  systemPrompt += `\n\n## Available Tools\nYou have ${toolNames.length} core tools available in this session:\n${toolNames.join(", ")}\n\nFor local machine operations, use the \`local_agent\` tool (exec, tools, discover actions). For cloud security tools, use \`kali\`. Do NOT reference CLI-local tools like read_file, write_file, edit_file, glob, grep, run_command — those are not available in this environment.`;
-
-  // Extended tools index — names + one-line descriptions only (saves ~20K tokens)
-  const extendedTools = opts?.extendedTools || [];
-  if (extendedTools.length > 0) {
-    systemPrompt += `\n\n## Extended Tools (call discover_tools to activate)\nThese tools are available but not loaded yet. Call \`discover_tools\` with the tool name(s) before using them:\n`;
-    for (const t of extendedTools) {
-      // First sentence only for compact display
-      const shortDesc = t.description.split(".")[0];
-      systemPrompt += `- **${t.name}**: ${shortDesc}\n`;
-    }
-  }
-
-  // --- DYNAMIC portion (changes per conversation, prepended to user message) ---
-  const dynamicParts = [];
-
-  // Client-provided session context (SSE chat sends this from SwiftUI/web)
-  if (opts?.clientContext && typeof opts.clientContext === "object") {
-    const context = opts.clientContext;
-    const ctxParts = [];
-    if (context.storeName) ctxParts.push(`Store: ${context.storeName}`);
-    if (context.locationName) {
-      let loc = `Location: ${context.locationName}`;
-      if (context.locationAddress) loc += ` (${context.locationAddress})`;
-      if (context.locationType) loc += ` [${context.locationType}]`;
-      ctxParts.push(loc);
-    }
-    if (context.userName) {
-      ctxParts.push(`User: ${context.userName}${opts.userEmail ? ` (${opts.userEmail})` : ""}`);
-    } else if (opts.userEmail) {
-      ctxParts.push(`User: ${opts.userEmail}`);
-    }
-    if (context.conversationType) {
-      ctxParts.push(`Channel: ${context.conversationTitle || context.conversationType} (${context.conversationType})`);
-    }
-    if (ctxParts.length) {
-      dynamicParts.push(`## Current Session Context\n${ctxParts.join("\n")}`);
-    }
-  }
-
-  // Parallel DB lookups: customer fetch + memory recall
-  const customerPromise = opts?.senderContext?.customerId && storeId ? Promise.resolve(supabase.from("v_store_customers").select("first_name, last_name, email, phone, loyalty_tier, total_orders, total_spent, lifetime_value").eq("id", opts.senderContext.customerId).eq("store_id", storeId).single()).then(({
-    data
-  }) => data).catch(() => null) : Promise.resolve(null);
-  const memoryPromise = storeId ? Promise.resolve(supabase.rpc("recall_memory", {
-    p_agent_id: agent.id,
-    p_query: message.substring(0, 200),
-    p_type: null,
-    p_limit: 5
-  })).then(({
-    data
-  }) => data).catch(err => {
-    log.warn({
-      err: err.message
-    }, "memory recall failed");
-    return null;
-  }) : Promise.resolve(null);
-  const [cust, memories] = await Promise.all([customerPromise, memoryPromise]);
-
-  // Channel-specific customer context injection
-  if (cust && opts?.senderContext?.customerId) {
-    dynamicParts.push(`## Current Customer\nName: ${cust.first_name} ${cust.last_name}\nEmail: ${cust.email || "N/A"}\nPhone: ${cust.phone || "N/A"}\nLoyalty: ${cust.loyalty_tier || "None"}\nOrders: ${cust.total_orders || 0}\nSpent: $${(cust.total_spent || 0).toFixed(2)}\nCustomer ID: ${opts.senderContext.customerId}`);
-  } else if (opts?.senderContext) {
-    const sc = opts.senderContext;
-    dynamicParts.push(`## Sender\nID: ${sc.senderId}\nName: ${sc.senderName || "Unknown"}\nChannel: ${sc.channelType || "unknown"}${sc.channelName ? ` (${sc.channelName})` : ""}\n(No customer record matched — use CRM tools to look them up if needed)`);
-  }
-
-  // Memory recall — inject relevant memories (capped at 5, 100 chars each)
-  if (memories?.length) {
-    const memBlock = memories.map(m => `- [${m.memory_type}] ${m.key}: ${JSON.stringify(m.value).substring(0, 100)}`).join("\n");
-    dynamicParts.push(`## Agent Memory\nRelevant memories from previous conversations:\n${memBlock}`);
-  }
-  const dynamicContext = dynamicParts.join("\n\n");
-  return {
-    systemPrompt,
-    dynamicContext
-  };
-}
-
-/** Persist everything after an agent turn — messages, audit, memory, cost.
- *  Called by both SSE chat and channel paths so nothing is ever missed. */
-async function persistAgentTurn(supabase, agent, opts) {
-  const {
-    conversationId,
-    storeId,
-    agentId,
-    agentModel,
-    traceId,
-    message,
-    result,
-    source,
-    chatStartTime,
-    chatEndTime,
-    userId,
-    userEmail,
-    senderContext
-  } = opts;
-
-  // ── Persist user + assistant messages to ai_messages ──
-  try {
-    await supabase.from("ai_messages").insert([{
-      conversation_id: conversationId,
-      role: "user",
-      content: [{
-        type: "text",
-        text: message
-      }],
-      token_count: Math.ceil(message.length / 4)
-    }, {
-      conversation_id: conversationId,
-      role: "assistant",
-      content: [{
-        type: "text",
-        text: result.finalText || ""
-      }],
-      is_tool_use: result.toolCallCount > 0,
-      tool_names: result.toolsUsed?.length ? result.toolsUsed : null,
-      token_count: result.tokens.input + result.tokens.output
-    }]);
-  } catch (err) {
-    log.error({
-      err: err.message
-    }, "message persist failed");
-  }
-
-  // ── Update conversation metadata ──
-  try {
-    await supabase.from("ai_conversations").update({
-      metadata: {
-        agentName: agent.name,
-        source,
-        model: agentModel,
-        lastTurnTokens: result.tokens.input + result.tokens.output,
-        lastToolCalls: result.toolCallCount,
-        lastDurationMs: chatEndTime - chatStartTime,
-        // Channel-specific (null when SSE chat — that's fine, no clutter)
-        ...(senderContext ? {
-          channel_type: senderContext.channelType || null,
-          channel_id: senderContext.channelId || null,
-          channel_name: senderContext.channelName || null,
-          sender_id: senderContext.senderId || null,
-          customer_id: senderContext.customerId || null,
-          customer_name: senderContext.customerName || null
-        } : {})
-      }
-    }).eq("id", conversationId);
-  } catch (err) {
-    log.error({
-      err: err.message
-    }, "conversation update failed");
-  }
-
-  // ── Telemetry: user message → ClickHouse ──
-  try {
-    queueSpan(auditRowToSpan({
-      action: "chat.user_message",
-      severity: "info",
-      store_id: storeId || null,
-      resource_type: "chat_message",
-      resource_id: agentId,
-      request_id: traceId,
-      conversation_id: conversationId,
-      user_id: userId || null,
-      user_email: userEmail || null,
-      source,
-      service_name: "agent-server",
-      span_kind: "INTERNAL",
-      status_code: "OK",
-      start_time: new Date().toISOString(),
-      end_time: new Date().toISOString(),
-      duration_ms: 0,
-      input_bytes: typeof message === "string" ? message.length : 0,
-      details: {
-        message_preview: message.substring(0, 200),
-        agent_id: agentId,
-        model: agentModel,
-        conversation_id: conversationId,
-        ...(senderContext ? {
-          channel_type: senderContext.channelType || null,
-          sender_id: senderContext.senderId || null,
-          customer_id: senderContext.customerId || null
-        } : {})
-      }
-    }));
-  } catch (err) {
-    log.error({
-      err: err.message
-    }, "audit user_message failed");
-  }
-
-  // ── Telemetry: assistant response → ClickHouse ──
-  try {
-    queueSpan(auditRowToSpan({
-      action: "chat.assistant_response",
-      severity: "info",
-      store_id: storeId || null,
-      resource_type: "chat_message",
-      resource_id: agentId,
-      request_id: traceId,
-      conversation_id: conversationId,
-      duration_ms: chatEndTime - chatStartTime,
-      user_id: userId || null,
-      user_email: userEmail || null,
-      source,
-      input_tokens: result.tokens.input,
-      output_tokens: result.tokens.output,
-      total_cost: result.costUsd,
-      model: agentModel,
-      trace_id: traceId,
-      span_kind: "INTERNAL",
-      service_name: "agent-server",
-      status_code: "OK",
-      start_time: new Date(chatStartTime).toISOString(),
-      end_time: new Date(chatEndTime).toISOString(),
-      stop_reason: result.stopReason || undefined,
-      turn_number: result.turnCount || 1,
-      details: {
-        response_preview: (result.finalText || "").substring(0, 500),
-        agent_id: agentId,
-        model: agentModel,
-        "gen_ai.request.model": agentModel,
-        "gen_ai.usage.input_tokens": result.tokens.input,
-        "gen_ai.usage.output_tokens": result.tokens.output,
-        "gen_ai.usage.cache_creation_tokens": result.tokens.cacheCreation || 0,
-        "gen_ai.usage.cache_read_tokens": result.tokens.cacheRead || 0,
-        "gen_ai.usage.cost": result.costUsd,
-        turn_count: result.turnCount || 1,
-        tool_calls: result.toolCallCount,
-        tool_names: result.toolsUsed,
-        conversation_id: conversationId,
-        session_cost_usd: result.costUsd,
-        cache_creation_tokens: result.tokens.cacheCreation || 0,
-        cache_read_tokens: result.tokens.cacheRead || 0,
-        cache_hit_rate: result.tokens.input > 0 ? Math.round((result.tokens.cacheRead || 0) / ((result.tokens.cacheRead || 0) + result.tokens.input) * 10000) / 100 : 0,
-        cache_cost_savings_pct: result.tokens.input > 0 && (result.tokens.cacheRead || 0) > 0 ? Math.round((result.tokens.cacheRead || 0) * 0.9 / ((result.tokens.cacheRead || 0) + result.tokens.input) * 10000) / 100 : 0,
-        loop_detector_stats: result.loopDetectorStats || null,
-        turns: result.turns || [],
-        ...(senderContext ? {
-          channel_type: senderContext.channelType || null,
-          channel_id: senderContext.channelId || null,
-          channel_name: senderContext.channelName || null,
-          sender_id: senderContext.senderId || null,
-          customer_id: senderContext.customerId || null,
-          customer_name: senderContext.customerName || null
-        } : {})
-      }
-    }));
-  } catch (err) {
-    log.error({
-      err: err.message
-    }, "audit assistant_response failed");
-  }
-
-  // ── Memory extraction — awaited with retry ──
-  if (storeId && result.finalText && result.finalText.length > 50) {
-    try {
-      await extractAndStoreMemories(supabase, getAnthropicClient(agent), agentId, storeId, message, result.finalText);
-    } catch (err1) {
-      // Retry once after 2s
-      try {
-        await new Promise(r => setTimeout(r, 2000));
-        await extractAndStoreMemories(supabase, getAnthropicClient(agent), agentId, storeId, message, result.finalText);
-      } catch (err2) {
-        log.error({
-          err: err2.message
-        }, "memory extract failed after retry");
-        queueSpan(auditRowToSpan({
-          action: "memory.extraction_failed",
-          severity: "warning",
-          store_id: storeId || null,
-          resource_type: "agent_memory",
-          resource_id: agentId,
-          conversation_id: conversationId,
-          user_id: userId || null,
-          user_email: userEmail || null,
-          error_type: classifyErrorType(err2.message) || undefined,
-          service_name: "agent-server",
-          span_kind: "INTERNAL",
-          status_code: "ERROR",
-          start_time: new Date().toISOString(),
-          end_time: new Date().toISOString(),
-          error_message: err2.message,
-          details: {
-            error: err2.message,
-            user_message_preview: message.substring(0, 100)
-          }
-        }));
-      }
-    }
-  }
-
-  // ── Cost budget tracking — awaited with retry ──
-  if (storeId && result.costUsd > 0) {
-    try {
-      await updateCostBudgets(supabase, storeId, agentId, result.costUsd);
-    } catch (err1) {
-      try {
-        await new Promise(r => setTimeout(r, 1000));
-        await updateCostBudgets(supabase, storeId, agentId, result.costUsd);
-      } catch (err2) {
-        log.error({
-          err: err2.message
-        }, "cost budget update failed after retry");
-        // Flag conversation metadata so budget sync can be reconciled later
-        await supabase.from("ai_conversations").update({
-          metadata: {
-            budget_sync_failed: true,
-            failed_cost_usd: result.costUsd
-          }
-        }).eq("id", conversationId).then(() => {});
-      }
-    }
-  }
-}
-
-// ============================================================================
-// AGENT CHAT HANDLER
-// ============================================================================
-
-async function handleAgentChat(req, res, supabase, body, user, isServiceRole, token, corsHeaders) {
-  const {
-    agentId,
-    message,
-    conversationHistory,
-    source,
-    conversationId,
-    context,
-    attachments
-  } = body;
-  let storeId = body.storeId;
-  if (!agentId || !message) {
-    jsonResponse(res, 400, {
-      error: "agentId and message required"
-    }, corsHeaders);
-    return;
-  }
-  if (typeof message === "string" && message.length > 100_000) {
-    jsonResponse(res, 400, {
-      error: "Message too long (max 100K characters)"
-    }, corsHeaders);
-    return;
-  }
-
-  // Server-derived store resolution — never trust client blindly
-  const resolvedStoreId = await resolveAndValidateStoreId(supabase, storeId, user, isServiceRole, token, body.userId);
-
-  // For service-role callers (workflows, internal), also try body.userId fallback
-  if (!resolvedStoreId && isServiceRole && body.userId) {
-    const {
-      data: srStores
-    } = await supabase.from("users").select("store_id").eq("auth_user_id", body.userId).not("store_id", "is", null).limit(1);
-    if (srStores?.length) {
-      storeId = srStores[0].store_id;
-    }
-  } else {
-    storeId = resolvedStoreId || undefined;
-  }
-
-  // Fail closed: non-service-role requests MUST have a resolved store
-  if (!storeId && !isServiceRole) {
-    jsonResponse(res, 400, {
-      error: "storeId required for agent chat"
-    }, corsHeaders);
-    return;
-  }
-  log.info({
-    storeId: storeId || "NONE",
-    source: body.source || "unknown",
-    isServiceRole,
-    userId: user?.id || body.userId || "NONE"
-  }, "agent-chat request");
-
-  // Agent chat rate limiting — per-store + concurrent cap
-  const rateLimitStoreId = storeId || agentId; // fallback to agentId if no store
-  const rateCheck = checkAgentChatRateLimit(rateLimitStoreId);
-  if (!rateCheck.allowed) {
-    res.writeHead(429, {
-      "Content-Type": "application/json",
-      ...corsHeaders
-    });
-    res.end(JSON.stringify({
-      error: rateCheck.error
-    }));
-    return;
-  }
-  agentChatConcurrent++;
-  try {
-    const userId = user?.id || body.userId || "";
-    const userEmail = user?.email || body.userEmail || null;
-    const agent = await loadAgentConfig(supabase, agentId, storeId);
-    if (!agent) {
-      jsonResponse(res, 404, {
-        error: "Agent not found"
-      }, corsHeaders);
-      return;
-    }
-    const {
-      core: coreTools,
-      extended: extendedTools
-    } = await loadTools(supabase);
-    setExtendedToolsCache(extendedTools); // Populate discover_tools handler cache
-    const {
-      rows: userToolRows,
-      defs: userToolDefs
-    } = storeId ? await loadUserTools(supabase, storeId) : {
-      rows: [],
-      defs: []
-    };
-    const tools = getToolsForAgent(agent, coreTools, userToolDefs);
-    const traceId = randomUUID();
-    const agentModel = agent.model || MODELS.SONNET;
-
-    // Resolve or create conversation
-    let activeConversationId;
-    if (conversationId) {
-      activeConversationId = conversationId;
-    } else {
-      let conv = await supabase.from("ai_conversations").insert({
-        store_id: storeId || null,
-        user_id: userId || null,
-        agent_id: agentId,
-        title: message.substring(0, 100),
-        metadata: {
-          agentName: agent.name,
-          source: source || "whale_chat"
-        }
-      }).select("id").single();
-      if (conv.error) {
-        log.error({
-          err: conv.error.message,
-          details: conv.error.details,
-          hint: conv.error.hint,
-          storeId,
-          userId,
-          agentId
-        }, "conversation create failed");
-        // Retry without user_id (may be FK constraint)
-        conv = await supabase.from("ai_conversations").insert({
-          store_id: storeId || null,
-          agent_id: agentId,
-          title: message.substring(0, 100),
-          metadata: {
-            agentName: agent.name,
-            source: source || "whale_chat",
-            userId,
-            userEmail
-          }
-        }).select("id").single();
-        if (conv.error) {
-          log.error({
-            err: conv.error.message,
-            details: conv.error.details,
-            hint: conv.error.hint
-          }, "conversation retry create failed");
-        }
-      }
-      activeConversationId = conv.data?.id || randomUUID();
-      log.info({
-        conversationId: activeConversationId,
-        fromDb: !!conv.data?.id
-      }, "conversation resolved");
-    }
-
-    // Build system prompt — shared helper ensures SSE chat + channel paths are identical
-    const {
-      systemPrompt,
-      dynamicContext
-    } = await buildAgentSystemPrompt(supabase, agent, storeId, message, tools, {
-      clientContext: context,
-      userId,
-      userEmail,
-      extendedTools: getExtendedToolsIndex()
-    });
-    const anthropic = getAnthropicClient(agent);
-
-    // Resolve all behavioral config from DB — single source of truth
-    const resolved = resolveAgentLoopConfig(agent, "sse");
-    if (resolved.defaultsUsed.length > 0) {
-      log.info({
-        agentId,
-        defaults: resolved.defaultsUsed
-      }, "agent config defaults applied");
-    }
-    const MAX_HISTORY_CHARS = resolved.maxHistoryChars;
-
-    // Build user message — multi-modal if image attachments present
-    let userContent;
-    if (attachments?.length) {
-      const contentBlocks = [];
-      for (const att of attachments) {
-        if (att.type === "image" && att.media_type && att.data) {
-          contentBlocks.push({
-            type: "image",
-            source: {
-              type: "base64",
-              media_type: att.media_type,
-              data: att.data
-            }
-          });
-        }
-      }
-      contentBlocks.push({
-        type: "text",
-        text: message
-      });
-      userContent = contentBlocks;
-    } else {
-      userContent = message;
-    }
-
-    // Prepend dynamic context to user message to keep system prompt static (cache-friendly)
-    const contextPrefix = dynamicContext ? `[Context]\n${dynamicContext}\n\n[User Message]\n` : "";
-    const finalUserContent = typeof userContent === "string" ? contextPrefix + userContent : [...(contextPrefix ? [{
-      type: "text",
-      text: contextPrefix
-    }] : []), ...userContent];
-    const messages = [...compactHistory(conversationHistory || [], MAX_HISTORY_CHARS), {
-      role: "user",
-      content: finalUserContent
-    }];
-
-    // Start SSE stream
-    res.writeHead(200, {
-      "Content-Type": "text/event-stream",
-      "Cache-Control": "no-cache",
-      Connection: "keep-alive",
-      ...corsHeaders
-    });
-
-    // Client disconnect detection
-    let clientDisconnected = false;
-    req.on("close", () => {
-      clientDisconnected = true;
-    });
-    const startedAt = Date.now();
-    const chatStartTime = Date.now();
-    try {
-      const result = await runServerAgentLoop({
-        anthropic,
-        supabase,
-        model: agentModel,
-        systemPrompt,
-        messages,
-        tools,
-        extendedTools,
-        // All behavioral knobs from resolved config — DB is single source of truth
-        maxTurns: resolved.maxTurns,
-        temperature: resolved.temperature,
-        maxTokens: resolved.maxTokens,
-        maxConcurrentTools: resolved.maxConcurrentTools,
-        enableDelegation: resolved.enableDelegation,
-        enableModelRouting: resolved.enableModelRouting,
-        contextOverrides: resolved.contextOverrides,
-        subagentMaxTokens: resolved.subagentMaxTokens,
-        subagentMaxTurns: resolved.subagentMaxTurns,
-        subagentTemperature: resolved.subagentTemperature,
-        // Loop detector thresholds from DB config
-        loopDetectorConfig: {
-          identicalCallLimit: resolved.loopIdenticalCallLimit,
-          consecutiveErrorLimit: resolved.loopConsecutiveErrorLimit,
-          turnErrorLimit: resolved.loopTurnErrorLimit,
-          window: resolved.loopWindow,
-          sessionToolErrorLimit: resolved.loopSessionToolErrorLimit,
-          consecutiveFailedTurnLimit: resolved.loopConsecutiveFailedTurnLimit,
-          fileReadLimit: resolved.loopFileReadLimit
-        },
-        batchErrorLimit: resolved.loopBatchErrorLimit,
-        storeId,
-        traceId,
-        userId,
-        userEmail,
-        source,
-        conversationId: activeConversationId,
-        agentId,
-        executeTool: async (toolName, args, sourceOverride, onToolProgress) => {
-          const toolArgs = {
-            ...args
-          };
-          if (!toolArgs.store_id && storeId) toolArgs.store_id = storeId;
-          return executeTool(supabase, toolName, toolArgs, storeId, traceId, userId, userEmail, sourceOverride || source, activeConversationId, userToolRows, agentId, onToolProgress, true);
-        },
-        onToolProgress: (name, progress) => sendSSE(res, {
-          type: "tool_progress",
-          name,
-          progress
-        }),
-        onText: text => sendSSE(res, {
-          type: "text",
-          text
-        }),
-        onToolStart: (name, input) => {
-          // Only send when input is available — deduplicates streaming double-fire
-          // (sse-parser fires onToolStart twice: once on content_block_start without input,
-          // once on content_block_stop with parsed input)
-          if (input !== undefined) {
-            sendSSE(res, {
-              type: "tool_start",
-              name,
-              input
-            });
-          }
-        },
-        onToolResult: (name, success, r) => sendSSE(res, {
-          type: "tool_result",
-          name,
-          success,
-          result: r
-        }),
-        onSubagentProgress: evt => {
-          sendSSE(res, {
-            type: "subagent",
-            subagentId: evt.subagentId,
-            subagentEvent: evt.event,
-            name: evt.toolName
-          });
-        },
-        clientDisconnected: {
-          get value() {
-            return clientDisconnected;
-          }
-        },
-        startedAt,
-        maxDurationMs: resolved.maxDurationMs
-      });
-
-      // Send usage SSE
-      sendSSE(res, {
-        type: "usage",
-        usage: {
-          input_tokens: result.tokens.input,
-          output_tokens: result.tokens.output,
-          cache_creation_tokens: result.tokens.cacheCreation,
-          cache_read_tokens: result.tokens.cacheRead,
-          cost_usd: result.costUsd
-        }
-      });
-
-      // Persist everything — shared helper ensures SSE chat + channel paths are identical
-      await persistAgentTurn(supabase, agent, {
-        conversationId: activeConversationId,
-        storeId,
-        agentId,
-        agentModel,
-        traceId,
-        message,
-        result,
-        source: source || "whale_chat",
-        chatStartTime,
-        chatEndTime: Date.now(),
-        userId,
-        userEmail
-      });
-      sendSSE(res, {
-        type: "done",
-        conversationId: activeConversationId
-      });
-    } catch (err) {
-      sendSSE(res, {
-        type: "error",
-        error: sanitizeError(err)
-      });
-    }
-    res.end();
-  } finally {
-    agentChatConcurrent--;
-  }
-}
-
-// ============================================================================
-// MEMORY EXTRACTION — extract key facts after agent conversation
-// ============================================================================
-
-async function extractAndStoreMemories(supabase, anthropic, agentId, storeId, userMessage, assistantResponse) {
-  const extraction = await anthropic.messages.create({
-    model: "claude-haiku-4-5-20251001",
-    max_tokens: 500,
-    system: `Extract key facts worth remembering from this conversation turn.
-Return JSON array: [{"key": "short_key", "value": {"detail": "..."}, "type": "short_term|long_term|entity"}]
-Rules:
-- Only extract genuinely useful facts (preferences, decisions, corrections, entities)
-- "entity" for people/businesses/products mentioned
-- "long_term" for preferences, patterns, decisions
-- "short_term" for context that may expire
-- Return [] if nothing worth remembering
-- Max 3 items per turn`,
-    messages: [{
-      role: "user",
-      content: `User: ${userMessage.substring(0, 500)}\n\nAssistant: ${assistantResponse.substring(0, 1000)}`
-    }]
-  });
-  const text = extraction.content.find(b => b.type === "text")?.text || "[]";
-  const match = text.match(/\[[\s\S]*\]/);
-  if (!match) return;
-  let items;
-  try {
-    items = JSON.parse(match[0]);
-  } catch {
-    return; // Malformed JSON from extraction
-  }
-  for (const item of items.slice(0, 3)) {
-    if (!item.key) continue;
-    await supabase.rpc("store_memory", {
-      p_agent_id: agentId,
-      p_store_id: storeId,
-      p_type: item.type || "short_term",
-      p_key: item.key,
-      p_value: item.value || {}
-    });
-  }
-}
-
-// ============================================================================
-// COST BUDGET TRACKING — increment active budgets after each conversation
-// ============================================================================
-
-// P0 FIX: Atomic cost budget increment via RPC (fixes TOCTOU race on concurrent updates)
-async function updateCostBudgets(supabase, storeId, agentId, costUsd) {
-  const {
-    error
-  } = await supabase.rpc("increment_cost_budget", {
-    p_store_id: storeId,
-    p_agent_id: agentId,
-    p_cost_usd: costUsd
-  });
-  if (error) {
-    console.error("[cost-budget] increment_cost_budget RPC failed:", error.message);
-  }
-}
-
 // ============================================================================
 // HTTP SERVER
 // ============================================================================
@@ -1395,17 +97,31 @@ const server = http.createServer(async (req, res) => {
     const ready = isReady();
     const status = ready ? 200 : 503;
     const agentStats = getGatewayStats();
+    const cbStats = getCircuitBreakerStats();
     jsonResponse(res, status, {
       status: ready ? "ok" : "starting",
       version: process.env.npm_package_version || "6.4.0",
       uptime: Math.floor(process.uptime()),
       pg_listen: pgListenReady,
       worker_pool: workerPoolReady,
-      local_agents: agentStats.total_agents
+      local_agents: agentStats.total_agents,
+      circuit_breakers: cbStats
     }, corsHeaders);
     return;
   }
 
+  // OpenAPI spec — public, no auth required (for Scalar docs viewer)
+  if (req.method === "GET" && req.url === "/openapi.json") {
+    const spec = generateOpenApiSpec();
+    res.writeHead(200, {
+      ...corsHeaders,
+      "Content-Type": "application/json",
+      "Cache-Control": "public, max-age=3600"
+    });
+    res.end(JSON.stringify(spec));
+    return;
+  }
+
   // CORS preflight
   if (req.method === "OPTIONS") {
     res.writeHead(204, corsHeaders);
@@ -1464,6 +180,7 @@ const server = http.createServer(async (req, res) => {
     }
 
     // H6 FIX: Enforce per-run and total client limits
+    const sseClients = getSseClients();
     const existingClients = sseClients.get(runId)?.size || 0;
     if (existingClients >= MAX_SSE_CLIENTS_PER_RUN) {
       jsonResponse(res, 429, {
@@ -1985,6 +702,43 @@ const server = http.createServer(async (req, res) => {
     }
     return;
   }
+
+  // ================================================================
+  // Session API routes (supports GET, POST)
+  // Must be before the method gate since GET /sessions is valid
+  // ================================================================
+  if (pathname.startsWith("/sessions")) {
+    const authHeader = req.headers.authorization;
+    const token = authHeader?.startsWith("Bearer ") ? authHeader.substring(7) : "";
+    if (!token) {
+      req.resume();
+      jsonResponse(res, 401, {
+        error: "Missing authorization"
+      }, corsHeaders);
+      return;
+    }
+    const supabase = getServiceClient();
+    let userId;
+    const isServiceRole = safeCompare(token, SUPABASE_SERVICE_ROLE_KEY) || safeCompare(token, SERVICE_ROLE_JWT);
+    if (!isServiceRole) {
+      const {
+        data: {
+          user: authUser
+        }
+      } = await supabase.auth.getUser(token);
+      if (authUser) userId = authUser.id;
+    }
+    const handled = await handleSessionRoutes(req, res, pathname, url, corsHeaders, supabase, {
+      userId,
+      isServiceRole
+    });
+    if (!handled) {
+      jsonResponse(res, 404, {
+        error: `No route: ${req.method} ${pathname}`
+      }, corsHeaders);
+    }
+    return;
+  }
   if (req.method !== "POST" && req.method !== "DELETE" && req.method !== "PUT") {
     jsonResponse(res, 405, {
       error: "Method not allowed"
@@ -2898,406 +1652,17 @@ const server = http.createServer(async (req, res) => {
   }
 });
 
-// Inject tool executor into workflow engine (avoids circular dependency)
-setToolExecutor((supabase, toolName, args, storeId, traceId) => {
-  // Store boundary validation: prevent workflows from accessing other stores
-  if (args.store_id && args.store_id !== storeId) {
-    return Promise.resolve({
-      success: false,
-      error: "Store boundary violation: workflow cannot access other stores"
-    });
-  }
-  args.store_id = storeId; // Force the workflow's store
-  return executeTool(supabase, toolName, args, storeId, traceId, null, null, "workflow_engine");
-});
-
-// Inject agent executor for "agent" step type in workflows
-setAgentExecutor(async (supabase, agentId, prompt, storeId, maxTurns = 5, onToken, traceId) => {
-  const agent = await loadAgentConfig(supabase, agentId, storeId);
-  if (!agent) return {
-    success: false,
-    error: `Agent ${agentId} not found`
-  };
-
-  // Workflow agents get all tools (no lazy loading — they run headless)
-  const {
-    all: allTools
-  } = await loadTools(supabase);
-  const {
-    rows: userToolRows,
-    defs: userToolDefs
-  } = await loadUserTools(supabase, storeId);
-  const tools = getToolsForAgent(agent, allTools, userToolDefs);
-  const agentModel = agent.model || MODELS.SONNET;
-
-  // Resolve all behavioral config from DB — workflow maxTurns caps agent config
-  const resolved = resolveAgentLoopConfig(agent, "workflow", maxTurns);
-  if (resolved.defaultsUsed.length > 0) {
-    log.info({
-      agentId,
-      callPath: "workflow",
-      defaults: resolved.defaultsUsed
-    }, "workflow agent config defaults applied");
-  }
-
-  // Build full system prompt — shared helper ensures workflow gets same context as SSE/channel
-  // (previously hand-built, missing location/customer context)
-  const {
-    systemPrompt
-  } = await buildAgentSystemPrompt(supabase, agent, storeId, prompt, tools);
-  try {
-    const result = await runServerAgentLoop({
-      anthropic: getAnthropicClient(agent),
-      supabase,
-      model: agentModel,
-      systemPrompt,
-      messages: [{
-        role: "user",
-        content: prompt
-      }],
-      tools,
-      // All behavioral knobs from resolved config — DB is single source of truth
-      maxTurns: resolved.maxTurns,
-      temperature: resolved.temperature,
-      maxTokens: resolved.maxTokens,
-      maxConcurrentTools: resolved.maxConcurrentTools,
-      enableDelegation: resolved.enableDelegation,
-      enableModelRouting: resolved.enableModelRouting,
-      contextOverrides: resolved.contextOverrides,
-      subagentMaxTokens: resolved.subagentMaxTokens,
-      subagentMaxTurns: resolved.subagentMaxTurns,
-      subagentTemperature: resolved.subagentTemperature,
-      loopDetectorConfig: {
-        identicalCallLimit: resolved.loopIdenticalCallLimit,
-        consecutiveErrorLimit: resolved.loopConsecutiveErrorLimit,
-        turnErrorLimit: resolved.loopTurnErrorLimit,
-        window: resolved.loopWindow,
-        sessionToolErrorLimit: resolved.loopSessionToolErrorLimit,
-        consecutiveFailedTurnLimit: resolved.loopConsecutiveFailedTurnLimit,
-        fileReadLimit: resolved.loopFileReadLimit
-      },
-      batchErrorLimit: resolved.loopBatchErrorLimit,
-      storeId,
-      source: "workflow_agent",
-      agentId,
-      traceId,
-      executeTool: async (toolName, args) => {
-        const toolArgs = {
-          ...args
-        };
-        if (!toolArgs.store_id) toolArgs.store_id = storeId;
-        return executeTool(supabase, toolName, toolArgs, storeId, traceId, null, null, "workflow_agent", undefined, userToolRows, agentId, undefined, true);
-      },
-      enableStreaming: !!onToken,
-      onText: onToken || undefined,
-      maxDurationMs: resolved.maxDurationMs
-    });
-    return {
-      success: true,
-      response: result.finalText || "(no response)"
-    };
-  } catch (err) {
-    return {
-      success: false,
-      error: sanitizeError(err)
-    };
-  }
-});
-
-// Inject token broadcaster for real-time agent step streaming
-setTokenBroadcaster((runId, stepKey, token) => {
-  broadcastToRun(runId, {
-    type: "agent_token",
-    run_id: runId,
-    step_key: stepKey,
-    token
-  });
-});
-
-// Inject step error broadcaster for real-time workflow error surfacing
-setStepErrorBroadcaster((runId, data) => {
-  broadcastToRun(runId, data);
-});
-
-// Shared agent invoker — used by channel messages, webchat, and any source
-// Uses shared helpers so it's IDENTICAL to CLI chat — same prompt, tools, memory, persistence, audit
-async function invokeAgentForChannel(supabase, agentId, message, storeId, conversationId, senderContext) {
-  const agent = await loadAgentConfig(supabase, agentId, storeId);
-  if (!agent) return {
-    success: false,
-    error: `Agent ${agentId} not found`
-  };
-  const {
-    core: coreTools,
-    extended: extendedTools
-  } = await loadTools(supabase);
-  setExtendedToolsCache(extendedTools);
-  const {
-    rows: userToolRows,
-    defs: userToolDefs
-  } = await loadUserTools(supabase, storeId);
-  const tools = getToolsForAgent(agent, coreTools, userToolDefs);
-  const agentModel = agent.model || MODELS.SONNET;
-
-  // Resolve all behavioral config from DB — channel has 15-turn safety cap built in
-  const resolved = resolveAgentLoopConfig(agent, "channel");
-  if (resolved.defaultsUsed.length > 0) {
-    log.info({
-      agentId,
-      callPath: "channel",
-      defaults: resolved.defaultsUsed
-    }, "channel agent config defaults applied");
-  }
-
-  // Build system prompt — shared helper, identical to SSE chat
-  const {
-    systemPrompt,
-    dynamicContext
-  } = await buildAgentSystemPrompt(supabase, agent, storeId, message, tools, {
-    senderContext,
-    extendedTools: getExtendedToolsIndex()
-  });
-
-  // Update ai_conversations with agent_id (fire-and-forget)
-  if (conversationId) {
-    Promise.resolve(supabase.from("ai_conversations").update({
-      agent_id: agentId
-    }).eq("id", conversationId).is("agent_id", null)).catch(() => {});
-  }
-
-  // Load conversation history with size-based compaction (same as SSE chat)
-  const MAX_HISTORY_CHARS = resolved.maxHistoryChars;
-  let loadedHistory = [];
-  if (conversationId) {
-    try {
-      const {
-        data: history
-      } = await supabase.from("ai_messages").select("role, content").eq("conversation_id", conversationId).order("created_at", {
-        ascending: true
-      }).limit(50);
-      if (history?.length) {
-        const raw = [];
-        for (const m of history) {
-          if (m.role === "user" || m.role === "assistant") {
-            raw.push({
-              role: m.role,
-              content: m.content
-            });
-          }
-        }
-        loadedHistory = compactHistory(raw, MAX_HISTORY_CHARS);
-      }
-    } catch {/* history not critical */}
-  }
-  // Prepend dynamic context to user message to keep system prompt static (cache-friendly)
-  const contextPrefix = dynamicContext ? `[Context]\n${dynamicContext}\n\n[User Message]\n` : "";
-  const finalMessage = contextPrefix + message;
-  const messages = [...loadedHistory, {
-    role: "user",
-    content: finalMessage
-  }];
-  const traceId = randomUUID();
-  const chatStartTime = Date.now();
-  try {
-    const result = await runServerAgentLoop({
-      anthropic: getAnthropicClient(agent),
-      supabase,
-      model: agentModel,
-      systemPrompt,
-      messages,
-      tools,
-      extendedTools,
-      // All behavioral knobs from resolved config — DB is single source of truth
-      maxTurns: resolved.maxTurns,
-      temperature: resolved.temperature,
-      maxTokens: resolved.maxTokens,
-      maxConcurrentTools: resolved.maxConcurrentTools,
-      enableDelegation: resolved.enableDelegation,
-      enableModelRouting: resolved.enableModelRouting,
-      contextOverrides: resolved.contextOverrides,
-      subagentMaxTokens: resolved.subagentMaxTokens,
-      subagentMaxTurns: resolved.subagentMaxTurns,
-      subagentTemperature: resolved.subagentTemperature,
-      loopDetectorConfig: {
-        identicalCallLimit: resolved.loopIdenticalCallLimit,
-        consecutiveErrorLimit: resolved.loopConsecutiveErrorLimit,
-        turnErrorLimit: resolved.loopTurnErrorLimit,
-        window: resolved.loopWindow,
-        sessionToolErrorLimit: resolved.loopSessionToolErrorLimit,
-        consecutiveFailedTurnLimit: resolved.loopConsecutiveFailedTurnLimit,
-        fileReadLimit: resolved.loopFileReadLimit
-      },
-      batchErrorLimit: resolved.loopBatchErrorLimit,
-      storeId,
-      source: "channel_agent",
-      agentId,
-      traceId,
-      conversationId,
-      executeTool: async (toolName, args) => {
-        const toolArgs = {
-          ...args
-        };
-        if (!toolArgs.store_id) toolArgs.store_id = storeId;
-        // Pass sender context as user identity so node-triggered tool calls aren't anonymous
-        const senderUserId = senderContext?.customerId || null;
-        const senderUserLabel = senderContext?.customerName || senderContext?.senderName || (senderContext?.senderId ? `${senderContext.channelType}:${senderContext.senderId}` : null);
-        return executeTool(supabase, toolName, toolArgs, storeId, traceId, senderUserId, senderUserLabel, "channel_agent", conversationId, userToolRows, agentId, undefined, true);
-      },
-      enableStreaming: false,
-      maxDurationMs: resolved.maxDurationMs
-    });
-
-    // Persist everything — shared helper, identical to SSE chat
-    await persistAgentTurn(supabase, agent, {
-      conversationId,
-      storeId,
-      agentId,
-      agentModel,
-      traceId,
-      message,
-      result,
-      source: "channel_agent",
-      chatStartTime,
-      chatEndTime: Date.now(),
-      userId: senderContext?.customerId || undefined,
-      userEmail: senderContext?.customerName || senderContext?.senderName || (senderContext?.senderId ? `${senderContext.channelType}:${senderContext.senderId}` : undefined),
-      senderContext
-    });
-    return {
-      success: true,
-      response: result.finalText || "(no response)"
-    };
-  } catch (err) {
-    return {
-      success: false,
-      error: err.message
-    };
-  }
-}
+// Wire executors into workflow engine (extracted modules — no inline duplicates)
+wireToolExecutor();
+wireAgentExecutor();
+wireBroadcasters();
 
 // Wire both channel and webchat to the same invoker
 setNodeAgentInvoker(invokeAgentForChannel);
 webchatAgentInvoker = invokeAgentForChannel;
 
-// ============================================================================
-// NODE OFFLINE DETECTION
-// ============================================================================
-
-async function enforceNodeOfflineStatus(supabase) {
-  const threshold = new Date(Date.now() - 3 * 60_000).toISOString(); // 3 missed heartbeats (60s interval)
-  const {
-    data
-  } = await supabase.from("nodes").update({
-    status: "offline"
-  }).eq("status", "online").lt("last_heartbeat", threshold).select("id");
-  return data?.length || 0;
-}
-
-// ============================================================================
-// PERSISTENT WORKFLOW WORKER LOOP (5-second interval)
-// ============================================================================
-
-// Phase 3.1: Increased from 5s to 15s — NOTIFY-driven execution handles the fast path
-// Worker loop is now a safety net for missed notifications
-const BASE_WORKER_INTERVAL_MS = 15_000;
-const MAX_WORKER_INTERVAL_MS = 60_000;
-let workerRunning = false;
-let consecutiveErrors = 0;
-let currentWorkerInterval = BASE_WORKER_INTERVAL_MS;
-async function workflowWorkerLoop() {
-  if (workerRunning) return; // Prevent concurrent runs
-  workerRunning = true;
-  try {
-    const supabase = getServiceClient();
-    const [stepResult, waitingResolved] = await Promise.all([processWorkflowSteps(supabase, 10), processWaitingSteps(supabase), Promise.resolve(supabase.rpc("expire_pending_waitpoints")).then(() => {}).catch(e => log.warn({
-      err: e.message
-    }, "expire_pending_waitpoints failed")) // Non-fatal
-    ]);
-
-    // Schedule triggers + timeout enforcement + event triggers + orphan cleanup + DLQ retries + node offline detection
-    const [scheduled, timedOut, eventsProcessed, orphansCleaned, dlqRetried, nodesOfflined] = await Promise.all([processScheduleTriggers(supabase).catch(e => {
-      log.warn({
-        err: e.message
-      }, "processScheduleTriggers failed");
-      return 0;
-    }), enforceWorkflowTimeouts(supabase).catch(e => {
-      log.warn({
-        err: e.message
-      }, "enforceWorkflowTimeouts failed");
-      return 0;
-    }), processEventTriggers(supabase).catch(e => {
-      log.warn({
-        err: e.message
-      }, "processEventTriggers failed");
-      return 0;
-    }), cleanupOrphanedSteps(supabase).catch(e => {
-      log.warn({
-        err: e.message
-      }, "cleanupOrphanedSteps failed");
-      return 0;
-    }), processDlqRetries(supabase).catch(e => {
-      log.warn({
-        err: e.message
-      }, "processDlqRetries failed");
-      return 0;
-    }), enforceNodeOfflineStatus(supabase).catch(e => {
-      log.warn({
-        err: e.message
-      }, "enforceNodeOfflineStatus failed");
-      return 0;
-    })]);
-    if (stepResult.processed > 0 || waitingResolved > 0 || scheduled > 0 || timedOut > 0 || eventsProcessed > 0 || stepResult.reclaimed > 0 || orphansCleaned > 0 || dlqRetried > 0 || nodesOfflined > 0) {
-      log.info({
-        processed: stepResult.processed,
-        errors: stepResult.errors,
-        reclaimed: stepResult.reclaimed || 0,
-        waiting: waitingResolved,
-        scheduled,
-        timedOut,
-        events: eventsProcessed,
-        orphans: orphansCleaned,
-        dlqRetries: dlqRetried,
-        nodesOfflined
-      }, "worker tick");
-    }
-
-    // Reset backoff on success
-    if (consecutiveErrors > 0) {
-      consecutiveErrors = 0;
-      if (currentWorkerInterval !== BASE_WORKER_INTERVAL_MS) {
-        currentWorkerInterval = BASE_WORKER_INTERVAL_MS;
-        clearInterval(workerInterval);
-        workerInterval = setInterval(workflowWorkerLoop, currentWorkerInterval);
-        log.info({
-          intervalMs: currentWorkerInterval
-        }, "worker interval reset after recovery");
-      }
-    }
-  } catch (err) {
-    consecutiveErrors++;
-    log.error({
-      err: sanitizeError(err),
-      consecutiveErrors
-    }, "worker error");
-
-    // Exponential backoff: 5s → 10s → 20s → 40s → 60s (cap)
-    if (consecutiveErrors >= 3) {
-      const newInterval = Math.min(BASE_WORKER_INTERVAL_MS * Math.pow(2, consecutiveErrors - 2), MAX_WORKER_INTERVAL_MS);
-      if (newInterval !== currentWorkerInterval) {
-        currentWorkerInterval = newInterval;
-        clearInterval(workerInterval);
-        workerInterval = setInterval(workflowWorkerLoop, currentWorkerInterval);
-        log.warn({
-          intervalMs: currentWorkerInterval,
-          consecutiveErrors
-        }, "worker interval increased due to repeated errors");
-      }
-    }
-  } finally {
-    workerRunning = false;
-  }
-}
-let workerInterval = setInterval(workflowWorkerLoop, BASE_WORKER_INTERVAL_MS);
+// Start persistent workflow worker loop (extracted module)
+startWorkerLoop();
 
 // Initialize shared Supabase client with retry logic before server starts
 initSupabase(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY);
@@ -3305,8 +1670,7 @@ server.listen(PORT, () => {
   log.info({
     port: PORT,
     supabaseUrl: SUPABASE_URL,
-    runtime: process.version,
-    workerIntervalMs: BASE_WORKER_INTERVAL_MS
+    runtime: process.version
   }, "server listening");
 
   // Initialize code worker pool for fast code step execution
@@ -3367,11 +1731,12 @@ async function gracefulShutdown(signal) {
     log.info("HTTP server closed");
   });
 
-  // 2. Clear workflow worker intervals
-  clearInterval(workerInterval);
+  // 2. Clear workflow worker + SSE cleanup intervals
+  stopWorkerLoop();
   clearInterval(sseCleanupInterval);
 
   // 3. Close all SSE client connections
+  const sseClients = getSseClients();
   for (const [, clients] of sseClients) {
     for (const res of clients) {
       try {
@@ -3415,11 +1780,7 @@ async function gracefulShutdown(signal) {
   rateLimiter.shutdown();
 
   // 5. Close pg LISTEN connection
-  if (pgClient) {
-    pgClient.end().catch(() => {});
-    pgClient = null;
-    log.info("pg LISTEN connection closed");
-  }
+  closePgClient();
 
   // 6. Wait for active requests to drain (up to 25 seconds)
   const drainStart = Date.now();