whale-code 6.4.0 → 6.5.0

package/dist/server/handlers/workflow-steps.js

@@ -12,7 +12,10 @@ import { batchClient } from "../lib/batch-client.js";
 import { getProvider } from "../../shared/constants.js";
 import { createLogger } from "../lib/logger.js";
 import { startSpan } from "../lib/otel.js";
+import { queueSpan, auditRowToSpan, classifyErrorType } from "../lib/clickhouse-buffer.js";
 const log = createLogger("workflow-steps");
+// In-memory sliding-window rate limiter for webhooks (replaces audit_logs read)
+const webhookRateCounters = new Map();
 // ============================================================================
 // CONSTANTS
 // ============================================================================
@@ -1053,13 +1056,13 @@ async function updateToolCircuitBreaker(supabase, toolId, success, errorMessage)
             circuit_breaker_state: "open", circuit_breaker_failures: newFailures,
             circuit_breaker_tripped_at: new Date().toISOString(),
         }).eq("id", toolId);
-        const { error: toolCbErr } = await supabase.from("audit_logs").insert({
+        queueSpan(auditRowToSpan({
             action: "workflow.circuit_breaker.tripped", severity: "warning",
             resource_type: "user_tool", resource_id: toolId, source: "workflow_engine",
+            service_name: "workflow-engine", span_kind: "INTERNAL", status_code: "OK",
+            start_time: new Date().toISOString(), end_time: new Date().toISOString(),
             details: { failures: newFailures, threshold: data.circuit_breaker_threshold },
-        });
-        if (toolCbErr)
-            log.warn({ err: toolCbErr.message, toolId }, "tool circuit breaker audit failed");
+        }));
     }
     else {
         await supabase.from("user_tools").update({ circuit_breaker_failures: newFailures }).eq("id", toolId);
@@ -1085,13 +1088,13 @@ async function handleWorkflowCircuitBreaker(supabase, workflowId, success, error
             circuit_breaker_state: "open", circuit_breaker_failures: newFailures,
             circuit_breaker_tripped_at: new Date().toISOString(),
         }).eq("id", workflowId);
-        const { error: wfCbErr } = await supabase.from("audit_logs").insert({
+        queueSpan(auditRowToSpan({
             action: "workflow.circuit_breaker.tripped", severity: "warning",
             resource_type: "workflow", resource_id: workflowId, source: "workflow_engine",
+            service_name: "workflow-engine", span_kind: "INTERNAL", status_code: "OK",
+            start_time: new Date().toISOString(), end_time: new Date().toISOString(),
             details: { failures: newFailures, threshold: data.circuit_breaker_threshold },
-        });
-        if (wfCbErr)
-            log.warn({ err: wfCbErr.message, workflowId }, "workflow circuit breaker audit failed");
+        }));
     }
     else {
         await supabase.from("workflows").update({ circuit_breaker_failures: newFailures }).eq("id", workflowId);
@@ -1286,6 +1289,49 @@ export async function processWaitingSteps(supabase) {
             }
         }
     }
+    // 1b. Goal steps waiting for goal completion
+    const { data: goalSteps } = await supabase
+        .from("workflow_step_runs")
+        .select("id, run_id, step_key, output, step_type")
+        .eq("status", "waiting")
+        .eq("step_type", "goal")
+        .limit(50);
+    if (goalSteps?.length) {
+        const goalIds = goalSteps
+            .map(s => s.output?.goal_id)
+            .filter(Boolean);
+        if (goalIds.length) {
+            const { data: goals } = await supabase
+                .from("goals")
+                .select("id, status, final_summary, failure_reason, total_cost_usd, total_rounds")
+                .in("id", goalIds)
+                .in("status", ["completed", "failed", "cancelled"]);
+            if (goals?.length) {
+                const goalMap = new Map(goals.map(g => [g.id, g]));
+                for (const step of goalSteps) {
+                    const goalId = step.output?.goal_id;
+                    const goal = goalMap.get(goalId);
+                    if (!goal)
+                        continue;
+                    const success = goal.status === "completed";
+                    await supabase.from("workflow_step_runs").update({
+                        status: success ? "success" : "failed",
+                        output: {
+                            goal_id: goalId,
+                            goal_status: goal.status,
+                            summary: goal.final_summary,
+                            total_cost_usd: goal.total_cost_usd,
+                            total_rounds: goal.total_rounds,
+                        },
+                        error_message: success ? null : goal.failure_reason,
+                        completed_at: new Date().toISOString(),
+                    }).eq("id", step.id);
+                    await accumulateAndAdvance(supabase, step.id, step.run_id, step.step_key, success, { goal_id: goalId, summary: goal.final_summary }, success ? null : goal.failure_reason);
+                    resolved++;
+                }
+            }
+        }
+    }
     // 2. P1 FIX: Use aggregate RPC to eliminate N+1 queries (was 4 queries per parent)
     const { data: aggregatedParents, error: aggErr } = await supabase.rpc("get_waiting_parents_with_children");
     if (aggErr) {
@@ -1779,18 +1825,21 @@ export async function executeAndAdvance(supabase, step, traceId) {
         if (cpError)
             log.warn({ err: cpError.message, runId: step.run_id, stepKey: step.step_key }, "checkpoint insert failed");
     }
-    // Audit
-    const { error: stepAuditErr } = await supabase.from("audit_logs").insert({
+    // Telemetry → ClickHouse
+    queueSpan(auditRowToSpan({
         action: `workflow.step.${result.success ? "completed" : "failed"}`,
         severity: result.success ? "info" : "error",
         store_id: step.store_id, resource_type: "workflow_step_run",
         resource_id: step.step_run_id, source: "workflow_engine", duration_ms: durationMs,
         request_id: traceId || null,
+        service_name: "workflow-engine", span_kind: "INTERNAL",
+        status_code: result.success ? "OK" : "ERROR",
+        start_time: new Date(Date.now() - durationMs).toISOString(),
+        end_time: new Date().toISOString(),
+        error_type: result.error ? classifyErrorType(result.error) : undefined,
         details: { workflow_id: step.workflow_id, run_id: step.run_id, step_key: step.step_key, step_type: step.step_type, attempt: step.attempt_count },
         error_message: result.error || null,
-    });
-    if (stepAuditErr)
-        log.warn({ err: stepAuditErr.message, runId: step.run_id }, "step audit insert failed");
+    }));
     // Surface step errors to clients via SSE broadcast + structured error_details
     if (!result.success && result.error) {
         await surfaceStepError(supabase, step, result.error);
@@ -2073,7 +2122,7 @@ export async function executeInlineChain(supabase, runId, depth = 0, traceId) {
         return;
     }
     // Steps that go async (delay, sub_workflow, parallel, for_each, approval) don't chain
-    const asyncTypes = new Set(["delay", "sub_workflow", "parallel", "for_each", "approval", "waitpoint"]);
+    const asyncTypes = new Set(["delay", "sub_workflow", "parallel", "for_each", "approval", "waitpoint", "goal"]);
     if (asyncTypes.has(step.step_type))
         return;
     // Chain to next step
@@ -2185,17 +2234,20 @@ export async function completeWorkflowRun(supabase, runId, workflowId, storeId,
     // Circuit breaker
     if (workflowId)
         await handleWorkflowCircuitBreaker(supabase, workflowId, status === "success", errorMessage);
-    // Audit
-    const { error: runAuditErr } = await supabase.from("audit_logs").insert({
+    // Telemetry → ClickHouse
+    queueSpan(auditRowToSpan({
         action: `workflow.run.${status}`, severity: status === "success" ? "info" : "error",
         store_id: storeId || null, resource_type: "workflow_run", resource_id: runId,
         source: "workflow_engine", duration_ms: durationMs,
         request_id: resolvedTraceId,
+        service_name: "workflow-engine", span_kind: "INTERNAL",
+        status_code: status === "success" ? "OK" : "ERROR",
+        start_time: new Date(Date.now() - (durationMs || 0)).toISOString(),
+        end_time: new Date().toISOString(),
+        error_type: errorMessage ? classifyErrorType(errorMessage) : undefined,
         details: { workflow_id: workflowId, run_id: runId },
         error_message: errorMessage || null,
-    });
-    if (runAuditErr)
-        log.warn({ err: runAuditErr.message, runId }, "run completion audit failed");
+    }));
     // Error notifications
     if (status === "failed" && workflowId) {
         await sendErrorNotification(supabase, workflowId, runId, storeId, errorMessage, errorStepKey);
@@ -2265,15 +2317,18 @@ export async function handleWebhookIngestion(supabase, slug, rawBody, headers, s
     const endpoint = endpoints?.[0];
     if (!endpoint)
         return { status: 404, body: { error: "Webhook endpoint not found" } };
-    // Rate limit
-    const oneMinAgo = new Date(Date.now() - 60_000).toISOString();
-    const { count: recentCount } = await supabase.from("audit_logs")
-        .select("id", { count: "exact", head: true })
-        .eq("resource_type", "webhook_endpoint").eq("resource_id", endpoint.id)
-        .gte("created_at", oneMinAgo);
-    if ((recentCount || 0) >= endpoint.max_requests_per_minute) {
+    // Rate limit — in-memory sliding window (no DB round-trip)
+    const now = Date.now();
+    let counter = webhookRateCounters.get(endpoint.id);
+    if (!counter) {
+        counter = { timestamps: [] };
+        webhookRateCounters.set(endpoint.id, counter);
+    }
+    counter.timestamps = counter.timestamps.filter(t => t > now - 60_000);
+    if (counter.timestamps.length >= endpoint.max_requests_per_minute) {
         return { status: 429, body: { error: "Rate limit exceeded" } };
     }
+    counter.timestamps.push(now);
     // HMAC verification
     if (endpoint.verify_signature) {
         const signature = headers["x-webhook-signature"] || headers["x-hub-signature-256"] || "";
@@ -2311,14 +2366,15 @@ export async function handleWebhookIngestion(supabase, slug, rawBody, headers, s
         last_received_at: new Date().toISOString(),
         total_received: (endpoint.total_received || 0) + 1,
     }).eq("id", endpoint.id);
-    // Audit
-    const { error: whAuditErr } = await supabase.from("audit_logs").insert({
+    // Telemetry → ClickHouse
+    queueSpan(auditRowToSpan({
         action: "webhook.received", severity: "info", store_id: endpoint.store_id,
         resource_type: "webhook_endpoint", resource_id: endpoint.id, source: "webhook",
+        service_name: "workflow-engine", span_kind: "SERVER", status_code: "OK",
+        start_time: new Date().toISOString(), end_time: new Date().toISOString(),
+        input_bytes: rawBody.length,
         details: { slug, workflow_id: endpoint.workflow_id },
-    });
-    if (whAuditErr)
-        log.warn({ err: whAuditErr.message, slug }, "webhook audit insert failed");
+    }));
     // Start workflow
     const { data: startResult } = await supabase.rpc("start_workflow_run", {
         p_workflow_id: endpoint.workflow_id, p_store_id: endpoint.store_id,

package/dist/server/handlers/workflows.js

@@ -962,7 +962,7 @@ export async function handleWorkflows(supabase, args, storeId) {
         }
         case "list_tools": {
             const { data, error } = await supabase.from("ai_tool_registry")
-                .select("name, description, category, definition")
+                .select("name, description, category, definition, tool_mode, is_read_only, requires_store_id")
                 .eq("is_active", true)
                 .or("tool_mode.is.null,tool_mode.neq.code")
                 .order("category").order("name");
@@ -979,6 +979,9 @@ export async function handleWorkflows(supabase, args, storeId) {
                     category: row.category || "other",
                     actions,
                     input_schema: inputSchema || null,
+                    tool_mode: row.tool_mode || null,
+                    is_read_only: row.is_read_only ?? false,
+                    requires_store_id: row.requires_store_id ?? false,
                 };
             });
             return { success: true, data: { tools } };