whale-code 6.4.0

package/dist/server/handlers/workflow-steps.js

@@ -0,0 +1,2349 @@
+// server/handlers/workflow-steps.ts — Step executor engine
+// Extracted from workflows.ts to separate step execution from workflow CRUD/management.
+//
+// Contains: step type executors, executeAndAdvance, inline chain execution,
+// circuit breakers, code execution (JS/Python), cron parser, schedule/timeout processing,
+// event trigger processing, flow control, webhook ingestion, and all step advancement helpers.
+import { createHmac, timingSafeEqual, randomUUID } from "node:crypto";
+import { resolveTemplate, evaluateCondition } from "../lib/template-resolver.js";
+import { sanitizeError } from "../../shared/agent-core.js";
+import { executeWithPool, initWorkerPool, getPoolStats, shutdownPool } from "../lib/code-worker-pool.js";
+import { batchClient } from "../lib/batch-client.js";
+import { getProvider } from "../../shared/constants.js";
+import { createLogger } from "../lib/logger.js";
+import { startSpan } from "../lib/otel.js";
+const log = createLogger("workflow-steps");
+// ============================================================================
+// CONSTANTS
+// ============================================================================
+const MAX_INLINE_DEPTH = 50;
+const CODE_TIMEOUT_MS = 5000;
+const CODE_OUTPUT_MAX = 102_400; // 100KB
+const MAX_FOR_EACH_ITEMS = 1000; // P2 FIX: Prevent unbounded for_each expansion
+const MAX_PARALLEL_CHILDREN = 100; // P1 FIX: Cap parallel step fan-out
+const GUEST_APPROVAL_SECRET = process.env.GUEST_APPROVAL_SECRET || process.env.FLY_INTERNAL_SECRET || "";
+const GUEST_APPROVAL_BASE_URL = "https://whale-agent.fly.dev/approvals/guest";
+// ============================================================================
+// GUEST APPROVAL — HMAC-signed URLs for unauthenticated approvers
+// ============================================================================
+export function generateGuestApprovalUrl(approvalId, action, expiresAt) {
+    if (!GUEST_APPROVAL_SECRET)
+        return null; // Guest approvals disabled — no signing secret configured
+    const payload = `${approvalId}:${action}:${expiresAt}`;
+    const sig = createHmac("sha256", GUEST_APPROVAL_SECRET).update(payload).digest("hex");
+    return `${GUEST_APPROVAL_BASE_URL}/${approvalId}?action=${action}&expires=${encodeURIComponent(expiresAt)}&sig=${sig}`;
+}
+export function verifyGuestApprovalSignature(approvalId, action, expiresAt, sig) {
+    if (!GUEST_APPROVAL_SECRET)
+        return false; // Guest approvals disabled — no signing secret configured
+    const payload = `${approvalId}:${action}:${expiresAt}`;
+    const expected = createHmac("sha256", GUEST_APPROVAL_SECRET).update(payload).digest("hex");
+    try {
+        return timingSafeEqual(Buffer.from(sig, "hex"), Buffer.from(expected, "hex"));
+    }
+    catch {
+        return false;
+    }
+}
+// ============================================================================
+// EVENT JOURNAL — append-only state transition log
+// ============================================================================
+export async function logWorkflowEvent(supabase, runId, eventType, payload, stepRunId) {
+    const { error } = await supabase.from("workflow_events").insert({
+        run_id: runId,
+        step_run_id: stepRunId || null,
+        event_type: eventType,
+        payload,
+    });
+    if (error)
+        log.warn({ err: error.message, runId, eventType }, "logWorkflowEvent insert failed");
+}
+// ============================================================================
+// FLOW CONTROL — concurrency + rate limiting at step level
+// ============================================================================
+async function checkFlowControl(supabase, step) {
+    const config = step.step_config;
+    // Per-step concurrency limit
+    // P0 FIX: Filter by store_id to prevent cross-tenant data access in flow control decisions
+    const concurrencyLimit = config.concurrency_limit;
+    if (concurrencyLimit && concurrencyLimit > 0) {
+        const concurrencyKey = config.concurrency_key || step.step_key;
+        const { count } = await supabase.from("workflow_step_runs")
+            .select("id", { count: "exact", head: true })
+            .eq("step_key", concurrencyKey)
+            .eq("status", "running")
+            .eq("store_id", step.store_id)
+            .neq("id", step.step_run_id); // exclude self
+        if ((count || 0) >= concurrencyLimit) {
+            return { allowed: false, reason: `Concurrency limit ${concurrencyLimit} reached for '${concurrencyKey}'` };
+        }
+    }
+    // Per-step rate limit (max N executions per window)
+    // P0 FIX: Filter by store_id to prevent cross-tenant data access in flow control decisions
+    const rateLimit = config.rate_limit;
+    const rateWindowSec = config.rate_window_seconds || 60;
+    if (rateLimit && rateLimit > 0) {
+        const windowStart = new Date(Date.now() - rateWindowSec * 1000).toISOString();
+        const { count } = await supabase.from("workflow_step_runs")
+            .select("id", { count: "exact", head: true })
+            .eq("step_key", step.step_key)
+            .eq("store_id", step.store_id)
+            .in("status", ["success", "running"])
+            .gte("started_at", windowStart);
+        if ((count || 0) >= rateLimit) {
+            return { allowed: false, reason: `Rate limit ${rateLimit}/${rateWindowSec}s reached for '${step.step_key}'` };
+        }
+    }
+    return { allowed: true };
+}
+let _executeTool = null;
+let _runAgentQuery = null;
+let _broadcastToken = null;
+let _broadcastStepError = null;
+export function setToolExecutor(fn) { _executeTool = fn; }
+export function setAgentExecutor(fn) { _runAgentQuery = fn; }
+export function setTokenBroadcaster(fn) { _broadcastToken = fn; }
+export function setStepErrorBroadcaster(fn) { _broadcastStepError = fn; }
+/** Broadcast a step error to SSE clients and persist error_details on the step run. */
+async function surfaceStepError(supabase, step, errorMessage) {
+    const timestamp = new Date().toISOString();
+    // 1. Persist structured error_details on the step run record
+    await supabase.from("workflow_step_runs").update({
+        error_details: {
+            step_name: step.step_key,
+            step_type: step.step_type,
+            error_message: errorMessage,
+            timestamp,
+        },
+    }).eq("id", step.step_run_id);
+    // 2. Broadcast via SSE so connected clients see the error in real time
+    if (_broadcastStepError) {
+        _broadcastStepError(step.run_id, {
+            type: "workflow_error",
+            workflow_id: step.workflow_id,
+            run_id: step.run_id,
+            step_name: step.step_key,
+            step_type: step.step_type,
+            error: errorMessage,
+            timestamp,
+        });
+    }
+}
+// ============================================================================
+// STEP EXECUTORS
+// ============================================================================
+async function executeToolStep(supabase, config, ctx, storeId, traceId) {
+    if (!_executeTool)
+        return { success: false, error: "Tool executor not initialized" };
+    const toolName = config.tool_name;
+    if (!toolName)
+        return { success: false, error: "No tool_name in step config" };
+    const argsTemplate = (config.args_template || config.args || {});
+    const resolvedArgs = resolveTemplate(argsTemplate, ctx);
+    // For email steps: auto-inject template_data from workflow context so {{variable}} placeholders resolve
+    if (toolName === "email" && (resolvedArgs.action === "send" || resolvedArgs.action === "send_template") && !resolvedArgs.template_data) {
+        const mergedData = { ...(ctx.trigger || {}) };
+        for (const [, stepData] of Object.entries(ctx.steps || {})) {
+            if (stepData?.output && typeof stepData.output === "object") {
+                Object.assign(mergedData, stepData.output);
+            }
+        }
+        if (Object.keys(mergedData).length > 0) {
+            resolvedArgs.template_data = mergedData;
+        }
+    }
+    if (config.tool_id) {
+        const cb = await checkToolCircuitBreaker(supabase, config.tool_id);
+        if (!cb.allowed)
+            return { success: false, error: cb.reason };
+    }
+    const result = await _executeTool(supabase, toolName, resolvedArgs, storeId, traceId);
+    if (config.tool_id)
+        await updateToolCircuitBreaker(supabase, config.tool_id, result.success, result.error);
+    return result.success
+        ? { success: true, output: result.data }
+        : { success: false, error: result.error };
+}
+function executeConditionStep(config, ctx) {
+    const expression = config.expression;
+    if (!expression)
+        return { success: false, error: "No expression in condition step" };
+    if (!config.on_true && !config.on_false) {
+        return { success: false, output: { error: "Condition step must have at least on_true or on_false defined" } };
+    }
+    const result = evaluateCondition(expression, ctx);
+    const branch = result ? (config.on_true || undefined) : (config.on_false || undefined);
+    return { success: true, output: { condition_result: result, branch }, branch };
+}
+function executeTransformStep(config, ctx) {
+    const mapping = config.mapping;
+    if (!mapping)
+        return { success: false, error: "No mapping in transform step" };
+    return { success: true, output: resolveTemplate(mapping, ctx) };
+}
+async function executeAgentStep(config, ctx, storeId, supabase, step, traceId) {
+    if (!_runAgentQuery)
+        return { success: false, error: "Agent executor not initialized" };
+    const agentId = config.agent_id;
+    if (!agentId)
+        return { success: false, error: "No agent_id in agent step config" };
+    const promptTemplate = (config.prompt_template || config.prompt || "");
+    const prompt = resolveTemplate(promptTemplate, ctx);
+    if (!prompt)
+        return { success: false, error: "No prompt resolved for agent step" };
+    // AI tool gating — inject allowed/blocked tool lists into prompt
+    const allowedTools = config.allowed_tools;
+    const blockedTools = config.blocked_tools;
+    const requireApprovalTools = config.require_approval_tools;
+    let gatedPrompt = prompt;
+    if (allowedTools?.length) {
+        gatedPrompt += `\n\n[SYSTEM: You may ONLY use these tools: ${allowedTools.join(", ")}. Refuse any other tool calls.]`;
+    }
+    if (blockedTools?.length) {
+        gatedPrompt += `\n\n[SYSTEM: You must NEVER use these tools: ${blockedTools.join(", ")}. Use alternatives instead.]`;
+    }
+    if (requireApprovalTools?.length && step) {
+        // Check if approval was already given (stored in step input from approval step)
+        const approvedTools = step.input?.approved_tools;
+        const pendingTools = requireApprovalTools.filter(t => !approvedTools?.includes(t));
+        if (pendingTools.length > 0) {
+            gatedPrompt += `\n\n[SYSTEM: The following tools require human approval before use: ${pendingTools.join(", ")}. Do NOT call them — describe what you would do and why, then stop.]`;
+        }
+    }
+    const maxTurns = config.max_turns || 5;
+    const useBatch = config.use_batch === true;
+    // Batch mode: single-turn LLM call via Batch API for ~50% cost savings.
+    // Only valid when no tool loop is needed (the batch API doesn't support agentic tool loops).
+    if (useBatch && maxTurns <= 1) {
+        const model = config.model || "claude-sonnet-4-6";
+        const provider = getProvider(model);
+        const batchProvider = (provider === "openai") ? "openai" : "anthropic";
+        const requestId = `wf_agent_${randomUUID().replace(/-/g, "").slice(0, 12)}`;
+        try {
+            const batchResult = await batchClient.processSingle(requestId, batchProvider, model, [{ role: "user", content: gatedPrompt }], { max_tokens: config.max_tokens || 4096, temperature: config.temperature });
+            return batchResult.success
+                ? { success: true, output: { response: batchResult.text || "", usage: batchResult.usage, batch: true } }
+                : { success: false, error: batchResult.error || "Batch agent request failed" };
+        }
+        catch (err) {
+            return { success: false, error: sanitizeError(err) };
+        }
+    }
+    // Wire up token broadcasting for SSE streaming to connected clients
+    const onToken = step && _broadcastToken
+        ? (token) => _broadcastToken(step.run_id, step.step_key, token)
+        : undefined;
+    const result = await _runAgentQuery(supabase, agentId, gatedPrompt, storeId, maxTurns, onToken, traceId);
+    return result.success
+        ? { success: true, output: { response: result.response } }
+        : { success: false, error: result.error };
+}
+// P1 FIX: Use shared SSRF guard module (enhanced with DNS resolve-then-check, IPv6-mapped, CGNAT)
+import { validateUrl } from "../lib/ssrf-guard.js";
+async function executeWebhookOutStep(config, ctx) {
+    const url = resolveTemplate(config.url, ctx);
+    if (!url)
+        return { success: false, error: "No URL in webhook_out step" };
+    // P0 FIX: Use async validateUrl (DNS resolve-then-check) instead of sync isBlockedUrl
+    const ssrfError = await validateUrl(url);
+    if (ssrfError)
+        return { success: false, error: `Blocked: ${ssrfError}` };
+    const method = (config.method || "POST").toUpperCase();
+    const headers = {};
+    if (config.headers && typeof config.headers === "object") {
+        for (const [k, v] of Object.entries(config.headers)) {
+            headers[k] = resolveTemplate(v, ctx);
+        }
+    }
+    let body;
+    if (method !== "GET" && method !== "HEAD") {
+        const bodyTemplate = config.body_template || {};
+        body = JSON.stringify(resolveTemplate(bodyTemplate, ctx));
+        if (!headers["Content-Type"])
+            headers["Content-Type"] = "application/json";
+    }
+    if (config.hmac_secret && body) {
+        const hmac = createHmac("sha256", config.hmac_secret).update(body).digest("hex");
+        headers["X-Webhook-Signature"] = `sha256=${hmac}`;
+    }
+    try {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 30_000);
+        const resp = await fetch(url, { method, headers, body, signal: controller.signal });
+        clearTimeout(timer);
+        const ct = resp.headers.get("content-type") || "";
+        const data = ct.includes("json") ? await resp.json() : await resp.text();
+        if (!resp.ok)
+            return { success: false, error: `HTTP ${resp.status}: ${String(data).substring(0, 500)}` };
+        return { success: true, output: { status: resp.status, data } };
+    }
+    catch (err) {
+        if (err.name === "AbortError")
+            return { success: false, error: "Webhook request timed out" };
+        return { success: false, error: sanitizeError(err) };
+    }
+}
+function executeNoopStep() {
+    return { success: true, output: { noop: true } };
+}
+// ============================================================================
+// BATCH LLM STEP — non-streaming LLM via Batch API (~50% cost savings)
+// ============================================================================
+/**
+ * Execute an LLM request via the Batch API instead of streaming.
+ * Designed for workflow steps that don't need real-time token streaming.
+ *
+ * Step config:
+ *   model: string          — model ID (e.g. "claude-sonnet-4-6", "gpt-5-mini")
+ *   prompt: string         — prompt template (resolved with ctx)
+ *   system: string         — optional system prompt template
+ *   max_tokens: number     — optional, default 4096
+ *   temperature: number    — optional
+ *   tools: array           — optional tool definitions for the LLM
+ */
+async function executeLlmBatchStep(config, ctx) {
+    const model = config.model;
+    if (!model)
+        return { success: false, error: "No model in llm_batch step config" };
+    const promptTemplate = (config.prompt_template || config.prompt || "");
+    const prompt = resolveTemplate(promptTemplate, ctx);
+    if (!prompt)
+        return { success: false, error: "No prompt resolved for llm_batch step" };
+    const systemTemplate = config.system;
+    const system = systemTemplate ? resolveTemplate(systemTemplate, ctx) : undefined;
+    const maxTokens = config.max_tokens || 4096;
+    const temperature = config.temperature;
+    const tools = config.tools;
+    // Determine provider from model ID
+    const provider = getProvider(model);
+    const batchProvider = (provider === "openai") ? "openai" : "anthropic";
+    const requestId = `wf_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
+    try {
+        const result = await batchClient.processSingle(requestId, batchProvider, model, [{ role: "user", content: prompt }], { system, tools, max_tokens: maxTokens, temperature });
+        if (result.success) {
+            return {
+                success: true,
+                output: {
+                    response: result.text || "",
+                    content: result.content,
+                    usage: result.usage,
+                    batch_request_id: requestId,
+                },
+            };
+        }
+        else {
+            return { success: false, error: result.error || "Batch LLM request failed" };
+        }
+    }
+    catch (err) {
+        return { success: false, error: sanitizeError(err) };
+    }
+}
+// ============================================================================
+// PHASE 2: APPROVAL STEP EXECUTOR
+// ============================================================================
+async function executeApprovalStep(supabase, step, ctx) {
+    const config = step.step_config;
+    // Second pass — step was resumed after approval response
+    if (step.input && typeof step.input === "object" && step.input.approval_status) {
+        const approvalData = step.input;
+        const isApproved = approvalData.approval_status === "approved" || approvalData.approval_status === "approve";
+        return {
+            success: true,
+            output: {
+                approved: isApproved,
+                status: approvalData.approval_status,
+                response_data: approvalData.approval_data,
+                responded_by: approvalData.responded_by,
+            },
+            branch: isApproved ? config.on_approve : config.on_reject,
+        };
+    }
+    // First pass — create approval request and wait
+    const title = resolveTemplate((config.title || "Approval Required"), ctx);
+    const description = config.description ? resolveTemplate(config.description, ctx) : null;
+    const prompt = config.prompt ? resolveTemplate(config.prompt, ctx) : null;
+    const options = config.options || ["approve", "reject"];
+    const timeoutSeconds = config.timeout_seconds || 86400;
+    const timeoutAction = config.timeout_action || "fail";
+    const channels = config.notification_channels || ["push"];
+    const expiresAt = new Date(Date.now() + timeoutSeconds * 1000).toISOString();
+    await supabase.from("workflow_approval_requests").insert({
+        store_id: step.store_id,
+        run_id: step.run_id,
+        step_run_id: step.step_run_id,
+        workflow_id: step.workflow_id,
+        title,
+        description,
+        prompt,
+        options,
+        form_schema: config.form_schema || null,
+        assigned_to: config.assigned_to || null,
+        assigned_role: config.assigned_role || null,
+        expires_at: expiresAt,
+        timeout_action: timeoutAction,
+        notification_channels: channels,
+    });
+    // Generate guest approval URLs (signed, no auth required) — only if signing secret is configured
+    const guestUrls = {};
+    const optionsList = Array.isArray(options) ? options : ["approve", "reject"];
+    for (const opt of optionsList) {
+        const url = generateGuestApprovalUrl(step.step_run_id, opt, expiresAt);
+        if (url)
+            guestUrls[opt] = url;
+    }
+    // Set step to waiting
+    await supabase.from("workflow_step_runs").update({
+        status: "waiting",
+        output: { waiting_for: "approval", title, expires_at: expiresAt, guest_urls: guestUrls },
+    }).eq("id", step.step_run_id);
+    return "waiting";
+}
+// ============================================================================
+// PHASE 7: ENHANCED CODE EXECUTION
+// ============================================================================
+// Re-export pool management for index.ts to initialize on startup
+export { initWorkerPool, getPoolStats, shutdownPool };
+// ============================================================================
+// CRON EXPRESSION PARSER — 5-field (min hour dom mon dow)
+// No external dependencies. Supports: *, */N, N-M, N,M, N
+// ============================================================================
+function parseCronField(field, min, max) {
+    const values = new Set();
+    for (const part of field.split(",")) {
+        const trimmed = part.trim();
+        if (trimmed === "*") {
+            for (let i = min; i <= max; i++)
+                values.add(i);
+        }
+        else if (trimmed.includes("/")) {
+            const [range, stepStr] = trimmed.split("/");
+            const step = parseInt(stepStr, 10);
+            if (isNaN(step) || step <= 0)
+                continue;
+            let start = min, end = max;
+            if (range !== "*") {
+                if (range.includes("-")) {
+                    [start, end] = range.split("-").map(Number);
+                }
+                else {
+                    start = parseInt(range, 10);
+                }
+            }
+            for (let i = start; i <= end; i += step)
+                values.add(i);
+        }
+        else if (trimmed.includes("-")) {
+            const [s, e] = trimmed.split("-").map(Number);
+            for (let i = s; i <= e; i++)
+                values.add(i);
+        }
+        else {
+            const n = parseInt(trimmed, 10);
+            if (!isNaN(n) && n >= min && n <= max)
+                values.add(n);
+        }
+    }
+    return [...values].sort((a, b) => a - b);
+}
+/**
+ * Compute the next occurrence of a 5-field cron expression after `after`.
+ * Returns null if expression is invalid or no match found within 366 days.
+ */
+/**
+ * Get the UTC offset in minutes for an IANA timezone at a specific instant.
+ * Uses Intl.DateTimeFormat.formatToParts to extract wall-clock components,
+ * then diffs against the UTC components of the same instant. No string parsing.
+ */
+function getUtcOffsetMinutes(date, tz) {
+    // Extract wall-clock parts in the target timezone
+    const fmt = new Intl.DateTimeFormat("en-US", {
+        timeZone: tz, year: "numeric", month: "numeric", day: "numeric",
+        hour: "numeric", minute: "numeric", second: "numeric", hour12: false,
+    });
+    const p = Object.fromEntries(fmt.formatToParts(date).map(x => [x.type, parseInt(x.value, 10)]));
+    // Build a pseudo-UTC timestamp from the wall-clock parts
+    const wallMs = Date.UTC(p.year, p.month - 1, p.day, p.hour % 24, p.minute, p.second);
+    // The offset is how far ahead the wall clock is from the actual UTC instant
+    return (wallMs - date.getTime()) / 60_000;
+}
+export function getNextCronTime(expression, after = new Date(), timezone) {
+    const parts = expression.trim().split(/\s+/);
+    if (parts.length !== 5)
+        return null;
+    const minutes = parseCronField(parts[0], 0, 59);
+    const hours = parseCronField(parts[1], 0, 23);
+    const doms = parseCronField(parts[2], 1, 31);
+    const months = parseCronField(parts[3], 1, 12);
+    const dows = parseCronField(parts[4], 0, 6); // 0=Sunday
+    if (!minutes.length || !hours.length || !doms.length || !months.length || !dows.length)
+        return null;
+    // DOM/DOW: POSIX semantics — when both are restricted, match EITHER (OR).
+    // When only one is restricted, match only that one.
+    const domRestricted = parts[2] !== "*";
+    const dowRestricted = parts[4] !== "*";
+    const useDomDowOr = domRestricted && dowRestricted;
+    // Validate timezone
+    let effectiveTz = "UTC";
+    if (timezone) {
+        try {
+            Intl.DateTimeFormat(undefined, { timeZone: timezone });
+            effectiveTz = timezone;
+        }
+        catch { /* invalid tz, stay UTC */ }
+    }
+    // Strategy: work in "local time" coordinates using a fake Date whose UTC fields
+    // represent the wall-clock time in the target timezone. This lets us use the fast
+    // jump-by-month/day/hour logic. Once we find a match, we convert back to real UTC.
+    //
+    // The offset is recomputed each time we cross a day boundary to handle DST transitions.
+    let offsetMin = effectiveTz === "UTC" ? 0 : getUtcOffsetMinutes(after, effectiveTz);
+    const localEpoch = after.getTime() + offsetMin * 60_000;
+    const candidate = new Date(localEpoch);
+    candidate.setUTCSeconds(0, 0);
+    candidate.setUTCMinutes(candidate.getUTCMinutes() + 1); // 1 minute after `after`
+    const maxMs = after.getTime() + 366 * 86_400_000;
+    let lastOffsetDay = candidate.getUTCDate(); // Track day for offset recomputation
+    // Iterate in local-time coordinates (fast jumps — same algorithm as before)
+    while (true) {
+        // Recompute offset on day boundaries to handle DST transitions correctly
+        const currentDay = candidate.getUTCDate();
+        if (effectiveTz !== "UTC" && currentDay !== lastOffsetDay) {
+            // Convert current candidate back to approximate UTC, then get fresh offset
+            const approxUtc = new Date(candidate.getTime() - offsetMin * 60_000);
+            const newOffset = getUtcOffsetMinutes(approxUtc, effectiveTz);
+            if (newOffset !== offsetMin) {
+                // DST changed — adjust candidate to maintain correct local-time coordinates
+                const drift = (newOffset - offsetMin) * 60_000;
+                candidate.setTime(candidate.getTime() + drift);
+                offsetMin = newOffset;
+            }
+            lastOffsetDay = candidate.getUTCDate();
+        }
+        // Safety: check if we've exceeded 366 days
+        const realUtc = candidate.getTime() - offsetMin * 60_000;
+        if (realUtc > maxMs)
+            return null;
+        const mo = candidate.getUTCMonth() + 1;
+        const day = candidate.getUTCDate();
+        const hr = candidate.getUTCHours();
+        const mi = candidate.getUTCMinutes();
+        const dow = candidate.getUTCDay();
+        if (!months.includes(mo)) {
+            candidate.setUTCMonth(candidate.getUTCMonth() + 1, 1);
+            candidate.setUTCHours(0, 0, 0, 0);
+            continue;
+        }
+        const domMatch = doms.includes(day);
+        const dowMatch = dows.includes(dow);
+        const dayMatch = useDomDowOr ? (domMatch || dowMatch) : (domMatch && dowMatch);
+        if (!dayMatch) {
+            candidate.setUTCDate(candidate.getUTCDate() + 1);
+            candidate.setUTCHours(0, 0, 0, 0);
+            continue;
+        }
+        if (!hours.includes(hr)) {
+            candidate.setUTCHours(candidate.getUTCHours() + 1, 0, 0, 0);
+            continue;
+        }
+        if (!minutes.includes(mi)) {
+            candidate.setUTCMinutes(candidate.getUTCMinutes() + 1, 0, 0);
+            continue;
+        }
+        // Match found in local coordinates — convert back to real UTC.
+        if (effectiveTz === "UTC")
+            return candidate;
+        const result = new Date(candidate.getTime() - offsetMin * 60_000);
+        // Verify: the result in UTC should map back to the same local time we matched.
+        // DST spring-forward (gap): 2:00-3:00 doesn't exist → verifyOffset differs → we skip.
+        // DST fall-back (ambiguity): 1:00-2:00 exists twice → first occurrence wins (same as cronie).
+        const verifyOffset = getUtcOffsetMinutes(result, effectiveTz);
+        if (verifyOffset !== offsetMin) {
+            candidate.setUTCMinutes(candidate.getUTCMinutes() + 1, 0, 0);
+            continue;
+        }
+        return result;
+    }
+}
+// ============================================================================
+// SCHEDULE TRIGGER PROCESSING — fires due cron workflows
+// ============================================================================
+export async function processScheduleTriggers(supabase) {
+    // Find workflows that are past due — supports both cron (recurring) and one-time (run_at)
+    const { data: dueWorkflows } = await supabase.from("workflows")
+        .select("id, store_id, cron_expression, timezone")
+        .not("next_run_at", "is", null)
+        .lte("next_run_at", new Date().toISOString())
+        .eq("is_active", true)
+        .eq("status", "active")
+        .limit(10);
+    if (!dueWorkflows?.length)
+        return 0;
+    let fired = 0;
+    for (const wf of dueWorkflows) {
+        try {
+            const isOneTime = !wf.cron_expression;
+            // Start the run
+            const { data: result, error } = await supabase.rpc("start_workflow_run", {
+                p_workflow_id: wf.id,
+                p_store_id: wf.store_id,
+                p_trigger_type: "schedule",
+                p_trigger_payload: {
+                    cron: wf.cron_expression || null,
+                    one_time: isOneTime,
+                    scheduled_at: new Date().toISOString(),
+                },
+                p_idempotency_key: wf.cron_expression
+                    ? `schedule:${wf.id}:${new Date().toISOString().slice(0, 16)}` // Minute-granularity dedup for cron
+                    : `schedule:${wf.id}:one_time`,
+            });
+            if (error || !result?.success) {
+                log.error({ workflowId: wf.id, err: error?.message || result?.error }, "failed to start scheduled workflow");
+                // Still update next_run_at to prevent infinite retries
+            }
+            else {
+                fired++;
+                // Generate trace_id for the new run
+                const traceId = randomUUID();
+                await supabase.from("workflow_runs").update({ trace_id: traceId }).eq("id", result.run_id);
+                // Inline execution
+                try {
+                    await executeInlineChain(supabase, result.run_id);
+                }
+                catch (err) {
+                    log.error({ runId: result.run_id, err: err.message }, "inline chain failed for scheduled run");
+                }
+            }
+            if (isOneTime) {
+                // One-time schedule: clear next_run_at and deactivate
+                await supabase.from("workflows").update({
+                    last_scheduled_at: new Date().toISOString(),
+                    next_run_at: null,
+                    is_active: false,
+                    status: "paused",
+                }).eq("id", wf.id);
+            }
+            else {
+                // Recurring cron: compute next run time
+                const nextRun = getNextCronTime(wf.cron_expression, new Date(), wf.timezone || undefined);
+                await supabase.from("workflows").update({
+                    last_scheduled_at: new Date().toISOString(),
+                    next_run_at: nextRun?.toISOString() || null,
+                }).eq("id", wf.id);
+            }
+        }
+        catch (err) {
+            log.error({ workflowId: wf.id, err: sanitizeError(err) }, "error processing scheduled workflow");
+        }
+    }
+    return fired;
+}
+// ============================================================================
+// WORKFLOW TIMEOUT ENFORCEMENT — cancel overtime runs
+// ============================================================================
+const DEFAULT_MAX_RUN_DURATION_SEC = 3600; // 1 hour hard ceiling for any workflow run
+export async function enforceWorkflowTimeouts(supabase) {
+    // Find running workflows that exceeded their duration limit
+    const { data: timedOut } = await supabase.from("workflow_runs")
+        .select("id, workflow_id, store_id, started_at, workflows!inner(max_run_duration_seconds, name)")
+        .eq("status", "running")
+        .not("started_at", "is", null)
+        .limit(50);
+    if (!timedOut?.length)
+        return 0;
+    let count = 0;
+    const now = Date.now();
+    for (const run of timedOut) {
+        const wf = run.workflows;
+        // Use configured max_duration or fall back to the hard ceiling
+        const maxDuration = (wf?.max_run_duration_seconds && wf.max_run_duration_seconds > 0)
+            ? wf.max_run_duration_seconds
+            : DEFAULT_MAX_RUN_DURATION_SEC;
+        const elapsed = now - new Date(run.started_at).getTime();
+        if (elapsed < maxDuration * 1000)
+            continue;
+        // This run has timed out
+        await completeWorkflowRun(supabase, run.id, run.workflow_id, run.store_id, "timed_out", `Workflow exceeded max duration of ${maxDuration}s (ran for ${Math.round(elapsed / 1000)}s)`);
+        // Archive to DLQ
+        await archiveToDlq(supabase, run.id, run.workflow_id, run.store_id, wf?.name);
+        count++;
+        log.warn({ runId: run.id, elapsedSec: Math.round(elapsed / 1000), maxDuration }, "workflow run timed out");
+    }
+    // Detect zombie runs: "running" with no active steps (all steps are terminal)
+    const { data: zombieRuns } = await supabase.from("workflow_runs")
+        .select("id, workflow_id, store_id, started_at, workflows!inner(name)")
+        .eq("status", "running")
+        .not("started_at", "is", null)
+        .limit(50);
+    if (zombieRuns?.length) {
+        for (const run of zombieRuns) {
+            const elapsed = now - new Date(run.started_at).getTime();
+            if (elapsed < 120_000)
+                continue; // Only check runs older than 2 min
+            const { data: activeSteps } = await supabase.from("workflow_step_runs")
+                .select("id").eq("run_id", run.id)
+                .in("status", ["pending", "running", "retrying", "waiting"]).limit(1);
+            if (!activeSteps?.length) {
+                // No active steps but run is still "running" — finalize it
+                await checkWorkflowCompletion(supabase, run.id, run.workflow_id);
+                count++;
+                log.warn({ runId: run.id, elapsedSec: Math.round(elapsed / 1000) }, "finalized zombie workflow run");
+            }
+        }
+    }
+    return count;
+}
+// ============================================================================
+// ORPHANED STEP CLEANUP — cancel step_runs whose parent run is terminal
+// ============================================================================
+let lastOrphanCleanupAt = 0;
+const ORPHAN_CLEANUP_INTERVAL_MS = 60_000; // Run at most once per minute
+export async function cleanupOrphanedSteps(supabase) {
+    if (Date.now() - lastOrphanCleanupAt < ORPHAN_CLEANUP_INTERVAL_MS)
+        return 0;
+    lastOrphanCleanupAt = Date.now();
+    // Find step_runs in non-terminal status whose parent run IS terminal.
+    // The !inner join + .in filter on the JOINED table ensures Postgres only returns
+    // rows where the run status is already terminal — no in-app filtering needed.
+    const { data: orphans } = await supabase.from("workflow_step_runs")
+        .select("id, run_id, step_key, status, workflow_runs!workflow_step_runs_run_id_fkey!inner(status)")
+        .in("status", ["pending", "retrying", "waiting"])
+        .in("workflow_runs.status", ["success", "failed", "cancelled", "timed_out"])
+        .limit(100);
+    if (!orphans?.length)
+        return 0;
+    let cleaned = 0;
+    for (const step of orphans) {
+        const runStatus = step.workflow_runs?.status;
+        await supabase.from("workflow_step_runs").update({
+            status: "cancelled",
+            error_message: `Orphaned: parent run already ${runStatus}`,
+            completed_at: new Date().toISOString(),
+        }).eq("id", step.id);
+        cleaned++;
+    }
+    if (cleaned > 0) {
+        log.info({ cleaned }, "cleaned up orphaned step runs");
+    }
+    return cleaned;
+}
+// ============================================================================
+// DLQ RETRY MECHANISM — retry transient DLQ failures
+// ============================================================================
+let lastDlqRetryAt = 0;
+const DLQ_RETRY_INTERVAL_MS = 60_000; // Run at most once per minute
+const DLQ_RETRY_BATCH_SIZE = 5;
+const DLQ_MAX_RETRY_ATTEMPTS = 3;
+export async function processDlqRetries(supabase) {
+    // Throttle: only run once per minute
+    if (Date.now() - lastDlqRetryAt < DLQ_RETRY_INTERVAL_MS)
+        return 0;
+    lastDlqRetryAt = Date.now();
+    // Fetch retryable DLQ entries: transient errors (timeout, network), not yet exhausted
+    const { data: entries } = await supabase.from("workflow_dlq")
+        .select("id, workflow_id, store_id, trigger_type, trigger_payload, error_message, retry_count, last_retry_at")
+        .or("error_message.ilike.%timed out%,error_message.ilike.%timeout%,error_message.ilike.%network%,error_message.ilike.%ECONNREFUSED%,error_message.ilike.%fetch failed%")
+        .eq("status", "pending")
+        .lt("retry_count", DLQ_MAX_RETRY_ATTEMPTS)
+        .order("created_at", { ascending: true })
+        .limit(DLQ_RETRY_BATCH_SIZE);
+    if (!entries?.length)
+        return 0;
+    let retried = 0;
+    const now = Date.now();
+    for (const entry of entries) {
+        try {
+            // Enforce per-entry exponential backoff: 2min, 4min, 8min
+            if (entry.last_retry_at) {
+                const backoffMs = Math.pow(2, entry.retry_count || 0) * 120_000;
+                const elapsed = now - new Date(entry.last_retry_at).getTime();
+                if (elapsed < backoffMs)
+                    continue; // Too soon for this entry
+            }
+            const { data: result, error } = await supabase.rpc("start_workflow_run", {
+                p_workflow_id: entry.workflow_id,
+                p_store_id: entry.store_id,
+                p_trigger_type: entry.trigger_type || "dlq_retry",
+                p_trigger_payload: { ...(entry.trigger_payload || {}), _dlq_retry: true, _dlq_entry_id: entry.id },
+                p_idempotency_key: `dlq_retry:${entry.id}:${(entry.retry_count || 0) + 1}`,
+            });
+            if (error || !result?.success) {
+                // Update retry count but keep in DLQ
+                await supabase.from("workflow_dlq").update({
+                    retry_count: (entry.retry_count || 0) + 1,
+                    last_error: error?.message || result?.error || "retry failed",
+                    last_retry_at: new Date().toISOString(),
+                }).eq("id", entry.id);
+                continue;
+            }
+            // Success — mark DLQ entry as retried
+            await supabase.from("workflow_dlq").update({
+                status: "retried",
+                retry_count: (entry.retry_count || 0) + 1,
+                retried_run_id: result.run_id,
+                last_retry_at: new Date().toISOString(),
+            }).eq("id", entry.id);
+            // Execute inline
+            if (result.run_id) {
+                const traceId = randomUUID();
+                await supabase.from("workflow_runs").update({ trace_id: traceId }).eq("id", result.run_id);
+                try {
+                    await executeInlineChain(supabase, result.run_id);
+                }
+                catch (err) {
+                    log.error({ runId: result.run_id, err: err.message }, "inline chain failed for DLQ retry");
+                }
+            }
+            retried++;
+            log.info({ dlqEntryId: entry.id, workflowId: entry.workflow_id, retryCount: (entry.retry_count || 0) + 1 }, "DLQ entry retried");
+        }
+        catch (err) {
+            log.warn({ dlqEntryId: entry.id, err: sanitizeError(err) }, "DLQ retry error");
+            await supabase.from("workflow_dlq").update({
+                retry_count: (entry.retry_count || 0) + 1,
+                last_retry_at: new Date().toISOString(),
+                last_error: sanitizeError(err),
+            }).eq("id", entry.id);
+        }
+    }
+    return retried;
+}
+// ============================================================================
+// EVENT TRIGGER PROCESSING — match inbound events to workflow subscriptions
+// ============================================================================
+export async function processEventTriggers(supabase) {
+    // Atomically claim a batch of pending events using FOR UPDATE SKIP LOCKED
+    // Falls back to SELECT+UPDATE if the RPC doesn't exist yet
+    let events = null;
+    const { data: claimed, error: claimErr } = await supabase.rpc("claim_pending_events", { batch_size: 20 });
+    if (!claimErr && claimed?.length) {
+        events = claimed;
+    }
+    else {
+        // Fallback: non-atomic claim (SELECT then UPDATE)
+        if (claimErr)
+            log.debug({ err: claimErr.message }, "claim_pending_events RPC unavailable, using fallback");
+        const { data: fallbackEvents } = await supabase.from("automation_events")
+            .select("id, store_id, event_type, event_payload, source")
+            .eq("status", "pending")
+            .order("created_at", { ascending: true })
+            .limit(20);
+        if (fallbackEvents?.length) {
+            const eventIds = fallbackEvents.map(e => e.id);
+            await supabase.from("automation_events")
+                .update({ status: "processing" })
+                .in("id", eventIds);
+            events = fallbackEvents;
+        }
+    }
+    if (!events?.length)
+        return 0;
+    // P2 FIX: Batch-load ALL active subscriptions once instead of per-event queries.
+    // Group by (store_id, event_type) for O(1) in-memory lookup per event.
+    const uniqueStoreIds = Array.from(new Set(events.map(e => e.store_id)));
+    const uniqueEventTypes = Array.from(new Set(events.map(e => e.event_type)));
+    const { data: allSubs } = await supabase.from("workflow_event_subscriptions")
+        .select("id, workflow_id, filter_expression, store_id, event_type")
+        .in("store_id", uniqueStoreIds)
+        .in("event_type", uniqueEventTypes)
+        .eq("is_active", true);
+    // Build lookup map: "store_id:event_type" -> subscriptions[]
+    const subsMap = new Map();
+    for (const sub of (allSubs || [])) {
+        const key = `${sub.store_id}:${sub.event_type}`;
+        if (!subsMap.has(key))
+            subsMap.set(key, []);
+        subsMap.get(key).push(sub);
+    }
+    let processed = 0;
+    for (const event of events) {
+        try {
+            // P2 FIX: In-memory lookup instead of per-event DB query
+            const subs = subsMap.get(`${event.store_id}:${event.event_type}`) || [];
+            if (!subs.length) {
+                // No subscribers — mark processed and move on
+                await supabase.from("automation_events")
+                    .update({ status: "processed", processed_at: new Date().toISOString() })
+                    .eq("id", event.id);
+                processed++;
+                continue;
+            }
+            for (const sub of subs) {
+                // Optional filter expression evaluation
+                if (sub.filter_expression) {
+                    try {
+                        const ctx = {
+                            trigger: event.event_payload || {},
+                            steps: {},
+                        };
+                        const pass = evaluateCondition(sub.filter_expression, ctx);
+                        if (!pass)
+                            continue; // Filter didn't match — skip this subscription
+                    }
+                    catch {
+                        // Filter eval error — skip rather than block
+                        continue;
+                    }
+                }
+                // Start a workflow run for each matching subscription
+                const idempotencyKey = `event:${event.id}:${sub.workflow_id}`;
+                const { data: result, error: startErr } = await supabase.rpc("start_workflow_run", {
+                    p_workflow_id: sub.workflow_id,
+                    p_store_id: event.store_id,
+                    p_trigger_type: "event",
+                    p_trigger_payload: {
+                        ...(event.event_payload || {}),
+                        _event_id: event.id,
+                        _event_type: event.event_type,
+                        _event_source: event.source,
+                    },
+                    p_idempotency_key: idempotencyKey,
+                });
+                if (startErr || !result?.success) {
+                    log.error({ workflowId: sub.workflow_id, eventId: event.id, err: startErr?.message || result?.error }, "failed to start event-triggered workflow");
+                    continue;
+                }
+                // Assign trace ID and run inline chain for immediate execution
+                if (result.run_id && !result.deduplicated) {
+                    const traceId = randomUUID();
+                    await supabase.from("workflow_runs").update({ trace_id: traceId }).eq("id", result.run_id);
+                    try {
+                        await executeInlineChain(supabase, result.run_id);
+                    }
+                    catch (err) {
+                        log.error({ runId: result.run_id, err: err.message }, "inline chain failed for event run");
+                    }
+                }
+            }
+            // Mark event as processed
+            await supabase.from("automation_events")
+                .update({ status: "processed", processed_at: new Date().toISOString() })
+                .eq("id", event.id);
+            processed++;
+        }
+        catch (err) {
+            // Mark event as failed
+            await supabase.from("automation_events")
+                .update({
+                status: "failed",
+                processed_at: new Date().toISOString(),
+                error_message: sanitizeError(err),
+            })
+                .eq("id", event.id);
+            log.error({ eventId: event.id, err: sanitizeError(err) }, "error processing event trigger");
+        }
+    }
+    return processed;
+}
+// ============================================================================
+// DEAD LETTER QUEUE — archive failed runs for investigation
+// ============================================================================
+async function archiveToDlq(supabase, runId, workflowId, storeId, workflowName) {
+    const { data: run } = await supabase.from("workflow_runs")
+        .select("error_message, error_step_key, trigger_type, trigger_payload, step_outputs, duration_ms")
+        .eq("id", runId).single();
+    if (!run)
+        return;
+    const { error } = await supabase.from("workflow_dlq").insert({
+        store_id: storeId,
+        run_id: runId,
+        workflow_id: workflowId,
+        workflow_name: workflowName || null,
+        error_message: run.error_message,
+        error_step_key: run.error_step_key,
+        trigger_type: run.trigger_type,
+        trigger_payload: run.trigger_payload || {},
+        step_outputs: run.step_outputs || {},
+        run_duration_ms: run.duration_ms,
+    });
+    if (error)
+        log.warn({ err: error.message, runId, workflowId }, "archiveToDlq insert failed");
+}
+/**
+ * Process-isolated JS code execution via persistent worker pool.
+ * Falls back to one-shot fork if pool is unavailable.
+ * A crash, OOM, or infinite loop in user code cannot take down the server.
+ */
+async function executeCodeStepIsolated(config, ctx) {
+    const code = config.code;
+    if (!code)
+        return { success: false, error: "No code in code step config" };
+    const language = config.language || "javascript";
+    if (language === "python") {
+        // P0 FIX: Python regex sandbox is trivially bypassable (string concatenation,
+        // __builtins__, globals() aliasing). Hard-fail until a real Python sandbox
+        // (e.g. Pyodide/WASM or containerized execution) is available.
+        return { success: false, error: "Python code execution is temporarily disabled — sandbox under hardening. Use JavaScript code steps instead." };
+    }
+    const timeoutMs = config.timeout_ms || CODE_TIMEOUT_MS;
+    try {
+        const result = await executeWithPool({
+            code,
+            context: { steps: ctx.steps, trigger: ctx.trigger, input: ctx.input },
+            timeoutMs,
+        });
+        // P3 FIX: Enforce same 100KB output cap as Python code steps
+        if (result.success && result.output) {
+            const outputStr = JSON.stringify(result.output);
+            if (outputStr.length > CODE_OUTPUT_MAX) {
+                result.output = { result: outputStr.slice(0, CODE_OUTPUT_MAX) + "\n[output truncated at 100KB]", truncated: true };
+            }
+        }
+        return result;
+    }
+    catch (err) {
+        // P0 FIX: Hard failure — no in-process fallback (SSRF risk via unrestricted fetch in sandbox)
+        log.error({ err: err.message }, "worker pool execution failed, no fallback");
+        return { success: false, error: "Code execution unavailable — worker pool failed" };
+    }
+}
+// P0 FIX: executeCodeStepInProcess removed — in-process fallback was a security risk
+// (unrestricted fetch in sandbox = SSRF). All code steps now require the worker pool.
+async function executePythonCode(_code, _ctx) {
+    return { success: false, error: "Python code execution is disabled — sandbox under hardening" };
+}
+// ============================================================================
+// CIRCUIT BREAKER (per user_tool)
+// ============================================================================
+async function checkToolCircuitBreaker(supabase, toolId) {
+    const { data } = await supabase.from("user_tools")
+        .select("circuit_breaker_state, circuit_breaker_tripped_at, circuit_breaker_cooldown_seconds")
+        .eq("id", toolId).single();
+    if (!data)
+        return { allowed: true };
+    if (data.circuit_breaker_state === "open") {
+        const trippedAt = new Date(data.circuit_breaker_tripped_at).getTime();
+        const cooldownMs = (data.circuit_breaker_cooldown_seconds || 300) * 1000;
+        if (Date.now() < trippedAt + cooldownMs) {
+            return { allowed: false, reason: `Circuit breaker open (cooldown until ${new Date(trippedAt + cooldownMs).toISOString()})` };
+        }
+        await supabase.from("user_tools").update({ circuit_breaker_state: "half_open" }).eq("id", toolId);
+    }
+    return { allowed: true };
+}
+// P1 FIX: Transient error patterns — these indicate network/infra issues, not broken tools.
+// Circuit breaker should only trip on persistent tool-level errors (auth, 404, bad config).
+const TRANSIENT_ERROR_PATTERNS = [
+    /timed?\s*out/i, /timeout/i, /ECONNREFUSED/i, /ECONNRESET/i, /ETIMEDOUT/i,
+    /fetch failed/i, /network/i, /socket hang up/i, /EPIPE/i, /EHOSTUNREACH/i,
+    /503 Service Unavailable/i, /502 Bad Gateway/i, /429 Too Many Requests/i,
+];
+function isTransientError(errorMessage) {
+    if (!errorMessage)
+        return false;
+    return TRANSIENT_ERROR_PATTERNS.some(pattern => pattern.test(errorMessage));
+}
+async function updateToolCircuitBreaker(supabase, toolId, success, errorMessage) {
+    if (success) {
+        await supabase.from("user_tools").update({ circuit_breaker_state: "closed", circuit_breaker_failures: 0 }).eq("id", toolId);
+        return;
+    }
+    // P1 FIX: Skip circuit breaker increment for transient errors (network, timeout, etc.)
+    if (isTransientError(errorMessage)) {
+        log.debug({ toolId, error: errorMessage }, "skipping circuit breaker increment for transient error");
+        return;
+    }
+    const { data } = await supabase.from("user_tools")
+        .select("circuit_breaker_failures, circuit_breaker_threshold").eq("id", toolId).single();
+    if (!data)
+        return;
+    const newFailures = (data.circuit_breaker_failures || 0) + 1;
+    if (newFailures >= (data.circuit_breaker_threshold || 5)) {
+        await supabase.from("user_tools").update({
+            circuit_breaker_state: "open", circuit_breaker_failures: newFailures,
+            circuit_breaker_tripped_at: new Date().toISOString(),
+        }).eq("id", toolId);
+        const { error: toolCbErr } = await supabase.from("audit_logs").insert({
+            action: "workflow.circuit_breaker.tripped", severity: "warning",
+            resource_type: "user_tool", resource_id: toolId, source: "workflow_engine",
+            details: { failures: newFailures, threshold: data.circuit_breaker_threshold },
+        });
+        if (toolCbErr)
+            log.warn({ err: toolCbErr.message, toolId }, "tool circuit breaker audit failed");
+    }
+    else {
+        await supabase.from("user_tools").update({ circuit_breaker_failures: newFailures }).eq("id", toolId);
+    }
+}
+async function handleWorkflowCircuitBreaker(supabase, workflowId, success, errorMessage) {
+    if (success) {
+        await supabase.from("workflows").update({ circuit_breaker_state: "closed", circuit_breaker_failures: 0 }).eq("id", workflowId);
+        return;
+    }
+    // P1 FIX: Skip circuit breaker increment for transient errors
+    if (isTransientError(errorMessage)) {
+        log.debug({ workflowId, error: errorMessage }, "skipping workflow circuit breaker increment for transient error");
+        return;
+    }
+    const { data } = await supabase.from("workflows")
+        .select("circuit_breaker_failures, circuit_breaker_threshold").eq("id", workflowId).single();
+    if (!data)
+        return;
+    const newFailures = (data.circuit_breaker_failures || 0) + 1;
+    if (newFailures >= (data.circuit_breaker_threshold || 5)) {
+        await supabase.from("workflows").update({
+            circuit_breaker_state: "open", circuit_breaker_failures: newFailures,
+            circuit_breaker_tripped_at: new Date().toISOString(),
+        }).eq("id", workflowId);
+        const { error: wfCbErr } = await supabase.from("audit_logs").insert({
+            action: "workflow.circuit_breaker.tripped", severity: "warning",
+            resource_type: "workflow", resource_id: workflowId, source: "workflow_engine",
+            details: { failures: newFailures, threshold: data.circuit_breaker_threshold },
+        });
+        if (wfCbErr)
+            log.warn({ err: wfCbErr.message, workflowId }, "workflow circuit breaker audit failed");
+    }
+    else {
+        await supabase.from("workflows").update({ circuit_breaker_failures: newFailures }).eq("id", workflowId);
+    }
+}
+// ============================================================================
+// CORE ENGINE
+// ============================================================================
+/**
+ * Reclaim steps stuck in "running" status longer than their timeout + buffer.
+ * This handles steps that crashed mid-execution (e.g., server restart, OOM).
+ * Steps are reset to "retrying" so the worker can re-execute them.
+ */
+async function reclaimStaleSteps(supabase) {
+    // P0 FIX: Use step.timeout_seconds * 2 (min 300s) as stale threshold to prevent
+    // double-execution. The old 60s buffer was too aggressive for long-running steps.
+    const MIN_STALE_THRESHOLD_MS = 300_000; // 5 min minimum before reclaim
+    const MAX_STALE_AGE_MS = 1_200_000; // Hard ceiling: 20 min = always stale
+    const { data: staleSteps } = await supabase.from("workflow_step_runs")
+        .select(`
+      id, step_key, run_id, attempt_count, max_attempts, started_at,
+      workflow_steps!inner(timeout_seconds)
+    `)
+        .eq("status", "running")
+        .not("started_at", "is", null)
+        .limit(50);
+    if (!staleSteps?.length)
+        return 0;
+    let reclaimed = 0;
+    const now = Date.now();
+    for (const step of staleSteps) {
+        const timeoutSec = step.workflow_steps?.timeout_seconds || 120;
+        // P0 FIX: threshold = max(300s, timeout * 2) — gives step enough time to complete
+        const staleThreshold = Math.min(Math.max(MIN_STALE_THRESHOLD_MS, timeoutSec * 2000), MAX_STALE_AGE_MS);
+        const elapsed = now - new Date(step.started_at).getTime();
+        if (elapsed < staleThreshold)
+            continue;
+        const exhaustedRetries = step.attempt_count >= step.max_attempts;
+        if (exhaustedRetries) {
+            // No retries left — mark as failed
+            await supabase.from("workflow_step_runs").update({
+                status: "failed",
+                error_message: `Step stale: running for ${Math.round(elapsed / 1000)}s with no response (retries exhausted)`,
+                completed_at: new Date().toISOString(),
+            }).eq("id", step.id);
+        }
+        else {
+            // Reset to retrying so worker picks it up again
+            await supabase.from("workflow_step_runs").update({
+                status: "retrying",
+                error_message: `Step stale: running for ${Math.round(elapsed / 1000)}s with no response (auto-reclaimed)`,
+                next_retry_at: new Date(now + 5000).toISOString(),
+            }).eq("id", step.id);
+        }
+        reclaimed++;
+        log.warn({ stepRunId: step.id, stepKey: step.step_key, runId: step.run_id, elapsedSec: Math.round(elapsed / 1000), exhaustedRetries }, "reclaimed stale step");
+    }
+    return reclaimed;
+}
+export async function processWorkflowSteps(supabase, batchSize = 10) {
+    // Reclaim stale steps first so they become available for claiming
+    const reclaimed = await reclaimStaleSteps(supabase).catch(e => {
+        log.warn({ err: e.message }, "reclaimStaleSteps failed");
+        return 0;
+    });
+    const { data: claimedRaw, error: claimErr } = await supabase.rpc("claim_pending_steps", {
+        batch_size: batchSize,
+    });
+    if (claimErr) {
+        log.error({ err: claimErr.message }, "workflow claim error");
+        return { processed: 0, errors: 1, reclaimed };
+    }
+    let claimed = Array.isArray(claimedRaw) ? claimedRaw : [];
+    if (claimed.length === 0)
+        return { processed: 0, errors: 0, reclaimed };
+    // Circuit breaker enforcement — skip steps from workflows with open breakers
+    const workflowIds = [...new Set(claimed.map(s => s.workflow_id))];
+    const { data: openBreakers } = await supabase.from("workflows")
+        .select("id")
+        .in("id", workflowIds)
+        .eq("circuit_breaker_state", "open");
+    if (openBreakers?.length) {
+        const blockedIds = new Set(openBreakers.map(w => w.id));
+        const blocked = claimed.filter(s => blockedIds.has(s.workflow_id));
+        claimed = claimed.filter(s => !blockedIds.has(s.workflow_id));
+        // Mark blocked steps as skipped and finalize affected runs
+        const affectedRuns = new Map();
+        for (const step of blocked) {
+            await supabase.from("workflow_step_runs").update({
+                status: "skipped",
+                error_message: "Workflow circuit breaker is open — step skipped",
+                completed_at: new Date().toISOString(),
+            }).eq("id", step.step_run_id);
+            affectedRuns.set(step.run_id, step.workflow_id);
+        }
+        // Check completion for affected runs — prevents zombie "running" state
+        for (const [runId, workflowId] of affectedRuns) {
+            await checkWorkflowCompletion(supabase, runId, workflowId).catch(e => log.warn({ runId, err: e.message }, "completion check after circuit breaker skip failed"));
+        }
+        if (blocked.length)
+            log.warn({ skippedSteps: blocked.length, blockedWorkflows: blockedIds.size }, "circuit breaker skipped steps");
+        if (claimed.length === 0)
+            return { processed: blocked.length, errors: 0, reclaimed };
+    }
+    log.info({ stepCount: claimed.length }, "processing workflow steps");
+    let errors = 0;
+    // Batch-fetch trace_ids for all runs in this batch
+    const runIds = [...new Set(claimed.map(s => s.run_id))];
+    const { data: runTraces } = await supabase.from("workflow_runs")
+        .select("id, trace_id").in("id", runIds);
+    const traceMap = new Map((runTraces || []).map(r => [r.id, r.trace_id]));
+    // Partition: email tool steps from for_each need sequential processing with delays
+    // to avoid overwhelming Resend's 2 req/s rate limit
+    const isForEachEmailStep = (s) => s.parent_step_run_id && s.step_type === "tool" &&
+        String(s.step_config?.tool_name || "").includes("email");
+    const emailForEachSteps = claimed.filter(isForEachEmailStep);
+    const otherSteps = claimed.filter(s => !isForEachEmailStep(s));
+    const processStep = async (step) => {
+        try {
+            await applyVersionOverrides(supabase, step);
+            await executeAndAdvance(supabase, step, traceMap.get(step.run_id));
+        }
+        catch (err) {
+            errors++;
+            const errMsg = sanitizeError(err);
+            log.error({ stepKey: step.step_key, runId: step.run_id, err: errMsg }, "step execution error");
+            await supabase.from("workflow_step_runs").update({
+                status: "failed", error_message: errMsg,
+                completed_at: new Date().toISOString(), duration_ms: 0,
+            }).eq("id", step.step_run_id);
+            // Surface error to clients (SSE broadcast + structured error_details on step run)
+            await surfaceStepError(supabase, step, errMsg);
+            // Check if this failure finalizes the run (prevents zombie runs from uncaught errors)
+            await checkWorkflowCompletion(supabase, step.run_id, step.workflow_id).catch(e => log.warn({ runId: step.run_id, err: e.message }, "completion check after step error failed"));
+        }
+    };
+    // Process non-email steps in parallel (existing behavior)
+    await Promise.all(otherSteps.map(processStep));
+    // Process email for_each children sequentially with 550ms throttle
+    for (let i = 0; i < emailForEachSteps.length; i++) {
+        if (i > 0)
+            await new Promise(r => setTimeout(r, 550));
+        await processStep(emailForEachSteps[i]);
+    }
+    return { processed: claimed.length, errors };
+}
+/**
+ * Check waiting steps: sub_workflow children completed, parallel/for_each children done.
+ * Called by the persistent worker loop alongside processWorkflowSteps.
+ */
+export async function processWaitingSteps(supabase) {
+    let resolved = 0;
+    // 0. Expire pending approvals (Phase 2)
+    try {
+        await supabase.rpc("expire_pending_approvals");
+    }
+    catch (err) {
+        // Non-fatal — RPC may not exist yet if migration not applied
+        log.warn({ err: sanitizeError(err) }, "expire_pending_approvals error");
+    }
+    // 1. Sub-workflow steps waiting for child runs to complete
+    const { data: subWfSteps } = await supabase
+        .from("workflow_step_runs")
+        .select("id, run_id, step_key, child_run_id, step_type")
+        .eq("status", "waiting")
+        .eq("step_type", "sub_workflow")
+        .not("child_run_id", "is", null)
+        .limit(50);
+    if (subWfSteps?.length) {
+        const childRunIds = subWfSteps.map(s => s.child_run_id).filter(Boolean);
+        const { data: childRuns } = await supabase
+            .from("workflow_runs")
+            .select("id, status, step_outputs, error_message")
+            .in("id", childRunIds)
+            .in("status", ["success", "failed"]);
+        if (childRuns?.length) {
+            const runMap = new Map(childRuns.map(r => [r.id, r]));
+            for (const step of subWfSteps) {
+                const childRun = runMap.get(step.child_run_id);
+                if (!childRun)
+                    continue;
+                const success = childRun.status === "success";
+                await supabase.from("workflow_step_runs").update({
+                    status: success ? "success" : "failed",
+                    output: childRun.step_outputs,
+                    error_message: success ? null : childRun.error_message,
+                    completed_at: new Date().toISOString(),
+                }).eq("id", step.id);
+                // Accumulate output + advance
+                await accumulateAndAdvance(supabase, step.id, step.run_id, step.step_key, success, childRun.step_outputs, childRun.error_message);
+                resolved++;
+            }
+        }
+    }
+    // 2. P1 FIX: Use aggregate RPC to eliminate N+1 queries (was 4 queries per parent)
+    const { data: aggregatedParents, error: aggErr } = await supabase.rpc("get_waiting_parents_with_children");
+    if (aggErr) {
+        // Fallback to old N+1 pattern if RPC doesn't exist yet
+        log.debug({ err: aggErr.message }, "get_waiting_parents_with_children RPC unavailable, using fallback");
+        const { data: waitingParents } = await supabase
+            .from("workflow_step_runs")
+            .select("id, run_id, step_key, step_type, output")
+            .eq("status", "waiting")
+            .in("step_type", ["parallel", "for_each"])
+            .limit(50);
+        if (waitingParents?.length) {
+            for (const parent of waitingParents) {
+                const { count: totalChildren } = await supabase
+                    .from("workflow_step_runs")
+                    .select("id", { count: "exact", head: true })
+                    .eq("parent_step_run_id", parent.id);
+                const { count: doneChildren } = await supabase
+                    .from("workflow_step_runs")
+                    .select("id", { count: "exact", head: true })
+                    .eq("parent_step_run_id", parent.id)
+                    .in("status", ["success", "failed", "skipped", "cancelled"]);
+                if (totalChildren && doneChildren && doneChildren >= totalChildren) {
+                    const { data: childOutputs } = await supabase
+                        .from("workflow_step_runs")
+                        .select("step_key, output, status, error_message")
+                        .eq("parent_step_run_id", parent.id)
+                        .order("created_at", { ascending: true });
+                    const outputs = (childOutputs || []).map(c => c.output);
+                    const failedKids = (childOutputs || []).filter(c => c.status === "failed");
+                    const allSuccess = failedKids.length === 0;
+                    await supabase.from("workflow_step_runs").update({
+                        status: allSuccess ? "success" : "failed",
+                        output: { children: outputs, total: totalChildren, failed: failedKids.length },
+                        error_message: allSuccess ? null : failedKids[0]?.error_message,
+                        completed_at: new Date().toISOString(),
+                    }).eq("id", parent.id);
+                    await accumulateAndAdvance(supabase, parent.id, parent.run_id, parent.step_key, allSuccess, { children: outputs }, allSuccess ? null : failedKids[0]?.error_message);
+                    resolved++;
+                }
+            }
+        }
+    }
+    else if (aggregatedParents?.length) {
+        for (const parent of aggregatedParents) {
+            if (parent.total_children > 0 && parent.done_children >= parent.total_children) {
+                const childOutputsArr = (parent.child_outputs || []);
+                const outputs = childOutputsArr.map((c) => c.output);
+                const allSuccess = parent.failed_children === 0;
+                const firstError = childOutputsArr.find((c) => c.status === "failed")?.error_message;
+                await supabase.from("workflow_step_runs").update({
+                    status: allSuccess ? "success" : "failed",
+                    output: { children: outputs, total: parent.total_children, failed: parent.failed_children },
+                    error_message: allSuccess ? null : firstError,
+                    completed_at: new Date().toISOString(),
+                }).eq("id", parent.parent_id);
+                await accumulateAndAdvance(supabase, parent.parent_id, parent.parent_run_id, parent.parent_step_key, allSuccess, { children: outputs }, allSuccess ? null : firstError);
+                resolved++;
+            }
+        }
+    }
+    return resolved;
+}
+/** Helper: after a waiting step resolves, accumulate output and advance the workflow. */
+async function accumulateAndAdvance(supabase, stepRunId, runId, stepKey, success, output, errorMessage) {
+    // Load run to get current step_outputs + workflow_id + on_success/on_failure
+    const { data: run } = await supabase.from("workflow_runs")
+        .select("workflow_id, step_outputs, store_id").eq("id", runId).single();
+    if (!run)
+        return;
+    const { data: stepDef } = await supabase.from("workflow_step_runs")
+        .select("step_id, step_key").eq("id", stepRunId).single();
+    if (!stepDef)
+        return;
+    // Phase 4: Try versioned step def first, fall back to live table
+    let wsDef = null;
+    const versionedSteps = await loadVersionedSteps(supabase, runId);
+    if (versionedSteps) {
+        const vStep = versionedSteps.find((s) => s.step_key === stepKey);
+        if (vStep)
+            wsDef = { on_success: vStep.on_success, on_failure: vStep.on_failure, max_retries: vStep.max_retries };
+    }
+    if (!wsDef) {
+        const { data } = await supabase.from("workflow_steps")
+            .select("on_success, on_failure, max_retries").eq("id", stepDef.step_id).single();
+        wsDef = data;
+    }
+    // P1 FIX: Atomic step output accumulation — use jsonb_set instead of read-modify-write
+    // This prevents lost updates when multiple steps complete concurrently
+    const stepOutput = { output, status: success ? "success" : "failed" };
+    await supabase.rpc("accumulate_step_output", {
+        p_run_id: runId,
+        p_step_key: stepKey,
+        p_step_output: stepOutput,
+    }).then(({ error: rpcErr }) => {
+        // Fallback to direct update if RPC doesn't exist yet (migration pending)
+        if (rpcErr) {
+            const newOutputs = { ...(run.step_outputs || {}), [stepKey]: stepOutput };
+            return supabase.from("workflow_runs").update({ step_outputs: newOutputs }).eq("id", runId);
+        }
+    });
+    if (success) {
+        if (wsDef?.on_success) {
+            await createNextStepRunByKey(supabase, runId, run.workflow_id, wsDef.on_success);
+        }
+        else {
+            await checkWorkflowCompletion(supabase, runId, run.workflow_id);
+        }
+    }
+    else {
+        if (wsDef?.on_failure) {
+            await createNextStepRunByKey(supabase, runId, run.workflow_id, wsDef.on_failure);
+        }
+        else {
+            await completeWorkflowRun(supabase, runId, run.workflow_id, run.store_id, "failed", errorMessage, stepKey);
+        }
+    }
+}
+export async function executeAndAdvance(supabase, step, traceId) {
+    const startTime = Date.now();
+    // Phase 3.3: OTEL span for step execution
+    const span = startSpan("workflow.step.execute", {
+        "workflow.run_id": step.run_id,
+        "workflow.step_key": step.step_key,
+        "workflow.step_type": step.step_type,
+        "workflow.attempt": step.attempt_count,
+        ...(traceId ? { "workflow.trace_id": traceId } : {}),
+    });
+    // Event journal — step started
+    await logWorkflowEvent(supabase, step.run_id, "step_started", {
+        step_key: step.step_key, step_type: step.step_type, attempt: step.attempt_count,
+    }, step.step_run_id);
+    // Step result caching — skip successful steps on retry (idempotent re-execution)
+    if (step.attempt_count > 1) {
+        const { data: prevRun } = await supabase.from("workflow_step_runs")
+            .select("status, output").eq("run_id", step.run_id).eq("step_key", step.step_key)
+            .eq("status", "success").neq("id", step.step_run_id).limit(1);
+        if (prevRun?.length) {
+            log.info({ stepKey: step.step_key, runId: step.run_id }, "step already succeeded, using cached result");
+            await supabase.from("workflow_step_runs").update({
+                status: "success", output: prevRun[0].output,
+                completed_at: new Date().toISOString(), duration_ms: Date.now() - startTime,
+            }).eq("id", step.step_run_id);
+            await logWorkflowEvent(supabase, step.run_id, "step_cached", { step_key: step.step_key }, step.step_run_id);
+            const nextStepKey = step.on_success;
+            if (!nextStepKey) {
+                await checkWorkflowCompletion(supabase, step.run_id, step.workflow_id);
+            }
+            else {
+                await createNextStepRunByKey(supabase, step.run_id, step.workflow_id, nextStepKey);
+            }
+            return;
+        }
+    }
+    // Flow control — check concurrency/rate limits before execution
+    const flowCheck = await checkFlowControl(supabase, step);
+    if (!flowCheck.allowed) {
+        // Requeue step with short delay for flow control backoff
+        await supabase.from("workflow_step_runs").update({
+            status: "retrying",
+            next_retry_at: new Date(Date.now() + 2000).toISOString(), // retry in 2s
+            output: { flow_control: flowCheck.reason },
+        }).eq("id", step.step_run_id);
+        await logWorkflowEvent(supabase, step.run_id, "step_throttled", { reason: flowCheck.reason }, step.step_run_id);
+        return;
+    }
+    // Build template context
+    const ctx = {
+        steps: {},
+        trigger: step.trigger_payload || {},
+        input: step.input || undefined,
+        workflow: {
+            id: step.workflow_id,
+            store_id: step.store_id,
+        },
+        run: {
+            id: step.run_id,
+            workflow_id: step.workflow_id,
+            store_id: step.store_id,
+        },
+    };
+    if (step.step_outputs && typeof step.step_outputs === "object") {
+        for (const [key, val] of Object.entries(step.step_outputs)) {
+            if (val && typeof val === "object") {
+                ctx.steps[key] = val;
+            }
+        }
+    }
+    let result;
+    // Enforce step-level timeout from step column (default 30s)
+    const stepTimeoutSec = step.timeout_seconds || step.step_config.timeout_seconds || 30;
+    const stepTimeoutMs = stepTimeoutSec * 1000;
+    let stepTimer;
+    const stepTimeoutPromise = new Promise((_, reject) => {
+        stepTimer = setTimeout(() => reject(new Error(`Step timed out after ${stepTimeoutSec}s`)), stepTimeoutMs);
+    });
+    try {
+        switch (step.step_type) {
+            case "tool":
+                result = await Promise.race([executeToolStep(supabase, step.step_config, ctx, step.store_id, traceId), stepTimeoutPromise]);
+                break;
+            case "condition":
+                result = executeConditionStep(step.step_config, ctx);
+                break;
+            case "transform":
+                result = executeTransformStep(step.step_config, ctx);
+                break;
+            case "delay": {
+                // First attempt: set the delay. Second attempt (after delay): success.
+                if (step.attempt_count <= 1) {
+                    const delaySec = step.step_config.seconds || 60;
+                    await supabase.from("workflow_step_runs").update({
+                        status: "retrying",
+                        output: { delay_seconds: delaySec, resume_at: new Date(Date.now() + delaySec * 1000).toISOString() },
+                        next_retry_at: new Date(Date.now() + delaySec * 1000).toISOString(),
+                    }).eq("id", step.step_run_id);
+                    clearTimeout(stepTimer);
+                    return; // Worker picks it up after delay
+                }
+                result = { success: true, output: { delayed: true, seconds: step.step_config.seconds } };
+                break;
+            }
+            case "agent":
+                result = await Promise.race([executeAgentStep(step.step_config, ctx, step.store_id, supabase, step, traceId), stepTimeoutPromise]);
+                break;
+            case "sub_workflow": {
+                const childWfId = resolveTemplate((step.step_config.workflow_id || ""), ctx);
+                if (!childWfId) {
+                    result = { success: false, error: "No workflow_id in sub_workflow config" };
+                    break;
+                }
+                const payloadTemplate = (step.step_config.trigger_payload_template || step.step_config.trigger_payload || {});
+                const payload = resolveTemplate(payloadTemplate, ctx);
+                const { data: startResult } = await supabase.rpc("start_workflow_run", {
+                    p_workflow_id: childWfId,
+                    p_store_id: step.store_id,
+                    p_trigger_type: "sub_workflow",
+                    p_trigger_payload: payload,
+                });
+                if (!startResult?.success) {
+                    result = { success: false, error: startResult?.error || "Failed to start sub-workflow" };
+                    break;
+                }
+                // Set to waiting — processWaitingSteps will resolve when child completes
+                await supabase.from("workflow_step_runs").update({
+                    status: "waiting",
+                    child_run_id: startResult.run_id,
+                    output: { child_run_id: startResult.run_id, child_workflow_id: childWfId },
+                }).eq("id", step.step_run_id);
+                clearTimeout(stepTimer);
+                return;
+            }
+            case "parallel": {
+                const stepKeys = (step.step_config.step_keys || step.step_config.child_steps || []);
+                if (stepKeys.length === 0) {
+                    result = { success: true, output: { parallel: true, steps: [] } };
+                    break;
+                }
+                if (stepKeys.length > MAX_PARALLEL_CHILDREN) {
+                    result = { success: false, error: `Parallel step has ${stepKeys.length} children, exceeding limit of ${MAX_PARALLEL_CHILDREN}` };
+                    break;
+                }
+                const { data: steps } = await supabase.from("workflow_steps")
+                    .select("id, step_key, step_type, max_retries")
+                    .eq("workflow_id", step.workflow_id).in("step_key", stepKeys);
+                if (steps?.length) {
+                    await supabase.from("workflow_step_runs").insert(steps.map(s => ({
+                        run_id: step.run_id, step_id: s.id, step_key: s.step_key,
+                        step_type: s.step_type, status: "pending",
+                        max_attempts: s.max_retries ?? 3, parent_step_run_id: step.step_run_id,
+                    })));
+                }
+                await supabase.from("workflow_step_runs").update({
+                    status: "waiting", output: { waiting_for: stepKeys },
+                }).eq("id", step.step_run_id);
+                clearTimeout(stepTimer);
+                return; // processWaitingSteps resolves when all children complete
+            }
+            case "for_each": {
+                const itemsExpr = step.step_config.items;
+                const targetStepKey = step.step_config.step_key;
+                if (!itemsExpr || !targetStepKey) {
+                    result = { success: false, error: "for_each requires items and step_key in config" };
+                    break;
+                }
+                const items = resolveTemplate(itemsExpr, ctx);
+                if (!Array.isArray(items)) {
+                    result = { success: false, error: `for_each items resolved to ${typeof items}, expected array` };
+                    break;
+                }
+                if (items.length === 0) {
+                    result = { success: true, output: { children: [], total: 0 } };
+                    break;
+                }
+                // P2 FIX: Enforce max items limit to prevent runaway step creation
+                const maxItems = step.step_config.max_items || MAX_FOR_EACH_ITEMS;
+                if (items.length > maxItems) {
+                    result = { success: false, error: `for_each exceeded maximum of ${maxItems} items (got ${items.length}). Increase limit in step config or paginate.` };
+                    break;
+                }
+                // Look up target step definition
+                const { data: targetStep } = await supabase.from("workflow_steps")
+                    .select("id, step_key, step_type, max_retries")
+                    .eq("workflow_id", step.workflow_id).eq("step_key", targetStepKey).single();
+                if (!targetStep) {
+                    result = { success: false, error: `for_each target step '${targetStepKey}' not found` };
+                    break;
+                }
+                // Create a step_run per item with the item as input
+                await supabase.from("workflow_step_runs").insert(items.map((item, idx) => ({
+                    run_id: step.run_id, step_id: targetStep.id,
+                    step_key: `${targetStepKey}[${idx}]`, step_type: targetStep.step_type,
+                    status: "pending", max_attempts: targetStep.max_retries ?? 3,
+                    parent_step_run_id: step.step_run_id, input: item,
+                })));
+                await supabase.from("workflow_step_runs").update({
+                    status: "waiting", output: { waiting_for_count: items.length, target_step: targetStepKey },
+                }).eq("id", step.step_run_id);
+                clearTimeout(stepTimer);
+                return;
+            }
+            case "code": {
+                result = await Promise.race([executeCodeStepIsolated(step.step_config, ctx), stepTimeoutPromise]);
+                break;
+            }
+            case "webhook_out":
+                result = await Promise.race([executeWebhookOutStep(step.step_config, ctx), stepTimeoutPromise]);
+                break;
+            case "noop":
+                result = executeNoopStep();
+                break;
+            case "llm_batch":
+                result = await Promise.race([executeLlmBatchStep(step.step_config, ctx), stepTimeoutPromise]);
+                break;
+            case "approval": {
+                const approvalResult = await executeApprovalStep(supabase, step, ctx);
+                if (approvalResult === "waiting") {
+                    clearTimeout(stepTimer);
+                    return;
+                }
+                result = approvalResult;
+                break;
+            }
+            // Custom step — POSTs workflow context to a user-defined URL and uses the response
+            case "custom": {
+                const customUrl = resolveTemplate((step.step_config.url || step.step_config.endpoint), ctx);
+                if (!customUrl) {
+                    result = { success: false, error: "Custom step requires url in config" };
+                    break;
+                }
+                // P0 FIX: Use async validateUrl (DNS resolve-then-check) instead of sync isBlockedUrl
+                const customSsrfError = await validateUrl(customUrl);
+                if (customSsrfError) {
+                    result = { success: false, error: `Custom step blocked: ${customSsrfError}` };
+                    break;
+                }
+                try {
+                    const customHeaders = { "Content-Type": "application/json" };
+                    if (step.step_config.headers && typeof step.step_config.headers === "object") {
+                        for (const [k, v] of Object.entries(step.step_config.headers)) {
+                            customHeaders[k] = resolveTemplate(v, ctx);
+                        }
+                    }
+                    const customBody = JSON.stringify({
+                        step_key: step.step_key,
+                        run_id: step.run_id,
+                        workflow_id: step.workflow_id,
+                        input: step.input,
+                        step_outputs: step.step_outputs,
+                        trigger_payload: step.trigger_payload,
+                        config: step.step_config.payload_config || {},
+                    });
+                    const ctrl = new AbortController();
+                    const timer = setTimeout(() => ctrl.abort(), 30_000);
+                    const resp = await fetch(customUrl, { method: "POST", headers: customHeaders, body: customBody, signal: ctrl.signal });
+                    clearTimeout(timer);
+                    const respData = resp.headers.get("content-type")?.includes("json")
+                        ? await resp.json() : await resp.text();
+                    if (!resp.ok) {
+                        result = { success: false, error: `Custom step HTTP ${resp.status}: ${String(respData).substring(0, 500)}` };
+                    }
+                    else {
+                        // Support branch routing from custom step response
+                        const branch = typeof respData === "object" && respData?.branch ? respData.branch : undefined;
+                        result = { success: true, output: respData, branch };
+                    }
+                }
+                catch (err) {
+                    result = { success: false, error: err.name === "AbortError" ? "Custom step timed out" : sanitizeError(err) };
+                }
+                break;
+            }
+            // Waitpoint — generalized wait-for-external-signal (subsumes approval, webhook callback, cross-workflow)
+            case "waitpoint": {
+                // Second pass — resumed with completion data
+                if (step.input && typeof step.input === "object" && step.input.waitpoint_completed) {
+                    result = { success: true, output: step.input.waitpoint_data || {} };
+                    break;
+                }
+                // First pass — create waitpoint token and pause
+                const waitpointToken = randomUUID();
+                const waitpointTimeout = step.step_config.timeout_seconds || 86400;
+                const waitpointExpires = new Date(Date.now() + waitpointTimeout * 1000).toISOString();
+                await supabase.from("waitpoint_tokens").insert({
+                    token: waitpointToken,
+                    run_id: step.run_id,
+                    step_run_id: step.step_run_id,
+                    store_id: step.store_id,
+                    expires_at: waitpointExpires,
+                    label: step.step_config.label || step.step_key,
+                });
+                await supabase.from("workflow_step_runs").update({
+                    status: "waiting",
+                    output: { waiting_for: "waitpoint", token: waitpointToken, expires_at: waitpointExpires },
+                }).eq("id", step.step_run_id);
+                await logWorkflowEvent(supabase, step.run_id, "waitpoint_created", { token: waitpointToken }, step.step_run_id);
+                clearTimeout(stepTimer);
+                return;
+            }
+            default:
+                result = { success: false, error: `Unknown step type: ${step.step_type}` };
+        }
+    }
+    catch (timeoutErr) {
+        result = { success: false, error: timeoutErr.message || `Step timed out after ${stepTimeoutSec}s` };
+    }
+    finally {
+        clearTimeout(stepTimer);
+    }
+    const durationMs = Date.now() - startTime;
+    // Phase 3.3: End OTEL span with result attributes
+    if (result.success) {
+        span.end({ "workflow.duration_ms": durationMs, "workflow.status": "success" });
+    }
+    else {
+        span.setError(result.error || "step failed");
+        span.end({ "workflow.duration_ms": durationMs, "workflow.status": "failed" });
+    }
+    // Event journal — step completed
+    await logWorkflowEvent(supabase, step.run_id, result.success ? "step_completed" : "step_failed", {
+        step_key: step.step_key, duration_ms: durationMs,
+        ...(result.error ? { error: result.error } : {}),
+        ...(result.branch ? { branch: result.branch } : {}),
+    }, step.step_run_id);
+    // Persist step result
+    await supabase.from("workflow_step_runs").update({
+        status: result.success ? "success" : "failed",
+        output: result.output || null,
+        error_message: result.error || null,
+        completed_at: new Date().toISOString(),
+        duration_ms: durationMs,
+    }).eq("id", step.step_run_id);
+    // P4 FIX: Atomically merge step output using jsonb_set to prevent race conditions
+    // Two concurrent steps can no longer overwrite each other's outputs
+    const stepOutput = { output: result.output, status: result.success ? "success" : "failed", duration_ms: durationMs };
+    const { error: rpcError } = await supabase.rpc("accumulate_step_output", {
+        p_run_id: step.run_id,
+        p_step_key: step.step_key,
+        p_step_output: stepOutput,
+    });
+    if (rpcError) {
+        // Retry once — transient connection errors are common under load
+        log.warn({ err: rpcError.message, runId: step.run_id, stepKey: step.step_key }, "accumulate_step_output RPC error, retrying once");
+        const { error: retryError } = await supabase.rpc("accumulate_step_output", {
+            p_run_id: step.run_id,
+            p_step_key: step.step_key,
+            p_step_output: stepOutput,
+        });
+        if (retryError) {
+            // Final fallback: advisory-lock-guarded RPC prevents parallel step race conditions
+            log.warn({ err: retryError.message, runId: step.run_id, stepKey: step.step_key }, "accumulate_step_output retry failed, using locked fallback");
+            const { error: lockError } = await supabase.rpc("accumulate_step_output_locked", {
+                p_run_id: step.run_id,
+                p_step_key: step.step_key,
+                p_step_output: stepOutput,
+            });
+            if (lockError) {
+                // P0 FIX: Throw so the step is marked as failed, not falsely as success
+                throw new Error(`Failed to accumulate step output after all fallbacks: ${lockError.message}`);
+            }
+        }
+    }
+    // Checkpoint — snapshot state after each step for replay/debugging
+    if (result.success) {
+        const { error: cpError } = await supabase.from("workflow_checkpoints").insert({
+            run_id: step.run_id, step_run_id: step.step_run_id, step_key: step.step_key,
+            step_outputs: { ...(step.step_outputs || {}), [step.step_key]: { output: result.output, status: "success", duration_ms: durationMs } },
+            trigger_payload: step.trigger_payload,
+            sequence_number: Object.keys(step.step_outputs || {}).length + 1,
+        });
+        if (cpError)
+            log.warn({ err: cpError.message, runId: step.run_id, stepKey: step.step_key }, "checkpoint insert failed");
+    }
+    // Audit
+    const { error: stepAuditErr } = await supabase.from("audit_logs").insert({
+        action: `workflow.step.${result.success ? "completed" : "failed"}`,
+        severity: result.success ? "info" : "error",
+        store_id: step.store_id, resource_type: "workflow_step_run",
+        resource_id: step.step_run_id, source: "workflow_engine", duration_ms: durationMs,
+        request_id: traceId || null,
+        details: { workflow_id: step.workflow_id, run_id: step.run_id, step_key: step.step_key, step_type: step.step_type, attempt: step.attempt_count },
+        error_message: result.error || null,
+    });
+    if (stepAuditErr)
+        log.warn({ err: stepAuditErr.message, runId: step.run_id }, "step audit insert failed");
+    // Surface step errors to clients via SSE broadcast + structured error_details
+    if (!result.success && result.error) {
+        await surfaceStepError(supabase, step, result.error);
+    }
+    // Child steps (parallel/for_each) — just save result, parent handles advancement
+    if (step.parent_step_run_id) {
+        // If child failed and has retries left, retry it (respects retry_policy)
+        if (!result.success && step.attempt_count < step.max_attempts) {
+            const retryPolicy = step.step_config.retry_policy;
+            const backoffType = retryPolicy?.backoff_type || "exponential";
+            const baseDelay = retryPolicy?.backoff_base_seconds || step.retry_delay_seconds || 10;
+            const maxBackoff = retryPolicy?.max_backoff_seconds || 300;
+            let backoffDelay;
+            switch (backoffType) {
+                case "fixed":
+                    backoffDelay = baseDelay;
+                    break;
+                case "linear":
+                    backoffDelay = baseDelay * step.attempt_count;
+                    break;
+                default:
+                    backoffDelay = baseDelay * Math.pow(2, step.attempt_count - 1);
+                    break;
+            }
+            backoffDelay = Math.min(backoffDelay, maxBackoff);
+            // P3 FIX: Add jitter (50%-100% of computed delay) to prevent thundering herd
+            backoffDelay *= (0.5 + Math.random() * 0.5);
+            await supabase.from("workflow_step_runs").update({
+                status: "retrying",
+                next_retry_at: new Date(Date.now() + backoffDelay * 1000).toISOString(),
+            }).eq("id", step.step_run_id);
+        }
+        return; // Parent's processWaitingSteps handles advancement
+    }
+    // Handle failure — configurable retry policy
+    if (!result.success) {
+        if (step.attempt_count < step.max_attempts) {
+            // Check retry_on filter — only retry if error matches pattern (if configured)
+            const retryPolicy = step.step_config.retry_policy;
+            const retryOn = retryPolicy?.retry_on;
+            const shouldRetry = !retryOn?.length || retryOn.some(pattern => result.error?.includes(pattern));
+            if (shouldRetry) {
+                const backoffType = retryPolicy?.backoff_type || "exponential";
+                const baseDelay = retryPolicy?.backoff_base_seconds || step.retry_delay_seconds || 10;
+                const maxBackoff = retryPolicy?.max_backoff_seconds || 300; // 5 min cap
+                let backoffDelay;
+                switch (backoffType) {
+                    case "fixed":
+                        backoffDelay = baseDelay;
+                        break;
+                    case "linear":
+                        backoffDelay = baseDelay * step.attempt_count;
+                        break;
+                    default:
+                        backoffDelay = baseDelay * Math.pow(2, step.attempt_count - 1);
+                        break; // exponential
+                }
+                backoffDelay = Math.min(backoffDelay, maxBackoff);
+                // P3 FIX: Add jitter (50%-100% of computed delay) to prevent thundering herd
+                backoffDelay *= (0.5 + Math.random() * 0.5);
+                await supabase.from("workflow_step_runs").update({
+                    status: "retrying",
+                    next_retry_at: new Date(Date.now() + backoffDelay * 1000).toISOString(),
+                }).eq("id", step.step_run_id);
+                await logWorkflowEvent(supabase, step.run_id, "step_retrying", {
+                    step_key: step.step_key, attempt: step.attempt_count, backoff_type: backoffType, delay_seconds: backoffDelay,
+                }, step.step_run_id);
+                return;
+            }
+            // retry_on filter didn't match — fall through to failure handling
+        }
+        if (step.on_failure) {
+            await createNextStepRunByKey(supabase, step.run_id, step.workflow_id, step.on_failure);
+        }
+        else {
+            await completeWorkflowRun(supabase, step.run_id, step.workflow_id, step.store_id, "failed", result.error, step.step_key);
+        }
+        return;
+    }
+    // Advance — condition steps use branch, otherwise on_success
+    const nextStepKey = result.branch || step.on_success;
+    if (!nextStepKey) {
+        await checkWorkflowCompletion(supabase, step.run_id, step.workflow_id);
+        return;
+    }
+    await createNextStepRunByKey(supabase, step.run_id, step.workflow_id, nextStepKey);
+}
+// ============================================================================
+// WORKFLOW ADVANCEMENT HELPERS
+// ============================================================================
+// P1 FIX: Per-run in-memory cache for versioned steps (30s TTL)
+// Prevents 4x redundant DB calls per step execution
+// P0 FIX: Capped at 500 entries with LRU eviction to prevent unbounded memory growth
+const VERSION_CACHE_MAX_SIZE = 500;
+const versionedStepsCache = new Map();
+const VERSION_CACHE_TTL_MS = 30_000;
+export function clearStepCache() {
+    versionedStepsCache.clear();
+}
+/** Insert into cache with LRU eviction when size exceeds limit */
+function versionedStepsCacheSet(key, value) {
+    // If key already exists, delete first so re-insertion moves it to end (most recent)
+    if (versionedStepsCache.has(key)) {
+        versionedStepsCache.delete(key);
+    }
+    versionedStepsCache.set(key, value);
+    // Evict oldest entries (Map iteration order = insertion order) if over limit
+    if (versionedStepsCache.size > VERSION_CACHE_MAX_SIZE) {
+        const excess = versionedStepsCache.size - VERSION_CACHE_MAX_SIZE;
+        let removed = 0;
+        for (const k of versionedStepsCache.keys()) {
+            if (removed >= excess)
+                break;
+            versionedStepsCache.delete(k);
+            removed++;
+        }
+    }
+}
+// Periodic cleanup to prevent memory leaks
+setInterval(() => {
+    const now = Date.now();
+    for (const [key, entry] of versionedStepsCache) {
+        if (now > entry.expiresAt)
+            versionedStepsCache.delete(key);
+    }
+}, 60_000);
+/**
+ * Load the versioned steps array for a run. Returns null if no version.
+ * Uses per-run in-memory cache with 30s TTL.
+ */
+async function loadVersionedSteps(supabase, runId) {
+    const cached = versionedStepsCache.get(runId);
+    if (cached && Date.now() < cached.expiresAt)
+        return cached.data;
+    const { data: run } = await supabase.from("workflow_runs")
+        .select("version_id").eq("id", runId).single();
+    if (!run?.version_id) {
+        versionedStepsCacheSet(runId, { data: null, expiresAt: Date.now() + VERSION_CACHE_TTL_MS });
+        return null;
+    }
+    const { data: version } = await supabase.from("workflow_versions")
+        .select("steps").eq("id", run.version_id).single();
+    const result = (version?.steps && Array.isArray(version.steps)) ? version.steps : null;
+    versionedStepsCacheSet(runId, { data: result, expiresAt: Date.now() + VERSION_CACHE_TTL_MS });
+    return result;
+}
+/**
+ * Apply versioned overrides to a claimed step. If the run has a version_id,
+ * replaces step_config, on_success, on_failure with values from the snapshot.
+ */
+async function applyVersionOverrides(supabase, step) {
+    const versionedSteps = await loadVersionedSteps(supabase, step.run_id);
+    if (!versionedSteps)
+        return;
+    const vStep = versionedSteps.find((s) => s.step_key === step.step_key);
+    if (vStep) {
+        step.step_config = vStep.step_config || step.step_config;
+        step.on_success = vStep.on_success ?? step.on_success;
+        step.on_failure = vStep.on_failure ?? step.on_failure;
+    }
+}
+/**
+ * Resolve a step definition by key. If the run has a version_id, load from
+ * the versioned snapshot. Otherwise, load from the live workflow_steps table.
+ */
+async function resolveStepDef(supabase, runId, workflowId, stepKey) {
+    const versionedSteps = await loadVersionedSteps(supabase, runId);
+    if (versionedSteps) {
+        const step = versionedSteps.find((s) => s.step_key === stepKey);
+        if (step) {
+            return { id: step.id, step_key: step.step_key, step_type: step.step_type, max_retries: step.max_retries ?? 3 };
+        }
+        return null;
+    }
+    // Live table
+    const { data } = await supabase.from("workflow_steps")
+        .select("id, step_key, step_type, max_retries")
+        .eq("workflow_id", workflowId).eq("step_key", stepKey).single();
+    return data;
+}
+async function createNextStepRunByKey(supabase, runId, workflowId, stepKey) {
+    const nextStep = await resolveStepDef(supabase, runId, workflowId, stepKey);
+    if (!nextStep) {
+        log.error({ stepKey, workflowId }, "step not found in workflow");
+        const { data: run } = await supabase.from("workflow_runs").select("store_id").eq("id", runId).single();
+        await completeWorkflowRun(supabase, runId, workflowId, run?.store_id, "failed", `Step '${stepKey}' not found`);
+        return null;
+    }
+    // Check step count limit
+    const { data: run } = await supabase.from("workflow_runs")
+        .select("store_id").eq("id", runId).single();
+    const { count } = await supabase.from("workflow_step_runs")
+        .select("id", { count: "exact", head: true }).eq("run_id", runId);
+    const { data: wf } = await supabase.from("workflows")
+        .select("max_steps_per_run").eq("id", workflowId).single();
+    if ((count || 0) >= (wf?.max_steps_per_run || 50)) {
+        await completeWorkflowRun(supabase, runId, workflowId, run?.store_id, "failed", `Step limit exceeded (${wf?.max_steps_per_run || 50})`);
+        return null;
+    }
+    const { data: inserted } = await supabase.from("workflow_step_runs").insert({
+        run_id: runId, step_id: nextStep.id, step_key: nextStep.step_key,
+        step_type: nextStep.step_type, status: "pending", max_attempts: nextStep.max_retries ?? 3,
+    }).select("id").single();
+    return inserted?.id || null;
+}
+// ============================================================================
+// PHASE 1: INLINE EXECUTION — execute steps immediately, no 5s wait
+// ============================================================================
+/**
+ * Claim a single pending step for a specific run using atomic RPC.
+ * P0 FIX: Uses claim_step_for_run RPC with FOR UPDATE SKIP LOCKED + attempt_count increment.
+ * Replaces the old SELECT-then-UPDATE pattern that never incremented attempt_count (infinite retries).
+ */
+async function claimStepForRun(supabase, runId) {
+    const { data, error } = await supabase.rpc("claim_step_for_run", { p_run_id: runId });
+    if (error) {
+        log.error({ err: error.message, runId }, "claim_step_for_run RPC failed");
+        return null;
+    }
+    // RPC returns an array of rows; we expect 0 or 1
+    const row = Array.isArray(data) ? data[0] : data;
+    if (!row)
+        return null;
+    return {
+        step_run_id: row.step_run_id,
+        run_id: row.run_id,
+        workflow_id: row.workflow_id,
+        store_id: row.store_id,
+        step_id: row.step_id,
+        step_key: row.step_key,
+        step_type: row.step_type,
+        step_config: row.step_config || {},
+        on_success: row.on_success,
+        on_failure: row.on_failure,
+        timeout_seconds: row.timeout_seconds || 60,
+        input_schema: row.input_schema,
+        step_outputs: row.step_outputs || {},
+        trigger_payload: row.trigger_payload || {},
+        attempt_count: row.attempt_count || 1,
+        max_attempts: row.max_attempts || 3,
+        max_steps_per_run: row.max_steps_per_run || 50,
+        input: row.input,
+        parent_step_run_id: row.parent_step_run_id,
+        retry_delay_seconds: row.retry_delay_seconds || 10,
+    };
+}
+/**
+ * Execute the first pending step of a run inline, then chain subsequent steps.
+ * Depth guard prevents unbounded recursion — worker loop catches anything left.
+ */
+export async function executeInlineChain(supabase, runId, depth = 0, traceId) {
+    if (depth >= MAX_INLINE_DEPTH) {
+        log.warn({ runId, depthLimit: MAX_INLINE_DEPTH }, "inline depth limit reached, deferring to worker");
+        return;
+    }
+    // Resolve traceId from run record if not passed (first depth only to avoid repeated queries)
+    if (!traceId && depth === 0) {
+        const { data: runData } = await supabase.from("workflow_runs")
+            .select("trace_id").eq("id", runId).single();
+        traceId = runData?.trace_id || undefined;
+    }
+    // H1 FIX: Claim step specifically for this run — cannot steal from other runs
+    const step = await claimStepForRun(supabase, runId);
+    if (!step)
+        return; // No pending steps for this run
+    // Phase 4: Override step_config/on_success/on_failure from version snapshot
+    await applyVersionOverrides(supabase, step);
+    try {
+        await executeAndAdvance(supabase, step, traceId);
+    }
+    catch (err) {
+        const errMsg = sanitizeError(err);
+        log.error({ stepKey: step.step_key, runId, err: errMsg }, "inline step execution error");
+        await supabase.from("workflow_step_runs").update({
+            status: "failed", error_message: errMsg,
+            completed_at: new Date().toISOString(), duration_ms: 0,
+        }).eq("id", step.step_run_id);
+        // Surface error to clients (SSE broadcast + structured error_details on step run)
+        await surfaceStepError(supabase, step, errMsg);
+        return;
+    }
+    // Steps that go async (delay, sub_workflow, parallel, for_each, approval) don't chain
+    const asyncTypes = new Set(["delay", "sub_workflow", "parallel", "for_each", "approval", "waitpoint"]);
+    if (asyncTypes.has(step.step_type))
+        return;
+    // Chain to next step
+    await executeInlineChain(supabase, runId, depth + 1, traceId);
+}
+async function checkWorkflowCompletion(supabase, runId, workflowId) {
+    // P1 FIX: Single atomic query to count all step statuses — eliminates race windows
+    // between multiple queries that could see inconsistent state
+    const { data: allSteps } = await supabase.from("workflow_step_runs")
+        .select("step_key, step_id, status, error_message")
+        .eq("run_id", runId);
+    if (!allSteps?.length) {
+        // No steps at all — mark as success (empty workflow)
+        const { data: run } = await supabase.from("workflow_runs").select("store_id").eq("id", runId).single();
+        await completeWorkflowRun(supabase, runId, workflowId, run?.store_id, "success");
+        return;
+    }
+    // Count in-memory from single query result
+    const activeStatuses = new Set(["pending", "running", "retrying", "waiting"]);
+    const activeSteps = allSteps.filter(s => activeStatuses.has(s.status));
+    if (activeSteps.length > 0)
+        return; // Still in progress
+    // All steps are terminal — determine outcome
+    const skippedSteps = allSteps.filter(s => s.status === "skipped");
+    const failedSteps = allSteps.filter(s => s.status === "failed");
+    // Filter to unhandled failures — steps where on_failure is null (no error handler)
+    // M14 FIX: Use versioned step defs when available
+    let failed = [];
+    if (failedSteps.length) {
+        const versionedSteps = await loadVersionedSteps(supabase, runId);
+        if (versionedSteps) {
+            // Use versioned definitions
+            const vStepMap = new Map(versionedSteps.map((s) => [s.step_key, s]));
+            failed = failedSteps.filter(s => {
+                const vStep = vStepMap.get(s.step_key);
+                return !vStep?.on_failure;
+            });
+        }
+        else {
+            // Fall back to live table
+            const stepIds = failedSteps.map(s => s.step_id).filter(Boolean);
+            if (stepIds.length) {
+                const { data: stepDefs } = await supabase.from("workflow_steps")
+                    .select("id, on_failure").in("id", stepIds);
+                const defMap = new Map((stepDefs || []).map(d => [d.id, d]));
+                failed = failedSteps.filter(s => {
+                    const def = defMap.get(s.step_id);
+                    return !def?.on_failure;
+                });
+            }
+        }
+    }
+    const { data: run } = await supabase.from("workflow_runs").select("store_id").eq("id", runId).single();
+    if (failed.length) {
+        await completeWorkflowRun(supabase, runId, workflowId, run?.store_id, "failed", failed[0].error_message, failed[0].step_key);
+    }
+    else if (skippedSteps.length) {
+        // All remaining steps were skipped (circuit breaker) — fail the run
+        await completeWorkflowRun(supabase, runId, workflowId, run?.store_id, "failed", `${skippedSteps.length} step(s) skipped by circuit breaker`, skippedSteps[0].step_key);
+    }
+    else {
+        await completeWorkflowRun(supabase, runId, workflowId, run?.store_id, "success");
+    }
+}
+export async function completeWorkflowRun(supabase, runId, workflowId, storeId, status, errorMessage, errorStepKey, traceId) {
+    const { data: run } = await supabase.from("workflow_runs")
+        .select("started_at, trace_id, metadata").eq("id", runId).single();
+    const durationMs = run?.started_at ? Date.now() - new Date(run.started_at).getTime() : null;
+    const resolvedTraceId = traceId || run?.trace_id || null;
+    // Aggregate error_count and error_log from failed step runs
+    const { data: failedStepRuns } = await supabase.from("workflow_step_runs")
+        .select("step_key, step_type, error_message, error_details")
+        .eq("run_id", runId)
+        .eq("status", "failed");
+    const errorCount = failedStepRuns?.length || 0;
+    const errorLog = (failedStepRuns || []).map(sr => ({
+        step_name: sr.step_key,
+        step_type: sr.step_type,
+        error_message: sr.error_message,
+        ...(sr.error_details ? { details: sr.error_details } : {}),
+    }));
+    const updatedMetadata = {
+        ...(run?.metadata || {}),
+        error_count: errorCount,
+        ...(errorLog.length ? { error_log: errorLog } : {}),
+    };
+    // Guard against double-completion: only update if still running
+    const { data: updatedRows, error: updateErr } = await supabase.from("workflow_runs").update({
+        status, error_message: errorMessage || null, error_step_key: errorStepKey || null,
+        completed_at: new Date().toISOString(), duration_ms: durationMs, current_step_key: null,
+        metadata: updatedMetadata,
+    }).eq("id", runId).eq("status", "running").select("id");
+    if (updateErr) {
+        log.warn({ err: updateErr.message, runId, status }, "completeWorkflowRun update failed");
+        return;
+    }
+    if (!updatedRows || updatedRows.length === 0) {
+        log.warn({ runId, status }, "completeWorkflowRun skipped — run already completed by another caller");
+        return;
+    }
+    // Cancel remaining pending steps
+    await supabase.from("workflow_step_runs").update({ status: "cancelled" })
+        .eq("run_id", runId).in("status", ["pending", "retrying", "waiting"]);
+    // Event journal — run completed
+    await logWorkflowEvent(supabase, runId, `run_${status}`, {
+        workflow_id: workflowId, duration_ms: durationMs,
+        ...(errorMessage ? { error: errorMessage } : {}),
+    });
+    // Circuit breaker
+    if (workflowId)
+        await handleWorkflowCircuitBreaker(supabase, workflowId, status === "success", errorMessage);
+    // Audit
+    const { error: runAuditErr } = await supabase.from("audit_logs").insert({
+        action: `workflow.run.${status}`, severity: status === "success" ? "info" : "error",
+        store_id: storeId || null, resource_type: "workflow_run", resource_id: runId,
+        source: "workflow_engine", duration_ms: durationMs,
+        request_id: resolvedTraceId,
+        details: { workflow_id: workflowId, run_id: runId },
+        error_message: errorMessage || null,
+    });
+    if (runAuditErr)
+        log.warn({ err: runAuditErr.message, runId }, "run completion audit failed");
+    // Error notifications
+    if (status === "failed" && workflowId) {
+        await sendErrorNotification(supabase, workflowId, runId, storeId, errorMessage, errorStepKey);
+        // Auto-archive to Dead Letter Queue
+        if (storeId) {
+            try {
+                await archiveToDlq(supabase, runId, workflowId, storeId);
+            }
+            catch (e) {
+                log.warn({ err: e?.message, runId }, "archiveToDlq failed");
+            }
+        }
+    }
+}
+async function sendErrorNotification(supabase, workflowId, runId, storeId, errorMessage, errorStepKey) {
+    const { data: wf } = await supabase.from("workflows")
+        .select("name, on_error_webhook_url, on_error_email").eq("id", workflowId).single();
+    if (!wf)
+        return;
+    const errorPayload = {
+        event: "workflow.run.failed",
+        workflow_id: workflowId, workflow_name: wf.name,
+        run_id: runId, error_message: errorMessage, error_step: errorStepKey,
+        timestamp: new Date().toISOString(),
+    };
+    // Webhook notification (with SSRF protection)
+    const errorWebhookSsrf = wf.on_error_webhook_url ? await validateUrl(wf.on_error_webhook_url) : "no URL";
+    if (wf.on_error_webhook_url && !errorWebhookSsrf) {
+        try {
+            const controller = new AbortController();
+            const timer = setTimeout(() => controller.abort(), 10_000);
+            await fetch(wf.on_error_webhook_url, {
+                method: "POST", headers: { "Content-Type": "application/json" },
+                body: JSON.stringify(errorPayload), signal: controller.signal,
+            });
+            clearTimeout(timer);
+        }
+        catch (err) {
+            log.error({ err: sanitizeError(err), workflowId }, "error notification webhook failed");
+        }
+    }
+    // Email notification
+    if (wf.on_error_email && _executeTool && storeId) {
+        try {
+            await _executeTool(supabase, "email", {
+                action: "send", to: wf.on_error_email,
+                subject: `Workflow "${wf.name}" failed`,
+                text: `Workflow "${wf.name}" failed at step "${errorStepKey || "unknown"}".\n\nError: ${errorMessage || "Unknown error"}\n\nRun ID: ${runId}\nTime: ${new Date().toISOString()}`,
+            }, storeId);
+        }
+        catch (err) {
+            log.error({ err: sanitizeError(err), workflowId }, "error notification email failed");
+        }
+    }
+}
+// ============================================================================
+// WEBHOOK INGESTION
+// ============================================================================
+export async function handleWebhookIngestion(supabase, slug, rawBody, headers, storeId) {
+    // P0 FIX: Multi-tenancy — scope webhook lookup by store_id when available
+    let endpointQuery = supabase.from("webhook_endpoints")
+        .select("*").eq("slug", slug).eq("is_active", true);
+    if (storeId) {
+        endpointQuery = endpointQuery.eq("store_id", storeId);
+    }
+    const { data: endpoints } = await endpointQuery.limit(1);
+    const endpoint = endpoints?.[0];
+    if (!endpoint)
+        return { status: 404, body: { error: "Webhook endpoint not found" } };
+    // Rate limit
+    const oneMinAgo = new Date(Date.now() - 60_000).toISOString();
+    const { count: recentCount } = await supabase.from("audit_logs")
+        .select("id", { count: "exact", head: true })
+        .eq("resource_type", "webhook_endpoint").eq("resource_id", endpoint.id)
+        .gte("created_at", oneMinAgo);
+    if ((recentCount || 0) >= endpoint.max_requests_per_minute) {
+        return { status: 429, body: { error: "Rate limit exceeded" } };
+    }
+    // HMAC verification
+    if (endpoint.verify_signature) {
+        const signature = headers["x-webhook-signature"] || headers["x-hub-signature-256"] || "";
+        if (!signature)
+            return { status: 401, body: { error: "Missing signature" } };
+        const expected = `sha256=${createHmac("sha256", endpoint.signing_secret).update(rawBody).digest("hex")}`;
+        try {
+            const sigBuf = Buffer.from(signature);
+            const expBuf = Buffer.from(expected);
+            if (sigBuf.length !== expBuf.length || !timingSafeEqual(sigBuf, expBuf)) {
+                return { status: 401, body: { error: "Invalid signature" } };
+            }
+        }
+        catch {
+            return { status: 401, body: { error: "Invalid signature" } };
+        }
+    }
+    // P3 FIX: Reject oversized payloads before parsing to prevent memory exhaustion
+    const MAX_TRIGGER_PAYLOAD_BYTES = 10_000_000; // 10MB
+    if (rawBody.length > MAX_TRIGGER_PAYLOAD_BYTES) {
+        return { status: 413, body: { error: `Trigger payload too large (${rawBody.length} bytes, max 10MB)` } };
+    }
+    let payload;
+    try {
+        payload = JSON.parse(rawBody);
+    }
+    catch {
+        payload = { raw: rawBody };
+    }
+    if (endpoint.payload_transform && typeof endpoint.payload_transform === "object") {
+        payload = resolveTemplate(endpoint.payload_transform, { steps: {}, trigger: payload });
+    }
+    // Update stats
+    await supabase.from("webhook_endpoints").update({
+        last_received_at: new Date().toISOString(),
+        total_received: (endpoint.total_received || 0) + 1,
+    }).eq("id", endpoint.id);
+    // Audit
+    const { error: whAuditErr } = await supabase.from("audit_logs").insert({
+        action: "webhook.received", severity: "info", store_id: endpoint.store_id,
+        resource_type: "webhook_endpoint", resource_id: endpoint.id, source: "webhook",
+        details: { slug, workflow_id: endpoint.workflow_id },
+    });
+    if (whAuditErr)
+        log.warn({ err: whAuditErr.message, slug }, "webhook audit insert failed");
+    // Start workflow
+    const { data: startResult } = await supabase.rpc("start_workflow_run", {
+        p_workflow_id: endpoint.workflow_id, p_store_id: endpoint.store_id,
+        p_trigger_type: "webhook", p_trigger_payload: payload,
+    });
+    if (!startResult?.success) {
+        return { status: 422, body: { error: startResult?.error || "Failed to start workflow" } };
+    }
+    const runId = startResult.run_id;
+    // Sync response — poll until workflow completes or timeout
+    if (endpoint.sync_response) {
+        const timeoutMs = (endpoint.sync_timeout_seconds || 30) * 1000;
+        const deadline = Date.now() + timeoutMs;
+        while (Date.now() < deadline) {
+            await new Promise(r => setTimeout(r, 500));
+            const { data: run } = await supabase.from("workflow_runs")
+                .select("status, step_outputs, error_message").eq("id", runId).single();
+            if (run?.status === "success") {
+                return { status: 200, body: { success: true, run_id: runId, output: run.step_outputs } };
+            }
+            if (run?.status === "failed") {
+                return { status: 422, body: { success: false, run_id: runId, error: run.error_message, output: run.step_outputs } };
+            }
+        }
+        return { status: 202, body: { success: true, run_id: runId, status: "running", message: "Workflow still in progress" } };
+    }
+    return { status: 200, body: { success: true, run_id: runId, deduplicated: startResult.deduplicated || false } };
+}