spine-framework 0.4.3 → 0.4.4

package/.framework/functions/_shared/trigger-engine.ts

@@ -37,7 +37,6 @@
 import { CoreContext } from './middleware'
 import { adminDb } from './db'
 import { emitAudit } from './audit'
-import { runPipeline } from './pipeline-runner'
 // CIL[END stable_id="~/.framework/functions/_shared/trigger-engine.ts:module_header:module_header"]
 // CIL[DOC-VECTOR type="context" content="purpose='This module serves as the trigger engine for processing events within the system, coordinating various actions based on incoming triggers.'; No verified external callers" status="ready"]
 // CIL[DOC-VECTOR type="blast_radius" content="No direct caller data from CIL graph; local impact only" status="ready"]
@@ -203,43 +202,102 @@ interface Trigger {
 // CIL[DOC-VECTOR type="implementation_pattern" content="" status="pending_human_input"]
 // CIL[DOC-VECTOR type="user_journey" content="" status="pending_human_input"]
 
-// CIL[DOC-MACRO purpose="This function checks for active triggers based on the specified event type and fires them if their conditions are met."]
-// CIL[DOC-RESPONSIBILITY responsibility="This function checks for active triggers based on the specified event type and fires them if their conditions are met. It exists to facilitate the execution of defined actions in response to specific events within the system."]
-// CIL[DOC-CONTEXT module="trigger-engine" layer="app-specific" related_files=".framework/functions/_shared/middleware,.framework/functions/_shared/db,.framework/functions/_shared/audit,.framework/functions/_shared/pipeline-runner"]
-// CIL[DOC-CONTRACT inputs="- eventType: EventType (required) — the type of event that triggers the action.\\n- entityType: string (required) — the type of entity associated with the event.\\n- entityId: string (required) — the unique identifier of the entity.\\n- entityData: any (required) — the data associated with the entity.\\n- ctx: CoreContext (required) — the context for the current operation." outputs="Promise<void>" errors="none" side_effects="Database writes to update trigger last triggered time and count, RPC calls to increment trigger count, and audit log emissions. dependencies="adminDb.from('triggers'), adminDb.from('types'), runPipeline, emitAudit, adminDb.from('triggers').update"]
-// CIL[DOC-MICRO logic_summary="This function checks for active triggers based on the specified event type and fires them if their conditions are met. It exists to facilitate the execution of defined actions in response to specific events within the system." notable_decisions="Logs errors to the console and emits audit logs for failures."]
-// CIL[DOC-CONFIDENCE pass="second" score="0.90" flags="none"]
+// CIL[DOC-MACRO purpose="Dispatch a trigger pipeline to the background pipeline-runner function, returning immediately without awaiting execution"]
+// CIL[DOC-RESPONSIBILITY responsibility="POST the payload to /.netlify/functions/pipeline-runner-background with the internal invoke secret so the pipeline runs in an isolated process with its own connection budget"]
+// CIL[DOC-CONTEXT module="trigger-engine" layer="shared-core" related_files=".framework/functions/pipeline-runner-background.ts"]
+// CIL[DOC-CONTRACT inputs="pipelineId, triggerData, ctx, opts { executionId, triggerId }" outputs="Promise<void>" errors="Logs dispatch failure to console; caller's queued execution row is left for the sweeper" side_effects="HTTP POST to internal function"]
+// CIL[DOC-MICRO logic_summary="Builds the dispatch URL from SITE_URL/URL env (falling back to localhost for netlify dev), sends X-Internal-Secret for machine auth, and awaits only the 202 ack bounded by a 5s AbortSignal timeout — the caller never waits for pipeline completion" notable_decisions="Does NOT await runPipeline — that was the connection-pool starvation bug; moving execution to a separate function invocation is the fix. The 5s timeout bounds the caller when Netlify's queue ack stalls; an aborted dispatch may still deliver server-side, and the pre-created 'queued' row is the sweeper's safety net either way"]
+// CIL[DOC-CONFIDENCE pass="first" score="0.9" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+/**
+ * POST a pipeline dispatch to the internal background function.
+ * Returns immediately; the pipeline runs in a separate process.
+ * Any dispatch failure leaves the pre-created execution row 'queued' —
+ * the sweeper in pipeline-runner.ts marks it 'failed' after 10 minutes.
+ */
+export async function dispatchTriggerPipeline(
+  pipelineId: string,
+  triggerData: any,
+  ctx: CoreContext,
+  opts: { executionId: string; triggerId: string }
+): Promise<void> {
+  const secret = (globalThis as any).process?.env?.INTERNAL_INVOKE_SECRET
+  if (!secret) {
+    console.error(`[${ctx.requestId}] INTERNAL_INVOKE_SECRET not set — cannot dispatch trigger pipeline ${pipelineId}`)
+    return
+  }
+
+  const siteUrl = (globalThis as any).process?.env?.SITE_URL ||
+                  (globalThis as any).process?.env?.URL ||
+                  'http://localhost:8888'
+  const url = `${siteUrl.replace(/\/$/, '')}/.netlify/functions/pipeline-runner-background`
+
+  try {
+    const response = await fetch(url, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Internal-Secret': secret,
+      },
+      body: JSON.stringify({
+        pipeline_id: pipelineId,
+        trigger_data: triggerData,
+        account_id: ctx.accountId,
+        execution_id: opts.executionId,
+        trigger_id: opts.triggerId,
+      }),
+      // Bound the caller's wait: the background function normally answers
+      // 202 in well under a second, but a stuck connection (deploy rollout,
+      // slow Netlify queue ack) once held a caller for ~34s.  Aborting only
+      // releases this caller — if Netlify already received the POST the
+      // queued invocation still runs; if not, the pre-created 'queued'
+      // execution row is left for the sweeper.
+      signal: AbortSignal.timeout(5000),
+    })
+
+    if (!response.ok) {
+      const errText = await response.text().catch(() => `HTTP ${response.status}`)
+      console.error(`[${ctx.requestId}] trigger dispatch failed for pipeline ${pipelineId} (execution ${opts.executionId}): HTTP ${response.status} — ${errText}`)
+    }
+  } catch (err: any) {
+    const timedOut = err?.name === 'AbortError' || err?.name === 'TimeoutError'
+    console.error(`[${ctx.requestId}] trigger dispatch ${timedOut ? 'timed out after 5s' : 'error'} for pipeline ${pipelineId} (execution ${opts.executionId}):`, err.message || err)
+  }
+}
+// CIL[END stable_id="~/.framework/functions/_shared/trigger-engine.ts:dispatchtriggerpipeline:function"]
+
+
+// CIL[DOC-MACRO purpose="Check for active triggers matching the event type, pre-create a queued pipeline_executions row, and dispatch to the background function"]
+// CIL[DOC-RESPONSIBILITY responsibility="Evaluate trigger conditions, reserve a queued execution row, dispatch to pipeline-runner-background, record trigger fire, and emit audit — returns without waiting for pipeline completion"]
+// CIL[DOC-CONTEXT module="trigger-engine" layer="shared-core" related_files=".framework/functions/_shared/middleware,.framework/functions/_shared/db,.framework/functions/_shared/audit,.framework/functions/pipeline-runner-background.ts"]
+// CIL[DOC-CONTRACT inputs="eventType, entityType, entityId, entityData, ctx, previousEntityData? (pre-update record snapshot for transition checks)" outputs="Promise<void>" errors="Logs errors; emits trigger.failed audit" side_effects="Inserts pipeline_executions row (status: queued), RPC record_trigger_fire, emits trigger.fired audit"]
+// CIL[DOC-MICRO logic_summary="Queries active triggers for the event type, resolves type_slug when needed for trigger filters, evaluates each trigger's conditions, then for matches: INSERTs a queued pipeline_executions row so the execution ID exists before dispatch, calls dispatchTriggerPipeline (which returns immediately), records the trigger fire, and emits the trigger.fired audit with execution_status='queued'" notable_decisions="Pre-creating the execution row is the key to preserving execution_id in the audit trail without awaiting pipeline completion; 'queued' rows left orphaned are swept by the stale-execution reaper"]
+// CIL[DOC-CONFIDENCE pass="first" score="0.9" flags="none"]
 // CIL[DOC-STABILITY stability="stable"]
 /**
- * This function checks for active triggers based on the specified event type and fires them if their conditions are met. It exists to facilitate the execution of defined actions in response to specific events within the system.
+ * Check for active triggers matching the event type and dispatch each to the
+ * background pipeline-runner function.  Returns immediately — pipeline
+ * completion is tracked in pipeline_executions independently.
  *
  * @inputSpec
- *   - eventType: EventType (required) — the type of event that triggers the action.
- *   - entityType: string (required) — the type of entity associated with the event.
- *   - entityId: string (required) — the unique identifier of the entity.
- *   - entityData: any (required) — the data associated with the entity.
- *   - ctx: CoreContext (required) — the context for the current operation.
+ *   - eventType: EventType — the event that occurred.
+ *   - entityType: string — the entity type (plural runtime table name).
+ *   - entityId: string — the affected entity ID.
+ *   - entityData: any — the entity snapshot at the time of the event.
+ *   - ctx: CoreContext — request context (account scope, request ID, principal).
  * @outputSpec  Promise<void>
- * @sideEffects Database writes to update trigger last triggered time and count, RPC calls to increment trigger count, and audit log emissions.
+ * @sideEffects INSERTs pipeline_executions rows (status: queued),
+ *              RPC record_trigger_fire per matching trigger,
+ *              emits trigger.fired / trigger.failed audit events.
  * @stability   stable
- * @calls       adminDb.from('triggers'), adminDb.from('types'), runPipeline, emitAudit, adminDb.from('triggers').update
- * @errorHandling Logs errors to the console and emits audit logs for failures.
- * @cil-generated
  */
-// CIL[BEGIN stable_id="~/.framework/functions/_shared/trigger-engine.ts:checkandfiretriggers:function" type="function" identity="checkAndFireTriggers" parent="~/.framework/functions/_shared/trigger-engine.ts"]
-// CIL[DOC-MACRO purpose="checkAndFireTriggers function"]
-// CIL[DOC-RESPONSIBILITY responsibility="Implements the checkAndFireTriggers function."]
-// CIL[DOC-CONTEXT module=".framework/functions/_shared/trigger-engine.ts" layer="app-specific" related_files=""]
-// CIL[DOC-CONTRACT inputs="none" outputs="none" errors="none" side_effects="none"]
-// CIL[DOC-MICRO logic_summary="Implements the checkAndFireTriggers function." notable_decisions="[]"]
-// CIL[DOC-CONFIDENCE pass="first" score="0.8" flags="none"]
-// CIL[DOC-STABILITY stability="stable"]
 export async function checkAndFireTriggers(
   eventType: EventType,
   entityType: string,
   entityId: string,
   entityData: any,
-  ctx: CoreContext
+  ctx: CoreContext,
+  previousEntityData?: any
 ): Promise<void> {
   const { data: triggers, error } = await adminDb
     .from('triggers')
@@ -270,11 +328,30 @@ export async function checkAndFireTriggers(
   for (const trigger of triggers) {
     try {
       const shouldFire = evaluateTriggerConditions(trigger, entityType, entityData)
-      
+
       if (!shouldFire) {
         continue
       }
 
+      // Transition-only triggers (config.fire_on_transition): fire only when
+      // the record moves INTO the matching state.  Without this, every
+      // update to an already-matching record re-fires the pipeline — e.g.
+      // each write to a published KB article re-dispatched auto_embed (3×
+      // per publish) and revoke fired on articles that never left
+      // 'published'.  When no previous snapshot is available the trigger
+      // fires normally (conservative: matches pre-flag behavior).
+      if (trigger.config?.fire_on_transition && previousEntityData) {
+        const prevEntityData = {
+          ...previousEntityData,
+          // Same record → same type; the slug was resolved on the new
+          // snapshot only, so carry it over for config.type_slug checks.
+          type_slug: previousEntityData.type_slug ?? entityData?.type_slug
+        }
+        if (evaluateTriggerConditions(trigger, entityType, prevEntityData)) {
+          continue
+        }
+      }
+
       const triggerData = {
         event: eventType,
         entity_id: entityId,
@@ -290,15 +367,46 @@ export async function checkAndFireTriggers(
         fired_at: new Date().toISOString()
       }
 
-      const result = await runPipeline(trigger.pipeline_id, triggerData, ctx)
-
-      await adminDb
-        .from('triggers')
-        .update({
-          last_triggered: new Date().toISOString(),
-          trigger_count: adminDb.rpc('increment_trigger_count', { p_trigger_id: trigger.id })
+      // Reserve the execution row before dispatch so the audit trail has a
+      // stable execution_id even though the pipeline has not started yet.
+      // The background function claims this row (queued → running) via
+      // runPipeline's opts.executionId parameter.  If dispatch fails, this
+      // row stays 'queued' and the sweeper marks it 'failed' after 10 min.
+      const { data: execRow } = await adminDb
+        .from('pipeline_executions')
+        .insert({
+          pipeline_id: trigger.pipeline_id,
+          status: 'queued',
+          trigger_data: triggerData,
+          account_id: ctx.accountId,
+          // Same guard as runPipeline's own insert: created_by is uuid-typed,
+          // non-UUID principals (machine ids like 'system', 'timer:*') → null.
+          created_by: ctx.principal?.id &&
+            /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(ctx.principal.id)
+            ? ctx.principal.id : null,
+          started_at: new Date().toISOString(),
+          updated_at: new Date().toISOString()
         })
-        .eq('id', trigger.id)
+        .select('id')
+        .single()
+
+      const executionId = execRow?.id ?? crypto.randomUUID()
+
+      await dispatchTriggerPipeline(trigger.pipeline_id, triggerData, ctx, {
+        executionId,
+        triggerId: trigger.id,
+      })
+
+      // Atomic bookkeeping: the previous update embedded an
+      // adminDb.rpc() Promise as a column value, so the whole write
+      // failed silently — trigger_count stayed 0 and last_triggered
+      // stayed null on every fire.
+      const { error: fireError } = await adminDb.rpc('record_trigger_fire', {
+        p_trigger_id: trigger.id
+      })
+      if (fireError) {
+        console.error(`[${ctx.requestId}] record_trigger_fire failed for ${trigger.id}:`, fireError.message)
+      }
 
       await emitAudit(ctx, 'trigger.fired', {
         type: 'trigger',
@@ -309,13 +417,13 @@ export async function checkAndFireTriggers(
         entity_type: entityType,
         entity_id: entityId,
         pipeline_id: trigger.pipeline_id,
-        execution_id: result.executionId,
-        execution_status: result.status
+        execution_id: executionId,
+        execution_status: 'queued'
       })
 
     } catch (error: any) {
       console.error(`[${ctx.requestId}] Trigger ${trigger.id} failed:`, error)
-      
+
       await emitAudit(ctx, 'trigger.failed', {
         type: 'trigger',
         id: trigger.id,
@@ -602,10 +710,11 @@ export async function fireUpdateTriggers(
   entityType: string,
   entityId: string,
   entityData: any,
-  ctx: CoreContext
+  ctx: CoreContext,
+  previousEntityData?: any
 ): Promise<void> {
   const eventType = `${eventNameFor(entityType)}_updated` as EventType
-  await checkAndFireTriggers(eventType, entityType, entityId, entityData, ctx)
+  await checkAndFireTriggers(eventType, entityType, entityId, entityData, ctx, previousEntityData)
 }
 // CIL[END stable_id="~/.framework/functions/_shared/trigger-engine.ts:fireupdatetriggers:function"]
 // CIL[DOC-VECTOR type="context" content="purpose='Triggers an update event for a specified entity type and ID, allowing for associated actions to be executed.'; No verified external callers" status="ready"]

package/.framework/functions/admin-data.ts

@@ -688,7 +688,11 @@ export const create = createHandler(async (ctx, body) => {
 
         const { data, error: err } = await ctx.db.from(entity).insert(raw).select().single()
         if (err) throw err
-        fireCreateTriggers(entity, data.id, raw, ctx).catch(console.error)
+        // Await the trigger check: the heavy work already offloads to
+        // pipeline-runner-background, so this only adds eval + queue insert
+        // + dispatch. Detached post-response work is CPU-frozen on Netlify
+        // and silently dropped trigger fires in production.
+        await fireCreateTriggers(entity, data.id, raw, ctx).catch(console.error)
         return await sanitizeRecordData(ctx, data, entity)
       }
 
@@ -808,7 +812,7 @@ export const create = createHandler(async (ctx, body) => {
       if (err) throw err
 
       const entityData = { ...dataToInsert, id: data.id }
-      fireCreateTriggers(entity, data.id, entityData, ctx).catch(console.error)
+      await fireCreateTriggers(entity, data.id, entityData, ctx).catch(console.error)
 
       return await sanitizeRecordData(ctx, data, entity)
     }
@@ -915,7 +919,9 @@ export const update = createHandler(async (ctx, body) => {
       if (err) throw err
 
       const entityData = { ...data, ...dataToUpdate }
-      fireUpdateTriggers(entity, id, entityData, ctx).catch(console.error)
+      // Await the trigger check — see note in the create path. Detached
+      // post-response work is CPU-frozen on Netlify and drops trigger fires.
+      await fireUpdateTriggers(entity, id, entityData, ctx, existing).catch(console.error)
 
       return await sanitizeRecordData(ctx, data, entity)
     }

package/.framework/functions/ai-agents.ts

@@ -30,7 +30,7 @@ import { adminDb } from './_shared/db'
 import { emitLog } from './_shared/audit'
 import { isSystemAdmin } from './_shared/principal'
 import { PermissionEngine, sanitizeRecordData } from './_shared/permissions'
-import { runAgent, kickAgentEpisode } from './_shared/agent-runner'
+import { runAgent, claimAgentEpisode, acquireEpisodeLease } from './_shared/agent-runner'
 import { checkRateLimit } from './_shared/rate-limit'
 import { detectInjection, emitSecurityEvent, getAppSecurityConfig } from './_shared/security'
 // CIL[END stable_id="~/.framework/functions/ai-agents.ts:module_header:module_header"]
@@ -479,20 +479,27 @@ export const run = createHandler(async (ctx, body) => {
     }
   }
 
+  // Episode mode runs inline up to this budget — the first turns execute
+  // at full request CPU, then the client's step loop continues the
+  // episode under the same lease. Sized to leave headroom under the
+  // platform's synchronous request limit (~26s) even with a slow turn.
+  const EPISODE_INLINE_BUDGET_MS = 8000
+
   const runMode = (mode as 'step' | 'complete' | 'episode') || 'complete'
 
-  // Episode single-flight: the in-flight driver stamps driver_id +
-  // driver_heartbeat_at into research_state. A different driver asking to
-  // step/kick while the lease is fresh gets 'running' instead of a second
-  // interleaved write pass. The owner itself is never refused (its own
-  // staleness check is finer-grained), and a lease expires after
-  // DRIVER_LEASE_MS so a crashed driver can't wedge the thread.
-  const DRIVER_LEASE_MS = 60000
+  // Episode single-flight: arbitration is the atomic episode_leases RPC
+  // (migration 063) — a claim INSERTs/locks the thread's lease row in one
+  // statement, so two entry paths can never both pass. The same driver
+  // re-acquiring renews; a different live driver is refused 'running';
+  // an expired lease is taken over so a crashed driver can't wedge the
+  // thread. When the RPC is absent (un-migrated deployment) the legacy
+  // non-atomic research_state check below runs instead.
   const existingState = (data as any)?.data?.research_state
   const heartbeatAt = Date.parse(existingState?.driver_heartbeat_at || '')
   const leaseHolder = existingState?.driver_id
   const callerToken = (body?.driver_token as string) || null
   const callerId = callerToken || ctx.requestId
+  const DRIVER_LEASE_MS = 60000
   const leaseLive =
     !!leaseHolder &&
     leaseHolder !== callerId &&
@@ -501,32 +508,91 @@ export const run = createHandler(async (ctx, body) => {
     Number.isFinite(heartbeatAt) &&
     Date.now() - heartbeatAt < DRIVER_LEASE_MS
 
-  if ((runMode === 'step' || runMode === 'complete') && leaseLive) {
-    return { ok: true, thread_id, status: 'running', driver_id: leaseHolder }
-  }
-
   if (runMode === 'episode') {
-    if (leaseLive) {
-      return { ok: true, thread_id, status: 'running', detached: true, already_running: true }
+    // Episode mode claims the lease and runs the first turns INLINE
+    // (deadline-bounded complete mode) — detached 'void runAgent' work is
+    // CPU-throttled after the response flush on serverless platforms
+    // (~37s dead window observed before its first persisted message),
+    // which is exactly the stall the caller's quiet-window then "rescues"
+    // by double-driving. stealDetached lets this request take over from
+    // a 'detached:' background kick instantly; the fenced driver aborts
+    // at its next heartbeat without side effects.
+    const kick = await claimAgentEpisode(thread_id, user_message || '', ctx, callerId, (data as any)?.data, { stealDetached: true })
+    if (!kick.claimed && kick.already_done) {
+      // Replay of an already-answered question — return the episode's
+      // existing final rather than 'running' so callers don't poll for
+      // work that isn't coming.
+      const { data: lastOutbound } = await adminDb
+        .from('messages')
+        .select('id, content, data')
+        .eq('thread_id', thread_id)
+        .eq('direction', 'outbound')
+        .order('created_at', { ascending: false })
+        .limit(1)
+      const final = Array.isArray(lastOutbound) ? lastOutbound[0] : lastOutbound
+      return {
+        ok: true,
+        thread_id,
+        status: 'complete',
+        already_done: true,
+        driver_id: kick.driver_id,
+        message_id: final?.id || null,
+        content: final?.content || '',
+        message_type: final?.data?.message_type || null,
+        data: final?.data || null,
+        agent_id: final?.data?.agent_id || (data as any)?.data?.agent_id || null,
+        prompt_config_id: final?.data?.prompt_config_id || (data as any)?.data?.prompt_config_id || null,
+      }
     }
-    // Debit agent_step only when a new driver will actually start work —
-    // lease-refused requests and ticket-creation kicks already debited or
-    // started nothing, and counting them would double-charge each episode.
-    await checkRateLimit(ctx, 'agent_step')
-    // Claim the lease before detaching so a step request landing during
-    // resolveAgentConfig still sees a live driver. Shared implementation lives
-    // in kickAgentEpisode (also used by ticket-creation handlers).
-    const leaseId = callerToken || ctx.requestId
-    const kick = await kickAgentEpisode(thread_id, user_message || '', ctx, leaseId, (data as any)?.data)
     if (!kick.claimed) {
-      return { ok: true, thread_id, status: 'running', detached: true, already_running: true }
+      return { ok: true, thread_id, status: 'running', already_running: true, driver_id: kick.driver_id }
+    }
+    // Debit agent_step only after a claim — refused requests start
+    // nothing and must not double-charge the episode.
+    await checkRateLimit(ctx, 'agent_step')
+    const message = await runAgent(thread_id, user_message || '', ctx, 'complete', kick.driver_id || callerToken || undefined, kick.lease_token, { deadlineMs: EPISODE_INLINE_BUDGET_MS })
+
+    const { data: updatedThread } = await adminDb
+      .from('threads')
+      .select('data')
+      .eq('id', thread_id)
+      .single()
+    const state = updatedThread?.data?.research_state || {}
+    return {
+      ok: true,
+      thread_id,
+      detached: false,
+      message_id: message?.id || null,
+      status: state.status || (message ? 'complete' : 'failed'),
+      turn_count: state.turn_count ?? 0,
+      content: message?.content || '',
+      message_type: message?.data?.message_type || null,
+      data: message?.data || null,
+      agent_id: message?.data?.agent_id || updatedThread?.data?.agent_id || null,
+      prompt_config_id: message?.data?.prompt_config_id || updatedThread?.data?.prompt_config_id || null,
     }
-    return { ok: true, thread_id, status: 'running', detached: true }
   }
 
-  // step/complete modes debit here — same rule: only when work will start.
+  // step/complete: claim the episode lease for this driver. 'held' means
+  // another live driver owns the episode — the caller watches persisted
+  // state instead of interleaving writes. A claimed lease survives a
+  // nonterminal step return so the same driver's next call renews it.
+  // stealDetached lets a request driver take over from a 'detached:'
+  // background kick without waiting out the TTL.
+  const claim = await acquireEpisodeLease(thread_id, ctx.accountId, callerId, undefined, true)
+  let leaseToken: string | undefined
+  if (claim?.outcome === 'held') {
+    return { ok: true, thread_id, status: 'running', driver_id: claim.holder_driver_id }
+  }
+  if (claim === null && leaseLive) {
+    return { ok: true, thread_id, status: 'running', driver_id: leaseHolder }
+  }
+  if (claim?.outcome === 'claimed') leaseToken = claim.lease_token
+
+  // step/complete modes debit here — only when work will start, so
+  // lease-refused requests never double-charge an episode.
   await checkRateLimit(ctx, 'agent_step')
-  const message = await runAgent(thread_id, user_message || '', ctx, runMode, callerToken || undefined)
+  const message = await runAgent(thread_id, user_message || '', ctx, runMode, callerId, leaseToken)
 
   const { data: updatedThread } = await adminDb
     .from('threads')

package/.framework/functions/pipeline-runner-background.ts

@@ -0,0 +1,120 @@
+// CIL[BEGIN stable_id="~/.framework/functions/pipeline-runner-background.ts:module_header:module_header" type="module_header" identity="module_header" parent="~/.framework/functions/pipeline-runner-background.ts"]
+// CIL[DOC-MACRO purpose="Background entry point for event-triggered pipeline executions — receives dispatched payloads from checkAndFireTriggers and runs the pipeline in an isolated process with its own connection budget"]
+// CIL[DOC-RESPONSIBILITY responsibility="Authenticate internal dispatch calls via INTERNAL_INVOKE_SECRET, reconstruct a machine RequestContext, and invoke runPipeline with the pre-created execution_id so the queued row transitions to running/completed rather than being duplicated"]
+// CIL[DOC-CONTEXT module="pipeline-runner-background" layer="functions" related_files=".framework/functions/_shared/trigger-engine.ts,.framework/functions/_shared/pipeline-runner.ts,.framework/functions/_shared/principal.ts,.framework/functions/_shared/middleware.ts"]
+// CIL[DOC-CONTRACT inputs="POST body { pipeline_id, trigger_data, account_id, execution_id, trigger_id } with X-Internal-Secret header" outputs="{ ok: true, execution_id } | { error }" errors="403 on missing/wrong secret, 400 on missing pipeline_id, 500 on pipeline failure" side_effects="Runs the named pipeline; updates pipeline_executions row from queued→running→terminal"]
+// CIL[DOC-MICRO logic_summary="Validates the shared internal secret (timing-safe, same as principal.isInternalInvokeAuthorized), builds a SYSTEM_PRINCIPAL RequestContext carrying the original account scope, then calls runPipeline with the caller's pre-created execution_id. Status 'queued' → 'running' is handled inside runPipeline via the new opts.executionId parameter." notable_decisions="Does NOT use createHandler — it would resolve the secret to anonymous and 401 before reaching the handler. Auth is checked inline and SYSTEM_PRINCIPAL provides the machine context directly."]
+// CIL[DOC-CONFIDENCE pass="first" score="0.9" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+// CIL[DOC-AUDIENCE audience="framework"]
+
+import { createHash, timingSafeEqual } from 'node:crypto'
+import { runPipeline, sweepStaleExecutions } from './_shared/pipeline-runner'
+import { SYSTEM_PRINCIPAL } from './_shared/principal'
+import { adminDb } from './_shared/db'
+// CIL[END stable_id="~/.framework/functions/pipeline-runner-background.ts:module_header:module_header"]
+
+
+// CIL[BEGIN stable_id="~/.framework/functions/pipeline-runner-background.ts:isdispatchauthorized:function" type="function" identity="isDispatchAuthorized" parent="~/.framework/functions/pipeline-runner-background.ts"]
+// CIL[DOC-MACRO purpose="Verify the X-Internal-Secret header using timing-safe comparison (mirrors principal.isInternalInvokeAuthorized)"]
+// CIL[DOC-RESPONSIBILITY responsibility="Return true only when the request carries a valid INTERNAL_INVOKE_SECRET"]
+// CIL[DOC-CONTEXT module="pipeline-runner-background" layer="functions" related_files=".framework/functions/_shared/principal.ts"]
+// CIL[DOC-CONTRACT inputs="event headers" outputs="boolean" errors="none" side_effects="none"]
+// CIL[DOC-MICRO logic_summary="SHA-256 both sides before timingSafeEqual so early-exit on length mismatch cannot leak the expected secret length via response timing — identical pattern to principal.isInternalInvokeAuthorized" notable_decisions="duplicated here rather than imported because isInternalInvokeAuthorized is unexported from principal.ts"]
+// CIL[DOC-CONFIDENCE pass="first" score="0.9" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+function isDispatchAuthorized(event: any): boolean {
+  const expected = (globalThis as any).process?.env?.INTERNAL_INVOKE_SECRET
+  if (!expected) return false
+
+  const provided = event.headers?.['x-internal-secret'] || event.headers?.['X-Internal-Secret']
+  if (!provided || typeof provided !== 'string') return false
+
+  const expectedHash = createHash('sha256').update(expected).digest()
+  const providedHash = createHash('sha256').update(provided).digest()
+
+  try {
+    return timingSafeEqual(expectedHash, providedHash)
+  } catch {
+    return false
+  }
+}
+// CIL[END stable_id="~/.framework/functions/pipeline-runner-background.ts:isdispatchauthorized:function"]
+
+
+// CIL[BEGIN stable_id="~/.framework/functions/pipeline-runner-background.ts:handler:constant" type="constant" identity="handler" parent="~/.framework/functions/pipeline-runner-background.ts"]
+// CIL[DOC-MACRO purpose="Netlify function handler for dispatched pipeline executions"]
+// CIL[DOC-RESPONSIBILITY responsibility="Authenticate, sweep stale executions, then runPipeline with the caller's pre-created execution_id"]
+// CIL[DOC-CONTEXT module="pipeline-runner-background" layer="functions" related_files=".framework/functions/_shared/pipeline-runner.ts"]
+// CIL[DOC-CONTRACT inputs="POST body { pipeline_id, trigger_data, account_id, execution_id, trigger_id }" outputs="{ ok, execution_id, status } | { error }" errors="403 unauthorized, 400 missing pipeline_id" side_effects="Pipeline runs; pipeline_executions transitions queued→running→terminal"]
+// CIL[DOC-MICRO logic_summary="Reject without INTERNAL_INVOKE_SECRET; build SYSTEM_PRINCIPAL ctx scoped to the dispatching account; runPipeline with opts.executionId so the pre-created row is claimed rather than duplicated; return the final ExecutionResult" notable_decisions="sweepStaleExecutions is called opportunistically on entry — same fire-and-forget pattern as runPipeline itself, but on a fresh process it costs nothing"]
+// CIL[DOC-CONFIDENCE pass="first" score="0.9" flags="none"]
+// CIL[DOC-STABILITY stability="stable"]
+export const handler = async (event: any, _context: any) => {
+  const requestId = crypto.randomUUID()
+
+  if (!isDispatchAuthorized(event)) {
+    return {
+      statusCode: 403,
+      body: JSON.stringify({ error: 'Forbidden — invalid internal secret' }),
+    }
+  }
+
+  let body: any = {}
+  try {
+    body = typeof event.body === 'string' ? JSON.parse(event.body || '{}') : (event.body || {})
+  } catch {
+    return { statusCode: 400, body: JSON.stringify({ error: 'Invalid JSON body' }) }
+  }
+
+  const { pipeline_id, trigger_data, account_id, execution_id, trigger_id } = body
+
+  if (!pipeline_id) {
+    return { statusCode: 400, body: JSON.stringify({ error: 'pipeline_id is required' }) }
+  }
+
+  // Opportunistic sweep — picks up any executions orphaned by a prior dead
+  // invocation before this one starts its own work.
+  void sweepStaleExecutions().catch(() => {})
+
+  const ctx = {
+    requestId,
+    principal: {
+      ...SYSTEM_PRINCIPAL,
+      provenance: {
+        ...SYSTEM_PRINCIPAL.provenance,
+        sourceType: 'trigger' as const,
+        invokedAt: new Date().toISOString(),
+        triggerId: trigger_id || null,
+      },
+    },
+    db: adminDb,
+    accountId: account_id || null,
+    appId: null,
+    query: {},
+    requestPath: '/.netlify/functions/pipeline-runner-background',
+    headers: event.headers || {},
+  }
+
+  try {
+    const result = await runPipeline(pipeline_id, trigger_data || {}, ctx as any, {
+      executionId: execution_id,
+    })
+
+    return {
+      statusCode: 200,
+      body: JSON.stringify({
+        ok: result.status === 'completed',
+        execution_id: result.executionId,
+        status: result.status,
+      }),
+    }
+  } catch (err: any) {
+    console.error(`[${requestId}] pipeline-runner-background failed for pipeline ${pipeline_id}:`, err)
+    return {
+      statusCode: 500,
+      body: JSON.stringify({ error: err.message || 'Pipeline execution failed', execution_id }),
+    }
+  }
+}
+// CIL[END stable_id="~/.framework/functions/pipeline-runner-background.ts:handler:constant"]

package/.framework/functions/system-cron.ts

@@ -10,7 +10,7 @@
 import { createHandler, RequestContext } from './_shared/middleware'
 import { adminDb } from './_shared/db'
 import { emitAudit } from './_shared/audit'
-import { runPipeline } from './_shared/pipeline-runner'
+import { runPipeline, sweepStaleExecutions } from './_shared/pipeline-runner'
 import { retryPendingDeliveries } from './_shared/retry-deliveries'
 // CIL[DOC-VECTOR type="context" content="purpose='Imports shared middleware, database, audit, pipeline and retry helpers for the system-cron Netlify function.'; No verified external callers; imports: .framework/functions/_shared/middleware.ts,.framework/functions/_shared/db.ts,.framework/functions/_shared/audit.ts"]
 // CIL[DOC-VECTOR type="blast_radius" content="No direct caller data from CIL graph; local impact only"]
@@ -77,6 +77,15 @@ export const handler = createHandler(async (ctx: RequestContext) => {
     durationMs: number
   }> = []
   
+  // Reap executions whose runner died mid-flight (detached process killed,
+  // function timeout, dispatched trigger never started). Runs every minute
+  // regardless of whether schedules/timers are due this tick.
+  try {
+    await sweepStaleExecutions()
+  } catch (sweepError) {
+    console.warn('[system-cron] stale-execution sweep failed:', sweepError)
+  }
+
   try {
     const { data: dueSchedules, error: schedulesError } = await adminDb.rpc('get_due_schedules', {
       p_now: new Date().toISOString()