threadwire 0.1.13 → 0.1.15

package/src/run-worker.js

@@ -2,23 +2,40 @@
 
 import {spawn as nodeSpawn} from "node:child_process"
 import {constants as osConstants} from "node:os"
-import {StringDecoder} from "node:string_decoder"
+import {JsonlRecordSpool, StdoutRecordTooLargeError} from "./jsonl-record-spool.js"
 import {buildProviderEnvironment} from "./telegram-ingress/config.js"
 
-const DEFAULT_MAX_STDOUT_RECORD_BYTES = 1_048_576
+export {StdoutRecordTooLargeError}
+
+const DEFAULT_MAX_STDOUT_RECORD_BYTES = 16_777_216
+const DEFAULT_STDOUT_RECORD_MEMORY_BYTES = 1_048_576
 const DEFAULT_TERMINATION_GRACE_PERIOD_MS = 5_000
+const DEFAULT_CLEANUP_CONFIRMATION_MS = 5_000
+const CLEANUP_PROBE_INTERVAL_MS = 10
 
 /**
- * @typedef {{executable: string, arguments: string[], cwd: string, environment?: NodeJS.ProcessEnv, provider?: string, parse: (record: unknown) => import("./types.js").WorkerEvent[], onEvent: (event: import("./types.js").WorkerEvent) => void | Promise<void>, onSpawn?: (pid: number) => void, onRecord?: (record: unknown) => void | Promise<void>, onStdoutChunk?: (chunk: Buffer) => void | Promise<void>, onStderrChunk?: (chunk: Buffer) => void | Promise<void>, spawnImplementation?: typeof nodeSpawn, maxStdoutRecordBytes?: number, terminationGracePeriodMs?: number, setTimer?: (callback: () => void, milliseconds: number) => unknown, clearTimer?: (handle: unknown) => void, platform?: NodeJS.Platform, killProcess?: (pid: number, signal: NodeJS.Signals) => boolean}} RunWorkerOptions
+ * @typedef {{executable: string, arguments: string[], cwd: string, environment?: NodeJS.ProcessEnv, provider?: string, parse: (record: unknown) => import("./types.js").WorkerEvent[], onEvent: (event: import("./types.js").WorkerEvent) => void | Promise<void>, onSpawn?: (pid: number) => void, onRecord?: (record: unknown) => void | Promise<void>, onStdoutChunk?: (chunk: Buffer) => void | Promise<void>, onStderrChunk?: (chunk: Buffer) => void | Promise<void>, completion?: (record: unknown) => boolean, signal?: AbortSignal, spawnImplementation?: typeof nodeSpawn, maxStdoutRecordBytes?: number, stdoutRecordMemoryBytes?: number, spoolDirectory?: string, terminationGracePeriodMs?: number, cleanupConfirmationMs?: number, setTimer?: (callback: () => void, milliseconds: number) => unknown, clearTimer?: (handle: unknown) => void, platform?: NodeJS.Platform, killProcess?: (pid: number, signal: NodeJS.Signals) => boolean, probeProcessGroup?: (pid: number) => boolean}} RunWorkerOptions
  */
 
 /** @param {RunWorkerOptions} options @returns {Promise<number>} */
 export function runWorker(options) {
   const maxStdoutRecordBytes = positiveSafeInteger(options.maxStdoutRecordBytes ?? DEFAULT_MAX_STDOUT_RECORD_BYTES, "maxStdoutRecordBytes")
+  const stdoutRecordMemoryBytes = positiveSafeInteger(options.stdoutRecordMemoryBytes ?? DEFAULT_STDOUT_RECORD_MEMORY_BYTES, "stdoutRecordMemoryBytes")
   const terminationGracePeriodMs = positiveSafeInteger(options.terminationGracePeriodMs ?? DEFAULT_TERMINATION_GRACE_PERIOD_MS, "terminationGracePeriodMs")
+  const cleanupConfirmationMs = positiveSafeInteger(options.cleanupConfirmationMs ?? DEFAULT_CLEANUP_CONFIRMATION_MS, "cleanupConfirmationMs")
   const platform = options.platform ?? process.platform
   const spawnImplementation = options.spawnImplementation ?? nodeSpawn
   const killProcess = options.killProcess ?? ((pid, signal) => process.kill(pid, signal))
+  // Attempt-owned liveness probe: signal 0 against the exact process group.
+  // Never a name match or a broad process scan.
+  const probeProcessGroup = options.probeProcessGroup ?? ((pid) => {
+    try {
+      process.kill(-pid, 0)
+      return true
+    } catch {
+      return false
+    }
+  })
   const setTimer = options.setTimer ?? ((callback, milliseconds) => setTimeout(callback, milliseconds))
   const clearTimer = options.clearTimer ?? ((handle) => clearTimeout(/** @type {ReturnType<typeof setTimeout>} */ (handle)))
   const useDetachedProcessGroup = platform !== "win32"
@@ -29,9 +46,14 @@ export function runWorker(options) {
     shell: false,
     stdio: ["ignore", "pipe", "pipe"]
   })
-  const stdoutDecoder = new StringDecoder("utf8")
-  let stdoutBuffer = ""
-  let stdoutRecordBytes = 0
+  // Byte-oriented record spool: small pending records stay in bounded memory,
+  // oversized ones spill to one private run-scoped file, and a separate
+  // absolute cap rejects unterminated output. Closed on every settlement path.
+  const spool = new JsonlRecordSpool({
+    memoryBytes: stdoutRecordMemoryBytes,
+    maxBytes: maxStdoutRecordBytes,
+    ...(options.spoolDirectory === undefined ? {} : {directory: options.spoolDirectory})
+  })
   let stderrSeen = false
 
   return new Promise((resolve, reject) => {
@@ -47,11 +69,18 @@ export function runWorker(options) {
     let shutdownError = null
     /** @type {unknown | null} */
     let terminationTimer = null
+    let protocolComplete = false
+    /** @type {NodeJS.Signals | null} */
+    let cancellationSignal = null
     const processGroupPid = useDetachedProcessGroup && isPositiveSafeInteger(child.pid) ? child.pid : null
     /** @param {NodeJS.Signals} signal */
     const forwardSignal = (signal) => signalChildTree(child, processGroupPid, signal, killProcess)
-    const onInterrupt = () => forwardSignal("SIGINT")
-    const onTerminate = () => forwardSignal("SIGTERM")
+    const groupMembersRemain = () => processGroupPid !== null && probeProcessGroup(processGroupPid)
+    // Owned-tree liveness: where a process group exists, the exact group probe
+    // covers the child and every descendant (a reaped-away zombie stays in the
+    // group until close); where none exists (win32), only the direct child's
+    // own close proves termination.
+    const ownedTreeGone = () => processGroupPid === null ? closeSeen : !groupMembersRemain()
     const clearTerminationTimer = () => {
       if (terminationTimer === null) return
       clearTimer(terminationTimer)
@@ -62,12 +91,17 @@ export function runWorker(options) {
       child.stderr?.removeListener("data", onStderr)
       child.removeListener("spawn", onSpawn)
       child.removeListener("error", onError)
-      removeSignalHandlers(onInterrupt, onTerminate)
+      options.signal?.removeEventListener("abort", onAbort)
     }
     const cleanup = () => {
       detachForShutdown()
       clearTerminationTimer()
       child.removeListener("close", onClose)
+      spool.close()
+      // Process signal handlers stay installed through every settlement path
+      // so a real parent SIGINT/SIGTERM during cleanup can never hit the
+      // default action and kill the Threadwire parent mid-finalization.
+      removeSignalHandlers(onInterrupt, onTerminate)
     }
     /** @param {Error} error */
     const rejectAndCleanup = (error) => {
@@ -76,22 +110,101 @@ export function runWorker(options) {
       cleanup()
       reject(error)
     }
+    /**
+     * Bounded cleanup confirmation after SIGKILL: settlement waits until the
+     * attempt-owned tree is confirmed gone (exact group probe, or the direct
+     * child's close where no process group exists). If it cannot be confirmed
+     * within cleanupConfirmationMs, settlement becomes a cleanup failure —
+     * never success. This is a cleanup bound, not an idle/tool timeout.
+     * @param {() => void} settleClean @param {() => void} settleCleanupFailure
+     */
+    const confirmOwnedCleanup = (settleClean, settleCleanupFailure) => {
+      const startedAt = Date.now()
+      const confirm = () => {
+        if (settled) return
+        if (ownedTreeGone()) {
+          terminationTimer = null
+          settleClean()
+          return
+        }
+        if (Date.now() - startedAt >= cleanupConfirmationMs) {
+          terminationTimer = null
+          settleCleanupFailure()
+          return
+        }
+        terminationTimer = setTimer(confirm, CLEANUP_PROBE_INTERVAL_MS)
+      }
+      confirm()
+    }
+    /**
+     * Grace expiry shared by protocol, cancellation, and failure escalation:
+     * SIGKILL whatever of the attempt-owned tree remains, then settle only
+     * after bounded cleanup confirmation — never while owned processes still
+     * exist.
+     * @param {() => void} settleClean @param {() => void} settleCleanupFailure
+     */
+    const escalateAndConfirm = (settleClean, settleCleanupFailure) => {
+      if (settled) return
+      terminationTimer = null
+      if (!ownedTreeGone()) {
+        try {
+          forwardSignal("SIGKILL")
+        } catch {
+          /* Owned processes may already be gone. */
+        }
+      }
+      confirmOwnedCleanup(settleClean, settleCleanupFailure)
+    }
+    /**
+     * Shared failure escalation: after the grace period, SIGKILL the exact
+     * attempt-owned group and reject with the original failure once cleanup is
+     * confirmed.
+     */
+    const armFailureEscalation = () => {
+      terminationTimer = setTimer(() => {
+        if (settled || shutdownError === null) return
+        const error = shutdownError
+        escalateAndConfirm(
+          () => rejectAndCleanup(error),
+          () => rejectAndCleanup(error)
+        )
+      }, terminationGracePeriodMs)
+    }
     /** @param {Error} error @param {boolean} [terminateChild] */
     const fail = (error, terminateChild = true) => {
       if (settled || shutdownError) return
+      // Failure always replaces any pending protocol-success or cancellation
+      // escalation atomically, so an explicit failure can never lose to a
+      // success timer.
+      clearTerminationTimer()
       if (!terminateChild) {
         rejectAndCleanup(error)
         return
       }
       if (closeSeen) {
-        if (processGroupPid === null) {
-          try {
-            forwardSignal("SIGTERM")
-          } catch {
-            /* Child may already be gone. */
+        if (ownedTreeGone()) {
+          // Safe post-close path: no surviving attempt-owned group, so no
+          // signal is sent (win32 still terminates the direct child handle).
+          if (processGroupPid === null) {
+            try {
+              forwardSignal("SIGTERM")
+            } catch {
+              /* Child may already be gone. */
+            }
           }
+          rejectAndCleanup(error)
+          return
         }
-        rejectAndCleanup(error)
+        // The direct child closed but attempt-owned descendants survive:
+        // terminate and escalate before rejecting the original failure.
+        shutdownError = error
+        detachForShutdown()
+        try {
+          forwardSignal("SIGTERM")
+        } catch {
+          /* Group members may already be gone. */
+        }
+        armFailureEscalation()
         return
       }
       shutdownError = error
@@ -101,17 +214,102 @@ export function runWorker(options) {
       } catch {
         /* Child may already be gone. */
       }
-      terminationTimer = setTimer(() => {
-        if (settled || shutdownError === null) return
-        terminationTimer = null
-        try {
-          forwardSignal("SIGKILL")
-        } catch {
-          /* Child may already be gone. */
-        }
-        rejectAndCleanup(shutdownError)
-      }, terminationGracePeriodMs)
+      armFailureEscalation()
+    }
+    /** Explicit caller cancellation/deadline: failure semantics, never success. */
+    const onAbort = () => {
+      if (cancellationSignal !== null) return
+      const reason = options.signal?.aborted ? options.signal.reason : undefined
+      fail(reason instanceof Error ? reason : new Error("Worker run aborted"))
+    }
+    const settleProtocolSuccess = () => {
+      // Protocol success can never settle over an explicit failure or abort.
+      if (settled || shutdownError) return
+      settled = true
+      cleanup()
+      resolve(0)
     }
+    /**
+     * Grace-period escalation after protocol completion. Success settles only
+     * once the attempt-owned tree is confirmed gone after SIGKILL; if cleanup
+     * cannot be confirmed within the bounded window, the run settles as a
+     * cleanup failure — never success.
+     */
+    const escalateProtocolTermination = () => {
+      escalateAndConfirm(
+        settleProtocolSuccess,
+        () => rejectAndCleanup(new CleanupConfirmationError("protocol completion"))
+      )
+    }
+    /**
+     * The provider protocol confirmed a completed run: stop consuming output
+     * and terminate the attempt-owned process tree instead of waiting for the
+     * provider to exit on its own. Success resolves exactly once with status 0;
+     * a provider or descendant that ignores SIGTERM is escalated and never
+     * leaked.
+     */
+    const completeProtocol = () => {
+      if (settled || shutdownError || protocolComplete) return
+      protocolComplete = true
+      child.stdout?.removeListener("data", onStdout)
+      child.stderr?.removeListener("data", onStderr)
+      // Process signal handlers deliberately stay installed: a parent
+      // SIGINT/SIGTERM during protocol settlement must be handled by onCancel,
+      // never hit the default action.
+      if (ownedTreeGone()) return
+      try {
+        forwardSignal("SIGTERM")
+      } catch {
+        /* Child may already be gone. */
+      }
+      terminationTimer = setTimer(escalateProtocolTermination, terminationGracePeriodMs)
+    }
+    const settleCancellation = () => {
+      if (settled || shutdownError) return
+      settled = true
+      cleanup()
+      resolve(signalExitCode(cancellationSignal))
+    }
+    /**
+     * Grace-period escalation for explicit parent cancellation: settle with the
+     * original signal-derived status only once the attempt-owned tree is
+     * confirmed gone, SIGKILLing the exact group when members survive and
+     * failing cleanup (never succeeding) when confirmation times out.
+     */
+    const escalateCancellation = () => {
+      escalateAndConfirm(
+        settleCancellation,
+        () => rejectAndCleanup(new CleanupConfirmationError(`cancellation (${String(cancellationSignal)})`))
+      )
+    }
+    /**
+     * Parent SIGINT/SIGTERM is explicit cancellation owned by Threadwire: stop
+     * consuming provider output, forward the signal to the exact attempt-owned
+     * process group, and arm the grace/escalation timer. During the
+     * protocol-complete settlement window it atomically replaces the pending
+     * protocol-success escalation: explicit cancellation always keeps the
+     * forwarded signal's exit semantics (never protocol success), settles
+     * exactly once, and cleans the owned tree first. After close (without
+     * protocol completion) it stays the safe no-signal no-op.
+     * @param {NodeJS.Signals} signal
+     */
+    const onCancel = (signal) => {
+      if (settled || shutdownError || cancellationSignal !== null) return
+      if (!protocolComplete && closeSeen) return
+      cancellationSignal = signal
+      // Atomically replace any pending protocol-success escalation timer.
+      clearTerminationTimer()
+      child.stdout?.removeListener("data", onStdout)
+      child.stderr?.removeListener("data", onStderr)
+      try {
+        forwardSignal(signal)
+      } catch {
+        /* Child may already be gone. */
+      }
+      terminationTimer = setTimer(escalateCancellation, terminationGracePeriodMs)
+    }
+    const onInterrupt = () => onCancel("SIGINT")
+    const onTerminate = () => onCancel("SIGTERM")
     /**
      * @param {() => Promise<void>} task
      * @param {NodeJS.ReadableStream | undefined} stream
@@ -120,7 +318,7 @@ export function runWorker(options) {
       stream?.pause()
       consumer = consumer
         .then(async () => {
-          if (settled || shutdownError) return
+          if (settled || shutdownError || protocolComplete || cancellationSignal !== null) return
           await task()
         })
         .then(() => {
@@ -133,18 +331,21 @@ export function runWorker(options) {
     }
     /** @param {Buffer | string} chunk */
     const onStdout = (chunk) => {
-      if (settled || shutdownError) return
+      if (settled || shutdownError || protocolComplete || cancellationSignal !== null) return
       enqueueConsumer(async () => {
         const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
         await options.onStdoutChunk?.(bytes)
-        stdoutRecordBytes = nextRecordByteCount(bytes, stdoutRecordBytes, maxStdoutRecordBytes)
-        stdoutBuffer += stdoutDecoder.write(bytes)
-        stdoutBuffer = await drainLines(stdoutBuffer, options.parse, options.onEvent, options.onRecord)
+        for (const record of spool.push(bytes)) {
+          // Once completion fires, stop consuming this chunk immediately:
+          // trailing records must never emit duplicate output or consumer
+          // failures.
+          if (await parseLine(record, options.parse, options.onEvent, options.onRecord, options.completion, completeProtocol)) break
+        }
       }, child.stdout)
     }
     /** @param {Buffer | string} chunk */
     const onStderr = (chunk) => {
-      if (settled || shutdownError) return
+      if (settled || shutdownError || protocolComplete || cancellationSignal !== null) return
       enqueueConsumer(
         async () => {
           await options.onStderrChunk?.(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))
@@ -176,24 +377,52 @@ export function runWorker(options) {
       }
       try {
         await consumer
-        if (settled) return
-        stdoutBuffer += stdoutDecoder.end()
-        if (stdoutBuffer.trim().length > 0) await parseLine(stdoutBuffer, options.parse, options.onEvent, options.onRecord)
+        // A consumer failure that landed during the wait owns settlement
+        // through its armed escalation; never resolve over it.
+        if (settled || shutdownError !== null) return
+        const tail = spool.end()
+        if (tail.trim().length > 0) {
+          await parseLine(tail, options.parse, options.onEvent, options.onRecord, options.completion, completeProtocol)
+        }
       } catch (error) {
         fail(toError(error), false)
         return
       }
+      // Protocol completion with surviving group members defers settlement to
+      // the escalation timer started by completeProtocol.
+      if (protocolComplete && !ownedTreeGone()) return
       settled = true
       cleanup()
-      resolve(closeCode ?? signalExitCode(closeSignal))
+      resolve(protocolComplete ? 0 : (closeCode ?? signalExitCode(closeSignal)))
     }
     /** @param {number | null} code @param {NodeJS.Signals | null} signal */
     const onClose = (code, signal) => {
       if (settled) return
       closeSeen = true
-      removeSignalHandlers(onInterrupt, onTerminate)
       closeCode = code
       closeSignal = signal
+      // An explicit failure (including an abort after protocol completion)
+      // always outranks protocol success and cancellation at close.
+      if (shutdownError !== null) {
+        // Surviving owned processes retain the escalation timer so SIGKILL
+        // still reaches the exact group before rejection.
+        if (!ownedTreeGone()) return
+        clearTerminationTimer()
+        void finalizeClose()
+        return
+      }
+      // Explicit cancellation settles with the forwarded signal's status once
+      // the owned tree is gone; surviving members keep the escalation alive.
+      if (cancellationSignal !== null) {
+        if (ownedTreeGone()) settleCancellation()
+        return
+      }
+      if (protocolComplete) {
+        // Settle promptly only when the entire attempt-owned tree is gone;
+        // otherwise the retained grace/escalation timer owns settlement.
+        if (ownedTreeGone()) settleProtocolSuccess()
+        return
+      }
       clearTerminationTimer()
       void finalizeClose()
     }
@@ -202,15 +431,20 @@ export function runWorker(options) {
     child.once("spawn", onSpawn)
     child.once("error", onError)
     child.once("close", onClose)
-    process.once("SIGINT", onInterrupt)
-    process.once("SIGTERM", onTerminate)
+    // Persistent until final cleanup: handlers must outlive every settlement
+    // window so a real parent signal can never hit the default action.
+    process.on("SIGINT", onInterrupt)
+    process.on("SIGTERM", onTerminate)
+    options.signal?.addEventListener("abort", onAbort, {once: true})
+    if (options.signal?.aborted) onAbort()
   })
 }
 
-export class StdoutRecordTooLargeError extends Error {
-  constructor() {
-    super("Worker stdout record exceeded the configured byte capacity")
-    this.name = "StdoutRecordTooLargeError"
+export class CleanupConfirmationError extends Error {
+  /** @param {string} context */
+  constructor(context) {
+    super(`Worker cleanup after ${context} could not be confirmed within the bounded confirmation window`)
+    this.name = "CleanupConfirmationError"
   }
 }
 
@@ -225,23 +459,15 @@ export function childEnvironment(environment = process.env, provider) {
   return childEnvironment
 }
 
-/** @param {string} buffer @param {RunWorkerOptions["parse"]} parse @param {RunWorkerOptions["onEvent"]} onEvent @param {RunWorkerOptions["onRecord"]} onRecord */
-async function drainLines(buffer, parse, onEvent, onRecord) {
-  const lines = buffer.split("\n")
-  const tail = lines.pop() ?? ""
-  for (const line of lines) await parseLine(line, parse, onEvent, onRecord)
-  return tail
-}
-
-/** @param {string} line @param {RunWorkerOptions["parse"]} parse @param {RunWorkerOptions["onEvent"]} onEvent @param {RunWorkerOptions["onRecord"]} onRecord */
-async function parseLine(line, parse, onEvent, onRecord) {
-  if (line.trim().length === 0) return
+/** @param {string} line @param {RunWorkerOptions["parse"]} parse @param {RunWorkerOptions["onEvent"]} onEvent @param {RunWorkerOptions["onRecord"]} onRecord @param {RunWorkerOptions["completion"]} completion @param {() => void} onProtocolComplete @returns {Promise<boolean>} whether protocol completion fired */
+async function parseLine(line, parse, onEvent, onRecord, completion, onProtocolComplete) {
+  if (line.trim().length === 0) return false
   let record
   try {
     record = JSON.parse(line)
   } catch {
     await onEvent({type: "diagnostic", level: "warning", summary: "Worker emitted an unreadable event"})
-    return
+    return false
   }
   await onRecord?.(record)
   let events
@@ -249,9 +475,14 @@ async function parseLine(line, parse, onEvent, onRecord) {
     events = parse(record)
   } catch {
     await onEvent({type: "diagnostic", level: "warning", summary: "Worker emitted an unreadable event"})
-    return
+    return false
   }
   for (const event of events) await onEvent(event)
+  if (completion?.(record) === true) {
+    onProtocolComplete()
+    return true
+  }
+  return false
 }
 
 /** @param {NodeJS.Signals | null} signal */
@@ -282,22 +513,6 @@ function toError(error) {
   return error instanceof Error ? error : new Error("Worker event consumer failed")
 }
 
-/** @param {Buffer} bytes @param {number} currentBytes @param {number} maximumBytes */
-function nextRecordByteCount(bytes, currentBytes, maximumBytes) {
-  let recordBytes = currentBytes
-  let segmentStart = 0
-  for (let index = 0; index < bytes.length; index += 1) {
-    if (bytes[index] !== 0x0a) continue
-    recordBytes += index - segmentStart
-    if (recordBytes > maximumBytes) throw new StdoutRecordTooLargeError()
-    recordBytes = 0
-    segmentStart = index + 1
-  }
-  recordBytes += bytes.length - segmentStart
-  if (recordBytes > maximumBytes) throw new StdoutRecordTooLargeError()
-  return recordBytes
-}
-
 /** @param {number} value @param {string} name */
 function positiveSafeInteger(value, name) {
   if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive safe integer`)

package/src/telegram-ingress/core.js

@@ -1,6 +1,7 @@
 // @ts-check
 
 import {createHash, randomUUID, timingSafeEqual} from "node:crypto"
+import {abortable} from "../absolute-deadline.js"
 import {evidenceTelegramDestination} from "../evidence-store.js"
 import {NoticeQueue} from "../notice-queue.js"
 import {createFetchTransport} from "../notifiers/fetch-transport.js"
@@ -30,8 +31,8 @@ import {parseCodeCommand, parseEvidenceCommand} from "./command.js"
  *   kimiBinding?: unknown,
  *   activity?: Pick<import("../activity-log.js").ActivityLog, "recordStarted" | "recordSession" | "close">,
  *   evidenceStore?: import("../evidence-store.js").EvidenceStore,
- *   isolatedRuntimeClient?: Pick<import("../isolated-runtime-client.js").IsolatedRuntimeClient, "preflight" | "run">,
- *   kimiIsolatedRuntimeClient?: Pick<import("../isolated-runtime-client.js").IsolatedRuntimeClient, "preflight" | "run">,
+ *   isolatedRuntimeClient?: Pick<import("../isolated-runtime-client.js").IsolatedRuntimeClient, "preflight" | "startRun">,
+ *   kimiIsolatedRuntimeClient?: Pick<import("../isolated-runtime-client.js").IsolatedRuntimeClient, "preflight" | "startRun">,
  *   onWorkerFailure?: (message: string) => void,
  *   onWorkerSettled?: () => void
  * }} IngressDependencies
@@ -127,30 +128,44 @@ export async function dispatchWorker(job, config, dependencies = {}) {
     })
     : undefined
 
-  const provider = createProviderImpl(job.provider, [], job.prompt, undefined, providerEnvironment)
-  const transport = createFetchTransportImpl(config.botToken, undefined, config.telegramRequestTimeoutMs)
-  const sender = createTelegramSenderImpl(job.target, transport)
-  const control = new WorkerControlImpl({
-    sender,
-    processNumber,
-    toolMessages: config.toolMessages ?? false,
-    NoticeQueueClass: NoticeQueueImpl,
-    RelayClass: RelayImpl
-  })
-  const evidenceOwner = dependencies.evidenceStore?.createOwnerScope({
-    destinationId: evidenceTelegramDestination(job.target, job.senderId),
-    runId: randomUUID()
-  })
-  const evidence = evidenceOwner === undefined ? undefined : await dependencies.evidenceStore?.createArtifact(evidenceOwner, {
-    contentType: "text/plain; charset=utf-8",
-    redactions: await collectEvidenceRedactions(dependencies.providerEnvironment ?? {})
-  })
+  /** @type {ReturnType<typeof createProvider>} */
+  let provider
+  /** @type {ReturnType<typeof createTelegramSender>} */
+  let sender
+  /** @type {WorkerControl} */
+  let control
+  /** @type {Awaited<ReturnType<import("../evidence-store.js").EvidenceStore["createArtifact"]>> | undefined} */
+  let evidence
+  try {
+    provider = createProviderImpl(job.provider, [], job.prompt, undefined, providerEnvironment)
+    const transport = createFetchTransportImpl(config.botToken, undefined, config.telegramRequestTimeoutMs)
+    sender = createTelegramSenderImpl(job.target, transport)
+    control = new WorkerControlImpl({
+      sender,
+      processNumber,
+      toolMessages: config.toolMessages ?? false,
+      NoticeQueueClass: NoticeQueueImpl,
+      RelayClass: RelayImpl
+    })
+    const evidenceOwner = dependencies.evidenceStore?.createOwnerScope({
+      destinationId: evidenceTelegramDestination(job.target, job.senderId),
+      runId: randomUUID()
+    })
+    evidence = evidenceOwner === undefined ? undefined : await dependencies.evidenceStore?.createArtifact(evidenceOwner, {
+      contentType: "text/plain; charset=utf-8",
+      redactions: await collectEvidenceRedactions(dependencies.providerEnvironment ?? {})
+    })
+  } catch (error) {
+    earlyKimiPreflight?.deadline?.close()
+    throw error
+  }
   let evidenceTransferred = false
   try {
     await evidence?.append("prompt", `prompt\n${job.prompt}\nprovider-stream\n`)
 
     if (isolatedRuntimeClient !== undefined) {
       evidenceTransferred = true
+      let isolatedDeadline = earlyKimiPreflight?.deadline
       try {
         const preflight = earlyKimiPreflight ?? await isolatedRuntimeClient.preflight({
           provider: job.provider,
@@ -160,7 +175,8 @@ export async function dispatchWorker(job, config, dependencies = {}) {
             : {repositoryRoot: cwd, cwd}),
           providerArguments: []
         })
-        const exitCode = await isolatedRuntimeClient.run({
+        isolatedDeadline = preflight.deadline
+        const isolatedRun = isolatedRuntimeClient.startRun({
           preflightId: preflight.preflightId,
           prompt: job.prompt,
           providerArguments: [],
@@ -175,6 +191,13 @@ export async function dispatchWorker(job, config, dependencies = {}) {
             onStderrChunk: (chunk) => evidence?.append("provider-stderr", chunk)
           })
         })
+        let exitCode
+        try {
+          exitCode = await abortable(isolatedRun.completion, preflight.deadline?.signal)
+        } catch (error) {
+          await isolatedRun.cancel(error instanceof Error ? error : new Error("Isolated runtime run failed"))
+          throw error
+        }
         if (exitCode !== 0) throw new Error(`Isolated ${job.provider === "kimi" ? "Kimi" : "Codex"} worker exited with status ${exitCode}`)
         await control.close()
         if (evidence !== undefined) {
@@ -187,6 +210,7 @@ export async function dispatchWorker(job, config, dependencies = {}) {
         await evidence?.abort()
         throw error
       } finally {
+        isolatedDeadline?.close()
         dependencies.onWorkerSettled?.()
       }
     }
@@ -235,6 +259,7 @@ export async function dispatchWorker(job, config, dependencies = {}) {
       environment: providerEnvironment,
       provider: provider.name,
       parse: provider.parse,
+      ...(provider.completion === undefined ? {} : {completion: provider.completion}),
       onEvent: async (event) => control.accept(event),
       onSpawn: (pid) => {
         if (pid !== undefined) dependencies.activity?.recordStarted(provider.name, pid)
@@ -318,7 +343,11 @@ export async function dispatchWorker(job, config, dependencies = {}) {
 
     await spawned
   } finally {
-    if (!evidenceTransferred) await evidence?.abort()
+    try {
+      if (!evidenceTransferred) await evidence?.abort()
+    } finally {
+      earlyKimiPreflight?.deadline?.close()
+    }
   }
 }