threadwire 0.1.25 → 0.1.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,9 +5,10 @@ import {randomBytes} from "node:crypto"
5
5
  import {once} from "node:events"
6
6
  import {lstat, realpath} from "node:fs/promises"
7
7
  import {createServer} from "node:http"
8
+ import {hostname} from "node:os"
8
9
  import {isAbsolute, join, normalize, relative} from "node:path"
9
10
  import {DockerApi, DockerApiError} from "./docker-api.js"
10
- import {buildKimiBindingValidatorSpec, buildKimiRelayContainerSpec, buildWorkerContainerSpec, isDigestImage} from "./isolated-worker.js"
11
+ import {buildGitMetadataSnapshotSpec, buildKimiBindingValidatorSpec, buildKimiRelayContainerSpec, buildKimiSourceSnapshotSpec, buildWorkerContainerSpec, isDigestImage} from "./isolated-worker.js"
11
12
  import {validateRelayWriteProviderArguments} from "./relay-write.js"
12
13
  import {codexSessionId, parseCodexEvent} from "./providers/codex.js"
13
14
  import {kimiSessionEnvelopeId, validateKimiProviderArguments} from "./providers/kimi.js"
@@ -15,7 +16,10 @@ import {parseApprovedKimiModels} from "./kimi-model-broker-policy.js"
15
16
  import {parseThreadwireBinding, resumeTaskIdentity, threadwireBindingDigest} from "./threadwire-binding.js"
16
17
  import {openStateRegistry, validateStateVolume} from "./isolated-state.js"
17
18
  import {assertNoNestedMounts} from "./mount-policy.js"
18
- import {AbsoluteDeadline, deadlineAfter} from "./absolute-deadline.js"
19
+ import {MUTATION_CAPABILITIES, parseMutationPolicy} from "./mutation-policy.js"
20
+ import {gitSnapshot, snapshotMutations, reconcileMutationResult} from "./mutation-snapshot.js"
21
+ import {AbsoluteDeadline, abortable, deadlineAfter} from "./absolute-deadline.js"
22
+ import {exactKeys, isRecord} from "./record-helpers.js"
19
23
 
20
24
  const MAX_PREFLIGHT_BYTES = 32_768
21
25
  const MAX_RUN_BYTES = 1_048_576
@@ -128,7 +132,8 @@ export async function startIsolatedRuntime(options = {}) {
128
132
  }
129
133
  if (request.method === "POST" && request.url === "/preflight") {
130
134
  const releaseIncoming = incomingRequests.acquire("preflight")
131
- const launchDeadlineAt = requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)
135
+ const launchDeadlineHeader = request.headers["x-threadwire-launch-deadline"]
136
+ const launchDeadlineAt = requestedDeadline(launchDeadlineHeader, workerTimeoutMs)
132
137
  const operation = new AbsoluteDeadline(minimumDeadline(Date.now() + preflightTimeoutMs, launchDeadlineAt), {
133
138
  signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime operation timed out", disconnectMessage: "Isolated runtime caller disconnected"
134
139
  })
@@ -149,8 +154,11 @@ export async function startIsolatedRuntime(options = {}) {
149
154
  try { validateStateVolume(volume, state.volume, state.labels) } catch { throw new ClientError(`${providerName(provider)} resume state is unavailable`) }
150
155
  } else state = stateRegistry.allocate(task)
151
156
  const preflightId = randomBytes(24).toString("base64url")
152
- preflights.add(preflightId, task, {...selection, provider, providerSelection, state, fingerprint, worktreeRoot, ...(launchDeadlineAt === undefined ? {} : {launchDeadlineAt})})
153
- sendJson(response, 200, {ok: true, preflightId, ...(launchDeadlineAt === undefined ? {} : {launchDeadlineAt})})
157
+ // Only an explicitly requested launch deadline is carried into the run;
158
+ // the workerTimeoutMs fallback caps only the preflight operation itself.
159
+ const explicitLaunchDeadline = launchDeadlineHeader === undefined ? undefined : launchDeadlineAt
160
+ preflights.add(preflightId, task, {...selection, provider, providerSelection, state, fingerprint, worktreeRoot, ...(explicitLaunchDeadline === undefined ? {} : {launchDeadlineAt: explicitLaunchDeadline})})
161
+ sendJson(response, 200, {ok: true, preflightId, ...(explicitLaunchDeadline === undefined ? {} : {launchDeadlineAt: explicitLaunchDeadline})})
154
162
  } finally {
155
163
  operation.close()
156
164
  releaseIncoming()
@@ -178,8 +186,13 @@ export async function startIsolatedRuntime(options = {}) {
178
186
  const operationSignal = runLifecycle === undefined
179
187
  ? shutdown.signal
180
188
  : AbortSignal.any([shutdown.signal, runLifecycle.signal])
181
- const operation = new AbsoluteDeadline(minimumDeadline(preflight.launchDeadlineAt,
182
- requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)), {
189
+ const runHeaderDeadline = request.headers["x-threadwire-launch-deadline"] === undefined
190
+ ? undefined
191
+ : requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)
192
+ const launchDeadlineAt = preflight.launchDeadlineAt === undefined
193
+ ? runHeaderDeadline
194
+ : (runHeaderDeadline === undefined ? preflight.launchDeadlineAt : Math.min(preflight.launchDeadlineAt, runHeaderDeadline))
195
+ const operation = new AbsoluteDeadline(launchDeadlineAt, {
183
196
  signal: operationSignal, request, response, timeoutMessage: "Isolated runtime launch timed out", disconnectMessage: "Isolated runtime caller disconnected",
184
197
  destroyRequestOnAbort: runLifecycle === undefined
185
198
  })
@@ -264,7 +277,7 @@ async function verifyPrerequisites(options) {
264
277
  if (typeof image.Id !== "string" || !isDigestImage(image.Id)) throw new Error("Worker image unavailable")
265
278
  const validatedWorktree = selection.provider === "kimi"
266
279
  ? await verifyKimiBindingAdmission(docker, selection.binding, operation.options())
267
- : await validateWorktreeSelection(selection.repositoryRoot, selection.cwd, allowedRoots)
280
+ : await validateWorktreeSelection(selection.repositoryRoot, selection.cwd, allowedRoots, selection.mutationPolicy)
268
281
  const health = await abortable(fetchImplementation(new URL("/healthz", brokerUrl), {
269
282
  method: "GET",
270
283
  redirect: "error",
@@ -320,11 +333,11 @@ async function verifyKimiBindingAdmission(docker, value, requestOptions) {
320
333
  return {fingerprint: [], root: binding.source.target}
321
334
  }
322
335
 
323
- export async function validateWorktree(repositoryRoot, cwd, allowedRoots) {
324
- return (await validateWorktreeSelection(repositoryRoot, cwd, allowedRoots)).fingerprint
336
+ export async function validateWorktree(repositoryRoot, cwd, allowedRoots, mutationPolicy) {
337
+ return (await validateWorktreeSelection(repositoryRoot, cwd, allowedRoots, mutationPolicy)).fingerprint
325
338
  }
326
339
 
327
- export async function validateWorktreeSelection(repositoryRoot, cwd, allowedRoots) {
340
+ export async function validateWorktreeSelection(repositoryRoot, cwd, allowedRoots, mutationPolicy) {
328
341
  const root = allowedRoots
329
342
  .filter((allowed) => within(allowed, repositoryRoot))
330
343
  .sort((first, second) => second.length - first.length)[0]
@@ -347,8 +360,16 @@ export async function validateWorktreeSelection(repositoryRoot, cwd, allowedRoot
347
360
  await assertNoNestedMounts(repositoryRoot)
348
361
  try {
349
362
  const gitEntry = await lstat(join(repositoryRoot, ".git"))
350
- if (gitEntry.isDirectory()) throw new ClientError("Common Git directories cannot enter the worker")
351
- if (!gitEntry.isFile() || gitEntry.isSymbolicLink()) throw new ClientError("Configured worktree is unavailable")
363
+ if (gitEntry.isDirectory()) {
364
+ // An explicit mutation policy is what authorizes a .git directory to
365
+ // enter the worker; the per-run policy is then enforced at container
366
+ // creation time (read-only worktree/.git mounts when denied). Without a
367
+ // policy the historical behavior is preserved: common Git directories are
368
+ // rejected to avoid leaking repository metadata into an unbounded run.
369
+ if (mutationPolicy === undefined) throw new ClientError("Common Git directories cannot enter the worker")
370
+ } else if (!gitEntry.isFile() || gitEntry.isSymbolicLink()) {
371
+ throw new ClientError("Configured worktree is unavailable")
372
+ }
352
373
  } catch (error) {
353
374
  if (error instanceof ClientError) throw error
354
375
  if (/** @type {NodeJS.ErrnoException} */ (error).code !== "ENOENT") throw error
@@ -382,6 +403,8 @@ export async function runIsolatedWorker(options) {
382
403
  let relayId
383
404
  let relayContainerName
384
405
  let containerId
406
+ let gitSnapshotContainerId
407
+ let gitMetadataVolumeName = undefined
385
408
  let grantToken
386
409
  let grantRenewalTimer
387
410
  let runTracked = false
@@ -509,6 +532,41 @@ export async function runIsolatedWorker(options) {
509
532
  const worktreeMetadata = provider === "kimi" ? {ino: "0"} : repository.find((entry) => entry.path === options.preflight.repositoryRoot)
510
533
  const cwdMetadata = provider === "kimi" ? {ino: "0"} : repository.find((entry) => entry.path === options.preflight.cwd)
511
534
  if (worktreeMetadata === undefined || cwdMetadata === undefined) throw new Error("Worktree fingerprint unavailable")
535
+ const mutationPolicy = options.preflight.mutationPolicy
536
+ const useGitMetadataVolume = provider === "codex" && options.worktreeVolume !== undefined && mutationPolicy !== undefined
537
+ && mutationPolicy.worktreeEdit === true && mutationPolicy.commit === false
538
+ let gitMetadataVolumeName
539
+ if (useGitMetadataVolume) {
540
+ runStage = "git-metadata-volume"
541
+ gitMetadataVolumeName = `${runId}-git`
542
+ await options.docker.createVolume(gitMetadataVolumeName, {...resourceLabels, "org.threadwire.role": "git-metadata-snapshot"}, operation.options())
543
+ runStage = "git-metadata-snapshot"
544
+ const snapshotImage = await resolveRuntimeImage(options.docker, operation)
545
+ const snapshotSpec = buildGitMetadataSnapshotSpec({
546
+ image: snapshotImage,
547
+ sourceVolume: options.worktreeVolume,
548
+ sourceSubpath: worktreeSubpath,
549
+ targetVolume: gitMetadataVolumeName,
550
+ name: `${runId}-git-snapshot`,
551
+ labels: resourceLabels
552
+ })
553
+ const snapshotContainer = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(`${runId}-git-snapshot`, snapshotSpec, operation.options()))
554
+ if (typeof snapshotContainer.Id !== "string") throw new Error("Git metadata snapshot helper creation failed")
555
+ gitSnapshotContainerId = snapshotContainer.Id
556
+ try {
557
+ await validateWorkerMountInventory(await options.docker.inspectContainer(gitSnapshotContainerId, operation.options()), snapshotSpec.HostConfig.Mounts)
558
+ await options.docker.startContainer(gitSnapshotContainerId, operation.options())
559
+ const snapshotWait = /** @type {{StatusCode?: unknown}} */ (await options.docker.waitContainer(gitSnapshotContainerId, operation.options()))
560
+ if (snapshotWait.StatusCode !== 0) {
561
+ const statusCode = Number(snapshotWait.StatusCode)
562
+ const category = statusCode === 1 ? "helper-logic" : statusCode === 2 ? "helper-runtime" : "helper-unknown"
563
+ process.stderr.write(`threadwire-runtime: git metadata snapshot helper exited ${statusCode} (${category})\n`)
564
+ throw new Error(`Git metadata snapshot helper failed (${category})`)
565
+ }
566
+ } finally {
567
+ await cleanupDocker(() => options.docker.removeContainer(gitSnapshotContainerId, operation.options())).catch(() => {})
568
+ }
569
+ }
512
570
  const spec = buildWorkerContainerSpec({
513
571
  provider,
514
572
  image: options.workerImage,
@@ -536,7 +594,9 @@ export async function runIsolatedWorker(options) {
536
594
  subpath: worktreeSubpath
537
595
  }
538
596
  }),
539
- ...(options.resumeSession === undefined ? {} : {resumeSession: options.resumeSession})
597
+ ...(gitMetadataVolumeName === undefined ? {} : {gitOverlayVolume: {name: gitMetadataVolumeName}}),
598
+ ...(options.resumeSession === undefined ? {} : {resumeSession: options.resumeSession}),
599
+ ...(options.preflight.mutationPolicy === undefined ? {} : {mutationPolicy: options.preflight.mutationPolicy})
540
600
  })
541
601
  runStage = "worker-create"
542
602
  const container = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(`${runId}-worker`, spec, operation.options()))
@@ -553,6 +613,23 @@ export async function runIsolatedWorker(options) {
553
613
  headers: {authorization: `Bearer ${options.brokerAdminToken}`}
554
614
  }), operation.signal)
555
615
  if (!activationResponse.ok) throw new Error("Model broker grant activation failed")
616
+ runStage = "snapshot-before"
617
+ const snapshotRepository = provider === "codex" ? options.preflight.repositoryRoot : undefined
618
+ const beforeSnapshot = mutationPolicy !== undefined
619
+ ? await captureMutationSnapshot({
620
+ provider,
621
+ docker: options.docker,
622
+ repositoryRoot: snapshotRepository,
623
+ binding: options.preflight.binding,
624
+ workerImage: options.workerImage,
625
+ runId,
626
+ namespace: options.stateRegistry.namespace,
627
+ lineage: state.lineage,
628
+ taskHash: state.labels["org.threadwire.task"],
629
+ when: "before",
630
+ operation
631
+ })
632
+ : undefined
556
633
  const leaseMs = grantLease(options.grantLeaseMs)
557
634
  let renewing = false
558
635
  let renewalConfirmed = false
@@ -611,11 +688,13 @@ export async function runIsolatedWorker(options) {
611
688
  rawChunks = provider === "kimi" ? [] : chunks.map((chunk) => ({channel: chunk.channel, data: chunk.data.toString("base64")}))
612
689
  if (provider === "kimi") for (const record of records) await publishRecord?.(record)
613
690
  }
614
- const result = {
691
+ let result = {
615
692
  exitCode: typeof wait.StatusCode === "number" ? wait.StatusCode : 1,
616
693
  records,
617
694
  rawChunks
618
695
  }
696
+ const mutationClaimResult = extractMutationClaim(result.records)
697
+ result.records = mutationClaimResult.records
619
698
  let sessionIds = [...new Set(result.records.filter((record) => record.type === "session").map((record) => record.sessionId))]
620
699
  if (provider === "kimi" && result.exitCode !== 0) {
621
700
  result.records = result.records.filter((record) => record.type !== "session")
@@ -628,6 +707,42 @@ export async function runIsolatedWorker(options) {
628
707
  throw new Error("Kimi worker did not preserve the exact resume session")
629
708
  }
630
709
  }
710
+ if (beforeSnapshot !== undefined) {
711
+ runStage = "snapshot-after"
712
+ const afterSnapshot = await captureMutationSnapshot({
713
+ provider,
714
+ docker: options.docker,
715
+ repositoryRoot: snapshotRepository,
716
+ binding: options.preflight.binding,
717
+ workerImage: options.workerImage,
718
+ runId,
719
+ namespace: options.stateRegistry.namespace,
720
+ lineage: state.lineage,
721
+ taskHash: state.labels["org.threadwire.task"],
722
+ when: "after",
723
+ operation
724
+ })
725
+ const observed = snapshotMutations(beforeSnapshot, afterSnapshot)
726
+ const reconciliation = reconcileMutationResult({
727
+ policy: /** @type {import("./mutation-policy.js").MutationPolicy} */ (mutationPolicy),
728
+ observed,
729
+ providerExitCode: result.exitCode,
730
+ providerClaim: mutationClaimResult.claim
731
+ })
732
+ result = {
733
+ ...result,
734
+ exitCode: reconciliation.exitCode,
735
+ observedMutations: reconciliation.observedMutations
736
+ }
737
+ if (reconciliation.blocker !== undefined) {
738
+ result.mutationBlocker = reconciliation.blocker
739
+ }
740
+ if (provider === "kimi" && result.exitCode !== 0) {
741
+ result.records = result.records.filter((record) => record.type !== "session")
742
+ sessionIds = []
743
+ deferredSessions.length = 0
744
+ }
745
+ }
631
746
  if (freshState && result.exitCode === 0) {
632
747
  if (sessionIds.length === 0) throw new Error(`${providerName(provider)} session state was not established`)
633
748
  await options.stateRegistry.register(sessionIds, state, options.sessionTtlMs, operation.signal)
@@ -654,9 +769,17 @@ export async function runIsolatedWorker(options) {
654
769
  throw error
655
770
  }
656
771
  }
772
+ if (result.observedMutations !== undefined) {
773
+ await options.onRecord?.({
774
+ type: "observed_mutations",
775
+ mutations: result.observedMutations,
776
+ ...(result.mutationBlocker === undefined ? {} : {blocker: result.mutationBlocker})
777
+ })
778
+ }
657
779
  workerResult = result
658
780
  } catch (error) {
659
- process.stderr.write(`threadwire-runtime: isolated run failed at ${runStage}\n`)
781
+ const message = error instanceof Error ? error.message : "unknown"
782
+ process.stderr.write(`threadwire-runtime: isolated run failed at ${runStage}: ${message}\n`)
660
783
  runError = error
661
784
  } finally {
662
785
  let resourcesRemoved = true
@@ -668,7 +791,11 @@ export async function runIsolatedWorker(options) {
668
791
  ? deadlineAfter(5_000, {timeoutMessage: "Isolated runtime emergency cleanup timeout"})
669
792
  : undefined
670
793
  const cleanupOperation = emergencyCleanup ?? operation
671
- const cleanupOptions = () => cleanupOperation.options()
794
+ const cleanupOptions = cleanupOperation.options()
795
+ // Caller/deadline aborts cannot authorize further graceful shutdown. Force
796
+ // an immediate KILL for the worker container so the bounded emergency
797
+ // cleanup window is not consumed by a TERM grace period.
798
+ const workerCleanupOptions = {...cleanupOptions, forceKill: operation.signal.aborted}
672
799
  if (grantToken !== undefined) {
673
800
  await abortable(options.fetchImplementation(new URL(`/admin/grants/${encodeURIComponent(grantToken)}`, options.brokerUrl), {
674
801
  method: "DELETE",
@@ -676,13 +803,13 @@ export async function runIsolatedWorker(options) {
676
803
  headers: {authorization: `Bearer ${options.brokerAdminToken}`}
677
804
  }), cleanupOperation.signal).catch(() => process.stderr.write("threadwire-isolated-runtime: grant cleanup failed\n"))
678
805
  }
679
- if (validatorId !== undefined) await cleanupDocker(() => options.docker.removeContainer(validatorId, cleanupOptions())).catch(() => {
806
+ if (validatorId !== undefined) await cleanupDocker(() => options.docker.removeContainer(validatorId, cleanupOptions)).catch(() => {
680
807
  resourcesRemoved = false
681
808
  process.stderr.write("threadwire-isolated-runtime: validator cleanup failed\n")
682
809
  })
683
810
  if (containerId !== undefined) {
684
811
  try {
685
- await stopAndRemoveWorkerContainer(options.docker, containerId, cleanupOptions())
812
+ await stopAndRemoveWorkerContainer(options.docker, containerId, workerCleanupOptions)
686
813
  } catch (error) {
687
814
  resourcesRemoved = false
688
815
  cleanupFailure = error instanceof ContainerCleanupConfirmationError
@@ -691,22 +818,26 @@ export async function runIsolatedWorker(options) {
691
818
  process.stderr.write("threadwire-isolated-runtime: worker cleanup failed\n")
692
819
  }
693
820
  }
694
- if (relayId !== undefined) await cleanupDocker(() => options.docker.removeContainer(relayId, cleanupOptions())).catch(() => {
821
+ if (gitMetadataVolumeName !== undefined) await cleanupDocker(() => options.docker.removeVolume(gitMetadataVolumeName, cleanupOptions)).catch(() => {
822
+ resourcesRemoved = false
823
+ process.stderr.write("threadwire-isolated-runtime: git metadata volume cleanup failed\n")
824
+ })
825
+ if (relayId !== undefined) await cleanupDocker(() => options.docker.removeContainer(relayId, cleanupOptions)).catch(() => {
695
826
  resourcesRemoved = false
696
827
  process.stderr.write("threadwire-isolated-runtime: relay cleanup failed\n")
697
828
  })
698
829
  if (networkId !== undefined) {
699
- if (provider !== "kimi") await cleanupDocker(() => options.docker.disconnectNetwork(networkId, options.brokerContainer, cleanupOptions())).catch(() => {})
700
- await cleanupDocker(() => options.docker.removeNetwork(networkId, cleanupOptions())).catch(() => {
830
+ if (provider !== "kimi") await cleanupDocker(() => options.docker.disconnectNetwork(networkId, options.brokerContainer, cleanupOptions)).catch(() => {})
831
+ await cleanupDocker(() => options.docker.removeNetwork(networkId, cleanupOptions)).catch(() => {
701
832
  resourcesRemoved = false
702
833
  process.stderr.write("threadwire-isolated-runtime: network cleanup failed\n")
703
834
  })
704
835
  }
705
- if (egressNetworkId !== undefined) await cleanupDocker(() => options.docker.removeNetwork(egressNetworkId, cleanupOptions())).catch(() => {
836
+ if (egressNetworkId !== undefined) await cleanupDocker(() => options.docker.removeNetwork(egressNetworkId, cleanupOptions)).catch(() => {
706
837
  resourcesRemoved = false
707
838
  process.stderr.write("threadwire-isolated-runtime: egress network cleanup failed\n")
708
839
  })
709
- if (freshState && !keepFreshState && resourcesRemoved) await cleanupDocker(() => options.docker.removeVolume(state.volume, cleanupOptions())).catch(() => { resourcesRemoved = false })
840
+ if (freshState && !keepFreshState && resourcesRemoved) await cleanupDocker(() => options.docker.removeVolume(state.volume, cleanupOptions)).catch(() => { resourcesRemoved = false })
710
841
  if (runTracked && resourcesRemoved) await options.stateRegistry.completeRun(runId, cleanupOperation.signal).catch(() => {})
711
842
  emergencyCleanup?.close()
712
843
  }
@@ -715,6 +846,89 @@ export async function runIsolatedWorker(options) {
715
846
  return workerResult
716
847
  }
717
848
 
849
+ /**
850
+ * Capture a credential-free pre/post mutation snapshot for the active provider.
851
+ * Codex snapshots the host-mounted repository root directly; Kimi snapshots the
852
+ * named source volume through an offline helper container because the runtime
853
+ * itself does not mount the task source volume. The helper uses the same
854
+ * immutable worker image that will run the worker, so it is available on the
855
+ * task daemon without inspecting the supervisor runtime container.
856
+ * @param {{provider: "codex" | "kimi", docker: DockerApi, repositoryRoot?: string, binding?: unknown, workerImage?: string, runId: string, namespace: string, lineage: string, taskHash: string, when: "before" | "after", operation: import("./absolute-deadline.js").AbsoluteDeadline}} options
857
+ */
858
+ export async function captureMutationSnapshot(options) {
859
+ if (options.provider === "kimi") {
860
+ if (typeof options.workerImage !== "string" || !isDigestImage(options.workerImage)) throw new Error("Kimi snapshot worker image is invalid")
861
+ return runKimiSourceSnapshot({
862
+ docker: options.docker,
863
+ binding: /** @type {import("./threadwire-binding.js").ThreadwireBinding} */ (options.binding),
864
+ image: options.workerImage,
865
+ runId: options.runId,
866
+ namespace: options.namespace,
867
+ lineage: options.lineage,
868
+ taskHash: options.taskHash,
869
+ when: options.when,
870
+ operation: options.operation
871
+ })
872
+ }
873
+ return gitSnapshot(/** @type {string} */ (options.repositoryRoot), {timeoutMs: 10_000}).catch((error) => {
874
+ throw new Error(`Mutation snapshot failed: ${error instanceof Error ? error.message : "unknown"}`)
875
+ })
876
+ }
877
+
878
+ /**
879
+ * Run an offline helper container that snapshots a Kimi source volume. The
880
+ * helper uses an explicit immutable image available on the task daemon, has no
881
+ * network, mounts only the source volume read-only, and carries no credentials.
882
+ * @param {{docker: DockerApi, binding: import("./threadwire-binding.js").ThreadwireBinding, image: string, runId: string, namespace: string, lineage: string, taskHash: string, when: "before" | "after", operation: import("./absolute-deadline.js").AbsoluteDeadline}} options
883
+ */
884
+ async function runKimiSourceSnapshot(options) {
885
+ const binding = parseThreadwireBinding(options.binding)
886
+ const spec = buildKimiSourceSnapshotSpec({
887
+ image: options.image,
888
+ sourceVolume: binding.source.volume,
889
+ sourceTarget: binding.source.target,
890
+ uid: binding.runtime.uid,
891
+ gid: binding.runtime.gid,
892
+ name: `${options.runId}-snapshot-${options.when}`,
893
+ labels: {
894
+ "org.threadwire.owner": "isolated-runtime",
895
+ "org.threadwire.namespace": options.namespace,
896
+ "org.threadwire.provider": "kimi",
897
+ "org.threadwire.run": options.runId,
898
+ "org.threadwire.lineage": options.lineage,
899
+ "org.threadwire.task": options.taskHash,
900
+ "org.threadwire.role": "snapshot"
901
+ }
902
+ })
903
+ const created = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(`${options.runId}-snapshot-${options.when}`, spec, options.operation.options()))
904
+ if (typeof created.Id !== "string") throw new Error("Kimi snapshot helper creation failed")
905
+ const containerId = created.Id
906
+ try {
907
+ await validateWorkerMountInventory(await options.docker.inspectContainer(containerId, options.operation.options()), spec.HostConfig.Mounts)
908
+ await options.docker.startContainer(containerId, options.operation.options())
909
+ const wait = /** @type {{StatusCode?: unknown}} */ (await options.docker.waitContainer(containerId, options.operation.options()))
910
+ if (wait.StatusCode !== 0) throw new Error("Kimi snapshot helper failed")
911
+ const logs = /** @type {Buffer} */ (await options.docker.logs(containerId, options.operation.options()))
912
+ const chunks = demultiplexDockerLogChunks(logs)
913
+ const stdout = Buffer.concat(chunks.filter((chunk) => chunk.channel === "stdout").map((chunk) => chunk.data)).toString("utf8").trim()
914
+ let parsed
915
+ try { parsed = JSON.parse(stdout) } catch { throw new Error("Kimi snapshot helper returned invalid JSON") }
916
+ if (!isRecord(parsed)) throw new Error("Kimi snapshot helper returned non-object")
917
+ return /** @type {import("./mutation-snapshot.js").GitSnapshot} */ (parsed)
918
+ } finally {
919
+ await cleanupDocker(() => options.docker.removeContainer(containerId, options.operation.options())).catch(() => {})
920
+ }
921
+ }
922
+
923
+ async function resolveRuntimeImage(docker, operation) {
924
+ const containerId = hostname()
925
+ const inspection = await docker.inspectContainer(containerId, operation.options()).catch(() => undefined)
926
+ if (!isRecord(inspection) || typeof inspection.Image !== "string" || !isDigestImage(inspection.Image)) {
927
+ throw new Error("Runtime self-image unavailable")
928
+ }
929
+ return inspection.Image
930
+ }
931
+
718
932
  async function cleanupDocker(operation) {
719
933
  return operation()
720
934
  }
@@ -722,23 +936,39 @@ async function cleanupDocker(operation) {
722
936
  async function stopAndRemoveWorkerContainer(docker, containerId, requestOptions) {
723
937
  let inspection = await docker.inspectContainer(containerId, requestOptions)
724
938
  if (typeof inspection?.State?.Running !== "boolean") throw new Error("Worker container state unavailable")
725
- if (inspection.State.Running) {
939
+ if (!inspection.State.Running) {
940
+ await docker.removeContainer(containerId, requestOptions)
941
+ return
942
+ }
943
+ // A caller/deadline abort bypasses the normal TERM grace to avoid spending
944
+ // the bounded emergency cleanup budget on a cooperative shutdown that may
945
+ // never arrive. Normal completion/failure cleanup still uses graceful stop.
946
+ if (requestOptions.forceKill) {
726
947
  try {
727
- await docker.stopContainer(containerId, WORKER_STOP_GRACE_SECONDS, requestOptions)
948
+ await docker.killContainer(containerId, "KILL", requestOptions)
728
949
  } catch (error) {
729
- if (!isContainerStopAlreadyStoppedError(error)) throw error
950
+ if (!isContainerKillConflictError(error)) throw error
730
951
  }
731
952
  inspection = await docker.inspectContainer(containerId, requestOptions)
732
- if (typeof inspection?.State?.Running !== "boolean") throw new Error("Worker container state unavailable")
733
- if (inspection.State.Running) {
734
- try {
735
- await docker.killContainer(containerId, "KILL", requestOptions)
736
- } catch (error) {
737
- if (!isContainerKillConflictError(error)) throw error
738
- }
739
- inspection = await docker.inspectContainer(containerId, requestOptions)
740
- if (inspection?.State?.Running !== false) throw new ContainerCleanupConfirmationError()
953
+ if (inspection?.State?.Running !== false) throw new ContainerCleanupConfirmationError()
954
+ await docker.removeContainer(containerId, requestOptions)
955
+ return
956
+ }
957
+ try {
958
+ await docker.stopContainer(containerId, WORKER_STOP_GRACE_SECONDS, requestOptions)
959
+ } catch (error) {
960
+ if (!isContainerStopAlreadyStoppedError(error)) throw error
961
+ }
962
+ inspection = await docker.inspectContainer(containerId, requestOptions)
963
+ if (typeof inspection?.State?.Running !== "boolean") throw new Error("Worker container state unavailable")
964
+ if (inspection.State.Running) {
965
+ try {
966
+ await docker.killContainer(containerId, "KILL", requestOptions)
967
+ } catch (error) {
968
+ if (!isContainerKillConflictError(error)) throw error
741
969
  }
970
+ inspection = await docker.inspectContainer(containerId, requestOptions)
971
+ if (inspection?.State?.Running !== false) throw new ContainerCleanupConfirmationError()
742
972
  }
743
973
  await docker.removeContainer(containerId, requestOptions)
744
974
  }
@@ -907,6 +1137,40 @@ function parseWorkerRecords(text, provider = "codex") {
907
1137
  return records
908
1138
  }
909
1139
 
1140
+ /**
1141
+ * Extract a closed provider mutation claim from worker records. The claim
1142
+ * contract is exactly `{type: "mutation_claim", mutations: {worktreeEdit: bool,
1143
+ * commit: bool, push: bool, githubWrite: bool}}`. The claim record is removed
1144
+ * from the returned records so it is never relayed as a worker event.
1145
+ * Malformed claims or multiple claims fail closed and abort reconciliation.
1146
+ * @param {unknown[]} records
1147
+ */
1148
+ export function extractMutationClaim(records) {
1149
+ const claims = records.filter((record) => isRecord(record) && record.type === "mutation_claim")
1150
+ if (claims.length === 0) return {records}
1151
+ if (claims.length > 1) throw new Error("Multiple provider mutation claims")
1152
+ const claim = claims[0]
1153
+ if (!exactKeys(claim, ["type", "mutations"])) throw new Error("Malformed provider mutation claim")
1154
+ const mutations = claim.mutations
1155
+ if (!isRecord(mutations) || !exactKeys(mutations, [...MUTATION_CAPABILITIES])) {
1156
+ throw new Error("Malformed provider mutation claim")
1157
+ }
1158
+ const parsed = {
1159
+ worktreeEdit: booleanClaimValue(mutations.worktreeEdit),
1160
+ commit: booleanClaimValue(mutations.commit),
1161
+ push: booleanClaimValue(mutations.push),
1162
+ githubWrite: booleanClaimValue(mutations.githubWrite)
1163
+ }
1164
+ const stripped = records.filter((record) => record !== claim)
1165
+ return {records: stripped, claim: parsed}
1166
+ }
1167
+
1168
+ /** @param {unknown} value */
1169
+ function booleanClaimValue(value) {
1170
+ if (typeof value !== "boolean") throw new Error("Malformed provider mutation claim")
1171
+ return value
1172
+ }
1173
+
910
1174
  function validatedKimiWorkerEnvelope(record) {
911
1175
  const sessionId = kimiSessionEnvelopeId(record)
912
1176
  if (sessionId !== undefined) return {type: "session", sessionId}
@@ -959,6 +1223,20 @@ export function demultiplexDockerLogChunks(buffer) {
959
1223
  return [{channel: "stdout", data: buffer}]
960
1224
  }
961
1225
 
1226
+ /**
1227
+ * @typedef {{
1228
+ * provider: "codex" | "kimi",
1229
+ * profile: string,
1230
+ * repositoryRoot: string,
1231
+ * cwd: string,
1232
+ * providerArguments: string[],
1233
+ * resumeSession?: string,
1234
+ * mutationPolicy?: import("./mutation-policy.js").MutationPolicy,
1235
+ * binding?: unknown
1236
+ * }} PreflightSelection
1237
+ */
1238
+
1239
+ /** @param {unknown} value @param {"codex" | "kimi"} [expectedProvider] @returns {PreflightSelection} */
962
1240
  export function parsePreflight(value, expectedProvider = "codex") {
963
1241
  if (!isRecord(value) || value.provider !== expectedProvider || (expectedProvider !== "codex" && expectedProvider !== "kimi")
964
1242
  || typeof value.profile !== "string" || !Array.isArray(value.providerArguments)
@@ -968,30 +1246,36 @@ export function parsePreflight(value, expectedProvider = "codex") {
968
1246
  if (value.resumeSession !== undefined && (typeof value.resumeSession !== "string" || /[\r\n]/u.test(value.resumeSession))) throw new ClientError("Invalid isolated runtime preflight")
969
1247
  if (expectedProvider === "kimi") {
970
1248
  // The Kimi request schema is closed: repositoryRoot/cwd are derived from
971
- // the binding, so any extraneous top-level field is rejected rather than
972
- // silently ignored.
973
- const allowedKeys = ["provider", "profile", "binding", "providerArguments", ...(value.resumeSession === undefined ? [] : ["resumeSession"])]
1249
+ // the binding, and mutationPolicy is the only optional extension.
1250
+ const allowedKeys = ["provider", "profile", "binding", "providerArguments",
1251
+ ...(value.resumeSession === undefined ? [] : ["resumeSession"]),
1252
+ ...(value.mutationPolicy === undefined ? [] : ["mutationPolicy"])]
974
1253
  if (!exactKeys(value, allowedKeys)) throw new ClientError("Invalid isolated runtime preflight")
975
1254
  let binding
976
1255
  try { binding = parseThreadwireBinding(value.binding) } catch { throw new ClientError("Invalid isolated runtime preflight") }
1256
+ const mutationPolicy = value.mutationPolicy === undefined ? undefined : parseMutationPolicy(JSON.stringify(value.mutationPolicy))
977
1257
  return {
978
1258
  provider: "kimi", profile: value.profile, binding,
979
1259
  repositoryRoot: binding.source.target, cwd: binding.runtime.workdir,
980
1260
  providerArguments: value.providerArguments,
981
- ...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession})
1261
+ ...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession}),
1262
+ ...(mutationPolicy === undefined ? {} : {mutationPolicy})
982
1263
  }
983
1264
  }
984
1265
  if (typeof value.repositoryRoot !== "string" || typeof value.cwd !== "string"
985
1266
  || !bounded(value.repositoryRoot, 4096) || !bounded(value.cwd, 4096)) throw new ClientError("Invalid isolated runtime preflight")
1267
+ const mutationPolicy = parseMutationPolicy(value.mutationPolicy === undefined ? undefined : JSON.stringify(value.mutationPolicy))
986
1268
  return {
987
1269
  provider: "codex", profile: value.profile, repositoryRoot: value.repositoryRoot, cwd: value.cwd,
988
1270
  providerArguments: value.providerArguments,
989
- ...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession})
1271
+ ...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession}),
1272
+ ...(mutationPolicy === undefined ? {} : {mutationPolicy})
990
1273
  }
991
1274
  }
992
1275
 
993
1276
  export function parseRun(value) {
994
- if (!isRecord(value) || typeof value.preflightId !== "string" || typeof value.prompt !== "string" || value.prompt.trim().length === 0 || !Array.isArray(value.providerArguments) || !value.providerArguments.every((item) => typeof item === "string")) {
1277
+ if (!isRecord(value) || typeof value.preflightId !== "string" || typeof value.prompt !== "string" || value.prompt.trim().length === 0 || !Array.isArray(value.providerArguments) || !value.providerArguments.every((item) => typeof item === "string")
1278
+ || value.mutationPolicy !== undefined) {
995
1279
  throw new ClientError("Invalid isolated runtime run")
996
1280
  }
997
1281
  if (!bounded(value.preflightId, 64) || !boundedPrompt(value.prompt, 524_288)
@@ -1284,31 +1568,12 @@ function sendJson(response, status, value) {
1284
1568
  response.end(JSON.stringify(value))
1285
1569
  }
1286
1570
 
1287
- function isRecord(value) {
1288
- return typeof value === "object" && value !== null && !Array.isArray(value)
1289
- }
1290
-
1291
- function exactKeys(value, expected) {
1292
- const actual = Object.keys(value).sort()
1293
- const wanted = [...expected].sort()
1294
- return actual.length === wanted.length && actual.every((key, index) => key === wanted[index])
1295
- }
1296
-
1297
- function abortable(promise, signal) {
1298
- if (signal.aborted) return Promise.reject(signal.reason)
1299
- return new Promise((resolve, reject) => {
1300
- const abort = () => reject(signal.reason instanceof Error ? signal.reason : new Error("Isolated runtime operation aborted"))
1301
- signal.addEventListener("abort", abort, {once: true})
1302
- promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort))
1303
- })
1304
- }
1305
-
1306
1571
  function listen(server, port, host) {
1307
1572
  return new Promise((resolve, reject) => {
1308
1573
  server.once("error", reject)
1309
1574
  server.listen(port, host, () => {
1310
- server.removeListener("error", reject)
1311
- resolve(undefined)
1575
+ server.off("error", reject)
1576
+ resolve()
1312
1577
  })
1313
1578
  })
1314
1579
  }
@@ -1484,6 +1749,24 @@ export async function reconcileDockerResources(docker, brokerContainer, stateReg
1484
1749
  activeLineages.add(identity.lineage)
1485
1750
  continue
1486
1751
  }
1752
+ // Remove any ephemeral per-run git metadata snapshot volume that was created
1753
+ // for a commit-denied run and then left behind by a supervisor crash. Active
1754
+ // runs and runs whose primary cleanup failed are skipped.
1755
+ const snapshots = await docker.listVolumes({
1756
+ "org.threadwire.owner": "isolated-runtime",
1757
+ "org.threadwire.namespace": namespace,
1758
+ "org.threadwire.run": identity.run,
1759
+ "org.threadwire.role": "git-metadata-snapshot"
1760
+ }, requestOptions).catch(() => undefined)
1761
+ if (Array.isArray(snapshots?.Volumes)) {
1762
+ for (const snapshot of snapshots.Volumes) {
1763
+ await docker.removeVolume(snapshot.Name, requestOptions).catch(() => { cleanupFailed.add(identity.run) })
1764
+ }
1765
+ }
1766
+ if (cleanupFailed.has(identity.run)) {
1767
+ activeLineages.add(identity.lineage)
1768
+ continue
1769
+ }
1487
1770
  await stateRegistry.completeRun(identity.run, requestOptions?.signal).catch(() => activeLineages.add(identity.lineage))
1488
1771
  }
1489
1772
  return [...activeLineages]
@@ -1533,17 +1816,30 @@ function ownedContainerIdentity(container, labels, persisted) {
1533
1816
  || typeof container.Config.WorkingDir !== "string" || !within("/worktree", container.Config.WorkingDir)
1534
1817
  || container.HostConfig.NetworkMode !== `${labels.run}-net`
1535
1818
  || !validWorkerSecurity(container.Config, container.HostConfig, labels.provider)
1536
- || !Array.isArray(container.HostConfig.Mounts) || container.HostConfig.Mounts.length !== 2) return undefined
1819
+ || !Array.isArray(container.HostConfig.Mounts) || (container.HostConfig.Mounts.length !== 2 && container.HostConfig.Mounts.length !== 3)) return undefined
1537
1820
  const inspectedLabels = validResourceLabels(container.Config.Labels, labels.namespace, true, labels.provider)
1538
1821
  if (!inspectedLabels || JSON.stringify(inspectedLabels) !== JSON.stringify(labels)) return undefined
1539
1822
  const state = container.HostConfig.Mounts.find((mount) => mount?.Target === "/home/worker")
1540
1823
  const worktree = container.HostConfig.Mounts.find((mount) => mount?.Target === "/worktree")
1824
+ const gitOverlay = container.HostConfig.Mounts.find((mount) => mount?.Target === "/worktree/.git")
1541
1825
  if (state?.Type !== "volume" || state.Source !== labels.volume || state.ReadOnly === true
1542
1826
  || Object.keys(state).some((key) => !["Type", "Source", "Target", "ReadOnly"].includes(key))) return undefined
1543
- if (!validWorktreeMount(worktree)) return undefined
1827
+ if (!validWorktreeMount(worktree, {allowReadOnly: gitOverlay === undefined})) return undefined
1828
+ if (container.HostConfig.Mounts.length === 3) {
1829
+ if (!validGitOverlayMount(gitOverlay, labels.run)) return undefined
1830
+ } else if (gitOverlay !== undefined) {
1831
+ return undefined
1832
+ }
1544
1833
  return runIdentityFrom(labels, container.Config.Image, worktree)
1545
1834
  }
1546
1835
 
1836
+ /** @param {unknown} mount @param {string} runId */
1837
+ function validGitOverlayMount(mount, runId) {
1838
+ return isRecord(mount) && mount.Type === "volume" && mount.Source === `${runId}-git` && mount.Target === "/worktree/.git"
1839
+ && mount.ReadOnly === true
1840
+ && Object.keys(mount).every((key) => ["Type", "Source", "Target", "ReadOnly"].includes(key))
1841
+ }
1842
+
1547
1843
  /**
1548
1844
  * Exact, fail-closed Kimi container identity. The persisted sealed run
1549
1845
  * identity is the single authority for the expected validator, relay, and
@@ -1766,11 +2062,13 @@ function validWorkerSecurity(config, host, provider = "codex") {
1766
2062
  && expected.every((name) => names.filter((entry) => entry === name).length === 1)
1767
2063
  && !names.some((name) => /^(?:CODEX|OPENAI|TELEGRAM|THREADWIRE_KIMI_(?:OAUTH|CONTROL))/u.test(name))
1768
2064
  }
1769
- if (config.Env.length !== 13 + resumeCount) return false
2065
+ const hasGitOverlay = config.Env.includes("THREADWIRE_GIT_OVERLAY_READ_ONLY=true")
1770
2066
  const expected = [
1771
2067
  "HOME", "CODEX_HOME", "TMPDIR", "OPENAI_BASE_URL", "OPENAI_API_KEY", "CODEX_API_KEY",
1772
2068
  "THREADWIRE_PROMPT", "THREADWIRE_CODEX_ARGUMENTS", "THREADWIRE_WORKTREE_INODE", "THREADWIRE_CWD_INODE"
1773
2069
  ]
2070
+ if (hasGitOverlay) expected.push("THREADWIRE_GIT_OVERLAY_READ_ONLY")
2071
+ if (config.Env.length !== expected.length + 3 + resumeCount) return false
1774
2072
  return expected.every((name) => names.filter((entry) => entry === name).length === 1)
1775
2073
  && config.Env.includes("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
1776
2074
  && config.Env.includes("NODE_VERSION=24.19.0") && config.Env.includes("YARN_VERSION=1.22.22")
@@ -1813,8 +2111,9 @@ function ownedKimiNetworkIdentity(network, labels, identity) {
1813
2111
  return true
1814
2112
  }
1815
2113
 
1816
- function validWorktreeMount(mount) {
1817
- if (!isRecord(mount) || mount.Target !== "/worktree" || mount.ReadOnly === true) return false
2114
+ function validWorktreeMount(mount, options = {}) {
2115
+ if (!isRecord(mount) || mount.Target !== "/worktree") return false
2116
+ if (!options.allowReadOnly && mount.ReadOnly === true) return false
1818
2117
  if (mount.Type === "bind") {
1819
2118
  return Object.keys(mount).every((key) => ["Type", "Source", "Target", "ReadOnly", "BindOptions"].includes(key))
1820
2119
  && typeof mount.Source === "string" && isAbsolute(mount.Source)