threadwire 0.1.6 → 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1117 @@
1
+ // @ts-nocheck
2
+ /* eslint-disable jsdoc/require-jsdoc */
3
+
4
+ import {randomBytes} from "node:crypto"
5
+ import {lstat, realpath} from "node:fs/promises"
6
+ import {createServer} from "node:http"
7
+ import {isAbsolute, join, normalize, relative} from "node:path"
8
+ import {DockerApi} from "./docker-api.js"
9
+ import {buildWorkerContainerSpec, isDigestImage} from "./isolated-worker.js"
10
+ import {validateRelayWriteProviderArguments} from "./relay-write.js"
11
+ import {codexSessionId, parseCodexEvent} from "./providers/codex.js"
12
+ import {kimiSessionEnvelopeId, validateKimiProviderArguments} from "./providers/kimi.js"
13
+ import {parseApprovedKimiModels} from "./kimi-model-broker-policy.js"
14
+ import {openStateRegistry, validateStateVolume} from "./isolated-state.js"
15
+ import {assertNoNestedMounts} from "./mount-policy.js"
16
+ import {AbsoluteDeadline, deadlineAfter} from "./absolute-deadline.js"
17
+
18
+ const MAX_PREFLIGHT_BYTES = 32_768
19
+ const MAX_RUN_BYTES = 1_048_576
20
+ const PREFLIGHT_TTL_MS = 60_000
21
+ const PREFLIGHT_CAPACITY = 128
22
+ const PREFLIGHT_TASK_CAPACITY = 8
23
+ const GC_INTERVAL_MS = 5_000
24
+ const ACTIVE_RUN_CAPACITY = 16
25
+ const ACTIVE_TASK_CAPACITY = 2
26
+ const WORKER_TIMEOUT_MS = 3_600_000
27
+ const PREFLIGHT_TIMEOUT_MS = 30_000
28
+ const MAX_RAW_OUTPUT_BYTES = 1_048_576
29
+
30
+ /** @param {{environment?: NodeJS.ProcessEnv, docker?: DockerApi, fetchImplementation?: typeof fetch, now?: () => number}} [options] */
31
+ export async function startIsolatedRuntime(options = {}) {
32
+ const environment = options.environment ?? process.env
33
+ const host = environment.THREADWIRE_ISOLATED_RUNTIME_HOST ?? "0.0.0.0"
34
+ const port = portValue(environment.THREADWIRE_ISOLATED_RUNTIME_PORT, 8790)
35
+ const provider = isolatedProvider(environment.THREADWIRE_ISOLATED_PROVIDER)
36
+ const controlToken = provider === "kimi"
37
+ ? authoritySetting(environment.THREADWIRE_ISOLATED_RUNTIME_CONTROL_TOKEN, "THREADWIRE_ISOLATED_RUNTIME_CONTROL_TOKEN")
38
+ : safeSetting(environment.THREADWIRE_ISOLATED_RUNTIME_CONTROL_TOKEN, "THREADWIRE_ISOLATED_RUNTIME_CONTROL_TOKEN")
39
+ const workerImageName = provider === "kimi" ? "THREADWIRE_KIMI_RELAY_WORKER_IMAGE" : "THREADWIRE_RELAY_WORKER_IMAGE"
40
+ const workerImage = safeSetting(environment[workerImageName], workerImageName)
41
+ if (!isDigestImage(workerImage)) throw new Error(`${workerImageName} must use an immutable digest`)
42
+ const brokerUrlName = provider === "kimi" ? "THREADWIRE_KIMI_MODEL_BROKER_URL" : "THREADWIRE_MODEL_BROKER_URL"
43
+ const brokerAdminName = provider === "kimi" ? "THREADWIRE_KIMI_MODEL_BROKER_ADMIN_TOKEN" : "THREADWIRE_MODEL_BROKER_ADMIN_TOKEN"
44
+ const brokerContainerName = provider === "kimi" ? "THREADWIRE_KIMI_MODEL_BROKER_CONTAINER" : "THREADWIRE_MODEL_BROKER_CONTAINER"
45
+ const brokerUrl = normalizedHttpUrl(environment[brokerUrlName], brokerUrlName)
46
+ const brokerAdminToken = provider === "kimi"
47
+ ? authoritySetting(environment[brokerAdminName], brokerAdminName)
48
+ : safeSetting(environment[brokerAdminName], brokerAdminName)
49
+ const brokerContainer = safeSetting(environment[brokerContainerName], brokerContainerName)
50
+ const allowedRoots = parseAllowedRoots(environment.THREADWIRE_ALLOWED_WORKTREE_ROOTS)
51
+ const worktreeVolume = environment.THREADWIRE_WORKTREE_VOLUME
52
+ const sessionTtlMs = positiveDuration(environment.THREADWIRE_SESSION_TTL_MS, 86_400_000)
53
+ const workerTimeoutMs = positiveDuration(environment.THREADWIRE_WORKER_TIMEOUT_MS, WORKER_TIMEOUT_MS)
54
+ const preflightTimeoutMs = positiveDuration(environment.THREADWIRE_PREFLIGHT_TIMEOUT_MS, PREFLIGHT_TIMEOUT_MS)
55
+ const docker = options.docker ?? new DockerApi()
56
+ const fetchImplementation = options.fetchImplementation ?? fetch
57
+ const now = options.now ?? Date.now
58
+ const allowedModels = provider === "kimi"
59
+ ? parseApprovedKimiModels(environment.THREADWIRE_ALLOWED_KIMI_MODELS)
60
+ : parseAllowedModels(environment.THREADWIRE_ALLOWED_CODEX_MODELS)
61
+ const stateRegistry = await openStateRegistry({
62
+ file: safeSetting(environment.THREADWIRE_STATE_REGISTRY_FILE, "THREADWIRE_STATE_REGISTRY_FILE"),
63
+ key: safeSetting(environment.THREADWIRE_STATE_AUTH_KEY, "THREADWIRE_STATE_AUTH_KEY"),
64
+ namespace: safeSetting(environment.THREADWIRE_STATE_NAMESPACE, "THREADWIRE_STATE_NAMESPACE"),
65
+ provider,
66
+ now
67
+ })
68
+ const preflights = new PreflightStore({
69
+ now,
70
+ ttlMs: options.preflightTtlMs ?? PREFLIGHT_TTL_MS,
71
+ capacity: options.preflightCapacity ?? PREFLIGHT_CAPACITY,
72
+ taskCapacity: options.preflightTaskCapacity ?? PREFLIGHT_TASK_CAPACITY
73
+ })
74
+ const activeLineages = new LineageReservations()
75
+ const activeRuns = new ActiveRunAdmission({
76
+ capacity: positiveInteger(environment.THREADWIRE_ACTIVE_RUN_CAPACITY, ACTIVE_RUN_CAPACITY),
77
+ taskCapacity: positiveInteger(environment.THREADWIRE_ACTIVE_TASK_CAPACITY, ACTIVE_TASK_CAPACITY)
78
+ })
79
+ const incomingRequests = new ActiveRunAdmission({capacity: ACTIVE_RUN_CAPACITY, taskCapacity: ACTIVE_RUN_CAPACITY})
80
+ const shutdown = new AbortController()
81
+ const sockets = new Set()
82
+ let stopping = false
83
+ let gcPromise = Promise.resolve()
84
+ const collectState = () => {
85
+ gcPromise = gcPromise.then(async () => {
86
+ if (shutdown.signal.aborted) return
87
+ const collection = deadlineAfter(Math.min(30_000, workerTimeoutMs), {signal: shutdown.signal, timeoutMessage: "Isolated runtime collection timeout"})
88
+ const releaseCollection = activeLineages.beginCollection()
89
+ try {
90
+ const active = new Set(activeLineages.values())
91
+ const current = {
92
+ *[Symbol.iterator]() { yield * active },
93
+ has(lineage) { return active.has(lineage) || activeLineages.has(lineage) }
94
+ }
95
+ for (const lineage of await reconcileDockerResources(docker, brokerContainer, stateRegistry, current, collection.options())) active.add(lineage)
96
+ await stateRegistry.collect(docker, {has: (lineage) => active.has(lineage) || activeLineages.has(lineage)}, collection.options())
97
+ } finally {
98
+ collection.close()
99
+ releaseCollection()
100
+ }
101
+ }).catch(() => {})
102
+ return gcPromise
103
+ }
104
+ const startupActive = await reconcileDockerResources(docker, brokerContainer, stateRegistry, new Set())
105
+ await stateRegistry.collect(docker, new Set(startupActive))
106
+ const gcTimer = setInterval(collectState, Math.min(GC_INTERVAL_MS, sessionTtlMs))
107
+ gcTimer.unref()
108
+
109
+ const server = createServer(async (request, response) => {
110
+ try {
111
+ if (stopping) throw new Error("Isolated runtime is shutting down")
112
+ requireControl(request.headers.authorization, controlToken)
113
+ if (request.method === "GET" && request.url === "/healthz") {
114
+ sendJson(response, 200, {ok: true})
115
+ return
116
+ }
117
+ if (request.method === "POST" && request.url === "/preflight") {
118
+ const releaseIncoming = incomingRequests.acquire("preflight")
119
+ const launchDeadlineAt = requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)
120
+ const operation = new AbsoluteDeadline(Math.min(Date.now() + preflightTimeoutMs, launchDeadlineAt), {
121
+ signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime operation timed out", disconnectMessage: "Isolated runtime caller disconnected"
122
+ })
123
+ try {
124
+ const body = await readJson(request, MAX_PREFLIGHT_BYTES, operation.signal)
125
+ const selection = parsePreflight(body, provider)
126
+ const task = taskIdentity(selection)
127
+ preflights.assertCapacity(task)
128
+ const providerSelection = runtimeProviderSelection(provider, selection.providerArguments, allowedModels)
129
+ const {fingerprint, root: worktreeRoot} = await verifyPrerequisites({
130
+ docker, workerImage, selection, allowedRoots, brokerUrl, brokerAdminToken, fetchImplementation, operation
131
+ })
132
+ let state
133
+ if (selection.resumeSession !== undefined) {
134
+ state = stateRegistry.lookup(selection.resumeSession, provider)
135
+ if (!state || state.provider !== provider || state.task !== task) throw new ClientError(`${providerName(provider)} resume state is unavailable`)
136
+ const volume = await docker.inspectVolume(state.volume, operation.options()).catch(() => undefined)
137
+ try { validateStateVolume(volume, state.volume, state.labels) } catch { throw new ClientError(`${providerName(provider)} resume state is unavailable`) }
138
+ } else state = stateRegistry.allocate(task)
139
+ const preflightId = randomBytes(24).toString("base64url")
140
+ preflights.add(preflightId, task, {...selection, providerSelection, state, fingerprint, worktreeRoot, launchDeadlineAt})
141
+ sendJson(response, 200, {ok: true, preflightId, launchDeadlineAt})
142
+ } finally {
143
+ operation.close()
144
+ releaseIncoming()
145
+ }
146
+ return
147
+ }
148
+ if (request.method === "POST" && request.url === "/run") {
149
+ const releaseIncoming = incomingRequests.acquire("run")
150
+ const bodyOperation = new AbsoluteDeadline(Date.now() + preflightTimeoutMs, {signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime operation timed out"})
151
+ let body
152
+ try {
153
+ body = parseRun(await readJson(request, MAX_RUN_BYTES, bodyOperation.signal))
154
+ } finally {
155
+ bodyOperation.close()
156
+ releaseIncoming()
157
+ }
158
+ const preflight = preflights.consume(body.preflightId)
159
+ if (!preflight) throw new ClientError("Isolated runtime preflight expired")
160
+ const operation = new AbsoluteDeadline(Math.min(preflight.launchDeadlineAt,
161
+ requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)), {
162
+ signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime launch timed out", disconnectMessage: "Isolated runtime caller disconnected"
163
+ })
164
+ const task = taskIdentity(preflight)
165
+ let releaseRun
166
+ let releaseLineage
167
+ try {
168
+ releaseRun = activeRuns.acquire(task)
169
+ releaseLineage = activeLineages.acquire(preflight.state.lineage)
170
+ if (body.resumeSession !== preflight.resumeSession) throw new ClientError(`${providerName(provider)} resume state does not match preflight`)
171
+ if (JSON.stringify(runtimeProviderSelection(provider, body.providerArguments, allowedModels)) !== JSON.stringify(preflight.providerSelection)) {
172
+ throw new ClientError(`${providerName(provider)} arguments do not match preflight`)
173
+ }
174
+ await revalidateFingerprint(preflight.fingerprint)
175
+ operation.throwIfAborted()
176
+ const result = await runIsolatedWorker({
177
+ docker, workerImage, brokerUrl, brokerAdminToken, brokerContainer,
178
+ fetchImplementation, preflight, ...body, task, now, stateRegistry, sessionTtlMs, operation,
179
+ ...(worktreeVolume === undefined ? {} : {worktreeVolume})
180
+ })
181
+ sendJson(response, 200, result)
182
+ } finally {
183
+ operation.close()
184
+ releaseLineage?.()
185
+ releaseRun?.()
186
+ void collectState()
187
+ }
188
+ return
189
+ }
190
+ response.writeHead(404)
191
+ response.end()
192
+ } catch (error) {
193
+ sendJson(response, error instanceof ClientError ? 400 : 503, {
194
+ error: error instanceof ClientError ? error.message : "Isolated runtime unavailable"
195
+ })
196
+ }
197
+ })
198
+ server.on("connection", (socket) => {
199
+ sockets.add(socket)
200
+ socket.once("close", () => sockets.delete(socket))
201
+ })
202
+ await listen(server, port, host)
203
+ return {
204
+ server,
205
+ close: async () => {
206
+ stopping = true
207
+ clearInterval(gcTimer)
208
+ preflights.close()
209
+ shutdown.abort(new Error("Isolated runtime is shutting down"))
210
+ for (const socket of sockets) socket.destroy()
211
+ await boundedClose(server, 1000)
212
+ await Promise.race([gcPromise, new Promise((resolve) => setTimeout(resolve, 1000))])
213
+ },
214
+ config: {host, port}
215
+ }
216
+ }
217
+
218
+ async function verifyPrerequisites(options) {
219
+ const {docker, workerImage, selection, allowedRoots, brokerUrl, brokerAdminToken, fetchImplementation, operation} = options
220
+ const version = /** @type {{ApiVersion?: unknown}} */ (await docker.version(operation.options()))
221
+ if (typeof version.ApiVersion !== "string") throw new Error("Docker API unavailable")
222
+ const image = /** @type {{Id?: unknown}} */ (await docker.inspectImage(workerImage, operation.options()))
223
+ if (typeof image.Id !== "string" || !isDigestImage(image.Id)) throw new Error("Worker image unavailable")
224
+ const validatedWorktree = await validateWorktreeSelection(selection.repositoryRoot, selection.cwd, allowedRoots)
225
+ const health = await abortable(fetchImplementation(new URL("/healthz", brokerUrl), {
226
+ method: "GET",
227
+ redirect: "error",
228
+ signal: operation.signal,
229
+ headers: {authorization: `Bearer ${brokerAdminToken}`}
230
+ }), operation.signal)
231
+ if (!health.ok) throw new Error("Model broker unavailable")
232
+ return validatedWorktree
233
+ }
234
+
235
+ export async function validateWorktree(repositoryRoot, cwd, allowedRoots) {
236
+ return (await validateWorktreeSelection(repositoryRoot, cwd, allowedRoots)).fingerprint
237
+ }
238
+
239
+ export async function validateWorktreeSelection(repositoryRoot, cwd, allowedRoots) {
240
+ const root = allowedRoots
241
+ .filter((allowed) => within(allowed, repositoryRoot))
242
+ .sort((first, second) => second.length - first.length)[0]
243
+ if (root === undefined) throw new ClientError("Configured worktree is unavailable")
244
+ const paths = ancestry(root, repositoryRoot)
245
+ for (const path of ancestry(repositoryRoot, cwd).slice(1)) if (!paths.includes(path)) paths.push(path)
246
+ /** @type {{path: string, dev: string, ino: string}[]} */
247
+ const fingerprint = []
248
+ for (const path of paths) {
249
+ if (!isAbsolute(path) || normalize(path) !== path || path === "/") throw new ClientError("Configured worktree is unavailable")
250
+ const metadata = await lstat(path, {bigint: true})
251
+ if (!metadata.isDirectory() || metadata.isSymbolicLink() || await realpath(path) !== path) {
252
+ throw new ClientError("Configured worktree is unavailable")
253
+ }
254
+ fingerprint.push({path, dev: String(metadata.dev), ino: String(metadata.ino)})
255
+ }
256
+ if (!within(repositoryRoot, cwd) || !allowedRoots.some((allowed) => within(allowed, repositoryRoot))) {
257
+ throw new ClientError("Configured worktree is unavailable")
258
+ }
259
+ await assertNoNestedMounts(repositoryRoot)
260
+ try {
261
+ const gitEntry = await lstat(join(repositoryRoot, ".git"))
262
+ if (gitEntry.isDirectory()) throw new ClientError("Common Git directories cannot enter the worker")
263
+ if (!gitEntry.isFile() || gitEntry.isSymbolicLink()) throw new ClientError("Configured worktree is unavailable")
264
+ } catch (error) {
265
+ if (error instanceof ClientError) throw error
266
+ if (/** @type {NodeJS.ErrnoException} */ (error).code !== "ENOENT") throw error
267
+ }
268
+ return {fingerprint, root}
269
+ }
270
+
271
+ export async function revalidateFingerprint(fingerprint) {
272
+ if (!Array.isArray(fingerprint)) throw new ClientError("Configured worktree changed after preflight")
273
+ for (const expected of fingerprint) {
274
+ const metadata = await lstat(expected.path, {bigint: true}).catch(() => undefined)
275
+ if (!metadata?.isDirectory() || metadata.isSymbolicLink() || String(metadata.dev) !== expected.dev || String(metadata.ino) !== expected.ino || await realpath(expected.path) !== expected.path) {
276
+ throw new ClientError("Configured worktree changed after preflight")
277
+ }
278
+ }
279
+ }
280
+
281
+ export async function runIsolatedWorker(options) {
282
+ const runId = `tw-${randomBytes(12).toString("hex")}`
283
+ const networkName = `${runId}-net`
284
+ const state = options.preflight.state
285
+ const provider = options.preflight.provider ?? "codex"
286
+ const providerSelection = options.preflight.providerSelection ?? options.preflight.codexArguments
287
+ if (providerSelection === undefined) throw new Error("Provider selection unavailable")
288
+ const freshState = options.resumeSession === undefined
289
+ let keepFreshState = false
290
+ let networkId
291
+ let containerId
292
+ let grantToken
293
+ let runTracked = false
294
+ const operation = options.operation
295
+ const worktreeSubpath = options.worktreeVolume === undefined
296
+ ? ""
297
+ : deriveVolumeWorktreeSubpath(options.preflight.worktreeRoot, options.preflight.repositoryRoot)
298
+ try {
299
+ operation.throwIfAborted()
300
+ await prepareStateVolume(options.docker, state, freshState, operation.options())
301
+ if (freshState) await options.stateRegistry.track(state, operation.signal)
302
+ const runIdentity = expectedRunIdentity(options, runId, networkName, state, worktreeSubpath)
303
+ const runSeal = options.stateRegistry.sealRun(runIdentity)
304
+ await options.stateRegistry.trackRun(runIdentity, operation.signal)
305
+ runTracked = true
306
+ const resourceLabels = {
307
+ "org.threadwire.namespace": options.stateRegistry.namespace,
308
+ "org.threadwire.provider": provider,
309
+ "org.threadwire.run": runId,
310
+ "org.threadwire.lineage": state.lineage,
311
+ "org.threadwire.task": state.labels["org.threadwire.task"],
312
+ "org.threadwire.state-volume": state.volume,
313
+ "org.threadwire.run-seal": runSeal
314
+ }
315
+ const network = /** @type {{Id?: unknown}} */ (await options.docker.createNetwork(networkName, resourceLabels, operation.options()))
316
+ if (typeof network.Id !== "string") throw new Error("Docker network creation failed")
317
+ networkId = network.Id
318
+ await options.docker.connectNetwork(networkId, options.brokerContainer, [], operation.options())
319
+ const networkInspection = await options.docker.inspectNetwork(networkId, operation.options())
320
+ const brokerAddress = brokerNetworkAddress(networkInspection, options.brokerContainer)
321
+ const grantResponse = await abortable(options.fetchImplementation(new URL("/admin/grants", options.brokerUrl), {
322
+ method: "POST",
323
+ redirect: "error",
324
+ signal: operation.signal,
325
+ headers: {authorization: `Bearer ${options.brokerAdminToken}`, "content-type": "application/json"},
326
+ body: JSON.stringify(provider === "kimi"
327
+ ? {
328
+ runId, provider, networkId, modelAlias: providerSelection.modelAlias, brokerAddress,
329
+ taskId: state.labels["org.threadwire.task"],
330
+ sessionId: options.resumeSession ?? `pending:${state.lineage}`,
331
+ ttlMs: Math.min(operation.remaining(), 3_600_000)
332
+ }
333
+ : {runId, provider, networkId, model: providerSelection.model, brokerAddress, ttlMs: Math.min(operation.remaining(), 3_600_000)})
334
+ }), operation.signal)
335
+ if (!grantResponse.ok) throw new Error("Model broker grant failed")
336
+ const grant = await grantResponse.json()
337
+ if (!isRecord(grant) || typeof grant.token !== "string" || !Number.isSafeInteger(grant.port)) throw new Error("Model broker grant failed")
338
+ grantToken = grant.token
339
+ const repository = /** @type {{path: string, ino: string}[]} */ (options.preflight.fingerprint)
340
+ const worktreeMetadata = repository.find((entry) => entry.path === options.preflight.repositoryRoot)
341
+ const cwdMetadata = repository.find((entry) => entry.path === options.preflight.cwd)
342
+ if (worktreeMetadata === undefined || cwdMetadata === undefined) throw new Error("Worktree fingerprint unavailable")
343
+ const spec = buildWorkerContainerSpec({
344
+ provider,
345
+ image: options.workerImage,
346
+ worktree: options.preflight.repositoryRoot,
347
+ networkName,
348
+ brokerToken: grantToken,
349
+ brokerUrl: `http://${brokerAddress}:${grant.port}/v1`,
350
+ prompt: options.prompt,
351
+ providerArguments: providerSelection.arguments,
352
+ ...(provider === "kimi" ? {model: providerSelection.model} : {}),
353
+ stateVolume: state.volume,
354
+ workingDirectory: `/worktree${relative(options.preflight.repositoryRoot, options.preflight.cwd) === "" ? "" : `/${relative(options.preflight.repositoryRoot, options.preflight.cwd)}`}`,
355
+ worktreeInode: worktreeMetadata.ino,
356
+ cwdInode: cwdMetadata.ino,
357
+ lineage: state.lineage,
358
+ namespace: options.stateRegistry.namespace,
359
+ runId,
360
+ taskHash: state.labels["org.threadwire.task"],
361
+ runSeal,
362
+ ...(options.worktreeVolume === undefined
363
+ ? {}
364
+ : {
365
+ worktreeVolume: {
366
+ name: options.worktreeVolume,
367
+ subpath: worktreeSubpath
368
+ }
369
+ }),
370
+ ...(options.resumeSession === undefined ? {} : {resumeSession: options.resumeSession})
371
+ })
372
+ const container = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(`${runId}-worker`, spec, operation.options()))
373
+ if (typeof container.Id !== "string") throw new Error("Docker worker creation failed")
374
+ containerId = container.Id
375
+ await revalidateFingerprint(options.preflight.fingerprint)
376
+ await validateWorkerMountInventory(await options.docker.inspectContainer(containerId, operation.options()), spec.HostConfig.Mounts)
377
+ const activationResponse = await abortable(options.fetchImplementation(
378
+ new URL(`/admin/grants/${encodeURIComponent(grantToken)}/activate`, options.brokerUrl), {
379
+ method: "POST",
380
+ redirect: "error",
381
+ signal: operation.signal,
382
+ headers: {authorization: `Bearer ${options.brokerAdminToken}`}
383
+ }), operation.signal)
384
+ if (!activationResponse.ok) throw new Error("Model broker grant activation failed")
385
+ await options.docker.startContainer(containerId, operation.options())
386
+ const wait = /** @type {{StatusCode?: unknown}} */ (await options.docker.waitContainer(containerId, operation.options()))
387
+ const logs = /** @type {Buffer} */ (await options.docker.logs(containerId, operation.options()))
388
+ const rawChunks = demultiplexDockerLogChunks(logs)
389
+ const result = {
390
+ exitCode: typeof wait.StatusCode === "number" ? wait.StatusCode : 1,
391
+ records: parseWorkerRecords(
392
+ Buffer.concat(rawChunks.filter((chunk) => chunk.channel === "stdout").map((chunk) => chunk.data)).toString("utf8"),
393
+ provider
394
+ ),
395
+ rawChunks: provider === "kimi"
396
+ ? []
397
+ : rawChunks.map((chunk) => ({channel: chunk.channel, data: chunk.data.toString("base64")}))
398
+ }
399
+ let sessionIds = [...new Set(result.records.filter((record) => record.type === "session").map((record) => record.sessionId))]
400
+ if (provider === "kimi" && result.exitCode !== 0) {
401
+ result.records = result.records.filter((record) => record.type !== "session")
402
+ sessionIds = []
403
+ }
404
+ if (provider === "kimi" && result.exitCode === 0) {
405
+ if (sessionIds.length !== 1) throw new Error("Kimi worker did not establish exactly one native session")
406
+ if (options.resumeSession !== undefined && sessionIds[0] !== options.resumeSession) {
407
+ throw new Error("Kimi worker did not preserve the exact resume session")
408
+ }
409
+ }
410
+ if (freshState && result.exitCode === 0) {
411
+ if (sessionIds.length === 0) throw new Error(`${providerName(provider)} session state was not established`)
412
+ await options.stateRegistry.register(sessionIds, state, options.sessionTtlMs, operation.signal)
413
+ keepFreshState = true
414
+ }
415
+ return result
416
+ } finally {
417
+ let resourcesRemoved = true
418
+ // A caller/deadline abort cannot authorize more work, but exact force
419
+ // teardown must remain possible. This supervisor-only emergency budget is
420
+ // bounded and is used solely for already-recorded resource identifiers.
421
+ const emergencyCleanup = operation.signal.aborted
422
+ ? deadlineAfter(5_000, {timeoutMessage: "Isolated runtime emergency cleanup timeout"})
423
+ : undefined
424
+ const cleanupOperation = emergencyCleanup ?? operation
425
+ const cleanupOptions = () => cleanupOperation.options()
426
+ if (grantToken !== undefined) {
427
+ await abortable(options.fetchImplementation(new URL(`/admin/grants/${encodeURIComponent(grantToken)}`, options.brokerUrl), {
428
+ method: "DELETE",
429
+ signal: cleanupOperation.signal,
430
+ headers: {authorization: `Bearer ${options.brokerAdminToken}`}
431
+ }), cleanupOperation.signal).catch(() => process.stderr.write("threadwire-isolated-runtime: grant cleanup failed\n"))
432
+ }
433
+ if (containerId !== undefined) await cleanupDocker(() => options.docker.removeContainer(containerId, cleanupOptions())).catch(() => {
434
+ resourcesRemoved = false
435
+ process.stderr.write("threadwire-isolated-runtime: worker cleanup failed\n")
436
+ })
437
+ if (networkId !== undefined) {
438
+ await cleanupDocker(() => options.docker.disconnectNetwork(networkId, options.brokerContainer, cleanupOptions())).catch(() => {})
439
+ await cleanupDocker(() => options.docker.removeNetwork(networkId, cleanupOptions())).catch(() => {
440
+ resourcesRemoved = false
441
+ process.stderr.write("threadwire-isolated-runtime: network cleanup failed\n")
442
+ })
443
+ }
444
+ if (freshState && !keepFreshState) await cleanupDocker(() => options.docker.removeVolume(state.volume, cleanupOptions())).catch(() => { resourcesRemoved = false })
445
+ if (runTracked && resourcesRemoved) await options.stateRegistry.completeRun(runId, cleanupOperation.signal).catch(() => {})
446
+ emergencyCleanup?.close()
447
+ }
448
+ }
449
+
450
+ async function cleanupDocker(operation) {
451
+ return operation()
452
+ }
453
+
454
+ export async function prepareStateVolume(docker, state, fresh, requestOptions) {
455
+ if (fresh) await docker.createVolume(state.volume, state.labels, requestOptions)
456
+ const volume = await docker.inspectVolume(state.volume, requestOptions).catch(() => undefined)
457
+ try {
458
+ validateStateVolume(volume, state.volume, state.labels)
459
+ } catch {
460
+ const provider = state.provider ?? "codex"
461
+ throw new ClientError(fresh ? `${providerName(provider)} state volume collision` : `${providerName(provider)} resume state is unavailable`)
462
+ }
463
+ }
464
+
465
+ export function validateWorkerMountInventory(container, expected) {
466
+ if (!isRecord(container) || !Array.isArray(container.Mounts) || container.Mounts.length !== expected.length) {
467
+ throw new Error("Worker mount topology unavailable")
468
+ }
469
+ for (const mount of expected) {
470
+ const actual = container.Mounts.find((entry) => isRecord(entry) && entry.Destination === mount.Target)
471
+ const sourceMatches = mount.Type === "volume" ? actual?.Name === mount.Source : actual?.Source === mount.Source
472
+ if (!actual || actual.Type !== mount.Type || !sourceMatches || actual.RW !== !mount.ReadOnly) {
473
+ throw new Error("Worker mount topology unavailable")
474
+ }
475
+ }
476
+ }
477
+
478
+ function parseWorkerRecords(text, provider = "codex") {
479
+ const records = []
480
+ let kimiSessionId
481
+ let conflictingKimiSession = false
482
+ for (const line of text.split("\n")) {
483
+ if (line.trim().length === 0) continue
484
+ try {
485
+ const record = JSON.parse(line)
486
+ if (!isRecord(record)) continue
487
+ if (provider === "kimi") {
488
+ const envelope = validatedKimiWorkerEnvelope(record)
489
+ if (envelope?.type === "session") {
490
+ if (kimiSessionId === undefined) {
491
+ kimiSessionId = envelope.sessionId
492
+ records.push(envelope)
493
+ } else if (kimiSessionId !== envelope.sessionId) conflictingKimiSession = true
494
+ } else if (envelope !== undefined) records.push(envelope)
495
+ continue
496
+ }
497
+ records.push(record)
498
+ if (record.type !== "worker-event") {
499
+ const sessionId = codexSessionId(record)
500
+ if (sessionId !== undefined) records.push({type: "session", sessionId})
501
+ for (const event of parseCodexEvent(record)) records.push({type: "worker-event", event})
502
+ }
503
+ } catch {
504
+ if (provider === "codex") {
505
+ records.push({type: "worker-event", event: {type: "diagnostic", level: "warning", summary: "Worker emitted an unreadable event"}})
506
+ }
507
+ }
508
+ }
509
+ if (conflictingKimiSession) throw new Error("Kimi worker emitted conflicting native sessions")
510
+ return records
511
+ }
512
+
513
+ function validatedKimiWorkerEnvelope(record) {
514
+ const sessionId = kimiSessionEnvelopeId(record)
515
+ if (sessionId !== undefined) return {type: "session", sessionId}
516
+ if (!exactKeys(record, ["type", "event"]) || record.type !== "worker-event" || !isRecord(record.event)) return undefined
517
+ const event = record.event
518
+ if (exactKeys(event, ["type", "text", "streamId"]) && event.type === "text-delta"
519
+ && typeof event.text === "string" && event.text.length <= 1_048_576 && event.streamId === "kimi:assistant") {
520
+ return {type: "worker-event", event: {type: "text-delta", text: event.text, streamId: "kimi:assistant"}}
521
+ }
522
+ if (exactKeys(event, ["type", "phase", "name", "key"]) && event.type === "tool"
523
+ && (event.phase === "started" || event.phase === "finished")
524
+ && typeof event.name === "string" && /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/u.test(event.name)
525
+ && typeof event.key === "string" && /^tool:[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u.test(event.key)) {
526
+ return {type: "worker-event", event: {type: "tool", phase: event.phase, name: event.name, key: event.key}}
527
+ }
528
+ if (exactKeys(event, ["type", "phase", "summary"]) && event.type === "lifecycle") {
529
+ const summaries = {
530
+ started: "Kimi worker started",
531
+ completed: "Kimi worker completed",
532
+ failed: "Kimi worker failed",
533
+ cancelled: "Kimi worker cancelled"
534
+ }
535
+ if (summaries[event.phase] === event.summary) {
536
+ return {type: "worker-event", event: {type: "lifecycle", phase: event.phase, summary: event.summary}}
537
+ }
538
+ }
539
+ return undefined
540
+ }
541
+
542
+ export function demultiplexDockerLogs(buffer) {
543
+ return Buffer.concat(demultiplexDockerLogChunks(buffer).map((chunk) => chunk.data)).toString("utf8")
544
+ }
545
+
546
+ export function demultiplexDockerLogChunks(buffer) {
547
+ let offset = 0
548
+ const output = []
549
+ let bytes = 0
550
+ while (offset + 8 <= buffer.length) {
551
+ const stream = buffer[offset]
552
+ const length = buffer.readUInt32BE(offset + 4)
553
+ if ((stream !== 1 && stream !== 2) || offset + 8 + length > buffer.length) throw new Error("Worker emitted invalid Docker logs")
554
+ bytes += length
555
+ if (bytes > MAX_RAW_OUTPUT_BYTES) throw new Error("Worker output exceeded capacity")
556
+ output.push({channel: stream === 1 ? "stdout" : "stderr", data: buffer.subarray(offset + 8, offset + 8 + length)})
557
+ offset += 8 + length
558
+ }
559
+ if (offset === buffer.length && output.length > 0) return output
560
+ if (buffer.length > MAX_RAW_OUTPUT_BYTES) throw new Error("Worker output exceeded capacity")
561
+ return [{channel: "stdout", data: buffer}]
562
+ }
563
+
564
+ export function parsePreflight(value, expectedProvider = "codex") {
565
+ if (!isRecord(value) || value.provider !== expectedProvider || (expectedProvider !== "codex" && expectedProvider !== "kimi")
566
+ || typeof value.profile !== "string" || typeof value.repositoryRoot !== "string" || typeof value.cwd !== "string"
567
+ || !Array.isArray(value.providerArguments) || !value.providerArguments.every((item) => typeof item === "string")) {
568
+ throw new ClientError("Invalid isolated runtime preflight")
569
+ }
570
+ if (!bounded(value.profile, 128) || !bounded(value.repositoryRoot, 4096) || !bounded(value.cwd, 4096)
571
+ || value.providerArguments.length > 16 || value.providerArguments.some((item) => !bounded(item, 512))
572
+ || (value.resumeSession !== undefined && !bounded(value.resumeSession, 512))) {
573
+ throw new ClientError("Invalid isolated runtime preflight")
574
+ }
575
+ if (value.resumeSession !== undefined && (typeof value.resumeSession !== "string" || /[\r\n]/u.test(value.resumeSession))) throw new ClientError("Invalid isolated runtime preflight")
576
+ return {
577
+ provider: expectedProvider,
578
+ profile: value.profile,
579
+ repositoryRoot: value.repositoryRoot,
580
+ cwd: value.cwd,
581
+ providerArguments: value.providerArguments,
582
+ ...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession})
583
+ }
584
+ }
585
+
586
+ export function parseRun(value) {
587
+ if (!isRecord(value) || typeof value.preflightId !== "string" || typeof value.prompt !== "string" || value.prompt.trim().length === 0 || !Array.isArray(value.providerArguments) || !value.providerArguments.every((item) => typeof item === "string")) {
588
+ throw new ClientError("Invalid isolated runtime run")
589
+ }
590
+ if (!bounded(value.preflightId, 64) || !bounded(value.prompt, 524_288)
591
+ || value.providerArguments.length > 16 || value.providerArguments.some((item) => !bounded(item, 512))
592
+ || (value.resumeSession !== undefined && !bounded(value.resumeSession, 512))) {
593
+ throw new ClientError("Invalid isolated runtime run")
594
+ }
595
+ if (value.resumeSession !== undefined && (typeof value.resumeSession !== "string" || /[\r\n]/u.test(value.resumeSession))) throw new ClientError("Invalid isolated runtime run")
596
+ return {
597
+ preflightId: value.preflightId,
598
+ prompt: value.prompt,
599
+ providerArguments: value.providerArguments,
600
+ ...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession})
601
+ }
602
+ }
603
+
604
+ function runtimeProviderSelection(provider, providerArguments, allowedModels) {
605
+ if (provider === "kimi") {
606
+ let selection
607
+ try { selection = validateKimiProviderArguments(providerArguments) } catch (error) {
608
+ throw new ClientError(error instanceof Error ? error.message : "Kimi provider arguments are unavailable")
609
+ }
610
+ const approved = allowedModels.get(selection.modelAlias)
611
+ if (approved === undefined) throw new ClientError("Kimi model is not allowed")
612
+ return {modelAlias: approved.alias, model: approved.model, arguments: selection.arguments}
613
+ }
614
+ let selection
615
+ try { selection = validateRelayWriteProviderArguments(providerArguments) } catch (error) {
616
+ throw new ClientError(error instanceof Error ? error.message : "Provider argument is unavailable in relay write mode")
617
+ }
618
+ if (!allowedModels.has(selection.model)) throw new ClientError("Codex model is not allowed")
619
+ return selection
620
+ }
621
+
622
+ function isolatedProvider(value) {
623
+ if (value === undefined || value === "codex") return "codex"
624
+ if (value === "kimi") return "kimi"
625
+ throw new Error("THREADWIRE_ISOLATED_PROVIDER must be codex or kimi")
626
+ }
627
+
628
+ function providerName(provider) { return provider === "kimi" ? "Kimi" : "Codex" }
629
+
630
+ function requestedDeadline(value, maximumMs) {
631
+ const parsed = Number(value)
632
+ const maximum = Date.now() + maximumMs
633
+ return Number.isSafeInteger(parsed) && parsed > Date.now() ? Math.min(parsed, maximum) : maximum
634
+ }
635
+
636
+ function parseAllowedRoots(value) {
637
+ let roots
638
+ try {
639
+ roots = JSON.parse(safeSetting(value, "THREADWIRE_ALLOWED_WORKTREE_ROOTS"))
640
+ } catch {
641
+ throw new Error("THREADWIRE_ALLOWED_WORKTREE_ROOTS must be a JSON array")
642
+ }
643
+ if (!Array.isArray(roots) || roots.length === 0 || !roots.every((root) => typeof root === "string" && isAbsolute(root) && normalize(root) === root && root !== "/")) {
644
+ throw new Error("THREADWIRE_ALLOWED_WORKTREE_ROOTS must contain normalized absolute paths")
645
+ }
646
+ return roots
647
+ }
648
+
649
+ function within(root, child) {
650
+ const path = relative(root, child)
651
+ return path === "" || (!path.startsWith("..") && !isAbsolute(path))
652
+ }
653
+
654
+ function ancestry(root, child) {
655
+ const remainder = relative(root, child)
656
+ const paths = [root]
657
+ let current = root
658
+ if (remainder === "") return paths
659
+ for (const component of remainder.split("/")) {
660
+ current = `${current}/${component}`
661
+ paths.push(current)
662
+ }
663
+ return paths
664
+ }
665
+
666
+ function taskIdentity(preflight) {
667
+ return `${preflight.provider ?? "codex"}\0${preflight.profile}\0${preflight.repositoryRoot}`
668
+ }
669
+
670
+ function brokerNetworkAddress(value, containerName) {
671
+ if (!isRecord(value) || !isRecord(value.Containers)) throw new Error("Broker network identity unavailable")
672
+ for (const [id, endpoint] of Object.entries(value.Containers)) {
673
+ if (!isRecord(endpoint)) continue
674
+ if (id === containerName || endpoint.Name === containerName) {
675
+ const address = endpoint.IPv4Address
676
+ if (typeof address !== "string" || !/^(?:\d{1,3}\.){3}\d{1,3}\/\d{1,2}$/u.test(address)) break
677
+ return address.split("/")[0]
678
+ }
679
+ }
680
+ throw new Error("Broker network identity unavailable")
681
+ }
682
+
683
+ async function readJson(request, capacity, signal) {
684
+ const chunks = []
685
+ let bytes = 0
686
+ for await (const chunk of request) {
687
+ if (signal?.aborted) throw signal.reason
688
+ const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
689
+ bytes += buffer.length
690
+ if (bytes > capacity) throw new ClientError("Request exceeded capacity")
691
+ chunks.push(buffer)
692
+ }
693
+ if (signal?.aborted) throw signal.reason
694
+ try {
695
+ return JSON.parse(Buffer.concat(chunks).toString("utf8"))
696
+ } catch {
697
+ throw new ClientError("Invalid request")
698
+ }
699
+ }
700
+
701
+ function requireControl(value, expected) {
702
+ if (value !== `Bearer ${expected}`) throw new ClientError("Request denied")
703
+ }
704
+
705
+ function normalizedHttpUrl(value, name) {
706
+ const url = new URL(safeSetting(value, name))
707
+ if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash) throw new Error(`${name} is invalid`)
708
+ return url
709
+ }
710
+
711
+ function safeSetting(value, name) {
712
+ if (typeof value !== "string" || value.length === 0 || /[\r\n]/u.test(value)) throw new Error(`${name} is required`)
713
+ return value
714
+ }
715
+
716
+ function authoritySetting(value, name) {
717
+ const authority = safeSetting(value, name)
718
+ if (authority.length < 32 || authority.length > 512) throw new Error(`${name} must be at least 32 characters`)
719
+ return authority
720
+ }
721
+
722
+ function portValue(value, fallback) {
723
+ if (value === undefined) return fallback
724
+ const number = Number(value)
725
+ if (!/^\d+$/u.test(value) || !Number.isSafeInteger(number) || number < 1 || number > 65535) throw new Error("Invalid isolated runtime port")
726
+ return number
727
+ }
728
+
729
+ function positiveDuration(value, fallback) {
730
+ if (value === undefined) return fallback
731
+ const number = Number(value)
732
+ if (!/^\d+$/u.test(value) || !Number.isSafeInteger(number) || number < 1 || number > 604_800_000) throw new Error("Invalid session TTL")
733
+ return number
734
+ }
735
+
736
+ function positiveInteger(value, fallback) {
737
+ if (value === undefined) return fallback
738
+ const number = Number(value)
739
+ if (!/^\d+$/u.test(value) || !Number.isSafeInteger(number) || number < 1 || number > 1024) throw new Error("Invalid isolated runtime capacity")
740
+ return number
741
+ }
742
+
743
+ function parseAllowedModels(value) {
744
+ let models
745
+ try {
746
+ models = JSON.parse(safeSetting(value, "THREADWIRE_ALLOWED_CODEX_MODELS"))
747
+ } catch {
748
+ throw new Error("THREADWIRE_ALLOWED_CODEX_MODELS must be a JSON array")
749
+ }
750
+ if (!Array.isArray(models) || models.length === 0 || !models.every((model) => typeof model === "string" && /^[A-Za-z0-9._-]{1,128}$/u.test(model))) {
751
+ throw new Error("THREADWIRE_ALLOWED_CODEX_MODELS must contain model identifiers")
752
+ }
753
+ return new Set(models)
754
+ }
755
+
756
+ function sendJson(response, status, value) {
757
+ response.writeHead(status, {"content-type": "application/json"})
758
+ response.end(JSON.stringify(value))
759
+ }
760
+
761
+ function isRecord(value) {
762
+ return typeof value === "object" && value !== null && !Array.isArray(value)
763
+ }
764
+
765
+ function exactKeys(value, expected) {
766
+ const actual = Object.keys(value).sort()
767
+ const wanted = [...expected].sort()
768
+ return actual.length === wanted.length && actual.every((key, index) => key === wanted[index])
769
+ }
770
+
771
+ function abortable(promise, signal) {
772
+ if (signal.aborted) return Promise.reject(signal.reason)
773
+ return new Promise((resolve, reject) => {
774
+ const abort = () => reject(signal.reason instanceof Error ? signal.reason : new Error("Isolated runtime operation aborted"))
775
+ signal.addEventListener("abort", abort, {once: true})
776
+ promise.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort))
777
+ })
778
+ }
779
+
780
+ function listen(server, port, host) {
781
+ return new Promise((resolve, reject) => {
782
+ server.once("error", reject)
783
+ server.listen(port, host, () => {
784
+ server.removeListener("error", reject)
785
+ resolve(undefined)
786
+ })
787
+ })
788
+ }
789
+
790
+ class ClientError extends Error {}
791
+
792
+ function bounded(value, maximum) {
793
+ return typeof value === "string" && value.length > 0 && value.length <= maximum && !/[\0\r\n]/u.test(value)
794
+ }
795
+
796
+ export class PreflightStore {
797
+ constructor({now = Date.now, ttlMs = PREFLIGHT_TTL_MS, capacity = PREFLIGHT_CAPACITY, taskCapacity = PREFLIGHT_TASK_CAPACITY} = {}) {
798
+ this.now = now
799
+ this.ttlMs = ttlMs
800
+ this.capacity = capacity
801
+ this.taskCapacity = taskCapacity
802
+ this.entries = new Map()
803
+ this.timer = setInterval(() => this.sweep(), Math.max(10, Math.min(ttlMs, 1000)))
804
+ this.timer.unref()
805
+ }
806
+ sweep() {
807
+ const current = this.now()
808
+ for (const [id, entry] of this.entries) if (entry.expiresAt <= current) this.entries.delete(id)
809
+ }
810
+ add(id, task, value) {
811
+ this.sweep()
812
+ this.assertCapacity(task)
813
+ this.entries.set(id, {task, expiresAt: this.now() + this.ttlMs, value})
814
+ }
815
+ assertCapacity(task) {
816
+ this.sweep()
817
+ let taskCount = 0
818
+ for (const entry of this.entries.values()) if (entry.task === task) taskCount++
819
+ if (this.entries.size >= this.capacity || taskCount >= this.taskCapacity) throw new ClientError("Isolated runtime preflight capacity exhausted")
820
+ }
821
+ consume(id) {
822
+ this.sweep()
823
+ const entry = this.entries.get(id)
824
+ this.entries.delete(id)
825
+ return entry?.value
826
+ }
827
+ get size() { return this.entries.size }
828
+ close() {
829
+ clearInterval(this.timer)
830
+ this.entries.clear()
831
+ }
832
+ }
833
+
834
+ export class LineageReservations {
835
+ constructor() {
836
+ this.active = new Set()
837
+ this.collecting = false
838
+ }
839
+ acquire(lineage) {
840
+ if (this.active.has(lineage)) throw new ClientError("Codex resume state is already active")
841
+ this.active.add(lineage)
842
+ let released = false
843
+ return () => {
844
+ if (released) return
845
+ released = true
846
+ this.active.delete(lineage)
847
+ }
848
+ }
849
+ beginCollection() {
850
+ if (this.collecting) throw new Error("Codex state collection overlap")
851
+ this.collecting = true
852
+ return () => { this.collecting = false }
853
+ }
854
+ values() { return this.active.values() }
855
+ has(lineage) { return this.active.has(lineage) }
856
+ }
857
+
858
+ export class ActiveRunAdmission {
859
+ constructor({capacity = ACTIVE_RUN_CAPACITY, taskCapacity = ACTIVE_TASK_CAPACITY} = {}) {
860
+ this.capacity = capacity
861
+ this.taskCapacity = taskCapacity
862
+ this.active = new Map()
863
+ }
864
+ acquire(task) {
865
+ const taskCount = this.active.get(task) ?? 0
866
+ let total = 0
867
+ for (const count of this.active.values()) total += count
868
+ if (total >= this.capacity || taskCount >= this.taskCapacity) throw new ClientError("Isolated runtime active-run capacity exhausted")
869
+ this.active.set(task, taskCount + 1)
870
+ let released = false
871
+ return () => {
872
+ if (released) return
873
+ released = true
874
+ const remaining = (this.active.get(task) ?? 1) - 1
875
+ if (remaining === 0) this.active.delete(task)
876
+ else this.active.set(task, remaining)
877
+ }
878
+ }
879
+ }
880
+
881
+ export class OperationDeadline extends AbsoluteDeadline {
882
+ constructor(timeoutMs, request, response, now = Date.now) {
883
+ super(now() + timeoutMs, {request, response, now, timeoutMessage: "Isolated runtime operation timed out", disconnectMessage: "Isolated runtime caller disconnected"})
884
+ }
885
+ }
886
+
887
+ function boundedClose(server, timeoutMs) {
888
+ return Promise.race([
889
+ new Promise((resolve) => server.close(() => resolve(undefined))),
890
+ new Promise((resolve) => {
891
+ const timer = setTimeout(resolve, timeoutMs)
892
+ timer.unref()
893
+ })
894
+ ])
895
+ }
896
+
897
+ export async function reconcileDockerResources(docker, brokerContainer, stateRegistry, inProcessLineages = new Set(), requestOptions) {
898
+ const namespace = stateRegistry.namespace
899
+ const activeLineages = new Set(inProcessLineages)
900
+ const activeRuns = new Set()
901
+ const ownedIdentities = new Map(stateRegistry.runRecords().map((identity) => [identity.run, identity]))
902
+ const cleanupFailed = new Set()
903
+ const containers = await docker.listContainers({"org.threadwire.owner": "isolated-runtime"}, true, requestOptions)
904
+ for (const container of Array.isArray(containers) ? containers : []) {
905
+ const labels = validResourceLabels(container?.Labels, namespace, true, stateRegistry.provider ?? "codex")
906
+ if (!labels) continue
907
+ const persisted = ownedIdentities.get(labels.run)
908
+ if (!persisted || !stateRegistry.ownsRunSeal(persisted, labels.seal, labels.legacy)) continue
909
+ const inspection = await docker.inspectContainer(container.Id, requestOptions).catch(() => undefined)
910
+ const identity = ownedContainerIdentity(inspection, labels)
911
+ if (!identity || !stateRegistry.ownsRun(identity, labels.seal, labels.legacy)) {
912
+ cleanupFailed.add(labels.run)
913
+ continue
914
+ }
915
+ if (inProcessLineages.has(labels.lineage)) {
916
+ activeLineages.add(labels.lineage)
917
+ activeRuns.add(labels.run)
918
+ continue
919
+ }
920
+ if (inProcessLineages.has(labels.lineage)) {
921
+ activeLineages.add(labels.lineage)
922
+ activeRuns.add(labels.run)
923
+ continue
924
+ }
925
+ await docker.removeContainer(container.Id, requestOptions).catch(() => cleanupFailed.add(labels.run))
926
+ }
927
+ const networks = await docker.listNetworks({"org.threadwire.owner": "isolated-runtime"}, requestOptions)
928
+ for (const network of Array.isArray(networks) ? networks : []) {
929
+ const labels = validResourceLabels(network?.Labels, namespace, false, stateRegistry.provider ?? "codex")
930
+ if (!labels || activeRuns.has(labels.run) || cleanupFailed.has(labels.run)) continue
931
+ const identity = ownedIdentities.get(labels.run)
932
+ if (!identity || !stateRegistry.ownsRunSeal(identity, labels.seal, labels.legacy)) continue
933
+ const inspection = await docker.inspectNetwork(network.Id, requestOptions).catch(() => undefined)
934
+ if (!identity || !ownedNetworkIdentity(inspection, labels, identity, brokerContainer)
935
+ || !stateRegistry.ownsRun(identity, labels.seal, labels.legacy)) {
936
+ cleanupFailed.add(labels.run)
937
+ continue
938
+ }
939
+ if (inProcessLineages.has(labels.lineage)) {
940
+ activeLineages.add(labels.lineage)
941
+ activeRuns.add(labels.run)
942
+ continue
943
+ }
944
+ const brokerEndpoint = Object.entries(inspection.Containers ?? {}).find(([, endpoint]) => endpoint?.Name === brokerContainer)
945
+ if (brokerEndpoint) await docker.disconnectNetwork(network.Id, brokerContainer, requestOptions).catch(() => {})
946
+ await docker.removeNetwork(network.Id, requestOptions).catch(() => cleanupFailed.add(labels.run))
947
+ }
948
+ for (const identity of ownedIdentities.values()) {
949
+ if (activeRuns.has(identity.run) || cleanupFailed.has(identity.run)) {
950
+ activeLineages.add(identity.lineage)
951
+ continue
952
+ }
953
+ if (inProcessLineages.has(identity.lineage)) {
954
+ activeLineages.add(identity.lineage)
955
+ continue
956
+ }
957
+ await stateRegistry.completeRun(identity.run, requestOptions?.signal).catch(() => activeLineages.add(identity.lineage))
958
+ }
959
+ return [...activeLineages]
960
+ }
961
+
962
+ function validResourceLabels(value, namespace, container, expectedProvider = "codex") {
963
+ if (!isRecord(value)) return undefined
964
+ const provider = value["org.threadwire.provider"] ?? "codex"
965
+ const legacy = value["org.threadwire.provider"] === undefined
966
+ const expectedKeys = [
967
+ "org.threadwire.owner", "org.threadwire.namespace", "org.threadwire.run",
968
+ "org.threadwire.lineage", "org.threadwire.task", "org.threadwire.state-volume",
969
+ "org.threadwire.run-seal",
970
+ ...(legacy ? [] : ["org.threadwire.provider"]),
971
+ ...(container ? ["org.threadwire.network"] : [])
972
+ ]
973
+ if (provider !== expectedProvider || (legacy && provider !== "codex")
974
+ || Object.keys(value).length !== expectedKeys.length || expectedKeys.some((key) => typeof value[key] !== "string")) return undefined
975
+ const run = value["org.threadwire.run"]
976
+ const lineage = value["org.threadwire.lineage"]
977
+ const volume = value["org.threadwire.state-volume"]
978
+ const seal = value["org.threadwire.run-seal"]
979
+ if (value["org.threadwire.owner"] !== "isolated-runtime" || value["org.threadwire.namespace"] !== namespace
980
+ || !/^tw-[0-9a-f]{24}$/u.test(run) || !/^[0-9a-f]{48}$/u.test(lineage)
981
+ || !/^[0-9a-f]{64}$/u.test(value["org.threadwire.task"])
982
+ || !/^[0-9a-f]{64}$/u.test(seal)
983
+ || volume !== `threadwire-state-${namespace}-${lineage}`
984
+ || (container && value["org.threadwire.network"] !== `${run}-net`)) return undefined
985
+ return {provider, legacy, run, lineage, volume, seal, task: value["org.threadwire.task"], namespace}
986
+ }
987
+
988
+ function ownedContainerIdentity(container, labels) {
989
+ if (!isRecord(container) || container.Name !== `/${labels.run}-worker`
990
+ || !isRecord(container.Config) || !isRecord(container.HostConfig)
991
+ || container.Config.Image !== container.Image || !isDigestImage(container.Config.Image)
992
+ || JSON.stringify(container.Config.Entrypoint) !== JSON.stringify(labels.provider === "kimi"
993
+ ? ["node", "/opt/threadwire/docker/kimi-worker-entrypoint.mjs"]
994
+ : ["/usr/local/libexec/threadwire/worker-entrypoint"])
995
+ || ![null, undefined].includes(container.Config.Cmd) || container.Config.User !== "10002:10002"
996
+ || typeof container.Config.WorkingDir !== "string" || !within("/worktree", container.Config.WorkingDir)
997
+ || container.HostConfig.NetworkMode !== `${labels.run}-net`
998
+ || !validWorkerSecurity(container.Config, container.HostConfig, labels.provider)
999
+ || !Array.isArray(container.HostConfig.Mounts) || container.HostConfig.Mounts.length !== 2) return undefined
1000
+ const inspectedLabels = validResourceLabels(container.Config.Labels, labels.namespace, true, labels.provider)
1001
+ if (!inspectedLabels || JSON.stringify(inspectedLabels) !== JSON.stringify(labels)) return undefined
1002
+ const state = container.HostConfig.Mounts.find((mount) => mount?.Target === "/home/worker")
1003
+ const worktree = container.HostConfig.Mounts.find((mount) => mount?.Target === "/worktree")
1004
+ if (state?.Type !== "volume" || state.Source !== labels.volume || state.ReadOnly === true
1005
+ || Object.keys(state).some((key) => !["Type", "Source", "Target", "ReadOnly"].includes(key))) return undefined
1006
+ if (!validWorktreeMount(worktree)) return undefined
1007
+ return runIdentityFrom(labels, container.Config.Image, worktree)
1008
+ }
1009
+
1010
+ function validWorkerSecurity(config, host, provider = "codex") {
1011
+ if (host.AutoRemove !== false || host.ReadonlyRootfs !== true
1012
+ || host.Privileged === true || JSON.stringify(host.CapDrop) !== JSON.stringify(["ALL"])
1013
+ || JSON.stringify(host.CapAdd ?? null) !== "null"
1014
+ || JSON.stringify(host.SecurityOpt) !== JSON.stringify(["no-new-privileges"])
1015
+ || host.PidsLimit !== 128 || host.Memory !== 1_073_741_824 || host.NanoCpus !== 2_000_000_000
1016
+ || !isRecord(host.Tmpfs) || Object.keys(host.Tmpfs).length !== 2
1017
+ || host.Tmpfs["/tmp"] !== "rw,noexec,nosuid,nodev,size=67108864,uid=10002,gid=10002"
1018
+ || host.Tmpfs["/run"] !== "rw,noexec,nosuid,nodev,size=1048576,uid=10002,gid=10002"
1019
+ || !Array.isArray(host.Ulimits) || host.Ulimits.length !== 3
1020
+ || ![
1021
+ ["nofile", 1024, 1024], ["core", 0, 0], ["fsize", 1_073_741_824, 1_073_741_824]
1022
+ ].every(([name, soft, hard], index) => host.Ulimits[index]?.Name === name
1023
+ && host.Ulimits[index]?.Soft === soft && host.Ulimits[index]?.Hard === hard)) return false
1024
+ const forbidden = {
1025
+ Binds: [null, []], Devices: [null, []], DeviceRequests: [null, []], PortBindings: [null, {}],
1026
+ Links: [null, []], ExtraHosts: [null, []], VolumesFrom: [null, []], Dns: [null, []],
1027
+ DnsSearch: [null, []], Sysctls: [null, {}]
1028
+ }
1029
+ for (const [key, defaults] of Object.entries(forbidden)) {
1030
+ if (!defaults.some((value) => JSON.stringify(host[key] ?? null) === JSON.stringify(value))) return false
1031
+ }
1032
+ if (![undefined, "", "private"].includes(host.PidMode) || ![undefined, "", "private"].includes(host.IpcMode)
1033
+ || ![undefined, ""].includes(host.UTSMode) || ![undefined, ""].includes(host.UsernsMode)
1034
+ || ![undefined, "", "private"].includes(host.CgroupnsMode) || host.PublishAllPorts === true) return false
1035
+ if (!Array.isArray(config.Env)) return false
1036
+ const names = config.Env.map((entry) => entry.split("=", 1)[0])
1037
+ const resumeCount = names.filter((name) => name === "THREADWIRE_RESUME_SESSION").length
1038
+ if (resumeCount > 1) return false
1039
+ if (provider === "kimi") {
1040
+ const expected = [
1041
+ "HOME", "TMPDIR", "THREADWIRE_PROMPT", "THREADWIRE_WORKTREE_INODE", "THREADWIRE_CWD_INODE",
1042
+ "KIMI_CODE_HOME", "THREADWIRE_KIMI_MODEL", "THREADWIRE_KIMI_BROKER_URL", "THREADWIRE_KIMI_BROKER_TOKEN"
1043
+ ]
1044
+ return names.length === expected.length + resumeCount
1045
+ && expected.every((name) => names.filter((entry) => entry === name).length === 1)
1046
+ && !names.some((name) => /^(?:CODEX|OPENAI|TELEGRAM|THREADWIRE_KIMI_(?:OAUTH|CONTROL))/u.test(name))
1047
+ }
1048
+ if (config.Env.length !== 13 + resumeCount) return false
1049
+ const expected = [
1050
+ "HOME", "CODEX_HOME", "TMPDIR", "OPENAI_BASE_URL", "OPENAI_API_KEY", "CODEX_API_KEY",
1051
+ "THREADWIRE_PROMPT", "THREADWIRE_CODEX_ARGUMENTS", "THREADWIRE_WORKTREE_INODE", "THREADWIRE_CWD_INODE"
1052
+ ]
1053
+ return expected.every((name) => names.filter((entry) => entry === name).length === 1)
1054
+ && config.Env.includes("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin")
1055
+ && config.Env.includes("NODE_VERSION=22.17.0") && config.Env.includes("YARN_VERSION=1.22.22")
1056
+ }
1057
+
1058
+ function ownedNetworkIdentity(network, labels, identity, brokerContainer) {
1059
+ if (!isRecord(network) || network.Name !== `${labels.run}-net` || network.Internal !== true
1060
+ || network.Driver !== "bridge" || network.Scope !== "local"
1061
+ || network.EnableIPv6 !== false || network.Attachable !== false || network.Ingress !== false
1062
+ || network.ConfigOnly !== false || !isRecord(network.Options) || Object.keys(network.Options).length !== 0
1063
+ || !isRecord(network.Containers)) return undefined
1064
+ const inspectedLabels = validResourceLabels(network.Labels, labels.namespace, false, labels.provider)
1065
+ if (!inspectedLabels || JSON.stringify(inspectedLabels) !== JSON.stringify(labels)) return undefined
1066
+ const endpoints = Object.values(network.Containers)
1067
+ if (endpoints.some((endpoint) => !isRecord(endpoint)
1068
+ || ![identity.container, brokerContainer].includes(endpoint.Name))) return undefined
1069
+ return true
1070
+ }
1071
+
1072
+ function validWorktreeMount(mount) {
1073
+ if (!isRecord(mount) || mount.Target !== "/worktree" || mount.ReadOnly === true) return false
1074
+ if (mount.Type === "bind") {
1075
+ return Object.keys(mount).every((key) => ["Type", "Source", "Target", "ReadOnly", "BindOptions"].includes(key))
1076
+ && typeof mount.Source === "string" && isAbsolute(mount.Source)
1077
+ && isRecord(mount.BindOptions) && Object.keys(mount.BindOptions).length === 1
1078
+ && mount.BindOptions.Propagation === "rprivate"
1079
+ }
1080
+ return mount.Type === "volume" && typeof mount.Source === "string"
1081
+ && Object.keys(mount).every((key) => ["Type", "Source", "Target", "ReadOnly", "VolumeOptions"].includes(key))
1082
+ && isRecord(mount.VolumeOptions) && Object.keys(mount.VolumeOptions).length === 1
1083
+ && typeof mount.VolumeOptions.Subpath === "string" && mount.VolumeOptions.Subpath.length > 0
1084
+ }
1085
+
1086
+ function runIdentityFrom(labels, image, worktree) {
1087
+ return {
1088
+ provider: labels.provider, namespace: labels.namespace, run: labels.run, lineage: labels.lineage, task: labels.task,
1089
+ volume: labels.volume, network: `${labels.run}-net`, container: `${labels.run}-worker`,
1090
+ image, worktreeType: worktree.Type, worktreeSource: worktree.Source,
1091
+ worktreeSubpath: worktree.Type === "volume" ? worktree.VolumeOptions.Subpath : ""
1092
+ }
1093
+ }
1094
+
1095
+ function expectedRunIdentity(options, run, network, state, worktreeSubpath) {
1096
+ const volumeMode = options.worktreeVolume !== undefined
1097
+ return {
1098
+ provider: options.preflight.provider ?? "codex",
1099
+ namespace: options.stateRegistry.namespace, run, lineage: state.lineage,
1100
+ task: state.labels["org.threadwire.task"], volume: state.volume, network,
1101
+ container: `${run}-worker`, image: options.workerImage,
1102
+ worktreeType: volumeMode ? "volume" : "bind",
1103
+ worktreeSource: volumeMode ? options.worktreeVolume : options.preflight.repositoryRoot,
1104
+ worktreeSubpath: volumeMode ? worktreeSubpath : ""
1105
+ }
1106
+ }
1107
+
1108
+ export function deriveVolumeWorktreeSubpath(worktreeRoot, repositoryRoot) {
1109
+ if (typeof worktreeRoot !== "string" || typeof repositoryRoot !== "string" || !within(worktreeRoot, repositoryRoot)) {
1110
+ throw new ClientError("Configured worktree has an invalid volume subpath")
1111
+ }
1112
+ const subpath = relative(worktreeRoot, repositoryRoot)
1113
+ if (subpath === "" || isAbsolute(subpath) || normalize(subpath) !== subpath || subpath.split("/").includes("..")) {
1114
+ throw new ClientError("Configured worktree has an invalid volume subpath")
1115
+ }
1116
+ return subpath
1117
+ }