threadwire 0.1.11 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +10 -1
- package/TELEGRAM-INGRESS.md +2 -0
- package/docs/container-runtime.md +39 -3
- package/docs/isolated-provider-runtime.md +90 -12
- package/package.json +1 -1
- package/scripts/verify-package.js +1 -0
- package/src/absolute-deadline.js +8 -5
- package/src/cli.js +14 -3
- package/src/docker-api.js +100 -14
- package/src/isolated-runtime-client.js +116 -44
- package/src/isolated-runtime.js +673 -64
- package/src/isolated-state.js +62 -11
- package/src/isolated-worker.js +231 -23
- package/src/kimi-model-broker-policy.js +16 -5
- package/src/kimi-model-broker.js +134 -92
- package/src/model-broker-policy.js +11 -8
- package/src/model-broker.js +14 -0
- package/src/providers/kimi.js +15 -3
- package/src/telegram-ingress/core.js +7 -5
- package/src/telegram-webhook.js +10 -0
- package/src/threadwire-binding.js +192 -0
- package/src/workspace-profile.js +31 -8
- package/threadwire.workspace-profiles.json +3 -2
package/src/isolated-runtime.js
CHANGED
|
@@ -2,15 +2,17 @@
|
|
|
2
2
|
/* eslint-disable jsdoc/require-jsdoc */
|
|
3
3
|
|
|
4
4
|
import {randomBytes} from "node:crypto"
|
|
5
|
+
import {once} from "node:events"
|
|
5
6
|
import {lstat, realpath} from "node:fs/promises"
|
|
6
7
|
import {createServer} from "node:http"
|
|
7
8
|
import {isAbsolute, join, normalize, relative} from "node:path"
|
|
8
9
|
import {DockerApi} from "./docker-api.js"
|
|
9
|
-
import {buildWorkerContainerSpec, isDigestImage} from "./isolated-worker.js"
|
|
10
|
+
import {buildKimiBindingValidatorSpec, buildKimiRelayContainerSpec, buildWorkerContainerSpec, isDigestImage} from "./isolated-worker.js"
|
|
10
11
|
import {validateRelayWriteProviderArguments} from "./relay-write.js"
|
|
11
12
|
import {codexSessionId, parseCodexEvent} from "./providers/codex.js"
|
|
12
13
|
import {kimiSessionEnvelopeId, validateKimiProviderArguments} from "./providers/kimi.js"
|
|
13
14
|
import {parseApprovedKimiModels} from "./kimi-model-broker-policy.js"
|
|
15
|
+
import {parseThreadwireBinding, resumeTaskIdentity, threadwireBindingDigest} from "./threadwire-binding.js"
|
|
14
16
|
import {openStateRegistry, validateStateVolume} from "./isolated-state.js"
|
|
15
17
|
import {assertNoNestedMounts} from "./mount-policy.js"
|
|
16
18
|
import {AbsoluteDeadline, deadlineAfter} from "./absolute-deadline.js"
|
|
@@ -23,7 +25,8 @@ const PREFLIGHT_TASK_CAPACITY = 8
|
|
|
23
25
|
const GC_INTERVAL_MS = 5_000
|
|
24
26
|
const ACTIVE_RUN_CAPACITY = 16
|
|
25
27
|
const ACTIVE_TASK_CAPACITY = 2
|
|
26
|
-
const
|
|
28
|
+
const GRANT_LEASE_MS = 60_000
|
|
29
|
+
const GRANT_LEASE_MAX_MS = 3_600_000
|
|
27
30
|
const PREFLIGHT_TIMEOUT_MS = 30_000
|
|
28
31
|
const MAX_RAW_OUTPUT_BYTES = 1_048_576
|
|
29
32
|
|
|
@@ -46,12 +49,20 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
46
49
|
const brokerAdminToken = provider === "kimi"
|
|
47
50
|
? authoritySetting(environment[brokerAdminName], brokerAdminName)
|
|
48
51
|
: safeSetting(environment[brokerAdminName], brokerAdminName)
|
|
49
|
-
const brokerContainer = safeSetting(environment[brokerContainerName], brokerContainerName)
|
|
50
|
-
const
|
|
52
|
+
const brokerContainer = provider === "kimi" ? undefined : safeSetting(environment[brokerContainerName], brokerContainerName)
|
|
53
|
+
const relayImage = provider === "kimi"
|
|
54
|
+
? safeSetting(environment.THREADWIRE_KIMI_MODEL_RELAY_IMAGE, "THREADWIRE_KIMI_MODEL_RELAY_IMAGE")
|
|
55
|
+
: undefined
|
|
56
|
+
if (relayImage !== undefined && !isDigestImage(relayImage)) throw new Error("THREADWIRE_KIMI_MODEL_RELAY_IMAGE must use an immutable digest")
|
|
57
|
+
const relayWorkerUrl = provider === "kimi"
|
|
58
|
+
? normalizedHttpUrl(environment.THREADWIRE_KIMI_MODEL_BROKER_WORKER_URL, "THREADWIRE_KIMI_MODEL_BROKER_WORKER_URL")
|
|
59
|
+
: undefined
|
|
60
|
+
const allowedRoots = provider === "kimi" ? [] : parseAllowedRoots(environment.THREADWIRE_ALLOWED_WORKTREE_ROOTS)
|
|
51
61
|
const worktreeVolume = environment.THREADWIRE_WORKTREE_VOLUME
|
|
52
62
|
const sessionTtlMs = positiveDuration(environment.THREADWIRE_SESSION_TTL_MS, 86_400_000)
|
|
53
|
-
const workerTimeoutMs =
|
|
63
|
+
const workerTimeoutMs = optionalDuration(environment.THREADWIRE_WORKER_TIMEOUT_MS)
|
|
54
64
|
const preflightTimeoutMs = positiveDuration(environment.THREADWIRE_PREFLIGHT_TIMEOUT_MS, PREFLIGHT_TIMEOUT_MS)
|
|
65
|
+
const grantLeaseMs = grantLeaseSetting(environment.THREADWIRE_GRANT_LEASE_MS)
|
|
55
66
|
const docker = options.docker ?? new DockerApi()
|
|
56
67
|
const fetchImplementation = options.fetchImplementation ?? fetch
|
|
57
68
|
const now = options.now ?? Date.now
|
|
@@ -84,7 +95,7 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
84
95
|
const collectState = () => {
|
|
85
96
|
gcPromise = gcPromise.then(async () => {
|
|
86
97
|
if (shutdown.signal.aborted) return
|
|
87
|
-
const collection = deadlineAfter(
|
|
98
|
+
const collection = deadlineAfter(30_000, {signal: shutdown.signal, timeoutMessage: "Isolated runtime collection timeout"})
|
|
88
99
|
const releaseCollection = activeLineages.beginCollection()
|
|
89
100
|
try {
|
|
90
101
|
const active = new Set(activeLineages.values())
|
|
@@ -117,7 +128,7 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
117
128
|
if (request.method === "POST" && request.url === "/preflight") {
|
|
118
129
|
const releaseIncoming = incomingRequests.acquire("preflight")
|
|
119
130
|
const launchDeadlineAt = requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)
|
|
120
|
-
const operation = new AbsoluteDeadline(
|
|
131
|
+
const operation = new AbsoluteDeadline(minimumDeadline(Date.now() + preflightTimeoutMs, launchDeadlineAt), {
|
|
121
132
|
signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime operation timed out", disconnectMessage: "Isolated runtime caller disconnected"
|
|
122
133
|
})
|
|
123
134
|
try {
|
|
@@ -137,8 +148,8 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
137
148
|
try { validateStateVolume(volume, state.volume, state.labels) } catch { throw new ClientError(`${providerName(provider)} resume state is unavailable`) }
|
|
138
149
|
} else state = stateRegistry.allocate(task)
|
|
139
150
|
const preflightId = randomBytes(24).toString("base64url")
|
|
140
|
-
preflights.add(preflightId, task, {...selection, providerSelection, state, fingerprint, worktreeRoot, launchDeadlineAt})
|
|
141
|
-
sendJson(response, 200, {ok: true, preflightId, launchDeadlineAt})
|
|
151
|
+
preflights.add(preflightId, task, {...selection, provider, providerSelection, state, fingerprint, worktreeRoot, ...(launchDeadlineAt === undefined ? {} : {launchDeadlineAt})})
|
|
152
|
+
sendJson(response, 200, {ok: true, preflightId, ...(launchDeadlineAt === undefined ? {} : {launchDeadlineAt})})
|
|
142
153
|
} finally {
|
|
143
154
|
operation.close()
|
|
144
155
|
releaseIncoming()
|
|
@@ -157,7 +168,7 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
157
168
|
}
|
|
158
169
|
const preflight = preflights.consume(body.preflightId)
|
|
159
170
|
if (!preflight) throw new ClientError("Isolated runtime preflight expired")
|
|
160
|
-
const operation = new AbsoluteDeadline(
|
|
171
|
+
const operation = new AbsoluteDeadline(minimumDeadline(preflight.launchDeadlineAt,
|
|
161
172
|
requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)), {
|
|
162
173
|
signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime launch timed out", disconnectMessage: "Isolated runtime caller disconnected"
|
|
163
174
|
})
|
|
@@ -173,12 +184,19 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
173
184
|
}
|
|
174
185
|
await revalidateFingerprint(preflight.fingerprint)
|
|
175
186
|
operation.throwIfAborted()
|
|
187
|
+
const writeFrame = async (frame) => {
|
|
188
|
+
if (!response.headersSent) response.writeHead(200, {"content-type": "application/x-ndjson"})
|
|
189
|
+
if (!response.write(`${JSON.stringify(frame)}\n`)) await abortable(once(response, "drain"), operation.signal)
|
|
190
|
+
}
|
|
176
191
|
const result = await runIsolatedWorker({
|
|
177
|
-
docker, workerImage, brokerUrl, brokerAdminToken, brokerContainer,
|
|
178
|
-
fetchImplementation, preflight, ...body, task, now, stateRegistry, sessionTtlMs, operation,
|
|
192
|
+
docker, workerImage, brokerUrl, brokerAdminToken, brokerContainer, relayImage, relayWorkerUrl,
|
|
193
|
+
fetchImplementation, preflight, ...body, task, now, stateRegistry, sessionTtlMs, operation, grantLeaseMs,
|
|
194
|
+
onRecord: (record) => writeFrame({type: "record", record}),
|
|
179
195
|
...(worktreeVolume === undefined ? {} : {worktreeVolume})
|
|
180
196
|
})
|
|
181
|
-
|
|
197
|
+
for (const chunk of result.rawChunks) await writeFrame({type: "chunk", ...chunk})
|
|
198
|
+
await writeFrame({type: "terminal", exitCode: result.exitCode})
|
|
199
|
+
response.end()
|
|
182
200
|
} finally {
|
|
183
201
|
operation.close()
|
|
184
202
|
releaseLineage?.()
|
|
@@ -190,6 +208,10 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
190
208
|
response.writeHead(404)
|
|
191
209
|
response.end()
|
|
192
210
|
} catch (error) {
|
|
211
|
+
if (response.headersSent) {
|
|
212
|
+
if (!response.writableEnded && !response.destroyed) response.end(`${JSON.stringify({type: "terminal", exitCode: 1})}\n`)
|
|
213
|
+
return
|
|
214
|
+
}
|
|
193
215
|
sendJson(response, error instanceof ClientError ? 400 : 503, {
|
|
194
216
|
error: error instanceof ClientError ? error.message : "Isolated runtime unavailable"
|
|
195
217
|
})
|
|
@@ -221,7 +243,9 @@ async function verifyPrerequisites(options) {
|
|
|
221
243
|
if (typeof version.ApiVersion !== "string") throw new Error("Docker API unavailable")
|
|
222
244
|
const image = /** @type {{Id?: unknown}} */ (await docker.inspectImage(workerImage, operation.options()))
|
|
223
245
|
if (typeof image.Id !== "string" || !isDigestImage(image.Id)) throw new Error("Worker image unavailable")
|
|
224
|
-
const validatedWorktree =
|
|
246
|
+
const validatedWorktree = selection.provider === "kimi"
|
|
247
|
+
? await verifyKimiBindingAdmission(docker, selection.binding, operation.options())
|
|
248
|
+
: await validateWorktreeSelection(selection.repositoryRoot, selection.cwd, allowedRoots)
|
|
225
249
|
const health = await abortable(fetchImplementation(new URL("/healthz", brokerUrl), {
|
|
226
250
|
method: "GET",
|
|
227
251
|
redirect: "error",
|
|
@@ -232,6 +256,51 @@ async function verifyPrerequisites(options) {
|
|
|
232
256
|
return validatedWorktree
|
|
233
257
|
}
|
|
234
258
|
|
|
259
|
+
async function verifyKimiBindingAdmission(docker, value, requestOptions) {
|
|
260
|
+
const binding = parseThreadwireBinding(value)
|
|
261
|
+
const source = await docker.inspectVolume(binding.source.volume, requestOptions)
|
|
262
|
+
const context = await docker.inspectVolume(binding.context.volume, requestOptions)
|
|
263
|
+
const expected = {"org.threadwire.owner": "threadwire-task", "org.threadwire.binding-schema": "1", "org.threadwire.task": binding.taskId}
|
|
264
|
+
const labelsMatch = (labels, extra) => isRecord(labels)
|
|
265
|
+
&& Object.entries({...expected, ...extra}).every(([key, item]) => labels[key] === item)
|
|
266
|
+
if (!isRecord(source) || source.Name !== binding.source.volume
|
|
267
|
+
|| !isRecord(source.Labels) || source.Labels["org.threadwire.owner"] !== "threadwire-task"
|
|
268
|
+
|| source.Labels["org.threadwire.task"] !== binding.taskId || source.Labels["org.threadwire.role"] !== "source"
|
|
269
|
+
|| (source.Labels["org.threadwire.binding-schema"] !== undefined && source.Labels["org.threadwire.binding-schema"] !== "1")) {
|
|
270
|
+
throw new ClientError("Threadwire binding is unavailable")
|
|
271
|
+
}
|
|
272
|
+
if (!isRecord(context) || context.Name !== binding.context.volume || !labelsMatch(context.Labels, {
|
|
273
|
+
"org.threadwire.role": "context",
|
|
274
|
+
"org.threadwire.context-manifest-sha256": binding.context.digests.manifest,
|
|
275
|
+
"org.threadwire.context-content-sha256": binding.context.digests.content,
|
|
276
|
+
"org.threadwire.context-image-id": binding.context.imageId
|
|
277
|
+
})) throw new ClientError("Threadwire binding is unavailable")
|
|
278
|
+
const lease = await docker.inspectContainer(binding.leaseContainerId, requestOptions)
|
|
279
|
+
const configuredMounts = lease?.HostConfig?.Mounts
|
|
280
|
+
const effectiveMounts = lease?.Mounts
|
|
281
|
+
const leaseLabels = lease?.Config?.Labels
|
|
282
|
+
const expectedUser = binding.runtime.uid + ":" + binding.runtime.gid
|
|
283
|
+
if (!isRecord(lease) || lease.Id !== binding.leaseContainerId || lease.Image !== binding.context.imageId || lease.State?.Running !== true
|
|
284
|
+
|| lease.Config?.User !== expectedUser || lease.Config?.WorkingDir !== binding.runtime.workdir
|
|
285
|
+
|| !labelsMatch(leaseLabels, {"org.threadwire.role": "lease"})
|
|
286
|
+
|| !Array.isArray(configuredMounts) || configuredMounts.length !== 2 || !Array.isArray(effectiveMounts) || effectiveMounts.length !== 2) {
|
|
287
|
+
throw new ClientError("Threadwire binding is unavailable")
|
|
288
|
+
}
|
|
289
|
+
const expectedMounts = [binding.source, binding.context]
|
|
290
|
+
for (const mount of expectedMounts) {
|
|
291
|
+
const configured = configuredMounts.find((item) => item?.Type === "volume" && item.Source === mount.volume && item.Target === mount.target)
|
|
292
|
+
const effective = effectiveMounts.find((item) => item?.Type === "volume" && item.Name === mount.volume && item.Destination === mount.target)
|
|
293
|
+
if (Boolean(configured?.ReadOnly) !== mount.readOnly || effective?.RW !== !mount.readOnly) throw new ClientError("Threadwire binding is unavailable")
|
|
294
|
+
}
|
|
295
|
+
for (const volume of [binding.source.volume, binding.context.volume]) {
|
|
296
|
+
const references = await docker.listVolumeReferences(volume, requestOptions)
|
|
297
|
+
if (!Array.isArray(references) || references.length !== 1 || references[0]?.Id !== binding.leaseContainerId) {
|
|
298
|
+
throw new ClientError("Threadwire binding is unavailable")
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return {fingerprint: [], root: binding.source.target}
|
|
302
|
+
}
|
|
303
|
+
|
|
235
304
|
export async function validateWorktree(repositoryRoot, cwd, allowedRoots) {
|
|
236
305
|
return (await validateWorktreeSelection(repositoryRoot, cwd, allowedRoots)).fingerprint
|
|
237
306
|
}
|
|
@@ -281,6 +350,7 @@ export async function revalidateFingerprint(fingerprint) {
|
|
|
281
350
|
export async function runIsolatedWorker(options) {
|
|
282
351
|
const runId = `tw-${randomBytes(12).toString("hex")}`
|
|
283
352
|
const networkName = `${runId}-net`
|
|
353
|
+
const egressNetworkName = `${runId}-egress`
|
|
284
354
|
const state = options.preflight.state
|
|
285
355
|
const provider = options.preflight.provider ?? "codex"
|
|
286
356
|
const providerSelection = options.preflight.providerSelection ?? options.preflight.codexArguments
|
|
@@ -288,22 +358,45 @@ export async function runIsolatedWorker(options) {
|
|
|
288
358
|
const freshState = options.resumeSession === undefined
|
|
289
359
|
let keepFreshState = false
|
|
290
360
|
let networkId
|
|
361
|
+
let egressNetworkId
|
|
362
|
+
let validatorId
|
|
363
|
+
let relayId
|
|
364
|
+
let relayContainerName
|
|
291
365
|
let containerId
|
|
292
366
|
let grantToken
|
|
367
|
+
let grantRenewalTimer
|
|
293
368
|
let runTracked = false
|
|
369
|
+
let runStage = "state"
|
|
294
370
|
const operation = options.operation
|
|
295
371
|
const worktreeSubpath = options.worktreeVolume === undefined
|
|
296
372
|
? ""
|
|
297
373
|
: deriveVolumeWorktreeSubpath(options.preflight.worktreeRoot, options.preflight.repositoryRoot)
|
|
298
374
|
try {
|
|
299
375
|
operation.throwIfAborted()
|
|
376
|
+
runStage = "state-volume"
|
|
300
377
|
await prepareStateVolume(options.docker, state, freshState, operation.options())
|
|
378
|
+
runStage = "lineage-track"
|
|
301
379
|
if (freshState) await options.stateRegistry.track(state, operation.signal)
|
|
302
|
-
|
|
380
|
+
runStage = "run-identity"
|
|
381
|
+
// Docker normalizes a created container's Config.Image to the image
|
|
382
|
+
// content Id, so the sealed Kimi identity pins the resolved relay image Id
|
|
383
|
+
// (not the configured reference) for restart reconciliation to recognize
|
|
384
|
+
// the genuine relay. The worker/validator image Id is the binding-sealed
|
|
385
|
+
// contextImageId.
|
|
386
|
+
let relayImageId
|
|
387
|
+
if (provider === "kimi") {
|
|
388
|
+
if (!isDigestImage(options.relayImage)) throw new Error("Kimi relay configuration unavailable")
|
|
389
|
+
const relayImageInspection = /** @type {{Id?: unknown}} */ (await options.docker.inspectImage(options.relayImage, operation.options()))
|
|
390
|
+
if (typeof relayImageInspection.Id !== "string" || !isDigestImage(relayImageInspection.Id)) throw new Error("Kimi relay image unavailable")
|
|
391
|
+
relayImageId = relayImageInspection.Id
|
|
392
|
+
}
|
|
393
|
+
const runIdentity = expectedRunIdentity(options, runId, networkName, state, worktreeSubpath, relayImageId)
|
|
303
394
|
const runSeal = options.stateRegistry.sealRun(runIdentity)
|
|
395
|
+
runStage = "run-track"
|
|
304
396
|
await options.stateRegistry.trackRun(runIdentity, operation.signal)
|
|
305
397
|
runTracked = true
|
|
306
398
|
const resourceLabels = {
|
|
399
|
+
"org.threadwire.owner": "isolated-runtime",
|
|
307
400
|
"org.threadwire.namespace": options.stateRegistry.namespace,
|
|
308
401
|
"org.threadwire.provider": provider,
|
|
309
402
|
"org.threadwire.run": runId,
|
|
@@ -312,12 +405,65 @@ export async function runIsolatedWorker(options) {
|
|
|
312
405
|
"org.threadwire.state-volume": state.volume,
|
|
313
406
|
"org.threadwire.run-seal": runSeal
|
|
314
407
|
}
|
|
408
|
+
if (provider === "kimi") {
|
|
409
|
+
runStage = "validator"
|
|
410
|
+
const validatorSpec = buildKimiBindingValidatorSpec({image: options.workerImage, binding: options.preflight.binding, labels: {...resourceLabels, "org.threadwire.role": "validator"}})
|
|
411
|
+
const validator = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(`${runId}-validator`, validatorSpec, operation.options()))
|
|
412
|
+
if (typeof validator.Id !== "string") throw new Error("Kimi binding validator creation failed")
|
|
413
|
+
validatorId = validator.Id
|
|
414
|
+
await validateWorkerMountInventory(await options.docker.inspectContainer(validatorId, operation.options()), validatorSpec.HostConfig.Mounts)
|
|
415
|
+
await options.docker.startContainer(validatorId, operation.options())
|
|
416
|
+
const validatorWait = await options.docker.waitContainer(validatorId, operation.options())
|
|
417
|
+
const validatorLogs = await options.docker.logs(validatorId, operation.options())
|
|
418
|
+
if (validatorWait.StatusCode !== 0) throw new Error("Kimi binding validation failed")
|
|
419
|
+
const validatorOutput = demultiplexDockerLogChunks(validatorLogs)
|
|
420
|
+
let attestation
|
|
421
|
+
try {
|
|
422
|
+
attestation = JSON.parse(Buffer.concat(validatorOutput.filter((chunk) => chunk.channel === "stdout").map((chunk) => chunk.data)).toString("utf8").trim())
|
|
423
|
+
} catch {
|
|
424
|
+
throw new Error("Kimi binding validation failed")
|
|
425
|
+
}
|
|
426
|
+
const binding = options.preflight.binding
|
|
427
|
+
if (!isRecord(attestation) || !exactKeys(attestation, ["content", "manifest", "ok", "revision"]) || attestation.ok !== true
|
|
428
|
+
|| attestation.revision !== binding.source.revision || attestation.manifest !== binding.context.digests.manifest
|
|
429
|
+
|| attestation.content !== binding.context.digests.content) throw new Error("Kimi binding validation failed")
|
|
430
|
+
await cleanupDocker(() => options.docker.removeContainer(validatorId, operation.options()))
|
|
431
|
+
validatorId = undefined
|
|
432
|
+
}
|
|
433
|
+
runStage = "network"
|
|
315
434
|
const network = /** @type {{Id?: unknown}} */ (await options.docker.createNetwork(networkName, resourceLabels, operation.options()))
|
|
316
435
|
if (typeof network.Id !== "string") throw new Error("Docker network creation failed")
|
|
317
436
|
networkId = network.Id
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
437
|
+
let workerBrokerUrl
|
|
438
|
+
let brokerAddress
|
|
439
|
+
if (provider === "kimi") {
|
|
440
|
+
if (!isDigestImage(options.relayImage) || !(options.relayWorkerUrl instanceof URL)) throw new Error("Kimi relay configuration unavailable")
|
|
441
|
+
runStage = "relay"
|
|
442
|
+
const egressNetwork = /** @type {{Id?: unknown}} */ (await options.docker.createEgressNetwork(egressNetworkName, resourceLabels, operation.options()))
|
|
443
|
+
if (typeof egressNetwork.Id !== "string") throw new Error("Docker egress network creation failed")
|
|
444
|
+
egressNetworkId = egressNetwork.Id
|
|
445
|
+
const relaySpec = buildKimiRelayContainerSpec({
|
|
446
|
+
image: options.relayImage,
|
|
447
|
+
upstreamUrl: options.relayWorkerUrl.toString(),
|
|
448
|
+
listenPort: 8788,
|
|
449
|
+
networkName: egressNetworkName,
|
|
450
|
+
labels: {...resourceLabels, "org.threadwire.role": "relay"}
|
|
451
|
+
})
|
|
452
|
+
relayContainerName = `${runId}-relay`
|
|
453
|
+
const relay = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(relayContainerName, relaySpec, operation.options()))
|
|
454
|
+
if (typeof relay.Id !== "string") throw new Error("Kimi relay creation failed")
|
|
455
|
+
relayId = relay.Id
|
|
456
|
+
await validateWorkerMountInventory(await options.docker.inspectContainer(relayId, operation.options()), relaySpec.HostConfig.Mounts)
|
|
457
|
+
await options.docker.connectNetwork(networkId, relayId, [`${runId}-relay`], operation.options())
|
|
458
|
+
await options.docker.startContainer(relayId, operation.options())
|
|
459
|
+
const networkInspection = await options.docker.inspectNetwork(networkId, operation.options())
|
|
460
|
+
workerBrokerUrl = `http://${brokerNetworkAddress(networkInspection, relayContainerName)}:8788/v1`
|
|
461
|
+
} else {
|
|
462
|
+
await options.docker.connectNetwork(networkId, options.brokerContainer, [], operation.options())
|
|
463
|
+
const networkInspection = await options.docker.inspectNetwork(networkId, operation.options())
|
|
464
|
+
brokerAddress = brokerNetworkAddress(networkInspection, options.brokerContainer)
|
|
465
|
+
}
|
|
466
|
+
runStage = "grant"
|
|
321
467
|
const grantResponse = await abortable(options.fetchImplementation(new URL("/admin/grants", options.brokerUrl), {
|
|
322
468
|
method: "POST",
|
|
323
469
|
redirect: "error",
|
|
@@ -325,20 +471,21 @@ export async function runIsolatedWorker(options) {
|
|
|
325
471
|
headers: {authorization: `Bearer ${options.brokerAdminToken}`, "content-type": "application/json"},
|
|
326
472
|
body: JSON.stringify(provider === "kimi"
|
|
327
473
|
? {
|
|
328
|
-
runId, provider, networkId, modelAlias: providerSelection.modelAlias,
|
|
474
|
+
runId, provider, networkId, modelAlias: providerSelection.modelAlias,
|
|
329
475
|
taskId: state.labels["org.threadwire.task"],
|
|
330
476
|
sessionId: options.resumeSession ?? `pending:${state.lineage}`,
|
|
331
|
-
ttlMs:
|
|
477
|
+
ttlMs: grantLease(options.grantLeaseMs)
|
|
332
478
|
}
|
|
333
|
-
: {runId, provider, networkId, model: providerSelection.model, brokerAddress, ttlMs:
|
|
479
|
+
: {runId, provider, networkId, model: providerSelection.model, brokerAddress, ttlMs: grantLease(options.grantLeaseMs)})
|
|
334
480
|
}), operation.signal)
|
|
481
|
+
process.stderr.write(`threadwire-runtime: grant response status=${grantResponse.status}\n`)
|
|
335
482
|
if (!grantResponse.ok) throw new Error("Model broker grant failed")
|
|
336
483
|
const grant = await grantResponse.json()
|
|
337
|
-
if (!isRecord(grant) || typeof grant.token !== "string" || !Number.isSafeInteger(grant.port)) throw new Error("Model broker grant failed")
|
|
484
|
+
if (!isRecord(grant) || typeof grant.token !== "string" || (provider === "codex" && !Number.isSafeInteger(grant.port))) throw new Error("Model broker grant failed")
|
|
338
485
|
grantToken = grant.token
|
|
339
486
|
const repository = /** @type {{path: string, ino: string}[]} */ (options.preflight.fingerprint)
|
|
340
|
-
const worktreeMetadata = repository.find((entry) => entry.path === options.preflight.repositoryRoot)
|
|
341
|
-
const cwdMetadata = repository.find((entry) => entry.path === options.preflight.cwd)
|
|
487
|
+
const worktreeMetadata = provider === "kimi" ? {ino: "0"} : repository.find((entry) => entry.path === options.preflight.repositoryRoot)
|
|
488
|
+
const cwdMetadata = provider === "kimi" ? {ino: "0"} : repository.find((entry) => entry.path === options.preflight.cwd)
|
|
342
489
|
if (worktreeMetadata === undefined || cwdMetadata === undefined) throw new Error("Worktree fingerprint unavailable")
|
|
343
490
|
const spec = buildWorkerContainerSpec({
|
|
344
491
|
provider,
|
|
@@ -346,10 +493,10 @@ export async function runIsolatedWorker(options) {
|
|
|
346
493
|
worktree: options.preflight.repositoryRoot,
|
|
347
494
|
networkName,
|
|
348
495
|
brokerToken: grantToken,
|
|
349
|
-
brokerUrl: `http://${brokerAddress}:${grant.port}/v1`,
|
|
496
|
+
brokerUrl: provider === "kimi" ? workerBrokerUrl : `http://${brokerAddress}:${grant.port}/v1`,
|
|
350
497
|
prompt: options.prompt,
|
|
351
498
|
providerArguments: providerSelection.arguments,
|
|
352
|
-
...(provider === "kimi" ? {model: providerSelection.model} : {}),
|
|
499
|
+
...(provider === "kimi" ? {model: providerSelection.model, binding: options.preflight.binding} : {}),
|
|
353
500
|
stateVolume: state.volume,
|
|
354
501
|
workingDirectory: `/worktree${relative(options.preflight.repositoryRoot, options.preflight.cwd) === "" ? "" : `/${relative(options.preflight.repositoryRoot, options.preflight.cwd)}`}`,
|
|
355
502
|
worktreeInode: worktreeMetadata.ino,
|
|
@@ -369,11 +516,13 @@ export async function runIsolatedWorker(options) {
|
|
|
369
516
|
}),
|
|
370
517
|
...(options.resumeSession === undefined ? {} : {resumeSession: options.resumeSession})
|
|
371
518
|
})
|
|
519
|
+
runStage = "worker-create"
|
|
372
520
|
const container = /** @type {{Id?: unknown}} */ (await options.docker.createContainer(`${runId}-worker`, spec, operation.options()))
|
|
373
521
|
if (typeof container.Id !== "string") throw new Error("Docker worker creation failed")
|
|
374
522
|
containerId = container.Id
|
|
375
523
|
await revalidateFingerprint(options.preflight.fingerprint)
|
|
376
524
|
await validateWorkerMountInventory(await options.docker.inspectContainer(containerId, operation.options()), spec.HostConfig.Mounts)
|
|
525
|
+
runStage = "grant-activation"
|
|
377
526
|
const activationResponse = await abortable(options.fetchImplementation(
|
|
378
527
|
new URL(`/admin/grants/${encodeURIComponent(grantToken)}/activate`, options.brokerUrl), {
|
|
379
528
|
method: "POST",
|
|
@@ -382,24 +531,74 @@ export async function runIsolatedWorker(options) {
|
|
|
382
531
|
headers: {authorization: `Bearer ${options.brokerAdminToken}`}
|
|
383
532
|
}), operation.signal)
|
|
384
533
|
if (!activationResponse.ok) throw new Error("Model broker grant activation failed")
|
|
534
|
+
const leaseMs = grantLease(options.grantLeaseMs)
|
|
535
|
+
let renewing = false
|
|
536
|
+
let renewalConfirmed = false
|
|
537
|
+
grantRenewalTimer = setInterval(() => {
|
|
538
|
+
if (renewing || operation.signal.aborted || grantToken === undefined) return
|
|
539
|
+
renewing = true
|
|
540
|
+
abortable(options.fetchImplementation(new URL(`/admin/grants/${encodeURIComponent(grantToken)}/renew`, options.brokerUrl), {
|
|
541
|
+
method: "POST", redirect: "error", signal: operation.signal,
|
|
542
|
+
headers: {authorization: `Bearer ${options.brokerAdminToken}`, "content-type": "application/json"},
|
|
543
|
+
body: JSON.stringify({ttlMs: leaseMs})
|
|
544
|
+
}), operation.signal).then((response) => {
|
|
545
|
+
if (!response.ok) throw new Error("Model broker grant renewal failed")
|
|
546
|
+
if (!renewalConfirmed) { renewalConfirmed = true; process.stderr.write("threadwire-runtime: grant renewal confirmed\n") }
|
|
547
|
+
}).catch((error) => {
|
|
548
|
+
process.stderr.write("threadwire-runtime: grant renewal failed\n")
|
|
549
|
+
operation.controller.abort(error instanceof Error ? error : new Error("Model broker grant renewal failed"))
|
|
550
|
+
}).finally(() => { renewing = false })
|
|
551
|
+
}, Math.max(1, Math.floor(leaseMs / 3)))
|
|
552
|
+
grantRenewalTimer.unref()
|
|
553
|
+
runStage = "worker-start"
|
|
385
554
|
await options.docker.startContainer(containerId, operation.options())
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
555
|
+
let wait
|
|
556
|
+
let records
|
|
557
|
+
let rawChunks
|
|
558
|
+
runStage = "stream"
|
|
559
|
+
// Session envelopes are buffered and only published once the run's success
|
|
560
|
+
// is known: a resume hint followed by a nonzero exit must never expose an
|
|
561
|
+
// unusable resume session. Non-session records stream immediately.
|
|
562
|
+
/** @type {unknown[]} */
|
|
563
|
+
const deferredSessions = []
|
|
564
|
+
const publishRecord = options.onRecord === undefined
|
|
565
|
+
? undefined
|
|
566
|
+
: (record) => {
|
|
567
|
+
if (provider === "kimi" && isRecord(record) && record.type === "session") {
|
|
568
|
+
deferredSessions.push(record)
|
|
569
|
+
return undefined
|
|
570
|
+
}
|
|
571
|
+
return options.onRecord(record)
|
|
572
|
+
}
|
|
573
|
+
if (provider === "kimi" && typeof options.docker.streamLogs === "function") {
|
|
574
|
+
const [streamedRecords, waitResult] = await Promise.all([
|
|
575
|
+
streamKimiWorkerRecords(options.docker, containerId, operation, publishRecord),
|
|
576
|
+
waitForKimiWorker(options.docker, containerId, operation)
|
|
577
|
+
])
|
|
578
|
+
records = streamedRecords
|
|
579
|
+
wait = /** @type {{StatusCode?: unknown}} */ (waitResult)
|
|
580
|
+
rawChunks = []
|
|
581
|
+
} else {
|
|
582
|
+
wait = /** @type {{StatusCode?: unknown}} */ (await options.docker.waitContainer(containerId, operation.options()))
|
|
583
|
+
const logs = /** @type {Buffer} */ (await options.docker.logs(containerId, operation.options()))
|
|
584
|
+
const chunks = demultiplexDockerLogChunks(logs)
|
|
585
|
+
records = parseWorkerRecords(
|
|
586
|
+
Buffer.concat(chunks.filter((chunk) => chunk.channel === "stdout").map((chunk) => chunk.data)).toString("utf8"),
|
|
587
|
+
provider
|
|
588
|
+
)
|
|
589
|
+
rawChunks = provider === "kimi" ? [] : chunks.map((chunk) => ({channel: chunk.channel, data: chunk.data.toString("base64")}))
|
|
590
|
+
if (provider === "kimi") for (const record of records) await publishRecord?.(record)
|
|
591
|
+
}
|
|
389
592
|
const result = {
|
|
390
593
|
exitCode: typeof wait.StatusCode === "number" ? wait.StatusCode : 1,
|
|
391
|
-
records
|
|
392
|
-
|
|
393
|
-
provider
|
|
394
|
-
),
|
|
395
|
-
rawChunks: provider === "kimi"
|
|
396
|
-
? []
|
|
397
|
-
: rawChunks.map((chunk) => ({channel: chunk.channel, data: chunk.data.toString("base64")}))
|
|
594
|
+
records,
|
|
595
|
+
rawChunks
|
|
398
596
|
}
|
|
399
597
|
let sessionIds = [...new Set(result.records.filter((record) => record.type === "session").map((record) => record.sessionId))]
|
|
400
598
|
if (provider === "kimi" && result.exitCode !== 0) {
|
|
401
599
|
result.records = result.records.filter((record) => record.type !== "session")
|
|
402
600
|
sessionIds = []
|
|
601
|
+
deferredSessions.length = 0
|
|
403
602
|
}
|
|
404
603
|
if (provider === "kimi" && result.exitCode === 0) {
|
|
405
604
|
if (sessionIds.length !== 1) throw new Error("Kimi worker did not establish exactly one native session")
|
|
@@ -412,9 +611,34 @@ export async function runIsolatedWorker(options) {
|
|
|
412
611
|
await options.stateRegistry.register(sessionIds, state, options.sessionTtlMs, operation.signal)
|
|
413
612
|
keepFreshState = true
|
|
414
613
|
}
|
|
614
|
+
// The run succeeded: publish any session envelopes deferred during streaming.
|
|
615
|
+
// Registration precedes delivery so a delivered identifier is always
|
|
616
|
+
// resumable; if delivery fails the client never received the identifier,
|
|
617
|
+
// so exactly the just-registered aliases are rolled back and the fresh
|
|
618
|
+
// state stays collectable by the finally-block volume cleanup instead of
|
|
619
|
+
// being retained until TTL.
|
|
620
|
+
if (result.exitCode === 0 && deferredSessions.length > 0) {
|
|
621
|
+
try {
|
|
622
|
+
for (const record of deferredSessions) await options.onRecord?.(record)
|
|
623
|
+
} catch (error) {
|
|
624
|
+
if (freshState && sessionIds.length > 0) {
|
|
625
|
+
await options.stateRegistry.unregister(sessionIds, state, operation.signal).catch((rollbackError) => {
|
|
626
|
+
process.stderr.write(`threadwire-runtime: session alias rollback failed: ${rollbackError instanceof Error ? rollbackError.message : "unknown"}\n`)
|
|
627
|
+
})
|
|
628
|
+
// The client never received a resumable identifier, so the fresh
|
|
629
|
+
// state must not be retained: let the finally-block remove it.
|
|
630
|
+
keepFreshState = false
|
|
631
|
+
}
|
|
632
|
+
throw error
|
|
633
|
+
}
|
|
634
|
+
}
|
|
415
635
|
return result
|
|
636
|
+
} catch (error) {
|
|
637
|
+
process.stderr.write(`threadwire-runtime: isolated run failed at ${runStage}\n`)
|
|
638
|
+
throw error
|
|
416
639
|
} finally {
|
|
417
640
|
let resourcesRemoved = true
|
|
641
|
+
if (grantRenewalTimer !== undefined) clearInterval(grantRenewalTimer)
|
|
418
642
|
// A caller/deadline abort cannot authorize more work, but exact force
|
|
419
643
|
// teardown must remain possible. This supervisor-only emergency budget is
|
|
420
644
|
// bounded and is used solely for already-recorded resource identifiers.
|
|
@@ -430,17 +654,29 @@ export async function runIsolatedWorker(options) {
|
|
|
430
654
|
headers: {authorization: `Bearer ${options.brokerAdminToken}`}
|
|
431
655
|
}), cleanupOperation.signal).catch(() => process.stderr.write("threadwire-isolated-runtime: grant cleanup failed\n"))
|
|
432
656
|
}
|
|
657
|
+
if (validatorId !== undefined) await cleanupDocker(() => options.docker.removeContainer(validatorId, cleanupOptions())).catch(() => {
|
|
658
|
+
resourcesRemoved = false
|
|
659
|
+
process.stderr.write("threadwire-isolated-runtime: validator cleanup failed\n")
|
|
660
|
+
})
|
|
433
661
|
if (containerId !== undefined) await cleanupDocker(() => options.docker.removeContainer(containerId, cleanupOptions())).catch(() => {
|
|
434
662
|
resourcesRemoved = false
|
|
435
663
|
process.stderr.write("threadwire-isolated-runtime: worker cleanup failed\n")
|
|
436
664
|
})
|
|
665
|
+
if (relayId !== undefined) await cleanupDocker(() => options.docker.removeContainer(relayId, cleanupOptions())).catch(() => {
|
|
666
|
+
resourcesRemoved = false
|
|
667
|
+
process.stderr.write("threadwire-isolated-runtime: relay cleanup failed\n")
|
|
668
|
+
})
|
|
437
669
|
if (networkId !== undefined) {
|
|
438
|
-
await cleanupDocker(() => options.docker.disconnectNetwork(networkId, options.brokerContainer, cleanupOptions())).catch(() => {})
|
|
670
|
+
if (provider !== "kimi") await cleanupDocker(() => options.docker.disconnectNetwork(networkId, options.brokerContainer, cleanupOptions())).catch(() => {})
|
|
439
671
|
await cleanupDocker(() => options.docker.removeNetwork(networkId, cleanupOptions())).catch(() => {
|
|
440
672
|
resourcesRemoved = false
|
|
441
673
|
process.stderr.write("threadwire-isolated-runtime: network cleanup failed\n")
|
|
442
674
|
})
|
|
443
675
|
}
|
|
676
|
+
if (egressNetworkId !== undefined) await cleanupDocker(() => options.docker.removeNetwork(egressNetworkId, cleanupOptions())).catch(() => {
|
|
677
|
+
resourcesRemoved = false
|
|
678
|
+
process.stderr.write("threadwire-isolated-runtime: egress network cleanup failed\n")
|
|
679
|
+
})
|
|
444
680
|
if (freshState && !keepFreshState) await cleanupDocker(() => options.docker.removeVolume(state.volume, cleanupOptions())).catch(() => { resourcesRemoved = false })
|
|
445
681
|
if (runTracked && resourcesRemoved) await options.stateRegistry.completeRun(runId, cleanupOperation.signal).catch(() => {})
|
|
446
682
|
emergencyCleanup?.close()
|
|
@@ -475,6 +711,95 @@ export function validateWorkerMountInventory(container, expected) {
|
|
|
475
711
|
}
|
|
476
712
|
}
|
|
477
713
|
|
|
714
|
+
async function waitForKimiWorker(docker, containerId, operation) {
|
|
715
|
+
try {
|
|
716
|
+
return await docker.waitContainer(containerId, operation.options())
|
|
717
|
+
} catch (error) {
|
|
718
|
+
const name = typeof error?.name === "string" && /^[A-Za-z][A-Za-z0-9]*$/u.test(error.name) ? error.name : "unknown"
|
|
719
|
+
const code = typeof error?.code === "string" && /^[A-Z0-9_]+$/u.test(error.code) ? error.code : "none"
|
|
720
|
+
const status = Number.isSafeInteger(error?.statusCode) ? error.statusCode : "none"
|
|
721
|
+
process.stderr.write(`threadwire-runtime: Kimi wait failure name=${name} code=${code} status=${status}\n`)
|
|
722
|
+
throw error
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
async function streamKimiWorkerRecords(docker, containerId, operation, onRecord) {
|
|
727
|
+
let dockerBuffer = Buffer.alloc(0)
|
|
728
|
+
let stdoutBuffer = Buffer.alloc(0)
|
|
729
|
+
let bytes = 0
|
|
730
|
+
let failure
|
|
731
|
+
/** @type {unknown[]} */
|
|
732
|
+
const records = []
|
|
733
|
+
let callbacks = Promise.resolve()
|
|
734
|
+
const fail = (error) => {
|
|
735
|
+
if (failure !== undefined) return
|
|
736
|
+
failure = error instanceof Error ? error : new Error("Kimi worker stream failed")
|
|
737
|
+
operation.controller?.abort(failure)
|
|
738
|
+
}
|
|
739
|
+
const consumeLine = (line) => {
|
|
740
|
+
let parsed
|
|
741
|
+
try { parsed = JSON.parse(line.toString("utf8")) } catch { return }
|
|
742
|
+
if (!isRecord(parsed)) return
|
|
743
|
+
const record = validatedKimiWorkerEnvelope(parsed)
|
|
744
|
+
if (record === undefined) return
|
|
745
|
+
records.push(record)
|
|
746
|
+
callbacks = callbacks.then(() => onRecord?.(record)).catch(fail)
|
|
747
|
+
}
|
|
748
|
+
const consumeDockerChunk = (chunk) => {
|
|
749
|
+
if (failure !== undefined) return
|
|
750
|
+
try {
|
|
751
|
+
dockerBuffer = Buffer.concat([dockerBuffer, Buffer.from(chunk)])
|
|
752
|
+
while (dockerBuffer.length >= 8) {
|
|
753
|
+
const stream = dockerBuffer[0]
|
|
754
|
+
const length = dockerBuffer.readUInt32BE(4)
|
|
755
|
+
if (stream !== 1 && stream !== 2) throw new Error("Worker emitted invalid Docker logs")
|
|
756
|
+
if (length > MAX_RAW_OUTPUT_BYTES) throw new Error("Worker output exceeded capacity")
|
|
757
|
+
if (dockerBuffer.length < 8 + length) break
|
|
758
|
+
const data = dockerBuffer.subarray(8, 8 + length)
|
|
759
|
+
dockerBuffer = dockerBuffer.subarray(8 + length)
|
|
760
|
+
bytes += data.length
|
|
761
|
+
if (bytes > MAX_RAW_OUTPUT_BYTES) throw new Error("Worker output exceeded capacity")
|
|
762
|
+
if (stream === 2) {
|
|
763
|
+
const text = data.toString("utf8")
|
|
764
|
+
if (/^threadwire-kimi-worker: provider (?:record )?failure (?:authentication|network|model|configuration|context|rate-limit|tool|unknown)\n$/u.test(text)
|
|
765
|
+
|| /^threadwire-kimi-worker: ignored record (?:non-object|role=[a-z0-9._-]+ type=[a-z0-9._-]+ keys=(?:unsafe|[a-z0-9_,]+))\n$/u.test(text)
|
|
766
|
+
|| /^threadwire-kimi-worker: internal failure stream\n$/u.test(text)
|
|
767
|
+
|| /^threadwire-kimi-worker: provider signal (?:SIGKILL|SIGTERM|SIGABRT)\n$/u.test(text)) process.stderr.write(data)
|
|
768
|
+
continue
|
|
769
|
+
}
|
|
770
|
+
stdoutBuffer = Buffer.concat([stdoutBuffer, data])
|
|
771
|
+
while (true) {
|
|
772
|
+
const newline = stdoutBuffer.indexOf(0x0a)
|
|
773
|
+
if (newline < 0) break
|
|
774
|
+
consumeLine(stdoutBuffer.subarray(0, newline))
|
|
775
|
+
stdoutBuffer = stdoutBuffer.subarray(newline + 1)
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
} catch (error) { fail(error) }
|
|
779
|
+
}
|
|
780
|
+
try {
|
|
781
|
+
await docker.streamLogs(containerId, consumeDockerChunk, operation.options())
|
|
782
|
+
} catch (error) {
|
|
783
|
+
fail(error)
|
|
784
|
+
}
|
|
785
|
+
if (dockerBuffer.length !== 0) fail(new Error("Worker emitted invalid Docker logs"))
|
|
786
|
+
if (stdoutBuffer.length !== 0) consumeLine(stdoutBuffer)
|
|
787
|
+
await callbacks
|
|
788
|
+
if (failure !== undefined) {
|
|
789
|
+
process.stderr.write(`threadwire-runtime: Kimi stream failure ${classifyKimiStreamFailure(failure)}\n`)
|
|
790
|
+
throw failure
|
|
791
|
+
}
|
|
792
|
+
return records
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
function classifyKimiStreamFailure(error) {
|
|
796
|
+
if (error.message === "Worker emitted invalid Docker logs") return "framing"
|
|
797
|
+
if (error.message === "Worker output exceeded capacity") return "capacity"
|
|
798
|
+
if (error.message === "Model broker grant renewal failed") return "grant-renewal"
|
|
799
|
+
if (/socket|closed|EPIPE|aborted/u.test(error.message)) return "client-disconnect"
|
|
800
|
+
return "unknown"
|
|
801
|
+
}
|
|
802
|
+
|
|
478
803
|
function parseWorkerRecords(text, provider = "codex") {
|
|
479
804
|
const records = []
|
|
480
805
|
let kimiSessionId
|
|
@@ -519,11 +844,12 @@ function validatedKimiWorkerEnvelope(record) {
|
|
|
519
844
|
&& typeof event.text === "string" && event.text.length <= 1_048_576 && event.streamId === "kimi:assistant") {
|
|
520
845
|
return {type: "worker-event", event: {type: "text-delta", text: event.text, streamId: "kimi:assistant"}}
|
|
521
846
|
}
|
|
522
|
-
if (exactKeys(event, ["type", "phase", "name", "key"]) && event.type === "tool"
|
|
847
|
+
if ((exactKeys(event, ["type", "phase", "name", "key"]) || exactKeys(event, ["type", "phase", "name", "key", "detail"])) && event.type === "tool"
|
|
523
848
|
&& (event.phase === "started" || event.phase === "finished")
|
|
524
849
|
&& typeof event.name === "string" && /^[A-Za-z_][A-Za-z0-9_.:-]{0,127}$/u.test(event.name)
|
|
525
|
-
&& typeof event.key === "string" && /^tool:[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u.test(event.key)
|
|
526
|
-
|
|
850
|
+
&& typeof event.key === "string" && /^tool:[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$/u.test(event.key)
|
|
851
|
+
&& (event.detail === undefined || (typeof event.detail === "string" && event.detail.length <= 512))) {
|
|
852
|
+
return {type: "worker-event", event: {type: "tool", phase: event.phase, name: event.name, key: event.key, ...(event.detail === undefined ? {} : {detail: event.detail})}}
|
|
527
853
|
}
|
|
528
854
|
if (exactKeys(event, ["type", "phase", "summary"]) && event.type === "lifecycle") {
|
|
529
855
|
const summaries = {
|
|
@@ -563,21 +889,30 @@ export function demultiplexDockerLogChunks(buffer) {
|
|
|
563
889
|
|
|
564
890
|
export function parsePreflight(value, expectedProvider = "codex") {
|
|
565
891
|
if (!isRecord(value) || value.provider !== expectedProvider || (expectedProvider !== "codex" && expectedProvider !== "kimi")
|
|
566
|
-
|| typeof value.profile !== "string" ||
|
|
567
|
-
|| !
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
if (!bounded(value.profile, 128) || !bounded(value.repositoryRoot, 4096) || !bounded(value.cwd, 4096)
|
|
571
|
-
|| value.providerArguments.length > 16 || value.providerArguments.some((item) => !bounded(item, 512))
|
|
572
|
-
|| (value.resumeSession !== undefined && !bounded(value.resumeSession, 512))) {
|
|
573
|
-
throw new ClientError("Invalid isolated runtime preflight")
|
|
574
|
-
}
|
|
892
|
+
|| typeof value.profile !== "string" || !Array.isArray(value.providerArguments)
|
|
893
|
+
|| !value.providerArguments.every((item) => typeof item === "string")) throw new ClientError("Invalid isolated runtime preflight")
|
|
894
|
+
if (!bounded(value.profile, 128) || value.providerArguments.length > 16 || value.providerArguments.some((item) => !bounded(item, 512))
|
|
895
|
+
|| (value.resumeSession !== undefined && !bounded(value.resumeSession, 512))) throw new ClientError("Invalid isolated runtime preflight")
|
|
575
896
|
if (value.resumeSession !== undefined && (typeof value.resumeSession !== "string" || /[\r\n]/u.test(value.resumeSession))) throw new ClientError("Invalid isolated runtime preflight")
|
|
897
|
+
if (expectedProvider === "kimi") {
|
|
898
|
+
// The Kimi request schema is closed: repositoryRoot/cwd are derived from
|
|
899
|
+
// the binding, so any extraneous top-level field is rejected rather than
|
|
900
|
+
// silently ignored.
|
|
901
|
+
const allowedKeys = ["provider", "profile", "binding", "providerArguments", ...(value.resumeSession === undefined ? [] : ["resumeSession"])]
|
|
902
|
+
if (!exactKeys(value, allowedKeys)) throw new ClientError("Invalid isolated runtime preflight")
|
|
903
|
+
let binding
|
|
904
|
+
try { binding = parseThreadwireBinding(value.binding) } catch { throw new ClientError("Invalid isolated runtime preflight") }
|
|
905
|
+
return {
|
|
906
|
+
provider: "kimi", profile: value.profile, binding,
|
|
907
|
+
repositoryRoot: binding.source.target, cwd: binding.runtime.workdir,
|
|
908
|
+
providerArguments: value.providerArguments,
|
|
909
|
+
...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession})
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
if (typeof value.repositoryRoot !== "string" || typeof value.cwd !== "string"
|
|
913
|
+
|| !bounded(value.repositoryRoot, 4096) || !bounded(value.cwd, 4096)) throw new ClientError("Invalid isolated runtime preflight")
|
|
576
914
|
return {
|
|
577
|
-
provider:
|
|
578
|
-
profile: value.profile,
|
|
579
|
-
repositoryRoot: value.repositoryRoot,
|
|
580
|
-
cwd: value.cwd,
|
|
915
|
+
provider: "codex", profile: value.profile, repositoryRoot: value.repositoryRoot, cwd: value.cwd,
|
|
581
916
|
providerArguments: value.providerArguments,
|
|
582
917
|
...(value.resumeSession === undefined ? {} : {resumeSession: value.resumeSession})
|
|
583
918
|
}
|
|
@@ -587,7 +922,7 @@ export function parseRun(value) {
|
|
|
587
922
|
if (!isRecord(value) || typeof value.preflightId !== "string" || typeof value.prompt !== "string" || value.prompt.trim().length === 0 || !Array.isArray(value.providerArguments) || !value.providerArguments.every((item) => typeof item === "string")) {
|
|
588
923
|
throw new ClientError("Invalid isolated runtime run")
|
|
589
924
|
}
|
|
590
|
-
if (!bounded(value.preflightId, 64) || !
|
|
925
|
+
if (!bounded(value.preflightId, 64) || !boundedPrompt(value.prompt, 524_288)
|
|
591
926
|
|| value.providerArguments.length > 16 || value.providerArguments.some((item) => !bounded(item, 512))
|
|
592
927
|
|| (value.resumeSession !== undefined && !bounded(value.resumeSession, 512))) {
|
|
593
928
|
throw new ClientError("Invalid isolated runtime run")
|
|
@@ -628,9 +963,37 @@ function isolatedProvider(value) {
|
|
|
628
963
|
function providerName(provider) { return provider === "kimi" ? "Kimi" : "Codex" }
|
|
629
964
|
|
|
630
965
|
function requestedDeadline(value, maximumMs) {
|
|
631
|
-
const parsed = Number(value)
|
|
632
|
-
const
|
|
633
|
-
|
|
966
|
+
const parsed = typeof value === "string" && value.length > 0 ? Number(value) : undefined
|
|
967
|
+
const requested = Number.isSafeInteger(parsed) && parsed > Date.now() ? parsed : undefined
|
|
968
|
+
const maximum = maximumMs === undefined ? undefined : Date.now() + maximumMs
|
|
969
|
+
return minimumDeadline(requested, maximum)
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
function minimumDeadline(first, second) {
|
|
973
|
+
if (first === undefined) return second
|
|
974
|
+
if (second === undefined) return first
|
|
975
|
+
return Math.min(first, second)
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
function grantLease(value) {
|
|
979
|
+
const lease = value ?? GRANT_LEASE_MS
|
|
980
|
+
if (!Number.isSafeInteger(lease) || lease < 1 || lease > GRANT_LEASE_MAX_MS) throw new Error("Invalid broker grant lease")
|
|
981
|
+
return lease
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
/**
|
|
985
|
+
* Fail-closed startup read of the broker grant lease. The default is 60
|
|
986
|
+
* seconds and the maximum is one hour; an invalid, zero, or excessive value
|
|
987
|
+
* fails startup rather than being silently coerced.
|
|
988
|
+
* @param {string | undefined} value
|
|
989
|
+
*/
|
|
990
|
+
function grantLeaseSetting(value) {
|
|
991
|
+
if (value === undefined) return GRANT_LEASE_MS
|
|
992
|
+
const number = Number(value)
|
|
993
|
+
if (!/^\d+$/u.test(value) || !Number.isSafeInteger(number) || number < 1 || number > GRANT_LEASE_MAX_MS) {
|
|
994
|
+
throw new Error("THREADWIRE_GRANT_LEASE_MS must be a positive millisecond integer no greater than one hour")
|
|
995
|
+
}
|
|
996
|
+
return number
|
|
634
997
|
}
|
|
635
998
|
|
|
636
999
|
function parseAllowedRoots(value) {
|
|
@@ -663,7 +1026,15 @@ function ancestry(root, child) {
|
|
|
663
1026
|
return paths
|
|
664
1027
|
}
|
|
665
1028
|
|
|
666
|
-
function taskIdentity(preflight) {
|
|
1029
|
+
export function taskIdentity(preflight) {
|
|
1030
|
+
// Kimi scopes resume/task identity to the exact binding so distinct task
|
|
1031
|
+
// bindings sharing a profile root never collapse together. The binding
|
|
1032
|
+
// identity intentionally excludes the mutable source revision and the
|
|
1033
|
+
// recreatable lease container, so an intended same-task resume stays valid
|
|
1034
|
+
// across checkout and lease rotation. Codex identity is unchanged.
|
|
1035
|
+
if ((preflight.provider ?? "codex") === "kimi") {
|
|
1036
|
+
return `kimi\0${preflight.profile}\0${resumeTaskIdentity(preflight.binding)}`
|
|
1037
|
+
}
|
|
667
1038
|
return `${preflight.provider ?? "codex"}\0${preflight.profile}\0${preflight.repositoryRoot}`
|
|
668
1039
|
}
|
|
669
1040
|
|
|
@@ -726,6 +1097,11 @@ function portValue(value, fallback) {
|
|
|
726
1097
|
return number
|
|
727
1098
|
}
|
|
728
1099
|
|
|
1100
|
+
function optionalDuration(value) {
|
|
1101
|
+
if (value === undefined || (typeof value === "string" && value.trim().length === 0)) return undefined
|
|
1102
|
+
return positiveDuration(value)
|
|
1103
|
+
}
|
|
1104
|
+
|
|
729
1105
|
function positiveDuration(value, fallback) {
|
|
730
1106
|
if (value === undefined) return fallback
|
|
731
1107
|
const number = Number(value)
|
|
@@ -793,6 +1169,10 @@ function bounded(value, maximum) {
|
|
|
793
1169
|
return typeof value === "string" && value.length > 0 && value.length <= maximum && !/[\0\r\n]/u.test(value)
|
|
794
1170
|
}
|
|
795
1171
|
|
|
1172
|
+
function boundedPrompt(value, maximum) {
|
|
1173
|
+
return typeof value === "string" && value.length > 0 && value.length <= maximum && !value.includes("\0")
|
|
1174
|
+
}
|
|
1175
|
+
|
|
796
1176
|
export class PreflightStore {
|
|
797
1177
|
constructor({now = Date.now, ttlMs = PREFLIGHT_TTL_MS, capacity = PREFLIGHT_CAPACITY, taskCapacity = PREFLIGHT_TASK_CAPACITY} = {}) {
|
|
798
1178
|
this.now = now
|
|
@@ -907,7 +1287,7 @@ export async function reconcileDockerResources(docker, brokerContainer, stateReg
|
|
|
907
1287
|
const persisted = ownedIdentities.get(labels.run)
|
|
908
1288
|
if (!persisted || !stateRegistry.ownsRunSeal(persisted, labels.seal, labels.legacy)) continue
|
|
909
1289
|
const inspection = await docker.inspectContainer(container.Id, requestOptions).catch(() => undefined)
|
|
910
|
-
const identity = ownedContainerIdentity(inspection, labels)
|
|
1290
|
+
const identity = ownedContainerIdentity(inspection, labels, persisted)
|
|
911
1291
|
if (!identity || !stateRegistry.ownsRun(identity, labels.seal, labels.legacy)) {
|
|
912
1292
|
cleanupFailed.add(labels.run)
|
|
913
1293
|
continue
|
|
@@ -963,12 +1343,17 @@ function validResourceLabels(value, namespace, container, expectedProvider = "co
|
|
|
963
1343
|
if (!isRecord(value)) return undefined
|
|
964
1344
|
const provider = value["org.threadwire.provider"] ?? "codex"
|
|
965
1345
|
const legacy = value["org.threadwire.provider"] === undefined
|
|
1346
|
+
// Kimi containers carry an exact role label instead of the Codex network
|
|
1347
|
+
// label; Kimi networks carry neither. Codex topology is unchanged.
|
|
1348
|
+
const kimi = provider === "kimi"
|
|
1349
|
+
const role = value["org.threadwire.role"]
|
|
966
1350
|
const expectedKeys = [
|
|
967
1351
|
"org.threadwire.owner", "org.threadwire.namespace", "org.threadwire.run",
|
|
968
1352
|
"org.threadwire.lineage", "org.threadwire.task", "org.threadwire.state-volume",
|
|
969
1353
|
"org.threadwire.run-seal",
|
|
970
1354
|
...(legacy ? [] : ["org.threadwire.provider"]),
|
|
971
|
-
...(container ? ["org.threadwire.network"] : [])
|
|
1355
|
+
...(container && !kimi ? ["org.threadwire.network"] : []),
|
|
1356
|
+
...(container && kimi ? ["org.threadwire.role"] : [])
|
|
972
1357
|
]
|
|
973
1358
|
if (provider !== expectedProvider || (legacy && provider !== "codex")
|
|
974
1359
|
|| Object.keys(value).length !== expectedKeys.length || expectedKeys.some((key) => typeof value[key] !== "string")) return undefined
|
|
@@ -981,11 +1366,13 @@ function validResourceLabels(value, namespace, container, expectedProvider = "co
|
|
|
981
1366
|
|| !/^[0-9a-f]{64}$/u.test(value["org.threadwire.task"])
|
|
982
1367
|
|| !/^[0-9a-f]{64}$/u.test(seal)
|
|
983
1368
|
|| volume !== `threadwire-state-${namespace}-${lineage}`
|
|
984
|
-
|| (container && value["org.threadwire.network"] !== `${run}-net`)
|
|
985
|
-
|
|
1369
|
+
|| (container && !kimi && value["org.threadwire.network"] !== `${run}-net`)
|
|
1370
|
+
|| (container && kimi && !["validator", "relay", "worker"].includes(role))) return undefined
|
|
1371
|
+
return {provider, legacy, run, lineage, volume, seal, task: value["org.threadwire.task"], namespace, ...(kimi && container ? {role} : {})}
|
|
986
1372
|
}
|
|
987
1373
|
|
|
988
|
-
function ownedContainerIdentity(container, labels) {
|
|
1374
|
+
function ownedContainerIdentity(container, labels, persisted) {
|
|
1375
|
+
if (labels.provider === "kimi") return ownedKimiContainerIdentity(container, labels, persisted)
|
|
989
1376
|
if (!isRecord(container) || container.Name !== `/${labels.run}-worker`
|
|
990
1377
|
|| !isRecord(container.Config) || !isRecord(container.HostConfig)
|
|
991
1378
|
|| container.Config.Image !== container.Image || !isDigestImage(container.Config.Image)
|
|
@@ -1007,6 +1394,190 @@ function ownedContainerIdentity(container, labels) {
|
|
|
1007
1394
|
return runIdentityFrom(labels, container.Config.Image, worktree)
|
|
1008
1395
|
}
|
|
1009
1396
|
|
|
1397
|
+
/**
|
|
1398
|
+
* Exact, fail-closed Kimi container identity. The persisted sealed run
|
|
1399
|
+
* identity is the single authority for the expected validator, relay, and
|
|
1400
|
+
* worker specs; any topology, security, mount, environment, image, or role
|
|
1401
|
+
* drift is rejected by returning undefined so the resource is retained as a
|
|
1402
|
+
* cleanup failure rather than adopted.
|
|
1403
|
+
*/
|
|
1404
|
+
function ownedKimiContainerIdentity(container, labels, persisted) {
|
|
1405
|
+
if (!isRecord(container) || !isRecord(container.Config) || !isRecord(container.HostConfig) || !isRecord(persisted)) return undefined
|
|
1406
|
+
const inspectedLabels = validResourceLabels(container.Config.Labels, labels.namespace, true, labels.provider)
|
|
1407
|
+
if (!inspectedLabels || JSON.stringify(inspectedLabels) !== JSON.stringify(labels)) return undefined
|
|
1408
|
+
const role = labels.role
|
|
1409
|
+
if (role === "validator") return ownedKimiValidatorIdentity(container, persisted) ? persisted : undefined
|
|
1410
|
+
if (role === "relay") return ownedKimiRelayIdentity(container, persisted) ? persisted : undefined
|
|
1411
|
+
if (role === "worker") return ownedKimiWorkerIdentity(container, persisted) ? persisted : undefined
|
|
1412
|
+
return undefined
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
/** @param {Record<string, unknown>} mount */
|
|
1416
|
+
function exactKimiMount(mount, source, target, readOnly) {
|
|
1417
|
+
return isRecord(mount) && mount.Type === "volume" && mount.Source === source && mount.Target === target
|
|
1418
|
+
&& Boolean(mount.ReadOnly) === readOnly
|
|
1419
|
+
&& Object.keys(mount).every((key) => ["Type", "Source", "Target", "ReadOnly"].includes(key))
|
|
1420
|
+
}
|
|
1421
|
+
|
|
1422
|
+
function ownedKimiValidatorIdentity(container, persisted) {
|
|
1423
|
+
const config = container.Config
|
|
1424
|
+
const host = container.HostConfig
|
|
1425
|
+
const uid = persisted.runtimeUid
|
|
1426
|
+
const gid = persisted.runtimeGid
|
|
1427
|
+
return container.Name === `/${persisted.validator}`
|
|
1428
|
+
&& config.Image === persisted.image && isDigestImage(config.Image)
|
|
1429
|
+
&& JSON.stringify(config.Entrypoint) === JSON.stringify(["node", "/opt/threadwire/docker/kimi-binding-validator.mjs"])
|
|
1430
|
+
&& config.User === `${uid}:${gid}` && config.WorkingDir === persisted.workdir
|
|
1431
|
+
&& config.NetworkDisabled === true
|
|
1432
|
+
&& validKimiValidatorEnv(config.Env, persisted)
|
|
1433
|
+
&& host.NetworkMode === "none"
|
|
1434
|
+
&& validKimiSecurityPosture(host)
|
|
1435
|
+
&& Array.isArray(host.Mounts) && host.Mounts.length === 2
|
|
1436
|
+
&& exactKimiMount(host.Mounts[0], persisted.sourceVolume, persisted.sourceTarget, true)
|
|
1437
|
+
&& exactKimiMount(host.Mounts[1], persisted.contextVolume, persisted.contextTarget, true)
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
function ownedKimiRelayIdentity(container, persisted) {
|
|
1441
|
+
const config = container.Config
|
|
1442
|
+
const host = container.HostConfig
|
|
1443
|
+
return container.Name === `/${persisted.relay}`
|
|
1444
|
+
&& config.Image === persisted.relayImage && isDigestImage(config.Image)
|
|
1445
|
+
&& JSON.stringify(config.Entrypoint) === JSON.stringify(["node", "/opt/threadwire/docker/kimi-model-relay-entrypoint.mjs"])
|
|
1446
|
+
&& config.User === "10003:10003"
|
|
1447
|
+
&& validKimiRelayEnv(config.Env, persisted)
|
|
1448
|
+
&& host.NetworkMode === persisted.egressNetwork
|
|
1449
|
+
&& validKimiSecurityPosture(host)
|
|
1450
|
+
&& Array.isArray(host.Mounts) && host.Mounts.length === 0
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
/**
|
|
1454
|
+
* The relay environment is credential-free. As with the validator, Docker
|
|
1455
|
+
* merges the two relay-owned keys with legitimate non-sensitive keys inherited
|
|
1456
|
+
* from the image (PATH, NODE_VERSION, YARN_VERSION, and any future base-image
|
|
1457
|
+
* additions), so the inspected set is the two relay-owned keys plus those
|
|
1458
|
+
* inherited keys. Config.Env must be an array: missing is never accepted. Each
|
|
1459
|
+
* relay-owned key must be present exactly once with the exact sealed value;
|
|
1460
|
+
* any duplicate, tampered value, or unknown relay-controlled or sensitive key
|
|
1461
|
+
* (any other THREADWIRE_*, CODEX, OPENAI, TELEGRAM, or KIMI_*) is rejected.
|
|
1462
|
+
* @param {unknown} value @param {Record<string, unknown>} persisted
|
|
1463
|
+
*/
|
|
1464
|
+
function validKimiRelayEnv(value, persisted) {
|
|
1465
|
+
if (!Array.isArray(value)) return false
|
|
1466
|
+
const expected = new Map([
|
|
1467
|
+
["THREADWIRE_KIMI_RELAY_UPSTREAM_URL", persisted.endpoint],
|
|
1468
|
+
["THREADWIRE_KIMI_RELAY_LISTEN_PORT", "8788"]
|
|
1469
|
+
])
|
|
1470
|
+
const seen = new Set()
|
|
1471
|
+
for (const entry of value) {
|
|
1472
|
+
if (typeof entry !== "string") return false
|
|
1473
|
+
const separator = entry.indexOf("=")
|
|
1474
|
+
const name = separator === -1 ? entry : entry.slice(0, separator)
|
|
1475
|
+
const sealed = expected.get(name)
|
|
1476
|
+
if (sealed !== undefined) {
|
|
1477
|
+
if (seen.has(name) || entry.slice(separator + 1) !== sealed) return false
|
|
1478
|
+
seen.add(name)
|
|
1479
|
+
continue
|
|
1480
|
+
}
|
|
1481
|
+
if (name.startsWith("THREADWIRE_") || name.startsWith("CODEX") || name.startsWith("OPENAI")
|
|
1482
|
+
|| name.startsWith("TELEGRAM") || name.startsWith("KIMI_")) return false
|
|
1483
|
+
}
|
|
1484
|
+
return seen.size === expected.size
|
|
1485
|
+
}
|
|
1486
|
+
|
|
1487
|
+
function ownedKimiWorkerIdentity(container, persisted) {
|
|
1488
|
+
const config = container.Config
|
|
1489
|
+
const host = container.HostConfig
|
|
1490
|
+
const uid = persisted.runtimeUid
|
|
1491
|
+
const gid = persisted.runtimeGid
|
|
1492
|
+
return container.Name === `/${persisted.worker}`
|
|
1493
|
+
&& config.Image === persisted.image && isDigestImage(config.Image)
|
|
1494
|
+
&& JSON.stringify(config.Entrypoint) === JSON.stringify(["node", "/opt/threadwire/docker/kimi-worker-entrypoint.mjs"])
|
|
1495
|
+
&& config.User === `${uid}:${gid}` && config.WorkingDir === persisted.workdir
|
|
1496
|
+
&& config.NetworkDisabled === false
|
|
1497
|
+
&& host.NetworkMode === persisted.network
|
|
1498
|
+
&& validKimiWorkerEnv(config.Env)
|
|
1499
|
+
&& validKimiSecurityPosture(host)
|
|
1500
|
+
&& Array.isArray(host.Mounts) && host.Mounts.length === 3
|
|
1501
|
+
&& exactKimiMount(host.Mounts[0], persisted.sourceVolume, persisted.sourceTarget, persisted.sourceReadOnly === "true")
|
|
1502
|
+
&& exactKimiMount(host.Mounts[1], persisted.contextVolume, persisted.contextTarget, true)
|
|
1503
|
+
&& exactKimiMount(host.Mounts[2], persisted.volume, "/state", false)
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1506
|
+
/**
|
|
1507
|
+
* The validator environment is credential-free. Docker merges the container's
|
|
1508
|
+
* request environment with arbitrary legitimate non-sensitive keys inherited
|
|
1509
|
+
* from the image (PATH, HOME, KIMI_CODE_HOME, TMPDIR, locale, and any future
|
|
1510
|
+
* base-image additions), so the inspected set is the five validator-owned
|
|
1511
|
+
* THREADWIRE_* keys plus those inherited keys. Config.Env must be an array:
|
|
1512
|
+
* missing is never accepted. Each validator-owned key must be present exactly
|
|
1513
|
+
* once with the exact value sealed in the persisted run identity; any unknown
|
|
1514
|
+
* validator-controlled or sensitive key (any other THREADWIRE_*, CODEX,
|
|
1515
|
+
* OPENAI, TELEGRAM, or KIMI_* except the known inherited KIMI_CODE_HOME),
|
|
1516
|
+
* a duplicate, or a tampered value is rejected.
|
|
1517
|
+
* @param {unknown} value @param {Record<string, unknown>} persisted
|
|
1518
|
+
*/
|
|
1519
|
+
function validKimiValidatorEnv(value, persisted) {
|
|
1520
|
+
if (!Array.isArray(value)) return false
|
|
1521
|
+
const expected = new Map([
|
|
1522
|
+
["THREADWIRE_TASK_ID", persisted.taskId],
|
|
1523
|
+
["THREADWIRE_EXPECTED_REVISION", persisted.expectedRevision],
|
|
1524
|
+
["THREADWIRE_CONTEXT_MANIFEST_DIGEST", persisted.contextManifestDigest],
|
|
1525
|
+
["THREADWIRE_CONTEXT_CONTENT_DIGEST", persisted.contextContentDigest],
|
|
1526
|
+
["THREADWIRE_CONTEXT_IMAGE_ID", persisted.contextImageId]
|
|
1527
|
+
])
|
|
1528
|
+
const seen = new Set()
|
|
1529
|
+
for (const entry of value) {
|
|
1530
|
+
if (typeof entry !== "string") return false
|
|
1531
|
+
const separator = entry.indexOf("=")
|
|
1532
|
+
const name = separator === -1 ? entry : entry.slice(0, separator)
|
|
1533
|
+
const sealed = expected.get(name)
|
|
1534
|
+
if (sealed !== undefined) {
|
|
1535
|
+
if (seen.has(name) || typeof sealed !== "string" || entry.slice(separator + 1) !== sealed) return false
|
|
1536
|
+
seen.add(name)
|
|
1537
|
+
continue
|
|
1538
|
+
}
|
|
1539
|
+
// Unknown validator-controlled or sensitive keys are never inherited.
|
|
1540
|
+
if (name.startsWith("THREADWIRE_") || name.startsWith("CODEX") || name.startsWith("OPENAI")
|
|
1541
|
+
|| name.startsWith("TELEGRAM") || (name.startsWith("KIMI_") && name !== "KIMI_CODE_HOME")) return false
|
|
1542
|
+
}
|
|
1543
|
+
return seen.size === expected.size
|
|
1544
|
+
}
|
|
1545
|
+
|
|
1546
|
+
/** The Kimi worker environment carries a per-run grant token and prompt, so only the exact name set is sealed. */
|
|
1547
|
+
function validKimiWorkerEnv(value) {
|
|
1548
|
+
if (!Array.isArray(value)) return false
|
|
1549
|
+
const names = value.map((entry) => typeof entry === "string" && entry.split("=", 1)[0])
|
|
1550
|
+
if (names.some((name) => name === false)) return false
|
|
1551
|
+
const resumeCount = names.filter((name) => name === "THREADWIRE_RESUME_SESSION").length
|
|
1552
|
+
if (resumeCount > 1) return false
|
|
1553
|
+
const expected = [
|
|
1554
|
+
"PATH", "NODE_VERSION", "YARN_VERSION", "HOME", "KIMI_CODE_HOME", "TMPDIR",
|
|
1555
|
+
"THREADWIRE_PROMPT", "THREADWIRE_KIMI_MODEL", "THREADWIRE_KIMI_BROKER_URL", "THREADWIRE_KIMI_BROKER_TOKEN",
|
|
1556
|
+
"THREADWIRE_SOURCE_TARGET", "THREADWIRE_CONTEXT_TARGET", "THREADWIRE_TASK_ID", "THREADWIRE_SOURCE_READ_ONLY",
|
|
1557
|
+
"THREADWIRE_EXPECTED_REVISION", "THREADWIRE_CONTEXT_MANIFEST_DIGEST", "THREADWIRE_CONTEXT_CONTENT_DIGEST", "THREADWIRE_CONTEXT_IMAGE_ID"
|
|
1558
|
+
]
|
|
1559
|
+
return names.length === expected.length + resumeCount
|
|
1560
|
+
&& expected.every((name) => names.filter((entry) => entry === name).length === 1)
|
|
1561
|
+
&& !names.some((name) => /^(?:CODEX|OPENAI|TELEGRAM|THREADWIRE_KIMI_(?:OAUTH|CONTROL))/u.test(name))
|
|
1562
|
+
}
|
|
1563
|
+
|
|
1564
|
+
/**
|
|
1565
|
+
* Shared fail-closed Kimi security posture. The read-only root, dropped
|
|
1566
|
+
* capabilities, and no-new-privileges posture must be exact, and no privilege
|
|
1567
|
+
* escalation, published ports, or host-namespace sharing may be introduced.
|
|
1568
|
+
*/
|
|
1569
|
+
function validKimiSecurityPosture(host) {
|
|
1570
|
+
return host.ReadonlyRootfs === true && host.Privileged !== true
|
|
1571
|
+
&& JSON.stringify(host.CapDrop) === JSON.stringify(["ALL"])
|
|
1572
|
+
&& JSON.stringify(host.CapAdd ?? null) === "null"
|
|
1573
|
+
&& JSON.stringify(host.SecurityOpt) === JSON.stringify(["no-new-privileges"])
|
|
1574
|
+
&& host.PublishAllPorts !== true
|
|
1575
|
+
&& [undefined, "", "private"].includes(host.PidMode) && [undefined, "", "private"].includes(host.IpcMode)
|
|
1576
|
+
&& [undefined, ""].includes(host.UTSMode) && [undefined, ""].includes(host.UsernsMode)
|
|
1577
|
+
&& [undefined, "", "private"].includes(host.CgroupnsMode)
|
|
1578
|
+
&& (host.Binds === undefined || host.Binds === null || (Array.isArray(host.Binds) && host.Binds.length === 0))
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1010
1581
|
function validWorkerSecurity(config, host, provider = "codex") {
|
|
1011
1582
|
if (host.AutoRemove !== false || host.ReadonlyRootfs !== true
|
|
1012
1583
|
|| host.Privileged === true || JSON.stringify(host.CapDrop) !== JSON.stringify(["ALL"])
|
|
@@ -1056,6 +1627,7 @@ function validWorkerSecurity(config, host, provider = "codex") {
|
|
|
1056
1627
|
}
|
|
1057
1628
|
|
|
1058
1629
|
function ownedNetworkIdentity(network, labels, identity, brokerContainer) {
|
|
1630
|
+
if (labels.provider === "kimi") return ownedKimiNetworkIdentity(network, labels, identity)
|
|
1059
1631
|
if (!isRecord(network) || network.Name !== `${labels.run}-net` || network.Internal !== true
|
|
1060
1632
|
|| network.Driver !== "bridge" || network.Scope !== "local"
|
|
1061
1633
|
|| network.EnableIPv6 !== false || network.Attachable !== false || network.Ingress !== false
|
|
@@ -1069,6 +1641,28 @@ function ownedNetworkIdentity(network, labels, identity, brokerContainer) {
|
|
|
1069
1641
|
return true
|
|
1070
1642
|
}
|
|
1071
1643
|
|
|
1644
|
+
/**
|
|
1645
|
+
* Exact Kimi task and egress networks. The internal task network and the
|
|
1646
|
+
* non-internal egress network are both sealed by the persisted run identity;
|
|
1647
|
+
* only the per-run relay may be attached, so any foreign endpoint, name, or
|
|
1648
|
+
* topology drift fails closed.
|
|
1649
|
+
*/
|
|
1650
|
+
function ownedKimiNetworkIdentity(network, labels, identity) {
|
|
1651
|
+
if (!isRecord(network) || !isRecord(network.Containers)) return undefined
|
|
1652
|
+
const expected = network.Name === identity.network ? {internal: true}
|
|
1653
|
+
: network.Name === identity.egressNetwork ? {internal: false}
|
|
1654
|
+
: undefined
|
|
1655
|
+
if (expected === undefined || network.Internal !== expected.internal
|
|
1656
|
+
|| network.Driver !== "bridge" || network.Scope !== "local"
|
|
1657
|
+
|| network.EnableIPv6 !== false || network.Attachable !== false || network.Ingress !== false
|
|
1658
|
+
|| network.ConfigOnly !== false || !isRecord(network.Options) || Object.keys(network.Options).length !== 0) return undefined
|
|
1659
|
+
const inspectedLabels = validResourceLabels(network.Labels, labels.namespace, false, labels.provider)
|
|
1660
|
+
if (!inspectedLabels || JSON.stringify(inspectedLabels) !== JSON.stringify(labels)) return undefined
|
|
1661
|
+
const endpoints = Object.values(network.Containers)
|
|
1662
|
+
if (endpoints.some((endpoint) => !isRecord(endpoint) || endpoint.Name !== identity.relay)) return undefined
|
|
1663
|
+
return true
|
|
1664
|
+
}
|
|
1665
|
+
|
|
1072
1666
|
function validWorktreeMount(mount) {
|
|
1073
1667
|
if (!isRecord(mount) || mount.Target !== "/worktree" || mount.ReadOnly === true) return false
|
|
1074
1668
|
if (mount.Type === "bind") {
|
|
@@ -1092,7 +1686,22 @@ function runIdentityFrom(labels, image, worktree) {
|
|
|
1092
1686
|
}
|
|
1093
1687
|
}
|
|
1094
1688
|
|
|
1095
|
-
function expectedRunIdentity(options, run, network, state, worktreeSubpath) {
|
|
1689
|
+
function expectedRunIdentity(options, run, network, state, worktreeSubpath, relayImageId) {
|
|
1690
|
+
if (options.preflight.provider === "kimi") {
|
|
1691
|
+
const binding = parseThreadwireBinding(options.preflight.binding)
|
|
1692
|
+
if (typeof options.relayImage !== "string" || !(options.relayWorkerUrl instanceof URL) || !isDigestImage(relayImageId)) throw new Error("Kimi run identity is unavailable")
|
|
1693
|
+
return {
|
|
1694
|
+
provider: "kimi", namespace: options.stateRegistry.namespace, run, lineage: state.lineage,
|
|
1695
|
+
task: state.labels["org.threadwire.task"], volume: state.volume, network, egressNetwork: `${run}-egress`,
|
|
1696
|
+
validator: `${run}-validator`, worker: `${run}-worker`, relay: `${run}-relay`, image: binding.context.imageId,
|
|
1697
|
+
relayImage: relayImageId, endpoint: options.relayWorkerUrl.toString(), bindingDigest: threadwireBindingDigest(binding),
|
|
1698
|
+
sourceVolume: binding.source.volume, sourceTarget: binding.source.target, sourceReadOnly: String(binding.source.readOnly),
|
|
1699
|
+
contextVolume: binding.context.volume, contextTarget: binding.context.target,
|
|
1700
|
+
contextManifestDigest: binding.context.digests.manifest, contextContentDigest: binding.context.digests.content,
|
|
1701
|
+
contextImageId: binding.context.imageId, runtimeUid: String(binding.runtime.uid), runtimeGid: String(binding.runtime.gid),
|
|
1702
|
+
workdir: binding.runtime.workdir, taskId: binding.taskId, expectedRevision: binding.source.revision
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1096
1705
|
const volumeMode = options.worktreeVolume !== undefined
|
|
1097
1706
|
return {
|
|
1098
1707
|
provider: options.preflight.provider ?? "codex",
|