threadwire 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +14 -8
- package/TELEGRAM-INGRESS.md +7 -7
- package/docs/capacity-admission.md +147 -0
- package/docs/container-runtime.md +18 -34
- package/docs/development-container.md +77 -0
- package/docs/isolated-provider-runtime.md +23 -20
- package/package.json +2 -3
- package/scripts/provider-shims/front-door.sh.template +1 -5
- package/scripts/verify-package.js +8 -3
- package/src/absolute-deadline.js +6 -3
- package/src/activity-log.js +1 -14
- package/src/cli.js +145 -60
- package/src/docker-api.js +8 -0
- package/src/isolated-runtime-client.js +124 -4
- package/src/isolated-runtime.js +145 -11
- package/src/jsonl-record-spool.js +178 -0
- package/src/kimi-oauth-store.js +16 -0
- package/src/provider-capacity-codex.js +176 -0
- package/src/provider-capacity-kimi.js +105 -0
- package/src/provider-capacity.js +292 -0
- package/src/providers/index.js +4 -4
- package/src/providers/kimi.js +55 -5
- package/src/run-worker.js +290 -75
- package/src/telegram-ingress/config.js +14 -8
- package/src/telegram-ingress/core.js +78 -47
- package/src/telegram-ingress/http.js +0 -6
- package/src/telegram-webhook.js +11 -10
- package/src/workspace-profile.js +0 -212
- package/threadwire.workspace-profiles.json +0 -12
package/src/isolated-runtime.js
CHANGED
|
@@ -29,6 +29,7 @@ const GRANT_LEASE_MS = 60_000
|
|
|
29
29
|
const GRANT_LEASE_MAX_MS = 3_600_000
|
|
30
30
|
const PREFLIGHT_TIMEOUT_MS = 30_000
|
|
31
31
|
const MAX_RAW_OUTPUT_BYTES = 1_048_576
|
|
32
|
+
const WORKER_STOP_GRACE_SECONDS = 3
|
|
32
33
|
|
|
33
34
|
/** @param {{environment?: NodeJS.ProcessEnv, docker?: DockerApi, fetchImplementation?: typeof fetch, now?: () => number}} [options] */
|
|
34
35
|
export async function startIsolatedRuntime(options = {}) {
|
|
@@ -160,21 +161,32 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
160
161
|
const releaseIncoming = incomingRequests.acquire("run")
|
|
161
162
|
const bodyOperation = new AbsoluteDeadline(Date.now() + preflightTimeoutMs, {signal: shutdown.signal, request, response, timeoutMessage: "Isolated runtime operation timed out"})
|
|
162
163
|
let body
|
|
164
|
+
let runLifecycle
|
|
163
165
|
try {
|
|
164
|
-
|
|
166
|
+
if (request.headers["x-threadwire-run-lifecycle"] === "1") {
|
|
167
|
+
runLifecycle = await readRunLifecycle(request, MAX_RUN_BYTES, bodyOperation.signal)
|
|
168
|
+
body = parseRun(runLifecycle.value)
|
|
169
|
+
} else {
|
|
170
|
+
body = parseRun(await readJson(request, MAX_RUN_BYTES, bodyOperation.signal))
|
|
171
|
+
}
|
|
165
172
|
} finally {
|
|
166
173
|
bodyOperation.close()
|
|
167
174
|
releaseIncoming()
|
|
168
175
|
}
|
|
169
176
|
const preflight = preflights.consume(body.preflightId)
|
|
170
177
|
if (!preflight) throw new ClientError("Isolated runtime preflight expired")
|
|
178
|
+
const operationSignal = runLifecycle === undefined
|
|
179
|
+
? shutdown.signal
|
|
180
|
+
: AbortSignal.any([shutdown.signal, runLifecycle.signal])
|
|
171
181
|
const operation = new AbsoluteDeadline(minimumDeadline(preflight.launchDeadlineAt,
|
|
172
182
|
requestedDeadline(request.headers["x-threadwire-launch-deadline"], workerTimeoutMs)), {
|
|
173
|
-
signal:
|
|
183
|
+
signal: operationSignal, request, response, timeoutMessage: "Isolated runtime launch timed out", disconnectMessage: "Isolated runtime caller disconnected",
|
|
184
|
+
destroyRequestOnAbort: runLifecycle === undefined
|
|
174
185
|
})
|
|
175
186
|
const task = taskIdentity(preflight)
|
|
176
187
|
let releaseRun
|
|
177
188
|
let releaseLineage
|
|
189
|
+
let retainLineage = false
|
|
178
190
|
try {
|
|
179
191
|
releaseRun = activeRuns.acquire(task)
|
|
180
192
|
releaseLineage = activeLineages.acquire(preflight.state.lineage)
|
|
@@ -188,6 +200,7 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
188
200
|
if (!response.headersSent) response.writeHead(200, {"content-type": "application/x-ndjson"})
|
|
189
201
|
if (!response.write(`${JSON.stringify(frame)}\n`)) await abortable(once(response, "drain"), operation.signal)
|
|
190
202
|
}
|
|
203
|
+
if (runLifecycle !== undefined && !response.headersSent) response.writeHead(200, {"content-type": "application/x-ndjson"})
|
|
191
204
|
const result = await runIsolatedWorker({
|
|
192
205
|
docker, workerImage, brokerUrl, brokerAdminToken, brokerContainer, relayImage, relayWorkerUrl,
|
|
193
206
|
fetchImplementation, preflight, ...body, task, now, stateRegistry, sessionTtlMs, operation, grantLeaseMs,
|
|
@@ -197,9 +210,13 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
197
210
|
for (const chunk of result.rawChunks) await writeFrame({type: "chunk", ...chunk})
|
|
198
211
|
await writeFrame({type: "terminal", exitCode: result.exitCode})
|
|
199
212
|
response.end()
|
|
213
|
+
} catch (error) {
|
|
214
|
+
if (error instanceof ContainerCleanupConfirmationError) retainLineage = true
|
|
215
|
+
throw error
|
|
200
216
|
} finally {
|
|
201
217
|
operation.close()
|
|
202
|
-
|
|
218
|
+
runLifecycle?.close()
|
|
219
|
+
if (!retainLineage) releaseLineage?.()
|
|
203
220
|
releaseRun?.()
|
|
204
221
|
void collectState()
|
|
205
222
|
}
|
|
@@ -209,7 +226,9 @@ export async function startIsolatedRuntime(options = {}) {
|
|
|
209
226
|
response.end()
|
|
210
227
|
} catch (error) {
|
|
211
228
|
if (response.headersSent) {
|
|
212
|
-
if (!response.writableEnded && !response.destroyed)
|
|
229
|
+
if (!response.writableEnded && !response.destroyed) {
|
|
230
|
+
response.end(error instanceof ContainerCleanupConfirmationError ? undefined : `${JSON.stringify({type: "terminal", exitCode: 1})}\n`)
|
|
231
|
+
}
|
|
213
232
|
return
|
|
214
233
|
}
|
|
215
234
|
sendJson(response, error instanceof ClientError ? 400 : 503, {
|
|
@@ -367,6 +386,9 @@ export async function runIsolatedWorker(options) {
|
|
|
367
386
|
let grantRenewalTimer
|
|
368
387
|
let runTracked = false
|
|
369
388
|
let runStage = "state"
|
|
389
|
+
let workerResult
|
|
390
|
+
let runError
|
|
391
|
+
let cleanupFailure
|
|
370
392
|
const operation = options.operation
|
|
371
393
|
const worktreeSubpath = options.worktreeVolume === undefined
|
|
372
394
|
? ""
|
|
@@ -632,10 +654,10 @@ export async function runIsolatedWorker(options) {
|
|
|
632
654
|
throw error
|
|
633
655
|
}
|
|
634
656
|
}
|
|
635
|
-
|
|
657
|
+
workerResult = result
|
|
636
658
|
} catch (error) {
|
|
637
659
|
process.stderr.write(`threadwire-runtime: isolated run failed at ${runStage}\n`)
|
|
638
|
-
|
|
660
|
+
runError = error
|
|
639
661
|
} finally {
|
|
640
662
|
let resourcesRemoved = true
|
|
641
663
|
if (grantRenewalTimer !== undefined) clearInterval(grantRenewalTimer)
|
|
@@ -658,10 +680,17 @@ export async function runIsolatedWorker(options) {
|
|
|
658
680
|
resourcesRemoved = false
|
|
659
681
|
process.stderr.write("threadwire-isolated-runtime: validator cleanup failed\n")
|
|
660
682
|
})
|
|
661
|
-
if (containerId !== undefined)
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
683
|
+
if (containerId !== undefined) {
|
|
684
|
+
try {
|
|
685
|
+
await stopAndRemoveWorkerContainer(options.docker, containerId, cleanupOptions())
|
|
686
|
+
} catch (error) {
|
|
687
|
+
resourcesRemoved = false
|
|
688
|
+
cleanupFailure = error instanceof ContainerCleanupConfirmationError
|
|
689
|
+
? error
|
|
690
|
+
: new ContainerCleanupConfirmationError({cause: error})
|
|
691
|
+
process.stderr.write("threadwire-isolated-runtime: worker cleanup failed\n")
|
|
692
|
+
}
|
|
693
|
+
}
|
|
665
694
|
if (relayId !== undefined) await cleanupDocker(() => options.docker.removeContainer(relayId, cleanupOptions())).catch(() => {
|
|
666
695
|
resourcesRemoved = false
|
|
667
696
|
process.stderr.write("threadwire-isolated-runtime: relay cleanup failed\n")
|
|
@@ -677,16 +706,43 @@ export async function runIsolatedWorker(options) {
|
|
|
677
706
|
resourcesRemoved = false
|
|
678
707
|
process.stderr.write("threadwire-isolated-runtime: egress network cleanup failed\n")
|
|
679
708
|
})
|
|
680
|
-
if (freshState && !keepFreshState) await cleanupDocker(() => options.docker.removeVolume(state.volume, cleanupOptions())).catch(() => { resourcesRemoved = false })
|
|
709
|
+
if (freshState && !keepFreshState && resourcesRemoved) await cleanupDocker(() => options.docker.removeVolume(state.volume, cleanupOptions())).catch(() => { resourcesRemoved = false })
|
|
681
710
|
if (runTracked && resourcesRemoved) await options.stateRegistry.completeRun(runId, cleanupOperation.signal).catch(() => {})
|
|
682
711
|
emergencyCleanup?.close()
|
|
683
712
|
}
|
|
713
|
+
if (cleanupFailure !== undefined) throw cleanupFailure
|
|
714
|
+
if (runError !== undefined) throw runError
|
|
715
|
+
return workerResult
|
|
684
716
|
}
|
|
685
717
|
|
|
686
718
|
async function cleanupDocker(operation) {
|
|
687
719
|
return operation()
|
|
688
720
|
}
|
|
689
721
|
|
|
722
|
+
async function stopAndRemoveWorkerContainer(docker, containerId, requestOptions) {
|
|
723
|
+
let inspection = await docker.inspectContainer(containerId, requestOptions)
|
|
724
|
+
if (typeof inspection?.State?.Running !== "boolean") throw new Error("Worker container state unavailable")
|
|
725
|
+
if (inspection.State.Running) {
|
|
726
|
+
await docker.stopContainer(containerId, WORKER_STOP_GRACE_SECONDS, requestOptions)
|
|
727
|
+
inspection = await docker.inspectContainer(containerId, requestOptions)
|
|
728
|
+
if (typeof inspection?.State?.Running !== "boolean") throw new Error("Worker container state unavailable")
|
|
729
|
+
if (inspection.State.Running) {
|
|
730
|
+
await docker.killContainer(containerId, "KILL", requestOptions)
|
|
731
|
+
inspection = await docker.inspectContainer(containerId, requestOptions)
|
|
732
|
+
if (inspection?.State?.Running !== false) throw new ContainerCleanupConfirmationError()
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
await docker.removeContainer(containerId, requestOptions)
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
class ContainerCleanupConfirmationError extends Error {
|
|
739
|
+
/** @param {{cause?: unknown}} [options] */
|
|
740
|
+
constructor(options = {}) {
|
|
741
|
+
super("Worker container cleanup could not be confirmed", options)
|
|
742
|
+
this.name = "ContainerCleanupConfirmationError"
|
|
743
|
+
}
|
|
744
|
+
}
|
|
745
|
+
|
|
690
746
|
export async function prepareStateVolume(docker, state, fresh, requestOptions) {
|
|
691
747
|
if (fresh) await docker.createVolume(state.volume, state.labels, requestOptions)
|
|
692
748
|
const volume = await docker.inspectVolume(state.volume, requestOptions).catch(() => undefined)
|
|
@@ -1069,6 +1125,84 @@ async function readJson(request, capacity, signal) {
|
|
|
1069
1125
|
}
|
|
1070
1126
|
}
|
|
1071
1127
|
|
|
1128
|
+
function readRunLifecycle(request, capacity, signal) {
|
|
1129
|
+
const cancellation = new AbortController()
|
|
1130
|
+
/** @type {Buffer[]} */
|
|
1131
|
+
const chunks = []
|
|
1132
|
+
let bytes = 0
|
|
1133
|
+
let parsed = false
|
|
1134
|
+
let settled = false
|
|
1135
|
+
const cancel = (reason) => cancellation.abort(reason instanceof Error ? reason : new Error("Isolated runtime caller cancelled"))
|
|
1136
|
+
const close = () => {
|
|
1137
|
+
request.off("data", onData)
|
|
1138
|
+
request.off("end", onEnd)
|
|
1139
|
+
request.off("aborted", onAborted)
|
|
1140
|
+
request.off("error", onError)
|
|
1141
|
+
signal?.removeEventListener("abort", onSignal)
|
|
1142
|
+
}
|
|
1143
|
+
/** @type {(value: {value: unknown, signal: AbortSignal, close: () => void}) => void} */
|
|
1144
|
+
let resolveLifecycle
|
|
1145
|
+
/** @type {(error: Error) => void} */
|
|
1146
|
+
let rejectLifecycle
|
|
1147
|
+
const fail = (error) => {
|
|
1148
|
+
if (parsed) {
|
|
1149
|
+
cancel(error)
|
|
1150
|
+
return
|
|
1151
|
+
}
|
|
1152
|
+
if (settled) return
|
|
1153
|
+
settled = true
|
|
1154
|
+
close()
|
|
1155
|
+
rejectLifecycle(error)
|
|
1156
|
+
}
|
|
1157
|
+
const onData = (chunk) => {
|
|
1158
|
+
if (parsed) {
|
|
1159
|
+
fail(new ClientError("Invalid request"))
|
|
1160
|
+
return
|
|
1161
|
+
}
|
|
1162
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
|
1163
|
+
bytes += buffer.length
|
|
1164
|
+
if (bytes > capacity) {
|
|
1165
|
+
fail(new ClientError("Request exceeded capacity"))
|
|
1166
|
+
return
|
|
1167
|
+
}
|
|
1168
|
+
chunks.push(buffer)
|
|
1169
|
+
const combined = Buffer.concat(chunks, bytes)
|
|
1170
|
+
const newline = combined.indexOf(0x0a)
|
|
1171
|
+
if (newline < 0) return
|
|
1172
|
+
if (newline !== combined.length - 1) {
|
|
1173
|
+
fail(new ClientError("Invalid request"))
|
|
1174
|
+
return
|
|
1175
|
+
}
|
|
1176
|
+
let value
|
|
1177
|
+
try {
|
|
1178
|
+
value = JSON.parse(combined.subarray(0, newline).toString("utf8"))
|
|
1179
|
+
} catch {
|
|
1180
|
+
fail(new ClientError("Invalid request"))
|
|
1181
|
+
return
|
|
1182
|
+
}
|
|
1183
|
+
parsed = true
|
|
1184
|
+
settled = true
|
|
1185
|
+
resolveLifecycle({value, signal: cancellation.signal, close})
|
|
1186
|
+
}
|
|
1187
|
+
const onEnd = () => parsed
|
|
1188
|
+
? cancel(new Error("Isolated runtime caller cancelled"))
|
|
1189
|
+
: fail(new ClientError("Invalid request"))
|
|
1190
|
+
const onAborted = () => fail(new Error("Isolated runtime caller disconnected"))
|
|
1191
|
+
const onError = (error) => fail(error)
|
|
1192
|
+
const onSignal = () => fail(signal.reason instanceof Error ? signal.reason : new Error("Isolated runtime operation aborted"))
|
|
1193
|
+
const lifecycle = new Promise((resolve, reject) => {
|
|
1194
|
+
resolveLifecycle = resolve
|
|
1195
|
+
rejectLifecycle = reject
|
|
1196
|
+
})
|
|
1197
|
+
request.on("data", onData)
|
|
1198
|
+
request.once("end", onEnd)
|
|
1199
|
+
request.once("aborted", onAborted)
|
|
1200
|
+
request.once("error", onError)
|
|
1201
|
+
if (signal?.aborted) onSignal()
|
|
1202
|
+
else signal?.addEventListener("abort", onSignal, {once: true})
|
|
1203
|
+
return lifecycle
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1072
1206
|
function requireControl(value, expected) {
|
|
1073
1207
|
if (value !== `Bearer ${expected}`) throw new ClientError("Request denied")
|
|
1074
1208
|
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {closeSync, mkdtempSync, openSync, readFileSync, rmSync, unlinkSync, writeSync} from "node:fs"
|
|
4
|
+
import {tmpdir} from "node:os"
|
|
5
|
+
import {isAbsolute, join, normalize} from "node:path"
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Single-owner byte-oriented spool for one worker's pending JSONL record.
|
|
9
|
+
*
|
|
10
|
+
* Worker stdout is consumed as raw bytes. A pending record (bytes not yet
|
|
11
|
+
* terminated by `\n`) is kept in bounded memory while it stays below
|
|
12
|
+
* `memoryBytes`; beyond that threshold it spills exactly once to one private
|
|
13
|
+
* run-scoped file, writing every byte segment exactly once. Newline scanning
|
|
14
|
+
* happens on raw bytes: `0x0a` can never appear inside a multibyte UTF-8
|
|
15
|
+
* sequence, so newline boundaries are preserved across arbitrary chunk splits
|
|
16
|
+
* and completed records always decode as well-formed UTF-8.
|
|
17
|
+
*
|
|
18
|
+
* `maxBytes` is a separate absolute pending-record cap: unterminated or
|
|
19
|
+
* malformed output can never consume unbounded memory, disk, or JSON.parse
|
|
20
|
+
* allocation. `close()` is idempotent and removes every spool artifact; the
|
|
21
|
+
* owner calls it on every settlement path.
|
|
22
|
+
*/
|
|
23
|
+
export class JsonlRecordSpool {
|
|
24
|
+
/** @param {{memoryBytes: number, maxBytes: number, directory?: string}} options */
|
|
25
|
+
constructor(options) {
|
|
26
|
+
this.memoryBytes = positiveSafeInteger(options.memoryBytes, "memoryBytes")
|
|
27
|
+
this.maxBytes = positiveSafeInteger(options.maxBytes, "maxBytes")
|
|
28
|
+
const directory = options.directory ?? tmpdir()
|
|
29
|
+
if (!isAbsolute(directory) || normalize(directory) !== directory) throw new Error("directory must be a normalized absolute path")
|
|
30
|
+
this.directory = directory
|
|
31
|
+
/** @type {Buffer[]} */
|
|
32
|
+
this.segments = []
|
|
33
|
+
this.pendingBytes = 0
|
|
34
|
+
/** @type {{fd: number, path: string} | null} */
|
|
35
|
+
this.file = null
|
|
36
|
+
/** @type {string | null} */
|
|
37
|
+
this.ownedDirectory = null
|
|
38
|
+
this.closed = false
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Consume one stdout chunk and yield each completed record in order, decoded
|
|
43
|
+
* as UTF-8 without the terminating newline. Scanning advances only as the
|
|
44
|
+
* owner consumes records, so terminal protocol completion can leave trailing
|
|
45
|
+
* bytes in the same chunk untouched.
|
|
46
|
+
* @param {Buffer} chunk
|
|
47
|
+
* @returns {Generator<string>}
|
|
48
|
+
* @yields {string}
|
|
49
|
+
*/
|
|
50
|
+
*push(chunk) {
|
|
51
|
+
if (this.closed) throw new Error("JSONL record spool is closed")
|
|
52
|
+
let segmentStart = 0
|
|
53
|
+
for (let index = 0; index < chunk.length; index += 1) {
|
|
54
|
+
if (chunk[index] !== 0x0a) continue
|
|
55
|
+
const record = this.completeRecord(chunk.subarray(segmentStart, index))
|
|
56
|
+
segmentStart = index + 1
|
|
57
|
+
try {
|
|
58
|
+
yield record
|
|
59
|
+
} finally {
|
|
60
|
+
this.removeOwnedDirectory()
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (segmentStart < chunk.length) this.append(chunk.subarray(segmentStart))
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* End of stream: return the pending record ("" when none) and release it.
|
|
68
|
+
* @returns {string}
|
|
69
|
+
*/
|
|
70
|
+
end() {
|
|
71
|
+
if (this.closed || this.pendingBytes === 0) return ""
|
|
72
|
+
let record
|
|
73
|
+
if (this.file === null) {
|
|
74
|
+
record = Buffer.concat(this.segments, this.pendingBytes).toString("utf8")
|
|
75
|
+
this.segments = []
|
|
76
|
+
} else {
|
|
77
|
+
const file = this.file
|
|
78
|
+
this.file = null
|
|
79
|
+
closeSync(file.fd)
|
|
80
|
+
record = readFileSync(file.path, "utf8")
|
|
81
|
+
unlinkSync(file.path)
|
|
82
|
+
this.removeOwnedDirectory()
|
|
83
|
+
}
|
|
84
|
+
this.pendingBytes = 0
|
|
85
|
+
return record
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** Idempotent: close the spool file and remove every run-scoped artifact. */
|
|
89
|
+
close() {
|
|
90
|
+
if (this.closed) return
|
|
91
|
+
this.closed = true
|
|
92
|
+
this.segments = []
|
|
93
|
+
const file = this.file
|
|
94
|
+
this.file = null
|
|
95
|
+
if (file !== null) {
|
|
96
|
+
closeSync(file.fd)
|
|
97
|
+
rmSync(file.path, {force: true})
|
|
98
|
+
}
|
|
99
|
+
this.removeOwnedDirectory()
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Finish one record: combine the pending bytes with the final segment and
|
|
104
|
+
* decode exactly once. A spilled record is closed, read back, and unlinked;
|
|
105
|
+
* the next oversized record spills into a freshly created file, so at most
|
|
106
|
+
* one private file ever exists for the run.
|
|
107
|
+
* @param {Buffer} finalSegment @returns {string}
|
|
108
|
+
*/
|
|
109
|
+
completeRecord(finalSegment) {
|
|
110
|
+
const recordBytes = this.pendingBytes + finalSegment.length
|
|
111
|
+
if (recordBytes > this.maxBytes) throw new StdoutRecordTooLargeError()
|
|
112
|
+
if (this.file === null && recordBytes <= this.memoryBytes) {
|
|
113
|
+
const record = this.pendingBytes === 0
|
|
114
|
+
? finalSegment.toString("utf8")
|
|
115
|
+
: Buffer.concat([...this.segments, finalSegment], recordBytes).toString("utf8")
|
|
116
|
+
this.segments = []
|
|
117
|
+
this.pendingBytes = 0
|
|
118
|
+
return record
|
|
119
|
+
}
|
|
120
|
+
if (this.file === null) this.append(finalSegment)
|
|
121
|
+
else {
|
|
122
|
+
this.pendingBytes = recordBytes
|
|
123
|
+
writeSync(this.file.fd, finalSegment)
|
|
124
|
+
}
|
|
125
|
+
const file = this.file
|
|
126
|
+
if (file === null) throw new Error("JSONL record spool file is unavailable")
|
|
127
|
+
this.file = null
|
|
128
|
+
closeSync(file.fd)
|
|
129
|
+
const record = readFileSync(file.path, "utf8")
|
|
130
|
+
unlinkSync(file.path)
|
|
131
|
+
this.pendingBytes = 0
|
|
132
|
+
return record
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** @param {Buffer} segment */
|
|
136
|
+
append(segment) {
|
|
137
|
+
this.pendingBytes += segment.length
|
|
138
|
+
if (this.pendingBytes > this.maxBytes) throw new StdoutRecordTooLargeError()
|
|
139
|
+
if (this.file !== null) {
|
|
140
|
+
writeSync(this.file.fd, segment)
|
|
141
|
+
return
|
|
142
|
+
}
|
|
143
|
+
if (this.pendingBytes <= this.memoryBytes) {
|
|
144
|
+
this.segments.push(segment)
|
|
145
|
+
return
|
|
146
|
+
}
|
|
147
|
+
// First threshold crossing: create the private run-scoped file and write
|
|
148
|
+
// each buffered segment exactly once, in order.
|
|
149
|
+
this.ownedDirectory = mkdtempSync(join(this.directory, "threadwire-record-"))
|
|
150
|
+
const path = join(this.ownedDirectory, "pending-record")
|
|
151
|
+
const fd = openSync(path, "wx", 0o600)
|
|
152
|
+
this.file = {fd, path}
|
|
153
|
+
for (const buffered of this.segments) writeSync(fd, buffered)
|
|
154
|
+
this.segments = []
|
|
155
|
+
writeSync(fd, segment)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Remove the current record's private spill directory, if one exists. */
|
|
159
|
+
removeOwnedDirectory() {
|
|
160
|
+
const directory = this.ownedDirectory
|
|
161
|
+
if (directory === null) return
|
|
162
|
+
rmSync(directory, {recursive: true, force: true})
|
|
163
|
+
this.ownedDirectory = null
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export class StdoutRecordTooLargeError extends Error {
|
|
168
|
+
constructor() {
|
|
169
|
+
super("Worker stdout record exceeded the configured byte capacity")
|
|
170
|
+
this.name = "StdoutRecordTooLargeError"
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** @param {number} value @param {string} name */
|
|
175
|
+
function positiveSafeInteger(value, name) {
|
|
176
|
+
if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive safe integer`)
|
|
177
|
+
return value
|
|
178
|
+
}
|
package/src/kimi-oauth-store.js
CHANGED
|
@@ -7,6 +7,7 @@ import {dirname, isAbsolute, join} from "node:path"
|
|
|
7
7
|
|
|
8
8
|
const MAX_CREDENTIAL_BYTES = 65_536
|
|
9
9
|
const MAX_TOKEN_BYTES = 8_192
|
|
10
|
+
const CURRENT_TOKEN_SAFETY_MARGIN_SECONDS = 30
|
|
10
11
|
const CLIENT_ID = "17e5f671-d194-4dfb-9706-5516cb48c098"
|
|
11
12
|
const TOKEN_URL = "https://auth.kimi.com/api/oauth/token"
|
|
12
13
|
const EXACT_KEYS = ["access_token", "expires_at", "expires_in", "refresh_token", "scope", "token_type"]
|
|
@@ -117,6 +118,19 @@ export async function openKimiOAuthStore(options) {
|
|
|
117
118
|
refreshInFlight = operation
|
|
118
119
|
return operation
|
|
119
120
|
},
|
|
121
|
+
/**
|
|
122
|
+
* Non-mutating read for observational callers: returns the already-loaded
|
|
123
|
+
* access token only when it stays valid beyond a fixed 30-second safety
|
|
124
|
+
* margin. It never fetches, refreshes, persists, renames, chmods, or
|
|
125
|
+
* tombstones; logged-out credentials fail with the fixed needs-login error
|
|
126
|
+
* and expired or near-expiry credentials fail with the fixed
|
|
127
|
+
* refresh-required error.
|
|
128
|
+
*/
|
|
129
|
+
async getCurrentAccessToken() {
|
|
130
|
+
if (credential.access_token.length === 0) throw needsLogin()
|
|
131
|
+
if (credential.expires_at - now() <= CURRENT_TOKEN_SAFETY_MARGIN_SECONDS) throw refreshRequired()
|
|
132
|
+
return credential.access_token
|
|
133
|
+
},
|
|
120
134
|
async health() {
|
|
121
135
|
return credential.access_token.length === 0
|
|
122
136
|
? {ready: false, needsLogin: true}
|
|
@@ -263,5 +277,7 @@ function record(value) { return typeof value === "object" && value !== null && !
|
|
|
263
277
|
function unavailable() { return new Error("Kimi OAuth credential file is unavailable") }
|
|
264
278
|
/** @returns {Error} */
|
|
265
279
|
function needsLogin() { return new Error("Kimi OAuth credential was rejected; re-login required") }
|
|
280
|
+
/** @returns {Error} */
|
|
281
|
+
function refreshRequired() { return new Error("Kimi OAuth credential requires refresh") }
|
|
266
282
|
/** @param {unknown} [cause] @returns {Error} */
|
|
267
283
|
function refreshUnavailable(cause) { return new Error("Kimi OAuth refresh is unavailable", cause === undefined ? undefined : {cause}) }
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {spawn} from "node:child_process"
|
|
4
|
+
import {providerExecutable} from "./providers/executable.js"
|
|
5
|
+
import {CapacityProbeError, DEFAULT_CAPACITY_TIMEOUT_MS, normalizeCodexRateLimits} from "./provider-capacity.js"
|
|
6
|
+
|
|
7
|
+
const DEFAULT_EXECUTABLE = "/opt/data/libexec/threadwire/codex"
|
|
8
|
+
const MAX_LINE_BYTES = 65_536
|
|
9
|
+
const MAX_LINES = 1_024
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Probe live Codex account capacity through `codex app-server` JSON-RPC over
|
|
13
|
+
* stdio: `initialize` followed by `account/rateLimits/read`. The child is
|
|
14
|
+
* always killed, its stderr is consumed but never forwarded, and server error
|
|
15
|
+
* detail (which can carry account or credential material) is never propagated —
|
|
16
|
+
* failures classify with fixed, credential-free messages.
|
|
17
|
+
* @param {{
|
|
18
|
+
* env?: NodeJS.ProcessEnv,
|
|
19
|
+
* spawnImplementation?: typeof spawn | undefined,
|
|
20
|
+
* timeoutMs?: number
|
|
21
|
+
* }} options
|
|
22
|
+
* @returns {Promise<import("./provider-capacity.js").CapacityWindows>}
|
|
23
|
+
*/
|
|
24
|
+
export async function probeCodexCapacity(options) {
|
|
25
|
+
const environment = options.env ?? process.env
|
|
26
|
+
const spawnImplementation = options.spawnImplementation ?? spawn
|
|
27
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_CAPACITY_TIMEOUT_MS
|
|
28
|
+
const executable = providerExecutable("THREADWIRE_CODEX_BIN", DEFAULT_EXECUTABLE, environment)
|
|
29
|
+
const result = await appServerExchange(spawnImplementation, executable, environment, timeoutMs)
|
|
30
|
+
return normalizeCodexRateLimits(result)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Run the bounded two-request JSON-RPC exchange and resolve with the raw
|
|
35
|
+
* `account/rateLimits/read` result for fail-closed normalization.
|
|
36
|
+
* @param {typeof spawn} spawnImplementation
|
|
37
|
+
* @param {string} executable
|
|
38
|
+
* @param {NodeJS.ProcessEnv} environment
|
|
39
|
+
* @param {number} timeoutMs
|
|
40
|
+
* @returns {Promise<unknown>}
|
|
41
|
+
*/
|
|
42
|
+
function appServerExchange(spawnImplementation, executable, environment, timeoutMs) {
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
/** @type {ReturnType<typeof spawn>} */
|
|
45
|
+
let child
|
|
46
|
+
try {
|
|
47
|
+
child = spawnImplementation(executable, ["app-server"], {env: environment, stdio: ["pipe", "pipe", "pipe"]})
|
|
48
|
+
} catch {
|
|
49
|
+
reject(new CapacityProbeError("unavailable", "Codex app-server is unavailable"))
|
|
50
|
+
return
|
|
51
|
+
}
|
|
52
|
+
let settled = false
|
|
53
|
+
let buffer = ""
|
|
54
|
+
let lines = 0
|
|
55
|
+
let phase = "initialize"
|
|
56
|
+
// The timer is always cleared by finish, so it never outlives the exchange.
|
|
57
|
+
const timer = setTimeout(() => {
|
|
58
|
+
fail(new CapacityProbeError("unavailable", "Codex app-server capacity probe timed out"))
|
|
59
|
+
}, timeoutMs)
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* @param {(value: unknown) => void} settle
|
|
63
|
+
* @param {unknown} value
|
|
64
|
+
*/
|
|
65
|
+
function finish(settle, value) {
|
|
66
|
+
if (settled) return
|
|
67
|
+
settled = true
|
|
68
|
+
clearTimeout(timer)
|
|
69
|
+
child.kill()
|
|
70
|
+
settle(value)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** @param {CapacityProbeError} error */
|
|
74
|
+
function fail(error) {
|
|
75
|
+
finish(reject, error)
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** @param {Record<string, unknown>} request */
|
|
79
|
+
function write(request) {
|
|
80
|
+
try {
|
|
81
|
+
child.stdin?.write(`${JSON.stringify(request)}\n`)
|
|
82
|
+
} catch {
|
|
83
|
+
fail(new CapacityProbeError("unavailable", "Codex app-server is unavailable"))
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** @param {string} line */
|
|
88
|
+
function handleLine(line) {
|
|
89
|
+
if (line.trim().length === 0) return
|
|
90
|
+
/** @type {unknown} */
|
|
91
|
+
let message
|
|
92
|
+
try {
|
|
93
|
+
message = JSON.parse(line)
|
|
94
|
+
} catch {
|
|
95
|
+
fail(new CapacityProbeError("protocol", "Codex app-server emitted a malformed record"))
|
|
96
|
+
return
|
|
97
|
+
}
|
|
98
|
+
if (!isRecord(message) || (message.id !== 1 && message.id !== 2)) return
|
|
99
|
+
if (message.error !== undefined) {
|
|
100
|
+
if (message.id === 1 || phase === "initialize") {
|
|
101
|
+
fail(new CapacityProbeError("protocol", "Codex app-server initialize failed"))
|
|
102
|
+
} else if (isAuthRejection(message.error)) {
|
|
103
|
+
fail(new CapacityProbeError("auth", "Codex rate-limits credential was rejected"))
|
|
104
|
+
} else {
|
|
105
|
+
fail(new CapacityProbeError("protocol", "Codex rate-limits request failed"))
|
|
106
|
+
}
|
|
107
|
+
return
|
|
108
|
+
}
|
|
109
|
+
if (message.id === 1) {
|
|
110
|
+
if (phase !== "initialize") {
|
|
111
|
+
fail(new CapacityProbeError("protocol", "Codex app-server answered out of order"))
|
|
112
|
+
return
|
|
113
|
+
}
|
|
114
|
+
phase = "read"
|
|
115
|
+
write({jsonrpc: "2.0", id: 2, method: "account/rateLimits/read"})
|
|
116
|
+
return
|
|
117
|
+
}
|
|
118
|
+
if (phase !== "read") {
|
|
119
|
+
fail(new CapacityProbeError("protocol", "Codex app-server answered out of order"))
|
|
120
|
+
return
|
|
121
|
+
}
|
|
122
|
+
finish(resolve, message.result)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
child.on("error", () => fail(new CapacityProbeError("unavailable", "Codex app-server is unavailable")))
|
|
126
|
+
child.on("close", () => fail(new CapacityProbeError("unavailable", "Codex app-server exited before answering")))
|
|
127
|
+
child.stdin?.on("error", () => {})
|
|
128
|
+
child.stderr?.on("data", () => {})
|
|
129
|
+
child.stdout?.on("data", (chunk) => {
|
|
130
|
+
if (settled) return
|
|
131
|
+
buffer += /** @type {Buffer} */ (chunk).toString("utf8")
|
|
132
|
+
let index = buffer.indexOf("\n")
|
|
133
|
+
while (index >= 0 && !settled) {
|
|
134
|
+
const line = buffer.slice(0, index)
|
|
135
|
+
buffer = buffer.slice(index + 1)
|
|
136
|
+
lines += 1
|
|
137
|
+
if (line.length > MAX_LINE_BYTES || lines > MAX_LINES) {
|
|
138
|
+
fail(new CapacityProbeError("protocol", "Codex app-server exceeded the bounded exchange"))
|
|
139
|
+
return
|
|
140
|
+
}
|
|
141
|
+
handleLine(line)
|
|
142
|
+
index = buffer.indexOf("\n")
|
|
143
|
+
}
|
|
144
|
+
if (!settled && buffer.length > MAX_LINE_BYTES) {
|
|
145
|
+
fail(new CapacityProbeError("protocol", "Codex app-server exceeded the bounded exchange"))
|
|
146
|
+
}
|
|
147
|
+
})
|
|
148
|
+
|
|
149
|
+
write({
|
|
150
|
+
jsonrpc: "2.0",
|
|
151
|
+
id: 1,
|
|
152
|
+
method: "initialize",
|
|
153
|
+
params: {clientInfo: {name: "threadwire-capacity", version: "1"}}
|
|
154
|
+
})
|
|
155
|
+
})
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* A JSON-RPC error whose server message indicates an authentication or login
|
|
160
|
+
* failure classifies as `auth`; everything else stays `protocol`. The server
|
|
161
|
+
* message itself is only inspected, never propagated.
|
|
162
|
+
* @param {unknown} value
|
|
163
|
+
* @returns {boolean}
|
|
164
|
+
*/
|
|
165
|
+
function isAuthRejection(value) {
|
|
166
|
+
if (!isRecord(value) || typeof value.message !== "string") return false
|
|
167
|
+
return /auth|login|unauthorized|token/iu.test(value.message)
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* @param {unknown} value
|
|
172
|
+
* @returns {value is Record<string, unknown>}
|
|
173
|
+
*/
|
|
174
|
+
function isRecord(value) {
|
|
175
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
176
|
+
}
|