threadwire 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +11 -2
- package/docs/capacity-admission.md +147 -0
- package/docs/development-container.md +77 -0
- package/package.json +1 -1
- package/scripts/verify-package.js +8 -0
- package/src/absolute-deadline.js +6 -3
- package/src/activity-log-status.js +291 -0
- package/src/activity-log.js +91 -3
- package/src/cli.js +153 -8
- package/src/docker-api.js +8 -0
- package/src/isolated-runtime-client.js +124 -4
- package/src/isolated-runtime.js +145 -11
- package/src/jsonl-record-spool.js +178 -0
- package/src/kimi-oauth-store.js +16 -0
- package/src/provider-capacity-codex.js +176 -0
- package/src/provider-capacity-kimi.js +105 -0
- package/src/provider-capacity.js +292 -0
- package/src/providers/index.js +110 -6
- package/src/providers/kimi.js +30 -0
- package/src/run-worker.js +286 -71
- package/src/telegram-ingress/core.js +51 -22
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
import {isAbsolute, resolve} from "node:path"
|
|
4
|
+
import {open} from "node:fs/promises"
|
|
5
|
+
|
|
6
|
+
export const ACTIVITY_LOG_READ_CAPACITY = 1_048_576
|
|
7
|
+
|
|
8
|
+
const KNOWN_TYPES = new Set([
|
|
9
|
+
"provider-started", "controller-started", "session-available", "activity", "health", "terminal"
|
|
10
|
+
])
|
|
11
|
+
|
|
12
|
+
const SESSION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,511}$/u
|
|
13
|
+
|
|
14
|
+
const VALIDATED_FACT_FIELDS = {
|
|
15
|
+
"provider-started": ["type", "provider", "pid"],
|
|
16
|
+
"controller-started": ["type", "pid"],
|
|
17
|
+
"session-available": ["type", "provider", "sessionId"],
|
|
18
|
+
"activity": ["type", "provider", "kind"],
|
|
19
|
+
"health": ["type", "provider", "disposition", "category"],
|
|
20
|
+
"terminal": ["type", "provider", "state", "exitCode"]
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const OPTIONAL_FACT_FIELDS = {
|
|
24
|
+
"provider-started": ["at"],
|
|
25
|
+
"controller-started": ["at"],
|
|
26
|
+
"session-available": ["at"],
|
|
27
|
+
"activity": ["at"],
|
|
28
|
+
"health": ["retryAfterMs", "at"],
|
|
29
|
+
"terminal": ["at"]
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const PROVIDERS = new Set(["codex", "claude", "kimi", "opencode"])
|
|
33
|
+
const ACTIVITY_KINDS = new Set(["delta", "tool", "lifecycle"])
|
|
34
|
+
const HEALTH_DISPOSITIONS = new Set(["retrying", "blocked"])
|
|
35
|
+
const HEALTH_CATEGORIES = new Set([
|
|
36
|
+
"authentication", "permission", "rate-limit", "quota", "billing",
|
|
37
|
+
"model", "network", "protocol", "unknown"
|
|
38
|
+
])
|
|
39
|
+
const TERMINAL_STATES = new Set(["completed", "failed", "cancelled"])
|
|
40
|
+
|
|
41
|
+
const MAX_SAFE_DATE_MS = 8640000000000000 // Date maximum
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* @typedef {{version: number, state: "running" | "retrying" | "blocked" | "completed" | "failed" | "cancelled" | "unknown", provider: string | null, controllerPid: number | null, providerPid: number | null, continuationHandle: string | null, startedAt: string | null, lastActivityAt: string | null, quietForMs: number, health: {disposition: "retrying" | "blocked", category: string, retryAfterMs?: number} | null, terminal: {state: "completed" | "failed" | "cancelled", exitCode: number} | null}} StatusDocument
|
|
45
|
+
*/
|
|
46
|
+
|
|
47
|
+
/** @param {string[]} arguments_ */
|
|
48
|
+
export function parseStatusArguments(arguments_) {
|
|
49
|
+
if (arguments_[0] !== "status") throw new Error("Usage: threadwire status --activity-log <absolute-path>")
|
|
50
|
+
const own = arguments_.slice(1)
|
|
51
|
+
if (own.length !== 2 || own[0] !== "--activity-log") throw new Error("Usage: threadwire status --activity-log <absolute-path>")
|
|
52
|
+
const path = own[1]
|
|
53
|
+
if (!path) throw new Error("--activity-log must be an absolute path")
|
|
54
|
+
const resolved = resolve(path)
|
|
55
|
+
if (!isAbsolute(path)) throw new Error("--activity-log must be an absolute path")
|
|
56
|
+
return {activityLog: resolved}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Read and validate an activity log file, with a bounded read capacity to
|
|
61
|
+
* prevent uncontrolled allocation. Skips at most one incomplete trailing line.
|
|
62
|
+
* Rejects malformed completed records and records with unsafe field names.
|
|
63
|
+
* @param {string} path
|
|
64
|
+
* @returns {Promise<Record<string, unknown>[]>}
|
|
65
|
+
*/
|
|
66
|
+
export async function readActivityLog(path) {
|
|
67
|
+
const fd = await open(path, "r")
|
|
68
|
+
try {
|
|
69
|
+
const stat = await fd.stat()
|
|
70
|
+
if (stat.size > ACTIVITY_LOG_READ_CAPACITY) throw new Error("Activity log exceeds read capacity")
|
|
71
|
+
const buffer = Buffer.alloc(stat.size > 0 ? stat.size : 0)
|
|
72
|
+
let bytesRead = 0
|
|
73
|
+
while (bytesRead < stat.size) {
|
|
74
|
+
const result = await fd.read(buffer, bytesRead, stat.size - bytesRead, bytesRead)
|
|
75
|
+
if (result.bytesRead === 0) break
|
|
76
|
+
bytesRead += result.bytesRead
|
|
77
|
+
}
|
|
78
|
+
return parseActivityLogContent(buffer.toString("utf8", 0, bytesRead))
|
|
79
|
+
} finally {
|
|
80
|
+
await fd.close()
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* @param {string} content
|
|
86
|
+
* @returns {Record<string, unknown>[]}
|
|
87
|
+
*/
|
|
88
|
+
function parseActivityLogContent(content) {
|
|
89
|
+
const lines = content.split("\n")
|
|
90
|
+
/** @type {Record<string, unknown>[]} */
|
|
91
|
+
const records = []
|
|
92
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
93
|
+
const line = /** @type {string} */ (lines[index])
|
|
94
|
+
if (line.trim().length === 0) continue
|
|
95
|
+
let record
|
|
96
|
+
try {
|
|
97
|
+
record = JSON.parse(line)
|
|
98
|
+
} catch {
|
|
99
|
+
if (index === lines.length - 1) break
|
|
100
|
+
throw new Error("Activity log contains an unreadable completed record")
|
|
101
|
+
}
|
|
102
|
+
if (!isRecord(record)) throw new Error("Activity log record is not an object")
|
|
103
|
+
validateRecordSchema(record)
|
|
104
|
+
records.push(record)
|
|
105
|
+
}
|
|
106
|
+
return records
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Pure computation: derives a closed status document from an array of
|
|
111
|
+
* validated activity-log facts.
|
|
112
|
+
* @param {Record<string, unknown>[]} records
|
|
113
|
+
* @param {{now?: () => number}} [options]
|
|
114
|
+
* @returns {StatusDocument}
|
|
115
|
+
*/
|
|
116
|
+
export function computeStatus(records, options = {}) {
|
|
117
|
+
const now = options.now ?? (() => Date.now())
|
|
118
|
+
const clock = now()
|
|
119
|
+
if (!Number.isSafeInteger(clock) || clock < 0 || clock > MAX_SAFE_DATE_MS) throw new Error("Clock must be a safe non-negative Date-range integer")
|
|
120
|
+
|
|
121
|
+
let providerStarted = null
|
|
122
|
+
/** @type {number | null} */
|
|
123
|
+
let controllerPid = null
|
|
124
|
+
/** @type {string | null} */
|
|
125
|
+
let continuationHandle = null
|
|
126
|
+
/** @type {{disposition: "retrying" | "blocked", category: string, retryAfterMs?: number} | null} */
|
|
127
|
+
let health = null
|
|
128
|
+
/** @type {{state: "completed" | "failed" | "cancelled", exitCode: number} | null} */
|
|
129
|
+
let terminal = null
|
|
130
|
+
let sawTerminal = false
|
|
131
|
+
/** @type {number | null} */
|
|
132
|
+
let startedAt = null
|
|
133
|
+
/** @type {number | null} */
|
|
134
|
+
let lastActivityAt = null
|
|
135
|
+
let lastHealthIndex = -1
|
|
136
|
+
let lastActivityIndex = -1
|
|
137
|
+
|
|
138
|
+
for (let index = 0; index < records.length; index += 1) {
|
|
139
|
+
const record = /** @type {Record<string, unknown>} */ (records[index])
|
|
140
|
+
// Validate timestamps on every record, since computeStatus may be
|
|
141
|
+
// called directly (not through readActivityLog).
|
|
142
|
+
const atValue = record.at
|
|
143
|
+
if (atValue !== undefined && (typeof atValue !== "number" || !Number.isSafeInteger(atValue) || atValue < 0 || atValue > MAX_SAFE_DATE_MS)) {
|
|
144
|
+
throw new Error("Invalid timestamp")
|
|
145
|
+
}
|
|
146
|
+
const type = record.type
|
|
147
|
+
if (typeof type !== "string" || !KNOWN_TYPES.has(type)) {
|
|
148
|
+
throw new Error(`Unknown activity log record type: ${String(type)}`)
|
|
149
|
+
}
|
|
150
|
+
if (type === "provider-started") {
|
|
151
|
+
if (providerStarted !== null) throw new Error("Duplicate provider-started record")
|
|
152
|
+
providerStarted = record
|
|
153
|
+
if (typeof atValue === "number" && startedAt === null) startedAt = atValue
|
|
154
|
+
else lastActivityAt = lastActivityAt ?? null
|
|
155
|
+
} else if (type === "controller-started") {
|
|
156
|
+
const pid = record.pid
|
|
157
|
+
if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0) {
|
|
158
|
+
throw new Error("Invalid controller PID in activity log")
|
|
159
|
+
}
|
|
160
|
+
controllerPid = pid
|
|
161
|
+
if (typeof atValue === "number" && startedAt === null) startedAt = atValue
|
|
162
|
+
if (typeof atValue === "number") lastActivityAt = atValue
|
|
163
|
+
} else if (type === "session-available") {
|
|
164
|
+
const id = typeof record.sessionId === "string" ? record.sessionId : null
|
|
165
|
+
if (id !== null && !SESSION_ID_PATTERN.test(id)) throw new Error("Invalid continuation handle in activity log")
|
|
166
|
+
continuationHandle = id
|
|
167
|
+
if (typeof atValue === "number") lastActivityAt = atValue
|
|
168
|
+
} else if (type === "activity") {
|
|
169
|
+
lastActivityIndex = index
|
|
170
|
+
if (typeof atValue === "number") lastActivityAt = atValue
|
|
171
|
+
} else if (type === "health") {
|
|
172
|
+
if (sawTerminal) throw new Error("Health record appears after terminal")
|
|
173
|
+
lastHealthIndex = index
|
|
174
|
+
health = {
|
|
175
|
+
disposition: /** @type {"retrying" | "blocked"} */ (record.disposition),
|
|
176
|
+
category: /** @type {string} */ (record.category)
|
|
177
|
+
}
|
|
178
|
+
if (typeof record.retryAfterMs === "number") health.retryAfterMs = record.retryAfterMs
|
|
179
|
+
if (typeof atValue === "number") lastActivityAt = atValue
|
|
180
|
+
// Blocked stays sticky regardless; terminal always overrides later.
|
|
181
|
+
} else if (type === "terminal") {
|
|
182
|
+
if (sawTerminal) throw new Error("Duplicate terminal record")
|
|
183
|
+
sawTerminal = true
|
|
184
|
+
terminal = {
|
|
185
|
+
state: /** @type {"completed" | "failed" | "cancelled"} */ (record.state),
|
|
186
|
+
exitCode: /** @type {number} */ (record.exitCode)
|
|
187
|
+
}
|
|
188
|
+
if (typeof atValue === "number") lastActivityAt = atValue
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Activity that appears after a retrying health event clears it back to running.
|
|
193
|
+
// Blocked health stays sticky until terminal overrides it.
|
|
194
|
+
if (health !== null && health.disposition === "retrying" && lastActivityIndex > lastHealthIndex) {
|
|
195
|
+
health = null
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** @type {number | null} */
|
|
199
|
+
let effectiveStart = startedAt
|
|
200
|
+
const ps = providerStarted
|
|
201
|
+
if (effectiveStart === null && ps !== null && typeof ps.at === "number") {
|
|
202
|
+
effectiveStart = /** @type {number} */ (ps.at)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const state = terminal !== null
|
|
206
|
+
? terminal.state
|
|
207
|
+
: health !== null
|
|
208
|
+
? (health.disposition === "blocked" ? "blocked" : "retrying")
|
|
209
|
+
: ps !== null
|
|
210
|
+
? "running"
|
|
211
|
+
: "unknown"
|
|
212
|
+
|
|
213
|
+
const quietForMs = lastActivityAt === null || lastActivityAt === undefined
|
|
214
|
+
? (effectiveStart === null ? 0 : Math.max(0, clock - effectiveStart))
|
|
215
|
+
: Math.max(0, clock - lastActivityAt)
|
|
216
|
+
|
|
217
|
+
return {
|
|
218
|
+
version: 1,
|
|
219
|
+
state,
|
|
220
|
+
provider: ps !== null && typeof ps.provider === "string"
|
|
221
|
+
? ps.provider
|
|
222
|
+
: null,
|
|
223
|
+
controllerPid,
|
|
224
|
+
providerPid: ps !== null && typeof ps.pid === "number"
|
|
225
|
+
? /** @type {number} */ (ps.pid)
|
|
226
|
+
: null,
|
|
227
|
+
continuationHandle,
|
|
228
|
+
startedAt: effectiveStart === null ? null : new Date(effectiveStart).toISOString(),
|
|
229
|
+
lastActivityAt: lastActivityAt === null ? null : new Date(lastActivityAt).toISOString(),
|
|
230
|
+
quietForMs,
|
|
231
|
+
health,
|
|
232
|
+
terminal
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Validate a single activity-log record against the closed schema.
|
|
238
|
+
* @param {Record<string, unknown>} record
|
|
239
|
+
*/
|
|
240
|
+
function validateRecordSchema(record) {
|
|
241
|
+
const type = record.type
|
|
242
|
+
if (typeof type !== "string") throw new Error("Activity log record missing type")
|
|
243
|
+
const requiredFields = VALIDATED_FACT_FIELDS[/** @type {keyof typeof VALIDATED_FACT_FIELDS} */ (type)]
|
|
244
|
+
if (requiredFields === undefined) throw new Error(`Unknown activity log record type: ${type}`)
|
|
245
|
+
const optionalFields = OPTIONAL_FACT_FIELDS[/** @type {keyof typeof OPTIONAL_FACT_FIELDS} */ (type)] ?? []
|
|
246
|
+
const allowed = new Set([...requiredFields, ...optionalFields])
|
|
247
|
+
|
|
248
|
+
const actual = Object.keys(record)
|
|
249
|
+
for (const field of actual) {
|
|
250
|
+
if (!allowed.has(field)) throw new Error(`Unexpected field in ${type} record: ${field}`)
|
|
251
|
+
}
|
|
252
|
+
for (const field of requiredFields) {
|
|
253
|
+
if (!(field in record)) throw new Error(`Missing required field in ${type} record: ${field}`)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
if (type === "provider-started") {
|
|
257
|
+
if (typeof record.provider !== "string" || !PROVIDERS.has(record.provider)) throw new Error("Invalid provider")
|
|
258
|
+
if (typeof record.pid !== "number" || !Number.isSafeInteger(record.pid) || record.pid <= 0) throw new Error("Invalid pid")
|
|
259
|
+
} else if (type === "controller-started") {
|
|
260
|
+
if (typeof record.pid !== "number" || !Number.isSafeInteger(record.pid) || record.pid <= 0) throw new Error("Invalid pid")
|
|
261
|
+
} else if (type === "session-available") {
|
|
262
|
+
if (typeof record.provider !== "string" || !PROVIDERS.has(record.provider)) throw new Error("Invalid provider")
|
|
263
|
+
if (typeof record.sessionId !== "string") throw new Error("Invalid sessionId")
|
|
264
|
+
} else if (type === "activity") {
|
|
265
|
+
if (typeof record.provider !== "string" || !PROVIDERS.has(record.provider)) throw new Error("Invalid provider")
|
|
266
|
+
if (typeof record.kind !== "string" || !ACTIVITY_KINDS.has(record.kind)) throw new Error("Invalid activity kind")
|
|
267
|
+
} else if (type === "health") {
|
|
268
|
+
if (typeof record.provider !== "string" || !PROVIDERS.has(record.provider)) throw new Error("Invalid provider")
|
|
269
|
+
if (typeof record.disposition !== "string" || !HEALTH_DISPOSITIONS.has(record.disposition)) throw new Error("Invalid health disposition")
|
|
270
|
+
if (typeof record.category !== "string" || !HEALTH_CATEGORIES.has(record.category)) throw new Error("Invalid health category")
|
|
271
|
+
if (record.retryAfterMs !== undefined) {
|
|
272
|
+
if (typeof record.retryAfterMs !== "number" || !Number.isSafeInteger(record.retryAfterMs) || record.retryAfterMs <= 0) {
|
|
273
|
+
throw new Error("Invalid retryAfterMs")
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
} else if (type === "terminal") {
|
|
277
|
+
if (typeof record.provider !== "string" || !PROVIDERS.has(record.provider)) throw new Error("Invalid provider")
|
|
278
|
+
if (typeof record.state !== "string" || !TERMINAL_STATES.has(record.state)) throw new Error("Invalid terminal state")
|
|
279
|
+
if (typeof record.exitCode !== "number" || !Number.isSafeInteger(record.exitCode) || record.exitCode < 0 || record.exitCode > 255) {
|
|
280
|
+
throw new Error("Invalid exitCode")
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
if (record.at !== undefined && (typeof record.at !== "number" || !Number.isSafeInteger(record.at) || record.at < 0 || record.at > MAX_SAFE_DATE_MS)) {
|
|
284
|
+
throw new Error("Invalid timestamp")
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/** @param {unknown} value @returns {value is Record<string, unknown>} */
|
|
289
|
+
function isRecord(value) {
|
|
290
|
+
return typeof value === "object" && value !== null && !Array.isArray(value)
|
|
291
|
+
}
|
package/src/activity-log.js
CHANGED
|
@@ -4,11 +4,38 @@ import {closeSync, openSync, writeSync} from "node:fs"
|
|
|
4
4
|
|
|
5
5
|
const SESSION_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,511}$/u
|
|
6
6
|
|
|
7
|
+
const ACTIVITY_KINDS = new Set(["delta", "tool", "lifecycle"])
|
|
8
|
+
const HEALTH_DISPOSITIONS = new Set(["retrying", "blocked"])
|
|
9
|
+
const HEALTH_CATEGORIES = new Set([
|
|
10
|
+
"authentication", "permission", "rate-limit", "quota", "billing",
|
|
11
|
+
"model", "network", "protocol", "unknown"
|
|
12
|
+
])
|
|
13
|
+
const TERMINAL_STATES = new Set(["completed", "failed", "cancelled"])
|
|
14
|
+
|
|
15
|
+
const SIGNAL_CANCELLED_EXIT_CODES = new Set([130, 143])
|
|
16
|
+
|
|
17
|
+
const ACTIVITY_THROTTLE_MS = 5000
|
|
18
|
+
|
|
7
19
|
export class ActivityLog {
|
|
8
|
-
/**
|
|
9
|
-
|
|
20
|
+
/**
|
|
21
|
+
* @param {string} path
|
|
22
|
+
* @param {{now?: () => number}} [options]
|
|
23
|
+
*/
|
|
24
|
+
constructor(path, options = {}) {
|
|
10
25
|
this.fileDescriptor = openSync(path, "a", 0o600)
|
|
11
26
|
this.closed = false
|
|
27
|
+
this.now = options.now ?? (() => Date.now())
|
|
28
|
+
this.terminalWritten = false
|
|
29
|
+
/** @type {string | null} */
|
|
30
|
+
this.lastHealthFingerprint = null
|
|
31
|
+
this.lastActivityAt = -ACTIVITY_THROTTLE_MS
|
|
32
|
+
this.activityCount = 0
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** @param {number} pid */
|
|
36
|
+
recordController(pid) {
|
|
37
|
+
if (!Number.isSafeInteger(pid) || pid <= 0) throw new Error("Controller PID is unavailable")
|
|
38
|
+
this.write({type: "controller-started", pid})
|
|
12
39
|
}
|
|
13
40
|
|
|
14
41
|
/** @param {"codex" | "claude" | "kimi" | "opencode"} provider @param {number} pid */
|
|
@@ -23,15 +50,76 @@ export class ActivityLog {
|
|
|
23
50
|
this.write({type: "session-available", provider, sessionId})
|
|
24
51
|
}
|
|
25
52
|
|
|
53
|
+
/**
|
|
54
|
+
* @param {"codex" | "claude" | "kimi" | "opencode"} provider
|
|
55
|
+
* @param {"delta" | "tool" | "lifecycle"} kind
|
|
56
|
+
*/
|
|
57
|
+
recordActivity(provider, kind) {
|
|
58
|
+
if (!ACTIVITY_KINDS.has(kind)) throw new Error(`Unknown activity kind: ${kind}`)
|
|
59
|
+
const now = this.now()
|
|
60
|
+
if (now - this.lastActivityAt < ACTIVITY_THROTTLE_MS) return
|
|
61
|
+
this.lastActivityAt = now
|
|
62
|
+
this.activityCount += 1
|
|
63
|
+
this.write({type: "activity", provider, kind})
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @param {"codex" | "claude" | "kimi" | "opencode"} provider
|
|
68
|
+
* @param {{disposition: "retrying" | "blocked", category: "authentication" | "permission" | "rate-limit" | "quota" | "billing" | "model" | "network" | "protocol" | "unknown", retryAfterMs?: number}} health
|
|
69
|
+
*/
|
|
70
|
+
recordHealth(provider, health) {
|
|
71
|
+
if (!HEALTH_DISPOSITIONS.has(health.disposition)) throw new Error("Unknown health disposition")
|
|
72
|
+
if (!HEALTH_CATEGORIES.has(health.category)) throw new Error("Unknown health category")
|
|
73
|
+
if (health.retryAfterMs !== undefined) {
|
|
74
|
+
if (!Number.isSafeInteger(health.retryAfterMs) || health.retryAfterMs <= 0) throw new Error("retryAfterMs must be a positive safe integer")
|
|
75
|
+
if (health.disposition !== "retrying") throw new Error("retryAfterMs is only valid for retrying disposition")
|
|
76
|
+
}
|
|
77
|
+
const fingerprint = `${health.disposition}\0${health.category}\0${health.retryAfterMs ?? ""}`
|
|
78
|
+
if (fingerprint === this.lastHealthFingerprint) return
|
|
79
|
+
this.lastHealthFingerprint = fingerprint
|
|
80
|
+
/** @type {{type: string, provider: string, disposition: string, category: string, retryAfterMs?: number}} */
|
|
81
|
+
const fact = {type: "health", provider, disposition: health.disposition, category: health.category}
|
|
82
|
+
if (health.retryAfterMs !== undefined) fact.retryAfterMs = health.retryAfterMs
|
|
83
|
+
this.write(fact)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* @param {"codex" | "claude" | "kimi" | "opencode"} provider
|
|
88
|
+
* @param {"completed" | "failed" | "cancelled"} state
|
|
89
|
+
* @param {number} exitCode
|
|
90
|
+
*/
|
|
91
|
+
recordTerminal(provider, state, exitCode) {
|
|
92
|
+
if (this.terminalWritten) return
|
|
93
|
+
if (!TERMINAL_STATES.has(state)) throw new Error("Unknown terminal state")
|
|
94
|
+
if (!Number.isSafeInteger(exitCode) || exitCode < 0 || exitCode > 255) throw new Error("exitCode must be an integer between 0 and 255")
|
|
95
|
+
this.terminalWritten = true
|
|
96
|
+
this.write({type: "terminal", provider, state, exitCode})
|
|
97
|
+
}
|
|
98
|
+
|
|
26
99
|
close() {
|
|
27
100
|
if (this.closed) return
|
|
28
101
|
this.closed = true
|
|
29
102
|
closeSync(this.fileDescriptor)
|
|
30
103
|
}
|
|
31
104
|
|
|
32
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* @param {Record<string, unknown>} fact
|
|
107
|
+
*/
|
|
33
108
|
write(fact) {
|
|
34
109
|
if (this.closed) throw new Error("Activity log is closed")
|
|
110
|
+
fact.at = this.now()
|
|
35
111
|
writeSync(this.fileDescriptor, `${JSON.stringify(fact)}\n`)
|
|
36
112
|
}
|
|
37
113
|
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Classify a terminal exit code. Known signal-derived codes (130 = SIGINT,
|
|
117
|
+
* 143 = SIGTERM) map to cancelled; all others map to completed (0) or failed
|
|
118
|
+
* (nonzero). This is the only place that picks the terminal state constant.
|
|
119
|
+
* @param {number} exitCode
|
|
120
|
+
* @returns {"completed" | "failed" | "cancelled"}
|
|
121
|
+
*/
|
|
122
|
+
export function terminalState(exitCode) {
|
|
123
|
+
if (SIGNAL_CANCELLED_EXIT_CODES.has(exitCode)) return "cancelled"
|
|
124
|
+
return exitCode === 0 ? "completed" : "failed"
|
|
125
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -8,7 +8,8 @@ import {createFetchTransport} from "./notifiers/fetch-transport.js"
|
|
|
8
8
|
import {createTelegramSender, parseTelegramTarget} from "./notifiers/telegram.js"
|
|
9
9
|
import {createProvider, PROVIDERS} from "./providers/index.js"
|
|
10
10
|
import {runWorker} from "./run-worker.js"
|
|
11
|
-
import {ActivityLog} from "./activity-log.js"
|
|
11
|
+
import {ActivityLog, terminalState} from "./activity-log.js"
|
|
12
|
+
import {computeStatus, parseStatusArguments, readActivityLog} from "./activity-log-status.js"
|
|
12
13
|
import {DelegatedResultAdmission, validateContinuationHandle} from "./delegated-result-admission.js"
|
|
13
14
|
import {buildProviderEnvironment, collectEvidenceRedactions, parseTelegramRequestTimeoutMs, resolveFileBackedSettings} from "./telegram-ingress/config.js"
|
|
14
15
|
import {WorkerControl} from "./worker-control.js"
|
|
@@ -18,6 +19,16 @@ import {isolatedRuntimeClientFromEnvironment} from "./isolated-runtime-client.js
|
|
|
18
19
|
import {validateRelayWriteProviderArguments} from "./relay-write.js"
|
|
19
20
|
import {abortable} from "./absolute-deadline.js"
|
|
20
21
|
import {NormalizedOutput} from "./normalized-output.js"
|
|
22
|
+
import {
|
|
23
|
+
CAPACITY_PROVIDERS,
|
|
24
|
+
DEFAULT_CAPACITY_TIMEOUT_MS,
|
|
25
|
+
DEFAULT_LONG_RESERVE_PERCENT,
|
|
26
|
+
DEFAULT_SHORT_RESERVE_PERCENT,
|
|
27
|
+
createCapacityProbe,
|
|
28
|
+
selectProvider
|
|
29
|
+
} from "./provider-capacity.js"
|
|
30
|
+
import {probeCodexCapacity} from "./provider-capacity-codex.js"
|
|
31
|
+
import {probeKimiCapacity} from "./provider-capacity-kimi.js"
|
|
21
32
|
|
|
22
33
|
const HELP = `Usage: threadwire run --provider <codex|claude|kimi|opencode> --target telegram:<chat-id> | telegram:<chat-id>:<thread-id>
|
|
23
34
|
[--process-number <positive-integer>] [--cwd <directory>]
|
|
@@ -28,10 +39,17 @@ const HELP = `Usage: threadwire run --provider <codex|claude|kimi|opencode> --ta
|
|
|
28
39
|
[--prompt <text> | --prompt-file <path> | stdin]
|
|
29
40
|
[-- <provider arguments...>]
|
|
30
41
|
threadwire evidence read --handle <opaque-handle>
|
|
31
|
-
(--bytes <offset>:<limit> | --lines <start>:<limit> | --query <literal> --context-bytes <limit>)
|
|
42
|
+
(--bytes <offset>:<limit> | --lines <start>:<limit> | --query <literal> --context-bytes <limit>)
|
|
43
|
+
threadwire capacity [--provider <codex|kimi>]...
|
|
44
|
+
[--short-reserve-percent <0-100>] [--long-reserve-percent <0-100>]
|
|
45
|
+
[--timeout-ms <positive-integer>]
|
|
46
|
+
threadwire status --activity-log <absolute-path>
|
|
47
|
+
(Emits one closed versioned JSON status document from the activity log.)`
|
|
32
48
|
|
|
33
49
|
/** @typedef {{provider: string, target: string, cwd: string, toolMessages: boolean, relayWrite: boolean, prompt?: string, promptFile?: string, processNumber?: number, maxOutputLength?: number, resumeSession?: string, transcript?: string, activityLog?: string, providerArguments: string[]}} ParsedArguments */
|
|
34
50
|
/** @typedef {{evidenceRead: true, request: unknown}} EvidenceParsedArguments */
|
|
51
|
+
/** @typedef {{capacity: true, providers: import("./provider-capacity.js").CapacityProvider[], shortReservePercent: number, longReservePercent: number, timeoutMs: number}} CapacityParsedArguments */
|
|
52
|
+
/** @typedef {{probe: (provider: import("./provider-capacity.js").CapacityProvider, timeoutMs?: number) => Promise<import("./provider-capacity.js").CapacityCandidate>}} CapacityProbeDependency */
|
|
35
53
|
/**
|
|
36
54
|
* @typedef {{
|
|
37
55
|
* env?: NodeJS.ProcessEnv,
|
|
@@ -43,7 +61,9 @@ const HELP = `Usage: threadwire run --provider <codex|claude|kimi|opencode> --ta
|
|
|
43
61
|
* evidenceStore?: EvidenceStore,
|
|
44
62
|
* evidenceOwnerScope?: {destinationId: string, runId: string},
|
|
45
63
|
* workerControlOptions?: Pick<ConstructorParameters<typeof WorkerControl>[0], "queueOptions">,
|
|
46
|
-
* isolatedRuntimeClient?: Pick<import("./isolated-runtime-client.js").IsolatedRuntimeClient, "preflight" | "
|
|
64
|
+
* isolatedRuntimeClient?: Pick<import("./isolated-runtime-client.js").IsolatedRuntimeClient, "preflight" | "startRun">,
|
|
65
|
+
* capacityProbe?: CapacityProbeDependency,
|
|
66
|
+
* now?: () => number
|
|
47
67
|
* }} MainDependencies
|
|
48
68
|
*/
|
|
49
69
|
|
|
@@ -135,6 +155,71 @@ function parseEvidenceArguments(arguments_) {
|
|
|
135
155
|
return {evidenceRead: true, request}
|
|
136
156
|
}
|
|
137
157
|
|
|
158
|
+
/** @param {string[]} arguments_ @returns {CapacityParsedArguments | {help: true}} */
|
|
159
|
+
function parseCapacityArguments(arguments_) {
|
|
160
|
+
if (arguments_.includes("--help") || arguments_.includes("-h")) return {help: true}
|
|
161
|
+
/** @type {import("./provider-capacity.js").CapacityProvider[]} */
|
|
162
|
+
const providers = []
|
|
163
|
+
let shortReservePercent = DEFAULT_SHORT_RESERVE_PERCENT
|
|
164
|
+
let longReservePercent = DEFAULT_LONG_RESERVE_PERCENT
|
|
165
|
+
let timeoutMs = DEFAULT_CAPACITY_TIMEOUT_MS
|
|
166
|
+
/** @type {Set<string>} */
|
|
167
|
+
const seen = new Set()
|
|
168
|
+
for (let index = 1; index < arguments_.length; index += 2) {
|
|
169
|
+
const option = arguments_[index]
|
|
170
|
+
const value = arguments_[index + 1]
|
|
171
|
+
if (!option || value === undefined) throw new Error(HELP)
|
|
172
|
+
if (option !== "--provider") {
|
|
173
|
+
if (seen.has(option)) throw new Error(`Duplicate capacity option: ${option}`)
|
|
174
|
+
seen.add(option)
|
|
175
|
+
}
|
|
176
|
+
if (option === "--provider") {
|
|
177
|
+
if (!CAPACITY_PROVIDERS.includes(/** @type {import("./provider-capacity.js").CapacityProvider} */ (value))) {
|
|
178
|
+
throw new Error(`--provider must be one of: ${CAPACITY_PROVIDERS.join(", ")}`)
|
|
179
|
+
}
|
|
180
|
+
const provider = /** @type {import("./provider-capacity.js").CapacityProvider} */ (value)
|
|
181
|
+
if (providers.includes(provider)) throw new Error(`Duplicate capacity provider: ${value}`)
|
|
182
|
+
providers.push(provider)
|
|
183
|
+
} else if (option === "--short-reserve-percent") shortReservePercent = reservePercent(value, option)
|
|
184
|
+
else if (option === "--long-reserve-percent") longReservePercent = reservePercent(value, option)
|
|
185
|
+
else if (option === "--timeout-ms") timeoutMs = positiveInteger(value, option)
|
|
186
|
+
else throw new Error(HELP)
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
capacity: true,
|
|
190
|
+
providers: providers.length === 0 ? [...CAPACITY_PROVIDERS] : providers,
|
|
191
|
+
shortReservePercent,
|
|
192
|
+
longReservePercent,
|
|
193
|
+
timeoutMs
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* The real probe never rejects (it converts failures into candidates); this
|
|
199
|
+
* guard keeps any probe implementation from leaking an unexpected error into
|
|
200
|
+
* output by degrading to a fixed, credential-free candidate.
|
|
201
|
+
* @param {CapacityProbeDependency} probe
|
|
202
|
+
* @param {import("./provider-capacity.js").CapacityProvider} provider
|
|
203
|
+
* @param {number} timeoutMs
|
|
204
|
+
* @returns {Promise<import("./provider-capacity.js").CapacityCandidate>}
|
|
205
|
+
*/
|
|
206
|
+
async function probeCapacityCandidate(probe, provider, timeoutMs) {
|
|
207
|
+
try {
|
|
208
|
+
return await probe.probe(provider, timeoutMs)
|
|
209
|
+
} catch {
|
|
210
|
+
return {provider, status: "unavailable", error: "Capacity probe failed"}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** @param {string} value @param {string} option */
|
|
215
|
+
function reservePercent(value, option) {
|
|
216
|
+
const number = Number(value)
|
|
217
|
+
if (!/^\d+$/u.test(value) || !Number.isSafeInteger(number) || number > 100) {
|
|
218
|
+
throw new Error(`${option} must be an integer between 0 and 100`)
|
|
219
|
+
}
|
|
220
|
+
return number
|
|
221
|
+
}
|
|
222
|
+
|
|
138
223
|
/** @param {string} value @param {boolean} zeroStart */
|
|
139
224
|
function rangePair(value, zeroStart) {
|
|
140
225
|
const match = /^(\d+):(\d+)$/u.exec(value)
|
|
@@ -162,11 +247,51 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
162
247
|
/** @type {DelegatedResultAdmission | undefined} */
|
|
163
248
|
let runAdmission
|
|
164
249
|
try {
|
|
165
|
-
|
|
250
|
+
if (arguments_[0] === "status") {
|
|
251
|
+
const statusParsed = parseStatusArguments(arguments_)
|
|
252
|
+
if (validateOnly) return 0
|
|
253
|
+
const records = await readActivityLog(statusParsed.activityLog)
|
|
254
|
+
const status = computeStatus(records, {
|
|
255
|
+
...(dependencies.now === undefined ? {} : {now: dependencies.now})
|
|
256
|
+
})
|
|
257
|
+
output.write(`${JSON.stringify(status)}\n`)
|
|
258
|
+
return 0
|
|
259
|
+
}
|
|
260
|
+
const parsed = arguments_[0] === "evidence"
|
|
261
|
+
? parseEvidenceArguments(arguments_)
|
|
262
|
+
: arguments_[0] === "capacity" ? parseCapacityArguments(arguments_) : parseArguments(arguments_)
|
|
166
263
|
if ("help" in parsed) {
|
|
167
264
|
if (!validateOnly) output.write(`${HELP}\n`)
|
|
168
265
|
return 0
|
|
169
266
|
}
|
|
267
|
+
if ("capacity" in parsed) {
|
|
268
|
+
if (validateOnly) return 0
|
|
269
|
+
const probe = dependencies.capacityProbe ?? createCapacityProbe({
|
|
270
|
+
env: sourceEnvironment,
|
|
271
|
+
probes: {
|
|
272
|
+
codex: ({env, timeoutMs}) => probeCodexCapacity({env, timeoutMs}),
|
|
273
|
+
kimi: ({env, timeoutMs}) => probeKimiCapacity({env, timeoutMs})
|
|
274
|
+
}
|
|
275
|
+
})
|
|
276
|
+
/** @type {import("./provider-capacity.js").CapacityCandidate[]} */
|
|
277
|
+
const candidates = []
|
|
278
|
+
for (const provider of parsed.providers) {
|
|
279
|
+
candidates.push(await probeCapacityCandidate(probe, provider, parsed.timeoutMs))
|
|
280
|
+
}
|
|
281
|
+
const {selection, rejected} = selectProvider(candidates, {
|
|
282
|
+
shortReservePercent: parsed.shortReservePercent,
|
|
283
|
+
longReservePercent: parsed.longReservePercent
|
|
284
|
+
})
|
|
285
|
+
output.write(`${JSON.stringify({
|
|
286
|
+
version: 1,
|
|
287
|
+
generatedAt: new Date((dependencies.now ?? Date.now)()).toISOString(),
|
|
288
|
+
reserves: {shortPercent: parsed.shortReservePercent, longPercent: parsed.longReservePercent},
|
|
289
|
+
candidates,
|
|
290
|
+
rejected,
|
|
291
|
+
selection
|
|
292
|
+
})}\n`)
|
|
293
|
+
return selection === null ? 2 : 0
|
|
294
|
+
}
|
|
170
295
|
if ("evidenceRead" in parsed) {
|
|
171
296
|
const root = evidenceRoot(sourceEnvironment.THREADWIRE_EVIDENCE_ROOT)
|
|
172
297
|
if (!root) throw new Error("THREADWIRE_EVIDENCE_ROOT is required")
|
|
@@ -261,6 +386,7 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
261
386
|
metrics
|
|
262
387
|
})
|
|
263
388
|
activity = parsed.activityLog === undefined ? undefined : new ActivityLog(parsed.activityLog)
|
|
389
|
+
activity?.recordController(process.pid)
|
|
264
390
|
/** @type {import("./run-worker.js").RunWorkerOptions} */
|
|
265
391
|
const workerOptions = {
|
|
266
392
|
executable: provider.executable,
|
|
@@ -269,11 +395,16 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
269
395
|
environment: providerEnvironment,
|
|
270
396
|
provider: provider.name,
|
|
271
397
|
parse: provider.parse,
|
|
398
|
+
...(provider.completion === undefined ? {} : {completion: provider.completion}),
|
|
399
|
+
...(launchDeadline === undefined ? {} : {signal: launchDeadline.signal}),
|
|
272
400
|
onEvent: (event) => {
|
|
273
401
|
validateNormalizedWorkerEvent(event)
|
|
274
402
|
if (event.type !== "text-delta") {
|
|
275
403
|
metrics.recordRejected(`${event.type}_progress`, Buffer.byteLength(JSON.stringify(event), "utf8"))
|
|
276
404
|
}
|
|
405
|
+
if (event.type === "text-delta") activity?.recordActivity(provider.name, "delta")
|
|
406
|
+
else if (event.type === "tool") activity?.recordActivity(provider.name, "tool")
|
|
407
|
+
else if (event.type === "lifecycle") activity?.recordActivity(provider.name, "lifecycle")
|
|
277
408
|
acceptAdmissionEvent(admission, event)
|
|
278
409
|
return control.accept(event)
|
|
279
410
|
},
|
|
@@ -287,6 +418,8 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
287
418
|
admission.setContinuationHandle(id)
|
|
288
419
|
activity?.recordSession(provider.name, id)
|
|
289
420
|
}
|
|
421
|
+
const health = provider.health(record)
|
|
422
|
+
if (health !== undefined) activity?.recordHealth(provider.name, health)
|
|
290
423
|
},
|
|
291
424
|
onStdoutChunk: (chunk) => {
|
|
292
425
|
metrics.recordRawChildChunk("provider_stdout", chunk.length)
|
|
@@ -301,8 +434,9 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
301
434
|
return evidence.append("provider-stderr", chunk).then(() => { evidencePayloadBytes += chunk.length })
|
|
302
435
|
}
|
|
303
436
|
}
|
|
304
|
-
|
|
305
|
-
|
|
437
|
+
let exitCode
|
|
438
|
+
if (usesIsolatedRuntime) {
|
|
439
|
+
const isolatedRun = /** @type {NonNullable<typeof isolatedRuntimeClient>} */ (isolatedRuntimeClient).startRun({
|
|
306
440
|
preflightId: /** @type {{preflightId: string}} */ (isolatedPreflight).preflightId,
|
|
307
441
|
prompt,
|
|
308
442
|
providerArguments: parsed.providerArguments,
|
|
@@ -313,7 +447,15 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
313
447
|
...(parsed.provider === "kimi" || workerOptions.onStderrChunk === undefined ? {} : {onStderrChunk: workerOptions.onStderrChunk}),
|
|
314
448
|
...(launchDeadline === undefined ? {} : {deadline: launchDeadline})
|
|
315
449
|
})
|
|
316
|
-
|
|
450
|
+
try {
|
|
451
|
+
exitCode = await abortable(isolatedRun.completion, launchDeadline?.signal)
|
|
452
|
+
} catch (error) {
|
|
453
|
+
await isolatedRun.cancel(error instanceof Error ? error : new Error("Isolated runtime run failed"))
|
|
454
|
+
throw error
|
|
455
|
+
}
|
|
456
|
+
} else {
|
|
457
|
+
exitCode = await (dependencies.workerRunner ?? runWorker)(workerOptions)
|
|
458
|
+
}
|
|
317
459
|
await boundedLaunch(control.close(), launchDeadline)
|
|
318
460
|
terminalExitCode = exitCode
|
|
319
461
|
} finally {
|
|
@@ -336,7 +478,10 @@ export async function main(arguments_, dependencies = {}) {
|
|
|
336
478
|
}
|
|
337
479
|
admission.complete({state: terminalExitCode === 0 ? "completed" : "failed", exitCode: terminalExitCode})
|
|
338
480
|
errorOutput.write(`threadwire-context-metrics ${JSON.stringify(metrics.conciseDiagnostic())}\n`)
|
|
339
|
-
activity
|
|
481
|
+
if (activity !== undefined) {
|
|
482
|
+
activity.recordTerminal(/** @type {"codex" | "claude" | "kimi" | "opencode"} */ (parsed.provider), terminalState(terminalExitCode), terminalExitCode)
|
|
483
|
+
activity.close()
|
|
484
|
+
}
|
|
340
485
|
await boundedLaunch(ownedEvidenceStore?.close(), launchDeadline).catch(() => {})
|
|
341
486
|
launchDeadline?.close()
|
|
342
487
|
}
|