pi-claude-supervisor 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.cn.md +21 -2
- package/README.md +30 -3
- package/docs/architecture.md +30 -0
- package/package.json +3 -3
- package/src/config.ts +78 -0
- package/src/cwd-lease.ts +5 -0
- package/src/decision-worker.ts +62 -3
- package/src/index.ts +103 -17
- package/src/policy.ts +61 -2
- package/src/supervisor.ts +271 -30
- package/src/worker/tmux-adapter.ts +50 -5
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,30 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented here.
|
|
4
4
|
|
|
5
|
+
## [0.8.0](https://github.com/btnalit/pi-claude-supervisor/compare/v0.7.3...v0.8.0) (2026-09-19)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
### Features
|
|
9
|
+
|
|
10
|
+
* **recover:** extend an expired task's budget from now, and discard records nobody will recover ([0bbf07f](https://github.com/btnalit/pi-claude-supervisor/commit/0bbf07f8a44eb5f9cfb3542a8b01115aa2c3703a))
|
|
11
|
+
* **supervisor:** turn the wall-clock deadline into a warned, graceful close-out ([4b5f92a](https://github.com/btnalit/pi-claude-supervisor/commit/4b5f92a61c99d9f1448d3c994975826897326ced))
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### Bug Fixes
|
|
15
|
+
|
|
16
|
+
* **recover:** point an adopted task's discard at re-adoption, guard the owed re-ask ([0fef7d8](https://github.com/btnalit/pi-claude-supervisor/commit/0fef7d850b3934543feb664b954f5d4cb92c63e0))
|
|
17
|
+
* **supervisor:** address the review of the deadline close-out ([721b0b8](https://github.com/btnalit/pi-claude-supervisor/commit/721b0b8d2c62f81811ad67bf42c396a1c0550903))
|
|
18
|
+
|
|
19
|
+
## [0.7.3](https://github.com/btnalit/pi-claude-supervisor/compare/v0.7.2...v0.7.3) (2026-09-17)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
### Bug Fixes
|
|
23
|
+
|
|
24
|
+
* **policy:** a name bound to literal text is not an unseen argument ([b93bbf6](https://github.com/btnalit/pi-claude-supervisor/commit/b93bbf6ae019fb7137588f113c0e1113ee942be7))
|
|
25
|
+
* **policy:** a tilde expands only at the start of a word ([9671b23](https://github.com/btnalit/pi-claude-supervisor/commit/9671b23cd6f112f28decc2e906c029c11f5f7d1c))
|
|
26
|
+
* **tmux:** a subagent hand-back is Claude's own prompt, not a human's ([90d40ba](https://github.com/btnalit/pi-claude-supervisor/commit/90d40baf357c5390f1699502235459e2d4230d55))
|
|
27
|
+
* **tmux:** remove a dead Worker's tmux socket and sweep leftovers at load ([db83d40](https://github.com/btnalit/pi-claude-supervisor/commit/db83d4016dc210739d386d038df220cd7c1c4479))
|
|
28
|
+
|
|
5
29
|
## [0.7.2](https://github.com/btnalit/pi-claude-supervisor/compare/v0.7.1...v0.7.2) (2026-09-17)
|
|
6
30
|
|
|
7
31
|
|
package/README.cn.md
CHANGED
|
@@ -194,8 +194,16 @@ stream-json` 的方式运行 Claude,完全没有终端界面;一旦设置
|
|
|
194
194
|
`PI_CLAUDE_SUPERVISOR_TRUSTED_CLAUDE` 显式固定)。
|
|
195
195
|
- Linux 上,cgroup v2 边界会清理每一个后代进程,包括 `setsid()` 后代;
|
|
196
196
|
`required` 模式会 fail closed,而不是回退到其他清理方式。
|
|
197
|
-
- 默认 4
|
|
198
|
-
|
|
197
|
+
- 每个任务有一个总时限(默认 4 小时,可用 `--deadline 8h` 按任务指定或用
|
|
198
|
+
`DEADLINE_MS` 全局设置)。到期不会直接杀掉工作:到期前
|
|
199
|
+
`DEADLINE_WARNING_MS`(默认 15 分钟)会提醒 Decision Worker 引导 Worker 收尾;
|
|
200
|
+
到期后进入 `DEADLINE_GRACE_MS`(默认 30 分钟)的收尾窗口,空闲的 Worker 会被直接
|
|
201
|
+
验收和 review 而不是被停止,`wait` 决策不再生效,修复轮会告诉 Worker 还剩多少
|
|
202
|
+
时间。只有收尾窗口也耗尽,Worker 才会被硬停(`worker_watchdog_timeout`);对
|
|
203
|
+
接管的交互式会话来说这个硬停只是 release:Claude 继续运行,但不再受监督。
|
|
204
|
+
收尾窗口只属于自动模式任务;手动任务仍在到期时停止,`DEADLINE_GRACE_MS=0`
|
|
205
|
+
让自动任务也恢复这一行为。20 分钟无输出 watchdog(`NO_OUTPUT_TIMEOUT_MS`)
|
|
206
|
+
随时会停止沉默的 Worker。
|
|
199
207
|
- 验收命令、证据收集和 Reviewer 共用一个 abort signal,因此 stop 或 shutdown
|
|
200
208
|
不必等待完整的命令或模型超时。
|
|
201
209
|
- 每个任务只持有一个 cwd 租约;并发任务需要各自独立的 worktree。
|
|
@@ -267,6 +275,10 @@ stream-json` 的方式运行 Claude,完全没有终端界面;一旦设置
|
|
|
267
275
|
| `EVIDENCE_MAX_BYTES` | `1048576`(1 MiB) | 每个任务收集的最大仓库证据字节数 |
|
|
268
276
|
| `EVIDENCE_MAX_UNTRACKED_FILES` | `512` | 每个任务作为证据收集的最大未跟踪文件数 |
|
|
269
277
|
| `REVIEW_TIMEOUT_MS` | `600000`(10 分钟) | 每轮独立 Reviewer 的总预算,含一次针对 provider 错误的重试 |
|
|
278
|
+
| `DEADLINE_MS` | `4h` | 每个任务的累计总时限(`8h`、`90m`、`2h30m` 或毫秒;5 分钟到 7 天);`0`(或 `0m`)关闭;`--deadline` 可按任务覆盖 |
|
|
279
|
+
| `DEADLINE_GRACE_MS` | `30m` | 自动任务到期后的收尾窗口:空闲的 Worker 会被验收而不是停止;`0` 恢复到期立即停止 |
|
|
280
|
+
| `DEADLINE_WARNING_MS` | `15m` | 到期前多久提醒并重新询问 Decision Worker;`0` 关闭提醒 |
|
|
281
|
+
| `NO_OUTPUT_TIMEOUT_MS` | `20m` | Worker 多久没有输出就停止;`0` 关闭该检查 |
|
|
270
282
|
| `EVENT_LOG_MAX_BYTES` | `67108864`(64 MiB) | `events.jsonl` 达到该大小后滚动,保留 5 份滚动文件 |
|
|
271
283
|
|
|
272
284
|
## 恢复、租约与状态
|
|
@@ -280,6 +292,13 @@ Worker,它不会静默恢复或重复执行任务。只有在租约证明旧 Wor
|
|
|
280
292
|
的 `TRANSPORT`/`TMUX_MODE` 配置来判断,因此在启动任务和恢复任务之间请不要
|
|
281
293
|
改变这两个配置。
|
|
282
294
|
|
|
295
|
+
因总时限到期而停止的任务会以 `deadline=expired … ago` 列出。普通 `recover`
|
|
296
|
+
会拒绝它;`recover --takeover --extend <duration> <task-id>` 从现在起再给这么
|
|
297
|
+
多预算(恢复后的 Supervisor 会把新时限持久化),`--extend 0` 则立即进入收尾:
|
|
298
|
+
新 Worker 的第一个 watchdog tick 就会对仓库现状做验收和 review,修复轮会告诉
|
|
299
|
+
它还剩多少时间。确定不再恢复的记录用 `/supervise discard <task-id>` 丢弃
|
|
300
|
+
(会话文件保留到保留期清理为止)。
|
|
301
|
+
|
|
283
302
|
每个任务在 `CWD_LEASE_DIR` 下持有一个 cwd 租约;并发任务需要各自独立的
|
|
284
303
|
worktree。无法读取的租约记录(损坏的 JSON、异常的结构)会被隔离到 quarantine
|
|
285
304
|
目录,而不会阻塞其他查找;`/supervise sessions` 会列出当前被隔离的记录,方便
|
package/README.md
CHANGED
|
@@ -69,8 +69,11 @@ Then, inside any Pi session:
|
|
|
69
69
|
```text
|
|
70
70
|
/supervise start implement the requested change
|
|
71
71
|
/supervise adopt-tmux my-tmux-session implement the requested change
|
|
72
|
+
/supervise start --deadline 8h a large multi-worktree change
|
|
72
73
|
```
|
|
73
74
|
|
|
75
|
+
Both accept `--spec <file>` and `--deadline <duration>` (`8h`, `90m`, `0` for
|
|
76
|
+
no deadline) ahead of the task text.
|
|
74
77
|
Watch a task with `/supervise status <task-id>` or `/supervise sessions`; for
|
|
75
78
|
a tmux transport, attach directly with the `tmux -S <socket> attach -t
|
|
76
79
|
<session>` command each of these prints. `/supervise stop <task-id>` closes
|
|
@@ -232,9 +235,20 @@ Supervisor being able to see it, or when you don't need to attach.
|
|
|
232
235
|
- On Linux, a cgroup v2 boundary cleans up every descendant, including
|
|
233
236
|
`setsid()` descendants; `required` mode fails closed instead of falling
|
|
234
237
|
back.
|
|
235
|
-
- A
|
|
236
|
-
|
|
237
|
-
(`
|
|
238
|
+
- A wall-clock deadline (4 hours by default, `--deadline 8h` per task or
|
|
239
|
+
`DEADLINE_MS`) bounds a task. Reaching it does not kill the work: the
|
|
240
|
+
Decision Worker is warned ahead of time (`DEADLINE_WARNING_MS`, 15 min) and
|
|
241
|
+
told to steer the Worker to a wrap-up, and once the deadline passes a
|
|
242
|
+
close-out window opens (`DEADLINE_GRACE_MS`, 30 min) in which an idle Worker
|
|
243
|
+
is verified and reviewed instead of stopped, a `wait` decision is no longer
|
|
244
|
+
honored, and a repair round tells the Worker how long it has left. Only when
|
|
245
|
+
the close-out window has also elapsed is the Worker stopped outright
|
|
246
|
+
(`worker_watchdog_timeout`), and on an adopted interactive session that stop
|
|
247
|
+
is a release: Claude keeps running, unsupervised. The close-out belongs to
|
|
248
|
+
automatic tasks; a manual task is stopped at the deadline as before, and
|
|
249
|
+
`DEADLINE_GRACE_MS=0` restores that for automatic ones too. A 20-minute
|
|
250
|
+
no-output watchdog (`NO_OUTPUT_TIMEOUT_MS`) still stops a silent Worker at
|
|
251
|
+
any time.
|
|
238
252
|
- Acceptance checks, evidence collection, and the Reviewer share an abort
|
|
239
253
|
signal, so a stop or shutdown does not wait for a full command or model
|
|
240
254
|
timeout.
|
|
@@ -309,6 +323,10 @@ Environment variables (or `~/.config/pi-claude-supervisor/env`), all prefixed
|
|
|
309
323
|
| `EVIDENCE_MAX_BYTES` | `1048576` (1 MiB) | Maximum repository evidence bytes collected per task |
|
|
310
324
|
| `EVIDENCE_MAX_UNTRACKED_FILES` | `512` | Maximum untracked files collected as evidence per task |
|
|
311
325
|
| `REVIEW_TIMEOUT_MS` | `600000` (10 min) | Total independent Reviewer budget per round, including one retry on a provider error |
|
|
326
|
+
| `DEADLINE_MS` | `4h` | Cumulative wall-clock budget per task (`8h`, `90m`, `2h30m` or ms; 5m–7d); `0` (or `0m`) disables it; `--deadline` overrides it per task |
|
|
327
|
+
| `DEADLINE_GRACE_MS` | `30m` | Close-out window after the deadline for automatic tasks: an idle Worker is verified instead of stopped; `0` restores the immediate stop |
|
|
328
|
+
| `DEADLINE_WARNING_MS` | `15m` | How long before the deadline the Decision Worker is warned and re-asked; `0` disables the warning |
|
|
329
|
+
| `NO_OUTPUT_TIMEOUT_MS` | `20m` | Stop a Worker that has produced no output for this long; `0` disables the check |
|
|
312
330
|
| `EVENT_LOG_MAX_BYTES` | `67108864` (64 MiB) | Rotates `events.jsonl` at this size; 5 rotated files are kept |
|
|
313
331
|
|
|
314
332
|
## Recovery, leases and state
|
|
@@ -325,6 +343,15 @@ interactive — it derives that from the current `TRANSPORT`/`TMUX_MODE`
|
|
|
325
343
|
configuration at recovery time, so do not change either between starting a
|
|
326
344
|
task and recovering it.
|
|
327
345
|
|
|
346
|
+
A task that stopped at its wall-clock deadline is listed with `deadline=expired
|
|
347
|
+
… ago`. Plain `recover` refuses it; `recover --takeover --extend <duration>
|
|
348
|
+
<task-id>` grants that much budget from now (the recovered Supervisor persists
|
|
349
|
+
the new deadline), and `--extend 0` opens the close-out at once, so the fresh
|
|
350
|
+
Worker's first watchdog tick verifies and reviews the repository as it stands
|
|
351
|
+
and any repair round tells it how long it has. A record nobody will recover is
|
|
352
|
+
dropped with `/supervise discard <task-id>` (its session file is kept until
|
|
353
|
+
retention pruning).
|
|
354
|
+
|
|
328
355
|
Every task holds one cwd lease under `CWD_LEASE_DIR`; concurrent tasks need
|
|
329
356
|
separate worktrees. A lease record that cannot be read (corrupt JSON,
|
|
330
357
|
unexpected shape) is quarantined instead of blocking other lookups, and
|
package/docs/architecture.md
CHANGED
|
@@ -401,6 +401,36 @@ not. JSONL sends are rejected while a request is active, and a valid terminal
|
|
|
401
401
|
result moves the session to `waiting`; only then may the next turn be sent. A paused
|
|
402
402
|
Worker does not consume its no-output budget; resume establishes a fresh no-output
|
|
403
403
|
baseline while the cumulative wall-clock deadline remains active.
|
|
404
|
+
|
|
405
|
+
The wall-clock deadline is a budget, not a kill switch. The watchdog drives it
|
|
406
|
+
through three phases, each recorded once per task: `worker_deadline_approaching`
|
|
407
|
+
(`deadlineWarningMs` before the deadline) refreshes the Decision Worker's clock
|
|
408
|
+
(`deadline.remainingMs` in its context and `deadlineRemainingMinutes` in every
|
|
409
|
+
event prompt) and, if the Worker idles under a `wait`, replays the last completed
|
|
410
|
+
turn so the Decision Worker can steer the Worker to integrate, commit and stop;
|
|
411
|
+
`worker_deadline_reached` opens the close-out window (`deadlineGraceMs`), during
|
|
412
|
+
which an idle automatic Worker is verified at once (`deadline_close_out`), a
|
|
413
|
+
`wait` decision on an idle Worker is applied as verify (`decision_overridden`), a
|
|
414
|
+
Worker mid-turn keeps its turn and its completed turn is decided with
|
|
415
|
+
`closeOut: true`, and a repair round tells the Worker how much of the window
|
|
416
|
+
remains; `worker_watchdog_timeout` (`worker deadline exceeded`) stops the Worker
|
|
417
|
+
only once the close-out window has also elapsed. The deadline never verifies
|
|
418
|
+
underneath a decision that is still in flight for the last turn, and a repeated
|
|
419
|
+
`wait` for the same turn re-arms the wait timer rather than being deduplicated.
|
|
420
|
+
A zero grace window restores the immediate stop at the deadline, and a manual
|
|
421
|
+
task (no Decision Worker to drive a close-out) always behaves that way; a repair
|
|
422
|
+
round is refused once less than a minute of the window is left, so the findings
|
|
423
|
+
stay on a blocked candidate instead of being cut short by the stop. For an adopted
|
|
424
|
+
interactive session the outright stop is a release (the adapter never kills a
|
|
425
|
+
session it does not own), so the Claude process keeps running unsupervised; the
|
|
426
|
+
close-out exists so that a task which merely ran long still ends with a verified
|
|
427
|
+
candidate instead of a silent hand-back. A record left behind by the outright
|
|
428
|
+
stop (`recoverable_failure`, so `active/interrupted`) is not a dead end either:
|
|
429
|
+
`recover --extend <duration>` re-persists a deadline measured from now
|
|
430
|
+
(`extendedDeadlineMs`), `--extend 0` recovers straight into the close-out, and
|
|
431
|
+
`discard` closes a record nobody will recover — refused while the task's cwd
|
|
432
|
+
lease exists, since a live owner means the task is running in another Pi and a
|
|
433
|
+
dead owner's lease is reclaimed only through `recover --takeover`'s cleanup proof.
|
|
404
434
|
Input writes are serialized with stop and are acknowledged through the stream
|
|
405
435
|
write callback before their idempotency key is consumed. Writes have a bounded
|
|
406
436
|
timeout, and `stop()` preempts a queued lifecycle operation by initiating adapter
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-claude-supervisor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "A policy-gated Pi supervisor for observing and verifying Claude Code workers.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -67,8 +67,8 @@
|
|
|
67
67
|
"devDependencies": {
|
|
68
68
|
"@earendil-works/pi-coding-agent": "0.85.1",
|
|
69
69
|
"@types/node": "22.20.1",
|
|
70
|
-
"typebox": "1.3.
|
|
70
|
+
"typebox": "1.3.32",
|
|
71
71
|
"typescript": "7.0.2",
|
|
72
|
-
"yaml": "2.9.
|
|
72
|
+
"yaml": "2.9.1"
|
|
73
73
|
}
|
|
74
74
|
}
|
package/src/config.ts
CHANGED
|
@@ -39,6 +39,10 @@ const allowed = new Set([
|
|
|
39
39
|
"PI_CLAUDE_SUPERVISOR_EVIDENCE_MAX_UNTRACKED_FILES",
|
|
40
40
|
"PI_CLAUDE_SUPERVISOR_REVIEW_TIMEOUT_MS",
|
|
41
41
|
"PI_CLAUDE_SUPERVISOR_EVENT_LOG_MAX_BYTES",
|
|
42
|
+
"PI_CLAUDE_SUPERVISOR_DEADLINE_MS",
|
|
43
|
+
"PI_CLAUDE_SUPERVISOR_DEADLINE_GRACE_MS",
|
|
44
|
+
"PI_CLAUDE_SUPERVISOR_DEADLINE_WARNING_MS",
|
|
45
|
+
"PI_CLAUDE_SUPERVISOR_NO_OUTPUT_TIMEOUT_MS",
|
|
42
46
|
]);
|
|
43
47
|
|
|
44
48
|
export interface AutonomyDefaults {
|
|
@@ -109,6 +113,71 @@ export function evidenceMaxUntrackedFiles(env: NodeJS.ProcessEnv = process.env):
|
|
|
109
113
|
return readBoundedInteger(env.PI_CLAUDE_SUPERVISOR_EVIDENCE_MAX_UNTRACKED_FILES, 512, 16, 10_000);
|
|
110
114
|
}
|
|
111
115
|
|
|
116
|
+
export const DEFAULT_DEADLINE_MS = 4 * 60 * 60_000;
|
|
117
|
+
export const DEFAULT_DEADLINE_GRACE_MS = 30 * 60_000;
|
|
118
|
+
export const DEFAULT_DEADLINE_WARNING_MS = 15 * 60_000;
|
|
119
|
+
export const DEFAULT_NO_OUTPUT_TIMEOUT_MS = 20 * 60_000;
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Cumulative wall-clock budget for a task's Worker turns (5 minutes to 7 days);
|
|
123
|
+
* "0" disables the deadline. A duration suffix (`8h`, `90m`, `2h30m`) is
|
|
124
|
+
* accepted as well as plain milliseconds.
|
|
125
|
+
*/
|
|
126
|
+
export function deadlineMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
127
|
+
return readBoundedDurationWithZeroOptOut(env.PI_CLAUDE_SUPERVISOR_DEADLINE_MS, DEFAULT_DEADLINE_MS, 5 * 60_000, 7 * 24 * 60 * 60_000);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Close-out window after the deadline: the Worker is asked to finish and the
|
|
132
|
+
* candidate is verified instead of the Worker being stopped outright. "0"
|
|
133
|
+
* restores the immediate stop at the deadline. Up to 24 hours.
|
|
134
|
+
*/
|
|
135
|
+
export function deadlineGraceMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
136
|
+
return readBoundedDurationWithZeroOptOut(env.PI_CLAUDE_SUPERVISOR_DEADLINE_GRACE_MS, DEFAULT_DEADLINE_GRACE_MS, 60_000, 24 * 60 * 60_000);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** How long before the deadline the Decision Worker is warned (up to 24 hours); "0" disables the warning. */
|
|
140
|
+
export function deadlineWarningMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
141
|
+
return readBoundedDurationWithZeroOptOut(env.PI_CLAUDE_SUPERVISOR_DEADLINE_WARNING_MS, DEFAULT_DEADLINE_WARNING_MS, 60_000, 24 * 60 * 60_000);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Stop a Worker that has produced no output for this long (1 minute to 24 hours); "0" disables the check. */
|
|
145
|
+
export function noOutputTimeoutMs(env: NodeJS.ProcessEnv = process.env): number {
|
|
146
|
+
return readBoundedDurationWithZeroOptOut(env.PI_CLAUDE_SUPERVISOR_NO_OUTPUT_TIMEOUT_MS, DEFAULT_NO_OUTPUT_TIMEOUT_MS, 60_000, 24 * 60 * 60_000);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const DURATION_UNITS_MS: Record<string, number> = { ms: 1, s: 1_000, m: 60_000, h: 60 * 60_000, d: 24 * 60 * 60_000 };
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Parse a duration such as `8h`, `90m`, `2h30m`, `45s` or plain milliseconds
|
|
153
|
+
* into milliseconds; undefined for anything else (including negative values).
|
|
154
|
+
*/
|
|
155
|
+
export function parseDurationMs(value: string | undefined): number | undefined {
|
|
156
|
+
const trimmed = value?.trim().toLowerCase();
|
|
157
|
+
if (!trimmed) return undefined;
|
|
158
|
+
if (/^\d+$/u.test(trimmed)) {
|
|
159
|
+
const parsed = Number(trimmed);
|
|
160
|
+
return Number.isSafeInteger(parsed) ? parsed : undefined;
|
|
161
|
+
}
|
|
162
|
+
if (!/^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d))+$/u.test(trimmed)) return undefined;
|
|
163
|
+
let total = 0;
|
|
164
|
+
for (const match of trimmed.matchAll(/(\d+(?:\.\d+)?)(ms|s|m|h|d)/gu)) {
|
|
165
|
+
total += Number(match[1]) * DURATION_UNITS_MS[match[2]];
|
|
166
|
+
}
|
|
167
|
+
return Number.isSafeInteger(total) ? total : undefined;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Format milliseconds as a compact duration (`2h13m`, `45s`, `0s`) for status lines and notices. */
|
|
171
|
+
export function formatDurationMs(ms: number): string {
|
|
172
|
+
const total = Math.max(0, Math.round(ms / 1_000));
|
|
173
|
+
const hours = Math.floor(total / 3_600);
|
|
174
|
+
const minutes = Math.floor((total % 3_600) / 60);
|
|
175
|
+
const seconds = total % 60;
|
|
176
|
+
if (hours > 0) return minutes > 0 ? `${hours}h${minutes}m` : `${hours}h`;
|
|
177
|
+
if (minutes > 0) return seconds > 0 && total < 600 ? `${minutes}m${seconds}s` : `${minutes}m`;
|
|
178
|
+
return `${seconds}s`;
|
|
179
|
+
}
|
|
180
|
+
|
|
112
181
|
export function loadSupervisorEnvironment(): string | undefined {
|
|
113
182
|
const path = process.env.PI_CLAUDE_SUPERVISOR_ENV_FILE ?? join(homedir(), ".config", "pi-claude-supervisor", "env");
|
|
114
183
|
if (!existsSync(path)) return undefined;
|
|
@@ -148,6 +217,15 @@ function readBoundedIntegerWithZeroOptOut(value: string | undefined, fallback: n
|
|
|
148
217
|
return readBoundedInteger(value, fallback, minimum, maximum);
|
|
149
218
|
}
|
|
150
219
|
|
|
220
|
+
/** Like readBoundedIntegerWithZeroOptOut, but also accepts a duration suffix (`8h`, `90m`); out-of-range values keep the fallback. */
|
|
221
|
+
function readBoundedDurationWithZeroOptOut(value: string | undefined, fallback: number, minimum: number, maximum: number): number {
|
|
222
|
+
if (value === undefined) return fallback;
|
|
223
|
+
const parsed = parseDurationMs(value);
|
|
224
|
+
// "0", "0m", "0h": any zero duration is the opt-out, not a below-minimum typo.
|
|
225
|
+
if (parsed === 0) return 0;
|
|
226
|
+
return parsed !== undefined && parsed >= minimum && parsed <= maximum ? parsed : fallback;
|
|
227
|
+
}
|
|
228
|
+
|
|
151
229
|
function readTrimmedString(value: string | undefined): string | undefined {
|
|
152
230
|
const trimmed = value?.trim();
|
|
153
231
|
return trimmed ? trimmed : undefined;
|
package/src/cwd-lease.ts
CHANGED
|
@@ -651,6 +651,11 @@ async function cgroupHasProcesses(path: string): Promise<boolean> {
|
|
|
651
651
|
}
|
|
652
652
|
}
|
|
653
653
|
|
|
654
|
+
/** True while the Pi process that acquired the lease is still alive (pid and start time both match). */
|
|
655
|
+
export async function leaseOwnerLive(lease: CwdLeaseRecord): Promise<boolean> {
|
|
656
|
+
return processIdentityLive(lease.ownerPid, lease.ownerStartTime);
|
|
657
|
+
}
|
|
658
|
+
|
|
654
659
|
async function canTakeoverLease(lease: CwdLeaseRecord): Promise<TakeoverProof | undefined> {
|
|
655
660
|
// The old supervisor owner must be gone. A dead owner is not enough when
|
|
656
661
|
// the detached Worker itself is still alive. Compare start time as well as
|
package/src/decision-worker.ts
CHANGED
|
@@ -26,6 +26,21 @@ export interface DecisionContext {
|
|
|
26
26
|
maxTurns: number;
|
|
27
27
|
repairRound?: number;
|
|
28
28
|
spec?: TaskSpec;
|
|
29
|
+
/** Wall-clock budget of the task; absent when no deadline is configured. */
|
|
30
|
+
deadline?: DecisionDeadlineContext;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface DecisionDeadlineContext {
|
|
34
|
+
/** Cumulative wall-clock budget for the task's Worker turns. */
|
|
35
|
+
totalMs: number;
|
|
36
|
+
/** Close-out window after the deadline before the Supervisor stops the Worker; 0 means the stop is immediate. */
|
|
37
|
+
graceMs: number;
|
|
38
|
+
/** Time left before the deadline; 0 once it has passed. */
|
|
39
|
+
remainingMs: number;
|
|
40
|
+
/** True once the deadline has passed and the task is in its close-out window. */
|
|
41
|
+
closeOut: boolean;
|
|
42
|
+
/** During close-out: time left before the Supervisor stops the Worker outright. */
|
|
43
|
+
closeOutRemainingMs?: number;
|
|
29
44
|
}
|
|
30
45
|
|
|
31
46
|
export interface DecisionWorkerLike {
|
|
@@ -277,7 +292,7 @@ Task id: ${redactText(context.taskId)}
|
|
|
277
292
|
Working directory: ${redactText(context.cwd)}
|
|
278
293
|
Maximum automatic turns: ${context.maxTurns}
|
|
279
294
|
Current repair round: ${context.repairRound ?? 0}
|
|
280
|
-
Task specification: ${boundedJson(context.spec ?? { goal: context.task })}
|
|
295
|
+
${deadlineInstructions(context.deadline)}Task specification: ${boundedJson(context.spec ?? { goal: context.task })}
|
|
281
296
|
|
|
282
297
|
Return exactly one JSON object and no markdown:
|
|
283
298
|
{"action":"continue|redirect|answer|allow_permission|deny_permission|verify|retry|stop|park|wait|noop",...}
|
|
@@ -303,7 +318,39 @@ wait for a human to be online. For an exited event choose verify, park or stop;
|
|
|
303
318
|
verify. Choose wait when the Worker's result says it is waiting for its own background agents,
|
|
304
319
|
tasks or monitors: their completion re-invokes the Worker automatically, a message would only
|
|
305
320
|
interrupt it, and the Supervisor asks you again if the Worker has not resumed within the wait
|
|
306
|
-
timeout. A completed turn or a permission request always requires a concrete action
|
|
321
|
+
timeout. A completed turn or a permission request always requires a concrete action.${deadlinePolicy(context.deadline)}`;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function deadlineInstructions(deadline: DecisionDeadlineContext | undefined): string {
|
|
325
|
+
if (!deadline) return "";
|
|
326
|
+
const closeOut = deadline.graceMs > 0
|
|
327
|
+
? `then a ${formatMinutes(deadline.graceMs)} close-out window before the Supervisor stops the Worker`
|
|
328
|
+
: "with no close-out window: the Supervisor stops the Worker at the deadline";
|
|
329
|
+
return `Wall-clock budget: ${formatMinutes(deadline.totalMs)} for the whole task, ${closeOut}.\n`;
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function deadlinePolicy(deadline: DecisionDeadlineContext | undefined): string {
|
|
333
|
+
if (!deadline) return "";
|
|
334
|
+
const wrapUp = `
|
|
335
|
+
Time budget: CURRENT CONTEXT reports deadlineRemainingMinutes. While it is small (roughly the
|
|
336
|
+
length of one build-and-test cycle), stop waiting on long background work: use continue or
|
|
337
|
+
redirect to tell Claude Code to integrate what is finished, run the required checks, commit,
|
|
338
|
+
and stop, so the candidate can be verified before the deadline.`;
|
|
339
|
+
if (deadline.graceMs <= 0) {
|
|
340
|
+
return `${wrapUp} There is no close-out window: a Worker still running at the deadline is
|
|
341
|
+
stopped outright, so choose verify as soon as the result is committed rather than waiting.`;
|
|
342
|
+
}
|
|
343
|
+
return `${wrapUp} Once closeOut is true the
|
|
344
|
+
deadline has passed and only closeOutRemainingMinutes are left before the Worker is stopped:
|
|
345
|
+
choose verify as soon as the Worker is idle, or one concise continue/redirect that tells it to
|
|
346
|
+
commit what is complete and stop; wait is no longer available during close-out and an idle
|
|
347
|
+
Worker is verified instead.`;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function formatMinutes(ms: number): string {
|
|
351
|
+
const minutes = Math.round(ms / 60_000);
|
|
352
|
+
if (minutes >= 120 && minutes % 60 === 0) return `${minutes / 60} hours`;
|
|
353
|
+
return `${minutes} minutes`;
|
|
307
354
|
}
|
|
308
355
|
|
|
309
356
|
/**
|
|
@@ -318,7 +365,19 @@ async function askDecision(
|
|
|
318
365
|
timeoutMs: number,
|
|
319
366
|
options: { instructionsPrefix?: string; onUsage?: (usage: PiUsageSample) => void } = {},
|
|
320
367
|
): Promise<string> {
|
|
321
|
-
const currentContext = {
|
|
368
|
+
const currentContext = {
|
|
369
|
+
state: context.state,
|
|
370
|
+
turn: context.turn,
|
|
371
|
+
maxTurns: context.maxTurns,
|
|
372
|
+
repairRound: context.repairRound ?? 0,
|
|
373
|
+
...(context.deadline ? {
|
|
374
|
+
deadlineRemainingMinutes: Math.round(context.deadline.remainingMs / 60_000),
|
|
375
|
+
closeOut: context.deadline.closeOut,
|
|
376
|
+
...(context.deadline.closeOut && context.deadline.closeOutRemainingMs !== undefined
|
|
377
|
+
? { closeOutRemainingMinutes: Math.round(context.deadline.closeOutRemainingMs / 60_000) }
|
|
378
|
+
: {}),
|
|
379
|
+
} : {}),
|
|
380
|
+
};
|
|
322
381
|
const prompt = `${options.instructionsPrefix ?? ""}UNTRUSTED SUPERVISOR EVENT:\n${boundedEventJson(event)}\n\nCURRENT CONTEXT:\n${boundedJson(currentContext)}\n\nChoose one action now.`;
|
|
323
382
|
return promptForText(session, prompt, timeoutMs, "Decision Worker request", MAX_DECISION_RESPONSE_BYTES, { role: "decision", onUsage: options.onUsage });
|
|
324
383
|
}
|
package/src/index.ts
CHANGED
|
@@ -8,13 +8,13 @@ import { EventLog } from "./events.ts";
|
|
|
8
8
|
import { redactSensitive } from "./redaction.ts";
|
|
9
9
|
import { ProcessWorkerAdapter } from "./worker/process-adapter.ts";
|
|
10
10
|
import { automaticWorkerEnvironment } from "./worker/environment.ts";
|
|
11
|
-
import { TmuxWorkerAdapter, attachCommand } from "./worker/tmux-adapter.ts";
|
|
12
|
-
import { Supervisor, type DecisionSessionClosedInfo, type HumanInterventionNotice, type SupervisorProgress, type SupervisorTokenUsage } from "./supervisor.ts";
|
|
11
|
+
import { TmuxWorkerAdapter, attachCommand, sweepDeadTmuxSockets } from "./worker/tmux-adapter.ts";
|
|
12
|
+
import { Supervisor, extendedDeadlineMs, type DecisionSessionClosedInfo, type HumanInterventionNotice, type SupervisorProgress, type SupervisorTokenUsage } from "./supervisor.ts";
|
|
13
13
|
import { evaluateCommand } from "./policy.ts";
|
|
14
14
|
import { HumanWebhookNotifier } from "./notifications.ts";
|
|
15
|
-
import { autoInstallHooks, autonomyDefaults, closeWorkerOnCompletion, decisionCompactionTokens, decisionModel, decisionSessionRetentionDays, eventLogMaxBytes, loadSupervisorEnvironment, progressHeartbeatMs, reviewTimeoutMs, reviewerModel, tmuxMode, workerAutocompactTokens, workerMcpConfigPath, workerModel } from "./config.ts";
|
|
15
|
+
import { autoInstallHooks, autonomyDefaults, closeWorkerOnCompletion, deadlineGraceMs, deadlineMs, deadlineWarningMs, decisionCompactionTokens, decisionModel, decisionSessionRetentionDays, eventLogMaxBytes, formatDurationMs, loadSupervisorEnvironment, noOutputTimeoutMs, parseDurationMs, progressHeartbeatMs, reviewTimeoutMs, reviewerModel, tmuxMode, workerAutocompactTokens, workerMcpConfigPath, workerModel } from "./config.ts";
|
|
16
16
|
import { DecisionSessionStore, type DecisionSessionRecord } from "./decision-session-store.ts";
|
|
17
|
-
import { CwdLeaseStore, type CwdLeaseHandle, pathsOverlap, workerIdentity } from "./cwd-lease.ts";
|
|
17
|
+
import { CwdLeaseStore, type CwdLeaseHandle, leaseOwnerLive, pathsOverlap, workerIdentity } from "./cwd-lease.ts";
|
|
18
18
|
import { normalizeTaskSpec } from "./acceptance.ts";
|
|
19
19
|
import { PiReadOnlyReviewer } from "./reviewer.ts";
|
|
20
20
|
import { resolvePiModel } from "./pi-model.ts";
|
|
@@ -128,6 +128,9 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
128
128
|
// with /supervise uninstall-hooks; the relay is a ~1 ms no-op when no
|
|
129
129
|
// Supervisor is listening, so leaving it installed costs nothing.
|
|
130
130
|
let hookInstallNotice: string | undefined;
|
|
131
|
+
// A Worker killed with its cgroup leaves its tmux socket behind; sweep the
|
|
132
|
+
// ones no server answers on so the temp directory does not fill with them.
|
|
133
|
+
if (transport === "tmux") sweepDeadTmuxSockets();
|
|
131
134
|
if (hookServer) {
|
|
132
135
|
hookServerReady = (async () => {
|
|
133
136
|
await writeRelayScript(relayPath);
|
|
@@ -366,12 +369,9 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
366
369
|
const [operation = "status", ...rest] = tokens;
|
|
367
370
|
let message = "";
|
|
368
371
|
if (operation === "start" || operation === "adopt-tmux") {
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
const specIndex = rest.findIndex((value) => value.startsWith("--spec="));
|
|
373
|
-
if (specIndex >= 0) specPath = rest.splice(specIndex, 1)[0]?.slice("--spec=".length);
|
|
374
|
-
}
|
|
372
|
+
const specPath = takeOption(rest, "--spec");
|
|
373
|
+
const deadlineOption = takeOption(rest, "--deadline");
|
|
374
|
+
const taskDeadlineMs = deadlineOption === undefined ? deadlineMs() : parseTaskDeadline(deadlineOption);
|
|
375
375
|
const tmuxSession = operation === "adopt-tmux" ? rest.shift() : undefined;
|
|
376
376
|
const task = rest.join(" ").trim();
|
|
377
377
|
const fileSpec = specPath ? await readTaskSpecFile(specPath, ctx.cwd) : undefined;
|
|
@@ -384,7 +384,7 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
384
384
|
// own events through hooks exactly like an owned one.
|
|
385
385
|
const taskAutomation = (operation !== "adopt-tmux" || tmuxMode() === "interactive") && automation && spec.autonomy.unattended;
|
|
386
386
|
const interactive = taskAutomation && adapter.capabilities().transport === "tmux" && tmuxMode() === "interactive";
|
|
387
|
-
if (!goal) throw new Error(operation === "adopt-tmux" ? "Usage: /supervise adopt-tmux [--spec <file>] <tmux-session> <task>" : "Usage: /supervise start [--spec <file>] <task>");
|
|
387
|
+
if (!goal) throw new Error(operation === "adopt-tmux" ? "Usage: /supervise adopt-tmux [--spec <file>] [--deadline <duration>] <tmux-session> <task>" : "Usage: /supervise start [--spec <file>] [--deadline <duration>] <task>");
|
|
388
388
|
if (operation === "adopt-tmux" && adapter.capabilities().transport !== "tmux") throw new Error("/supervise adopt-tmux requires PI_CLAUDE_SUPERVISOR_TRANSPORT=tmux");
|
|
389
389
|
if (operation === "adopt-tmux" && interactive && !await userHooksInstalled(claudeUserSettingsPath())) {
|
|
390
390
|
await hookServerReady?.catch(() => {});
|
|
@@ -455,6 +455,10 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
455
455
|
approval,
|
|
456
456
|
automation: taskAutomation,
|
|
457
457
|
interactive,
|
|
458
|
+
deadlineMs: taskDeadlineMs,
|
|
459
|
+
deadlineGraceMs: deadlineGraceMs(),
|
|
460
|
+
deadlineWarningMs: deadlineWarningMs(),
|
|
461
|
+
noOutputTimeoutMs: noOutputTimeoutMs(),
|
|
458
462
|
hookSource: interactive ? hookServer : undefined,
|
|
459
463
|
hookSettingsPath,
|
|
460
464
|
keepWorkerOnCompletion: interactive ? !closeWorkerOnCompletion() : undefined,
|
|
@@ -640,9 +644,13 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
640
644
|
pendingCwds.delete(cwdKey);
|
|
641
645
|
}
|
|
642
646
|
} else if (operation === "recover") {
|
|
647
|
+
const usage = "Usage: /supervise recover [--takeover] [--extend <duration>] <task-id>";
|
|
648
|
+
const extendOption = takeOption(rest, "--extend", ["--takeover"]);
|
|
643
649
|
const takeover = rest.includes("--takeover");
|
|
644
650
|
const taskId = rest.find((value) => value !== "--takeover");
|
|
645
|
-
if (!taskId || rest.some((value) => value !== "--takeover" && value !== taskId)) throw new Error(
|
|
651
|
+
if (!taskId || rest.some((value) => value !== "--takeover" && value !== taskId)) throw new Error(usage);
|
|
652
|
+
const extendMs = extendOption === undefined ? undefined : parseDurationMs(extendOption);
|
|
653
|
+
if (extendOption !== undefined && extendMs === undefined) throw new Error(`--extend expects a duration such as 30m, 2h or 0 (close out now): ${extendOption}`);
|
|
646
654
|
if (shuttingDown) throw new Error("Pi session is shutting down");
|
|
647
655
|
if (sessions.has(taskId)) throw new Error(`Task session is already loaded: ${taskId}`);
|
|
648
656
|
const record = await decisionStore.load(taskId);
|
|
@@ -651,7 +659,17 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
651
659
|
if (staleRecovery && !takeover) throw new Error(`Decision Worker recovery is stale (${record.recoveryState}); retry with --takeover only after verifying the old Worker is gone: ${taskId}`);
|
|
652
660
|
if (!await decisionStore.sessionFileExists(taskId)) throw new Error(`Decision Worker session file is missing or unsafe: ${taskId}`);
|
|
653
661
|
if (record.maxTurns > 0 && record.turn >= record.maxTurns) throw new Error(`Cannot recover task after its turn budget was exhausted: ${taskId}`);
|
|
654
|
-
if (
|
|
662
|
+
if (extendMs !== undefined && record.deadlineMs <= 0) throw new Error(`Task has no wall-clock deadline to extend: ${taskId}`);
|
|
663
|
+
const elapsedMs = Math.max(0, Date.now() - Date.parse(record.startedAt));
|
|
664
|
+
// A task past its budget is not lost: `--extend` grants a fresh budget
|
|
665
|
+
// from now (0 opens the close-out at once, verifying the repository as
|
|
666
|
+
// it stands), which the recovered Supervisor persists as the deadline.
|
|
667
|
+
const recoveryDeadlineMs = extendMs === undefined ? record.deadlineMs : extendedDeadlineMs(record.deadlineMs, elapsedMs, extendMs);
|
|
668
|
+
// With --extend the new deadline is measured from now by construction
|
|
669
|
+
// (`--extend 0` deliberately lands on it, so the close-out opens at once).
|
|
670
|
+
if (extendMs === undefined && recoveryDeadlineMs > 0 && elapsedMs >= recoveryDeadlineMs) {
|
|
671
|
+
throw new Error(`Cannot recover task after its wall-clock deadline (${formatDurationMs(elapsedMs - recoveryDeadlineMs)} ago); pass --extend <duration> to grant a close-out budget from now, or /supervise discard ${taskId} to drop the record`);
|
|
672
|
+
}
|
|
655
673
|
const cwdKey = await canonicalCwd(record.cwd);
|
|
656
674
|
await releaseSettledReservations();
|
|
657
675
|
if (shuttingDown) throw new Error("Pi session is shutting down");
|
|
@@ -783,7 +801,9 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
783
801
|
: undefined,
|
|
784
802
|
...(automaticRecovery && record.resolvedExecutable ? { expectedClaudeExecutable: record.resolvedExecutable } : {}),
|
|
785
803
|
maxTurns: record.maxTurns,
|
|
786
|
-
deadlineMs:
|
|
804
|
+
deadlineMs: recoveryDeadlineMs,
|
|
805
|
+
deadlineGraceMs: deadlineGraceMs(),
|
|
806
|
+
deadlineWarningMs: deadlineWarningMs(),
|
|
787
807
|
noOutputTimeoutMs: record.noOutputTimeoutMs,
|
|
788
808
|
startedAt: record.startedAt,
|
|
789
809
|
baseCommit: record.baseCommit,
|
|
@@ -903,6 +923,33 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
903
923
|
pendingStartSessions.delete(session);
|
|
904
924
|
pendingCwds.delete(cwdKey);
|
|
905
925
|
}
|
|
926
|
+
} else if (operation === "discard") {
|
|
927
|
+
const [taskId, ...extra] = rest;
|
|
928
|
+
if (!taskId || extra.length > 0) throw new Error("Usage: /supervise discard <task-id>");
|
|
929
|
+
if (sessions.has(taskId)) throw new Error(`Task session is loaded in this Pi; use /supervise stop ${taskId} instead`);
|
|
930
|
+
const record = await decisionStore.load(taskId);
|
|
931
|
+
if (!record || record.state !== "active") throw new Error(`No recoverable Decision Worker session: ${taskId}`);
|
|
932
|
+
// "starting"/"registered"/"recovered_idle" mean another Pi owns a
|
|
933
|
+
// recovery of this task; only a settled record may be dropped here.
|
|
934
|
+
if (!["ready", "interrupted"].includes(record.recoveryState)) throw new Error(`Decision Worker recovery is in progress elsewhere (${record.recoveryState}); discard only after that Pi has released it: ${taskId}`);
|
|
935
|
+
// The cwd lease is the ownership record: a live owner means the task
|
|
936
|
+
// is running in another Pi ("ready" is also its normal state); a dead
|
|
937
|
+
// owner's lease still guards the cwd against a possibly-live detached
|
|
938
|
+
// Worker and is only reclaimed through recover --takeover's cleanup
|
|
939
|
+
// proof, which needs this record. Neither may be discarded from here.
|
|
940
|
+
const lease = (await cwdLeaseStore.list()).find((candidate) => candidate.taskId === taskId);
|
|
941
|
+
if (lease) {
|
|
942
|
+
if (await leaseOwnerLive(lease)) throw new Error(`Task is still owned by a live Pi (pid ${lease.ownerPid}); stop it there instead of discarding it: ${taskId}`);
|
|
943
|
+
// An adopted session's tmux server is the user's own and stays
|
|
944
|
+
// alive, so a takeover can never prove the Worker gone; adopting
|
|
945
|
+
// the session again hands the dead owner's lease over in place.
|
|
946
|
+
if (lease.worker?.ownership === "adopted" && lease.worker.sessionName) {
|
|
947
|
+
throw new Error(`Task still holds the cwd lease for ${redactText(lease.cwd)} (owner pid ${lease.ownerPid} is gone); adopt the session again with /supervise adopt-tmux ${lease.worker.sessionName} <task> to take the lease over, then discard this record`);
|
|
948
|
+
}
|
|
949
|
+
throw new Error(`Task still holds the cwd lease for ${redactText(lease.cwd)} (owner pid ${lease.ownerPid} is gone); run /supervise recover --takeover --extend 0 ${taskId} to prove the old Worker is gone and close the task out, then stop it if needed`);
|
|
950
|
+
}
|
|
951
|
+
await decisionStore.close(taskId);
|
|
952
|
+
message = `Discarded recoverable task ${taskId} (cwd ${redactText(record.cwd)}); its Decision Worker session file is kept until retention pruning`;
|
|
906
953
|
} else if (operation === "sessions") {
|
|
907
954
|
const recoverable = await decisionStore.list({ activeOnly: true });
|
|
908
955
|
message = formatSessions(sessions, recoverable, tmuxModeLabel());
|
|
@@ -913,7 +960,7 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
913
960
|
} else if (operation === "status") {
|
|
914
961
|
const { session, sessionId } = resolveSession(sessions, activeTaskId, rest, true);
|
|
915
962
|
message = session
|
|
916
|
-
? `task=${sessionId} state=${session.state} worker=${session.handle?.id ?? "none"}${tmuxModeLabel() ? ` mode=${tmuxModeLabel()}` : ""} ${formatUsageDetail(session.usage)}`
|
|
963
|
+
? `task=${sessionId} state=${session.state} worker=${session.handle?.id ?? "none"}${tmuxModeLabel() ? ` mode=${tmuxModeLabel()}` : ""}${formatDeadline(session.deadline)} ${formatUsageDetail(session.usage)}`
|
|
917
964
|
: formatSessions(sessions, await decisionStore.list({ activeOnly: true }), tmuxModeLabel());
|
|
918
965
|
} else if (operation === "capabilities") {
|
|
919
966
|
message = JSON.stringify(adapter.capabilities(), null, 2);
|
|
@@ -963,7 +1010,7 @@ export default function piClaudeSupervisor(pi: ExtensionAPI): void {
|
|
|
963
1010
|
} else if (operation === "resume-auto") {
|
|
964
1011
|
await session.resumeAutomation(); message = `Automatic decisions resumed: ${sessionId}.`;
|
|
965
1012
|
} else {
|
|
966
|
-
throw new Error("Usage: /supervise start|adopt-tmux|recover [--takeover]|sessions|status|poll [all|taskId]|send [taskId]|pause [taskId]|resume [taskId]|stop [taskId]|verify [taskId]|approve [taskId] <allow|deny>|takeover [taskId]|resume-auto [taskId]|capabilities|install-hooks|uninstall-hooks");
|
|
1013
|
+
throw new Error("Usage: /supervise start [--spec <file>] [--deadline <duration>]|adopt-tmux [--spec <file>] [--deadline <duration>]|recover [--takeover] [--extend <duration>]|discard <task-id>|sessions|status|poll [all|taskId]|send [taskId]|pause [taskId]|resume [taskId]|stop [taskId]|verify [taskId]|approve [taskId] <allow|deny>|takeover [taskId]|resume-auto [taskId]|capabilities|install-hooks|uninstall-hooks");
|
|
967
1014
|
}
|
|
968
1015
|
}
|
|
969
1016
|
if (hookInstallNotice) { notify(ctx, hookInstallNotice); hookInstallNotice = undefined; }
|
|
@@ -1056,13 +1103,52 @@ function requiresWorkerCleanup(error: unknown): boolean {
|
|
|
1056
1103
|
return Boolean(error && typeof error === "object" && (error as { workerCleanupRequired?: unknown }).workerCleanupRequired === true);
|
|
1057
1104
|
}
|
|
1058
1105
|
|
|
1106
|
+
/**
|
|
1107
|
+
* Remove a `--name value` or `--name=value` option and return its value.
|
|
1108
|
+
* `--name value` is recognised only in the leading option block (walking
|
|
1109
|
+
* over other options and their values; `flags` take no value), so a task
|
|
1110
|
+
* description that mentions `--deadline` is left alone; `--name=value` is
|
|
1111
|
+
* accepted anywhere, as `--spec=` always was.
|
|
1112
|
+
*/
|
|
1113
|
+
function takeOption(rest: string[], name: string, flags: readonly string[] = []): string | undefined {
|
|
1114
|
+
for (let index = 0; index < rest.length && rest[index].startsWith("--"); index += 1) {
|
|
1115
|
+
if (rest[index] === name) return rest.splice(index, 2)[1];
|
|
1116
|
+
if (rest[index].startsWith(`${name}=`)) return rest.splice(index, 1)[0]?.slice(name.length + 1);
|
|
1117
|
+
// Another option: its value (when it has one and it is not inline) is the next token.
|
|
1118
|
+
if (!flags.includes(rest[index]) && !rest[index].includes("=")) index += 1;
|
|
1119
|
+
}
|
|
1120
|
+
const inline = rest.findIndex((value) => value.startsWith(`${name}=`));
|
|
1121
|
+
return inline >= 0 ? rest.splice(inline, 1)[0]?.slice(name.length + 1) : undefined;
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
/** `--deadline` accepts `8h`, `90m`, `2h30m`, plain milliseconds, or `0` to disable the deadline for this task; the same 5m–7d range as `DEADLINE_MS`. */
|
|
1125
|
+
function parseTaskDeadline(value: string): number {
|
|
1126
|
+
const parsed = parseDurationMs(value);
|
|
1127
|
+
if (parsed === undefined) throw new Error(`--deadline expects a duration such as 8h, 90m or 0 (disabled): ${value}`);
|
|
1128
|
+
if (parsed !== 0 && (parsed < 5 * 60_000 || parsed > 7 * 24 * 60 * 60_000)) throw new Error("--deadline must be between 5m and 7d, or 0 to disable the deadline");
|
|
1129
|
+
return parsed;
|
|
1130
|
+
}
|
|
1131
|
+
|
|
1132
|
+
/** A recoverable record's budget: how much is left, or how long ago it expired (recover then needs `--extend`). */
|
|
1133
|
+
function formatRecordDeadline(record: DecisionSessionRecord): string {
|
|
1134
|
+
if (record.deadlineMs <= 0) return "";
|
|
1135
|
+
const remaining = record.deadlineMs - (Date.now() - Date.parse(record.startedAt));
|
|
1136
|
+
return remaining > 0 ? ` deadline=${formatDurationMs(remaining)} left` : ` deadline=expired ${formatDurationMs(-remaining)} ago (recover --extend)`;
|
|
1137
|
+
}
|
|
1138
|
+
|
|
1139
|
+
function formatDeadline(deadline: Supervisor["deadline"]): string {
|
|
1140
|
+
if (!deadline) return "";
|
|
1141
|
+
if (!deadline.closeOut) return ` deadline=${formatDurationMs(deadline.remainingMs)} left`;
|
|
1142
|
+
return ` deadline=close-out (${formatDurationMs(deadline.closeOutRemainingMs ?? 0)} left)`;
|
|
1143
|
+
}
|
|
1144
|
+
|
|
1059
1145
|
function formatSessions(sessions: Map<string, Supervisor>, recoverable: DecisionSessionRecord[] = [], tmuxModeLabel?: string): string {
|
|
1060
1146
|
const modeSuffix = tmuxModeLabel ? ` mode=${tmuxModeLabel}` : "";
|
|
1061
1147
|
const active = [...sessions.entries()]
|
|
1062
1148
|
.map(([taskId, session]) => `${taskId} state=${session.state} cwd=${session.task?.cwd ?? "-"} worker=${session.handle?.id ?? "-"}${modeSuffix}`);
|
|
1063
1149
|
const pending = recoverable
|
|
1064
1150
|
.filter((record) => !sessions.has(record.taskId))
|
|
1065
|
-
.map((record) => `${record.taskId} state=recoverable recovery=${record.recoveryState} cwd=${record.cwd} worker=${record.recoveryWorker?.id ?? "-"}${modeSuffix}`);
|
|
1151
|
+
.map((record) => `${record.taskId} state=recoverable recovery=${record.recoveryState} cwd=${record.cwd} worker=${record.recoveryWorker?.id ?? "-"}${modeSuffix}${formatRecordDeadline(record)}`);
|
|
1066
1152
|
return [...active, ...pending].join("\n") || "No task sessions.";
|
|
1067
1153
|
}
|
|
1068
1154
|
|
package/src/policy.ts
CHANGED
|
@@ -333,7 +333,8 @@ function evaluateCommandInternal(command: string, depth: number): PolicyResult {
|
|
|
333
333
|
}
|
|
334
334
|
|
|
335
335
|
function evaluateTokens(rawTokens: readonly ShellToken[], depth: number): PolicyResult {
|
|
336
|
-
const { tokens, embedded } = resolveDataTokens(rawTokens);
|
|
336
|
+
const { tokens: dataResolved, embedded } = resolveDataTokens(rawTokens);
|
|
337
|
+
const tokens = resolveLiteralBindings(dataResolved);
|
|
337
338
|
if (depth < 4) {
|
|
338
339
|
for (const body of embedded) {
|
|
339
340
|
const nestedResult = evaluateCommandInternal(body, depth + 1);
|
|
@@ -363,6 +364,61 @@ function evaluateTokens(rawTokens: readonly ShellToken[], depth: number): Policy
|
|
|
363
364
|
return { decision: "allow", reason: "command is allowed for unattended local development" };
|
|
364
365
|
}
|
|
365
366
|
|
|
367
|
+
/**
|
|
368
|
+
* `NAME=literal` and `for NAME in literal…` bind a name to text the policy can
|
|
369
|
+
* see, so a later `$NAME` is not an unseen argument: `for c in 5dae138 feff500;
|
|
370
|
+
* do git show $c; done` and `S=/tmp/x && cat > $S/log` are literal commands.
|
|
371
|
+
* Every bound value is substituted (a loop over `status push` yields
|
|
372
|
+
* `git status push …`, which the boundary checks still catch), and a name
|
|
373
|
+
* rebound to dynamic text is forgotten again.
|
|
374
|
+
*/
|
|
375
|
+
function resolveLiteralBindings(tokens: readonly ShellToken[]): ShellToken[] {
|
|
376
|
+
const bindings = new Map<string, string>();
|
|
377
|
+
const resolved: ShellToken[] = [];
|
|
378
|
+
const substitute = (token: ShellToken): ShellToken => {
|
|
379
|
+
if (token.operator || token.data || !token.dynamic || bindings.size === 0) return token;
|
|
380
|
+
const value = token.value.replace(/\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?/gu, (match, name: string) => bindings.get(name) ?? match);
|
|
381
|
+
if (value === token.value) return token;
|
|
382
|
+
const dynamic = /[$`*?[\]{}~]/u.test(value) && !/^[[\]{}]+$/u.test(value);
|
|
383
|
+
return { ...token, value, dynamic };
|
|
384
|
+
};
|
|
385
|
+
let commandPosition = true;
|
|
386
|
+
for (let index = 0; index < tokens.length; index += 1) {
|
|
387
|
+
const token = substitute(tokens[index]!);
|
|
388
|
+
resolved.push(token);
|
|
389
|
+
if (token.operator) { commandPosition = SEGMENT_SPLIT_OPERATORS.has(token.value); continue; }
|
|
390
|
+
const word = token.value;
|
|
391
|
+
if (commandPosition) {
|
|
392
|
+
const assignment = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/su.exec(word);
|
|
393
|
+
if (assignment) {
|
|
394
|
+
if (token.dynamic) bindings.delete(assignment[1]!);
|
|
395
|
+
else bindings.set(assignment[1]!, assignment[2]!);
|
|
396
|
+
continue;
|
|
397
|
+
}
|
|
398
|
+
if (word.toLowerCase() === "for") {
|
|
399
|
+
const name = tokens[index + 1]?.value;
|
|
400
|
+
const inWord = tokens[index + 2]?.value.toLowerCase();
|
|
401
|
+
if (name && /^[A-Za-z_][A-Za-z0-9_]*$/u.test(name) && inWord === "in") {
|
|
402
|
+
const values: string[] = [];
|
|
403
|
+
let cursor = index + 3;
|
|
404
|
+
let literal = true;
|
|
405
|
+
for (; cursor < tokens.length; cursor += 1) {
|
|
406
|
+
const item = tokens[cursor]!;
|
|
407
|
+
if (item.operator || item.value.toLowerCase() === "do") break;
|
|
408
|
+
const substituted = substitute(item);
|
|
409
|
+
if (substituted.dynamic || substituted.data) literal = false;
|
|
410
|
+
values.push(substituted.value);
|
|
411
|
+
}
|
|
412
|
+
if (literal && values.length > 0) bindings.set(name, values.join(" "));
|
|
413
|
+
else bindings.delete(name);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
if (!COMMAND_POSITION_KEYWORDS.has(word.toLowerCase())) commandPosition = false;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
return resolved;
|
|
420
|
+
}
|
|
421
|
+
|
|
366
422
|
/**
|
|
367
423
|
* A quoted heredoc body means whatever its consumer makes of it. Fed to a shell
|
|
368
424
|
* (`bash <<'EOF'`, `cat <<'EOF' | sh`, `eval "$(cat <<'EOF' …)"`) it is a
|
|
@@ -534,7 +590,10 @@ function lexShell(input: string): { tokens: ShellToken[]; error?: string } {
|
|
|
534
590
|
// Brace, tilde and pathname expansion can change command names, targets or
|
|
535
591
|
// Git refs after this lexical pass. Treat all unquoted expansion markers as
|
|
536
592
|
// dynamic rather than attempting to model Bash's expansion order.
|
|
537
|
-
|
|
593
|
+
// A tilde expands only at the start of a word (or of an assignment value);
|
|
594
|
+
// `HEAD~1` and `a~b` are literal text.
|
|
595
|
+
if (character === "~") { if (value === "" || /[=:]$/u.test(value)) dynamic = true; value += character; started = true; continue; }
|
|
596
|
+
if ("*?[]{}".includes(character)) { dynamic = true; value += character; started = true; continue; }
|
|
538
597
|
if (";&|<>".includes(character)) {
|
|
539
598
|
if (character === "<" && next === "<") {
|
|
540
599
|
const third = input[index + 2];
|
package/src/supervisor.ts
CHANGED
|
@@ -3,7 +3,8 @@ import { join } from "node:path";
|
|
|
3
3
|
import { EventLog, type SupervisorEvent } from "./events.ts";
|
|
4
4
|
import { SupervisorStateMachine } from "./state.ts";
|
|
5
5
|
import { evaluatePermission, isRoutinePermission } from "./policy.ts";
|
|
6
|
-
import { PiDecisionWorker, type DecisionAction, type DecisionWorkerFactory, type DecisionWorkerLike, type PiModel } from "./decision-worker.ts";
|
|
6
|
+
import { PiDecisionWorker, type DecisionAction, type DecisionContext, type DecisionDeadlineContext, type DecisionWorkerFactory, type DecisionWorkerLike, type PiModel } from "./decision-worker.ts";
|
|
7
|
+
import { DEFAULT_DEADLINE_GRACE_MS, DEFAULT_DEADLINE_MS, DEFAULT_DEADLINE_WARNING_MS, DEFAULT_NO_OUTPUT_TIMEOUT_MS, formatDurationMs } from "./config.ts";
|
|
7
8
|
import { collectRepositoryEvidence, repositoryBranch, repositoryCommitExists, repositoryHead, repositoryIsAncestor, repositoryWorkTree, verifyAll, type RepositoryEvidence, type VerificationCommand } from "./verifier.ts";
|
|
8
9
|
import { normalizeTaskSpec } from "./acceptance.ts";
|
|
9
10
|
import { redactSensitive } from "./redaction.ts";
|
|
@@ -28,6 +29,8 @@ import type {
|
|
|
28
29
|
} from "./types.ts";
|
|
29
30
|
|
|
30
31
|
const DEFAULT_REVIEW_TIMEOUT_MS = 600_000;
|
|
32
|
+
/** A repair round shorter than this cannot finish inside the close-out window; block the candidate instead. */
|
|
33
|
+
const MIN_CLOSE_OUT_REPAIR_MS = 60_000;
|
|
31
34
|
|
|
32
35
|
export interface DecisionSessionReadyInfo {
|
|
33
36
|
taskId: string;
|
|
@@ -96,6 +99,17 @@ export interface SupervisorStartOptions {
|
|
|
96
99
|
maxTurns?: number;
|
|
97
100
|
/** Maximum wall-clock runtime; defaults to 4 hours for long development tasks. Set to 0 to disable. */
|
|
98
101
|
deadlineMs?: number;
|
|
102
|
+
/**
|
|
103
|
+
* Close-out window after the deadline: the Decision Worker is told the
|
|
104
|
+
* budget is spent, an idle Worker is verified instead of stopped, and the
|
|
105
|
+
* Worker is stopped outright only once this window has also elapsed.
|
|
106
|
+
* Defaults to 30 minutes; 0 stops the Worker at the deadline as before.
|
|
107
|
+
* Only automatic tasks have a close-out (there is nobody else to verify);
|
|
108
|
+
* a manual task is stopped at the deadline regardless of this value.
|
|
109
|
+
*/
|
|
110
|
+
deadlineGraceMs?: number;
|
|
111
|
+
/** Warn the Decision Worker this long before the deadline; defaults to 15 minutes. Set to 0 to disable. */
|
|
112
|
+
deadlineWarningMs?: number;
|
|
99
113
|
/** Maximum time without worker output; defaults to 20 minutes. Set to 0 to disable. */
|
|
100
114
|
noOutputTimeoutMs?: number;
|
|
101
115
|
/** After a `wait` decision, how long the Worker may stay silent before the Decision Worker is asked again; defaults to 10 minutes. Set to 0 to disable. */
|
|
@@ -198,6 +212,8 @@ export class Supervisor {
|
|
|
198
212
|
#workerOutput = "";
|
|
199
213
|
#lastWorkerResult?: Record<string, unknown>;
|
|
200
214
|
#lastTurnCompleted?: WorkerEvent;
|
|
215
|
+
/** Key of the completed turn whose decision is still in flight; the deadline must not verify underneath it. */
|
|
216
|
+
#pendingDecisionKey?: string;
|
|
201
217
|
/** Armed by a `wait` decision: re-asks the Decision Worker if the Worker never resumes on its own. */
|
|
202
218
|
#waitTimer?: NodeJS.Timeout;
|
|
203
219
|
#waitTimeoutMs = 10 * 60_000;
|
|
@@ -210,8 +226,17 @@ export class Supervisor {
|
|
|
210
226
|
#lifecycleTail: Promise<void> = Promise.resolve();
|
|
211
227
|
#pendingEvents: Array<Omit<SupervisorEvent, "seq" | "at">> = [];
|
|
212
228
|
#preemptiveStop?: Promise<void>;
|
|
213
|
-
#deadlineMs =
|
|
214
|
-
#
|
|
229
|
+
#deadlineMs = DEFAULT_DEADLINE_MS;
|
|
230
|
+
#deadlineGraceMs = DEFAULT_DEADLINE_GRACE_MS;
|
|
231
|
+
#deadlineWarningMs = DEFAULT_DEADLINE_WARNING_MS;
|
|
232
|
+
/**
|
|
233
|
+
* Each deadline notice is emitted once per task; the close-out itself is
|
|
234
|
+
* derived from elapsed time. `warningReplayed` records whether the warning's
|
|
235
|
+
* re-ask reached the Decision Worker, or is still owed because a decision
|
|
236
|
+
* was in flight when the warning fired.
|
|
237
|
+
*/
|
|
238
|
+
#deadlineNotices = { approaching: false, reached: false, warningReplayed: false };
|
|
239
|
+
#noOutputTimeoutMs = DEFAULT_NO_OUTPUT_TIMEOUT_MS;
|
|
215
240
|
#noOutputBaselineAt?: number;
|
|
216
241
|
#verificationAbortController?: AbortController;
|
|
217
242
|
#progressPhase?: SupervisorProgressPhase;
|
|
@@ -274,6 +299,8 @@ export class Supervisor {
|
|
|
274
299
|
get candidateParked() { return this.#candidateParked; }
|
|
275
300
|
/** True after the persistent worker was detached from this Supervisor. */
|
|
276
301
|
get released() { return this.#released; }
|
|
302
|
+
/** Wall-clock budget of the active task, or undefined when no deadline is configured. */
|
|
303
|
+
get deadline(): DecisionDeadlineContext | undefined { return this.#deadlineContext(); }
|
|
277
304
|
/** Structural copy of the Worker + Pi-side token/cost accounting for this task. */
|
|
278
305
|
get usage(): SupervisorTokenUsage {
|
|
279
306
|
return {
|
|
@@ -301,6 +328,7 @@ export class Supervisor {
|
|
|
301
328
|
this.#workerOutput = "";
|
|
302
329
|
this.#lastWorkerResult = undefined;
|
|
303
330
|
this.#lastTurnCompleted = undefined;
|
|
331
|
+
this.#pendingDecisionKey = undefined;
|
|
304
332
|
this.#preemptiveStop = undefined;
|
|
305
333
|
this.#automation = (options.automation ?? false) && spec.autonomy.unattended;
|
|
306
334
|
this.#onDecisionSessionProgress = options.onDecisionSessionProgress;
|
|
@@ -327,8 +355,13 @@ export class Supervisor {
|
|
|
327
355
|
this.#repairRound = options.initialRepairRound ?? 0;
|
|
328
356
|
this.#lastFindingSignature = options.initialFindingSignature;
|
|
329
357
|
this.#turn = options.initialTurn ?? 0;
|
|
330
|
-
this.#deadlineMs = options.deadlineMs ??
|
|
331
|
-
|
|
358
|
+
this.#deadlineMs = options.deadlineMs ?? DEFAULT_DEADLINE_MS;
|
|
359
|
+
// The close-out is driven by the Decision Worker; without automation the
|
|
360
|
+
// deadline keeps its plain meaning and the Worker is stopped when it passes.
|
|
361
|
+
this.#deadlineGraceMs = this.#automation ? Math.max(0, options.deadlineGraceMs ?? DEFAULT_DEADLINE_GRACE_MS) : 0;
|
|
362
|
+
this.#deadlineWarningMs = Math.max(0, options.deadlineWarningMs ?? DEFAULT_DEADLINE_WARNING_MS);
|
|
363
|
+
this.#deadlineNotices = { approaching: false, reached: false, warningReplayed: false };
|
|
364
|
+
this.#noOutputTimeoutMs = options.noOutputTimeoutMs ?? DEFAULT_NO_OUTPUT_TIMEOUT_MS;
|
|
332
365
|
this.#waitTimeoutMs = options.waitTimeoutMs ?? 10 * 60_000;
|
|
333
366
|
this.#noOutputBaselineAt = undefined;
|
|
334
367
|
this.#verificationAbortController = undefined;
|
|
@@ -420,8 +453,9 @@ export class Supervisor {
|
|
|
420
453
|
this.#assertStartNotAborted(startAbortController.signal);
|
|
421
454
|
if (this.#automation) {
|
|
422
455
|
const createDecisionWorker: DecisionWorkerFactory = options.decisionWorkerFactory ?? ((decisionOptions) => new PiDecisionWorker(decisionOptions));
|
|
456
|
+
const deadline = this.#deadlineContext();
|
|
423
457
|
this.#decision = createDecisionWorker({
|
|
424
|
-
context: { taskId, task: spec.goal, cwd: options.cwd, state: this.#machine.state, turn: this.#turn, maxTurns: this.#task.maxTurns, repairRound: this.#repairRound, spec },
|
|
458
|
+
context: { taskId, task: spec.goal, cwd: options.cwd, state: this.#machine.state, turn: this.#turn, maxTurns: this.#task.maxTurns, repairRound: this.#repairRound, spec, ...(deadline ? { deadline } : {}) },
|
|
425
459
|
sessionFile: options.decisionSessionFile,
|
|
426
460
|
sessionDir: options.decisionSessionDir ? join(options.decisionSessionDir, taskId) : undefined,
|
|
427
461
|
model: this.#decisionModel,
|
|
@@ -733,8 +767,7 @@ export class Supervisor {
|
|
|
733
767
|
// recorded above so resumeAutomation can replay it once automation resumes.
|
|
734
768
|
const suppressTurnCompletedForHuman = this.#humanRequired && event.type === "turn_completed";
|
|
735
769
|
if (this.#decision && !skipDecisionNotify && !suppressTurnCompletedForHuman && (event.type === "permission_request" || event.type === "turn_completed" || event.type === "exited")) {
|
|
736
|
-
this.#
|
|
737
|
-
this.#decision.notify(event);
|
|
770
|
+
this.#notifyDecision(event);
|
|
738
771
|
}
|
|
739
772
|
this.#handledEvents.add(key);
|
|
740
773
|
this.#deferredWorkerEvents.delete(key);
|
|
@@ -822,6 +855,7 @@ export class Supervisor {
|
|
|
822
855
|
|
|
823
856
|
async #decisionFailure(event: WorkerEvent, error: unknown): Promise<void> {
|
|
824
857
|
return this.#exclusive(async () => {
|
|
858
|
+
this.#settlePendingDecision(event);
|
|
825
859
|
if (this.#releasing || this.#released) {
|
|
826
860
|
await this.#appendDecisionIgnored(event, undefined);
|
|
827
861
|
return;
|
|
@@ -847,6 +881,7 @@ export class Supervisor {
|
|
|
847
881
|
|
|
848
882
|
async #applyDecision(action: DecisionAction, event: WorkerEvent): Promise<void> {
|
|
849
883
|
return this.#exclusive(async () => {
|
|
884
|
+
this.#settlePendingDecision(event);
|
|
850
885
|
const task = this.#task;
|
|
851
886
|
const handle = this.#handle;
|
|
852
887
|
if (this.#releasing || this.#released) {
|
|
@@ -858,9 +893,14 @@ export class Supervisor {
|
|
|
858
893
|
await this.#appendEvent({ type: "decision_deferred", taskId: task.taskId, workerId: handle.id, data: { action: action.action, reason: action.reason, eventType: event.type } }).catch(() => {});
|
|
859
894
|
return;
|
|
860
895
|
}
|
|
896
|
+
// A repeated `wait` for the same event (the wait timer's re-ask, or a
|
|
897
|
+
// deadline-phase replay) must re-arm the timer, so it is never deduped;
|
|
898
|
+
// every other action is applied once per event.
|
|
861
899
|
const actionKey = `${workerEventKey(event)}:${action.action}`;
|
|
862
|
-
if (
|
|
863
|
-
|
|
900
|
+
if (action.action !== "wait") {
|
|
901
|
+
if (this.#handledEvents.has(actionKey)) return;
|
|
902
|
+
this.#handledEvents.add(actionKey);
|
|
903
|
+
}
|
|
864
904
|
await this.#appendEvent({ type: "decision_made", taskId: task.taskId, workerId: handle.id, data: { action: action.action, reason: action.reason, confidence: action.confidence } });
|
|
865
905
|
if (action.action === "allow_permission" || action.action === "deny_permission") {
|
|
866
906
|
if (event.type !== "permission_request" || !this.#adapter.respondPermission) {
|
|
@@ -914,23 +954,33 @@ export class Supervisor {
|
|
|
914
954
|
return;
|
|
915
955
|
}
|
|
916
956
|
if (action.action === "wait") {
|
|
957
|
+
// The deadline warning fired while this decision was in flight, so it
|
|
958
|
+
// was made from a pre-warning view of the clock: ask once more with
|
|
959
|
+
// the current clock before settling for a wait.
|
|
960
|
+
if (this.#deadlineNotices.approaching && !this.#deadlineNotices.warningReplayed && !this.#inCloseOut() && this.#decision?.replay
|
|
961
|
+
&& this.#machine.state === "waiting" && event.type === "turn_completed" && await this.#workerIdle(handle)) {
|
|
962
|
+
this.#deadlineNotices.warningReplayed = true;
|
|
963
|
+
this.#clearWaitTimer();
|
|
964
|
+
this.#notifyDecision(event, true);
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
// Once the deadline has passed there is nothing left to wait for: an
|
|
968
|
+
// idle Worker is verified now, before the close-out window runs out.
|
|
969
|
+
// A Worker that has resumed on its own keeps its turn; its next
|
|
970
|
+
// completed turn is decided under close-out as usual.
|
|
971
|
+
if (this.#inCloseOut() && this.#machine.state === "waiting" && await this.#workerIdle(handle)) {
|
|
972
|
+
await this.#noteDeadlineReached();
|
|
973
|
+
await this.#appendEvent({ type: "decision_overridden", taskId: task.taskId, workerId: handle.id, data: { action: "wait", override: "verify", reason: "task deadline reached; wait is unavailable during close-out", eventType: event.type } });
|
|
974
|
+
await this.#startVerification(handle, event, "deadline close-out");
|
|
975
|
+
return;
|
|
976
|
+
}
|
|
917
977
|
// The Worker will be re-invoked by its own background work; send
|
|
918
978
|
// nothing, but re-ask if it stays silent for the wait timeout.
|
|
919
979
|
this.#armWaitTimer(event);
|
|
920
980
|
return;
|
|
921
981
|
}
|
|
922
982
|
if (action.action === "verify") {
|
|
923
|
-
|
|
924
|
-
this.#machine.transition("verifying");
|
|
925
|
-
await this.#verifyInternal();
|
|
926
|
-
return;
|
|
927
|
-
}
|
|
928
|
-
if (this.#machine.state === "waiting") {
|
|
929
|
-
await this.#adapter.stop(handle, "Decision Worker requested verification");
|
|
930
|
-
await this.#pollInternal(true);
|
|
931
|
-
}
|
|
932
|
-
if (this.#machine.state === "verifying") await this.#verifyInternal();
|
|
933
|
-
else await this.#parkCandidate(`Decision Worker requested verification from state ${this.#machine.state}`, event);
|
|
983
|
+
await this.#startVerification(handle, event, "Decision Worker");
|
|
934
984
|
return;
|
|
935
985
|
}
|
|
936
986
|
if (action.action === "noop") {
|
|
@@ -960,6 +1010,48 @@ export class Supervisor {
|
|
|
960
1010
|
});
|
|
961
1011
|
}
|
|
962
1012
|
|
|
1013
|
+
/**
|
|
1014
|
+
* Move an idle Worker into verification the way a `verify` decision does:
|
|
1015
|
+
* in place when the transport supports it, otherwise by stopping the
|
|
1016
|
+
* Worker first. `origin` names who asked, for the stop reason and the park
|
|
1017
|
+
* message.
|
|
1018
|
+
*/
|
|
1019
|
+
async #startVerification(handle: WorkerHandle, event: WorkerEvent | undefined, origin: string): Promise<void> {
|
|
1020
|
+
if (this.#machine.state === "waiting" && canRepairInPlace(this.#adapter)) {
|
|
1021
|
+
this.#machine.transition("verifying");
|
|
1022
|
+
await this.#verifyInternal();
|
|
1023
|
+
return;
|
|
1024
|
+
}
|
|
1025
|
+
if (this.#machine.state === "waiting") {
|
|
1026
|
+
await this.#adapter.stop(handle, `${origin} requested verification`);
|
|
1027
|
+
await this.#pollInternal(true);
|
|
1028
|
+
}
|
|
1029
|
+
if (this.#machine.state === "verifying") await this.#verifyInternal();
|
|
1030
|
+
else await this.#parkCandidate(`${origin} requested verification from state ${this.#machine.state}`, event);
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
/** Refresh the Decision Worker's context and deliver (or re-deliver) an event; a completed turn is then pending a decision. */
|
|
1034
|
+
#notifyDecision(event: WorkerEvent, replay = false): void {
|
|
1035
|
+
if (!this.#decision) return;
|
|
1036
|
+
// A Decision Worker without replay support gets nothing re-delivered, so
|
|
1037
|
+
// nothing must be marked as pending on its account.
|
|
1038
|
+
if (replay && !this.#decision.replay) return;
|
|
1039
|
+
this.#decision.updateContext(this.#decisionContextPatch());
|
|
1040
|
+
if (event.type === "turn_completed") this.#pendingDecisionKey = workerEventKey(event);
|
|
1041
|
+
if (replay) this.#decision.replay!(event);
|
|
1042
|
+
else this.#decision.notify(event);
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
#settlePendingDecision(event: WorkerEvent): void {
|
|
1046
|
+
if (this.#pendingDecisionKey && this.#pendingDecisionKey === workerEventKey(event)) this.#pendingDecisionKey = undefined;
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
/** True when the adapter reports no active Worker turn; a status failure counts as busy. */
|
|
1050
|
+
async #workerIdle(handle: WorkerHandle): Promise<boolean> {
|
|
1051
|
+
const status = await this.#adapter.getStatus(handle).catch(() => undefined);
|
|
1052
|
+
return Boolean(status) && !status!.activeRequests;
|
|
1053
|
+
}
|
|
1054
|
+
|
|
963
1055
|
/**
|
|
964
1056
|
* A decision about a completed turn is stale once the Worker has started a
|
|
965
1057
|
* new turn on its own (a background agent, task or monitor of its own
|
|
@@ -992,8 +1084,7 @@ export class Supervisor {
|
|
|
992
1084
|
const status = await this.#adapter.getStatus(handle).catch(() => undefined);
|
|
993
1085
|
if (status?.activeRequests) return;
|
|
994
1086
|
await this.#appendEvent({ type: "wait_expired", taskId: this.#task?.taskId, workerId: handle.id, data: { waitTimeoutMs: this.#waitTimeoutMs } }).catch(() => {});
|
|
995
|
-
this.#
|
|
996
|
-
this.#decision.replay?.(event);
|
|
1087
|
+
this.#notifyDecision(event, true);
|
|
997
1088
|
}).catch(() => { /* the watchdog still covers a silent Worker */ });
|
|
998
1089
|
}, this.#waitTimeoutMs);
|
|
999
1090
|
this.#waitTimer.unref?.();
|
|
@@ -1134,8 +1225,7 @@ export class Supervisor {
|
|
|
1134
1225
|
// it has resumed on its own, its next Stop brings a fresh turn.
|
|
1135
1226
|
const status = await this.#adapter.getStatus(this.#handle).catch(() => undefined);
|
|
1136
1227
|
if (status?.activeRequests) return;
|
|
1137
|
-
this.#
|
|
1138
|
-
this.#decision.replay?.(this.#lastTurnCompleted);
|
|
1228
|
+
this.#notifyDecision(this.#lastTurnCompleted, true);
|
|
1139
1229
|
}
|
|
1140
1230
|
});
|
|
1141
1231
|
}
|
|
@@ -1610,6 +1700,13 @@ export class Supervisor {
|
|
|
1610
1700
|
await this.#appendEvent({ type: "candidate_blocked", taskId: task.taskId, workerId: handle.id, data: { reason: `${reason}; Worker transport does not support in-place repair` } });
|
|
1611
1701
|
return false;
|
|
1612
1702
|
}
|
|
1703
|
+
// A repair round that the outright stop would cut short only wastes the
|
|
1704
|
+
// findings: keep them on a blocked candidate instead.
|
|
1705
|
+
const closeOut = this.#deadlineContext();
|
|
1706
|
+
if (closeOut?.closeOut && (closeOut.closeOutRemainingMs ?? 0) < MIN_CLOSE_OUT_REPAIR_MS) {
|
|
1707
|
+
await this.#appendEvent({ type: "candidate_blocked", taskId: task.taskId, workerId: handle.id, data: { reason: `${reason}; the task's close-out window is exhausted (${formatDurationMs(closeOut.closeOutRemainingMs ?? 0)} left), no repair round can finish` } });
|
|
1708
|
+
return false;
|
|
1709
|
+
}
|
|
1613
1710
|
const status = await this.#adapter.getStatus(handle);
|
|
1614
1711
|
if (!status.running || !["running", "waiting", "verifying"].includes(this.#machine.state)) {
|
|
1615
1712
|
await this.#appendEvent({ type: "candidate_blocked", taskId: task.taskId, workerId: handle.id, data: { reason: `${reason}; Worker is no longer available for automatic repair` } });
|
|
@@ -1617,8 +1714,15 @@ export class Supervisor {
|
|
|
1617
1714
|
}
|
|
1618
1715
|
this.#repairRound += 1;
|
|
1619
1716
|
task.repairRound = this.#repairRound;
|
|
1620
|
-
|
|
1621
|
-
|
|
1717
|
+
// After the deadline a repair round races the close-out window: tell the
|
|
1718
|
+
// Worker how long it has so it commits what is complete instead of
|
|
1719
|
+
// starting more work the outright stop would discard.
|
|
1720
|
+
const deadline = this.#deadlineContext();
|
|
1721
|
+
const closeOutHint = deadline?.closeOut && deadline.closeOutRemainingMs !== undefined
|
|
1722
|
+
? ` The task's wall-clock budget is spent: you have about ${formatDurationMs(deadline.closeOutRemainingMs)} before the Supervisor stops this session. Address only what is required above, commit what is complete, and stop.`
|
|
1723
|
+
: "";
|
|
1724
|
+
const instruction = repairInstruction(result, reason, this.#repairRound) + closeOutHint;
|
|
1725
|
+
await this.#appendEvent({ type: "repair_requested", taskId: task.taskId, workerId: handle.id, data: { round: this.#repairRound, reason, instruction, ...(deadline?.closeOut ? { closeOutRemainingMs: deadline.closeOutRemainingMs } : {}) } });
|
|
1622
1726
|
this.#reportProgress("repair", `sending repair round ${this.#repairRound}`, true);
|
|
1623
1727
|
if (this.#machine.state === "verifying") this.#machine.transition("running");
|
|
1624
1728
|
this.#repairSendInProgress = true;
|
|
@@ -1826,7 +1930,7 @@ export class Supervisor {
|
|
|
1826
1930
|
}
|
|
1827
1931
|
this.#reportProgress("worker", `Worker ${this.#machine.state}; heartbeat`, false);
|
|
1828
1932
|
const now = Date.now();
|
|
1829
|
-
const
|
|
1933
|
+
const elapsed = this.#deadlineElapsedMs(now);
|
|
1830
1934
|
// The deadline is cumulative across recovery, but the no-output timer
|
|
1831
1935
|
// starts when this worker process starts. Otherwise a slow Decision Worker
|
|
1832
1936
|
// startup or a recovered task can be stopped on its first watchdog tick
|
|
@@ -1834,12 +1938,24 @@ export class Supervisor {
|
|
|
1834
1938
|
const workerStartedAt = Date.parse(this.#handle.startedAt);
|
|
1835
1939
|
const observedLastOutputAt = status.lastOutputAt ? Date.parse(status.lastOutputAt) : workerStartedAt;
|
|
1836
1940
|
const lastOutputAt = Math.max(observedLastOutputAt, this.#noOutputBaselineAt ?? 0);
|
|
1837
|
-
const
|
|
1941
|
+
const deadlineReached = this.#deadlineMs > 0 && elapsed >= this.#deadlineMs;
|
|
1942
|
+
// The deadline itself opens a close-out window; the outright stop waits
|
|
1943
|
+
// for the grace period as well (an empty grace keeps the old immediate stop).
|
|
1944
|
+
const reason = deadlineReached && elapsed >= this.#deadlineMs + this.#deadlineGraceMs
|
|
1838
1945
|
? "worker deadline exceeded"
|
|
1839
1946
|
: this.#machine.state !== "paused" && this.#noOutputTimeoutMs > 0 && now - lastOutputAt >= this.#noOutputTimeoutMs
|
|
1840
1947
|
? "worker produced no output before timeout"
|
|
1841
1948
|
: undefined;
|
|
1842
|
-
if (!reason)
|
|
1949
|
+
if (!reason) {
|
|
1950
|
+
if (deadlineReached) await this.#enterCloseOut(status, elapsed);
|
|
1951
|
+
else if (this.#deadlineMs > 0 && this.#deadlineWarningMs > 0 && !this.#deadlineNotices.approaching && this.#deadlineMs - elapsed <= this.#deadlineWarningMs) {
|
|
1952
|
+
await this.#warnDeadlineApproaching(status, this.#deadlineMs - elapsed);
|
|
1953
|
+
}
|
|
1954
|
+
return;
|
|
1955
|
+
}
|
|
1956
|
+
// A close-out window that was skipped entirely (a task recovered past its
|
|
1957
|
+
// budget) still leaves the deadline notice ahead of the stop in the log.
|
|
1958
|
+
if (deadlineReached && this.#deadlineGraceMs > 0) await this.#noteDeadlineReached(elapsed).catch(() => {});
|
|
1843
1959
|
let timeoutEventError: unknown;
|
|
1844
1960
|
try {
|
|
1845
1961
|
await this.#appendEvent({ type: "worker_watchdog_timeout", taskId: this.#task.taskId, workerId: this.#handle.id, data: { reason } });
|
|
@@ -1856,6 +1972,121 @@ export class Supervisor {
|
|
|
1856
1972
|
if (timeoutEventError) throw timeoutEventError;
|
|
1857
1973
|
}
|
|
1858
1974
|
|
|
1975
|
+
#deadlineElapsedMs(now = Date.now()): number {
|
|
1976
|
+
return this.#task ? Math.max(0, now - Date.parse(this.#task.startedAt)) : 0;
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
/**
|
|
1980
|
+
* True while the task is in its close-out window: the deadline has passed
|
|
1981
|
+
* and a grace window exists. With no grace there is no close-out, only the
|
|
1982
|
+
* outright stop on the next watchdog tick.
|
|
1983
|
+
*/
|
|
1984
|
+
#inCloseOut(now = Date.now()): boolean {
|
|
1985
|
+
return Boolean(this.#task) && this.#deadlineMs > 0 && this.#deadlineGraceMs > 0 && this.#deadlineElapsedMs(now) >= this.#deadlineMs;
|
|
1986
|
+
}
|
|
1987
|
+
|
|
1988
|
+
#deadlineContext(now = Date.now()): DecisionDeadlineContext | undefined {
|
|
1989
|
+
if (!this.#task || this.#deadlineMs <= 0) return undefined;
|
|
1990
|
+
const elapsed = this.#deadlineElapsedMs(now);
|
|
1991
|
+
const closeOut = this.#deadlineGraceMs > 0 && elapsed >= this.#deadlineMs;
|
|
1992
|
+
return {
|
|
1993
|
+
totalMs: this.#deadlineMs,
|
|
1994
|
+
graceMs: this.#deadlineGraceMs,
|
|
1995
|
+
remainingMs: Math.max(0, this.#deadlineMs - elapsed),
|
|
1996
|
+
closeOut,
|
|
1997
|
+
...(closeOut ? { closeOutRemainingMs: Math.max(0, this.#deadlineMs + this.#deadlineGraceMs - elapsed) } : {}),
|
|
1998
|
+
};
|
|
1999
|
+
}
|
|
2000
|
+
|
|
2001
|
+
/** The per-event context refresh sent to the Decision Worker before every notification or replay. */
|
|
2002
|
+
#decisionContextPatch(): Partial<DecisionContext> {
|
|
2003
|
+
const deadline = this.#deadlineContext();
|
|
2004
|
+
return { state: this.#machine.state, turn: this.#turn, repairRound: this.#repairRound, ...(deadline ? { deadline } : {}) };
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
/**
|
|
2008
|
+
* The deadline is near: record it once, refresh the Decision Worker's view
|
|
2009
|
+
* of the clock and, if the Worker is idle under a `wait`, ask the Decision
|
|
2010
|
+
* Worker again so it can steer the Worker toward a wrap-up while there is
|
|
2011
|
+
* still time to verify the result.
|
|
2012
|
+
*/
|
|
2013
|
+
async #warnDeadlineApproaching(status: WorkerStatus, remainingMs: number): Promise<void> {
|
|
2014
|
+
if (!this.#task || !this.#handle) return;
|
|
2015
|
+
this.#deadlineNotices.approaching = true;
|
|
2016
|
+
// A failed append leaves the event queued for the next flush; the clock
|
|
2017
|
+
// refresh below must happen either way.
|
|
2018
|
+
let appendError: unknown;
|
|
2019
|
+
try {
|
|
2020
|
+
await this.#appendEvent({ type: "worker_deadline_approaching", taskId: this.#task.taskId, workerId: this.#handle.id, data: { deadlineMs: this.#deadlineMs, graceMs: this.#deadlineGraceMs, remainingMs } });
|
|
2021
|
+
} catch (error) {
|
|
2022
|
+
appendError = error;
|
|
2023
|
+
}
|
|
2024
|
+
this.#reportProgress("worker", `deadline in ${formatDurationMs(remainingMs)}${this.#deadlineGraceMs > 0 ? `; close-out window ${formatDurationMs(this.#deadlineGraceMs)}` : ""}`, true);
|
|
2025
|
+
if (!this.#decision) {
|
|
2026
|
+
this.#deadlineNotices.warningReplayed = true;
|
|
2027
|
+
if (appendError) throw appendError;
|
|
2028
|
+
return;
|
|
2029
|
+
}
|
|
2030
|
+
this.#decision.updateContext(this.#decisionContextPatch());
|
|
2031
|
+
// A decision in flight was prompted with the pre-warning clock; the
|
|
2032
|
+
// re-ask is then owed to its `wait`, see #applyDecision. Any other
|
|
2033
|
+
// reason not to re-ask now (human, busy Worker, no turn) is final.
|
|
2034
|
+
if (!this.#pendingDecisionKey && (this.#humanRequired || this.#candidateParked || this.#machine.state !== "waiting" || !this.#lastTurnCompleted || status.activeRequests)) {
|
|
2035
|
+
this.#deadlineNotices.warningReplayed = true;
|
|
2036
|
+
} else if (!this.#pendingDecisionKey) {
|
|
2037
|
+
this.#deadlineNotices.warningReplayed = true;
|
|
2038
|
+
this.#clearWaitTimer();
|
|
2039
|
+
this.#notifyDecision(this.#lastTurnCompleted!, true);
|
|
2040
|
+
}
|
|
2041
|
+
if (appendError) throw appendError;
|
|
2042
|
+
}
|
|
2043
|
+
|
|
2044
|
+
/** Record the deadline once, whichever path notices it first (the watchdog tick or a decision applied under close-out). */
|
|
2045
|
+
async #noteDeadlineReached(elapsed = this.#deadlineElapsedMs()): Promise<void> {
|
|
2046
|
+
if (this.#deadlineNotices.reached || !this.#task || !this.#handle) return;
|
|
2047
|
+
this.#deadlineNotices.reached = true;
|
|
2048
|
+
const closeOutRemainingMs = Math.max(0, this.#deadlineMs + this.#deadlineGraceMs - elapsed);
|
|
2049
|
+
let appendError: unknown;
|
|
2050
|
+
try {
|
|
2051
|
+
await this.#appendEvent({ type: "worker_deadline_reached", taskId: this.#task.taskId, workerId: this.#handle.id, data: { deadlineMs: this.#deadlineMs, graceMs: this.#deadlineGraceMs, elapsedMs: elapsed, closeOutRemainingMs } });
|
|
2052
|
+
} catch (error) {
|
|
2053
|
+
appendError = error;
|
|
2054
|
+
}
|
|
2055
|
+
this.#reportProgress("worker", `deadline reached after ${formatDurationMs(elapsed)}; closing out within ${formatDurationMs(closeOutRemainingMs)}`, true);
|
|
2056
|
+
this.#decision?.updateContext(this.#decisionContextPatch());
|
|
2057
|
+
if (appendError) throw appendError;
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
/**
|
|
2061
|
+
* The deadline has passed. Record it once, then drive the close-out: an
|
|
2062
|
+
* idle automatic Worker is verified now; a busy one is decided under
|
|
2063
|
+
* close-out when its turn completes; the outright stop waits for the grace
|
|
2064
|
+
* window. Without automation the notice is the operator's cue to verify.
|
|
2065
|
+
*/
|
|
2066
|
+
async #enterCloseOut(status: WorkerStatus, elapsed: number): Promise<void> {
|
|
2067
|
+
const task = this.#task;
|
|
2068
|
+
const handle = this.#handle;
|
|
2069
|
+
if (!task || !handle) return;
|
|
2070
|
+
await this.#noteDeadlineReached(elapsed);
|
|
2071
|
+
if (!this.#automation || !this.#decision || this.#humanRequired || this.#candidateParked) return;
|
|
2072
|
+
// An idle Worker that never completed a turn under this Supervisor (an
|
|
2073
|
+
// adopted session, or one recovered past its budget) still reports
|
|
2074
|
+
// `running`: the running -> waiting classification lives in #pollInternal,
|
|
2075
|
+
// which nothing else calls while the Worker is alive.
|
|
2076
|
+
if (this.#machine.state === "running" && !status.activeRequests) await this.#pollInternal();
|
|
2077
|
+
// A decision still in flight for the last turn owns the next step: it is
|
|
2078
|
+
// applied under close-out (a `wait` becomes verify) once it arrives.
|
|
2079
|
+
if (this.#pendingDecisionKey || this.#machine.state !== "waiting" || this.#verificationAbortController || status.activeRequests) return;
|
|
2080
|
+
this.#clearWaitTimer();
|
|
2081
|
+
await this.#appendEvent({ type: "deadline_close_out", taskId: task.taskId, workerId: handle.id, data: { action: "verify", reason: "task deadline reached with an idle Worker" } });
|
|
2082
|
+
try {
|
|
2083
|
+
await this.#startVerification(handle, this.#lastTurnCompleted, "deadline close-out");
|
|
2084
|
+
} catch (error) {
|
|
2085
|
+
// Verification failures are parked inside #verifyInternal; audit the rest.
|
|
2086
|
+
await this.#appendEvent({ type: "worker_event_error", taskId: task.taskId, workerId: handle.id, data: { error: safeMessage(error), eventType: "deadline_close_out" } }).catch(() => {});
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
|
|
1859
2090
|
/**
|
|
1860
2091
|
* Best-effort accounting for a Worker `result` stream-json record; `total_cost_usd`
|
|
1861
2092
|
* is cumulative for the Worker's own process session, so a value smaller than the
|
|
@@ -2104,6 +2335,16 @@ function removeFlagWithValue(args: readonly string[], flag: string): string[] {
|
|
|
2104
2335
|
return result;
|
|
2105
2336
|
}
|
|
2106
2337
|
|
|
2338
|
+
/**
|
|
2339
|
+
* The deadline (in ms since the task started) that grants `extendMs` more
|
|
2340
|
+
* from now: measured from the later of the current deadline and the present,
|
|
2341
|
+
* so extending an expired task by 30 minutes means 30 minutes from now, and
|
|
2342
|
+
* extending by 0 opens its close-out immediately.
|
|
2343
|
+
*/
|
|
2344
|
+
export function extendedDeadlineMs(currentDeadlineMs: number, elapsedMs: number, extendMs: number): number {
|
|
2345
|
+
return Math.max(currentDeadlineMs, elapsedMs) + Math.max(0, extendMs);
|
|
2346
|
+
}
|
|
2347
|
+
|
|
2107
2348
|
function isProtectedBranch(branch: string): boolean {
|
|
2108
2349
|
return /^(?:main|master|trunk|integration|develop)$/iu.test(branch) || /(?:^|\/)(?:main|master|integration)$/iu.test(branch);
|
|
2109
2350
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
-
import { spawn } from "node:child_process";
|
|
3
|
-
import { constants as fsConstants } from "node:fs";
|
|
2
|
+
import { spawn, spawnSync } from "node:child_process";
|
|
3
|
+
import { constants as fsConstants, lstatSync, readdirSync, unlinkSync } from "node:fs";
|
|
4
4
|
import { access, chmod, lstat, mkdir, open, readdir, readFile, realpath, rm, rmdir, stat, truncate, writeFile } from "node:fs/promises";
|
|
5
5
|
import { tmpdir } from "node:os";
|
|
6
6
|
import { delimiter, dirname, isAbsolute, join } from "node:path";
|
|
@@ -798,6 +798,7 @@ export class TmuxWorkerAdapter implements WorkerAdapter {
|
|
|
798
798
|
if (record.owned) {
|
|
799
799
|
await this.#stopGuardian(record).catch((error) => { record.cleanupError ??= asError(error); });
|
|
800
800
|
await this.#run(record, ["kill-server"], undefined, undefined, true).catch(() => {});
|
|
801
|
+
removeDeadTmuxSocket(record.handle.tmuxSocket);
|
|
801
802
|
} else await this.#detachPipe(record);
|
|
802
803
|
}
|
|
803
804
|
const deadline = Date.now() + 25_000;
|
|
@@ -1670,6 +1671,7 @@ export class TmuxWorkerAdapter implements WorkerAdapter {
|
|
|
1670
1671
|
if (isMissingSession(error)) record.serverKilled = true;
|
|
1671
1672
|
else record.cleanupError = asError(error);
|
|
1672
1673
|
}
|
|
1674
|
+
if (record.serverKilled) removeDeadTmuxSocket(record.handle.tmuxSocket);
|
|
1673
1675
|
if (!record.serverKilled) await this.#ensurePaneGone(record);
|
|
1674
1676
|
if (record.cgroupPath) {
|
|
1675
1677
|
try {
|
|
@@ -2281,9 +2283,49 @@ function normalizeForMatch(value: string): string {
|
|
|
2281
2283
|
return value.trim().replace(/\s+/gu, " ");
|
|
2282
2284
|
}
|
|
2283
2285
|
|
|
2284
|
-
/**
|
|
2286
|
+
/**
|
|
2287
|
+
* A tmux server removes its socket when it exits normally, but not when it is
|
|
2288
|
+
* killed with its cgroup; a dead `pi-cs-*.sock` is unlinked once no server
|
|
2289
|
+
* answers on it. `sweepDeadTmuxSockets` does the same for every socket the
|
|
2290
|
+
* adapter's naming scheme left behind in the temp directory.
|
|
2291
|
+
*/
|
|
2292
|
+
function removeDeadTmuxSocket(socketPath: string | undefined): void {
|
|
2293
|
+
if (!socketPath || !/[\\/]pi-cs-[0-9a-f-]+\.sock$/u.test(socketPath)) return;
|
|
2294
|
+
try {
|
|
2295
|
+
const probe = spawnSync("tmux", ["-S", socketPath, "list-sessions"], { stdio: "ignore", timeout: 2_000 });
|
|
2296
|
+
// A server that answers, or a socket something else holds open (the probe
|
|
2297
|
+
// timed out), is left alone.
|
|
2298
|
+
if (probe.status === 0 || probe.signal) return;
|
|
2299
|
+
unlinkSync(socketPath);
|
|
2300
|
+
} catch { /* already gone, or not ours to remove */ }
|
|
2301
|
+
}
|
|
2302
|
+
|
|
2303
|
+
export function sweepDeadTmuxSockets(directory = tmpdir()): number {
|
|
2304
|
+
let removed = 0;
|
|
2305
|
+
let names: string[];
|
|
2306
|
+
try { names = readdirSync(directory); } catch { return 0; }
|
|
2307
|
+
for (const name of names) {
|
|
2308
|
+
if (!/^pi-cs-[0-9a-f-]+\.sock$/u.test(name)) continue;
|
|
2309
|
+
const socketPath = join(directory, name);
|
|
2310
|
+
try {
|
|
2311
|
+
if (!lstatSync(socketPath).isSocket()) continue;
|
|
2312
|
+
const probe = spawnSync("tmux", ["-S", socketPath, "list-sessions"], { stdio: "ignore", timeout: 2_000 });
|
|
2313
|
+
if (probe.status === 0 || probe.signal) continue;
|
|
2314
|
+
unlinkSync(socketPath);
|
|
2315
|
+
removed += 1;
|
|
2316
|
+
} catch { /* skip */ }
|
|
2317
|
+
}
|
|
2318
|
+
return removed;
|
|
2319
|
+
}
|
|
2320
|
+
|
|
2321
|
+
/**
|
|
2322
|
+
* True for a prompt Claude Code injected itself, which a human never typed:
|
|
2323
|
+
* `<task-notification>`, `<system-reminder>`, `<agent-message …>` and the
|
|
2324
|
+
* other hyphenated runtime wrappers it frames delivered content with. A
|
|
2325
|
+
* person's own message does not begin with such a tag.
|
|
2326
|
+
*/
|
|
2285
2327
|
function isClaudeRuntimePrompt(prompt: string): boolean {
|
|
2286
|
-
return /^\s*<(
|
|
2328
|
+
return /^\s*<[a-z]+(?:-[a-z]+)+(?:\s|>)/u.test(prompt);
|
|
2287
2329
|
}
|
|
2288
2330
|
|
|
2289
2331
|
/**
|
|
@@ -2407,7 +2449,10 @@ function isPaneIdentityError(error: unknown): boolean {
|
|
|
2407
2449
|
}
|
|
2408
2450
|
|
|
2409
2451
|
function isMissingSession(error: unknown): boolean {
|
|
2410
|
-
|
|
2452
|
+
// tmux leaves its socket behind on exit ("no server running") unless it was
|
|
2453
|
+
// unlinked, in which case it reports "error connecting … (No such file or
|
|
2454
|
+
// directory)"; both mean the same thing here.
|
|
2455
|
+
return error instanceof Error && /(can't find session|no server running|session not found|failed to connect|error connecting to [^\n]*No such file or directory)/iu.test(error.message);
|
|
2411
2456
|
}
|
|
2412
2457
|
|
|
2413
2458
|
function isMissingFile(error: unknown): boolean {
|