pi-crew 0.11.39 → 0.11.41
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +94 -0
- package/dist/index.mjs +495 -495
- package/package.json +1 -1
- package/skills/real-test-pi-crew/SKILL.md +5 -0
- package/src/runtime/foreground-watchdog.ts +213 -26
- package/src/runtime/process/reap-group.ts +18 -1
- package/src/runtime/process/run-descendant-sweep.ts +24 -2
- package/src/runtime/task-runner/live-executor.ts +146 -45
package/package.json
CHANGED
|
@@ -120,6 +120,7 @@ End-to-end verification discipline for pi-crew changes. Distilled from the broke
|
|
|
120
120
|
| `src/config/` (kể cả migration-validator) | T1 → T11c |
|
|
121
121
|
| `src/runtime/child-pi/` (spawn/kill/steer) | T7 → T9g |
|
|
122
122
|
| `src/runtime/process/**` (reap-group.ts / run-descendant-sweep.ts) + `child-pi-kill.ts` exited-root branch + `orphan-reaper.ts` identity admission + `crash-recovery.ts` terminal-sweep lane + state fields `childPgid/childStartTime/childUid` + heartbeat `pgid/starttime/uid` (wave-11 class: runtime spawn/kill/teardown + startup reconcile + state schema ADDITIVE) | T1 → pinned unit `test/unit/runtime/process/{reap-group,run-descendant-sweep}.test.ts` + `test/unit/runtime/recovery/reconcile-terminal-run-descendant-sweep.test.ts` + `test/unit/runtime/stale-reconciler-run-descendant-sweep.test.ts` + `test/unit/runtime/stale-reconciler-reap-escalation.test.ts` (5 file / 78 tests @ `f578c223`) → **integration live repro** `node scripts/test-runner.mjs test/integration/run-descendant-sweep-live.test.ts` (A1–A5: spawn leak THẬT → sweep reap ~100 ms; **negative control bắt buộc SỐNG**: no-marker / interactive-tty / parent-còn-sống) → off-path proof `PI_CREW_REAP_GROUP=0` + `PI_CREW_REAP_RUN_DESCENDANTS=0` (unit xanh trên CẢ hai trạng thái switch; KHÔNG để ambient `=0` sót lại làm run thật câm) → T9a `team action='doctor' focus='zombies'` (mảng `runDescendants` report-only + dòng cảnh báo no descent proof ⇒ NOT reaped) |
|
|
123
|
+
| `test/**` — bất kỳ fixture nào spawn một process THẬT (đặc biệt `detached: true`), hoặc test fixture rò rỉ process (orphan còn sống sau khi assert throw) | T1 file vừa sửa (`node scripts/test-runner.mjs <file>` ×2 — teardown là timing-sensitive) + `pgrep -af "setInterva[l]"` phải RỖNG sau mỗi lần chạy + (khi đụng helper) `node scripts/test-runner.mjs test/unit/helpers/fixture-process.test.ts`. **Rule (wave 12c, `test/helpers/README.md`)**: fixture DETACHED ⇒ `spawnFixture`+`reapFixture` trong `finally` (pid-only `child.kill()` KHÔNG phải teardown: không chạm tree, vô hiệu trước fixture nuốt SIGTERM, và không chạy nếu throw TRƯỚC `finally`/trong setup hook) + snapshot identity ngay sau spawn (chống pgid reuse) + `registerFixtureTrap()` cho cửa sổ abort sớm; fixture NON-DETACHED ⇒ pid kill + comment lý do (group reap giết runner). Cảnh báo: `spawnFixture` THROW khi `detached:false` — đúng theo thiết kế |
|
|
123
124
|
| `src/runtime/mcp-proxy.ts` / `src/runtime/model/pi-args.ts` (worker argv build + MCP role-gate — G2/G3 surface, SDD-2) | T7 (`argv-flags.smoke.ts` với `PI_CREW_SMOKE=1`) → T9b-W loadout theo role (post-G2: read-only KHÔNG có MCP trên live-session path là ĐÚNG — xem mục full-loadout sanity 9b) + pinned suites `mcp-proxy.test.ts` / `phase6-runtime-hardening.test.ts` / `child-pi-timeout`+`child-pi-surface` (argv/spill) |
|
|
124
125
|
| `src/runtime/async-runner.ts` / `src/runtime/background-runner.ts` (spawn, allowlist, entry-guard, `executeBackgroundRun` core) | T1 → T9b async (event `async.spawned` KHÔNG được có `data.inline:true` — battery phải chứng minh runner DETACHED thật) → T7; đụng `BACKGROUND_RUNNER_ENV_ALLOWLIST` → thêm T10b/T10c (quirk F4-2026-08-30: mux env bị strip ⇒ async headless âm thầm) |
|
|
125
126
|
| `src/runtime/surface/` | T10a → T10b → T10c (dò herdr bằng `HERDR_ENV=1`, KHÔNG phải `$TMUX`) |
|
|
@@ -194,6 +195,8 @@ End-to-end verification discipline for pi-crew changes. Distilled from the broke
|
|
|
194
195
|
|
|
195
196
|
> **2026-10-24 đính chính (wave 11c — battery findings F1+F2, mục 4b/7b của `docs/real-test/reports/real-test-2026-10-24-wave11-battery.md`)**: (1) **F2 sync-run miss ĐÃ FIX** — run sync/foreground không có `manifest.async.pid` nên leak mang `PI_CREW_PARENT_PID=<session pi pid>` không bao giờ qua descent proof (đúng rule 11 — an toàn nhưng bỏ sót vĩnh viễn). Từ 11c mọi manifest mới record `sessionPid` (= `process.pid` tại `createRunManifest`); `sweepTerminalRunDescendants` + doctor dry-run đưa nó vào `proofParents` với guard dead-only như mọi parent khác — session còn sống ⇒ KHÔNG là proof (user có thể đang dùng). Run async vẫn dùng `async.pid` primary. Battery addendum sẽ chứng minh bằng E2E sync-leak thật. (2) **F1 test-isolation ĐÃ FIX** — suite `run-descendant-sweep.test.ts` giờ pin `PI_CREW_REAP_RUN_DESCENDANTS=1` trong beforeEach (restore verbatim afterEach, pattern của `stale-reconciler-run-descendant-sweep`): 17/78 test không đỏ nữa khi ambient `=0`; pinned 78→82 (a20 + g8–g10); decision-table yêu cầu "unit xanh trên CẢ HAI trạng thái switch" giữ nguyên.
|
|
196
197
|
|
|
198
|
+
> **2026-10-24 update (wave 12 — test-fixture teardown, commits `75542295`→`a97714d2` + wave-12c F4/F6; spec `fixwave12-fixture-teardown-2026-10-24/SPEC.md`)**: lớp leak THỨ HAI, khác wave 11: 2 process `node -e "process.on('SIGTERM',()=>{}); setInterval(...)"` sống >6 h sau khi run `team_20261024024905_4cf882558c7c2b6d` completed — **test fixture**, không phải worker descendant: spawn `detached:true` (group riêng ⇒ group kill của runner không tới được) + teardown pid-only `child.kill()` (vô hiệu trước fixture nuốt SIGTERM) + không handle ⇒ không ai reap. Wave-11 sweep từ chối reap là **hành vi ĐÚNG** (không proof descent). (1) **Surface mới `test/helpers/fixture-process.ts`** — `spawnFixture` (THROW khi `detached:false`: child non-detached chia sẻ group của RUNNER, group kill sẽ giết runner + mọi file sibling; snapshot `readProcIdentity` NGAY sau spawn, normalise pgid=pid), `reapFixture` (group reap + escalation SIGTERM→SIGKILL, refusal chứng minh pid lạ ⇒ KHÔNG signal gì, pid-fallback cho non-Linux, không bao giờ throw), `registerFixtureTrap` (`after()` + `process.on("exit")` sync flush cho cửa sổ abort sớm). Rule viết 1 lần: `test/helpers/README.md`. (2) **GAP-1 (HIGH, wave 12c-F3)** — `background-runner-sigint-cleanup`: grandchild detached của harness giờ được reap theo group trong `finally` (harness giữ pid-only ĐÚNG — non-detached). (3) **GAP-2/GAP-3 (wave 12c-F3)** — fake `rg` tự ghi pid + reap group; `inference-broker` giữ `ChildProcess`, spawn detached + `reapFixture`/trap. (4) **GAP-4 (wave 12c-F4)** — `reap-group.test.ts`: teardown qua `reapTree` (leader-snapshot + group reap **TRƯỚC**, rồi pid kills idempotent) và `spawnTree()` tự reap group trên đường FAIL (timeout 15 s / spawn error / JSON hỏng) — trước đó leader biết, kids không ⇒ mồ côi toàn bộ. GAP-5 sweep: `guest-zombie` (detached thật) chuyển sang helper; 4 file còn lại là NON-detached ⇒ giữ pid kill + comment `// non-detached: pid-only by design (shares runner's group)`. (5) **Wave-12a bug tự gây** — thêm `test/unit/helpers/` mà không đăng ký shard weight ⇒ 4 test shard-partition fail (`ebe0138c`). (6) Battery cho wave này: T1 từng file đã sửa ×2 + `pgrep -af "setInterva[l]"` RỖNG + `test:unit` không tăng fail so baseline. (7) **Wave 12d lesson** — teardown test có async writer (`appendEventAsync` → `fs.promises.mkdir`) phải dùng `removeDirWithRetry` (`test/helpers/rm-retry.ts`: settle 250 ms + retry EBUSY/EPERM/ENOTEMPTY); `rmSync` trần ⇒ `ENOTEMPTY` dưới load vì writer tái tạo `runs/<runId>/` sau khi dir đã bị xoá.
|
|
199
|
+
|
|
197
200
|
## Core principle: disk ≠ live Pi
|
|
198
201
|
|
|
199
202
|
Two locations hold pi-crew state:
|
|
@@ -1264,6 +1267,7 @@ When a tier fails, the recovery is usually quick. Match the symptom to the cause
|
|
|
1264
1267
|
| `team action='schedule' interval=<ms>` always says `Invalid schedule "<N>ms". Use "5m", …` | handle-schedule builds `${interval}ms`; both interval parsers lacked an `ms` unit — every numeric interval died. Fixed 2026-09-21 | The suggested "5m" formats only work via `cron`/`once` params, not `interval` (interval is schema-typed as a plain ms number) |
|
|
1265
1268
|
| Runs keep appearing under one scheduled job; children of YOUR pi process; machine thrashing | **Interval ≥ 2^31-1 ms (≥24.8 days) overflowed the Node timer to a 1ms hot loop** (fixed 2026-09-22 — clamped hops). Check `.crew/audit/prune.jsonl` for interleaved auto-prunes. Contain: remove job, then `pkill -f '[b]ackground-runner.ts --cwd <root>'` | Any new timer scheduling must clamp below MAX_TIMER_DELAY_MS — grep for `setInterval(`/`setTimeout(` with computed delays |
|
|
1266
1269
|
| Live sweep/reap probe chạy "xanh" nhưng KHÔNG process nào bị reap (doctor `runDescendants` liệt kê mà không có gì chết) | **Ambient `PI_CREW_REAP_GROUP=0` / `PI_CREW_REAP_RUN_DESCENDANTS=0` còn trong shell** (sót từ probe off-path trước) — cả 2 switch default ON và được FORWARD sang background runner, ambient là đủ để câm sạch cả sweep lẫn group-reap; ngược lại quên set khi test off-path thì nhánh degrade không được chứng minh | Trước MỌI live repro: `env \| grep PI_CREW_REAP` phải RỖNG; chỉ set `=0` CHỦ ĐÍCH khi chạy nhánh off-path (unit phải xanh trên CẢ hai trạng thái switch), unset ngay sau (wave 11 lesson — 2 kill switch KHÔNG được set ngoài ý muốn theo cả 2 hướng) |
|
|
1270
|
+
| Process MỒ CÔI còn sống mang marker `PI_CREW_KIND=subagent` / `PI_CREW_TASK_ID` (hoặc đúng shape `node -e "setInterval(...)"`) trong khi run đã `completed` từ lâu, parent đã chết và process đã re-parent sang systemd | **Test-fixture teardown, KHÔNG phải runtime bug** (wave 12 incident: 2 fixture sống >6 h): fixture spawn DETACHED (group riêng ⇒ group kill của test runner không tới) + teardown pid-only `child.kill()` (vô hiệu trước fixture nuốt SIGTERM) + không ai giữ handle. Wave-11 sweep TỪ CHỐI reap là ĐÚNG — không chứng minh được descent của một run đã terminal | Kiểm `test/helpers/README.md`: detached ⇒ `spawnFixture`+`reapFixture` trong `finally`; non-detached ⇒ pid kill + comment. Grep suite vừa chạy tìm `detached: true` + `finally { … child.kill(` = thủ phạm; sửa teardown rồi `pgrep -af "setInterva[l]"` phải RỖNG. Đừng "fix" bằng cách nới wave-11 sweep cho process không proof descent |
|
|
1267
1271
|
|
|
1268
1272
|
## Release gate — battery → push → GitHub Actions GREEN → publish (canonical, 2026-10-14)
|
|
1269
1273
|
|
|
@@ -1442,6 +1446,7 @@ Use this to answer "đủ full tính năng chưa?" without re-deriving. Every us
|
|
|
1442
1446
|
| Steer delivery incl. boot window (pre-bind retry) | `src/prompt/prompt-runtime.ts` (poll + `createSeenSteerIdSet.unmark`), `src/runtime/child-pi/child-pi-spawn.ts:283` (env), `src/runtime/task-runner/child-executor.ts:562` (filename) | `prompt-runtime-steer-boot-window.test.ts` (4) + **Tier 9g** live (watcher at +0.5s; lab variant with `PI_CREW_STEERING_FILE` + `pi --extension <repo>/src/prompt/prompt-runtime.ts`) |
|
|
1443
1447
|
| Worker process-GROUP reaping (wave 11a F5 — đường group-signal DUY NHẤT) | `src/runtime/process/reap-group.ts` (`reapGroup` / `assertSafeGroup` guard-ladder 8 rung + escalation SIGTERM→poll→SIGKILL + `flushReapEscalationsSync`), wired `child-pi-kill.ts` exited-root (snapshot pgid/starttime trước khi root chết) + `orphan-reaper.ts` identity-proven admission | unit `node scripts/test-runner.mjs test/unit/runtime/process/reap-group.test.ts`; kill switch `PI_CREW_REAP_GROUP=0` (default ON — degrade về pid-only pre-wave; được FORWARD sang background runner); live: spawn worker có group con → kill root → group bị reap; **negative control bắt buộc: process cùng uid nhưng ngoài ladder (không leader-proof / không snapshot khớp) phải SỐNG** |
|
|
1444
1448
|
| Run-descendant sweep (wave 11b F7 — reap leak sống sót SAU run terminal) | `src/runtime/process/run-descendant-sweep.ts` (`collectRunDescendants`/`reapRunDescendants`, 1 snapshot `/proc`/pass) + `crash-recovery.ts` `selectUnsweptTerminalRunIds` lane (terminal ≥ 60 s, oldest-first, cap 25/pass, window 200, retry `descendantSweepPending` 10 phút) | unit sweep suites (xem decision-table row wave 11) + **integration live repro** `node scripts/test-runner.mjs test/integration/run-descendant-sweep-live.test.ts` (A1–A5) + report `docs/real-test/reports/real-test-2026-10-24-wave11-descendant-reaping.md`; kill switch `PI_CREW_REAP_RUN_DESCENDANTS=0` (no-op hoàn toàn); marker ladder `PI_CREW_BROKER_RUN_ID` (nhánh run-id cần recorded parent ĐÃ CHẾT) → `PI_CREW_KIND=subagent`+`PI_CREW_TASK_ID` → `PI_CREW_PARENT_PID`; **negative control bắt buộc: no-marker / interactive-tty / parent-sống phải SỐNG** |
|
|
1449
|
+
| Fixture test spawn/reap (wave 12 — `test/helpers/`): đường DUY NHẤT một test được spawn + reap một process tree | `test/helpers/fixture-process.ts` (`spawnFixture` — từ chối `detached:false`, snapshot identity ngay sau spawn; `reapFixture` — group reap qua `reapGroup` + escalation SIGTERM→SIGKILL, pid-fallback cho non-Linux, KHÔNG BAO GIỜ throw; `registerFixtureTrap` — `after()` + `process.on("exit")` flush) + rule viết 1 lần ở `test/helpers/README.md` | unit `test/unit/helpers/fixture-process.test.ts` (h1–h5: refuse non-detached, reap tree nuốt SIGTERM < 2 s, idempotent, mismatch ⇒ không signal, trap 1 lần); regression theo file: `pgrep -af "setInterva[l]"` RỖNG sau mỗi lần chạy; GAP-4 pin `reap-group.test.ts` (teardown `reapTree` = leader-snapshot + group reap TRƯỚC, kể cả đường `spawnTree()` fail giữa đường) |
|
|
1445
1450
|
|
|
1446
1451
|
---
|
|
1447
1452
|
|
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
* assistant has no way to know when a run completes or gets stuck without
|
|
7
7
|
* manual polling. This watchdog monitors active runs and:
|
|
8
8
|
*
|
|
9
|
-
* 1. Detects hung runs (active status,
|
|
9
|
+
* 1. Detects hung runs (active status, manifest not updated for >2 min with
|
|
10
|
+
* no agent activity evidence — wave 13: a live registry claim or a fresh
|
|
11
|
+
* task heartbeat withholds the verdict, see CLAIMED_RUN_HUNG_MS)
|
|
10
12
|
* 2. Injects a followUp message via pi.sendUserMessage() so the assistant
|
|
11
13
|
* is automatically notified — no manual sleep+check needed.
|
|
12
14
|
* 3. Cleans up after itself when the run completes or the session ends.
|
|
@@ -19,6 +21,7 @@
|
|
|
19
21
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
22
|
import { getCrewEnv, getCrewEnvInt } from "../config/env-vars.ts";
|
|
21
23
|
import { TEAM_TERMINAL_TASK_STATUSES } from "../state/contracts.ts";
|
|
24
|
+
import { activeRunEntries } from "../state/stores/active-run-registry.ts";
|
|
22
25
|
import { loadRunManifestById } from "../state/stores/state-store.ts";
|
|
23
26
|
import type { TeamTaskState } from "../state/types.ts";
|
|
24
27
|
import { readCrewAgents } from "./crew-agent-records.ts";
|
|
@@ -128,6 +131,170 @@ function resolveCheckIntervalMs(overrideMs: number | undefined): number {
|
|
|
128
131
|
return DEFAULT_CHECK_INTERVAL_MS;
|
|
129
132
|
}
|
|
130
133
|
|
|
134
|
+
// ── Wave 13 (F1, GAP-0): hung-verdict liveness short-circuit ────────────────
|
|
135
|
+
//
|
|
136
|
+
// Live evidence (2026-10-24/25): five false "appears hung" alerts in one
|
|
137
|
+
// session, all direct single-agent runs executing ONE long tool call (a
|
|
138
|
+
// 20-min CI poll loop, a 20-min test suite). The hung verdict
|
|
139
|
+
// (isLikelyOrphanedActiveRun: manifest stale ≥2 min + no agent activity
|
|
140
|
+
// within 5 min) is structurally blind while a tool executes — every
|
|
141
|
+
// liveness write is event-driven and no pi events fire mid-tool — so a
|
|
142
|
+
// HEALTHY run reads as hung ~5-10 min into a long tool. The two signals
|
|
143
|
+
// other consumers already trust, and this verdict ignores, are consulted
|
|
144
|
+
// here IN THE WATCHDOG (never inside isLikelyOrphanedActiveRun — that
|
|
145
|
+
// predicate is shared with the dashboard's isDisplayActiveRun, GAP-4):
|
|
146
|
+
// (a) a LIVE active-run-registry claim, and
|
|
147
|
+
// (b) a fresh task heartbeat (kept fresh through long tools by F2's
|
|
148
|
+
// live-executor timer).
|
|
149
|
+
// The two branches are INDEPENDENT ORs: the registry drops non-async
|
|
150
|
+
// entries once the manifest is >30 min stale, so past that horizon only the
|
|
151
|
+
// heartbeat carries liveness — never require both.
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Wave 13 (F1, GAP-0): hard ceiling for how long a LIVE active-run-registry
|
|
155
|
+
* claim can shield a run from the watchdog's hung verdict. A claimed run
|
|
156
|
+
* whose NEWEST liveness evidence (claim assertion time, task heartbeats/
|
|
157
|
+
* activity) is older than this still alerts — a genuine in-process deadlock
|
|
158
|
+
* (claim held, nothing progressing) is still caught — while healthy claimed
|
|
159
|
+
* runs mid-long-tool are not false-alarmed by the 2-min ladder. 45 min is
|
|
160
|
+
* 1.5× the registry's own 30-min freshness horizon and 9× the 5-min activity
|
|
161
|
+
* shield, so only a truly stuck claim trips it. Unclaimed runs keep the
|
|
162
|
+
* fast 2-min ladder unchanged; ORPHANED_ACTIVE_RUN_MS (process-status.ts,
|
|
163
|
+
* shared with the dashboard — GAP-4) is NOT touched.
|
|
164
|
+
*/
|
|
165
|
+
export const CLAIMED_RUN_HUNG_MS = 45 * 60 * 1000;
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Wave 13 (F1): freshness window for task-heartbeat liveness evidence —
|
|
169
|
+
* mirrors ACTIVE_EVIDENCE_TTL_MS in stale-reconciler.ts (5 min).
|
|
170
|
+
*/
|
|
171
|
+
const TASK_HEARTBEAT_EVIDENCE_MS = 5 * 60 * 1000;
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Wave 13 (F1): read-only active-run-registry claim consult result.
|
|
175
|
+
* `claimedAtMs` is the registry entry's `updatedAt` (the moment the claim
|
|
176
|
+
* was asserted — registerActiveRun fires at dispatch); undefined when the
|
|
177
|
+
* entry carries no parseable timestamp.
|
|
178
|
+
*/
|
|
179
|
+
interface LiveRunClaim {
|
|
180
|
+
claimedAtMs?: number;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Wave 13 (F1, GAP-0): consult the active-run registry for a LIVE claim on
|
|
185
|
+
* `runId`. Read-only mirror of findLiveRegistryEntry (stale-reconciler G24):
|
|
186
|
+
* activeRunEntries() applies the full liveness filter itself (terminal
|
|
187
|
+
* status, dead async PID, >30-min-stale non-async manifests, symlink
|
|
188
|
+
* safety), so an entry found here means some live session/runner claims the
|
|
189
|
+
* run RIGHT NOW. Inlined via the registry store instead of importing
|
|
190
|
+
* stale-reconciler (its reaping graph must not enter the watchdog's import
|
|
191
|
+
* closure); the async-PID half of hasLiveRunClaim is deliberately omitted —
|
|
192
|
+
* async runs never reach this verdict path (isLikelyOrphanedActiveRun
|
|
193
|
+
* excludes them). Fail-open: any registry error ⇒ no claim — a broken
|
|
194
|
+
* registry must never shield a genuinely hung run (G24 contract).
|
|
195
|
+
*/
|
|
196
|
+
function consultActiveRunRegistry(runId: string): LiveRunClaim | undefined {
|
|
197
|
+
try {
|
|
198
|
+
const entry = activeRunEntries().find((item) => item.runId === runId);
|
|
199
|
+
if (!entry) return undefined;
|
|
200
|
+
const claimedAtMs = entry.updatedAt ? Date.parse(entry.updatedAt) : Number.NaN;
|
|
201
|
+
return { claimedAtMs: Number.isFinite(claimedAtMs) ? claimedAtMs : undefined };
|
|
202
|
+
} catch {
|
|
203
|
+
return undefined;
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
let liveClaimConsult: (runId: string) => LiveRunClaim | undefined = consultActiveRunRegistry;
|
|
208
|
+
|
|
209
|
+
/** @internal Test seam — replace the registry claim consult (null = real). */
|
|
210
|
+
export function setLiveClaimConsultForTests(consult: ((runId: string) => LiveRunClaim | undefined) | null): void {
|
|
211
|
+
liveClaimConsult = consult ?? consultActiveRunRegistry;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/**
|
|
215
|
+
* Wave 13 (F1): does any running/waiting task carry FRESH live evidence?
|
|
216
|
+
* Behavioral mirror of hasRecentActiveEvidence (stale-reconciler.ts): a
|
|
217
|
+
* running/waiting task whose `heartbeat.lastSeenAt` (with `alive !== false`)
|
|
218
|
+
* or `agentProgress.lastActivityAt` is within TASK_HEARTBEAT_EVIDENCE_MS
|
|
219
|
+
* counts. Absent/unreadable tasks (`[]`) ⇒ false — fail-open to the ladder,
|
|
220
|
+
* same as WD-STALE-RECORD.
|
|
221
|
+
*/
|
|
222
|
+
function hasFreshTaskHeartbeatEvidence(tasks: TeamTaskState[] | undefined, now: number): boolean {
|
|
223
|
+
if (!Array.isArray(tasks)) return false;
|
|
224
|
+
return tasks.some((task) => {
|
|
225
|
+
if (task.status !== "running" && task.status !== "waiting") return false;
|
|
226
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
227
|
+
if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= TASK_HEARTBEAT_EVIDENCE_MS) return true;
|
|
228
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
229
|
+
return Number.isFinite(activityAt) && now - activityAt <= TASK_HEARTBEAT_EVIDENCE_MS;
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Wave 13 (F1): newest liveness-evidence timestamp (epoch ms) among the
|
|
235
|
+
* claim assertion time and the running/waiting tasks' heartbeat/activity
|
|
236
|
+
* marks — everything EXCEPT manifest.updatedAt, which is the frozen
|
|
237
|
+
* artifact under judgment, not evidence (the registry's own filter already
|
|
238
|
+
* bounds it at 30 min for entry presence; counting it here would make the
|
|
239
|
+
* CLAIMED_RUN_HUNG_MS ceiling unreachable). undefined when no signal
|
|
240
|
+
* carries a finite timestamp.
|
|
241
|
+
*/
|
|
242
|
+
function newestClaimEvidenceMs(tasks: TeamTaskState[] | undefined, claim: LiveRunClaim): number | undefined {
|
|
243
|
+
let newest: number | undefined;
|
|
244
|
+
const consider = (ms: number): void => {
|
|
245
|
+
if (Number.isFinite(ms) && (newest === undefined || ms > newest)) newest = ms;
|
|
246
|
+
};
|
|
247
|
+
if (claim.claimedAtMs !== undefined) consider(claim.claimedAtMs);
|
|
248
|
+
for (const task of tasks ?? []) {
|
|
249
|
+
if (task.status !== "running" && task.status !== "waiting") continue;
|
|
250
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
251
|
+
consider(heartbeatAt);
|
|
252
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
253
|
+
consider(activityAt);
|
|
254
|
+
}
|
|
255
|
+
return newest;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Wave 13 (F1): hung-verdict liveness short-circuit result. */
|
|
259
|
+
interface LivenessShortCircuit {
|
|
260
|
+
/** True ⇒ the hung verdict is withheld this cycle (live evidence present). */
|
|
261
|
+
suppress: boolean;
|
|
262
|
+
/** Debug-log reason (PI_CREW_DEBUG-gated). */
|
|
263
|
+
reason: string;
|
|
264
|
+
/** Claim present but its evidence aged past CLAIMED_RUN_HUNG_MS — the alert detail must name it. */
|
|
265
|
+
claimExhausted: boolean;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Wave 13 (F1, GAP-0): should the hung verdict for this cycle be withheld?
|
|
270
|
+
* Two INDEPENDENT OR branches (never require both):
|
|
271
|
+
* (b) a fresh task heartbeat (≤5 min) ⇒ suppress outright;
|
|
272
|
+
* (a) a live registry claim ⇒ suppress while the newest liveness evidence
|
|
273
|
+
* is ≤ CLAIMED_RUN_HUNG_MS old; past the ceiling the ladder proceeds
|
|
274
|
+
* (claimExhausted ⇒ the alert names the claimed-run threshold).
|
|
275
|
+
* No claim and no fresh heartbeat ⇒ the existing 2-min ladder is unchanged
|
|
276
|
+
* (an unclaimed zombie still alerts fast).
|
|
277
|
+
*/
|
|
278
|
+
function assessClaimedRunLiveness(runId: string, tasks: TeamTaskState[] | undefined, now: number): LivenessShortCircuit {
|
|
279
|
+
if (hasFreshTaskHeartbeatEvidence(tasks, now)) {
|
|
280
|
+
return { suppress: true, reason: "fresh task heartbeat", claimExhausted: false };
|
|
281
|
+
}
|
|
282
|
+
const claim = liveClaimConsult(runId);
|
|
283
|
+
if (claim) {
|
|
284
|
+
const evidenceMs = newestClaimEvidenceMs(tasks, claim);
|
|
285
|
+
const evidenceAgeMs = evidenceMs !== undefined ? now - evidenceMs : undefined;
|
|
286
|
+
if (evidenceAgeMs !== undefined && evidenceAgeMs <= CLAIMED_RUN_HUNG_MS) {
|
|
287
|
+
return {
|
|
288
|
+
suppress: true,
|
|
289
|
+
reason: `live registry claim (newest evidence ${Math.max(0, Math.round(evidenceAgeMs / 1000))}s old)`,
|
|
290
|
+
claimExhausted: false,
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
return { suppress: false, reason: "claim evidence older than CLAIMED_RUN_HUNG_MS", claimExhausted: true };
|
|
294
|
+
}
|
|
295
|
+
return { suppress: false, reason: "no live claim", claimExhausted: false };
|
|
296
|
+
}
|
|
297
|
+
|
|
131
298
|
/**
|
|
132
299
|
* Pre-emit state re-verify loader. Injectable for tests to simulate the run
|
|
133
300
|
* going terminal between the cycle's initial read and the emit point.
|
|
@@ -471,39 +638,59 @@ export function startForegroundWatchdog(opts: WatchdogOptions): void {
|
|
|
471
638
|
// Keep monitoring: if the manifest eventually finalizes, the
|
|
472
639
|
// terminal branch delivers the completion notice.
|
|
473
640
|
} else {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
641
|
+
// Wave 13 (F1, GAP-0): liveness short-circuit BEFORE any hung alert.
|
|
642
|
+
// The verdict above is structurally blind while a tool executes; a
|
|
643
|
+
// live registry claim or fresh task heartbeat withholds it. A
|
|
644
|
+
// suppressed cycle means the run left the hung state — reset the
|
|
645
|
+
// no-progress counter so a later genuine alert gets the full drip cap.
|
|
646
|
+
const liveness = assessClaimedRunLiveness(runId, reverified.tasks, now);
|
|
647
|
+
if (liveness.suppress) {
|
|
648
|
+
consecutiveHungNotices = 0;
|
|
649
|
+
debugLog(`[foreground-watchdog] suppress hung verdict for ${runId}: ${liveness.reason}`);
|
|
650
|
+
} else {
|
|
651
|
+
// Wave 13 (F3, GAP-2): the detail carries its EVIDENCE time (the
|
|
652
|
+
// manifest read the verdict was based on) — "as of <ISO>" — not the
|
|
653
|
+
// emit time, so a user re-checking can tell a stale/queued alert
|
|
654
|
+
// apart from one about the last few minutes. An exhausted claim
|
|
655
|
+
// names the claimed-run threshold so the user knows it is not the
|
|
656
|
+
// 2-min ladder.
|
|
657
|
+
const claimNote = liveness.claimExhausted
|
|
658
|
+
? `, claimed-run hung threshold exceeded (no live evidence for >${Math.round(CLAIMED_RUN_HUNG_MS / 60_000)} min)`
|
|
659
|
+
: "";
|
|
660
|
+
const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}, as of ${manifest.updatedAt}${claimNote}`;
|
|
661
|
+
consecutiveHungNotices += 1;
|
|
662
|
+
try {
|
|
663
|
+
if (consecutiveHungNotices <= HUNG_NOTICE_CAP) {
|
|
664
|
+
// WD-DEDUP: alert-once per runId at module scope — even a
|
|
665
|
+
// fresh watchdog instance must not re-alert the same run.
|
|
666
|
+
if (!hungAlertAlreadySent(runId)) {
|
|
667
|
+
// Mark BEFORE queueing so a racing cycle cannot double-queue.
|
|
668
|
+
updateHungAlertRecord(runId, { alerted: true });
|
|
669
|
+
// WD-DELIVER-GUARD: the host queues a followUp while the
|
|
670
|
+
// session streams and delivers it arbitrarily later — so
|
|
671
|
+
// hand it over only at the idle boundary, re-verifying the
|
|
672
|
+
// run state at THAT moment.
|
|
673
|
+
queueHungAlertDelivery(
|
|
674
|
+
pi,
|
|
675
|
+
cwd,
|
|
676
|
+
runId,
|
|
677
|
+
`pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
|
|
678
|
+
opts.isSessionIdle,
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
} else if (consecutiveHungNotices === HUNG_NOTICE_CAP + 1) {
|
|
682
|
+
// One-time handoff, then silence — still monitoring, no more drips.
|
|
487
683
|
queueHungAlertDelivery(
|
|
488
684
|
pi,
|
|
489
685
|
cwd,
|
|
490
686
|
runId,
|
|
491
|
-
`pi-crew watchdog: run ${runId}
|
|
687
|
+
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
492
688
|
opts.isSessionIdle,
|
|
493
689
|
);
|
|
494
690
|
}
|
|
495
|
-
}
|
|
496
|
-
|
|
497
|
-
queueHungAlertDelivery(
|
|
498
|
-
pi,
|
|
499
|
-
cwd,
|
|
500
|
-
runId,
|
|
501
|
-
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
502
|
-
opts.isSessionIdle,
|
|
503
|
-
);
|
|
691
|
+
} catch {
|
|
692
|
+
/* non-critical */
|
|
504
693
|
}
|
|
505
|
-
} catch {
|
|
506
|
-
/* non-critical */
|
|
507
694
|
}
|
|
508
695
|
}
|
|
509
696
|
// Keep monitoring (ARCH-5: silently past the cap — no more notices).
|
|
@@ -112,6 +112,18 @@ export interface ReapGroupOptions {
|
|
|
112
112
|
expected?: ProcIdentity;
|
|
113
113
|
/** Diagnostic label used in log messages. */
|
|
114
114
|
name?: string;
|
|
115
|
+
/**
|
|
116
|
+
* uid source for the same-uid rungs (5 and the dead-path uid proof). Default
|
|
117
|
+
* `() => process.getuid?.()` — production behaviour is unchanged. The seam
|
|
118
|
+
* exists because `process.getuid` is undefined on win32, which made the uid
|
|
119
|
+
* rungs untestable there without a process-wide monkeypatch.
|
|
120
|
+
*
|
|
121
|
+
* Read PER CALL and never cached: unlike `selfIdentity()` (which is cached
|
|
122
|
+
* because a process cannot change its group), this is an injected value with
|
|
123
|
+
* no invariant of its own, and caching it would make the first caller win for
|
|
124
|
+
* the lifetime of the module.
|
|
125
|
+
*/
|
|
126
|
+
getuid?: () => number | undefined;
|
|
115
127
|
}
|
|
116
128
|
|
|
117
129
|
/** Sync verdict + the authoritative async verdict. */
|
|
@@ -226,6 +238,11 @@ export function __setSelfIdentityReaderForTest(reader: ((pid: number) => ProcIde
|
|
|
226
238
|
selfIdentityReader = reader ?? readProcIdentity;
|
|
227
239
|
}
|
|
228
240
|
|
|
241
|
+
/** Default uid source for the same-uid rungs: the real process uid, when the
|
|
242
|
+
* runtime has one (POSIX). win32 has none ⇒ `undefined` ⇒ the rungs degrade
|
|
243
|
+
* exactly as they did before the seam existed. */
|
|
244
|
+
const defaultGetuid = (): number | undefined => process.getuid?.();
|
|
245
|
+
|
|
229
246
|
/**
|
|
230
247
|
* Run the refusal ladder. Returns the FIRST failing rung, or undefined when the
|
|
231
248
|
* group is safe to signal. Pure with respect to injected `readIdentity` /
|
|
@@ -251,7 +268,7 @@ export function assertSafeGroup(pgid: number, candidatePid: number, opts: ReapGr
|
|
|
251
268
|
// state in which `kill(-pgid)` could take out the host session.
|
|
252
269
|
if (self.pgid === undefined) return "identity-missing";
|
|
253
270
|
if (pgid === process.pid || pgid === process.ppid || pgid === self.pgid) return "pgid-init-or-self";
|
|
254
|
-
const selfUid =
|
|
271
|
+
const selfUid = (opts.getuid ?? defaultGetuid)();
|
|
255
272
|
// Rung 4: a LIVE candidate identity, or (dead candidate) leader-proof.
|
|
256
273
|
const live = readIdentity(candidatePid);
|
|
257
274
|
if (!live) {
|
|
@@ -232,6 +232,18 @@ export interface SweepOptions {
|
|
|
232
232
|
* and the doctor path are unchanged.
|
|
233
233
|
*/
|
|
234
234
|
snapshot?: ProcSnapshot;
|
|
235
|
+
/**
|
|
236
|
+
* uid source for the same-uid rung (safety rung 3). Default
|
|
237
|
+
* `() => process.getuid?.()` — production behaviour is unchanged. The seam
|
|
238
|
+
* exists because `process.getuid` is undefined on win32, which made the rung
|
|
239
|
+
* untestable there without a process-wide monkeypatch.
|
|
240
|
+
*
|
|
241
|
+
* It is honoured by BOTH halves of a sweep — discovery (`buildProcSnapshot`,
|
|
242
|
+
* `collectRunDescendants`) and the group reap — so an injected uid is never
|
|
243
|
+
* compared here and then ignored one call deeper (see the propagation note at
|
|
244
|
+
* the `reapGroup` call site in `reapRunDescendants`).
|
|
245
|
+
*/
|
|
246
|
+
getuid?: () => number | undefined;
|
|
235
247
|
}
|
|
236
248
|
|
|
237
249
|
/** Sweep verdict: what was signalled, what was refused, what is still in flight. */
|
|
@@ -248,6 +260,10 @@ export interface RunSweepResult {
|
|
|
248
260
|
descendants: RunDescendant[];
|
|
249
261
|
}
|
|
250
262
|
|
|
263
|
+
/** Default uid source: the real process uid when the runtime has one (POSIX),
|
|
264
|
+
* `undefined` on win32 — the pre-seam behaviour, unchanged. */
|
|
265
|
+
const defaultGetuid = (): number | undefined => process.getuid?.();
|
|
266
|
+
|
|
251
267
|
// ── default /proc readers (all best-effort, none throw) ──────────────────────
|
|
252
268
|
|
|
253
269
|
/** `/proc/<pid>/environ` as a key=value record; undefined when unreadable. */
|
|
@@ -399,7 +415,7 @@ export function buildProcSnapshot(opts: SweepOptions = {}): ProcSnapshot {
|
|
|
399
415
|
const readIdentity = opts.readIdentity ?? readProcIdentity;
|
|
400
416
|
const readEnviron = opts.readEnviron ?? readProcEnviron;
|
|
401
417
|
const readTtyNr = opts.readTtyNr ?? readProcTtyNr;
|
|
402
|
-
const selfUid =
|
|
418
|
+
const selfUid = (opts.getuid ?? defaultGetuid)();
|
|
403
419
|
const pids: number[] = [];
|
|
404
420
|
const entries = new Map<number, ProcSnapshotEntry>();
|
|
405
421
|
for (const pid of (opts.listPids ?? listProcPids)()) {
|
|
@@ -482,7 +498,7 @@ export function collectRunDescendants(markers: RunMarkers, opts: SweepOptions =
|
|
|
482
498
|
if ((opts.platform ?? process.platform) !== "linux") return [];
|
|
483
499
|
const readIdentity = opts.readIdentity ?? readProcIdentity;
|
|
484
500
|
const candidateStartMs = opts.candidateStartMs ?? defaultCandidateStartMs;
|
|
485
|
-
const selfUid =
|
|
501
|
+
const selfUid = (opts.getuid ?? defaultGetuid)();
|
|
486
502
|
// Our own group: never a candidate's target group (belt to reap-group's rung).
|
|
487
503
|
const selfPgrp = readIdentity(process.pid)?.pgid;
|
|
488
504
|
// 11b.4 (LOW-4): ONE /proc view per call — supplied by the caller (a whole
|
|
@@ -715,6 +731,12 @@ export function reapRunDescendants(markers: RunMarkers, opts: SweepOptions = {})
|
|
|
715
731
|
...(opts.gracefulMs !== undefined ? { gracefulMs: opts.gracefulMs } : {}),
|
|
716
732
|
...(opts.capMs !== undefined ? { capMs: opts.capMs } : {}),
|
|
717
733
|
...(opts.platform !== undefined ? { platform: opts.platform } : {}),
|
|
734
|
+
// F5 propagation (wave 12b): the uid source MUST reach the ladder too.
|
|
735
|
+
// Discovery already used `opts.getuid`; forwarding only at that half would
|
|
736
|
+
// let a caller assert one uid policy and execute another (the sweep would
|
|
737
|
+
// prove a group safe against the injected uid while the ladder compared
|
|
738
|
+
// against the real one) — a silent split-brain, worse than no seam at all.
|
|
739
|
+
...(opts.getuid !== undefined ? { getuid: opts.getuid } : {}),
|
|
718
740
|
});
|
|
719
741
|
if (reap.refusal !== undefined) {
|
|
720
742
|
// Nothing was signalled. Leave the key unset so a later tick can retry
|