pi-crew 0.11.37 → 0.11.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,9 +10,17 @@
10
10
  * 2. Injects a followUp message via pi.sendUserMessage() so the assistant
11
11
  * is automatically notified — no manual sleep+check needed.
12
12
  * 3. Cleans up after itself when the run completes or the session ends.
13
+ *
14
+ * Hung alerts are additionally DELIVERY-gated (WD-DELIVER-GUARD): the message
15
+ * is only handed to the host when the host will deliver it immediately, and
16
+ * the run state is re-read at that instant — an alert whose run finished in
17
+ * the meantime is dropped instead of nagging the user minutes late.
13
18
  */
14
19
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
20
+ import { getCrewEnv, getCrewEnvInt } from "../config/env-vars.ts";
21
+ import { TEAM_TERMINAL_TASK_STATUSES } from "../state/contracts.ts";
15
22
  import { loadRunManifestById } from "../state/stores/state-store.ts";
23
+ import type { TeamTaskState } from "../state/types.ts";
16
24
  import { readCrewAgents } from "./crew-agent-records.ts";
17
25
  import { isActiveRunStatus, isLikelyOrphanedActiveRun } from "./process-status.ts";
18
26
 
@@ -20,10 +28,22 @@ export interface WatchdogOptions {
20
28
  pi: ExtensionAPI;
21
29
  cwd: string;
22
30
  runId: string;
23
- /** Check interval in ms. Default: 5 minutes. */
31
+ /**
32
+ * Check interval in ms. Default: 5 minutes, or the
33
+ * `PI_CREW_WATCHDOG_INTERVAL_MS` override when set (WD-INTERVAL).
34
+ */
24
35
  checkIntervalMs?: number;
25
36
  /** Maximum time to monitor in ms. Default: 2 hours. */
26
37
  maxMonitorMs?: number;
38
+ /**
39
+ * WD-DELIVER-GUARD (2026-10-23): capability-guarded probe for "the host will
40
+ * deliver a followUp message NOW" (`ctx.isIdle()`). When it reports false,
41
+ * a hung alert is held in this module's pending map and retried at the idle
42
+ * boundary instead of being handed to the host's unretractable followUp
43
+ * queue (where it can land minutes later, for a run that already finished).
44
+ * Absent / throwing → treated as idle (pre-guard behavior, fail-open).
45
+ */
46
+ isSessionIdle?: () => boolean | undefined;
27
47
  }
28
48
 
29
49
  const DEFAULT_CHECK_INTERVAL_MS = 300_000; // 5 minutes
@@ -32,6 +52,302 @@ const DEFAULT_MAX_MONITOR_MS = 7_200_000; // 2 hours
32
52
  /** Active watchdog timers — keyed by runId for cleanup. */
33
53
  const activeWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
34
54
 
55
+ /**
56
+ * WD-DEDUP (2026-10-23): hung-alert dedup state, keyed by runId at module
57
+ * scope so a *new* watchdog instance (or a re-armed one after stop/start)
58
+ * cannot re-alert for a run the user already heard about. Bounded FIFO —
59
+ * the oldest entry is evicted once the cap is reached, so long-lived
60
+ * sessions cannot leak memory on this map.
61
+ */
62
+ interface HungAlertRecord {
63
+ /** A "run … appears hung" alert has already been emitted for this runId. */
64
+ alerted: boolean;
65
+ /** The one-time re-verify-skip debug log has already been emitted. */
66
+ skipLogged: boolean;
67
+ /**
68
+ * WD-STALE-RECORD (2026-10-23): the one-time "record stale (all tasks
69
+ * terminal) — skipped hung alert" notice has already been emitted.
70
+ */
71
+ staleNoticeSent?: boolean;
72
+ /**
73
+ * WD-DELIVER-GUARD (2026-10-23): the one-time "drop hung alert at delivery"
74
+ * debug log has already been emitted for this runId.
75
+ */
76
+ droppedAtDeliveryLogged?: boolean;
77
+ }
78
+ const HUNG_ALERT_RECORDS_CAP = 256;
79
+ const hungAlertRecords = new Map<string, HungAlertRecord>();
80
+
81
+ function hungAlertAlreadySent(runId: string): boolean {
82
+ return hungAlertRecords.get(runId)?.alerted === true;
83
+ }
84
+
85
+ /** Insert/update a record, evicting the oldest entry when the cap is hit. */
86
+ function updateHungAlertRecord(runId: string, patch: Partial<HungAlertRecord>): void {
87
+ if (!hungAlertRecords.has(runId) && hungAlertRecords.size >= HUNG_ALERT_RECORDS_CAP) {
88
+ const oldest = hungAlertRecords.keys().next().value;
89
+ if (oldest !== undefined) hungAlertRecords.delete(oldest);
90
+ }
91
+ const prev = hungAlertRecords.get(runId) ?? { alerted: false, skipLogged: false, staleNoticeSent: false };
92
+ hungAlertRecords.set(runId, { ...prev, ...patch });
93
+ }
94
+
95
+ /** Debug log gated behind PI_CREW_DEBUG (same convention as background-runner). */
96
+ function debugLog(message: string): void {
97
+ if (getCrewEnv("PI_CREW_DEBUG")) console.log(message);
98
+ }
99
+
100
+ /**
101
+ * WD-STALE-RECORD (2026-10-23): a run record whose manifest is still `running`
102
+ * but whose EVERY task is terminal is a stale record, not a hung run. The
103
+ * worker(s) finished; the manifest write is merely behind (finalize racing, or
104
+ * a finalize write lost to a crash). Alerting "appears hung" for such a run is
105
+ * the exact pain users reported (alert for a run that is actually done), so the
106
+ * re-verify gate treats it as terminal and stays silent.
107
+ *
108
+ * Fail-open by design: no tasks recorded (`[]`), a non-array, or an unreadable
109
+ * tasks.json (loadRunManifestById degrades to `tasks: []`) returns false so the
110
+ * legacy alert behavior is preserved — a broken tasks file must never silence
111
+ * a genuinely hung run.
112
+ */
113
+ function areAllTasksTerminal(tasks: TeamTaskState[] | undefined): boolean {
114
+ if (!Array.isArray(tasks) || tasks.length === 0) return false;
115
+ return tasks.every((task) => TEAM_TERMINAL_TASK_STATUSES.has(task.status));
116
+ }
117
+
118
+ /**
119
+ * WD-INTERVAL (2026-10-23): resolve the poll interval. An explicit
120
+ * `checkIntervalMs` (tests / callers) wins, then the `PI_CREW_WATCHDOG_INTERVAL_MS`
121
+ * env override (live repro at 20s instead of 5 min), then the 5-minute default.
122
+ * Only positive finite values are honored — `0`/negative/garbage fall through.
123
+ */
124
+ function resolveCheckIntervalMs(overrideMs: number | undefined): number {
125
+ if (typeof overrideMs === "number" && Number.isFinite(overrideMs) && overrideMs > 0) return overrideMs;
126
+ const configured = getCrewEnvInt("PI_CREW_WATCHDOG_INTERVAL_MS");
127
+ if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) return configured;
128
+ return DEFAULT_CHECK_INTERVAL_MS;
129
+ }
130
+
131
+ /**
132
+ * Pre-emit state re-verify loader. Injectable for tests to simulate the run
133
+ * going terminal between the cycle's initial read and the emit point.
134
+ */
135
+ type ReverifyLoader = typeof loadRunManifestById;
136
+ let reverifyLoader: ReverifyLoader = loadRunManifestById;
137
+
138
+ /**
139
+ * WD-DELIVER-GUARD (2026-10-23) — the residual the emit-time gate cannot see.
140
+ *
141
+ * Real incident (run team_20261023050900): the alert was EMITTED while the run
142
+ * was still running (so emit-time re-verify was correct), then
143
+ * `pi.sendUserMessage(..., {deliverAs:"followUp"})` pushed it into the host's
144
+ * follow-up queue while the session was streaming (`agent-session.js`:
145
+ * `if (this.isStreaming) this.agent.followUp(appMessage)`) and the host
146
+ * DELIVERED it 28 minutes later — after `run.completed` had landed. The user
147
+ * still saw "appears hung" for a finished run. `clearQueue()` is all-or-
148
+ * nothing, so a queued item cannot be retracted: the only fix is to not hand
149
+ * it over until delivery is imminent.
150
+ *
151
+ * Mechanism: while the session is streaming, the alert stays in OUR pending
152
+ * map; at the idle boundary (the moment the host delivers immediately) the run
153
+ * state is re-read one final time. Run terminal (completed/failed/cancelled/
154
+ * blocked) or all tasks terminal ⇒ DROP (debug-logged once per runId, no
155
+ * user-facing noise). FAIL-OPEN: an unreadable/absent record still delivers —
156
+ * a broken state file must never swallow a genuine hung alert.
157
+ *
158
+ * BOTH exits re-verify: the idle boundary AND the delivery cap. A session that
159
+ * never idles (one tool call blocking past the cap) is not a license to hand
160
+ * over a stale alert — the cap exit runs the same classify/verify, so an
161
+ * unverified delivery is impossible. "Late beats never" only applies to alerts
162
+ * whose run still reads as hung.
163
+ */
164
+ const DEFAULT_DELIVER_RETRY_MS = 500;
165
+ const DEFAULT_DELIVER_CAP_MS = 10 * 60_000; // 10 min
166
+ /** Defensive bound — a burst of distinct hung runs must not grow the map. */
167
+ const HUNG_DELIVERY_CAP = 256;
168
+
169
+ interface PendingHungDelivery {
170
+ /** Full alert text, handed over verbatim at the idle boundary. */
171
+ text: string;
172
+ /** Past this instant the alert is handed over anyway (late beats never). */
173
+ deadline: number;
174
+ /** Idle probe captured at queue time (see WatchdogOptions.isSessionIdle). */
175
+ isSessionIdle?: () => boolean | undefined;
176
+ timer?: ReturnType<typeof setTimeout>;
177
+ }
178
+
179
+ /** Hung alerts queued but not yet handed to the host, keyed by runId. */
180
+ const pendingHungDeliveries = new Map<string, PendingHungDelivery>();
181
+
182
+ let deliverRetryMs = DEFAULT_DELIVER_RETRY_MS;
183
+ let deliverCapMs = DEFAULT_DELIVER_CAP_MS;
184
+ /** Delivery-time state loader (separate seam from the emit-time re-verify). */
185
+ let deliveryLoader: ReverifyLoader = loadRunManifestById;
186
+
187
+ /** @internal Test seam — reset module-level hung-alert dedup + delivery state. */
188
+ export function resetHungAlertDedupForTests(): void {
189
+ hungAlertRecords.clear();
190
+ for (const entry of pendingHungDeliveries.values()) {
191
+ if (entry.timer !== undefined) clearTimeout(entry.timer);
192
+ }
193
+ pendingHungDeliveries.clear();
194
+ reverifyLoader = loadRunManifestById;
195
+ deliveryLoader = loadRunManifestById;
196
+ deliverRetryMs = DEFAULT_DELIVER_RETRY_MS;
197
+ deliverCapMs = DEFAULT_DELIVER_CAP_MS;
198
+ }
199
+
200
+ /** @internal Test seam — snapshot of the dedup map size (cap/leak assertions). */
201
+ export function hungAlertDedupSizeForTests(): number {
202
+ return hungAlertRecords.size;
203
+ }
204
+
205
+ /** @internal Test seam — force-mark a runId as already alerted (cap assertions). */
206
+ export function markHungAlertForTests(runId: string): void {
207
+ updateHungAlertRecord(runId, { alerted: true });
208
+ }
209
+
210
+ /** @internal Test seam — replace the pre-emit re-verify loader (null = real). */
211
+ export function setReverifyLoaderForTests(loader: ReverifyLoader | null): void {
212
+ reverifyLoader = loader ?? loadRunManifestById;
213
+ }
214
+
215
+ /** @internal Test seam — replace the DELIVERY-time state loader (null = real). */
216
+ export function setDeliveryLoaderForTests(loader: ReverifyLoader | null): void {
217
+ deliveryLoader = loader ?? loadRunManifestById;
218
+ }
219
+
220
+ /** @internal Test seam — shrink the idle-gate retry/cap timings. */
221
+ export function setDeliveryTimingForTests(retryMs?: number, capMs?: number): void {
222
+ deliverRetryMs = typeof retryMs === "number" && retryMs > 0 ? retryMs : DEFAULT_DELIVER_RETRY_MS;
223
+ deliverCapMs = typeof capMs === "number" && capMs > 0 ? capMs : DEFAULT_DELIVER_CAP_MS;
224
+ }
225
+
226
+ /** @internal Test seam — hung alerts queued but not yet handed to the host. */
227
+ export function hungAlertDeliveryPendingForTests(): number {
228
+ return pendingHungDeliveries.size;
229
+ }
230
+
231
+ /**
232
+ * Delivery-time verdict for a queued hung alert. `hung` = hand it to the host;
233
+ * `dropped` = the run is provably no longer hung, stay silent.
234
+ */
235
+ type DeliveryVerdict = { kind: "hung" } | { kind: "dropped"; reason: string };
236
+
237
+ /**
238
+ * Read the run state at handoff time. This is the WD-DELIVER-GUARD decision
239
+ * point: it runs immediately before `pi.sendUserMessage`, i.e. at the moment
240
+ * the host will actually deliver the message (session idle).
241
+ *
242
+ * Drop conditions (the exact user-visible pain):
243
+ * • manifest status terminal (completed/failed/cancelled/blocked), or
244
+ * • manifest still active BUT every task is terminal (stale record — the
245
+ * finalize/worker write is merely behind).
246
+ * FAIL-OPEN: a loader that throws, or a vanished manifest, yields `hung` — an
247
+ * unreadable record must never swallow a genuine alert.
248
+ */
249
+ function classifyAtDelivery(cwd: string, runId: string): DeliveryVerdict {
250
+ let loaded: Awaited<ReturnType<ReverifyLoader>>;
251
+ try {
252
+ loaded = deliveryLoader(cwd, runId);
253
+ } catch {
254
+ return { kind: "hung" }; // unreadable state → fail-open
255
+ }
256
+ if (!loaded) return { kind: "hung" }; // indeterminate → fail-open
257
+ if (!isActiveRunStatus(loaded.manifest.status)) {
258
+ return { kind: "dropped", reason: `status=${loaded.manifest.status}` };
259
+ }
260
+ if (areAllTasksTerminal(loaded.tasks)) {
261
+ return { kind: "dropped", reason: `all ${loaded.tasks.length} task(s) terminal (manifest ${loaded.manifest.status})` };
262
+ }
263
+ return { kind: "hung" };
264
+ }
265
+
266
+ /** True when the host would deliver a followUp message right now. */
267
+ function sessionLooksIdle(probe: (() => boolean | undefined) | undefined): boolean {
268
+ if (typeof probe !== "function") return true; // no capability → pre-guard behavior
269
+ try {
270
+ return probe() !== false;
271
+ } catch {
272
+ return true;
273
+ }
274
+ }
275
+
276
+ /**
277
+ * Try to hand one pending hung alert to the host. Re-arms itself while the
278
+ * session is streaming (the host would queue instead of deliver) and, at the
279
+ * idle boundary or past the cap, re-reads the run state ONCE and either drops
280
+ * or delivers.
281
+ */
282
+ function attemptHungAlertDelivery(pi: ExtensionAPI, cwd: string, runId: string, force: boolean): void {
283
+ const entry = pendingHungDeliveries.get(runId);
284
+ if (!entry) return;
285
+ if (entry.timer !== undefined) {
286
+ clearTimeout(entry.timer);
287
+ entry.timer = undefined;
288
+ }
289
+ if (!force && !sessionLooksIdle(entry.isSessionIdle) && Date.now() < entry.deadline) {
290
+ const timer = setTimeout(() => attemptHungAlertDelivery(pi, cwd, runId, false), deliverRetryMs);
291
+ timer.unref();
292
+ entry.timer = timer;
293
+ return;
294
+ }
295
+
296
+ // Fallthrough covers BOTH exits: the idle boundary and the cap expiry
297
+ // (`Date.now() >= entry.deadline`, session still streaming), plus any forced
298
+ // flush from the map cap. No exit skips the re-verify — an alert that got
299
+ // old is not evidence the run is still hung, so the same drop conditions
300
+ // apply here and the debug log names the reason it was dropped.
301
+ const verdict = classifyAtDelivery(cwd, runId);
302
+ pendingHungDeliveries.delete(runId);
303
+ if (verdict.kind === "dropped") {
304
+ if (!hungAlertRecords.get(runId)?.droppedAtDeliveryLogged) {
305
+ updateHungAlertRecord(runId, { droppedAtDeliveryLogged: true });
306
+ debugLog(`[foreground-watchdog] drop hung alert for ${runId} at delivery: run is no longer hung (${verdict.reason})`);
307
+ }
308
+ return;
309
+ }
310
+ try {
311
+ pi.sendUserMessage(entry.text, { deliverAs: "followUp" });
312
+ } catch {
313
+ /* non-critical */
314
+ }
315
+ }
316
+
317
+ /**
318
+ * Queue a hung alert for guarded delivery (WD-DELIVER-GUARD). One in flight
319
+ * per runId — a second notice for the same run while the first is still
320
+ * pending is dropped as redundant drip.
321
+ */
322
+ function queueHungAlertDelivery(
323
+ pi: ExtensionAPI,
324
+ cwd: string,
325
+ runId: string,
326
+ text: string,
327
+ isSessionIdle: (() => boolean | undefined) | undefined,
328
+ ): void {
329
+ if (pendingHungDeliveries.has(runId)) {
330
+ debugLog(`[foreground-watchdog] hung alert for ${runId} already pending delivery — not stacking another`);
331
+ return;
332
+ }
333
+ pendingHungDeliveries.set(runId, { text, deadline: Date.now() + deliverCapMs, isSessionIdle });
334
+ if (pendingHungDeliveries.size > HUNG_DELIVERY_CAP) {
335
+ // Defensive: never let the map grow and never delay a real alert —
336
+ // flush the oldest pending alert immediately (verify-then-send).
337
+ const oldest = pendingHungDeliveries.keys().next().value;
338
+ if (oldest !== undefined) attemptHungAlertDelivery(pi, cwd, oldest, true);
339
+ }
340
+ attemptHungAlertDelivery(pi, cwd, runId, false);
341
+ }
342
+
343
+ /** Drop a still-pending (not yet handed-over) hung alert for a runId. */
344
+ function cancelPendingHungDelivery(runId: string): void {
345
+ const entry = pendingHungDeliveries.get(runId);
346
+ if (!entry) return;
347
+ if (entry.timer !== undefined) clearTimeout(entry.timer);
348
+ pendingHungDeliveries.delete(runId);
349
+ }
350
+
35
351
  /** Stop a specific watchdog by runId. */
36
352
  export function stopWatchdog(runId: string): void {
37
353
  const timer = activeWatchdogs.get(runId);
@@ -39,6 +355,7 @@ export function stopWatchdog(runId: string): void {
39
355
  clearTimeout(timer);
40
356
  activeWatchdogs.delete(runId);
41
357
  }
358
+ cancelPendingHungDelivery(runId);
42
359
  }
43
360
 
44
361
  /** @internal Stop all active watchdogs. Called on session shutdown (RC-01). */
@@ -47,6 +364,10 @@ export function stopAllWatchdogs(): void {
47
364
  clearTimeout(timer);
48
365
  }
49
366
  activeWatchdogs.clear();
367
+ for (const entry of pendingHungDeliveries.values()) {
368
+ if (entry.timer !== undefined) clearTimeout(entry.timer);
369
+ }
370
+ pendingHungDeliveries.clear();
50
371
  }
51
372
 
52
373
  /**
@@ -62,7 +383,7 @@ export function stopAllWatchdogs(): void {
62
383
  */
63
384
  export function startForegroundWatchdog(opts: WatchdogOptions): void {
64
385
  const { pi, cwd, runId } = opts;
65
- const checkIntervalMs = opts.checkIntervalMs ?? DEFAULT_CHECK_INTERVAL_MS;
386
+ const checkIntervalMs = resolveCheckIntervalMs(opts.checkIntervalMs);
66
387
  const maxMonitorMs = opts.maxMonitorMs ?? DEFAULT_MAX_MONITOR_MS;
67
388
  const startTime = Date.now();
68
389
 
@@ -111,23 +432,79 @@ export function startForegroundWatchdog(opts: WatchdogOptions): void {
111
432
  const agents = readCrewAgents(manifest);
112
433
  const now = Date.now();
113
434
  if (isLikelyOrphanedActiveRun(manifest, agents, now)) {
114
- const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}`;
115
- consecutiveHungNotices += 1;
116
- try {
117
- if (consecutiveHungNotices <= HUNG_NOTICE_CAP) {
118
- pi.sendUserMessage(
119
- `pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
120
- { deliverAs: "followUp" },
121
- );
122
- } else if (consecutiveHungNotices === HUNG_NOTICE_CAP + 1) {
123
- // One-time handoff, then silence — still monitoring, no more drips.
124
- pi.sendUserMessage(
125
- `pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
126
- { deliverAs: "followUp" },
435
+ // WD-DEDUP (2026-10-23): re-verify the run state right before any
436
+ // emit. A check armed by an older poll cycle can race the run
437
+ // reaching a terminal status — alerting "appears hung" for a run
438
+ // that already finished is exactly the repeated-alert pain.
439
+ // Terminal (or vanished) at re-verify → skip the emit entirely;
440
+ // the next cycle's terminal branch delivers the completion notice.
441
+ const reverified = reverifyLoader(cwd, runId);
442
+ if (!reverified || !isActiveRunStatus(reverified.manifest.status)) {
443
+ const reason = reverified ? `status=${reverified.manifest.status}` : "manifest gone";
444
+ if (!hungAlertRecords.get(runId)?.skipLogged) {
445
+ updateHungAlertRecord(runId, { skipLogged: true });
446
+ debugLog(`[foreground-watchdog] skip hung alert for ${runId}: re-verify saw ${reason}`);
447
+ }
448
+ // Fall through to the shared schedule — keep monitoring; the next
449
+ // cycle hits the terminal branch (completion notice) or !loaded.
450
+ } else if (areAllTasksTerminal(reverified.tasks)) {
451
+ // WD-STALE-RECORD: manifest still says running, but every task is
452
+ // terminal → the run is done and only the record is behind. Stay
453
+ // silent about "hung" and tell the user ONCE that the record is
454
+ // stale (per runId, module-scope — no spam across restarts).
455
+ if (!hungAlertRecords.get(runId)?.skipLogged) {
456
+ updateHungAlertRecord(runId, { skipLogged: true });
457
+ debugLog(
458
+ `[foreground-watchdog] skip hung alert for ${runId}: record stale (manifest ${reverified.manifest.status}, all ${reverified.tasks.length} task(s) terminal)`,
127
459
  );
128
460
  }
129
- } catch {
130
- /* non-critical */
461
+ if (!hungAlertRecords.get(runId)?.staleNoticeSent) {
462
+ updateHungAlertRecord(runId, { staleNoticeSent: true });
463
+ try {
464
+ pi.sendUserMessage(`pi-crew watchdog: run ${runId} record stale (all tasks terminal) — skipped hung alert.`, {
465
+ deliverAs: "followUp",
466
+ });
467
+ } catch {
468
+ /* non-critical */
469
+ }
470
+ }
471
+ // Keep monitoring: if the manifest eventually finalizes, the
472
+ // terminal branch delivers the completion notice.
473
+ } else {
474
+ const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}`;
475
+ consecutiveHungNotices += 1;
476
+ try {
477
+ if (consecutiveHungNotices <= HUNG_NOTICE_CAP) {
478
+ // WD-DEDUP: alert-once per runId at module scope — even a
479
+ // fresh watchdog instance must not re-alert the same run.
480
+ if (!hungAlertAlreadySent(runId)) {
481
+ // Mark BEFORE queueing so a racing cycle cannot double-queue.
482
+ updateHungAlertRecord(runId, { alerted: true });
483
+ // WD-DELIVER-GUARD: the host queues a followUp while the
484
+ // session streams and delivers it arbitrarily later — so
485
+ // hand it over only at the idle boundary, re-verifying the
486
+ // run state at THAT moment.
487
+ queueHungAlertDelivery(
488
+ pi,
489
+ cwd,
490
+ runId,
491
+ `pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
492
+ opts.isSessionIdle,
493
+ );
494
+ }
495
+ } else if (consecutiveHungNotices === HUNG_NOTICE_CAP + 1) {
496
+ // One-time handoff, then silence — still monitoring, no more drips.
497
+ queueHungAlertDelivery(
498
+ pi,
499
+ cwd,
500
+ runId,
501
+ `pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
502
+ opts.isSessionIdle,
503
+ );
504
+ }
505
+ } catch {
506
+ /* non-critical */
507
+ }
131
508
  }
132
509
  // Keep monitoring (ARCH-5: silently past the cap — no more notices).
133
510
  } else {
@@ -3,6 +3,7 @@ import * as os from "node:os";
3
3
  import * as path from "node:path";
4
4
  import { getCrewEnv } from "../../config/env-vars.ts";
5
5
  import { errors } from "../../errors.ts";
6
+ import type { UsageState } from "../../state/types.ts";
6
7
  import { logInternalError } from "../../utils/internal-error.ts";
7
8
  import { fuzzyResolveModelId } from "./model-resolver.ts";
8
9
  import { checkModelScope } from "./model-scope.ts";
@@ -19,6 +20,10 @@ export interface ModelAttemptSummary {
19
20
  success: boolean;
20
21
  exitCode?: number | null;
21
22
  error?: string;
23
+ /** W10-7 (v0.11.38): THIS attempt's own usage (from its parsed output) —
24
+ * failed-fallback attempts burned real tokens that task.usage (the FINAL
25
+ * run's usage) never shows; post-execution reconciles the two. */
26
+ usage?: UsageState;
22
27
  }
23
28
 
24
29
  export interface ModelLike {
@@ -1193,6 +1193,9 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
1193
1193
  success: !error,
1194
1194
  exitCode,
1195
1195
  error,
1196
+ // W10-7: capture THIS attempt's usage before the loop can move on —
1197
+ // failed attempts' tokens otherwise vanish from every persisted record.
1198
+ ...(parsedOutput?.usage ? { usage: parsedOutput.usage } : {}),
1196
1199
  };
1197
1200
  modelAttempts.push(attempt);
1198
1201
  task = {
@@ -27,6 +27,7 @@ import type {
27
27
  TeamRunManifest,
28
28
  TeamTaskState,
29
29
  TeamTaskStatus,
30
+ UsageState,
30
31
  VerificationEvidence,
31
32
  } from "../../state/types.ts";
32
33
  import { captureWorktreeDiffAsync, captureWorktreeDiffStatAsync } from "../../worktree/worktree-manager.ts";
@@ -38,6 +39,7 @@ import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../heartbeat/worker
38
39
  import type { ModelAttemptSummary } from "../model/model-fallback.ts";
39
40
  import { isStderrOnlyResult, type OutputValidationResult, validateWorkerOutput } from "../output/output-validator.ts";
40
41
  import type { ParsedPiJsonOutput } from "../output/pi-json-output.ts";
42
+ import { permissionForRole } from "../role-permission.ts";
41
43
  import { writeTaskSharedOutput } from "../task-output-context.ts";
42
44
  import { evaluateCompletionMutationGuard } from "../verification/completion-guard.ts";
43
45
  import { createVerificationEvidence } from "../verification/green-contract.ts";
@@ -139,6 +141,87 @@ export type VerifierVerdict = "pass" | "fail" | "none";
139
141
  */
140
142
  const VERIFIER_VERDICT_LINE = /^[\s*_>#`=-]*(PASS|FAIL)[\s*_`]*(?=[:.\u2014\u2013-]|$|\()/i;
141
143
 
144
+ /** W10-5: the critic/oracle decision-drift guardian marker (see
145
+ * agents/critic.md + agents/oracle.md "Decision-drift guardian" sections). */
146
+ const DRIFT_BLOCK_MARKER = /DRIFT-BLOCK\b/i;
147
+
148
+ /**
149
+ * W10-8 (pi-subagents lesson #4): bounded `git status --porcelain` probe for
150
+ * READ-ONLY lanes — any entry (modified OR untracked) is an unexpected
151
+ * mutation. Returns undefined when the tree is clean or git fails (fail-open:
152
+ * a broken probe must not break task finalization). Note-only, never a fail.
153
+ */
154
+ export async function detectReadLaneMutation(worktreePath: string): Promise<NonNullable<TeamTaskState["unexpectedMutation"]> | undefined> {
155
+ try {
156
+ // W10-8 one-shot git probe per read-only task — child_process is
157
+ // already in the platform; dynamic keeps it off the static import graph.
158
+ // LAZY: see above
159
+ const { execFile } = await import("node:child_process");
160
+ const out: string = await new Promise((resolve, reject) => {
161
+ execFile("git", ["status", "--porcelain"], { cwd: worktreePath, encoding: "utf-8", timeout: 10_000 }, (err, stdout) =>
162
+ err ? reject(err) : resolve(stdout),
163
+ );
164
+ });
165
+ const entries = out
166
+ .split("\n")
167
+ .map((l) => l.trim())
168
+ .filter((l) => l.length > 0);
169
+ if (entries.length === 0) return undefined;
170
+ return {
171
+ filesChanged: entries.length,
172
+ sample: entries.slice(0, 20).map((e) => e.slice(0, 200)),
173
+ };
174
+ } catch {
175
+ return undefined;
176
+ }
177
+ }
178
+
179
+ /**
180
+ * W10-7 (v0.11.38, pi-subagents lesson #14 — usage reconciliation): compare
181
+ * the SUM of every model attempt's own cost (failed fallback attempts burn
182
+ * real tokens too, but task.usage carries only the FINAL run's numbers)
183
+ * against the final cost. Returns a usageReconcile note ONLY when the gap
184
+ * exceeds 5% AND at least two attempts reported usage — a single attempt is
185
+ * trivially self-consistent. Observability only: never fails the task.
186
+ */
187
+ export function reconcileUsage(
188
+ attempts: ModelAttemptSummary[] | undefined,
189
+ finalUsage: UsageState | undefined,
190
+ ): { usageReconcile: NonNullable<TeamTaskState["usageReconcile"]> } | undefined {
191
+ if (!attempts || attempts.length < 2) return undefined;
192
+ // W10-7 batch-A threshold guard: with no final usage to reconcile AGAINST
193
+ // (finalUsage missing, or its cost not a finite number) the comparison is
194
+ // meaningless — an attempt-sum alone cannot prove a gap, and comparing to
195
+ // 0 would file a false 100%-gap alarm against missing data. No note.
196
+ if (typeof finalUsage?.cost !== "number" || !Number.isFinite(finalUsage.cost)) return undefined;
197
+ const costs = attempts.map((a) => (typeof a.usage?.cost === "number" && Number.isFinite(a.usage.cost) ? a.usage.cost : undefined));
198
+ const withUsage = costs.filter((c): c is number => c !== undefined);
199
+ if (withUsage.length < 2) return undefined;
200
+ const attemptTotal = withUsage.reduce((sum, c) => sum + c, 0);
201
+ const finalCost = finalUsage.cost;
202
+ if (attemptTotal <= 0) return undefined;
203
+ const deltaPct = Math.abs(attemptTotal - finalCost) / attemptTotal;
204
+ if (deltaPct <= 0.05) return undefined;
205
+ return {
206
+ usageReconcile: {
207
+ attemptTotalCostUsd: Number(attemptTotal.toFixed(6)),
208
+ finalRunCostUsd: finalCost,
209
+ deltaPct: Number((deltaPct * 100).toFixed(1)),
210
+ attemptsWithUsage: withUsage.length,
211
+ },
212
+ };
213
+ }
214
+
215
+ /**
216
+ * W10-8 visibility (batch A): the unexpectedMutation note lives on the task
217
+ * record where it is easy to miss — stamping it also emits ONE stderr line
218
+ * so a READ-ONLY lane violation shows up in the run log without new
219
+ * dependencies. Exported for the console-capture pin; fire-and-forget.
220
+ */
221
+ export function warnReadLaneMutation(taskId: string, note: { filesChanged: number }): void {
222
+ console.warn(`[pi-crew:read-lane] unexpected mutation in READ-ONLY lane: ${taskId} files=${note.filesChanged}`);
223
+ }
224
+
142
225
  /**
143
226
  * M3-2 (R47-3/V4, fix-wave 6): read the verifier verdict out of the
144
227
  * authoritative result sources.
@@ -173,7 +256,18 @@ export function extractVerifierVerdict(...sources: Array<string | undefined>): V
173
256
  if (match[1]?.toUpperCase() === "FAIL") return "fail";
174
257
  sawPass = true;
175
258
  }
176
- if (sawPass) return "pass";
259
+ if (sawPass) {
260
+ // W10-5 (v0.11.38, pi-subagents lesson #20 — oracle/drift-guardian): a
261
+ // verdict that flags a decision-drift contradiction (the `DRIFT-BLOCK:`
262
+ // marker the critic/oracle prompts mandate) is SELF-CONTRADICTORY as a
263
+ // pass — the worker saw a contradiction with an inherited decision and
264
+ // approved anyway. Degrade to "none" (not verified): the existing
265
+ // self-attested downgrade path then demands human/leader review instead
266
+ // of a green. FAIL still wins above — a flagged drift alongside FAIL is
267
+ // already the honest verdict.
268
+ if (DRIFT_BLOCK_MARKER.test(source)) return "none";
269
+ return "pass";
270
+ }
177
271
  }
178
272
  return "none";
179
273
  }
@@ -320,6 +414,7 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
320
414
  producer: task.id,
321
415
  })
322
416
  : undefined;
417
+
323
418
  const diffStatArtifact = workspace.worktreePath
324
419
  ? writeArtifact(manifest.artifactsRoot, {
325
420
  kind: "metadata",
@@ -328,6 +423,14 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
328
423
  producer: task.id,
329
424
  })
330
425
  : undefined;
426
+ // W10-8 (v0.11.38, pi-subagents lesson #4 — staged-files guard): a READ-ONLY
427
+ // lane that dirtied its worktree is a contract violation worth surfacing —
428
+ // note only (the diff artifact above is the evidence; the leader decides).
429
+ // `git diff` alone misses UNTRACKED files, hence status --porcelain.
430
+ const readLaneMutation =
431
+ workspace.worktreePath && permissionForRole(task.role) === "read_only"
432
+ ? await detectReadLaneMutation(workspace.worktreePath)
433
+ : undefined;
331
434
 
332
435
  // Capture unified patches from edit tool results
333
436
  const patchArtifact = parsedOutput?.patches?.length
@@ -715,6 +818,9 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
715
818
  exitCode = exitCode === 0 ? 1 : exitCode;
716
819
  }
717
820
 
821
+ // W10-8 batch-A visibility: emit the one-line stderr event exactly when
822
+ // the note below is stamped (only on detection).
823
+ if (readLaneMutation) warnReadLaneMutation(task.id, readLaneMutation);
718
824
  task = {
719
825
  ...task,
720
826
  status: error ? "failed" : noYield ? "needs_attention" : "completed",
@@ -722,6 +828,11 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
722
828
  exitCode,
723
829
  modelAttempts,
724
830
  usage: parsedOutput?.usage,
831
+ // W10-7 (v0.11.38): reconcile per-attempt costs (failed fallbacks burn
832
+ // real tokens that the final run's usage never shows) against the final
833
+ // cost — observability note only, >5% gap with >1 usage-carrying attempt.
834
+ ...(reconcileUsage(modelAttempts, parsedOutput?.usage) ?? {}),
835
+ ...(readLaneMutation ? { unexpectedMutation: readLaneMutation } : {}),
725
836
  jsonEvents: parsedOutput?.jsonEvents,
726
837
  agentProgress:
727
838
  error && task.agentProgress?.currentTool