pi-crew 0.11.37 → 0.11.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/agents/critic.md +13 -0
- package/agents/oracle.md +14 -0
- package/dist/index.mjs +517 -516
- package/package.json +3 -2
- package/skills/delegation-patterns/SKILL.md +16 -0
- package/skills/real-test-pi-crew/SKILL.md +4 -1
- package/src/config/env-allowlist-registry.ts +8 -0
- package/src/config/env-vars.ts +11 -0
- package/src/extension/registration/foreground-run-controller.ts +21 -1
- package/src/prompt/prompt-runtime.ts +67 -5
- package/src/runtime/child-pi/child-pi-spawn.ts +16 -0
- package/src/runtime/child-pi/child-pi.ts +3 -1
- package/src/runtime/foreground-watchdog.ts +394 -17
- package/src/runtime/model/model-fallback.ts +5 -0
- package/src/runtime/task-runner/child-executor.ts +3 -0
- package/src/runtime/task-runner/post-execution.ts +112 -1
- package/src/runtime/verification/verify-gate-memo.ts +127 -0
- package/src/runtime/verification/verify-gate.ts +127 -1
- package/src/state/coordination/mailbox.ts +68 -0
- package/src/state/types.ts +19 -0
|
@@ -10,9 +10,17 @@
|
|
|
10
10
|
* 2. Injects a followUp message via pi.sendUserMessage() so the assistant
|
|
11
11
|
* is automatically notified — no manual sleep+check needed.
|
|
12
12
|
* 3. Cleans up after itself when the run completes or the session ends.
|
|
13
|
+
*
|
|
14
|
+
* Hung alerts are additionally DELIVERY-gated (WD-DELIVER-GUARD): the message
|
|
15
|
+
* is only handed to the host when the host will deliver it immediately, and
|
|
16
|
+
* the run state is re-read at that instant — an alert whose run finished in
|
|
17
|
+
* the meantime is dropped instead of nagging the user minutes late.
|
|
13
18
|
*/
|
|
14
19
|
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
|
+
import { getCrewEnv, getCrewEnvInt } from "../config/env-vars.ts";
|
|
21
|
+
import { TEAM_TERMINAL_TASK_STATUSES } from "../state/contracts.ts";
|
|
15
22
|
import { loadRunManifestById } from "../state/stores/state-store.ts";
|
|
23
|
+
import type { TeamTaskState } from "../state/types.ts";
|
|
16
24
|
import { readCrewAgents } from "./crew-agent-records.ts";
|
|
17
25
|
import { isActiveRunStatus, isLikelyOrphanedActiveRun } from "./process-status.ts";
|
|
18
26
|
|
|
@@ -20,10 +28,22 @@ export interface WatchdogOptions {
|
|
|
20
28
|
pi: ExtensionAPI;
|
|
21
29
|
cwd: string;
|
|
22
30
|
runId: string;
|
|
23
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* Check interval in ms. Default: 5 minutes, or the
|
|
33
|
+
* `PI_CREW_WATCHDOG_INTERVAL_MS` override when set (WD-INTERVAL).
|
|
34
|
+
*/
|
|
24
35
|
checkIntervalMs?: number;
|
|
25
36
|
/** Maximum time to monitor in ms. Default: 2 hours. */
|
|
26
37
|
maxMonitorMs?: number;
|
|
38
|
+
/**
|
|
39
|
+
* WD-DELIVER-GUARD (2026-10-23): capability-guarded probe for "the host will
|
|
40
|
+
* deliver a followUp message NOW" (`ctx.isIdle()`). When it reports false,
|
|
41
|
+
* a hung alert is held in this module's pending map and retried at the idle
|
|
42
|
+
* boundary instead of being handed to the host's unretractable followUp
|
|
43
|
+
* queue (where it can land minutes later, for a run that already finished).
|
|
44
|
+
* Absent / throwing → treated as idle (pre-guard behavior, fail-open).
|
|
45
|
+
*/
|
|
46
|
+
isSessionIdle?: () => boolean | undefined;
|
|
27
47
|
}
|
|
28
48
|
|
|
29
49
|
const DEFAULT_CHECK_INTERVAL_MS = 300_000; // 5 minutes
|
|
@@ -32,6 +52,302 @@ const DEFAULT_MAX_MONITOR_MS = 7_200_000; // 2 hours
|
|
|
32
52
|
/** Active watchdog timers — keyed by runId for cleanup. */
|
|
33
53
|
const activeWatchdogs = new Map<string, ReturnType<typeof setTimeout>>();
|
|
34
54
|
|
|
55
|
+
/**
|
|
56
|
+
* WD-DEDUP (2026-10-23): hung-alert dedup state, keyed by runId at module
|
|
57
|
+
* scope so a *new* watchdog instance (or a re-armed one after stop/start)
|
|
58
|
+
* cannot re-alert for a run the user already heard about. Bounded FIFO —
|
|
59
|
+
* the oldest entry is evicted once the cap is reached, so long-lived
|
|
60
|
+
* sessions cannot leak memory on this map.
|
|
61
|
+
*/
|
|
62
|
+
interface HungAlertRecord {
|
|
63
|
+
/** A "run … appears hung" alert has already been emitted for this runId. */
|
|
64
|
+
alerted: boolean;
|
|
65
|
+
/** The one-time re-verify-skip debug log has already been emitted. */
|
|
66
|
+
skipLogged: boolean;
|
|
67
|
+
/**
|
|
68
|
+
* WD-STALE-RECORD (2026-10-23): the one-time "record stale (all tasks
|
|
69
|
+
* terminal) — skipped hung alert" notice has already been emitted.
|
|
70
|
+
*/
|
|
71
|
+
staleNoticeSent?: boolean;
|
|
72
|
+
/**
|
|
73
|
+
* WD-DELIVER-GUARD (2026-10-23): the one-time "drop hung alert at delivery"
|
|
74
|
+
* debug log has already been emitted for this runId.
|
|
75
|
+
*/
|
|
76
|
+
droppedAtDeliveryLogged?: boolean;
|
|
77
|
+
}
|
|
78
|
+
const HUNG_ALERT_RECORDS_CAP = 256;
|
|
79
|
+
const hungAlertRecords = new Map<string, HungAlertRecord>();
|
|
80
|
+
|
|
81
|
+
function hungAlertAlreadySent(runId: string): boolean {
|
|
82
|
+
return hungAlertRecords.get(runId)?.alerted === true;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/** Insert/update a record, evicting the oldest entry when the cap is hit. */
|
|
86
|
+
function updateHungAlertRecord(runId: string, patch: Partial<HungAlertRecord>): void {
|
|
87
|
+
if (!hungAlertRecords.has(runId) && hungAlertRecords.size >= HUNG_ALERT_RECORDS_CAP) {
|
|
88
|
+
const oldest = hungAlertRecords.keys().next().value;
|
|
89
|
+
if (oldest !== undefined) hungAlertRecords.delete(oldest);
|
|
90
|
+
}
|
|
91
|
+
const prev = hungAlertRecords.get(runId) ?? { alerted: false, skipLogged: false, staleNoticeSent: false };
|
|
92
|
+
hungAlertRecords.set(runId, { ...prev, ...patch });
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** Debug log gated behind PI_CREW_DEBUG (same convention as background-runner). */
|
|
96
|
+
function debugLog(message: string): void {
|
|
97
|
+
if (getCrewEnv("PI_CREW_DEBUG")) console.log(message);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* WD-STALE-RECORD (2026-10-23): a run record whose manifest is still `running`
|
|
102
|
+
* but whose EVERY task is terminal is a stale record, not a hung run. The
|
|
103
|
+
* worker(s) finished; the manifest write is merely behind (finalize racing, or
|
|
104
|
+
* a finalize write lost to a crash). Alerting "appears hung" for such a run is
|
|
105
|
+
* the exact pain users reported (alert for a run that is actually done), so the
|
|
106
|
+
* re-verify gate treats it as terminal and stays silent.
|
|
107
|
+
*
|
|
108
|
+
* Fail-open by design: no tasks recorded (`[]`), a non-array, or an unreadable
|
|
109
|
+
* tasks.json (loadRunManifestById degrades to `tasks: []`) returns false so the
|
|
110
|
+
* legacy alert behavior is preserved — a broken tasks file must never silence
|
|
111
|
+
* a genuinely hung run.
|
|
112
|
+
*/
|
|
113
|
+
function areAllTasksTerminal(tasks: TeamTaskState[] | undefined): boolean {
|
|
114
|
+
if (!Array.isArray(tasks) || tasks.length === 0) return false;
|
|
115
|
+
return tasks.every((task) => TEAM_TERMINAL_TASK_STATUSES.has(task.status));
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* WD-INTERVAL (2026-10-23): resolve the poll interval. An explicit
|
|
120
|
+
* `checkIntervalMs` (tests / callers) wins, then the `PI_CREW_WATCHDOG_INTERVAL_MS`
|
|
121
|
+
* env override (live repro at 20s instead of 5 min), then the 5-minute default.
|
|
122
|
+
* Only positive finite values are honored — `0`/negative/garbage fall through.
|
|
123
|
+
*/
|
|
124
|
+
function resolveCheckIntervalMs(overrideMs: number | undefined): number {
|
|
125
|
+
if (typeof overrideMs === "number" && Number.isFinite(overrideMs) && overrideMs > 0) return overrideMs;
|
|
126
|
+
const configured = getCrewEnvInt("PI_CREW_WATCHDOG_INTERVAL_MS");
|
|
127
|
+
if (typeof configured === "number" && Number.isFinite(configured) && configured > 0) return configured;
|
|
128
|
+
return DEFAULT_CHECK_INTERVAL_MS;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* Pre-emit state re-verify loader. Injectable for tests to simulate the run
|
|
133
|
+
* going terminal between the cycle's initial read and the emit point.
|
|
134
|
+
*/
|
|
135
|
+
type ReverifyLoader = typeof loadRunManifestById;
|
|
136
|
+
let reverifyLoader: ReverifyLoader = loadRunManifestById;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* WD-DELIVER-GUARD (2026-10-23) — the residual the emit-time gate cannot see.
|
|
140
|
+
*
|
|
141
|
+
* Real incident (run team_20261023050900): the alert was EMITTED while the run
|
|
142
|
+
* was still running (so emit-time re-verify was correct), then
|
|
143
|
+
* `pi.sendUserMessage(..., {deliverAs:"followUp"})` pushed it into the host's
|
|
144
|
+
* follow-up queue while the session was streaming (`agent-session.js`:
|
|
145
|
+
* `if (this.isStreaming) this.agent.followUp(appMessage)`) and the host
|
|
146
|
+
* DELIVERED it 28 minutes later — after `run.completed` had landed. The user
|
|
147
|
+
* still saw "appears hung" for a finished run. `clearQueue()` is all-or-
|
|
148
|
+
* nothing, so a queued item cannot be retracted: the only fix is to not hand
|
|
149
|
+
* it over until delivery is imminent.
|
|
150
|
+
*
|
|
151
|
+
* Mechanism: while the session is streaming, the alert stays in OUR pending
|
|
152
|
+
* map; at the idle boundary (the moment the host delivers immediately) the run
|
|
153
|
+
* state is re-read one final time. Run terminal (completed/failed/cancelled/
|
|
154
|
+
* blocked) or all tasks terminal ⇒ DROP (debug-logged once per runId, no
|
|
155
|
+
* user-facing noise). FAIL-OPEN: an unreadable/absent record still delivers —
|
|
156
|
+
* a broken state file must never swallow a genuine hung alert.
|
|
157
|
+
*
|
|
158
|
+
* BOTH exits re-verify: the idle boundary AND the delivery cap. A session that
|
|
159
|
+
* never idles (one tool call blocking past the cap) is not a license to hand
|
|
160
|
+
* over a stale alert — the cap exit runs the same classify/verify, so an
|
|
161
|
+
* unverified delivery is impossible. "Late beats never" only applies to alerts
|
|
162
|
+
* whose run still reads as hung.
|
|
163
|
+
*/
|
|
164
|
+
const DEFAULT_DELIVER_RETRY_MS = 500;
|
|
165
|
+
const DEFAULT_DELIVER_CAP_MS = 10 * 60_000; // 10 min
|
|
166
|
+
/** Defensive bound — a burst of distinct hung runs must not grow the map. */
|
|
167
|
+
const HUNG_DELIVERY_CAP = 256;
|
|
168
|
+
|
|
169
|
+
interface PendingHungDelivery {
|
|
170
|
+
/** Full alert text, handed over verbatim at the idle boundary. */
|
|
171
|
+
text: string;
|
|
172
|
+
/** Past this instant the alert is handed over anyway (late beats never). */
|
|
173
|
+
deadline: number;
|
|
174
|
+
/** Idle probe captured at queue time (see WatchdogOptions.isSessionIdle). */
|
|
175
|
+
isSessionIdle?: () => boolean | undefined;
|
|
176
|
+
timer?: ReturnType<typeof setTimeout>;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Hung alerts queued but not yet handed to the host, keyed by runId. */
|
|
180
|
+
const pendingHungDeliveries = new Map<string, PendingHungDelivery>();
|
|
181
|
+
|
|
182
|
+
let deliverRetryMs = DEFAULT_DELIVER_RETRY_MS;
|
|
183
|
+
let deliverCapMs = DEFAULT_DELIVER_CAP_MS;
|
|
184
|
+
/** Delivery-time state loader (separate seam from the emit-time re-verify). */
|
|
185
|
+
let deliveryLoader: ReverifyLoader = loadRunManifestById;
|
|
186
|
+
|
|
187
|
+
/** @internal Test seam — reset module-level hung-alert dedup + delivery state. */
|
|
188
|
+
export function resetHungAlertDedupForTests(): void {
|
|
189
|
+
hungAlertRecords.clear();
|
|
190
|
+
for (const entry of pendingHungDeliveries.values()) {
|
|
191
|
+
if (entry.timer !== undefined) clearTimeout(entry.timer);
|
|
192
|
+
}
|
|
193
|
+
pendingHungDeliveries.clear();
|
|
194
|
+
reverifyLoader = loadRunManifestById;
|
|
195
|
+
deliveryLoader = loadRunManifestById;
|
|
196
|
+
deliverRetryMs = DEFAULT_DELIVER_RETRY_MS;
|
|
197
|
+
deliverCapMs = DEFAULT_DELIVER_CAP_MS;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** @internal Test seam — snapshot of the dedup map size (cap/leak assertions). */
|
|
201
|
+
export function hungAlertDedupSizeForTests(): number {
|
|
202
|
+
return hungAlertRecords.size;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** @internal Test seam — force-mark a runId as already alerted (cap assertions). */
|
|
206
|
+
export function markHungAlertForTests(runId: string): void {
|
|
207
|
+
updateHungAlertRecord(runId, { alerted: true });
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** @internal Test seam — replace the pre-emit re-verify loader (null = real). */
|
|
211
|
+
export function setReverifyLoaderForTests(loader: ReverifyLoader | null): void {
|
|
212
|
+
reverifyLoader = loader ?? loadRunManifestById;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** @internal Test seam — replace the DELIVERY-time state loader (null = real). */
|
|
216
|
+
export function setDeliveryLoaderForTests(loader: ReverifyLoader | null): void {
|
|
217
|
+
deliveryLoader = loader ?? loadRunManifestById;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** @internal Test seam — shrink the idle-gate retry/cap timings. */
|
|
221
|
+
export function setDeliveryTimingForTests(retryMs?: number, capMs?: number): void {
|
|
222
|
+
deliverRetryMs = typeof retryMs === "number" && retryMs > 0 ? retryMs : DEFAULT_DELIVER_RETRY_MS;
|
|
223
|
+
deliverCapMs = typeof capMs === "number" && capMs > 0 ? capMs : DEFAULT_DELIVER_CAP_MS;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** @internal Test seam — hung alerts queued but not yet handed to the host. */
|
|
227
|
+
export function hungAlertDeliveryPendingForTests(): number {
|
|
228
|
+
return pendingHungDeliveries.size;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Delivery-time verdict for a queued hung alert. `hung` = hand it to the host;
|
|
233
|
+
* `dropped` = the run is provably no longer hung, stay silent.
|
|
234
|
+
*/
|
|
235
|
+
type DeliveryVerdict = { kind: "hung" } | { kind: "dropped"; reason: string };
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Read the run state at handoff time. This is the WD-DELIVER-GUARD decision
|
|
239
|
+
* point: it runs immediately before `pi.sendUserMessage`, i.e. at the moment
|
|
240
|
+
* the host will actually deliver the message (session idle).
|
|
241
|
+
*
|
|
242
|
+
* Drop conditions (the exact user-visible pain):
|
|
243
|
+
* • manifest status terminal (completed/failed/cancelled/blocked), or
|
|
244
|
+
* • manifest still active BUT every task is terminal (stale record — the
|
|
245
|
+
* finalize/worker write is merely behind).
|
|
246
|
+
* FAIL-OPEN: a loader that throws, or a vanished manifest, yields `hung` — an
|
|
247
|
+
* unreadable record must never swallow a genuine alert.
|
|
248
|
+
*/
|
|
249
|
+
function classifyAtDelivery(cwd: string, runId: string): DeliveryVerdict {
|
|
250
|
+
let loaded: Awaited<ReturnType<ReverifyLoader>>;
|
|
251
|
+
try {
|
|
252
|
+
loaded = deliveryLoader(cwd, runId);
|
|
253
|
+
} catch {
|
|
254
|
+
return { kind: "hung" }; // unreadable state → fail-open
|
|
255
|
+
}
|
|
256
|
+
if (!loaded) return { kind: "hung" }; // indeterminate → fail-open
|
|
257
|
+
if (!isActiveRunStatus(loaded.manifest.status)) {
|
|
258
|
+
return { kind: "dropped", reason: `status=${loaded.manifest.status}` };
|
|
259
|
+
}
|
|
260
|
+
if (areAllTasksTerminal(loaded.tasks)) {
|
|
261
|
+
return { kind: "dropped", reason: `all ${loaded.tasks.length} task(s) terminal (manifest ${loaded.manifest.status})` };
|
|
262
|
+
}
|
|
263
|
+
return { kind: "hung" };
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** True when the host would deliver a followUp message right now. */
|
|
267
|
+
function sessionLooksIdle(probe: (() => boolean | undefined) | undefined): boolean {
|
|
268
|
+
if (typeof probe !== "function") return true; // no capability → pre-guard behavior
|
|
269
|
+
try {
|
|
270
|
+
return probe() !== false;
|
|
271
|
+
} catch {
|
|
272
|
+
return true;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Try to hand one pending hung alert to the host. Re-arms itself while the
|
|
278
|
+
* session is streaming (the host would queue instead of deliver) and, at the
|
|
279
|
+
* idle boundary or past the cap, re-reads the run state ONCE and either drops
|
|
280
|
+
* or delivers.
|
|
281
|
+
*/
|
|
282
|
+
function attemptHungAlertDelivery(pi: ExtensionAPI, cwd: string, runId: string, force: boolean): void {
|
|
283
|
+
const entry = pendingHungDeliveries.get(runId);
|
|
284
|
+
if (!entry) return;
|
|
285
|
+
if (entry.timer !== undefined) {
|
|
286
|
+
clearTimeout(entry.timer);
|
|
287
|
+
entry.timer = undefined;
|
|
288
|
+
}
|
|
289
|
+
if (!force && !sessionLooksIdle(entry.isSessionIdle) && Date.now() < entry.deadline) {
|
|
290
|
+
const timer = setTimeout(() => attemptHungAlertDelivery(pi, cwd, runId, false), deliverRetryMs);
|
|
291
|
+
timer.unref();
|
|
292
|
+
entry.timer = timer;
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// Fallthrough covers BOTH exits: the idle boundary and the cap expiry
|
|
297
|
+
// (`Date.now() >= entry.deadline`, session still streaming), plus any forced
|
|
298
|
+
// flush from the map cap. No exit skips the re-verify — an alert that got
|
|
299
|
+
// old is not evidence the run is still hung, so the same drop conditions
|
|
300
|
+
// apply here and the debug log names the reason it was dropped.
|
|
301
|
+
const verdict = classifyAtDelivery(cwd, runId);
|
|
302
|
+
pendingHungDeliveries.delete(runId);
|
|
303
|
+
if (verdict.kind === "dropped") {
|
|
304
|
+
if (!hungAlertRecords.get(runId)?.droppedAtDeliveryLogged) {
|
|
305
|
+
updateHungAlertRecord(runId, { droppedAtDeliveryLogged: true });
|
|
306
|
+
debugLog(`[foreground-watchdog] drop hung alert for ${runId} at delivery: run is no longer hung (${verdict.reason})`);
|
|
307
|
+
}
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
try {
|
|
311
|
+
pi.sendUserMessage(entry.text, { deliverAs: "followUp" });
|
|
312
|
+
} catch {
|
|
313
|
+
/* non-critical */
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* Queue a hung alert for guarded delivery (WD-DELIVER-GUARD). One in flight
|
|
319
|
+
* per runId — a second notice for the same run while the first is still
|
|
320
|
+
* pending is dropped as redundant drip.
|
|
321
|
+
*/
|
|
322
|
+
function queueHungAlertDelivery(
|
|
323
|
+
pi: ExtensionAPI,
|
|
324
|
+
cwd: string,
|
|
325
|
+
runId: string,
|
|
326
|
+
text: string,
|
|
327
|
+
isSessionIdle: (() => boolean | undefined) | undefined,
|
|
328
|
+
): void {
|
|
329
|
+
if (pendingHungDeliveries.has(runId)) {
|
|
330
|
+
debugLog(`[foreground-watchdog] hung alert for ${runId} already pending delivery — not stacking another`);
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
pendingHungDeliveries.set(runId, { text, deadline: Date.now() + deliverCapMs, isSessionIdle });
|
|
334
|
+
if (pendingHungDeliveries.size > HUNG_DELIVERY_CAP) {
|
|
335
|
+
// Defensive: never let the map grow and never delay a real alert —
|
|
336
|
+
// flush the oldest pending alert immediately (verify-then-send).
|
|
337
|
+
const oldest = pendingHungDeliveries.keys().next().value;
|
|
338
|
+
if (oldest !== undefined) attemptHungAlertDelivery(pi, cwd, oldest, true);
|
|
339
|
+
}
|
|
340
|
+
attemptHungAlertDelivery(pi, cwd, runId, false);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** Drop a still-pending (not yet handed-over) hung alert for a runId. */
|
|
344
|
+
function cancelPendingHungDelivery(runId: string): void {
|
|
345
|
+
const entry = pendingHungDeliveries.get(runId);
|
|
346
|
+
if (!entry) return;
|
|
347
|
+
if (entry.timer !== undefined) clearTimeout(entry.timer);
|
|
348
|
+
pendingHungDeliveries.delete(runId);
|
|
349
|
+
}
|
|
350
|
+
|
|
35
351
|
/** Stop a specific watchdog by runId. */
|
|
36
352
|
export function stopWatchdog(runId: string): void {
|
|
37
353
|
const timer = activeWatchdogs.get(runId);
|
|
@@ -39,6 +355,7 @@ export function stopWatchdog(runId: string): void {
|
|
|
39
355
|
clearTimeout(timer);
|
|
40
356
|
activeWatchdogs.delete(runId);
|
|
41
357
|
}
|
|
358
|
+
cancelPendingHungDelivery(runId);
|
|
42
359
|
}
|
|
43
360
|
|
|
44
361
|
/** @internal Stop all active watchdogs. Called on session shutdown (RC-01). */
|
|
@@ -47,6 +364,10 @@ export function stopAllWatchdogs(): void {
|
|
|
47
364
|
clearTimeout(timer);
|
|
48
365
|
}
|
|
49
366
|
activeWatchdogs.clear();
|
|
367
|
+
for (const entry of pendingHungDeliveries.values()) {
|
|
368
|
+
if (entry.timer !== undefined) clearTimeout(entry.timer);
|
|
369
|
+
}
|
|
370
|
+
pendingHungDeliveries.clear();
|
|
50
371
|
}
|
|
51
372
|
|
|
52
373
|
/**
|
|
@@ -62,7 +383,7 @@ export function stopAllWatchdogs(): void {
|
|
|
62
383
|
*/
|
|
63
384
|
export function startForegroundWatchdog(opts: WatchdogOptions): void {
|
|
64
385
|
const { pi, cwd, runId } = opts;
|
|
65
|
-
const checkIntervalMs = opts.checkIntervalMs
|
|
386
|
+
const checkIntervalMs = resolveCheckIntervalMs(opts.checkIntervalMs);
|
|
66
387
|
const maxMonitorMs = opts.maxMonitorMs ?? DEFAULT_MAX_MONITOR_MS;
|
|
67
388
|
const startTime = Date.now();
|
|
68
389
|
|
|
@@ -111,23 +432,79 @@ export function startForegroundWatchdog(opts: WatchdogOptions): void {
|
|
|
111
432
|
const agents = readCrewAgents(manifest);
|
|
112
433
|
const now = Date.now();
|
|
113
434
|
if (isLikelyOrphanedActiveRun(manifest, agents, now)) {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
435
|
+
// WD-DEDUP (2026-10-23): re-verify the run state right before any
|
|
436
|
+
// emit. A check armed by an older poll cycle can race the run
|
|
437
|
+
// reaching a terminal status — alerting "appears hung" for a run
|
|
438
|
+
// that already finished is exactly the repeated-alert pain.
|
|
439
|
+
// Terminal (or vanished) at re-verify → skip the emit entirely;
|
|
440
|
+
// the next cycle's terminal branch delivers the completion notice.
|
|
441
|
+
const reverified = reverifyLoader(cwd, runId);
|
|
442
|
+
if (!reverified || !isActiveRunStatus(reverified.manifest.status)) {
|
|
443
|
+
const reason = reverified ? `status=${reverified.manifest.status}` : "manifest gone";
|
|
444
|
+
if (!hungAlertRecords.get(runId)?.skipLogged) {
|
|
445
|
+
updateHungAlertRecord(runId, { skipLogged: true });
|
|
446
|
+
debugLog(`[foreground-watchdog] skip hung alert for ${runId}: re-verify saw ${reason}`);
|
|
447
|
+
}
|
|
448
|
+
// Fall through to the shared schedule — keep monitoring; the next
|
|
449
|
+
// cycle hits the terminal branch (completion notice) or !loaded.
|
|
450
|
+
} else if (areAllTasksTerminal(reverified.tasks)) {
|
|
451
|
+
// WD-STALE-RECORD: manifest still says running, but every task is
|
|
452
|
+
// terminal → the run is done and only the record is behind. Stay
|
|
453
|
+
// silent about "hung" and tell the user ONCE that the record is
|
|
454
|
+
// stale (per runId, module-scope — no spam across restarts).
|
|
455
|
+
if (!hungAlertRecords.get(runId)?.skipLogged) {
|
|
456
|
+
updateHungAlertRecord(runId, { skipLogged: true });
|
|
457
|
+
debugLog(
|
|
458
|
+
`[foreground-watchdog] skip hung alert for ${runId}: record stale (manifest ${reverified.manifest.status}, all ${reverified.tasks.length} task(s) terminal)`,
|
|
127
459
|
);
|
|
128
460
|
}
|
|
129
|
-
|
|
130
|
-
|
|
461
|
+
if (!hungAlertRecords.get(runId)?.staleNoticeSent) {
|
|
462
|
+
updateHungAlertRecord(runId, { staleNoticeSent: true });
|
|
463
|
+
try {
|
|
464
|
+
pi.sendUserMessage(`pi-crew watchdog: run ${runId} record stale (all tasks terminal) — skipped hung alert.`, {
|
|
465
|
+
deliverAs: "followUp",
|
|
466
|
+
});
|
|
467
|
+
} catch {
|
|
468
|
+
/* non-critical */
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
// Keep monitoring: if the manifest eventually finalizes, the
|
|
472
|
+
// terminal branch delivers the completion notice.
|
|
473
|
+
} else {
|
|
474
|
+
const detail = `status=${manifest.status}, updatedAt=${manifest.updatedAt}, agents=${agents.length}`;
|
|
475
|
+
consecutiveHungNotices += 1;
|
|
476
|
+
try {
|
|
477
|
+
if (consecutiveHungNotices <= HUNG_NOTICE_CAP) {
|
|
478
|
+
// WD-DEDUP: alert-once per runId at module scope — even a
|
|
479
|
+
// fresh watchdog instance must not re-alert the same run.
|
|
480
|
+
if (!hungAlertAlreadySent(runId)) {
|
|
481
|
+
// Mark BEFORE queueing so a racing cycle cannot double-queue.
|
|
482
|
+
updateHungAlertRecord(runId, { alerted: true });
|
|
483
|
+
// WD-DELIVER-GUARD: the host queues a followUp while the
|
|
484
|
+
// session streams and delivers it arbitrarily later — so
|
|
485
|
+
// hand it over only at the idle boundary, re-verifying the
|
|
486
|
+
// run state at THAT moment.
|
|
487
|
+
queueHungAlertDelivery(
|
|
488
|
+
pi,
|
|
489
|
+
cwd,
|
|
490
|
+
runId,
|
|
491
|
+
`pi-crew watchdog: run ${runId} appears hung (${detail}). Consider running team action='cancel' runId='${runId}' or team action='doctor'.`,
|
|
492
|
+
opts.isSessionIdle,
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
} else if (consecutiveHungNotices === HUNG_NOTICE_CAP + 1) {
|
|
496
|
+
// One-time handoff, then silence — still monitoring, no more drips.
|
|
497
|
+
queueHungAlertDelivery(
|
|
498
|
+
pi,
|
|
499
|
+
cwd,
|
|
500
|
+
runId,
|
|
501
|
+
`pi-crew watchdog: run ${runId} is still hung after ${consecutiveHungNotices} checks — going quiet now. Intervene via team action='cancel' runId='${runId}', team action='doctor', or leave it; a completion notice will still fire.`,
|
|
502
|
+
opts.isSessionIdle,
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
} catch {
|
|
506
|
+
/* non-critical */
|
|
507
|
+
}
|
|
131
508
|
}
|
|
132
509
|
// Keep monitoring (ARCH-5: silently past the cap — no more notices).
|
|
133
510
|
} else {
|
|
@@ -3,6 +3,7 @@ import * as os from "node:os";
|
|
|
3
3
|
import * as path from "node:path";
|
|
4
4
|
import { getCrewEnv } from "../../config/env-vars.ts";
|
|
5
5
|
import { errors } from "../../errors.ts";
|
|
6
|
+
import type { UsageState } from "../../state/types.ts";
|
|
6
7
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
7
8
|
import { fuzzyResolveModelId } from "./model-resolver.ts";
|
|
8
9
|
import { checkModelScope } from "./model-scope.ts";
|
|
@@ -19,6 +20,10 @@ export interface ModelAttemptSummary {
|
|
|
19
20
|
success: boolean;
|
|
20
21
|
exitCode?: number | null;
|
|
21
22
|
error?: string;
|
|
23
|
+
/** W10-7 (v0.11.38): THIS attempt's own usage (from its parsed output) —
|
|
24
|
+
* failed-fallback attempts burned real tokens that task.usage (the FINAL
|
|
25
|
+
* run's usage) never shows; post-execution reconciles the two. */
|
|
26
|
+
usage?: UsageState;
|
|
22
27
|
}
|
|
23
28
|
|
|
24
29
|
export interface ModelLike {
|
|
@@ -1193,6 +1193,9 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
1193
1193
|
success: !error,
|
|
1194
1194
|
exitCode,
|
|
1195
1195
|
error,
|
|
1196
|
+
// W10-7: capture THIS attempt's usage before the loop can move on —
|
|
1197
|
+
// failed attempts' tokens otherwise vanish from every persisted record.
|
|
1198
|
+
...(parsedOutput?.usage ? { usage: parsedOutput.usage } : {}),
|
|
1196
1199
|
};
|
|
1197
1200
|
modelAttempts.push(attempt);
|
|
1198
1201
|
task = {
|
|
@@ -27,6 +27,7 @@ import type {
|
|
|
27
27
|
TeamRunManifest,
|
|
28
28
|
TeamTaskState,
|
|
29
29
|
TeamTaskStatus,
|
|
30
|
+
UsageState,
|
|
30
31
|
VerificationEvidence,
|
|
31
32
|
} from "../../state/types.ts";
|
|
32
33
|
import { captureWorktreeDiffAsync, captureWorktreeDiffStatAsync } from "../../worktree/worktree-manager.ts";
|
|
@@ -38,6 +39,7 @@ import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../heartbeat/worker
|
|
|
38
39
|
import type { ModelAttemptSummary } from "../model/model-fallback.ts";
|
|
39
40
|
import { isStderrOnlyResult, type OutputValidationResult, validateWorkerOutput } from "../output/output-validator.ts";
|
|
40
41
|
import type { ParsedPiJsonOutput } from "../output/pi-json-output.ts";
|
|
42
|
+
import { permissionForRole } from "../role-permission.ts";
|
|
41
43
|
import { writeTaskSharedOutput } from "../task-output-context.ts";
|
|
42
44
|
import { evaluateCompletionMutationGuard } from "../verification/completion-guard.ts";
|
|
43
45
|
import { createVerificationEvidence } from "../verification/green-contract.ts";
|
|
@@ -139,6 +141,87 @@ export type VerifierVerdict = "pass" | "fail" | "none";
|
|
|
139
141
|
*/
|
|
140
142
|
const VERIFIER_VERDICT_LINE = /^[\s*_>#`=-]*(PASS|FAIL)[\s*_`]*(?=[:.\u2014\u2013-]|$|\()/i;
|
|
141
143
|
|
|
144
|
+
/** W10-5: the critic/oracle decision-drift guardian marker (see
|
|
145
|
+
* agents/critic.md + agents/oracle.md "Decision-drift guardian" sections). */
|
|
146
|
+
const DRIFT_BLOCK_MARKER = /DRIFT-BLOCK\b/i;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* W10-8 (pi-subagents lesson #4): bounded `git status --porcelain` probe for
|
|
150
|
+
* READ-ONLY lanes — any entry (modified OR untracked) is an unexpected
|
|
151
|
+
* mutation. Returns undefined when the tree is clean or git fails (fail-open:
|
|
152
|
+
* a broken probe must not break task finalization). Note-only, never a fail.
|
|
153
|
+
*/
|
|
154
|
+
export async function detectReadLaneMutation(worktreePath: string): Promise<NonNullable<TeamTaskState["unexpectedMutation"]> | undefined> {
|
|
155
|
+
try {
|
|
156
|
+
// W10-8 one-shot git probe per read-only task — child_process is
|
|
157
|
+
// already in the platform; dynamic keeps it off the static import graph.
|
|
158
|
+
// LAZY: see above
|
|
159
|
+
const { execFile } = await import("node:child_process");
|
|
160
|
+
const out: string = await new Promise((resolve, reject) => {
|
|
161
|
+
execFile("git", ["status", "--porcelain"], { cwd: worktreePath, encoding: "utf-8", timeout: 10_000 }, (err, stdout) =>
|
|
162
|
+
err ? reject(err) : resolve(stdout),
|
|
163
|
+
);
|
|
164
|
+
});
|
|
165
|
+
const entries = out
|
|
166
|
+
.split("\n")
|
|
167
|
+
.map((l) => l.trim())
|
|
168
|
+
.filter((l) => l.length > 0);
|
|
169
|
+
if (entries.length === 0) return undefined;
|
|
170
|
+
return {
|
|
171
|
+
filesChanged: entries.length,
|
|
172
|
+
sample: entries.slice(0, 20).map((e) => e.slice(0, 200)),
|
|
173
|
+
};
|
|
174
|
+
} catch {
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* W10-7 (v0.11.38, pi-subagents lesson #14 — usage reconciliation): compare
|
|
181
|
+
* the SUM of every model attempt's own cost (failed fallback attempts burn
|
|
182
|
+
* real tokens too, but task.usage carries only the FINAL run's numbers)
|
|
183
|
+
* against the final cost. Returns a usageReconcile note ONLY when the gap
|
|
184
|
+
* exceeds 5% AND at least two attempts reported usage — a single attempt is
|
|
185
|
+
* trivially self-consistent. Observability only: never fails the task.
|
|
186
|
+
*/
|
|
187
|
+
export function reconcileUsage(
|
|
188
|
+
attempts: ModelAttemptSummary[] | undefined,
|
|
189
|
+
finalUsage: UsageState | undefined,
|
|
190
|
+
): { usageReconcile: NonNullable<TeamTaskState["usageReconcile"]> } | undefined {
|
|
191
|
+
if (!attempts || attempts.length < 2) return undefined;
|
|
192
|
+
// W10-7 batch-A threshold guard: with no final usage to reconcile AGAINST
|
|
193
|
+
// (finalUsage missing, or its cost not a finite number) the comparison is
|
|
194
|
+
// meaningless — an attempt-sum alone cannot prove a gap, and comparing to
|
|
195
|
+
// 0 would file a false 100%-gap alarm against missing data. No note.
|
|
196
|
+
if (typeof finalUsage?.cost !== "number" || !Number.isFinite(finalUsage.cost)) return undefined;
|
|
197
|
+
const costs = attempts.map((a) => (typeof a.usage?.cost === "number" && Number.isFinite(a.usage.cost) ? a.usage.cost : undefined));
|
|
198
|
+
const withUsage = costs.filter((c): c is number => c !== undefined);
|
|
199
|
+
if (withUsage.length < 2) return undefined;
|
|
200
|
+
const attemptTotal = withUsage.reduce((sum, c) => sum + c, 0);
|
|
201
|
+
const finalCost = finalUsage.cost;
|
|
202
|
+
if (attemptTotal <= 0) return undefined;
|
|
203
|
+
const deltaPct = Math.abs(attemptTotal - finalCost) / attemptTotal;
|
|
204
|
+
if (deltaPct <= 0.05) return undefined;
|
|
205
|
+
return {
|
|
206
|
+
usageReconcile: {
|
|
207
|
+
attemptTotalCostUsd: Number(attemptTotal.toFixed(6)),
|
|
208
|
+
finalRunCostUsd: finalCost,
|
|
209
|
+
deltaPct: Number((deltaPct * 100).toFixed(1)),
|
|
210
|
+
attemptsWithUsage: withUsage.length,
|
|
211
|
+
},
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* W10-8 visibility (batch A): the unexpectedMutation note lives on the task
|
|
217
|
+
* record where it is easy to miss — stamping it also emits ONE stderr line
|
|
218
|
+
* so a READ-ONLY lane violation shows up in the run log without new
|
|
219
|
+
* dependencies. Exported for the console-capture pin; fire-and-forget.
|
|
220
|
+
*/
|
|
221
|
+
export function warnReadLaneMutation(taskId: string, note: { filesChanged: number }): void {
|
|
222
|
+
console.warn(`[pi-crew:read-lane] unexpected mutation in READ-ONLY lane: ${taskId} files=${note.filesChanged}`);
|
|
223
|
+
}
|
|
224
|
+
|
|
142
225
|
/**
|
|
143
226
|
* M3-2 (R47-3/V4, fix-wave 6): read the verifier verdict out of the
|
|
144
227
|
* authoritative result sources.
|
|
@@ -173,7 +256,18 @@ export function extractVerifierVerdict(...sources: Array<string | undefined>): V
|
|
|
173
256
|
if (match[1]?.toUpperCase() === "FAIL") return "fail";
|
|
174
257
|
sawPass = true;
|
|
175
258
|
}
|
|
176
|
-
if (sawPass)
|
|
259
|
+
if (sawPass) {
|
|
260
|
+
// W10-5 (v0.11.38, pi-subagents lesson #20 — oracle/drift-guardian): a
|
|
261
|
+
// verdict that flags a decision-drift contradiction (the `DRIFT-BLOCK:`
|
|
262
|
+
// marker the critic/oracle prompts mandate) is SELF-CONTRADICTORY as a
|
|
263
|
+
// pass — the worker saw a contradiction with an inherited decision and
|
|
264
|
+
// approved anyway. Degrade to "none" (not verified): the existing
|
|
265
|
+
// self-attested downgrade path then demands human/leader review instead
|
|
266
|
+
// of a green. FAIL still wins above — a flagged drift alongside FAIL is
|
|
267
|
+
// already the honest verdict.
|
|
268
|
+
if (DRIFT_BLOCK_MARKER.test(source)) return "none";
|
|
269
|
+
return "pass";
|
|
270
|
+
}
|
|
177
271
|
}
|
|
178
272
|
return "none";
|
|
179
273
|
}
|
|
@@ -320,6 +414,7 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
320
414
|
producer: task.id,
|
|
321
415
|
})
|
|
322
416
|
: undefined;
|
|
417
|
+
|
|
323
418
|
const diffStatArtifact = workspace.worktreePath
|
|
324
419
|
? writeArtifact(manifest.artifactsRoot, {
|
|
325
420
|
kind: "metadata",
|
|
@@ -328,6 +423,14 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
328
423
|
producer: task.id,
|
|
329
424
|
})
|
|
330
425
|
: undefined;
|
|
426
|
+
// W10-8 (v0.11.38, pi-subagents lesson #4 — staged-files guard): a READ-ONLY
|
|
427
|
+
// lane that dirtied its worktree is a contract violation worth surfacing —
|
|
428
|
+
// note only (the diff artifact above is the evidence; the leader decides).
|
|
429
|
+
// `git diff` alone misses UNTRACKED files, hence status --porcelain.
|
|
430
|
+
const readLaneMutation =
|
|
431
|
+
workspace.worktreePath && permissionForRole(task.role) === "read_only"
|
|
432
|
+
? await detectReadLaneMutation(workspace.worktreePath)
|
|
433
|
+
: undefined;
|
|
331
434
|
|
|
332
435
|
// Capture unified patches from edit tool results
|
|
333
436
|
const patchArtifact = parsedOutput?.patches?.length
|
|
@@ -715,6 +818,9 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
715
818
|
exitCode = exitCode === 0 ? 1 : exitCode;
|
|
716
819
|
}
|
|
717
820
|
|
|
821
|
+
// W10-8 batch-A visibility: emit the one-line stderr event exactly when
|
|
822
|
+
// the note below is stamped (only on detection).
|
|
823
|
+
if (readLaneMutation) warnReadLaneMutation(task.id, readLaneMutation);
|
|
718
824
|
task = {
|
|
719
825
|
...task,
|
|
720
826
|
status: error ? "failed" : noYield ? "needs_attention" : "completed",
|
|
@@ -722,6 +828,11 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
722
828
|
exitCode,
|
|
723
829
|
modelAttempts,
|
|
724
830
|
usage: parsedOutput?.usage,
|
|
831
|
+
// W10-7 (v0.11.38): reconcile per-attempt costs (failed fallbacks burn
|
|
832
|
+
// real tokens that the final run's usage never shows) against the final
|
|
833
|
+
// cost — observability note only, >5% gap with >1 usage-carrying attempt.
|
|
834
|
+
...(reconcileUsage(modelAttempts, parsedOutput?.usage) ?? {}),
|
|
835
|
+
...(readLaneMutation ? { unexpectedMutation: readLaneMutation } : {}),
|
|
725
836
|
jsonEvents: parsedOutput?.jsonEvents,
|
|
726
837
|
agentProgress:
|
|
727
838
|
error && task.agentProgress?.currentTool
|