pi-crew 0.11.41 → 0.11.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/dist/index.mjs +495 -495
- package/package.json +1 -1
- package/src/runtime/process/zombie-scanner.ts +97 -42
- package/src/runtime/process-status.ts +28 -1
- package/src/state/event-log/event-log.ts +42 -0
- package/src/ui/agents-jobs-browser.ts +34 -4
- package/src/ui/run-dashboard.ts +68 -13
package/package.json
CHANGED
|
@@ -97,11 +97,51 @@ export interface ForeignPiProcess {
|
|
|
97
97
|
cmd: string;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
+
/** Wave-14 F4 (GAP-3): injectable seam for `scanZombieSubagents`.
|
|
101
|
+
*
|
|
102
|
+
* Every ambient read the scanner performs is replaceable so classification
|
|
103
|
+
* tests can drive a SYNTHETIC /proc on every OS (previously 7 tests skipped
|
|
104
|
+
* forever on macOS/Windows CI). Defaults preserve behavior EXACTLY — each
|
|
105
|
+
* read-site resolves `opts.X ?? <the previous inline expression>` (the same
|
|
106
|
+
* injection pattern as reap-group.ts ReapGroupOptions, wave 12b). Production
|
|
107
|
+
* callers are no-arg (doctor.ts, orphan-reaper.ts) and see zero change. */
|
|
108
|
+
export interface ZombieScanOptions {
|
|
109
|
+
/** Platform gate. Default `process.platform`. */
|
|
110
|
+
platform?: NodeJS.Platform;
|
|
111
|
+
/** Real uid of the scanner process. Default `tryGetUid` (try/catch around
|
|
112
|
+
* `process.getuid?.()` — undefined on win32). */
|
|
113
|
+
getuid?: () => number | undefined;
|
|
114
|
+
/** File reader for /proc paths. Default `fs.readFileSync(p, "utf-8")` with
|
|
115
|
+
* read errors mapped to `undefined` (the previous per-helper catch). */
|
|
116
|
+
readProcFile?: (p: string) => string | undefined;
|
|
117
|
+
/** Candidate pid enumeration. Default: readdir of the real `/proc`. */
|
|
118
|
+
listProcPids?: () => number[];
|
|
119
|
+
/** Pid liveness probe. Default `isPidAlive` (`process.kill(pid, 0)` with the
|
|
120
|
+
* R7-A2 ESRCH/ENOENT semantics). */
|
|
121
|
+
pidAlive?: (pid: number) => boolean;
|
|
122
|
+
/** Scanner's own parent pid — the ancestor-chain starting point. Default
|
|
123
|
+
* `process.ppid`. Injection point for deterministic ancestor tests. */
|
|
124
|
+
getParentPid?: () => number;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Normalized /proc read: `undefined` = unreadable (missing, EPERM, …). */
|
|
128
|
+
type ReadProcFile = (p: string) => string | undefined;
|
|
129
|
+
|
|
130
|
+
/** Default seam reader — real fs with the previous per-helper catch folded in. */
|
|
131
|
+
const defaultReadProcFile: ReadProcFile = (p) => {
|
|
132
|
+
try {
|
|
133
|
+
return fs.readFileSync(p, "utf-8");
|
|
134
|
+
} catch {
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
|
|
100
139
|
/** Read /proc/<pid>/environ as a key=value record. Returns {} if unreadable. */
|
|
101
|
-
function readProcEnviron(pid: number): Record<string, string> {
|
|
140
|
+
function readProcEnviron(pid: number, read: ReadProcFile = defaultReadProcFile): Record<string, string> {
|
|
102
141
|
try {
|
|
103
142
|
// /proc/<pid>/environ is NUL-separated key=value pairs.
|
|
104
|
-
const raw =
|
|
143
|
+
const raw = read(`/proc/${pid}/environ`);
|
|
144
|
+
if (raw === undefined) return {};
|
|
105
145
|
const out: Record<string, string> = {};
|
|
106
146
|
for (const entry of raw.split("\0")) {
|
|
107
147
|
const eq = entry.indexOf("=");
|
|
@@ -114,9 +154,13 @@ function readProcEnviron(pid: number): Record<string, string> {
|
|
|
114
154
|
}
|
|
115
155
|
|
|
116
156
|
/** Read /proc/<pid>/stat to get ppid + starttime + elapsed. Returns undefined if unreadable. */
|
|
117
|
-
function readProcStat(
|
|
157
|
+
function readProcStat(
|
|
158
|
+
pid: number,
|
|
159
|
+
read: ReadProcFile = defaultReadProcFile,
|
|
160
|
+
): { ppid: number; starttime: number | undefined; elapsedSec: number | undefined } | undefined {
|
|
118
161
|
try {
|
|
119
|
-
const stat =
|
|
162
|
+
const stat = read(`/proc/${pid}/stat`);
|
|
163
|
+
if (stat === undefined) return undefined;
|
|
120
164
|
// stat format: pid (comm) state ppid ... starttime ...
|
|
121
165
|
// Shared paren-aware cut (proc-stat.ts) — MUST index identically with
|
|
122
166
|
// surface-spawn's PI_CREW_PARENT_START_TIME capture.
|
|
@@ -125,7 +169,7 @@ function readProcStat(pid: number): { ppid: number; starttime: number | undefine
|
|
|
125
169
|
const ppid = Number.parseInt(rest[PROC_STAT_PPID_INDEX] ?? "", 10);
|
|
126
170
|
const starttimeTicksRaw = Number.parseInt(rest[PROC_STAT_STARTTIME_INDEX] ?? "", 10);
|
|
127
171
|
const starttime = Number.isFinite(starttimeTicksRaw) ? starttimeTicksRaw : undefined;
|
|
128
|
-
const elapsedSec = computeElapsedSec(starttime);
|
|
172
|
+
const elapsedSec = computeElapsedSec(starttime, read);
|
|
129
173
|
return { ppid: Number.isFinite(ppid) ? ppid : 0, starttime, elapsedSec };
|
|
130
174
|
} catch {
|
|
131
175
|
return undefined;
|
|
@@ -133,9 +177,10 @@ function readProcStat(pid: number): { ppid: number; starttime: number | undefine
|
|
|
133
177
|
}
|
|
134
178
|
|
|
135
179
|
/** /proc/uptime first field (seconds since boot); undefined when unreadable. */
|
|
136
|
-
function readUptimeSec(): number | undefined {
|
|
180
|
+
function readUptimeSec(read: ReadProcFile = defaultReadProcFile): number | undefined {
|
|
137
181
|
try {
|
|
138
|
-
const uptimeRaw =
|
|
182
|
+
const uptimeRaw = read("/proc/uptime");
|
|
183
|
+
if (uptimeRaw === undefined) return undefined;
|
|
139
184
|
const uptimeSec = Number.parseFloat(uptimeRaw.split(" ")[0] ?? "");
|
|
140
185
|
return Number.isFinite(uptimeSec) ? uptimeSec : undefined;
|
|
141
186
|
} catch {
|
|
@@ -148,22 +193,22 @@ function readUptimeSec(): number | undefined {
|
|
|
148
193
|
* ticks — boot time = now - uptime, so startedAt = boot + starttime/CLK_TCK.
|
|
149
194
|
* Best-effort ISO string; undefined when /proc/uptime is unavailable.
|
|
150
195
|
*/
|
|
151
|
-
function computeStartedAtIso(starttime: number | undefined): string | undefined {
|
|
196
|
+
function computeStartedAtIso(starttime: number | undefined, read: ReadProcFile = defaultReadProcFile): string | undefined {
|
|
152
197
|
if (starttime === undefined || !Number.isFinite(starttime)) return undefined;
|
|
153
|
-
const uptimeSec = readUptimeSec();
|
|
198
|
+
const uptimeSec = readUptimeSec(read);
|
|
154
199
|
if (uptimeSec === undefined) return undefined;
|
|
155
200
|
const bootMs = Date.now() - uptimeSec * 1000;
|
|
156
201
|
return new Date(bootMs + (starttime / 100) * 1000).toISOString();
|
|
157
202
|
}
|
|
158
203
|
|
|
159
|
-
function computeElapsedSec(starttimeTicks: number | undefined): number | undefined {
|
|
204
|
+
function computeElapsedSec(starttimeTicks: number | undefined, read: ReadProcFile = defaultReadProcFile): number | undefined {
|
|
160
205
|
if (starttimeTicks === undefined || !Number.isFinite(starttimeTicks)) return undefined;
|
|
161
206
|
try {
|
|
162
207
|
// Linux CLK_TCK is virtually always 100 (sysconf(_SC_CLK_TCK)). Reading it
|
|
163
208
|
// portably from Node requires a native addon; hardcoding 100 matches every
|
|
164
209
|
// mainstream Linux distro and keeps this dependency-free.
|
|
165
210
|
const ticksPerSec = 100;
|
|
166
|
-
const uptimeSec = readUptimeSec();
|
|
211
|
+
const uptimeSec = readUptimeSec(read);
|
|
167
212
|
if (uptimeSec === undefined) return undefined;
|
|
168
213
|
// starttime (ticks since boot) → process age in seconds = uptime - starttime/ticksPerSec.
|
|
169
214
|
const startAgeSec = starttimeTicks / ticksPerSec;
|
|
@@ -188,24 +233,26 @@ function isPidAlive(pid: number): boolean {
|
|
|
188
233
|
}
|
|
189
234
|
}
|
|
190
235
|
|
|
191
|
-
function readProcCmdline(pid: number): string {
|
|
192
|
-
return readProcCmdlineTokens(pid).join(" ").trim() || `pid ${pid}`;
|
|
236
|
+
function readProcCmdline(pid: number, read: ReadProcFile = defaultReadProcFile): string {
|
|
237
|
+
return readProcCmdlineTokens(pid, read).join(" ").trim() || `pid ${pid}`;
|
|
193
238
|
}
|
|
194
239
|
|
|
195
240
|
/** /proc/<pid>/cmdline as argv tokens (NUL-split). Empty array if unreadable. */
|
|
196
|
-
function readProcCmdlineTokens(pid: number): string[] {
|
|
241
|
+
function readProcCmdlineTokens(pid: number, read: ReadProcFile = defaultReadProcFile): string[] {
|
|
197
242
|
try {
|
|
198
243
|
// /proc/<pid>/cmdline is NUL-separated argv.
|
|
199
|
-
const raw =
|
|
244
|
+
const raw = read(`/proc/${pid}/cmdline`);
|
|
245
|
+
if (raw === undefined) return [];
|
|
200
246
|
return raw.split("\0").filter(Boolean);
|
|
201
247
|
} catch {
|
|
202
248
|
return [];
|
|
203
249
|
}
|
|
204
250
|
}
|
|
205
251
|
|
|
206
|
-
function readProcRssKb(pid: number): number {
|
|
252
|
+
function readProcRssKb(pid: number, read: ReadProcFile = defaultReadProcFile): number {
|
|
207
253
|
try {
|
|
208
|
-
const status =
|
|
254
|
+
const status = read(`/proc/${pid}/status`);
|
|
255
|
+
if (status === undefined) return 0;
|
|
209
256
|
const match = status.match(/^VmRSS:\s+(\d+)\s+kB/m);
|
|
210
257
|
return match ? Number.parseInt(match[1] ?? "", 10) : 0;
|
|
211
258
|
} catch {
|
|
@@ -221,8 +268,8 @@ function readProcRssKb(pid: number): number {
|
|
|
221
268
|
* non-Linux platforms the scanner returns an empty result with a note in
|
|
222
269
|
* `errors` (zombie detection is best-effort; the doctor report still renders).
|
|
223
270
|
*/
|
|
224
|
-
function listCandidatePids(): number[] {
|
|
225
|
-
if (
|
|
271
|
+
function listCandidatePids(platform: NodeJS.Platform = process.platform): number[] {
|
|
272
|
+
if (platform !== "linux") return [];
|
|
226
273
|
const pids: number[] = [];
|
|
227
274
|
try {
|
|
228
275
|
for (const entry of fs.readdirSync("/proc")) {
|
|
@@ -241,23 +288,30 @@ function listCandidatePids(): number[] {
|
|
|
241
288
|
* any scan errors. Callers decide what to do with the result; this module
|
|
242
289
|
* has no side effects.
|
|
243
290
|
*/
|
|
244
|
-
export function scanZombieSubagents(): ZombieScanResult {
|
|
291
|
+
export function scanZombieSubagents(opts: ZombieScanOptions = {}): ZombieScanResult {
|
|
292
|
+
// Wave-14 F4: resolve the seam ONCE — every default below is the exact
|
|
293
|
+
// previous inline expression, so no-arg callers are byte-for-byte behavior
|
|
294
|
+
// preserved (doctor.ts:990, orphan-reaper.ts:306).
|
|
295
|
+
const platform = opts.platform ?? process.platform;
|
|
296
|
+
const read = opts.readProcFile ?? defaultReadProcFile;
|
|
297
|
+
const pidAlive = opts.pidAlive ?? isPidAlive;
|
|
245
298
|
const result: ZombieScanResult = { zombies: [], live: [], foreign: [], errors: [] };
|
|
246
|
-
if (
|
|
247
|
-
result.errors.push("zombie scan is Linux-only (/proc required); skipping on " +
|
|
299
|
+
if (platform !== "linux") {
|
|
300
|
+
result.errors.push("zombie scan is Linux-only (/proc required); skipping on " + platform);
|
|
248
301
|
return result;
|
|
249
302
|
}
|
|
250
303
|
|
|
251
|
-
const myUid = tryGetUid();
|
|
304
|
+
const myUid = (opts.getuid ?? tryGetUid)();
|
|
252
305
|
// R3-16: self + ancestors are live hosting sessions (main session / crew
|
|
253
306
|
// host chain) — never reportable as foreign, even when they carry markers.
|
|
254
|
-
const ancestors = collectAncestorPids();
|
|
255
|
-
|
|
307
|
+
const ancestors = collectAncestorPids(read, opts.getParentPid);
|
|
308
|
+
const listPids = opts.listProcPids ?? (() => listCandidatePids(platform));
|
|
309
|
+
for (const pid of listPids()) {
|
|
256
310
|
try {
|
|
257
311
|
// Cheap rejection first: only inspect processes we own (avoid scanning system procs).
|
|
258
|
-
if (myUid !== undefined && getProcUid(pid) !== myUid) continue;
|
|
312
|
+
if (myUid !== undefined && getProcUid(pid, read) !== myUid) continue;
|
|
259
313
|
|
|
260
|
-
const environ = readProcEnviron(pid);
|
|
314
|
+
const environ = readProcEnviron(pid, read);
|
|
261
315
|
// AUTHORITATIVE GATE: a process is a pi-crew sub-agent ONLY if it carries
|
|
262
316
|
// PI_CREW_KIND=subagent. The user's main session never sets this, so it can
|
|
263
317
|
// never be matched — this is the fix for accidentally killing main sessions.
|
|
@@ -271,18 +325,18 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
271
325
|
// them (pi-crew also scrubs these markers from worker env).
|
|
272
326
|
const hasPiMarker = environ.AI_AGENT === "pi" || environ.PI_CODING_AGENT === "true";
|
|
273
327
|
if (hasPiMarker && pid !== process.pid && !ancestors.has(pid)) {
|
|
274
|
-
const tokens = readProcCmdlineTokens(pid);
|
|
328
|
+
const tokens = readProcCmdlineTokens(pid, read);
|
|
275
329
|
if (looksLikePiBinary(tokens)) {
|
|
276
|
-
const foreignStat = readProcStat(pid);
|
|
330
|
+
const foreignStat = readProcStat(pid, read);
|
|
277
331
|
const ppid = foreignStat?.ppid ?? 0;
|
|
278
332
|
result.foreign.push({
|
|
279
333
|
pid,
|
|
280
334
|
ppid,
|
|
281
|
-
orphaned: ppid <= 1 || !
|
|
335
|
+
orphaned: ppid <= 1 || !pidAlive(ppid),
|
|
282
336
|
headless: isHeadlessArgv(tokens),
|
|
283
|
-
rssKb: readProcRssKb(pid),
|
|
337
|
+
rssKb: readProcRssKb(pid, read),
|
|
284
338
|
elapsedSec: foreignStat?.elapsedSec,
|
|
285
|
-
cmd: readProcCmdline(pid),
|
|
339
|
+
cmd: readProcCmdline(pid, read),
|
|
286
340
|
});
|
|
287
341
|
}
|
|
288
342
|
}
|
|
@@ -290,7 +344,7 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
290
344
|
}
|
|
291
345
|
|
|
292
346
|
const crewParentPid = Number.parseInt(environ.PI_CREW_PARENT_PID ?? "", 10);
|
|
293
|
-
const stat = readProcStat(pid);
|
|
347
|
+
const stat = readProcStat(pid, read);
|
|
294
348
|
// Surface identity (T12): strict allowlist — an unrelated PI_CREW_SURFACE
|
|
295
349
|
// value must not half-populate the fields doctor relies on for pane close.
|
|
296
350
|
const surface = environ.PI_CREW_SURFACE === "tmux" || environ.PI_CREW_SURFACE === "herdr" ? environ.PI_CREW_SURFACE : undefined;
|
|
@@ -298,17 +352,17 @@ export function scanZombieSubagents(): ZombieScanResult {
|
|
|
298
352
|
pid,
|
|
299
353
|
ppid: stat?.ppid ?? 0,
|
|
300
354
|
crewParentPid: Number.isFinite(crewParentPid) ? crewParentPid : 0,
|
|
301
|
-
parentAlive: Number.isFinite(crewParentPid) &&
|
|
355
|
+
parentAlive: Number.isFinite(crewParentPid) && pidAlive(crewParentPid),
|
|
302
356
|
role: environ.PI_CREW_ROLE,
|
|
303
357
|
surface,
|
|
304
358
|
surfacePaneId: environ.PI_CREW_SURFACE_PANE || undefined,
|
|
305
|
-
rssKb: readProcRssKb(pid),
|
|
359
|
+
rssKb: readProcRssKb(pid, read),
|
|
306
360
|
elapsedSec: stat?.elapsedSec,
|
|
307
361
|
// M-7: starttime (ticks) + derived wall-clock start — pid identity pin
|
|
308
362
|
// for kill-time PID-reuse checks and prompt-kill triage.
|
|
309
363
|
starttime: stat?.starttime,
|
|
310
|
-
startedAt: computeStartedAtIso(stat?.starttime),
|
|
311
|
-
cmd: readProcCmdline(pid),
|
|
364
|
+
startedAt: computeStartedAtIso(stat?.starttime, read),
|
|
365
|
+
cmd: readProcCmdline(pid, read),
|
|
312
366
|
};
|
|
313
367
|
|
|
314
368
|
if (entry.parentAlive) {
|
|
@@ -338,10 +392,11 @@ function tryGetUid(): number | undefined {
|
|
|
338
392
|
}
|
|
339
393
|
}
|
|
340
394
|
|
|
341
|
-
function getProcUid(pid: number): number | undefined {
|
|
395
|
+
function getProcUid(pid: number, read: ReadProcFile = defaultReadProcFile): number | undefined {
|
|
342
396
|
try {
|
|
343
397
|
// /proc/<pid>/status has Uid: <real> <eff> <sav> <fs>
|
|
344
|
-
const status =
|
|
398
|
+
const status = read(`/proc/${pid}/status`);
|
|
399
|
+
if (status === undefined) return undefined;
|
|
345
400
|
const match = status.match(/^Uid:\s+(\d+)/m);
|
|
346
401
|
return match ? Number.parseInt(match[1] ?? "", 10) : undefined;
|
|
347
402
|
} catch {
|
|
@@ -393,13 +448,13 @@ function isHeadlessArgv(tokens: string[]): boolean {
|
|
|
393
448
|
* it sit on this chain and carry markers when pi-in-pi). Bounded at 64 hops to
|
|
394
449
|
* survive a hypothetical pid cycle.
|
|
395
450
|
*/
|
|
396
|
-
function collectAncestorPids(): Set<number> {
|
|
451
|
+
function collectAncestorPids(read: ReadProcFile = defaultReadProcFile, getParentPid?: () => number): Set<number> {
|
|
397
452
|
const chain = new Set<number>();
|
|
398
|
-
let current = process.ppid;
|
|
453
|
+
let current = getParentPid !== undefined ? getParentPid() : process.ppid;
|
|
399
454
|
let guard = 0;
|
|
400
455
|
while (current > 1 && guard++ < 64) {
|
|
401
456
|
chain.add(current);
|
|
402
|
-
const stat = readProcStat(current);
|
|
457
|
+
const stat = readProcStat(current, read);
|
|
403
458
|
if (!stat || stat.ppid <= 0) break;
|
|
404
459
|
current = stat.ppid;
|
|
405
460
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { TeamRunManifest } from "../state/types.ts";
|
|
1
|
+
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
2
2
|
import type { CrewAgentRecord } from "./crew-agent-runtime.ts";
|
|
3
3
|
|
|
4
4
|
export { hasAsyncStartMarker } from "./async-marker.ts";
|
|
@@ -92,6 +92,33 @@ export function hasRecentAgentActivityEvidence(agent: CrewAgentRecord, now = Dat
|
|
|
92
92
|
return Number.isFinite(at) && now - at < livenessMs;
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Wave 15 (F1, GAP-0): does any running/waiting task carry FRESH live
|
|
97
|
+
* evidence — a `heartbeat.lastSeenAt` (with `alive !== false`) or the
|
|
98
|
+
* `agentProgress.lastActivityAt` fallback — within `ttlMs`? Behavioral
|
|
99
|
+
* mirror of the watchdog's private helper (foreground-watchdog.ts) and
|
|
100
|
+
* hasRecentActiveEvidence (stale-reconciler.ts), so watchdog, reconciler
|
|
101
|
+
* and display stop growing a third divergent private copy. PURE (GAP-4:
|
|
102
|
+
* the shared display predicates keep their signatures — call sites that
|
|
103
|
+
* already hold the tasks OR this in). Default TTL 5 min keeps the constant
|
|
104
|
+
* family aligned (AGENT_ACTIVITY_LIVENESS_MS = TASK_HEARTBEAT_EVIDENCE_MS
|
|
105
|
+
* = ACTIVE_EVIDENCE_TTL_MS). Undefined/empty/all-terminal ⇒ false.
|
|
106
|
+
*/
|
|
107
|
+
export function hasFreshTaskHeartbeatEvidence(
|
|
108
|
+
tasks: TeamTaskState[] | undefined,
|
|
109
|
+
now = Date.now(),
|
|
110
|
+
ttlMs = AGENT_ACTIVITY_LIVENESS_MS,
|
|
111
|
+
): boolean {
|
|
112
|
+
if (!Array.isArray(tasks)) return false;
|
|
113
|
+
return tasks.some((task) => {
|
|
114
|
+
if (task.status !== "running" && task.status !== "waiting") return false;
|
|
115
|
+
const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
|
|
116
|
+
if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= ttlMs) return true;
|
|
117
|
+
const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
|
|
118
|
+
return Number.isFinite(activityAt) && now - activityAt <= ttlMs;
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
95
122
|
export function isLikelyOrphanedActiveRun(
|
|
96
123
|
run: TeamRunManifest,
|
|
97
124
|
agents: CrewAgentRecord[] = [],
|
|
@@ -1376,6 +1376,48 @@ export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEv
|
|
|
1376
1376
|
appendEventAsync(eventsPath, event).catch((error) => logInternalError("event-log.fire-and-forget", error, eventsPath));
|
|
1377
1377
|
}
|
|
1378
1378
|
|
|
1379
|
+
/**
|
|
1380
|
+
* Wave-14 F1 (GAP-0): await a run's in-flight event-log writers.
|
|
1381
|
+
*
|
|
1382
|
+
* Safe to call from tests before removing a run state dir: awaits all
|
|
1383
|
+
* in-flight async appends for the given events path (all paths when omitted)
|
|
1384
|
+
* and flushes its buffer. No-op when nothing is in flight.
|
|
1385
|
+
*
|
|
1386
|
+
* - With `eventsPath`: awaiting the per-path queue TAIL settles the entire
|
|
1387
|
+
* write chain (tail-chaining guarantees the tail completes last), then
|
|
1388
|
+
* flushes that path's buffered tier via the internal `flushOneEventLogBuffer`
|
|
1389
|
+
* (per-path flush is a simple direct call — no complexity caveat).
|
|
1390
|
+
* The Map entry may already have self-deleted (compare-and-delete on
|
|
1391
|
+
* settle) — `?? Promise.resolve()` makes that a clean no-op.
|
|
1392
|
+
* - Without `eventsPath`: awaits every queue tail + `flushEventLogBuffer()`.
|
|
1393
|
+
*
|
|
1394
|
+
* Never throws: a rejected tail is absorbed by `allSettled` (a teardown
|
|
1395
|
+
* helper must never fail because an old write failed — the error-reset path
|
|
1396
|
+
* in appendEventAsync already logs and resets the queue). The buffer flush
|
|
1397
|
+
* is best-effort for the same reason. The MAX_ASYNC_QUEUES (256) cap is
|
|
1398
|
+
* irrelevant here: drain-by-key touches one entry, global drain iterates
|
|
1399
|
+
* whatever currently exists. `drainAsyncQueues` stays private — this is the
|
|
1400
|
+
* public, path-scoped surface.
|
|
1401
|
+
*/
|
|
1402
|
+
export async function drainEventLogWriters(eventsPath?: string): Promise<void> {
|
|
1403
|
+
if (eventsPath !== undefined) {
|
|
1404
|
+
const tail = asyncQueues.get(eventsPath) ?? Promise.resolve();
|
|
1405
|
+
await Promise.allSettled([tail]);
|
|
1406
|
+
try {
|
|
1407
|
+
await flushOneEventLogBuffer(eventsPath);
|
|
1408
|
+
} catch {
|
|
1409
|
+
/* best-effort — teardown must not fail on a flush error */
|
|
1410
|
+
}
|
|
1411
|
+
return;
|
|
1412
|
+
}
|
|
1413
|
+
await Promise.allSettled([...asyncQueues.values()]);
|
|
1414
|
+
try {
|
|
1415
|
+
await flushEventLogBuffer();
|
|
1416
|
+
} catch {
|
|
1417
|
+
/* best-effort */
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
|
|
1379
1421
|
// Auto-flush on process exit so buffered events do not silently leak.
|
|
1380
1422
|
// Defense-in-depth: SIGTERM/SIGINT use setImmediate so the handler returns
|
|
1381
1423
|
// immediately and the main thread is not blocked by sync I/O.
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
* dashboard overlay, so headless sessions never reach it.
|
|
50
50
|
*/
|
|
51
51
|
|
|
52
|
-
import { existsSync, readdirSync } from "node:fs";
|
|
52
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
53
53
|
import { join } from "node:path";
|
|
54
54
|
import { fileURLToPath } from "node:url";
|
|
55
55
|
import { matchesKey } from "@earendil-works/pi-tui";
|
|
@@ -60,13 +60,14 @@ import type { CrewAgentRecord } from "../runtime/crew-agent-runtime.ts";
|
|
|
60
60
|
import type { LiveAgentHandle } from "../runtime/live-session/live-agent-manager.ts";
|
|
61
61
|
import type { ManifestCache } from "../runtime/manifest-cache.ts";
|
|
62
62
|
import { createManifestCache } from "../runtime/manifest-cache.ts";
|
|
63
|
-
import { isDisplayActiveRun } from "../runtime/process-status.ts";
|
|
63
|
+
import { hasFreshTaskHeartbeatEvidence, isActiveRunStatus, isDisplayActiveRun } from "../runtime/process-status.ts";
|
|
64
64
|
import type { ScheduledJob } from "../runtime/scheduling/scheduler.ts";
|
|
65
65
|
import { surfaceGateEnvSnapshot } from "../runtime/surface/resolve-surface.ts";
|
|
66
66
|
import { getTaskUsage } from "../runtime/usage-tracker.ts";
|
|
67
67
|
import { readJsonFile } from "../state/atomic-write.ts";
|
|
68
68
|
import { loadRunManifestById } from "../state/stores/state-store.ts";
|
|
69
|
-
import type { TeamRunManifest } from "../state/types.ts";
|
|
69
|
+
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
70
|
+
import { readJsonFileCoalesced } from "../utils/file-coalescer.ts";
|
|
70
71
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
71
72
|
import { formatRelativeTime } from "../utils/relative-time.ts";
|
|
72
73
|
import { pad, sanitizeLine, truncate, visibleWidth } from "../utils/visual.ts";
|
|
@@ -202,6 +203,28 @@ function watchAgentTranscriptScript(): string | undefined {
|
|
|
202
203
|
return candidates.find((c) => existsSync(c));
|
|
203
204
|
}
|
|
204
205
|
|
|
206
|
+
/** Tasks-read TTL for the legacy-discovery heartbeat check (coalesced, live-run-sidebar pattern). */
|
|
207
|
+
const LEGACY_TASKS_READ_TTL_MS = 1000;
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Wave 15 (F2, GAP-0): coalesced tasks.json read for the legacy-discovery
|
|
211
|
+
* loop — mirrors run-dashboard/live-run-sidebar's readRunTasks. Returns
|
|
212
|
+
* undefined on unreadable so a broken tasks.json can NEVER fabricate
|
|
213
|
+
* liveness evidence (fail-closed).
|
|
214
|
+
*/
|
|
215
|
+
function readLegacyRunTasks(run: TeamRunManifest): TeamTaskState[] | undefined {
|
|
216
|
+
const parse = (): TeamTaskState[] => {
|
|
217
|
+
if (!existsSync(run.tasksPath)) return [];
|
|
218
|
+
const parsed = JSON.parse(readFileSync(run.tasksPath, "utf-8")) as unknown;
|
|
219
|
+
return Array.isArray(parsed) ? (parsed as TeamTaskState[]) : [];
|
|
220
|
+
};
|
|
221
|
+
try {
|
|
222
|
+
return readJsonFileCoalesced(run.tasksPath, LEGACY_TASKS_READ_TTL_MS, parse);
|
|
223
|
+
} catch {
|
|
224
|
+
return undefined;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
|
|
205
228
|
/** tok/s for record-based entries (no live handle in this process). */
|
|
206
229
|
function recordTokPerSec(record: CrewAgentRecord, nowMs: number): number | undefined {
|
|
207
230
|
if (record.status !== "running") return undefined;
|
|
@@ -476,7 +499,14 @@ export class AgentsJobsBrowser {
|
|
|
476
499
|
if (run.status !== "running" && run.status !== "queued") continue;
|
|
477
500
|
try {
|
|
478
501
|
const agents = readCrewAgents(run);
|
|
479
|
-
|
|
502
|
+
// Wave 15 (F2, GAP-0): tight heartbeat guard — a legacy-uncovered run
|
|
503
|
+
// whose manifest froze mid-long-tool stays listed while its task
|
|
504
|
+
// heartbeats are fresh (this loop already does per-run I/O — one more
|
|
505
|
+
// coalesced read is in-pattern). Terminal/async runs are never
|
|
506
|
+
// resurrected (isActiveRunStatus && !run.async).
|
|
507
|
+
const heartbeatLive =
|
|
508
|
+
isActiveRunStatus(run.status) && !run.async && hasFreshTaskHeartbeatEvidence(readLegacyRunTasks(run), nowMs);
|
|
509
|
+
if (isDisplayActiveRun(run, agents, nowMs) || heartbeatLive) runs.push({ run, agents });
|
|
480
510
|
} catch {
|
|
481
511
|
/* unreadable — skip */
|
|
482
512
|
}
|
package/src/ui/run-dashboard.ts
CHANGED
|
@@ -10,7 +10,12 @@ import { readCrewAgents } from "../runtime/crew-agent-records.ts";
|
|
|
10
10
|
import type { CrewAgentRecord } from "../runtime/crew-agent-runtime.ts";
|
|
11
11
|
import { getLiveAgentContextPercent } from "../runtime/live-session/live-agent-manager.ts";
|
|
12
12
|
import { isPlanApprovalPending } from "../runtime/plan-approval.ts";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
hasFreshTaskHeartbeatEvidence,
|
|
15
|
+
isActiveRunStatus,
|
|
16
|
+
isDisplayActiveRun,
|
|
17
|
+
isLikelyOrphanedActiveRun,
|
|
18
|
+
} from "../runtime/process-status.ts";
|
|
14
19
|
import type { ScheduledJob } from "../runtime/scheduling/scheduler.ts";
|
|
15
20
|
import type { TeamRunManifest, TeamTaskState, UsageState } from "../state/types.ts";
|
|
16
21
|
import { aggregateUsage } from "../state/usage.ts";
|
|
@@ -363,6 +368,19 @@ function taskForAgent(tasks: TeamTaskState[], agent: CrewAgentRecord): TeamTaskS
|
|
|
363
368
|
return tasks.find((task) => task.id === agent.taskId);
|
|
364
369
|
}
|
|
365
370
|
|
|
371
|
+
/**
|
|
372
|
+
* Wave 15 (F2, GAP-0): tight heartbeat-liveness guard for ACTIVE display —
|
|
373
|
+
* mirrors the orphan predicate's own preconditions (active status, non-async)
|
|
374
|
+
* so a terminal or async run is never resurrected by a heartbeat. A healthy
|
|
375
|
+
* direct run mid-long-tool freezes manifest `updatedAt` and agent progress
|
|
376
|
+
* (event-driven writes) but keeps stamping `task.heartbeat.lastSeenAt`, so
|
|
377
|
+
* fresh evidence means the run is alive. Call sites OR this in ONLY — every
|
|
378
|
+
* non-stale branch and layout stays untouched.
|
|
379
|
+
*/
|
|
380
|
+
function heartbeatLiveForDisplay(run: TeamRunManifest, tasks: TeamTaskState[] | undefined, now = Date.now()): boolean {
|
|
381
|
+
return isActiveRunStatus(run.status) && !run.async && hasFreshTaskHeartbeatEvidence(tasks, now);
|
|
382
|
+
}
|
|
383
|
+
|
|
366
384
|
function modelForTask(task: TeamTaskState | undefined): string | undefined {
|
|
367
385
|
const attempts = task?.modelAttempts;
|
|
368
386
|
if (!attempts?.length) return undefined;
|
|
@@ -467,7 +485,9 @@ function runLabel(
|
|
|
467
485
|
resolve?: SnapshotResolver,
|
|
468
486
|
): string {
|
|
469
487
|
const agents = agentsFor(run, snapshotCache, resolve);
|
|
470
|
-
|
|
488
|
+
// Wave 15 (F2): a stale-manifest verdict is VETOED by fresh task-heartbeat
|
|
489
|
+
// evidence (lazy read — only consulted when the orphan predicate fires).
|
|
490
|
+
const stale = isLikelyOrphanedActiveRun(run, agents) && !heartbeatLiveForDisplay(run, readRunTasks(run, snapshotCache, resolve));
|
|
471
491
|
const running = agents.find((agent) => agent.status === "running");
|
|
472
492
|
const queued = agents.find((agent) => agent.status === "queued");
|
|
473
493
|
const step = stale
|
|
@@ -523,7 +543,12 @@ function resolveRuns(runs: TeamRunManifest[], snapshotCache?: RunSnapshotCache,
|
|
|
523
543
|
const snapshot = resolve ? resolve(run) : snapshotFor(run, snapshotCache);
|
|
524
544
|
const agents = snapshot?.agents ?? agentsFor(run, snapshotCache, resolve);
|
|
525
545
|
const displayRun = snapshot?.manifest ?? run;
|
|
526
|
-
|
|
546
|
+
// Wave 15 (F2): fresh task heartbeats veto the "stale" label.
|
|
547
|
+
const status: RunStatus =
|
|
548
|
+
isLikelyOrphanedActiveRun(displayRun, agents) &&
|
|
549
|
+
!heartbeatLiveForDisplay(displayRun, snapshot?.tasks ?? readRunTasks(run, snapshotCache, resolve))
|
|
550
|
+
? "stale"
|
|
551
|
+
: (displayRun.status as RunStatus);
|
|
527
552
|
map.set(run.runId, { manifest: run, snapshot, agents, status });
|
|
528
553
|
}
|
|
529
554
|
return map;
|
|
@@ -536,12 +561,19 @@ function groupedRuns(
|
|
|
536
561
|
): Array<{ label: string; run?: TeamRunManifest }> {
|
|
537
562
|
const resolved = resolveRuns(runs, snapshotCache, resolve);
|
|
538
563
|
const rows: Array<{ label: string; run?: TeamRunManifest }> = [];
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
)
|
|
542
|
-
const
|
|
543
|
-
|
|
544
|
-
|
|
564
|
+
// Wave 15 (F2): a run with fresh task-heartbeat evidence stays display-active
|
|
565
|
+
// even when the manifest froze mid-long-tool (tight guard — never resurrects
|
|
566
|
+
// terminal/async runs).
|
|
567
|
+
const displayActive = (run: TeamRunManifest): boolean => {
|
|
568
|
+
const entry = resolved.get(run.runId);
|
|
569
|
+
const displayRun = entry?.snapshot?.manifest ?? run;
|
|
570
|
+
return (
|
|
571
|
+
isDisplayActiveRun(displayRun, entry?.agents ?? []) ||
|
|
572
|
+
heartbeatLiveForDisplay(displayRun, entry?.snapshot?.tasks ?? readRunTasks(run, snapshotCache, resolve))
|
|
573
|
+
);
|
|
574
|
+
};
|
|
575
|
+
const active = runs.filter((run) => displayActive(run));
|
|
576
|
+
const rest = runs.filter((run) => !displayActive(run));
|
|
545
577
|
if (active.length) rows.push({ label: "Active" }, ...active.map((run) => ({ label: run.runId, run })));
|
|
546
578
|
if (rest.length) rows.push({ label: "Recent" }, ...rest.map((run) => ({ label: run.runId, run })));
|
|
547
579
|
return rows;
|
|
@@ -754,7 +786,10 @@ export class RunDashboard implements DashboardComponent {
|
|
|
754
786
|
const snapshot = resolve ? resolve(run) : snapshotFor(run, this.options.snapshotCache);
|
|
755
787
|
const displayRun = snapshot?.manifest ?? run;
|
|
756
788
|
const agents = snapshot?.agents ?? agentsFor(run, this.options.snapshotCache, resolve);
|
|
757
|
-
|
|
789
|
+
// Wave 15 (F2): fresh task heartbeats veto the "stale" label.
|
|
790
|
+
const stale =
|
|
791
|
+
isLikelyOrphanedActiveRun(displayRun, agents) &&
|
|
792
|
+
!heartbeatLiveForDisplay(displayRun, snapshot?.tasks ?? readRunTasks(run, this.options.snapshotCache, resolve));
|
|
758
793
|
const status: RunStatus = stale ? "stale" : (displayRun.status as RunStatus);
|
|
759
794
|
if (status === "running" || agents.some((agent) => agent.status === "running")) hasRunning = true;
|
|
760
795
|
return snapshot?.signature ?? `${displayRun.runId}:${displayRun.status}:${displayRun.updatedAt}:${status}`;
|
|
@@ -963,7 +998,15 @@ export class RunDashboard implements DashboardComponent {
|
|
|
963
998
|
const snap = snapshotOnce(rowItem.run);
|
|
964
999
|
const run = snap?.manifest ?? rowItem.run;
|
|
965
1000
|
const agents = snap?.agents ?? agentsFor(rowItem.run, this.options.snapshotCache, snapshotOnce);
|
|
966
|
-
|
|
1001
|
+
// Wave 15 (F2): fresh task heartbeats veto the "stale" label (status line).
|
|
1002
|
+
const status: RunStatus =
|
|
1003
|
+
isLikelyOrphanedActiveRun(run, agents) &&
|
|
1004
|
+
!heartbeatLiveForDisplay(
|
|
1005
|
+
run,
|
|
1006
|
+
snap?.tasks ?? readRunTasks(rowItem.run, this.options.snapshotCache, snapshotOnce),
|
|
1007
|
+
)
|
|
1008
|
+
? "stale"
|
|
1009
|
+
: (run.status as RunStatus);
|
|
967
1010
|
const label = runLabel(run, idx === this.selected, this.options.snapshotCache, budget, snapshotOnce);
|
|
968
1011
|
lines.push(row(applyStatusColor(this.theme, status, label)));
|
|
969
1012
|
}
|
|
@@ -977,7 +1020,15 @@ export class RunDashboard implements DashboardComponent {
|
|
|
977
1020
|
const snap = snapshotOnce(rowItem.run);
|
|
978
1021
|
const run = snap?.manifest ?? rowItem.run;
|
|
979
1022
|
const agents = snap?.agents ?? agentsFor(rowItem.run, this.options.snapshotCache, snapshotOnce);
|
|
980
|
-
|
|
1023
|
+
// Wave 15 (F2): fresh task heartbeats veto the "stale" label (windowed status line).
|
|
1024
|
+
const status: RunStatus =
|
|
1025
|
+
isLikelyOrphanedActiveRun(run, agents) &&
|
|
1026
|
+
!heartbeatLiveForDisplay(
|
|
1027
|
+
run,
|
|
1028
|
+
snap?.tasks ?? readRunTasks(rowItem.run, this.options.snapshotCache, snapshotOnce),
|
|
1029
|
+
)
|
|
1030
|
+
? "stale"
|
|
1031
|
+
: (run.status as RunStatus);
|
|
981
1032
|
const label = runLabel(run, gi === this.selected, this.options.snapshotCache, budget, snapshotOnce);
|
|
982
1033
|
lines.push(row(applyStatusColor(this.theme, status, label)));
|
|
983
1034
|
}
|
|
@@ -996,8 +1047,12 @@ export class RunDashboard implements DashboardComponent {
|
|
|
996
1047
|
const snap = snapshotOnce(selectedRun);
|
|
997
1048
|
const r = snap?.manifest ?? selectedRun;
|
|
998
1049
|
const agents = snap?.agents ?? agentsFor(selectedRun, this.options.snapshotCache, snapshotOnce);
|
|
999
|
-
const statusStr: RunStatus = isLikelyOrphanedActiveRun(r, agents) ? "stale" : (r.status as RunStatus);
|
|
1000
1050
|
const selectedTasks = snap?.tasks ?? readRunTasks(r, this.options.snapshotCache, snapshotOnce);
|
|
1051
|
+
// Wave 15 (F2): fresh task heartbeats veto the "stale" label (detail header).
|
|
1052
|
+
const statusStr: RunStatus =
|
|
1053
|
+
isLikelyOrphanedActiveRun(r, agents) && !heartbeatLiveForDisplay(r, selectedTasks)
|
|
1054
|
+
? "stale"
|
|
1055
|
+
: (r.status as RunStatus);
|
|
1001
1056
|
lines.push(section("run"));
|
|
1002
1057
|
lines.push(row(`${fg("accent", "▸")} ${truncate(sanitizeUntrustedLine(r.goal), budget - 4)}`));
|
|
1003
1058
|
// L-2: surface the failure/cancellation reason inline for terminal runs.
|