pi-goal-list-loop-audit 0.35.13 → 0.35.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +977 -0
- package/INSTALL.md +13 -3
- package/LIST-PHILOSOPHY.md +90 -0
- package/PLAN.md +316 -0
- package/README.md +201 -26
- package/docs/DESIGN-subagent-visibility.md +76 -0
- package/docs/DESIGN.md +59 -8
- package/docs/GLLA-POSITIONING-AND-DECOMPOSITION-2026-08-08.md +6 -1
- package/docs/INDEX.md +4 -0
- package/docs/VISION-ASSIST.md +2 -2
- package/extensions/auditor-extensions.ts +289 -0
- package/extensions/context-hygiene.ts +129 -0
- package/extensions/faulty-objective-recovery.ts +24 -0
- package/extensions/glla-state-root.ts +99 -0
- package/extensions/goal-agents-panel.ts +211 -0
- package/extensions/goal-commands.ts +226 -10
- package/extensions/goal-continuation.ts +119 -21
- package/extensions/goal-heartbeat.ts +560 -35
- package/extensions/goal-loop-auditor-process.ts +126 -4
- package/extensions/goal-loop-auditor.ts +3 -5
- package/extensions/goal-loop-backoff.ts +79 -0
- package/extensions/goal-loop-core.ts +197 -15
- package/extensions/goal-loop-dispatch.ts +3 -2
- package/extensions/goal-loop-display.ts +141 -8
- package/extensions/goal-loop-forever.ts +52 -3
- package/extensions/goal-loop-shield.ts +132 -26
- package/extensions/goal-loop-stats.ts +3 -2
- package/extensions/goal-loop.ts +101 -4
- package/extensions/goal-recovery.ts +8 -2
- package/extensions/goal-settings.ts +61 -14
- package/extensions/goal-state.ts +20 -0
- package/extensions/loops/goal-activation.ts +426 -45
- package/extensions/loops/goal-auditor-hooks.ts +39 -3
- package/extensions/loops/goal-auditor-surface.ts +32 -0
- package/extensions/loops/goal-list-queue.ts +72 -12
- package/extensions/loops/goal-orchestrator.ts +33 -1
- package/extensions/loops/goal-runtime-globals.ts +5 -0
- package/extensions/loops/goal-session.ts +68 -8
- package/extensions/loops/goal-settings-ui.ts +116 -5
- package/extensions/loops/goal-tools.ts +104 -29
- package/extensions/loops/goal-ui.ts +106 -1
- package/extensions/loops/goal.ts +7 -7
- package/extensions/main-model-recovery.ts +10 -0
- package/extensions/multi-model-picker.ts +45 -7
- package/extensions/payload-guard.ts +172 -0
- package/extensions/reviewer.ts +3 -1
- package/extensions/settings-menu.ts +39 -4
- package/package.json +9 -3
- package/prompts/goal-loop-plan-loop.md +31 -0
- package/prompts/goal-loop-plan.md +51 -0
- package/schemas/goal.schema.json +4 -1
- package/scripts/auditor-extension-fixture.mjs +19 -0
- package/scripts/goal-auditor-launch.mjs +19 -1
- package/scripts/goal-auditor-worker.mjs +15 -0
- package/scripts/verify-auditor-extensions-offline.mjs +72 -0
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
// stranded_audit_recovered, subagent_hang_detected, ...).
|
|
20
20
|
// ============================================================================
|
|
21
21
|
|
|
22
|
+
import { randomUUID } from "node:crypto";
|
|
22
23
|
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
23
24
|
import { state } from "./goal-state.js";
|
|
24
25
|
import {
|
|
@@ -26,6 +27,7 @@ import {
|
|
|
26
27
|
appendLedger,
|
|
27
28
|
nowIso,
|
|
28
29
|
resolveEffectiveAggressiveSettings,
|
|
30
|
+
supervisorPaused,
|
|
29
31
|
type Goal,
|
|
30
32
|
} from "./goal-loop-core.js";
|
|
31
33
|
import { loadSettings } from "./goal-settings.js";
|
|
@@ -39,7 +41,7 @@ import {
|
|
|
39
41
|
shouldWedgeAlert,
|
|
40
42
|
} from "./goal-loop-backoff.js";
|
|
41
43
|
import { isLoopActive, loopTimerPending, scheduleLoopTick } from "./goal-loop.js";
|
|
42
|
-
import { mainModelRecoveryActive, markCompletionAuditRecoveryPending } from "./goal-recovery.js";
|
|
44
|
+
import { mainModelRecoveryActive, markCompletionAuditRecoveryPending, probeMainModelRecovery } from "./goal-recovery.js";
|
|
43
45
|
import type { ContinuationDispatch } from "./goal-loop-dispatch.js";
|
|
44
46
|
|
|
45
47
|
/** goal.ts-owned module lets the heartbeat reads/writes through this accessor.
|
|
@@ -175,6 +177,108 @@ const ZOMBIE_RUN_SILENT_MS = 20 * 60_000;
|
|
|
175
177
|
const ZOMBIE_RUN_ABORT_GRACE_MS = 10 * 60_000;
|
|
176
178
|
const ZOMBIE_RUN_ALERT_THROTTLE_MS = 10 * 60_000;
|
|
177
179
|
|
|
180
|
+
// v0.35.26 (issue #13): ONE shared name set for "the parent is legitimately
|
|
181
|
+
// stream-silent because it is blocked on a subagent call" — consumed by both
|
|
182
|
+
// the zombie stand-down and the wedge-alert hint so the two sites cannot
|
|
183
|
+
// drift apart again (that drift is exactly the bug). The legacy built-in
|
|
184
|
+
// names (Agent / get_subagent_result / steer_subagent) are joined by the
|
|
185
|
+
// pi-subagents extension's registrations: its foreground dispatch tool is
|
|
186
|
+
// named "subagent" and its blocking background wait is "subagent_wait"
|
|
187
|
+
// (field 2026-08-21: a healthy foreground child wrote Postgres records for
|
|
188
|
+
// 30 productive minutes while the parent was stream-silent on `subagent` —
|
|
189
|
+
// the watchdog aborted it at the grace boundary anyway).
|
|
190
|
+
const SUBAGENT_WAIT_TOOL_NAMES: ReadonlySet<string> = new Set([
|
|
191
|
+
"Agent",
|
|
192
|
+
"get_subagent_result",
|
|
193
|
+
"steer_subagent",
|
|
194
|
+
"subagent",
|
|
195
|
+
"subagent_wait",
|
|
196
|
+
]);
|
|
197
|
+
const isSubagentWaitCall = (t: { name?: string }): boolean =>
|
|
198
|
+
typeof t.name === "string" && SUBAGENT_WAIT_TOOL_NAMES.has(t.name);
|
|
199
|
+
|
|
200
|
+
// ============================================================================
|
|
201
|
+
// v0.35.28 (issue #16): due-wait backstop — the durable invariant that a
|
|
202
|
+
// pauseKind "wait" actually resumes when its pauseResumeAt lapses.
|
|
203
|
+
//
|
|
204
|
+
// Field report: a goal sat paused for 30+ minutes past its scheduled
|
|
205
|
+
// auto-resume while the agent narrated "the system should have auto-resumed
|
|
206
|
+
// by now". Investigation found auto-resume relied SOLELY on in-memory
|
|
207
|
+
// timers: agent-authored waits (pause_goal kind="wait") armed NO timer at
|
|
208
|
+
// all while their own copy promised automatic continuation; error-brake
|
|
209
|
+
// cooldowns were not re-armed on session_start; and every scheduled resume
|
|
210
|
+
// died with the session that created it. The heartbeat now compares wall
|
|
211
|
+
// time against pauseResumeAt on every tick and re-fires the route.
|
|
212
|
+
const WAIT_OVERDUE_GRACE_MS = 90 * 1_000;
|
|
213
|
+
/** One backstop attempt per (goalId:resumeAt) pair — the underlying route
|
|
214
|
+
* rewrites pauseResumeAt when it re-parks, which re-arms the key. */
|
|
215
|
+
let lastOverdueWaitKey = "";
|
|
216
|
+
|
|
217
|
+
function overdueWaitDue(): boolean {
|
|
218
|
+
const goal = state.goal;
|
|
219
|
+
if (!goal || goal.status !== "paused" || goal.pauseKind !== "wait") return false;
|
|
220
|
+
if (typeof goal.pauseResumeAt !== "string") return false;
|
|
221
|
+
const dueMs = Date.now() - Date.parse(goal.pauseResumeAt);
|
|
222
|
+
if (!Number.isFinite(dueMs) || dueMs < WAIT_OVERDUE_GRACE_MS) return false;
|
|
223
|
+
// Manual /glla pause and the v0.35.23 load hold freeze ALL automatic
|
|
224
|
+
// dispatch — an overdue wait under them stays frozen until an explicit
|
|
225
|
+
// resume releases the hold (same consent boundary as session_start).
|
|
226
|
+
if (supervisorPaused(state)) return false;
|
|
227
|
+
return `${goal.id}:${goal.pauseResumeAt}` !== lastOverdueWaitKey;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function overdueWaitBackstop(ctx: ExtensionContext): void {
|
|
231
|
+
// v0.35.48 (audit finding): this backstop MUTATES DURABLE STATE — parked →
|
|
232
|
+
// active, pauseResumeAt cleared — and then dispatches. It may only fire
|
|
233
|
+
// when the dispatch surface can actually carry the resume; otherwise it
|
|
234
|
+
// manufactures an ACTIVE goal with nothing scheduled until a fresh
|
|
235
|
+
// session_start (the paused-is-safe invariant, broken).
|
|
236
|
+
// - mid-handoff or stale-latched windows: stay parked; when a healthy
|
|
237
|
+
// session rebinds, subsequent ticks re-arm the backstop naturally.
|
|
238
|
+
// - active main-model recovery: ONLY recovery-routed waits may fire
|
|
239
|
+
// (the probe route re-parks with a FRESH resumeAt on failure); an
|
|
240
|
+
// unrelated agent-authored wait would dispatch into the same broken
|
|
241
|
+
// model, so it stays parked until recovery resolves.
|
|
242
|
+
// These gates sit BEFORE the lastOverdueWaitKey latch: a skipped window
|
|
243
|
+
// must stay retriable on later ticks, never latched as "already done".
|
|
244
|
+
if (flags.sessionHandoffPending || flags.staleTerminalDone || flags.extensionApiStale) return;
|
|
245
|
+
if (mainModelRecoveryActive() && !(state.goal?.pauseReason ?? "").startsWith("main model recovery")) return;
|
|
246
|
+
if (!overdueWaitDue()) return;
|
|
247
|
+
const goal = state.goal!;
|
|
248
|
+
const reason = goal.pauseReason ?? "";
|
|
249
|
+
lastOverdueWaitKey = `${goal.id}:${goal.pauseResumeAt}`;
|
|
250
|
+
appendLedger(ctx.cwd, "wait_pause_overdue_resume", {
|
|
251
|
+
goalId: goal.id,
|
|
252
|
+
pauseResumeAt: goal.pauseResumeAt,
|
|
253
|
+
overdueMs: Date.now() - Date.parse(goal.pauseResumeAt!),
|
|
254
|
+
reason: reason.slice(0, 160),
|
|
255
|
+
route: reason.startsWith("main model recovery") ? "main-model-probe" : "continuation",
|
|
256
|
+
});
|
|
257
|
+
if (reason.startsWith("main model recovery")) {
|
|
258
|
+
void probeMainModelRecovery(ctx).catch(() => { /* re-parks with a fresh resumeAt on failure */ });
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
// Agent-authored waits and error-brake cooldowns alike: the stated wait
|
|
262
|
+
// condition's deadline has passed — clear the park and re-dispatch, with
|
|
263
|
+
// a recovery stamp so the continuation prompt tells the agent it was
|
|
264
|
+
// ITSELF that was recovered (issue #16 part 2).
|
|
265
|
+
updateGoal({
|
|
266
|
+
status: "active",
|
|
267
|
+
pauseKind: undefined,
|
|
268
|
+
pauseResumeAt: undefined,
|
|
269
|
+
pauseReason: undefined,
|
|
270
|
+
pauseSuggestedAction: undefined,
|
|
271
|
+
autoResumedAt: new Date().toISOString(),
|
|
272
|
+
autoResumedEvent: `overdue wait resumed (${reason.slice(0, 80) || "time-gated wait"})`,
|
|
273
|
+
}, ctx);
|
|
274
|
+
scheduleContinuation(ctx, true);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Test-only: reset the due-wait backstop latch between isolated rigs. */
|
|
278
|
+
export function __testOnlyResetOverdueWaitBackstop(): void {
|
|
279
|
+
lastOverdueWaitKey = "";
|
|
280
|
+
}
|
|
281
|
+
|
|
178
282
|
function zombieRunSilentMs(): number {
|
|
179
283
|
return zombieRunSilentMsOverride ?? ZOMBIE_RUN_SILENT_MS;
|
|
180
284
|
}
|
|
@@ -200,6 +304,15 @@ export function __testOnlyResetZombieRunWatchdog(): void {
|
|
|
200
304
|
lastZombieAbortKey = "";
|
|
201
305
|
}
|
|
202
306
|
|
|
307
|
+
/** v0.35.17: release the one-shot abort latch when the bounded automatic
|
|
308
|
+
* retry dispatches a fresh attempt. Without this, a retry whose send never
|
|
309
|
+
* even starts (no agent_start to advance the stream clock) would reproduce
|
|
310
|
+
* the exact abort key of its parent episode and the heartbeat could warn but
|
|
311
|
+
* NEVER re-abort — a permanently BUSY-silent session. */
|
|
312
|
+
export function releaseZombieAbortKey(): void {
|
|
313
|
+
lastZombieAbortKey = "";
|
|
314
|
+
}
|
|
315
|
+
|
|
203
316
|
let lastUnansweredAlertAt = 0;
|
|
204
317
|
|
|
205
318
|
// v0.35.4: one-shot-per-episode latch for the context-starved warning. The
|
|
@@ -226,9 +339,11 @@ let heartbeatStaleDebounce = HEARTBEAT_STALE_DEBOUNCE;
|
|
|
226
339
|
// (tool uses or assistant output tokens) for SUBAGENT_HANG_NO_PROGRESS_MS
|
|
227
340
|
// (5m default — SHORTER than the auditor's 10m, because a hung subagent
|
|
228
341
|
// costs parent tokens on every turn) is surfaced to the user and ledgered
|
|
229
|
-
// `subagent_hang_detected`.
|
|
230
|
-
//
|
|
342
|
+
// `subagent_hang_detected`. A much longer second threshold can request one
|
|
343
|
+
// child-specific abort; zero disables that action while retaining detection.
|
|
231
344
|
const SUBAGENT_HANG_NO_PROGRESS_MS = 5 * 60_000;
|
|
345
|
+
const SUBAGENT_HANG_ESCALATION_DEFAULT_MINUTES = 30;
|
|
346
|
+
const SUBAGENT_HANG_ESCALATION_MIN_MS = 5 * 60_000;
|
|
232
347
|
// v0.34.102 (field: pully W161 rehearsal agent aac4ab1e, 2026-08-08T08:15
|
|
233
348
|
// → 10:13 — a 118-minute wedge where `subagent_hang_detected` NEVER fired
|
|
234
349
|
// because the manager record was unreachable and classifyHungSubagents
|
|
@@ -265,6 +380,8 @@ interface SubagentHangProbe {
|
|
|
265
380
|
agentType?: string;
|
|
266
381
|
summary?: string;
|
|
267
382
|
spawnedAt: number;
|
|
383
|
+
/** Host generation that observed this child spawn. */
|
|
384
|
+
ownerGeneration: number;
|
|
268
385
|
/** Last time the subagent delivered NEW progress (tool use / output tokens). */
|
|
269
386
|
lastProgressAt: number;
|
|
270
387
|
/** Last polled record.toolUses — progress when it increases. */
|
|
@@ -273,6 +390,9 @@ interface SubagentHangProbe {
|
|
|
273
390
|
lastOutputTokens: number;
|
|
274
391
|
/** Throttle: last user-facing hang warning for this subagent. */
|
|
275
392
|
hangAlertedAt?: number;
|
|
393
|
+
/** One-shot action state for the current no-progress episode. */
|
|
394
|
+
hangAction?: "abort-requested" | "unavailable" | "failed";
|
|
395
|
+
hangActionAt?: number;
|
|
276
396
|
endedAt?: number;
|
|
277
397
|
}
|
|
278
398
|
|
|
@@ -285,8 +405,18 @@ const subagentHangProbes = new Map<string, SubagentHangProbe>();
|
|
|
285
405
|
* cross-extension stream event. Defensive: absent when pi-subagents isn't
|
|
286
406
|
* loaded or the record shape changes (falls back to event-only evidence). */
|
|
287
407
|
const SUBAGENT_MANAGER_KEY = Symbol.for("pi-subagents:manager");
|
|
288
|
-
type SubagentRecordPoll = {
|
|
289
|
-
|
|
408
|
+
type SubagentRecordPoll = {
|
|
409
|
+
toolUses?: number;
|
|
410
|
+
lifetimeUsage?: { output?: number };
|
|
411
|
+
status?: string;
|
|
412
|
+
/** Present on pi-subagents records; required before automatic control. */
|
|
413
|
+
parentAgentId?: string;
|
|
414
|
+
};
|
|
415
|
+
type SubagentManagerPoll = {
|
|
416
|
+
getRecord?: (id: string) => SubagentRecordPoll | undefined;
|
|
417
|
+
/** Optional future registry capability; never fall back to a parent abort. */
|
|
418
|
+
abort?: (id: string) => boolean;
|
|
419
|
+
};
|
|
290
420
|
function subagentManagerPoller(): SubagentManagerPoll {
|
|
291
421
|
try {
|
|
292
422
|
return ((globalThis as any)[SUBAGENT_MANAGER_KEY] ?? {}) as SubagentManagerPoll;
|
|
@@ -295,25 +425,158 @@ function subagentManagerPoller(): SubagentManagerPoll {
|
|
|
295
425
|
}
|
|
296
426
|
}
|
|
297
427
|
|
|
428
|
+
/** Minimal pi.events surface used by the pinned pi-subagents RPC bridge. */
|
|
429
|
+
export interface SubagentRpcEventBus {
|
|
430
|
+
on(event: string, handler: (data: unknown) => void): () => void;
|
|
431
|
+
emit(event: string, data: unknown): void;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
type SubagentStopRpcResult =
|
|
435
|
+
| { kind: "requested" }
|
|
436
|
+
| { kind: "unavailable"; reason: string }
|
|
437
|
+
| { kind: "failed"; error: string }
|
|
438
|
+
| { kind: "stale" };
|
|
439
|
+
|
|
440
|
+
type SubagentRpcBinding = {
|
|
441
|
+
events: SubagentRpcEventBus;
|
|
442
|
+
generation: number;
|
|
443
|
+
ready: boolean;
|
|
444
|
+
};
|
|
445
|
+
|
|
446
|
+
const SUBAGENT_RPC_STOP_TIMEOUT_MS = 2_000;
|
|
447
|
+
let subagentRpcBinding: SubagentRpcBinding | null = null;
|
|
448
|
+
const observedSubagentRpcBuses = new Set<SubagentRpcEventBus>();
|
|
449
|
+
const readySubagentRpcBuses = new Set<SubagentRpcEventBus>();
|
|
450
|
+
|
|
451
|
+
/**
|
|
452
|
+
* Observe readiness at factory time. Extension factories run before lifecycle
|
|
453
|
+
* events, so this catches `subagents:ready` even when pi-subagents' handler is
|
|
454
|
+
* registered before GLLA's `session_start` handler. The event bus is only
|
|
455
|
+
* usable for control after bindSubagentRpcHost admits the current host.
|
|
456
|
+
*/
|
|
457
|
+
export function observeSubagentRpcReadiness(events: SubagentRpcEventBus): void {
|
|
458
|
+
if (observedSubagentRpcBuses.has(events)) return;
|
|
459
|
+
observedSubagentRpcBuses.add(events);
|
|
460
|
+
try {
|
|
461
|
+
events.on("subagents:ready", () => {
|
|
462
|
+
readySubagentRpcBuses.add(events);
|
|
463
|
+
if (subagentRpcBinding?.events === events) subagentRpcBinding.ready = true;
|
|
464
|
+
});
|
|
465
|
+
} catch {
|
|
466
|
+
// A factory-time event bus can be unavailable on older hosts; control will
|
|
467
|
+
// remain fail-closed and produce the durable unavailable escalation.
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/** Bind the RPC stop capability to an admitted MAIN host generation. */
|
|
472
|
+
export function bindSubagentRpcHost(events: SubagentRpcEventBus, generation: number): void {
|
|
473
|
+
observeSubagentRpcReadiness(events);
|
|
474
|
+
const retainedReady = subagentRpcBinding?.events === events && subagentRpcBinding.ready;
|
|
475
|
+
subagentRpcBinding = {
|
|
476
|
+
events,
|
|
477
|
+
generation,
|
|
478
|
+
ready: retainedReady || readySubagentRpcBuses.has(events),
|
|
479
|
+
};
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/** Clear a host-bound RPC capability before its event bus becomes stale. */
|
|
483
|
+
export function releaseSubagentRpcHost(events?: SubagentRpcEventBus): void {
|
|
484
|
+
if (events && subagentRpcBinding?.events !== events) return;
|
|
485
|
+
const boundEvents = subagentRpcBinding?.events ?? events;
|
|
486
|
+
subagentRpcBinding = null;
|
|
487
|
+
if (boundEvents) readySubagentRpcBuses.delete(boundEvents);
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
function requestSubagentStopViaRpc(recordId: string, generation: number): Promise<SubagentStopRpcResult> | undefined {
|
|
491
|
+
const binding = subagentRpcBinding;
|
|
492
|
+
if (!binding) return undefined;
|
|
493
|
+
if (!binding.ready) {
|
|
494
|
+
return Promise.resolve({ kind: "unavailable", reason: "the subagents RPC service is not ready" });
|
|
495
|
+
}
|
|
496
|
+
if (binding.generation !== generation || flags.sessionGeneration !== generation) {
|
|
497
|
+
return Promise.resolve({ kind: "stale" });
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
const requestId = randomUUID();
|
|
501
|
+
const replyChannel = `subagents:rpc:stop:reply:${requestId}`;
|
|
502
|
+
return new Promise<SubagentStopRpcResult>((resolve) => {
|
|
503
|
+
let settled = false;
|
|
504
|
+
let unsubscribe = () => {};
|
|
505
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
506
|
+
const finish = (result: SubagentStopRpcResult): void => {
|
|
507
|
+
if (settled) return;
|
|
508
|
+
settled = true;
|
|
509
|
+
unsubscribe();
|
|
510
|
+
if (timeout) clearTimeout(timeout);
|
|
511
|
+
resolve(result);
|
|
512
|
+
};
|
|
513
|
+
try {
|
|
514
|
+
unsubscribe = binding.events.on(replyChannel, (raw) => {
|
|
515
|
+
if (subagentRpcBinding?.events !== binding.events
|
|
516
|
+
|| subagentRpcBinding.generation !== generation
|
|
517
|
+
|| flags.sessionGeneration !== generation) {
|
|
518
|
+
finish({ kind: "stale" });
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
const reply = raw as { success?: boolean; error?: unknown } | null;
|
|
522
|
+
if (reply?.success === true) finish({ kind: "requested" });
|
|
523
|
+
else finish({ kind: "failed", error: typeof reply?.error === "string" ? reply.error : "the RPC service rejected the request" });
|
|
524
|
+
});
|
|
525
|
+
// Arm the timeout before emitting: a compatible host/test bus may reply
|
|
526
|
+
// synchronously, and finish() must then be able to clear the timer.
|
|
527
|
+
timeout = setTimeout(() => finish({ kind: "unavailable", reason: "the subagents stop RPC timed out" }), SUBAGENT_RPC_STOP_TIMEOUT_MS);
|
|
528
|
+
timeout.unref?.();
|
|
529
|
+
binding.events.emit("subagents:rpc:stop", { requestId, agentId: recordId });
|
|
530
|
+
} catch (error) {
|
|
531
|
+
finish({ kind: "unavailable", reason: error instanceof Error ? error.message : String(error) });
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
let subagentHangEscalationMsOverride: number | null = null;
|
|
537
|
+
function subagentHangEscalationMs(cwd: string): number {
|
|
538
|
+
if (subagentHangEscalationMsOverride !== null) return Math.max(0, subagentHangEscalationMsOverride);
|
|
539
|
+
const minutes = loadSettings(cwd).subagentHangEscalationMinutes;
|
|
540
|
+
if (minutes === 0) return 0;
|
|
541
|
+
if (typeof minutes !== "number" || !Number.isFinite(minutes) || minutes < 5) {
|
|
542
|
+
return SUBAGENT_HANG_ESCALATION_DEFAULT_MINUTES * 60_000;
|
|
543
|
+
}
|
|
544
|
+
return minutes * 60_000;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function ownerGeneration(): number {
|
|
548
|
+
try { return flags?.sessionGeneration ?? 0; } catch { return 0; }
|
|
549
|
+
}
|
|
550
|
+
|
|
298
551
|
export function upsertSubagentHangProbe(recordId: string, agentType: string | undefined, summary: string | undefined, now = Date.now()): void {
|
|
299
552
|
const existing = subagentHangProbes.get(recordId);
|
|
300
553
|
if (existing) {
|
|
301
554
|
// Re-observation (resume / re-run): fresh evidence + refreshed metadata.
|
|
302
555
|
existing.lastProgressAt = now;
|
|
556
|
+
existing.ownerGeneration = ownerGeneration();
|
|
303
557
|
existing.endedAt = undefined;
|
|
558
|
+
existing.hangAlertedAt = undefined;
|
|
559
|
+
existing.hangAction = undefined;
|
|
560
|
+
existing.hangActionAt = undefined;
|
|
304
561
|
if (agentType) existing.agentType = agentType;
|
|
305
562
|
if (summary) existing.summary = summary;
|
|
306
563
|
return;
|
|
307
564
|
}
|
|
308
565
|
subagentHangProbes.set(recordId, {
|
|
309
566
|
recordId, agentType, summary,
|
|
567
|
+
ownerGeneration: ownerGeneration(),
|
|
310
568
|
spawnedAt: now, lastProgressAt: now, lastToolUses: 0, lastOutputTokens: 0,
|
|
311
569
|
});
|
|
312
570
|
}
|
|
313
571
|
|
|
314
572
|
export function markSubagentHangProgress(recordId: string, now = Date.now()): void {
|
|
315
573
|
const p = subagentHangProbes.get(recordId);
|
|
316
|
-
if (p)
|
|
574
|
+
if (p) {
|
|
575
|
+
p.lastProgressAt = now;
|
|
576
|
+
p.hangAlertedAt = undefined;
|
|
577
|
+
p.hangAction = undefined;
|
|
578
|
+
p.hangActionAt = undefined;
|
|
579
|
+
}
|
|
317
580
|
}
|
|
318
581
|
|
|
319
582
|
export function endSubagentHangProbe(recordId: string, now = Date.now()): void {
|
|
@@ -334,6 +597,9 @@ export function classifyHungSubagents(
|
|
|
334
597
|
lastProgressAt: number;
|
|
335
598
|
lastToolUses: number;
|
|
336
599
|
lastOutputTokens: number;
|
|
600
|
+
hangAlertedAt?: number;
|
|
601
|
+
hangAction?: "abort-requested" | "unavailable" | "failed";
|
|
602
|
+
hangActionAt?: number;
|
|
337
603
|
endedAt?: number;
|
|
338
604
|
}>,
|
|
339
605
|
getRecord: (id: string) => SubagentRecordPoll | undefined,
|
|
@@ -362,6 +628,9 @@ export function classifyHungSubagents(
|
|
|
362
628
|
p.lastToolUses = toolUses;
|
|
363
629
|
p.lastOutputTokens = output;
|
|
364
630
|
p.lastProgressAt = now;
|
|
631
|
+
p.hangAlertedAt = undefined;
|
|
632
|
+
p.hangAction = undefined;
|
|
633
|
+
p.hangActionAt = undefined;
|
|
365
634
|
continue;
|
|
366
635
|
}
|
|
367
636
|
const silentMs = now - p.lastProgressAt;
|
|
@@ -370,8 +639,177 @@ export function classifyHungSubagents(
|
|
|
370
639
|
return hung;
|
|
371
640
|
}
|
|
372
641
|
|
|
642
|
+
function isLiveSubagentRecord(rec: SubagentRecordPoll | undefined): boolean {
|
|
643
|
+
return !!rec && (rec.status === "running" || rec.status === "steered" || rec.status === "queued");
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/** Only a child with fresh evidence keeps the generic parent zombie watchdog
|
|
647
|
+
* stood down. A classified stale child must not shield an unrelated parent
|
|
648
|
+
* turn forever; an event-only child gets the longer event-evidence window. */
|
|
649
|
+
function hasHealthySubagentHangProbe(now = Date.now()): boolean {
|
|
650
|
+
const poll = subagentManagerPoller();
|
|
651
|
+
for (const probe of subagentHangProbes.values()) {
|
|
652
|
+
if (probe.endedAt !== undefined || probe.hangActionAt !== undefined) continue;
|
|
653
|
+
const rec = poll.getRecord?.(probe.recordId);
|
|
654
|
+
if (rec && isLiveSubagentRecord(rec)) {
|
|
655
|
+
const toolUses = rec.toolUses ?? 0;
|
|
656
|
+
const output = rec.lifetimeUsage?.output ?? 0;
|
|
657
|
+
if (toolUses > probe.lastToolUses || output > probe.lastOutputTokens
|
|
658
|
+
|| now - probe.lastProgressAt < SUBAGENT_HANG_NO_PROGRESS_MS) return true;
|
|
659
|
+
} else if (!rec && now - probe.lastProgressAt < SUBAGENT_HANG_EVENT_ONLY_MS) {
|
|
660
|
+
return true;
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
return false;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function hasStaleSubagentHangProbe(now = Date.now()): boolean {
|
|
667
|
+
const poll = subagentManagerPoller();
|
|
668
|
+
for (const probe of subagentHangProbes.values()) {
|
|
669
|
+
if (probe.endedAt !== undefined || probe.hangActionAt !== undefined) continue;
|
|
670
|
+
const rec = poll.getRecord?.(probe.recordId);
|
|
671
|
+
if (rec && isLiveSubagentRecord(rec) && now - probe.lastProgressAt >= SUBAGENT_HANG_NO_PROGRESS_MS) return true;
|
|
672
|
+
if (!rec && now - probe.lastProgressAt >= SUBAGENT_HANG_EVENT_ONLY_MS) return true;
|
|
673
|
+
}
|
|
674
|
+
return false;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
function requestSubagentHangAction(
|
|
678
|
+
ctx: ExtensionContext,
|
|
679
|
+
probe: SubagentHangProbe,
|
|
680
|
+
silentMs: number,
|
|
681
|
+
poll: SubagentManagerPoll,
|
|
682
|
+
now: number,
|
|
683
|
+
escalationMs: number,
|
|
684
|
+
): void {
|
|
685
|
+
if (escalationMs <= 0 || silentMs < escalationMs || probe.hangActionAt !== undefined) return;
|
|
686
|
+
// Claim the episode before any capability call. A rejected/throwing manager
|
|
687
|
+
// must not produce a ledger/notify storm on every 15s heartbeat tick.
|
|
688
|
+
probe.hangActionAt = now;
|
|
689
|
+
const label = [probe.agentType, probe.summary].filter(Boolean).join(" ") || "subagent";
|
|
690
|
+
const common = {
|
|
691
|
+
recordId: probe.recordId,
|
|
692
|
+
agentType: probe.agentType,
|
|
693
|
+
summary: probe.summary,
|
|
694
|
+
silentMs,
|
|
695
|
+
generation: flags.sessionGeneration,
|
|
696
|
+
ownerGeneration: probe.ownerGeneration,
|
|
697
|
+
at: nowIso(),
|
|
698
|
+
};
|
|
699
|
+
const unavailable = (reason: string): void => {
|
|
700
|
+
probe.hangAction = "unavailable";
|
|
701
|
+
appendLedger(ctx.cwd, "subagent_hang_action_unavailable", { ...common, reason });
|
|
702
|
+
const msg = `glla: ${label} stayed frozen for ${Math.max(1, Math.round(silentMs / 60_000))}m, but no safe child-abort capability is available (${reason}). The warning and partial evidence are preserved; inspect the Agents panel and decide whether to interrupt the parent.`;
|
|
703
|
+
ctx.ui.notify(msg, "warning");
|
|
704
|
+
notifyExternal(ctx, msg);
|
|
705
|
+
};
|
|
706
|
+
|
|
707
|
+
if (probe.ownerGeneration !== flags.sessionGeneration) {
|
|
708
|
+
unavailable("the child belongs to an older host generation");
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
const rec = poll.getRecord?.(probe.recordId);
|
|
712
|
+
if (!rec) {
|
|
713
|
+
unavailable("the manager record is unavailable");
|
|
714
|
+
return;
|
|
715
|
+
}
|
|
716
|
+
if (!isLiveSubagentRecord(rec)) {
|
|
717
|
+
unavailable(`the child is no longer running (status=${rec.status ?? "unknown"})`);
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
// Automatic control is limited to top-level records. The capability is
|
|
721
|
+
// deliberately required to expose the ownership field; guessing about a
|
|
722
|
+
// nested child could kill work owned by another subagent.
|
|
723
|
+
if (!("parentAgentId" in rec) || rec.parentAgentId !== undefined) {
|
|
724
|
+
unavailable("the child is nested or its ownership is unknown");
|
|
725
|
+
return;
|
|
726
|
+
}
|
|
727
|
+
const toolUses = rec.toolUses ?? 0;
|
|
728
|
+
const output = rec.lifetimeUsage?.output ?? 0;
|
|
729
|
+
if (toolUses > probe.lastToolUses || output > probe.lastOutputTokens) {
|
|
730
|
+
// Progress arrived between classification and the capability call. Drop
|
|
731
|
+
// the latch so a later genuine no-progress episode can be evaluated.
|
|
732
|
+
probe.hangActionAt = undefined;
|
|
733
|
+
probe.hangAction = undefined;
|
|
734
|
+
probe.lastToolUses = toolUses;
|
|
735
|
+
probe.lastOutputTokens = output;
|
|
736
|
+
probe.lastProgressAt = now;
|
|
737
|
+
return;
|
|
738
|
+
}
|
|
739
|
+
const recordRequestedAbort = (): void => {
|
|
740
|
+
probe.hangAction = "abort-requested";
|
|
741
|
+
appendLedger(ctx.cwd, "subagent_hang_action_requested", { ...common, action: "abort", via: "subagents:rpc:stop" });
|
|
742
|
+
const msg = `glla: requested a bounded abort for frozen ${label} after ${Math.max(1, Math.round(silentMs / 60_000))}m with no progress. The parent was not aborted; partial child output remains available for inspection. /goal resume or /list resume retries the owning work after it settles.`;
|
|
743
|
+
ctx.ui.notify(msg, "warning");
|
|
744
|
+
notifyExternal(ctx, msg);
|
|
745
|
+
};
|
|
746
|
+
const applyRpcResult = (result: SubagentStopRpcResult): void => {
|
|
747
|
+
// A reply from a disposed host must not mutate the successor's ledger/UI.
|
|
748
|
+
if (result.kind === "stale"
|
|
749
|
+
|| flags.sessionGeneration !== probe.ownerGeneration
|
|
750
|
+
|| flags.sessionGeneration !== common.generation) return;
|
|
751
|
+
if (result.kind === "requested") {
|
|
752
|
+
recordRequestedAbort();
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
if (result.kind === "unavailable") {
|
|
756
|
+
unavailable(result.reason);
|
|
757
|
+
return;
|
|
758
|
+
}
|
|
759
|
+
appendLedger(ctx.cwd, "subagent_hang_action_failed", { ...common, action: "abort", error: result.error, via: "subagents:rpc:stop" });
|
|
760
|
+
probe.hangAction = "failed";
|
|
761
|
+
ctx.ui.notify(`glla: the child-stop RPC failed for ${label}; the parent was not aborted. Inspect the Agents panel and use an explicit interrupt if needed.`, "warning");
|
|
762
|
+
notifyExternal(ctx, `glla: the child-stop RPC failed for ${label}; the parent was not aborted. Inspect the Agents panel and use an explicit interrupt if needed.`);
|
|
763
|
+
};
|
|
764
|
+
|
|
765
|
+
// v0.35.65: the pinned pi-subagents registry intentionally exposes only
|
|
766
|
+
// polling; production child control is the admitted root-session RPC.
|
|
767
|
+
const rpcRequest = requestSubagentStopViaRpc(probe.recordId, common.generation);
|
|
768
|
+
if (rpcRequest) {
|
|
769
|
+
void rpcRequest.then(applyRpcResult);
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
// Keep compatibility with a future registry capability and with the focused
|
|
774
|
+
// unit seam, but never infer a parent abort from a missing child capability.
|
|
775
|
+
if (typeof poll.abort !== "function") {
|
|
776
|
+
unavailable("the manager exposes no child-specific abort method or stop RPC");
|
|
777
|
+
return;
|
|
778
|
+
}
|
|
779
|
+
let accepted = false;
|
|
780
|
+
try {
|
|
781
|
+
accepted = poll.abort(probe.recordId);
|
|
782
|
+
} catch (error) {
|
|
783
|
+
appendLedger(ctx.cwd, "subagent_hang_action_failed", {
|
|
784
|
+
...common,
|
|
785
|
+
action: "abort",
|
|
786
|
+
error: error instanceof Error ? error.message : String(error),
|
|
787
|
+
});
|
|
788
|
+
probe.hangAction = "failed";
|
|
789
|
+
ctx.ui.notify(`glla: could not request a child-specific abort for ${label}; the parent was not aborted. Inspect the Agents panel and use an explicit interrupt if needed.`, "warning");
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
if (!accepted) {
|
|
793
|
+
appendLedger(ctx.cwd, "subagent_hang_action_failed", { ...common, action: "abort", error: "manager rejected the running child" });
|
|
794
|
+
probe.hangAction = "failed";
|
|
795
|
+
ctx.ui.notify(`glla: the manager rejected the child-specific abort for ${label}; the parent was not aborted. Inspect the Agents panel and use an explicit interrupt if needed.`, "warning");
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
recordRequestedAbort();
|
|
799
|
+
}
|
|
800
|
+
|
|
373
801
|
function heartbeatTick(): void {
|
|
374
802
|
if (flags.zombieStoodDown || flags.initialSessionLoadPending) return; // blank startup waits for pi to bind a real session
|
|
803
|
+
// v0.35.15: a MANUAL `/glla pause` freezes the supervisor's automatic
|
|
804
|
+
// machinery — re-arms, stale probes, zombie cleanup, refires, everything
|
|
805
|
+
// this tick drives. The user explicitly asked for a hands-off window; no
|
|
806
|
+
// heartbeat side-effect may fire inside it. `/glla resume` restores it.
|
|
807
|
+
// v0.35.23 (note.md Next #2): the automatic LOAD HOLD is different — the
|
|
808
|
+
// tick stays ALIVE under it because host-loss supervision is safety
|
|
809
|
+
// machinery, not automation (a held plane must not become an unprobed
|
|
810
|
+
// idle plane). Everything this tick could DISPATCH downstream checks the
|
|
811
|
+
// same hold via supervisorPaused() and refuses; only probes/ledger run.
|
|
812
|
+
if (typeof state.supervisorPausedAt === "number") return;
|
|
375
813
|
// A completed/paused/held plane has no host-bound work to supervise. Do
|
|
376
814
|
// not probe the retained ExtensionAPI in that idle state: pi may dispose a
|
|
377
815
|
// session handle during an unrelated session transition, and reporting
|
|
@@ -390,7 +828,11 @@ function heartbeatTick(): void {
|
|
|
390
828
|
|| state.loop?.stopReason?.startsWith("extension api stale");
|
|
391
829
|
const parkedCompletionAuditRecovery = state.goal?.status === "paused"
|
|
392
830
|
&& state.goal.pendingCompletion?.phase === "recovery-pending";
|
|
393
|
-
|
|
831
|
+
// v0.35.28 (issue #16): a lapsed wait-pause is host-bound work too — the
|
|
832
|
+
// heartbeat owns its durable due-time backstop (see overdueWaitBackstop).
|
|
833
|
+
if (overdueWaitDue()) {
|
|
834
|
+
// fall through: the backstop below needs a heartbeat opportunity
|
|
835
|
+
} else if (state.goal?.status !== "active"
|
|
394
836
|
&& state.goal?.status !== "auditing"
|
|
395
837
|
&& !isLoopActive()
|
|
396
838
|
&& !staleRecoveryDebt
|
|
@@ -497,19 +939,19 @@ function heartbeatTick(): void {
|
|
|
497
939
|
// 74305f7e while the MAIN model was also in recovery; heartbeatTick
|
|
498
940
|
// returned at the `mainModelRecoveryActive()` gate BELOW, so the
|
|
499
941
|
// subagent hang scan never ran and the 12m+ wedge produced ZERO
|
|
500
|
-
// `subagent_hang_detected`). The scan
|
|
501
|
-
//
|
|
502
|
-
//
|
|
503
|
-
// subagents and the main model at the same time.
|
|
942
|
+
// `subagent_hang_detected`). The scan still runs before that gate.
|
|
943
|
+
// Detection remains warning/telemetry-first; only a much longer, confirmed
|
|
944
|
+
// frozen top-level record may request the manager's child-specific abort.
|
|
504
945
|
if (subagentHangProbes.size > 0) {
|
|
505
946
|
const nowMs = Date.now();
|
|
506
947
|
const poll = subagentManagerPoller();
|
|
948
|
+
const escalationMs = subagentHangEscalationMs(ctx.cwd);
|
|
507
949
|
const hung = classifyHungSubagents([...subagentHangProbes.values()], (id) => poll.getRecord?.(id), nowMs);
|
|
508
950
|
pruneEndedSubagentHangProbes(nowMs);
|
|
509
951
|
for (const h of hung) {
|
|
510
952
|
const p = subagentHangProbes.get(h.recordId);
|
|
511
|
-
if (!p
|
|
512
|
-
p.hangAlertedAt
|
|
953
|
+
if (!p) continue;
|
|
954
|
+
const shouldAlert = p.hangAlertedAt === undefined || nowMs - p.hangAlertedAt >= SUBAGENT_HANG_ALERT_THROTTLE_MS;
|
|
513
955
|
const label = [p.agentType, p.summary].filter(Boolean).join(" ");
|
|
514
956
|
const mins = Math.max(1, Math.round(h.silentMs / 60_000));
|
|
515
957
|
// v0.34.102: classifyHungSubagents cannot tell us the evidence class
|
|
@@ -519,23 +961,31 @@ function heartbeatTick(): void {
|
|
|
519
961
|
// only its event trail (spawn/compacted/steered) remains. Name it so
|
|
520
962
|
// the user knows whether the Agents panel can still show a live child.
|
|
521
963
|
const stillTracked = poll.getRecord?.(p.recordId) !== undefined;
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
964
|
+
if (shouldAlert) {
|
|
965
|
+
p.hangAlertedAt = nowMs;
|
|
966
|
+
appendLedger(ctx.cwd, "subagent_hang_detected", {
|
|
967
|
+
recordId: p.recordId,
|
|
968
|
+
agentType: p.agentType,
|
|
969
|
+
summary: p.summary,
|
|
970
|
+
silentMs: h.silentMs,
|
|
971
|
+
spawnedAt: new Date(p.spawnedAt).toISOString(),
|
|
972
|
+
evidence: stillTracked ? "record-frozen" : "event-only",
|
|
973
|
+
at: nowIso(),
|
|
974
|
+
});
|
|
975
|
+
const msg = `glla: subagent${label ? ` (${label})` : ""} shows no progress for ${mins}m — ${stillTracked
|
|
976
|
+
? "still running with no new tool calls or output"
|
|
977
|
+
: "its manager record is unreachable and it produced no events (spawn/compaction/steer)"}. It may be hung; a longer frozen interval will request a bounded child abort when safe.`;
|
|
978
|
+
ctx.ui.notify(msg, "warning");
|
|
979
|
+
notifyExternal(ctx, msg);
|
|
980
|
+
}
|
|
981
|
+
requestSubagentHangAction(ctx, p, h.silentMs, poll, nowMs, escalationMs);
|
|
536
982
|
}
|
|
537
983
|
}
|
|
538
|
-
if (mainModelRecoveryActive())
|
|
984
|
+
if (mainModelRecoveryActive()) {
|
|
985
|
+
overdueWaitBackstop(ctx);
|
|
986
|
+
return;
|
|
987
|
+
}
|
|
988
|
+
overdueWaitBackstop(ctx);
|
|
539
989
|
let idle = false;
|
|
540
990
|
let pending = false;
|
|
541
991
|
try {
|
|
@@ -598,12 +1048,17 @@ function heartbeatTick(): void {
|
|
|
598
1048
|
// abort and parked a run whose child was legitimately working). A parent
|
|
599
1049
|
// BUSY-waiting on Agent / steer_subagent / get_subagent_result is
|
|
600
1050
|
// EXPECTED to be stream-silent; child liveness is the subagent-hang
|
|
601
|
-
// watchdog's detection
|
|
602
|
-
// wait. Stand down
|
|
603
|
-
//
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
1051
|
+
// watchdog's detection/action territory and the wedge alert names the
|
|
1052
|
+
// wait. Stand down only while a probe still has fresh evidence. Once a
|
|
1053
|
+
// child is classified stale or an action was requested, it must not shield
|
|
1054
|
+
// an unrelated parent turn from its own bounded cleanup.
|
|
1055
|
+
const inFlightSubagentTool = [...flags.inFlightToolCalls.values()].some(isSubagentWaitCall);
|
|
1056
|
+
const healthySubagentWait = hasHealthySubagentHangProbe();
|
|
1057
|
+
const staleSubagent = hasStaleSubagentHangProbe();
|
|
1058
|
+
// Keep the legacy tool-name path for a foreground wait whose lifecycle
|
|
1059
|
+
// event has not reached the probe registry yet. Once a probe is already
|
|
1060
|
+
// stale, however, the same ambiguous tool name must not shield cleanup.
|
|
1061
|
+
const subagentWaitInFlight = healthySubagentWait || (inFlightSubagentTool && !staleSubagent);
|
|
607
1062
|
if (subagentWaitInFlight) {
|
|
608
1063
|
if (streamSilentMs >= zombieAbortMs && !flags.abortedStandDown) {
|
|
609
1064
|
appendLedger(ctx.cwd, "zombie_run_stood_down_subagent_wait", { streamSilentMs });
|
|
@@ -745,7 +1200,7 @@ function heartbeatTick(): void {
|
|
|
745
1200
|
// whose tool-use counter stops moving is hung, not thinking.
|
|
746
1201
|
const subWaits = new Set(
|
|
747
1202
|
[...flags.inFlightToolCalls.values()]
|
|
748
|
-
.filter(
|
|
1203
|
+
.filter(isSubagentWaitCall)
|
|
749
1204
|
.map((t) => t.name),
|
|
750
1205
|
);
|
|
751
1206
|
const subHint = subWaits.size > 0
|
|
@@ -845,6 +1300,69 @@ export function __testOnlySetHeartbeatStaleDebounce(n: number | null): void {
|
|
|
845
1300
|
heartbeatStaleDebounce = n ?? HEARTBEAT_STALE_DEBOUNCE;
|
|
846
1301
|
}
|
|
847
1302
|
|
|
1303
|
+
/** v0.35.29 (issue #15): production snapshot of the tracked-subagent table
|
|
1304
|
+
* for /glla agents and the widget segment. Read-only copies — never mutates
|
|
1305
|
+
* probe counters (unlike classifyHungSubagents, which advances them).
|
|
1306
|
+
* Hung classification mirrors the heartbeat scan's semantics without its
|
|
1307
|
+
* mutation: no new progress for >= SUBAGENT_HANG_NO_PROGRESS_MS with the
|
|
1308
|
+
* manager record still pollable = record-frozen; without a record, the longer
|
|
1309
|
+
* SUBAGENT_HANG_EVENT_ONLY_MS window applies = event-only. */
|
|
1310
|
+
export interface SubagentAgentView {
|
|
1311
|
+
recordId: string;
|
|
1312
|
+
agentType?: string;
|
|
1313
|
+
summary?: string;
|
|
1314
|
+
status: "running" | "hung" | "ended";
|
|
1315
|
+
spawnedAt: number;
|
|
1316
|
+
lastProgressAt: number;
|
|
1317
|
+
toolUses: number;
|
|
1318
|
+
outputTokens: number;
|
|
1319
|
+
/** ms since the last NEW progress (tool use / output tokens). */
|
|
1320
|
+
silentMs: number;
|
|
1321
|
+
evidence: "record-frozen" | "event-only" | "live";
|
|
1322
|
+
hangAlertedAt?: number;
|
|
1323
|
+
action?: "abort-requested" | "unavailable" | "failed";
|
|
1324
|
+
endedAt?: number;
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
export function getSubagentAgentsSnapshot(now = Date.now()): { agents: SubagentAgentView[]; managerAvailable: boolean } {
|
|
1328
|
+
const poll = subagentManagerPoller();
|
|
1329
|
+
const managerAvailable = typeof poll.getRecord === "function";
|
|
1330
|
+
const agents: SubagentAgentView[] = [];
|
|
1331
|
+
for (const probe of subagentHangProbes.values()) {
|
|
1332
|
+
const ended = probe.endedAt !== undefined;
|
|
1333
|
+
let status: SubagentAgentView["status"] = "running";
|
|
1334
|
+
let evidence: SubagentAgentView["evidence"] = "live";
|
|
1335
|
+
let record: SubagentRecordPoll | undefined;
|
|
1336
|
+
if (!ended) {
|
|
1337
|
+
record = poll.getRecord?.(probe.recordId);
|
|
1338
|
+
const stillTracked = record !== undefined;
|
|
1339
|
+
const silentMs = now - probe.lastProgressAt;
|
|
1340
|
+
if (record && !isLiveSubagentRecord(record)) {
|
|
1341
|
+
status = "ended";
|
|
1342
|
+
} else if (silentMs >= (stillTracked ? SUBAGENT_HANG_NO_PROGRESS_MS : SUBAGENT_HANG_EVENT_ONLY_MS)) {
|
|
1343
|
+
status = "hung";
|
|
1344
|
+
evidence = stillTracked ? "record-frozen" : "event-only";
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
agents.push({
|
|
1348
|
+
recordId: probe.recordId,
|
|
1349
|
+
...(probe.agentType ? { agentType: probe.agentType } : {}),
|
|
1350
|
+
...(probe.summary ? { summary: probe.summary } : {}),
|
|
1351
|
+
status,
|
|
1352
|
+
spawnedAt: probe.spawnedAt,
|
|
1353
|
+
lastProgressAt: probe.lastProgressAt,
|
|
1354
|
+
toolUses: probe.lastToolUses,
|
|
1355
|
+
outputTokens: probe.lastOutputTokens,
|
|
1356
|
+
silentMs: Math.max(0, now - probe.lastProgressAt),
|
|
1357
|
+
evidence,
|
|
1358
|
+
...(probe.hangAlertedAt !== undefined ? { hangAlertedAt: probe.hangAlertedAt } : {}),
|
|
1359
|
+
...(probe.hangAction !== undefined ? { action: probe.hangAction } : {}),
|
|
1360
|
+
...(probe.endedAt !== undefined ? { endedAt: probe.endedAt } : {}),
|
|
1361
|
+
});
|
|
1362
|
+
}
|
|
1363
|
+
return { agents, managerAvailable };
|
|
1364
|
+
}
|
|
1365
|
+
|
|
848
1366
|
/** Test-only: snapshot of the live subagent-hang probe registry. Returns
|
|
849
1367
|
* the LIVE probe objects (tests backdate lastProgressAt / read hangAlertedAt)
|
|
850
1368
|
* — read-only misuse is on the test author. Never called by production
|
|
@@ -853,8 +1371,15 @@ export function __testOnlySubagentHangProbes(): SubagentHangProbe[] {
|
|
|
853
1371
|
return [...subagentHangProbes.values()];
|
|
854
1372
|
}
|
|
855
1373
|
|
|
1374
|
+
/** Test-only: override the long frozen-child action threshold. `0` keeps
|
|
1375
|
+
* detection warning-only; null restores the settings-backed default. */
|
|
1376
|
+
export function __testOnlySetSubagentHangEscalationMs(ms: number | null): void {
|
|
1377
|
+
subagentHangEscalationMsOverride = ms;
|
|
1378
|
+
}
|
|
1379
|
+
|
|
856
1380
|
/** Test-only: clear the subagent-hang probe registry (between tests). Never
|
|
857
1381
|
* called by production code. */
|
|
858
1382
|
export function __testOnlyClearSubagentHangProbes(): void {
|
|
859
1383
|
subagentHangProbes.clear();
|
|
1384
|
+
subagentHangEscalationMsOverride = null;
|
|
860
1385
|
}
|