pi-crew 0.11.1 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/CHANGELOG.md +39 -9
  2. package/README.md +161 -1036
  3. package/agents/verifier.md +18 -7
  4. package/dist/index.mjs +744 -91462
  5. package/docs/README.md +57 -46
  6. package/docs/architecture.md +87 -33
  7. package/docs/commands-reference.md +9 -5
  8. package/docs/troubleshooting.md +3 -2
  9. package/package.json +1 -3
  10. package/schema.json +29 -0
  11. package/skills/real-test-pi-crew/SKILL.md +193 -36
  12. package/src/agents/agent-config.ts +1 -1
  13. package/src/agents/discover-agents.ts +1 -1
  14. package/src/config/config-validation.ts +15 -0
  15. package/src/config/config.ts +47 -13
  16. package/src/config/env-vars.ts +35 -0
  17. package/src/config/types.ts +19 -0
  18. package/src/errors.ts +2 -2
  19. package/src/extension/async-notifier.ts +23 -0
  20. package/src/extension/help.ts +21 -10
  21. package/src/extension/knowledge-injection.ts +2 -1
  22. package/src/extension/management.ts +8 -3
  23. package/src/extension/notification-sink.ts +17 -0
  24. package/src/extension/registration/command-utils.ts +28 -2
  25. package/src/extension/registration/commands/dashboard.ts +11 -1
  26. package/src/extension/registration/commands/manage.ts +31 -15
  27. package/src/extension/registration/commands/run.ts +24 -2
  28. package/src/extension/registration/commands/shared.ts +23 -0
  29. package/src/extension/registration/commands/status.ts +25 -2
  30. package/src/extension/registration/context-builder.ts +8 -2
  31. package/src/extension/registration/health-notify-policy.ts +100 -0
  32. package/src/extension/registration/lazy-configurers.ts +35 -0
  33. package/src/extension/registration/lifecycle-handlers.ts +91 -30
  34. package/src/extension/registration/lifecycle.ts +75 -10
  35. package/src/extension/registration/observability.ts +98 -35
  36. package/src/extension/registration/registration-types.ts +7 -5
  37. package/src/extension/registration/runtime-cleanup.ts +9 -3
  38. package/src/extension/registration/subagent-helpers.ts +38 -0
  39. package/src/extension/registration/wire-cross-extension.ts +28 -0
  40. package/src/extension/run-compare.ts +220 -0
  41. package/src/extension/run-export.ts +37 -5
  42. package/src/extension/run-maintenance.ts +155 -5
  43. package/src/extension/team-tool/dispatch/index.ts +3 -2
  44. package/src/extension/team-tool/dispatch/manage.ts +5 -2
  45. package/src/extension/team-tool/goal.ts +4 -1
  46. package/src/extension/team-tool/handle-settings.ts +19 -2
  47. package/src/extension/team-tool/health-monitor.ts +21 -7
  48. package/src/extension/team-tool/lifecycle-actions.ts +49 -1
  49. package/src/extension/team-tool/plan.ts +10 -0
  50. package/src/extension/team-tool/routing-hint.ts +63 -0
  51. package/src/extension/team-tool/status.ts +4 -0
  52. package/src/extension/team-tool.ts +52 -6
  53. package/src/extension/webhook-notify.ts +382 -0
  54. package/src/observability/metric-sink.ts +12 -2
  55. package/src/prompt/prompt-runtime.ts +82 -31
  56. package/src/prompt/worker-events-channel.ts +12 -0
  57. package/src/runtime/README.md +1 -1
  58. package/src/runtime/async-runner.ts +87 -1
  59. package/src/runtime/background-runner.ts +313 -234
  60. package/src/runtime/broker/crew-broker.ts +17 -11
  61. package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
  62. package/src/runtime/broker/wait-status-cache.ts +1 -1
  63. package/src/runtime/child-pi/child-pi-timers.ts +1 -1
  64. package/src/runtime/child-pi/mock-fixtures.ts +48 -0
  65. package/src/runtime/crew-agent-records.ts +337 -45
  66. package/src/runtime/deadletter.ts +43 -1
  67. package/src/runtime/delegate-spawn.ts +5 -1
  68. package/src/runtime/dispatch-batch.ts +72 -5
  69. package/src/runtime/goal-workflow/goal-loop-runner.ts +73 -4
  70. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  71. package/src/runtime/model/model-fallback.ts +21 -1
  72. package/src/runtime/model/pi-args.ts +8 -10
  73. package/src/runtime/recovery/crash-recovery.ts +25 -1
  74. package/src/runtime/run-worker.ts +12 -1
  75. package/src/runtime/scheduling/global-worker-cap.ts +13 -6
  76. package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
  77. package/src/runtime/scheduling/scheduler.ts +49 -13
  78. package/src/runtime/scheduling/semaphore.ts +148 -20
  79. package/src/runtime/scratchpad/README.md +1 -1
  80. package/src/runtime/scratchpad/protocol.ts +1 -1
  81. package/src/runtime/settings-store.ts +1 -1
  82. package/src/runtime/skill-instructions.ts +22 -0
  83. package/src/runtime/stale-reconciler.ts +85 -13
  84. package/src/runtime/task-runner/pre-execution.ts +26 -2
  85. package/src/runtime/task-runner/prompt-builder.ts +142 -45
  86. package/src/runtime/task-runner.ts +21 -1
  87. package/src/runtime/team-runner.ts +38 -1
  88. package/src/runtime/workspace-lock.ts +4 -1
  89. package/src/schema/config-schema.ts +14 -0
  90. package/src/schema/team-tool-schema.ts +17 -0
  91. package/src/state/atomic-write.ts +53 -0
  92. package/src/state/contracts.ts +109 -0
  93. package/src/state/coordination/locks.ts +191 -33
  94. package/src/state/coordination/mailbox.ts +140 -15
  95. package/src/state/crew-init.ts +87 -12
  96. package/src/state/event-log/cursor.ts +37 -1
  97. package/src/state/event-log/event-log-rotation.ts +72 -7
  98. package/src/state/stores/active-run-registry.ts +13 -1
  99. package/src/state/stores/state-store.ts +112 -22
  100. package/src/state/types.ts +4 -0
  101. package/src/ui/dashboard-panes/agents-pane.ts +11 -2
  102. package/src/ui/heartbeat-aggregator.ts +34 -0
  103. package/src/ui/keybinding-map.ts +22 -4
  104. package/src/ui/live-conversation-overlay.ts +6 -3
  105. package/src/ui/run-dashboard.ts +98 -5
  106. package/src/ui/run-snapshot-cache.ts +18 -1
  107. package/src/ui/spinner.ts +26 -2
  108. package/src/ui/tool-progress-formatter.ts +2 -1
  109. package/src/ui/tool-renderers/brief-mode.ts +2 -1
  110. package/src/ui/tool-renderers/index.ts +3 -3
  111. package/src/utils/incremental-reader.ts +11 -3
  112. package/src/utils/paths.ts +94 -12
  113. package/src/utils/project-markers.ts +40 -0
  114. package/src/worktree/worktree-manager.ts +206 -26
  115. package/workflows/distill.workflow.md +3 -3
  116. package/workflows/fast-fix.workflow.md +1 -1
  117. package/workflows/plan-execute.workflow.md +1 -1
  118. package/workflows/review.workflow.md +1 -1
  119. package/workflows/strict-fast-fix.workflow.md +1 -1
  120. package/docs/migration-v0.4-v0.5.md +0 -208
  121. package/docs/runtime-flow.md +0 -148
@@ -1,10 +1,28 @@
1
1
  import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
2
+ import { loadRunManifestById } from "../../../state/stores/state-store.ts";
2
3
  import { suggestRunIds } from "../../command-completions.ts";
3
4
  import { piTeamsHelp } from "../../help.ts";
4
- import { commandText, notifyCommandResult } from "../command-utils.ts";
5
+ import { commandText, NOTIFY_TEXT_CAP, notifyCommandResult } from "../command-utils.ts";
5
6
  import type { RegisterTeamCommandsDeps } from "./shared.ts";
6
7
  import { handleTeamTool, teamCommandContext } from "./shared.ts";
7
8
 
9
+ /**
10
+ * W5 (slash-commands fix spec): `/team-events` pointer. When the events
11
+ * listing is about to be clipped by the 800-char notification cap, resolve
12
+ * the run's on-disk events log so the message can end with a reachable path.
13
+ * Derived from the SAME source the `events` team action reads
14
+ * (inspect.ts): locateRunCwd + loadRunManifestById → manifest.eventsPath.
15
+ * The dynamic team-tool import is free here — the awaited handleTeamTool
16
+ * call above it already loaded that module into the ESM cache.
17
+ */
18
+ async function eventsLogFooter(runId: string | undefined, text: string, cwd: string): Promise<string | undefined> {
19
+ if (!runId || text.length <= NOTIFY_TEXT_CAP) return undefined;
20
+ const { locateRunCwd } = await import("../../team-tool.ts"); // LAZY: defer team-tool chain to call site (handleTeamTool already cached it).
21
+ const runCwd = locateRunCwd(runId, cwd);
22
+ const manifest = runCwd ? loadRunManifestById(runCwd, runId)?.manifest : undefined;
23
+ return manifest ? ` Full log: ${manifest.eventsPath}` : undefined;
24
+ }
25
+
8
26
  export function registerStatusCommands(pi: ExtensionAPI, deps: RegisterTeamCommandsDeps): void {
9
27
  pi.registerCommand("teams", {
10
28
  description: "List pi-crew teams, workflows, and agents",
@@ -34,7 +52,12 @@ export function registerStatusCommands(pi: ExtensionAPI, deps: RegisterTeamComma
34
52
  getRunSnapshotCache: deps.getRunSnapshotCache,
35
53
  },
36
54
  );
37
- await notifyCommandResult(ctx, commandText(result));
55
+ const text = commandText(result);
56
+ await notifyCommandResult(ctx, text, {
57
+ // W5: only `team-events` output is a truncatable on-disk log —
58
+ // point at the full file when the listing gets clipped.
59
+ truncatedFooter: action === "events" ? await eventsLogFooter(runId, text, ctx.cwd) : undefined,
60
+ });
38
61
  },
39
62
  });
40
63
  }
@@ -58,6 +58,7 @@ export function buildRegistrationContext(pi: ExtensionAPI): RegistrationContext
58
58
  autoRepairTimer: undefined,
59
59
  tempReconcileTimer: undefined,
60
60
  otlpExporter: undefined,
61
+ initPromise: undefined,
61
62
  } as ObservabilityState,
62
63
  lifecycleState: {
63
64
  notifierStarted: false,
@@ -121,7 +122,12 @@ export function buildRegistrationContext(pi: ExtensionAPI): RegistrationContext
121
122
  };
122
123
 
123
124
  ctx.getManifestCache = (cwd: string) => {
124
- if (ctx.manifestCache && ctx.cacheCwd === cwd) return ctx.manifestCache;
125
+ // F13 (RR-018): a disposed cache pair must be recreated EVEN WHEN the cwd
126
+ // is unchanged. Session cleanup disposes the caches but used to leave
127
+ // `cacheCwd` pointing at them, so same-cwd access returned the disposed
128
+ // instance (no disposed flag existed). isDisposed() makes the pair's
129
+ // liveness part of the swap check.
130
+ if (ctx.manifestCache && ctx.cacheCwd === cwd && !ctx.runSnapshotCache.isDisposed()) return ctx.manifestCache;
125
131
  if (ctx.manifestCache) ctx.manifestCache.dispose();
126
132
  if (ctx.runSnapshotCache) ctx.runSnapshotCache.dispose?.();
127
133
  ctx.cacheCwd = cwd;
@@ -130,7 +136,7 @@ export function buildRegistrationContext(pi: ExtensionAPI): RegistrationContext
130
136
  return ctx.manifestCache;
131
137
  };
132
138
  ctx.getRunSnapshotCache = (cwd: string) => {
133
- if (ctx.cacheCwd !== cwd) ctx.getManifestCache(cwd);
139
+ if (ctx.cacheCwd !== cwd || ctx.runSnapshotCache.isDisposed()) ctx.getManifestCache(cwd);
134
140
  return ctx.runSnapshotCache;
135
141
  };
136
142
 
@@ -0,0 +1,100 @@
1
+ /**
2
+ * health-notify-policy.ts — FINDING 6 (2026-09-23 battery): ambient
3
+ * notification re-delivery storms.
4
+ *
5
+ * Observed live: a zombie-run leak made the health monitor re-fire the same
6
+ * `recovery_missing_heartbeat` notification every cooldown window (5 min) for
7
+ * hours. Each fire queued a follow-up message on the host session while the
8
+ * parent was busy; the host drains follow-ups ONE PER TURN BOUNDARY
9
+ * (followUpMode "one-at-a-time"), so the queued duplicates kept dripping into
10
+ * the parent conversation for ~5 hours AFTER the runs were deleted and the
11
+ * monitor had written its `clear` entries.
12
+ *
13
+ * Policy fixes (both sides of the storm):
14
+ * 1. BOUNDED RE-FIRE — a given notification key may fire at most
15
+ * MAX_HEALTH_NOTIFY_FIRES times unless the underlying fingerprint changes
16
+ * (a different dead/missing/task-count shape is a NEW situation and gets a
17
+ * fresh budget). Previously the 5-min cooldown re-armed forever.
18
+ * 2. RESET-ON-CLEAR — the clear path resets the fire budget, so a genuine
19
+ * recurrence after a resolved incident notifies again (up to the cap).
20
+ *
21
+ * The host-queue purge of already-queued copies is handled separately
22
+ * (purgeQueuedAmbientNotifications in subagent-helpers.ts) — this module only
23
+ * stops pi-crew from FEEDING the queue.
24
+ */
25
+
26
+ /** Maximum notifications emitted per key before a fingerprint change. */
27
+ export const MAX_HEALTH_NOTIFY_FIRES = 3;
28
+
29
+ export interface AutoRecoveryEntry {
30
+ insertedAt: number;
31
+ lastAccessAt: number;
32
+ /** Times this key has fired under the current fingerprint. */
33
+ fires?: number;
34
+ /** Shape of the situation last notified (see healthNotifyFingerprint). */
35
+ fingerprint?: string;
36
+ }
37
+
38
+ export interface HealthNotifyState {
39
+ /** Map keyed `${kind}_${runId}` — typically ctx.autoRecoveryLast. */
40
+ entries: Map<string, AutoRecoveryEntry>;
41
+ /** LRU cap — typically ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES. */
42
+ maxEntries: number;
43
+ }
44
+
45
+ /** Compact fingerprint of the health situation: same counts = same situation. */
46
+ export function healthNotifyFingerprint(summary: { dead: number; missing: number }, totalTasks: number): string {
47
+ return `${summary.dead}/${summary.missing}/${totalTasks}`;
48
+ }
49
+
50
+ /**
51
+ * Decide whether a health notification may fire now, and record the decision.
52
+ *
53
+ * Fires when: (a) never fired before, (b) cooldown elapsed AND the same
54
+ * fingerprint still has budget, or (c) the fingerprint CHANGED (new situation
55
+ * — fresh budget). Blocks within the cooldown window and after the budget is
56
+ * exhausted. Evicts oldest-access entries at the cap (LRU, mirrors the
57
+ * previous inline behavior).
58
+ */
59
+ export function recordHealthNotifyDecision(
60
+ state: HealthNotifyState,
61
+ key: string,
62
+ fingerprint: string,
63
+ now: number,
64
+ cooldownMs = 5 * 60_000,
65
+ maxFires = MAX_HEALTH_NOTIFY_FIRES,
66
+ ): boolean {
67
+ const previous = state.entries.get(key);
68
+ if (previous !== undefined && now - previous.lastAccessAt < cooldownMs) return false;
69
+ if (previous !== undefined && previous.fires !== undefined && previous.fires >= maxFires && previous.fingerprint === fingerprint) {
70
+ // Budget exhausted for an unchanged situation. Still refresh access so
71
+ // LRU ordering reflects interest in this key.
72
+ previous.lastAccessAt = now;
73
+ return false;
74
+ }
75
+ while (state.entries.size >= state.maxEntries) {
76
+ let oldestKey: string | undefined;
77
+ let oldestAccess = Infinity;
78
+ for (const [k, v] of state.entries) {
79
+ if (v.lastAccessAt < oldestAccess) {
80
+ oldestAccess = v.lastAccessAt;
81
+ oldestKey = k;
82
+ }
83
+ }
84
+ if (oldestKey === undefined) break;
85
+ state.entries.delete(oldestKey);
86
+ }
87
+ const fires = previous?.fingerprint === fingerprint ? (previous.fires ?? 0) + 1 : 1;
88
+ state.entries.set(key, {
89
+ insertedAt: previous?.insertedAt ?? now,
90
+ lastAccessAt: now,
91
+ fires,
92
+ fingerprint,
93
+ });
94
+ return true;
95
+ }
96
+
97
+ /** Reset a key's fire budget (clear path: resolved incident gets fresh budget). */
98
+ export function resetHealthNotifyEntry(state: HealthNotifyState, key: string): void {
99
+ state.entries.delete(key);
100
+ }
@@ -24,6 +24,7 @@ import { reconcileAllStaleRuns } from "../../runtime/recovery/crash-recovery.ts"
24
24
  import { reconcileOrphanedTempWorkspaces } from "../../runtime/stale-reconciler.ts";
25
25
  import { requestPowerbarUpdate } from "../../ui/powerbar-publisher.ts";
26
26
  import { logInternalError } from "../../utils/internal-error.ts";
27
+ import { extractSessionId } from "../../utils/session-utils.ts";
27
28
  import type { RegistrationContext } from "./registration-types.ts";
28
29
  import { sendAgentWakeUp, sendFollowUp } from "./subagent-helpers.ts";
29
30
 
@@ -48,6 +49,32 @@ export function installLazyConfigurers(pi: ExtensionAPI, ctx: RegistrationContex
48
49
  ctx.configureDeliveryCoordinator = (): void => {
49
50
  void configureDeliveryCoordinatorImpl(pi, ctx);
50
51
  };
52
+ installTurnReconcileHook(pi, ctx);
53
+ }
54
+
55
+ /**
56
+ * F12 (RR-018): register the before_agent_start reconcile hook EXACTLY ONCE
57
+ * for the extension lifetime. The callback resolves the CURRENT session
58
+ * (`ctx.currentCtx`) at fire time instead of capturing the configuring
59
+ * session's ExtensionContext, so a stale session's hook can never re-activate,
60
+ * double-reconcile a turn, or flip the shared manifest-cache cwd back to an
61
+ * old session's directory. (There is no pi.off unregister surface, so the old
62
+ * per-session registration accumulated one dead hook per session switch.)
63
+ */
64
+ function installTurnReconcileHook(pi: ExtensionAPI, ctx: RegistrationContext): void {
65
+ try {
66
+ pi.on?.("before_agent_start", () => {
67
+ const current = ctx.currentCtx;
68
+ if (!current || ctx.cleanedUp) return;
69
+ try {
70
+ reconcileAllStaleRuns(current.cwd, ctx.getManifestCache(current.cwd), undefined, extractSessionId(current));
71
+ } catch (error) {
72
+ logInternalError("register.autoRepair.turnHook", error);
73
+ }
74
+ });
75
+ } catch {
76
+ /* older Pi without before_agent_start — rely on the interval alone */
77
+ }
51
78
  }
52
79
 
53
80
  /**
@@ -58,6 +85,9 @@ export function installLazyConfigurers(pi: ExtensionAPI, ctx: RegistrationContex
58
85
  * `ctx.currentCtx` and caches.
59
86
  */
60
87
  async function configureNotificationsImpl(pi: ExtensionAPI, ctx: RegistrationContext, extCtx: ExtensionContext): Promise<void> {
88
+ // F11: capture the owning generation BEFORE the first await — the deps
89
+ // closure below re-checks it after lifecycle.ts's lazy imports resolve.
90
+ const ownerGeneration = ctx.sessionGeneration;
61
91
  try {
62
92
  // LAZY: registration/lifecycle is heavy (notification-router + sink)
63
93
  const lifecycleModule = await import("./lifecycle.ts");
@@ -68,6 +98,7 @@ async function configureNotificationsImpl(pi: ExtensionAPI, ctx: RegistrationCon
68
98
  getManifestCache: ctx.getManifestCache,
69
99
  getRunSnapshotCache: ctx.getRunSnapshotCache,
70
100
  requestPowerbarUpdate,
101
+ isOwnerSession: () => !ctx.cleanedUp && ctx.sessionGeneration === ownerGeneration,
71
102
  });
72
103
  } catch (error) {
73
104
  logInternalError("register.configureNotifications", error);
@@ -90,6 +121,8 @@ async function configureObservabilityImpl(pi: ExtensionAPI, ctx: RegistrationCon
90
121
  getManifestCache: ctx.getManifestCache,
91
122
  notifyOperator: ctx.notifyOperator,
92
123
  isCleanedUp: () => ctx.cleanedUp,
124
+ // F11: reuse the existing sessionGeneration counter — never a parallel one.
125
+ getSessionGeneration: () => ctx.sessionGeneration,
93
126
  reconcileStaleRuns: (cwd, cache, currentSessionId) => reconcileAllStaleRuns(cwd, cache, undefined, currentSessionId),
94
127
  reconcileOrphanedTempWorkspaces: (now, opts) => reconcileOrphanedTempWorkspaces(now, opts),
95
128
  cleanupOrphanTempDirs,
@@ -108,6 +141,7 @@ async function configureObservabilityImpl(pi: ExtensionAPI, ctx: RegistrationCon
108
141
  * No dynamic import needed — lifecycle is already loaded by configureNotifications.
109
142
  */
110
143
  async function configureDeliveryCoordinatorImpl(pi: ExtensionAPI, ctx: RegistrationContext): Promise<void> {
144
+ const ownerGeneration = ctx.sessionGeneration;
111
145
  try {
112
146
  // LAZY: lifecycle.ts may not be loaded yet if configureNotifications hasn't fired.
113
147
  const lifecycleModule = await import("./lifecycle.ts");
@@ -117,6 +151,7 @@ async function configureDeliveryCoordinatorImpl(pi: ExtensionAPI, ctx: Registrat
117
151
  notifyOperator: ctx.notifyOperator,
118
152
  sendFollowUp,
119
153
  sendAgentWakeUp,
154
+ isOwnerSession: () => !ctx.cleanedUp && ctx.sessionGeneration === ownerGeneration,
120
155
  });
121
156
  } catch (error) {
122
157
  logInternalError("register.configureDeliveryCoordinator", error);
@@ -20,7 +20,12 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
20
20
  import { loadConfig } from "../../config/config.ts";
21
21
  import { DEFAULT_UI } from "../../config/defaults.ts";
22
22
  import { getCrewEnv } from "../../config/env-vars.ts";
23
- import { pruneFinishedRuns, pruneUserLevelRuns } from "../../extension/run-maintenance.ts";
23
+ import {
24
+ pruneFinishedRuns,
25
+ pruneUserLevelRuns,
26
+ resolveAutoPruneAgeFloorMs,
27
+ resolveAutoPruneKeep,
28
+ } from "../../extension/run-maintenance.ts";
24
29
  import { type BrokerSpawnCredentials, setActiveBrokerIssuer, setActiveBrokerRevoker } from "../../runtime/broker/broker-issuer.ts";
25
30
  import { CrewBroker } from "../../runtime/broker/crew-broker.ts";
26
31
  import { terminateActiveChildPiProcesses } from "../../runtime/child-pi/child-pi.ts";
@@ -37,9 +42,10 @@ import { CrewScheduler, type ScheduledJob } from "../../runtime/scheduling/sched
37
42
  import { tryRegisterSessionCleanup } from "../../runtime/session-resources.ts";
38
43
  import { createSessionSnapshot } from "../../runtime/session-snapshot.ts";
39
44
  import { applyCrewSettingsTiersToConfig, loadCrewSettingsTiers, scheduledJobsHiddenCountOf } from "../../runtime/settings-store.ts";
45
+ import { loadTasksWithRecovery } from "../../state/stores/manifest-io.ts";
40
46
  import { loadRunManifestById } from "../../state/stores/state-store.ts";
41
47
  import type { TeamRunManifest } from "../../state/types.ts";
42
- import { summarizeHeartbeats } from "../../ui/heartbeat-aggregator.ts";
48
+ import { overlayFreshTaskStatuses, summarizeHeartbeats } from "../../ui/heartbeat-aggregator.ts";
43
49
  import { installInlinePanel } from "../../ui/inline-panel/index.ts";
44
50
  import { clearSessionSwitchInFlight, markSessionSwitchInFlight } from "../../ui/inline-panel/view-session-store.ts";
45
51
  import { requestRender, setExtensionWidget, toPiWidgetPlacement } from "../../ui/pi-ui-compat.ts";
@@ -64,8 +70,11 @@ import { notifyActiveRuns } from "../session-summary.ts";
64
70
  import { persistScheduledJobUpdate, registerCrewScheduler, stashScheduledJobsHiddenCount } from "../team-tool/handle-schedule.ts";
65
71
  import { handleTeamTool } from "../team-tool.ts";
66
72
  import { runArtifactCleanup } from "./artifact-cleanup.ts";
73
+ import { healthNotifyFingerprint, recordHealthNotifyDecision, resetHealthNotifyEntry } from "./health-notify-policy.ts";
67
74
  import type { RegistrationContext } from "./registration-types.ts";
68
75
  import { createScheduleEventNotifier } from "./schedule-toast-bridge.ts";
76
+ import { purgeQueuedAmbientNotifications } from "./subagent-helpers.ts";
77
+ import { refreshCrossExtensionWiringForSession } from "./wire-cross-extension.ts";
69
78
 
70
79
  /**
71
80
  * Register all session-lifecycle handlers on the ExtensionAPI. The caller
@@ -251,6 +260,10 @@ function installSessionStartHandler(pi: ExtensionAPI, ctx: RegistrationContext):
251
260
  // it can issue tokens for child runs in this session. The controller
252
261
  // already gates by flag + root-session; this is a no-op when disabled.
253
262
  ctx.brokerController?.setSessionId(currentSessionId);
263
+ // F13 (RR-018): RPC wiring is extension-lifetime and survives session
264
+ // switches; rebind the crew global registry to THIS session's manifest
265
+ // cache (idempotent — see wire-cross-extension.ts).
266
+ refreshCrossExtensionWiringForSession(pi, ctx);
254
267
 
255
268
  // Defer ALL heavy cleanup to after the session_start handler returns.
256
269
  // These operations involve synchronous directory scanning (readdirSync, readFileSync)
@@ -460,7 +473,12 @@ async function runDeferredSessionCleanup(
460
473
 
461
474
  // Auto-prune finished project-level run directories
462
475
  try {
463
- const { removed } = pruneFinishedRuns(extensionCtx.cwd, 10);
476
+ // DP-01: keep + age-floor are env-driven; the age floor means a run
477
+ // finished <24h ago survives a session restart even beyond top-keep.
478
+ const { removed } = pruneFinishedRuns(extensionCtx.cwd, resolveAutoPruneKeep(), {
479
+ ageFloorMs: resolveAutoPruneAgeFloorMs(),
480
+ intent: "session-start-auto",
481
+ });
464
482
  if (removed.length > 0) {
465
483
  ctx.notifyOperator({
466
484
  id: `auto_prune_project`,
@@ -476,7 +494,10 @@ async function runDeferredSessionCleanup(
476
494
 
477
495
  // Auto-prune finished user-level run directories
478
496
  try {
479
- const { removed } = pruneUserLevelRuns(10);
497
+ const { removed } = pruneUserLevelRuns(resolveAutoPruneKeep(), {
498
+ ageFloorMs: resolveAutoPruneAgeFloorMs(),
499
+ intent: "session-start-auto",
500
+ });
480
501
  if (removed.length > 0) {
481
502
  ctx.notifyOperator({
482
503
  id: `auto_prune_user`,
@@ -648,6 +669,18 @@ export function evictRunFromManifests(manifests: TeamRunManifest[], runId: strin
648
669
  return manifests.filter((m) => m.runId !== runId);
649
670
  }
650
671
 
672
+ /**
673
+ * F14 (RR-019): cheap signature of the preloaded-manifest frame. Only a
674
+ * CHANGED signature lets `backgroundPreload` count as render activity
675
+ * (`schedule()`); an identical signature means "nothing the widget would
676
+ * paint differently" and must NOT reset the render scheduler's idle
677
+ * counters (that defeated the R1 idle stop — see setupRenderLoop). Covers
678
+ * run identity, status transitions, and updatedAt bumps. Exported for tests.
679
+ */
680
+ export function manifestsFrameSignature(manifests: TeamRunManifest[]): string {
681
+ return manifests.map((m) => `${m.runId}:${m.status}:${m.updatedAt}`).join("|");
682
+ }
683
+
651
684
  /**
652
685
  * Apply a runEventBus payload to a preloaded-manifest frame: evict the run on
653
686
  * terminal events, pass through unchanged otherwise. This is the exact logic
@@ -677,16 +710,24 @@ function setupRenderLoop(
677
710
  let lastPreloadedManifests: TeamRunManifest[] = [];
678
711
  let lastFrameManifestCache: ReturnType<typeof createManifestCache> | undefined;
679
712
  let lastFrameSnapshotCache: ReturnType<typeof createRunSnapshotCache> | undefined;
713
+ // F14: signature of the last frame backgroundPreload saw — only a change
714
+ // here counts as render activity.
715
+ let lastFrameSignature: string | undefined;
680
716
 
681
717
  const ownerGeneration = ctx.sessionGeneration;
682
718
 
683
- const buildFrame = async (): Promise<boolean> => {
684
- if (!ctx.currentCtx) return false;
719
+ const buildFrame = async (): Promise<{ ok: boolean; changed: boolean }> => {
720
+ if (!ctx.currentCtx) return { ok: false, changed: false };
685
721
  lastPreloadedConfig = loadConfig(ctx.currentCtx.cwd);
686
722
  lastFrameManifestCache = ctx.getManifestCache(ctx.currentCtx.cwd);
687
723
  lastFrameSnapshotCache = ctx.getRunSnapshotCache(ctx.currentCtx.cwd);
688
724
  const manifests = lastFrameManifestCache.list(20);
689
725
  lastPreloadedManifests = manifests;
726
+ // F14 (RR-019): compute the frame signature BEFORE the preload await so
727
+ // `changed` reflects what this tick saw, not a mid-preload mutation.
728
+ const frameSignature = manifestsFrameSignature(manifests);
729
+ const frameChanged = frameSignature !== lastFrameSignature;
730
+ lastFrameSignature = frameSignature;
690
731
  // pts/2 hang fix: reconcile per-run watchers against the ACTIVE set only.
691
732
  {
692
733
  const onRunChange = (runId: string): void => {
@@ -718,16 +759,23 @@ function setupRenderLoop(
718
759
  }
719
760
  const runIds = manifests.map((r) => r.runId);
720
761
  await lastFrameSnapshotCache.preloadAllStale(runIds);
721
- return true;
762
+ return { ok: true, changed: frameChanged };
722
763
  };
723
764
 
724
765
  const backgroundPreload = (): void => {
725
766
  if (!ctx.currentCtx || preloading) return;
726
767
  preloading = true;
727
768
  buildFrame()
728
- .then((ok) => {
769
+ .then(({ ok, changed }) => {
729
770
  preloading = false;
730
- if (ok) ctx.renderScheduler?.schedule();
771
+ // F14 (RR-019): preload completion is maintenance, NOT user activity.
772
+ // Only a genuinely CHANGED frame may call schedule() — an unchanged
773
+ // frame calling schedule() reset `lastEventAt`/`idleFallbackRenders`
774
+ // and re-armed the fallback loop that fallbackLoop() had just
775
+ // deliberately stopped (R1), rendering forever while idle. Real
776
+ // work still animates: runEventBus + fs.watch subscriptions call
777
+ // schedule() directly on genuine events.
778
+ if (ok && changed) ctx.renderScheduler?.schedule();
731
779
  })
732
780
  .catch((error: unknown) => {
733
781
  preloading = false;
@@ -796,7 +844,16 @@ function setupRenderLoop(
796
844
  const clearHealthNotifications = (runId: string): void => {
797
845
  for (const kind of ["recovery_dead_workers", "recovery_missing_heartbeat"]) {
798
846
  const key = `${kind}_${runId}`;
799
- ctx.autoRecoveryLast.delete(key);
847
+ // FINDING 6: reset the fire budget (a genuine recurrence after
848
+ // this clear re-notifies) AND opportunistically purge
849
+ // still-QUEUED host follow-up copies of the original warning —
850
+ // the host drains the queue one message per turn boundary, so
851
+ // stale copies otherwise drip in for hours after the clear
852
+ // (live: 5h of "missing heartbeat" replays after the 2026-09-23
853
+ // zombie cleanup). The purge is feature-detected: hosts without
854
+ // clearQueuedUserMessagesMatching (e.g. 0.87.0) no-op here; the
855
+ // fire-cap policy bounds the backlog on every host.
856
+ resetHealthNotifyEntry({ entries: ctx.autoRecoveryLast, maxEntries: ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES }, key);
800
857
  ctx.notifyOperator({
801
858
  id: key,
802
859
  clear: true,
@@ -805,6 +862,10 @@ function setupRenderLoop(
805
862
  runId,
806
863
  title: `Cleared ${kind} for ${runId}`,
807
864
  });
865
+ purgeQueuedAmbientNotifications(
866
+ pi,
867
+ (text) => text.includes(runId) && (text.includes("dead worker") || text.includes("missing heartbeat")),
868
+ );
808
869
  }
809
870
  };
810
871
  for (const run of sessionManifests) {
@@ -838,29 +899,29 @@ function setupRenderLoop(
838
899
  clearHealthNotifications(run.runId);
839
900
  continue;
840
901
  }
841
- const summary = summarizeHeartbeats(snapshot, { now });
902
+ // GATE 3 (FINDING 5) — task-status truth lives on DISK, not in the
903
+ // snapshot cache. A worker parked on `ask` writes running → waiting
904
+ // to tasks.json immediately, but the cached snapshot can lag that
905
+ // transition; in the window a parked (alive, silent-by-design)
906
+ // worker counted as active-without-heartbeat and fired a false
907
+ // "dead worker" (live: team_20260923100114, 01_explore parked on
908
+ // ask). Overlay FRESH statuses before summarizing; divergence also
909
+ // invalidates the stale cache entry so the next tick rebuilds it.
910
+ const freshTasks = loadTasksWithRecovery(freshManifest.tasksPath, freshManifest.eventsPath, run.runId);
911
+ const overlaid = overlayFreshTaskStatuses(snapshot, freshTasks);
912
+ if (overlaid !== snapshot) snapshotCache.invalidate(run.runId);
913
+ const summary = summarizeHeartbeats(overlaid, { now });
914
+ const fingerprint = healthNotifyFingerprint(summary, overlaid.tasks.length);
842
915
  const maybeNotifyHealth = (kind: string, count: number, title: string, body: string): void => {
843
916
  if (count <= 0) return;
844
917
  const key = `${kind}_${run.runId}`;
845
- const previous = ctx.autoRecoveryLast.get(key);
846
- if (previous !== undefined && now - previous.lastAccessAt < 5 * 60_000) return;
847
- // Defensive cap: evict entry with oldest lastAccessAt before inserting/updating.
848
- while (ctx.autoRecoveryLast.size >= ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES) {
849
- let oldestKey: string | undefined;
850
- let oldestAccess = Infinity;
851
- for (const [k, v] of ctx.autoRecoveryLast) {
852
- if (v.lastAccessAt < oldestAccess) {
853
- oldestAccess = v.lastAccessAt;
854
- oldestKey = k;
855
- }
856
- }
857
- if (oldestKey === undefined) break;
858
- ctx.autoRecoveryLast.delete(oldestKey);
859
- }
860
- ctx.autoRecoveryLast.set(key, {
861
- insertedAt: now,
862
- lastAccessAt: now,
863
- });
918
+ // FINDING 6: bounded re-fire — ≤ MAX_HEALTH_NOTIFY_FIRES per
919
+ // UNCHANGED fingerprint (5-min cooldown between fires). The
920
+ // old forever-re-arming cooldown fed the host follow-up queue
921
+ // duplicates that drained one-per-turn-boundary for hours.
922
+ // Policy detail + LRU eviction: health-notify-policy.ts.
923
+ const state = { entries: ctx.autoRecoveryLast, maxEntries: ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES };
924
+ if (!recordHealthNotifyDecision(state, key, fingerprint, now)) return;
864
925
  ctx.notifyOperator({
865
926
  id: key,
866
927
  severity: "warning",
@@ -22,6 +22,7 @@ import { projectCrewRoot } from "../../utils/paths.ts";
22
22
  import type { AsyncNotifierState } from "../async-notifier.ts";
23
23
  import type { NotificationDescriptor, NotificationRouter } from "../notification-router.ts";
24
24
  import type { NotificationSink } from "../notification-sink.ts";
25
+ import { createWebhookNotifier } from "../webhook-notify.ts";
25
26
 
26
27
  /**
27
28
  * Mutable state owned by register.ts and read/written by this module.
@@ -59,6 +60,30 @@ export function startLifecycleWatchers(ctx: ExtensionContext, state: LifecycleSt
59
60
  const loadedConfig = loadConfig(ctx.cwd);
60
61
  state.notifierStarted = true;
61
62
  try {
63
+ // US-030 (docs/specs/US-030.md): outbound webhook sink built from config.
64
+ // Gated on the notifications master switch (`notifications.enabled !== false`,
65
+ // same gate as configureNotifications), opt-in per `webhook.url`, and
66
+ // SSRF-guarded inside the factory. Inactive config → shared no-op
67
+ // singleton (zero network calls, zero hot-path allocation).
68
+ const notificationsConfig = loadedConfig.config.notifications;
69
+ const webhookNotifier =
70
+ notificationsConfig?.enabled === false
71
+ ? undefined
72
+ : createWebhookNotifier(
73
+ { webhook: notificationsConfig?.webhook, quietHours: notificationsConfig?.quietHours },
74
+ {
75
+ onFailure: (failure) => {
76
+ // Spec: failures surface as a `webhook.failed` event (crew.*-prefixed
77
+ // pi event, mirroring crew.run.completed / crew.task.overflow).
78
+ // logInternalError already fired inside the notifier.
79
+ deps.pi.events?.emit?.("crew.webhook.failed", {
80
+ runId: failure.runId,
81
+ attempts: failure.attempts,
82
+ error: failure.error,
83
+ });
84
+ },
85
+ },
86
+ );
62
87
  // LAZY: async-notifier pulls in debounce + cron helpers — defer
63
88
  // until the first lifecycle install (deferred import within module).
64
89
  void import("../async-notifier.ts").then(({ startAsyncRunNotifier }) => {
@@ -66,6 +91,7 @@ export function startLifecycleWatchers(ctx: ExtensionContext, state: LifecycleSt
66
91
  startAsyncRunNotifier(ctx, deps.notifierState, loadedConfig.config.notifierIntervalMs ?? DEFAULT_UI.notifierIntervalMs, {
67
92
  generation: deps.ownerGeneration,
68
93
  isCurrent: (generation) => generation === deps.ownerGeneration && deps.isContextCurrent(ctx, deps.ownerGeneration),
94
+ webhookNotifier,
69
95
  });
70
96
  });
71
97
  return true;
@@ -107,6 +133,13 @@ export interface NotificationsDeps {
107
133
  ctx: ExtensionContext,
108
134
  notificationCount: number,
109
135
  ) => void;
136
+ /**
137
+ * F11 (RR-018): ownership predicate — true only while the session that
138
+ * started this configure is still current. Checked after every lazy-import
139
+ * await boundary before publishing into shared state. Optional for older
140
+ * hand-built deps; defaults to "still owner".
141
+ */
142
+ isOwnerSession?: () => boolean;
110
143
  }
111
144
 
112
145
  /**
@@ -117,6 +150,11 @@ export interface NotificationsDeps {
117
150
  * - `config.notifications?.enabled === false` → no-op.
118
151
  */
119
152
  export async function configureNotifications(ctx: ExtensionContext, state: LifecycleState, deps: NotificationsDeps): Promise<void> {
153
+ // F11 (RR-018): ownership predicate — re-verified after every await below.
154
+ // A continuation whose session was cleaned up or superseded must not
155
+ // publish resources into shared state. Optional so hand-built deps from
156
+ // older call sites keep working (defaults to "still owner").
157
+ const stillOwns = deps.isOwnerSession ?? (() => true);
120
158
  state.notificationRouter?.dispose();
121
159
  state.notificationSink?.dispose();
122
160
  state.notificationRouter = undefined;
@@ -126,26 +164,29 @@ export async function configureNotifications(ctx: ExtensionContext, state: Lifec
126
164
 
127
165
  // LAZY: notification router — wires events to sinks
128
166
  const { NotificationRouter } = await import("../notification-router.ts");
167
+ if (!stillOwns()) return;
129
168
  // LAZY: JSONL sink — file-backed notification storage
130
169
  const { createJsonlSink } = await import("../notification-sink.ts");
170
+ if (!stillOwns()) return;
131
171
  // LAZY: follow-up helpers — shared with subagent handoff
132
172
  const { sendFollowUp } = await import("./subagent-helpers.ts");
173
+ if (!stillOwns()) return;
133
174
  // LAZY: widget updater — refreshes the TUI status widget
134
175
  const { updateCrewWidget } = await import("../../ui/widget/index.ts");
176
+ if (!stillOwns()) return;
135
177
 
136
- if (config.telemetry?.enabled !== false) {
137
- state.notificationSink = createJsonlSink(
138
- projectCrewRoot(ctx.cwd),
139
- config.notifications?.sinkRetentionDays ?? DEFAULT_NOTIFICATIONS.sinkRetentionDays,
140
- );
141
- }
142
- state.notificationRouter = new NotificationRouter(
178
+ // F11: create into locals — publish only while ownership still holds.
179
+ const notificationSink =
180
+ config.telemetry?.enabled !== false
181
+ ? createJsonlSink(projectCrewRoot(ctx.cwd), config.notifications?.sinkRetentionDays ?? DEFAULT_NOTIFICATIONS.sinkRetentionDays)
182
+ : undefined;
183
+ const notificationRouter = new NotificationRouter(
143
184
  {
144
185
  dedupWindowMs: config.notifications?.dedupWindowMs ?? DEFAULT_NOTIFICATIONS.dedupWindowMs,
145
186
  batchWindowMs: config.notifications?.batchWindowMs ?? DEFAULT_NOTIFICATIONS.batchWindowMs,
146
187
  quietHours: config.notifications?.quietHours,
147
188
  severityFilter: config.notifications?.severityFilter ?? [...DEFAULT_NOTIFICATIONS.severityFilter],
148
- sink: (notification) => state.notificationSink?.write(notification),
189
+ sink: (notification) => notificationSink?.write(notification),
149
190
  },
150
191
  (notification: NotificationDescriptor) => {
151
192
  if (notification.clear) {
@@ -183,6 +224,15 @@ export async function configureNotifications(ctx: ExtensionContext, state: Lifec
183
224
  }
184
225
  },
185
226
  );
227
+ if (!stillOwns()) {
228
+ // F11: ownership lost while suspended — dispose what was just created
229
+ // instead of publishing (no orphans).
230
+ notificationRouter.dispose();
231
+ notificationSink?.dispose();
232
+ return;
233
+ }
234
+ state.notificationSink = notificationSink;
235
+ state.notificationRouter = notificationRouter;
186
236
  }
187
237
 
188
238
  /** Dispose notification router + sink. */
@@ -200,6 +250,8 @@ export interface DeliveryDeps {
200
250
  notifyOperator: (notification: NotificationDescriptor) => void;
201
251
  sendFollowUp: (pi: ExtensionAPI, message: string) => void;
202
252
  sendAgentWakeUp: (pi: ExtensionAPI, message: string) => void;
253
+ /** F11 (RR-018): same ownership predicate as NotificationsDeps.isOwnerSession. */
254
+ isOwnerSession?: () => boolean;
203
255
  }
204
256
 
205
257
  /**
@@ -207,15 +259,19 @@ export interface DeliveryDeps {
207
259
  * register.ts as part of H3-L2 (delivery lifecycle service).
208
260
  */
209
261
  export async function configureDeliveryCoordinator(state: LifecycleState, deps: DeliveryDeps): Promise<void> {
262
+ // F11: ownership predicate — re-verified after the lazy imports below.
263
+ const stillOwns = deps.isOwnerSession ?? (() => true);
210
264
  state.deliveryCoordinator?.dispose();
211
265
  state.deliveryCoordinator = undefined;
212
266
  state.overflowTracker?.dispose();
213
267
  state.overflowTracker = undefined;
214
268
  // LAZY: delivery coordinator — batches notification fan-out
215
269
  const { DeliveryCoordinator } = await import("../../runtime/delivery-coordinator.ts");
270
+ if (!stillOwns()) return;
216
271
  // LAZY: overflow tracker — recovers from backlog overflow
217
272
  const { OverflowRecoveryTracker } = await import("../../runtime/recovery/overflow-recovery.ts");
218
- state.deliveryCoordinator = new DeliveryCoordinator({
273
+ if (!stillOwns()) return;
274
+ const deliveryCoordinator = new DeliveryCoordinator({
219
275
  emit: (event, data) => {
220
276
  deps.pi.events?.emit?.(event, data);
221
277
  },
@@ -226,7 +282,7 @@ export async function configureDeliveryCoordinator(state: LifecycleState, deps:
226
282
  deps.sendAgentWakeUp(deps.pi, message);
227
283
  },
228
284
  });
229
- state.overflowTracker = new OverflowRecoveryTracker({
285
+ const overflowTracker = new OverflowRecoveryTracker({
230
286
  onPhaseChange: (phaseState, previousPhase) => {
231
287
  if (deps.observabilityState.metricRegistry) {
232
288
  deps.observabilityState.metricRegistry
@@ -254,6 +310,15 @@ export async function configureDeliveryCoordinator(state: LifecycleState, deps:
254
310
  });
255
311
  },
256
312
  });
313
+ if (!stillOwns()) {
314
+ // F11: ownership lost while suspended — dispose what was just created
315
+ // instead of publishing (no orphans).
316
+ deliveryCoordinator.dispose();
317
+ overflowTracker.dispose();
318
+ return;
319
+ }
320
+ state.deliveryCoordinator = deliveryCoordinator;
321
+ state.overflowTracker = overflowTracker;
257
322
  }
258
323
 
259
324
  /** Dispose delivery coordinator + overflow tracker. */