pi-crew 0.11.0 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/CHANGELOG.md +155 -9
  2. package/README.md +161 -1037
  3. package/agents/verifier.md +18 -7
  4. package/dist/index.mjs +744 -90644
  5. package/docs/README.md +57 -46
  6. package/docs/architecture.md +87 -33
  7. package/docs/commands-reference.md +9 -5
  8. package/docs/troubleshooting.md +3 -2
  9. package/package.json +1 -3
  10. package/schema.json +39 -0
  11. package/skills/real-test-pi-crew/REPORT-TEMPLATE.md +2 -0
  12. package/skills/real-test-pi-crew/SKILL.md +371 -34
  13. package/src/agents/agent-config.ts +1 -1
  14. package/src/agents/discover-agents.ts +1 -1
  15. package/src/config/config-validation.ts +15 -2
  16. package/src/config/config.ts +47 -13
  17. package/src/config/defaults.ts +0 -1
  18. package/src/config/env-vars.ts +35 -0
  19. package/src/config/types.ts +19 -5
  20. package/src/errors.ts +2 -2
  21. package/src/extension/async-notifier.ts +23 -0
  22. package/src/extension/crew-vibes/config.ts +0 -21
  23. package/src/extension/crew-vibes/index.ts +0 -2
  24. package/src/extension/crew-vibes/render.ts +1 -50
  25. package/src/extension/help.ts +21 -12
  26. package/src/extension/knowledge-injection.ts +2 -1
  27. package/src/extension/management.ts +8 -3
  28. package/src/extension/notification-sink.ts +17 -0
  29. package/src/extension/register.ts +7 -2
  30. package/src/extension/registration/command-utils.ts +28 -2
  31. package/src/extension/registration/commands/dashboard.ts +11 -1
  32. package/src/extension/registration/commands/manage.ts +36 -19
  33. package/src/extension/registration/commands/run.ts +24 -2
  34. package/src/extension/registration/commands/shared.ts +23 -1
  35. package/src/extension/registration/commands/status.ts +25 -2
  36. package/src/extension/registration/context-builder.ts +8 -2
  37. package/src/extension/registration/health-notify-policy.ts +100 -0
  38. package/src/extension/registration/lazy-configurers.ts +35 -0
  39. package/src/extension/registration/lifecycle-handlers.ts +91 -30
  40. package/src/extension/registration/lifecycle.ts +75 -10
  41. package/src/extension/registration/observability.ts +98 -35
  42. package/src/extension/registration/registration-types.ts +7 -5
  43. package/src/extension/registration/runtime-cleanup.ts +9 -3
  44. package/src/extension/registration/subagent-helpers.ts +38 -0
  45. package/src/extension/registration/subagent-tools.ts +16 -6
  46. package/src/extension/registration/team-tool.ts +10 -3
  47. package/src/extension/registration/terminal-status-wiring.ts +172 -0
  48. package/src/extension/registration/viewers.ts +6 -0
  49. package/src/extension/registration/wire-cross-extension.ts +28 -0
  50. package/src/extension/run-compare.ts +220 -0
  51. package/src/extension/run-export.ts +37 -5
  52. package/src/extension/run-maintenance.ts +155 -5
  53. package/src/extension/team-tool/dispatch/index.ts +3 -2
  54. package/src/extension/team-tool/dispatch/manage.ts +5 -2
  55. package/src/extension/team-tool/goal.ts +4 -1
  56. package/src/extension/team-tool/handle-settings.ts +33 -4
  57. package/src/extension/team-tool/health-monitor.ts +21 -7
  58. package/src/extension/team-tool/lifecycle-actions.ts +49 -1
  59. package/src/extension/team-tool/plan.ts +10 -0
  60. package/src/extension/team-tool/routing-hint.ts +63 -0
  61. package/src/extension/team-tool/status.ts +4 -0
  62. package/src/extension/team-tool.ts +52 -6
  63. package/src/extension/webhook-notify.ts +382 -0
  64. package/src/observability/metric-sink.ts +12 -2
  65. package/src/prompt/prompt-runtime.ts +82 -31
  66. package/src/prompt/worker-events-channel.ts +12 -0
  67. package/src/runtime/README.md +1 -1
  68. package/src/runtime/async-runner.ts +87 -1
  69. package/src/runtime/background-runner.ts +313 -234
  70. package/src/runtime/broker/crew-broker.ts +17 -11
  71. package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
  72. package/src/runtime/broker/wait-status-cache.ts +1 -1
  73. package/src/runtime/child-pi/child-pi-timers.ts +1 -1
  74. package/src/runtime/child-pi/mock-fixtures.ts +48 -0
  75. package/src/runtime/crew-agent-records.ts +337 -45
  76. package/src/runtime/deadletter.ts +43 -1
  77. package/src/runtime/delegate-spawn.ts +5 -1
  78. package/src/runtime/dispatch-batch.ts +72 -5
  79. package/src/runtime/goal-workflow/goal-loop-runner.ts +73 -4
  80. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  81. package/src/runtime/model/model-fallback.ts +21 -1
  82. package/src/runtime/model/pi-args.ts +8 -10
  83. package/src/runtime/recovery/crash-recovery.ts +25 -1
  84. package/src/runtime/run-worker.ts +12 -1
  85. package/src/runtime/scheduling/global-worker-cap.ts +13 -6
  86. package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
  87. package/src/runtime/scheduling/scheduler.ts +49 -13
  88. package/src/runtime/scheduling/semaphore.ts +148 -20
  89. package/src/runtime/scratchpad/README.md +1 -1
  90. package/src/runtime/scratchpad/protocol.ts +1 -1
  91. package/src/runtime/settings-store.ts +1 -1
  92. package/src/runtime/skill-instructions.ts +22 -0
  93. package/src/runtime/stale-reconciler.ts +85 -13
  94. package/src/runtime/task-display.ts +1 -1
  95. package/src/runtime/task-runner/pre-execution.ts +26 -2
  96. package/src/runtime/task-runner/prompt-builder.ts +142 -45
  97. package/src/runtime/task-runner.ts +21 -1
  98. package/src/runtime/team-runner.ts +38 -1
  99. package/src/runtime/workspace-lock.ts +4 -1
  100. package/src/schema/config-schema.ts +18 -0
  101. package/src/schema/team-tool-schema.ts +17 -0
  102. package/src/state/atomic-write.ts +53 -0
  103. package/src/state/contracts.ts +109 -0
  104. package/src/state/coordination/locks.ts +191 -33
  105. package/src/state/coordination/mailbox.ts +140 -15
  106. package/src/state/crew-init.ts +87 -12
  107. package/src/state/event-log/cursor.ts +37 -1
  108. package/src/state/event-log/event-log-rotation.ts +72 -7
  109. package/src/state/stores/active-run-registry.ts +13 -1
  110. package/src/state/stores/state-store.ts +112 -22
  111. package/src/state/types.ts +4 -0
  112. package/src/ui/adaptive-card.ts +65 -0
  113. package/src/ui/agents-jobs-browser.ts +70 -64
  114. package/src/ui/card-colors.ts +36 -7
  115. package/src/ui/dashboard-panes/agents-pane.ts +55 -14
  116. package/src/ui/dashboard-panes/cancellation-pane.ts +0 -42
  117. package/src/ui/dashboard-panes/health-pane.ts +7 -5
  118. package/src/ui/dashboard-panes/mailbox-pane.ts +22 -6
  119. package/src/ui/dashboard-panes/metrics-pane.ts +15 -7
  120. package/src/ui/dashboard-panes/pane-theme.ts +21 -0
  121. package/src/ui/dashboard-panes/plan-pane.ts +63 -30
  122. package/src/ui/dashboard-panes/progress-pane.ts +3 -2
  123. package/src/ui/dashboard-panes/schedules-pane.ts +44 -21
  124. package/src/ui/dashboard-panes/transcript-pane.ts +11 -5
  125. package/src/ui/dwf-phase-display.ts +3 -20
  126. package/src/ui/format-helpers.ts +22 -0
  127. package/src/ui/heartbeat-aggregator.ts +34 -0
  128. package/src/ui/inline-panel/crew-editor.ts +13 -3
  129. package/src/ui/inline-panel/index.ts +60 -4
  130. package/src/ui/keybinding-map.ts +251 -35
  131. package/src/ui/live-conversation-overlay.ts +180 -47
  132. package/src/ui/live-run-sidebar.ts +134 -55
  133. package/src/ui/mascot.ts +32 -16
  134. package/src/ui/overlays/agent-picker-overlay.ts +81 -26
  135. package/src/ui/overlays/confirm-overlay.ts +55 -29
  136. package/src/ui/overlays/help-overlay.ts +108 -53
  137. package/src/ui/overlays/mailbox-compose-overlay.ts +89 -50
  138. package/src/ui/overlays/mailbox-detail-overlay.ts +137 -57
  139. package/src/ui/powerbar-publisher.ts +0 -1
  140. package/src/ui/rail.ts +333 -0
  141. package/src/ui/run-dashboard.ts +193 -79
  142. package/src/ui/run-snapshot-cache.ts +18 -1
  143. package/src/ui/settings-overlay.ts +81 -39
  144. package/src/ui/spinner.ts +26 -2
  145. package/src/ui/terminal-status.ts +7 -1
  146. package/src/ui/theme-adapter.ts +0 -45
  147. package/src/ui/theme-discovery.ts +12 -6
  148. package/src/ui/tool-progress-formatter.ts +128 -9
  149. package/src/ui/tool-renderers/brief-mode.ts +10 -67
  150. package/src/ui/tool-renderers/index.ts +374 -523
  151. package/src/ui/transcript-viewer.ts +30 -12
  152. package/src/ui/widget/index.ts +32 -52
  153. package/src/ui/widget/task-list.ts +64 -32
  154. package/src/ui/widget/widget-formatters.ts +3 -402
  155. package/src/ui/widget/widget-model.ts +28 -7
  156. package/src/ui/widget/widget-renderer.ts +201 -128
  157. package/src/ui/widget/widget-types.ts +0 -2
  158. package/src/utils/incremental-reader.ts +11 -3
  159. package/src/utils/paths.ts +94 -12
  160. package/src/utils/project-markers.ts +40 -0
  161. package/src/utils/visual.ts +0 -4
  162. package/src/worktree/worktree-manager.ts +206 -26
  163. package/workflows/distill.workflow.md +3 -3
  164. package/workflows/fast-fix.workflow.md +1 -1
  165. package/workflows/plan-execute.workflow.md +1 -1
  166. package/workflows/review.workflow.md +1 -1
  167. package/workflows/strict-fast-fix.workflow.md +1 -1
  168. package/docs/migration-v0.4-v0.5.md +0 -208
  169. package/docs/runtime-flow.md +0 -148
  170. package/src/extension/crew-vibes/figures.ts +0 -22
  171. package/src/extension/crew-vibes/font-detect.ts +0 -71
  172. package/src/ui/dynamic-border.ts +0 -35
  173. package/src/ui/loaders.ts +0 -6
  174. package/src/ui/overlay-stack.ts +0 -148
@@ -20,7 +20,12 @@ import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-a
20
20
  import { loadConfig } from "../../config/config.ts";
21
21
  import { DEFAULT_UI } from "../../config/defaults.ts";
22
22
  import { getCrewEnv } from "../../config/env-vars.ts";
23
- import { pruneFinishedRuns, pruneUserLevelRuns } from "../../extension/run-maintenance.ts";
23
+ import {
24
+ pruneFinishedRuns,
25
+ pruneUserLevelRuns,
26
+ resolveAutoPruneAgeFloorMs,
27
+ resolveAutoPruneKeep,
28
+ } from "../../extension/run-maintenance.ts";
24
29
  import { type BrokerSpawnCredentials, setActiveBrokerIssuer, setActiveBrokerRevoker } from "../../runtime/broker/broker-issuer.ts";
25
30
  import { CrewBroker } from "../../runtime/broker/crew-broker.ts";
26
31
  import { terminateActiveChildPiProcesses } from "../../runtime/child-pi/child-pi.ts";
@@ -37,9 +42,10 @@ import { CrewScheduler, type ScheduledJob } from "../../runtime/scheduling/sched
37
42
  import { tryRegisterSessionCleanup } from "../../runtime/session-resources.ts";
38
43
  import { createSessionSnapshot } from "../../runtime/session-snapshot.ts";
39
44
  import { applyCrewSettingsTiersToConfig, loadCrewSettingsTiers, scheduledJobsHiddenCountOf } from "../../runtime/settings-store.ts";
45
+ import { loadTasksWithRecovery } from "../../state/stores/manifest-io.ts";
40
46
  import { loadRunManifestById } from "../../state/stores/state-store.ts";
41
47
  import type { TeamRunManifest } from "../../state/types.ts";
42
- import { summarizeHeartbeats } from "../../ui/heartbeat-aggregator.ts";
48
+ import { overlayFreshTaskStatuses, summarizeHeartbeats } from "../../ui/heartbeat-aggregator.ts";
43
49
  import { installInlinePanel } from "../../ui/inline-panel/index.ts";
44
50
  import { clearSessionSwitchInFlight, markSessionSwitchInFlight } from "../../ui/inline-panel/view-session-store.ts";
45
51
  import { requestRender, setExtensionWidget, toPiWidgetPlacement } from "../../ui/pi-ui-compat.ts";
@@ -64,8 +70,11 @@ import { notifyActiveRuns } from "../session-summary.ts";
64
70
  import { persistScheduledJobUpdate, registerCrewScheduler, stashScheduledJobsHiddenCount } from "../team-tool/handle-schedule.ts";
65
71
  import { handleTeamTool } from "../team-tool.ts";
66
72
  import { runArtifactCleanup } from "./artifact-cleanup.ts";
73
+ import { healthNotifyFingerprint, recordHealthNotifyDecision, resetHealthNotifyEntry } from "./health-notify-policy.ts";
67
74
  import type { RegistrationContext } from "./registration-types.ts";
68
75
  import { createScheduleEventNotifier } from "./schedule-toast-bridge.ts";
76
+ import { purgeQueuedAmbientNotifications } from "./subagent-helpers.ts";
77
+ import { refreshCrossExtensionWiringForSession } from "./wire-cross-extension.ts";
69
78
 
70
79
  /**
71
80
  * Register all session-lifecycle handlers on the ExtensionAPI. The caller
@@ -251,6 +260,10 @@ function installSessionStartHandler(pi: ExtensionAPI, ctx: RegistrationContext):
251
260
  // it can issue tokens for child runs in this session. The controller
252
261
  // already gates by flag + root-session; this is a no-op when disabled.
253
262
  ctx.brokerController?.setSessionId(currentSessionId);
263
+ // F13 (RR-018): RPC wiring is extension-lifetime and survives session
264
+ // switches; rebind the crew global registry to THIS session's manifest
265
+ // cache (idempotent — see wire-cross-extension.ts).
266
+ refreshCrossExtensionWiringForSession(pi, ctx);
254
267
 
255
268
  // Defer ALL heavy cleanup to after the session_start handler returns.
256
269
  // These operations involve synchronous directory scanning (readdirSync, readFileSync)
@@ -460,7 +473,12 @@ async function runDeferredSessionCleanup(
460
473
 
461
474
  // Auto-prune finished project-level run directories
462
475
  try {
463
- const { removed } = pruneFinishedRuns(extensionCtx.cwd, 10);
476
+ // DP-01: keep + age-floor are env-driven; the age floor means a run
477
+ // finished <24h ago survives a session restart even beyond top-keep.
478
+ const { removed } = pruneFinishedRuns(extensionCtx.cwd, resolveAutoPruneKeep(), {
479
+ ageFloorMs: resolveAutoPruneAgeFloorMs(),
480
+ intent: "session-start-auto",
481
+ });
464
482
  if (removed.length > 0) {
465
483
  ctx.notifyOperator({
466
484
  id: `auto_prune_project`,
@@ -476,7 +494,10 @@ async function runDeferredSessionCleanup(
476
494
 
477
495
  // Auto-prune finished user-level run directories
478
496
  try {
479
- const { removed } = pruneUserLevelRuns(10);
497
+ const { removed } = pruneUserLevelRuns(resolveAutoPruneKeep(), {
498
+ ageFloorMs: resolveAutoPruneAgeFloorMs(),
499
+ intent: "session-start-auto",
500
+ });
480
501
  if (removed.length > 0) {
481
502
  ctx.notifyOperator({
482
503
  id: `auto_prune_user`,
@@ -648,6 +669,18 @@ export function evictRunFromManifests(manifests: TeamRunManifest[], runId: strin
648
669
  return manifests.filter((m) => m.runId !== runId);
649
670
  }
650
671
 
672
+ /**
673
+ * F14 (RR-019): cheap signature of the preloaded-manifest frame. Only a
674
+ * CHANGED signature lets `backgroundPreload` count as render activity
675
+ * (`schedule()`); an identical signature means "nothing the widget would
676
+ * paint differently" and must NOT reset the render scheduler's idle
677
+ * counters (that defeated the R1 idle stop — see setupRenderLoop). Covers
678
+ * run identity, status transitions, and updatedAt bumps. Exported for tests.
679
+ */
680
+ export function manifestsFrameSignature(manifests: TeamRunManifest[]): string {
681
+ return manifests.map((m) => `${m.runId}:${m.status}:${m.updatedAt}`).join("|");
682
+ }
683
+
651
684
  /**
652
685
  * Apply a runEventBus payload to a preloaded-manifest frame: evict the run on
653
686
  * terminal events, pass through unchanged otherwise. This is the exact logic
@@ -677,16 +710,24 @@ function setupRenderLoop(
677
710
  let lastPreloadedManifests: TeamRunManifest[] = [];
678
711
  let lastFrameManifestCache: ReturnType<typeof createManifestCache> | undefined;
679
712
  let lastFrameSnapshotCache: ReturnType<typeof createRunSnapshotCache> | undefined;
713
+ // F14: signature of the last frame backgroundPreload saw — only a change
714
+ // here counts as render activity.
715
+ let lastFrameSignature: string | undefined;
680
716
 
681
717
  const ownerGeneration = ctx.sessionGeneration;
682
718
 
683
- const buildFrame = async (): Promise<boolean> => {
684
- if (!ctx.currentCtx) return false;
719
+ const buildFrame = async (): Promise<{ ok: boolean; changed: boolean }> => {
720
+ if (!ctx.currentCtx) return { ok: false, changed: false };
685
721
  lastPreloadedConfig = loadConfig(ctx.currentCtx.cwd);
686
722
  lastFrameManifestCache = ctx.getManifestCache(ctx.currentCtx.cwd);
687
723
  lastFrameSnapshotCache = ctx.getRunSnapshotCache(ctx.currentCtx.cwd);
688
724
  const manifests = lastFrameManifestCache.list(20);
689
725
  lastPreloadedManifests = manifests;
726
+ // F14 (RR-019): compute the frame signature BEFORE the preload await so
727
+ // `changed` reflects what this tick saw, not a mid-preload mutation.
728
+ const frameSignature = manifestsFrameSignature(manifests);
729
+ const frameChanged = frameSignature !== lastFrameSignature;
730
+ lastFrameSignature = frameSignature;
690
731
  // pts/2 hang fix: reconcile per-run watchers against the ACTIVE set only.
691
732
  {
692
733
  const onRunChange = (runId: string): void => {
@@ -718,16 +759,23 @@ function setupRenderLoop(
718
759
  }
719
760
  const runIds = manifests.map((r) => r.runId);
720
761
  await lastFrameSnapshotCache.preloadAllStale(runIds);
721
- return true;
762
+ return { ok: true, changed: frameChanged };
722
763
  };
723
764
 
724
765
  const backgroundPreload = (): void => {
725
766
  if (!ctx.currentCtx || preloading) return;
726
767
  preloading = true;
727
768
  buildFrame()
728
- .then((ok) => {
769
+ .then(({ ok, changed }) => {
729
770
  preloading = false;
730
- if (ok) ctx.renderScheduler?.schedule();
771
+ // F14 (RR-019): preload completion is maintenance, NOT user activity.
772
+ // Only a genuinely CHANGED frame may call schedule() — an unchanged
773
+ // frame calling schedule() reset `lastEventAt`/`idleFallbackRenders`
774
+ // and re-armed the fallback loop that fallbackLoop() had just
775
+ // deliberately stopped (R1), rendering forever while idle. Real
776
+ // work still animates: runEventBus + fs.watch subscriptions call
777
+ // schedule() directly on genuine events.
778
+ if (ok && changed) ctx.renderScheduler?.schedule();
731
779
  })
732
780
  .catch((error: unknown) => {
733
781
  preloading = false;
@@ -796,7 +844,16 @@ function setupRenderLoop(
796
844
  const clearHealthNotifications = (runId: string): void => {
797
845
  for (const kind of ["recovery_dead_workers", "recovery_missing_heartbeat"]) {
798
846
  const key = `${kind}_${runId}`;
799
- ctx.autoRecoveryLast.delete(key);
847
+ // FINDING 6: reset the fire budget (a genuine recurrence after
848
+ // this clear re-notifies) AND opportunistically purge
849
+ // still-QUEUED host follow-up copies of the original warning —
850
+ // the host drains the queue one message per turn boundary, so
851
+ // stale copies otherwise drip in for hours after the clear
852
+ // (live: 5h of "missing heartbeat" replays after the 2026-09-23
853
+ // zombie cleanup). The purge is feature-detected: hosts without
854
+ // clearQueuedUserMessagesMatching (e.g. 0.87.0) no-op here; the
855
+ // fire-cap policy bounds the backlog on every host.
856
+ resetHealthNotifyEntry({ entries: ctx.autoRecoveryLast, maxEntries: ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES }, key);
800
857
  ctx.notifyOperator({
801
858
  id: key,
802
859
  clear: true,
@@ -805,6 +862,10 @@ function setupRenderLoop(
805
862
  runId,
806
863
  title: `Cleared ${kind} for ${runId}`,
807
864
  });
865
+ purgeQueuedAmbientNotifications(
866
+ pi,
867
+ (text) => text.includes(runId) && (text.includes("dead worker") || text.includes("missing heartbeat")),
868
+ );
808
869
  }
809
870
  };
810
871
  for (const run of sessionManifests) {
@@ -838,29 +899,29 @@ function setupRenderLoop(
838
899
  clearHealthNotifications(run.runId);
839
900
  continue;
840
901
  }
841
- const summary = summarizeHeartbeats(snapshot, { now });
902
+ // GATE 3 (FINDING 5) — task-status truth lives on DISK, not in the
903
+ // snapshot cache. A worker parked on `ask` writes running → waiting
904
+ // to tasks.json immediately, but the cached snapshot can lag that
905
+ // transition; in the window a parked (alive, silent-by-design)
906
+ // worker counted as active-without-heartbeat and fired a false
907
+ // "dead worker" (live: team_20260923100114, 01_explore parked on
908
+ // ask). Overlay FRESH statuses before summarizing; divergence also
909
+ // invalidates the stale cache entry so the next tick rebuilds it.
910
+ const freshTasks = loadTasksWithRecovery(freshManifest.tasksPath, freshManifest.eventsPath, run.runId);
911
+ const overlaid = overlayFreshTaskStatuses(snapshot, freshTasks);
912
+ if (overlaid !== snapshot) snapshotCache.invalidate(run.runId);
913
+ const summary = summarizeHeartbeats(overlaid, { now });
914
+ const fingerprint = healthNotifyFingerprint(summary, overlaid.tasks.length);
842
915
  const maybeNotifyHealth = (kind: string, count: number, title: string, body: string): void => {
843
916
  if (count <= 0) return;
844
917
  const key = `${kind}_${run.runId}`;
845
- const previous = ctx.autoRecoveryLast.get(key);
846
- if (previous !== undefined && now - previous.lastAccessAt < 5 * 60_000) return;
847
- // Defensive cap: evict entry with oldest lastAccessAt before inserting/updating.
848
- while (ctx.autoRecoveryLast.size >= ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES) {
849
- let oldestKey: string | undefined;
850
- let oldestAccess = Infinity;
851
- for (const [k, v] of ctx.autoRecoveryLast) {
852
- if (v.lastAccessAt < oldestAccess) {
853
- oldestAccess = v.lastAccessAt;
854
- oldestKey = k;
855
- }
856
- }
857
- if (oldestKey === undefined) break;
858
- ctx.autoRecoveryLast.delete(oldestKey);
859
- }
860
- ctx.autoRecoveryLast.set(key, {
861
- insertedAt: now,
862
- lastAccessAt: now,
863
- });
918
+ // FINDING 6: bounded re-fire — ≤ MAX_HEALTH_NOTIFY_FIRES per
919
+ // UNCHANGED fingerprint (5-min cooldown between fires). The
920
+ // old forever-re-arming cooldown fed the host follow-up queue
921
+ // duplicates that drained one-per-turn-boundary for hours.
922
+ // Policy detail + LRU eviction: health-notify-policy.ts.
923
+ const state = { entries: ctx.autoRecoveryLast, maxEntries: ctx.AUTO_RECOVERY_LAST_MAX_ENTRIES };
924
+ if (!recordHealthNotifyDecision(state, key, fingerprint, now)) return;
864
925
  ctx.notifyOperator({
865
926
  id: key,
866
927
  severity: "warning",
@@ -22,6 +22,7 @@ import { projectCrewRoot } from "../../utils/paths.ts";
22
22
  import type { AsyncNotifierState } from "../async-notifier.ts";
23
23
  import type { NotificationDescriptor, NotificationRouter } from "../notification-router.ts";
24
24
  import type { NotificationSink } from "../notification-sink.ts";
25
+ import { createWebhookNotifier } from "../webhook-notify.ts";
25
26
 
26
27
  /**
27
28
  * Mutable state owned by register.ts and read/written by this module.
@@ -59,6 +60,30 @@ export function startLifecycleWatchers(ctx: ExtensionContext, state: LifecycleSt
59
60
  const loadedConfig = loadConfig(ctx.cwd);
60
61
  state.notifierStarted = true;
61
62
  try {
63
+ // US-030 (docs/specs/US-030.md): outbound webhook sink built from config.
64
+ // Gated on the notifications master switch (`notifications.enabled !== false`,
65
+ // same gate as configureNotifications), opt-in per `webhook.url`, and
66
+ // SSRF-guarded inside the factory. Inactive config → shared no-op
67
+ // singleton (zero network calls, zero hot-path allocation).
68
+ const notificationsConfig = loadedConfig.config.notifications;
69
+ const webhookNotifier =
70
+ notificationsConfig?.enabled === false
71
+ ? undefined
72
+ : createWebhookNotifier(
73
+ { webhook: notificationsConfig?.webhook, quietHours: notificationsConfig?.quietHours },
74
+ {
75
+ onFailure: (failure) => {
76
+ // Spec: failures surface as a `webhook.failed` event (crew.*-prefixed
77
+ // pi event, mirroring crew.run.completed / crew.task.overflow).
78
+ // logInternalError already fired inside the notifier.
79
+ deps.pi.events?.emit?.("crew.webhook.failed", {
80
+ runId: failure.runId,
81
+ attempts: failure.attempts,
82
+ error: failure.error,
83
+ });
84
+ },
85
+ },
86
+ );
62
87
  // LAZY: async-notifier pulls in debounce + cron helpers — defer
63
88
  // until the first lifecycle install (deferred import within module).
64
89
  void import("../async-notifier.ts").then(({ startAsyncRunNotifier }) => {
@@ -66,6 +91,7 @@ export function startLifecycleWatchers(ctx: ExtensionContext, state: LifecycleSt
66
91
  startAsyncRunNotifier(ctx, deps.notifierState, loadedConfig.config.notifierIntervalMs ?? DEFAULT_UI.notifierIntervalMs, {
67
92
  generation: deps.ownerGeneration,
68
93
  isCurrent: (generation) => generation === deps.ownerGeneration && deps.isContextCurrent(ctx, deps.ownerGeneration),
94
+ webhookNotifier,
69
95
  });
70
96
  });
71
97
  return true;
@@ -107,6 +133,13 @@ export interface NotificationsDeps {
107
133
  ctx: ExtensionContext,
108
134
  notificationCount: number,
109
135
  ) => void;
136
+ /**
137
+ * F11 (RR-018): ownership predicate — true only while the session that
138
+ * started this configure is still current. Checked after every lazy-import
139
+ * await boundary before publishing into shared state. Optional for older
140
+ * hand-built deps; defaults to "still owner".
141
+ */
142
+ isOwnerSession?: () => boolean;
110
143
  }
111
144
 
112
145
  /**
@@ -117,6 +150,11 @@ export interface NotificationsDeps {
117
150
  * - `config.notifications?.enabled === false` → no-op.
118
151
  */
119
152
  export async function configureNotifications(ctx: ExtensionContext, state: LifecycleState, deps: NotificationsDeps): Promise<void> {
153
+ // F11 (RR-018): ownership predicate — re-verified after every await below.
154
+ // A continuation whose session was cleaned up or superseded must not
155
+ // publish resources into shared state. Optional so hand-built deps from
156
+ // older call sites keep working (defaults to "still owner").
157
+ const stillOwns = deps.isOwnerSession ?? (() => true);
120
158
  state.notificationRouter?.dispose();
121
159
  state.notificationSink?.dispose();
122
160
  state.notificationRouter = undefined;
@@ -126,26 +164,29 @@ export async function configureNotifications(ctx: ExtensionContext, state: Lifec
126
164
 
127
165
  // LAZY: notification router — wires events to sinks
128
166
  const { NotificationRouter } = await import("../notification-router.ts");
167
+ if (!stillOwns()) return;
129
168
  // LAZY: JSONL sink — file-backed notification storage
130
169
  const { createJsonlSink } = await import("../notification-sink.ts");
170
+ if (!stillOwns()) return;
131
171
  // LAZY: follow-up helpers — shared with subagent handoff
132
172
  const { sendFollowUp } = await import("./subagent-helpers.ts");
173
+ if (!stillOwns()) return;
133
174
  // LAZY: widget updater — refreshes the TUI status widget
134
175
  const { updateCrewWidget } = await import("../../ui/widget/index.ts");
176
+ if (!stillOwns()) return;
135
177
 
136
- if (config.telemetry?.enabled !== false) {
137
- state.notificationSink = createJsonlSink(
138
- projectCrewRoot(ctx.cwd),
139
- config.notifications?.sinkRetentionDays ?? DEFAULT_NOTIFICATIONS.sinkRetentionDays,
140
- );
141
- }
142
- state.notificationRouter = new NotificationRouter(
178
+ // F11: create into locals — publish only while ownership still holds.
179
+ const notificationSink =
180
+ config.telemetry?.enabled !== false
181
+ ? createJsonlSink(projectCrewRoot(ctx.cwd), config.notifications?.sinkRetentionDays ?? DEFAULT_NOTIFICATIONS.sinkRetentionDays)
182
+ : undefined;
183
+ const notificationRouter = new NotificationRouter(
143
184
  {
144
185
  dedupWindowMs: config.notifications?.dedupWindowMs ?? DEFAULT_NOTIFICATIONS.dedupWindowMs,
145
186
  batchWindowMs: config.notifications?.batchWindowMs ?? DEFAULT_NOTIFICATIONS.batchWindowMs,
146
187
  quietHours: config.notifications?.quietHours,
147
188
  severityFilter: config.notifications?.severityFilter ?? [...DEFAULT_NOTIFICATIONS.severityFilter],
148
- sink: (notification) => state.notificationSink?.write(notification),
189
+ sink: (notification) => notificationSink?.write(notification),
149
190
  },
150
191
  (notification: NotificationDescriptor) => {
151
192
  if (notification.clear) {
@@ -183,6 +224,15 @@ export async function configureNotifications(ctx: ExtensionContext, state: Lifec
183
224
  }
184
225
  },
185
226
  );
227
+ if (!stillOwns()) {
228
+ // F11: ownership lost while suspended — dispose what was just created
229
+ // instead of publishing (no orphans).
230
+ notificationRouter.dispose();
231
+ notificationSink?.dispose();
232
+ return;
233
+ }
234
+ state.notificationSink = notificationSink;
235
+ state.notificationRouter = notificationRouter;
186
236
  }
187
237
 
188
238
  /** Dispose notification router + sink. */
@@ -200,6 +250,8 @@ export interface DeliveryDeps {
200
250
  notifyOperator: (notification: NotificationDescriptor) => void;
201
251
  sendFollowUp: (pi: ExtensionAPI, message: string) => void;
202
252
  sendAgentWakeUp: (pi: ExtensionAPI, message: string) => void;
253
+ /** F11 (RR-018): same ownership predicate as NotificationsDeps.isOwnerSession. */
254
+ isOwnerSession?: () => boolean;
203
255
  }
204
256
 
205
257
  /**
@@ -207,15 +259,19 @@ export interface DeliveryDeps {
207
259
  * register.ts as part of H3-L2 (delivery lifecycle service).
208
260
  */
209
261
  export async function configureDeliveryCoordinator(state: LifecycleState, deps: DeliveryDeps): Promise<void> {
262
+ // F11: ownership predicate — re-verified after the lazy imports below.
263
+ const stillOwns = deps.isOwnerSession ?? (() => true);
210
264
  state.deliveryCoordinator?.dispose();
211
265
  state.deliveryCoordinator = undefined;
212
266
  state.overflowTracker?.dispose();
213
267
  state.overflowTracker = undefined;
214
268
  // LAZY: delivery coordinator — batches notification fan-out
215
269
  const { DeliveryCoordinator } = await import("../../runtime/delivery-coordinator.ts");
270
+ if (!stillOwns()) return;
216
271
  // LAZY: overflow tracker — recovers from backlog overflow
217
272
  const { OverflowRecoveryTracker } = await import("../../runtime/recovery/overflow-recovery.ts");
218
- state.deliveryCoordinator = new DeliveryCoordinator({
273
+ if (!stillOwns()) return;
274
+ const deliveryCoordinator = new DeliveryCoordinator({
219
275
  emit: (event, data) => {
220
276
  deps.pi.events?.emit?.(event, data);
221
277
  },
@@ -226,7 +282,7 @@ export async function configureDeliveryCoordinator(state: LifecycleState, deps:
226
282
  deps.sendAgentWakeUp(deps.pi, message);
227
283
  },
228
284
  });
229
- state.overflowTracker = new OverflowRecoveryTracker({
285
+ const overflowTracker = new OverflowRecoveryTracker({
230
286
  onPhaseChange: (phaseState, previousPhase) => {
231
287
  if (deps.observabilityState.metricRegistry) {
232
288
  deps.observabilityState.metricRegistry
@@ -254,6 +310,15 @@ export async function configureDeliveryCoordinator(state: LifecycleState, deps:
254
310
  });
255
311
  },
256
312
  });
313
+ if (!stillOwns()) {
314
+ // F11: ownership lost while suspended — dispose what was just created
315
+ // instead of publishing (no orphans).
316
+ deliveryCoordinator.dispose();
317
+ overflowTracker.dispose();
318
+ return;
319
+ }
320
+ state.deliveryCoordinator = deliveryCoordinator;
321
+ state.overflowTracker = overflowTracker;
257
322
  }
258
323
 
259
324
  /** Dispose delivery coordinator + overflow tracker. */
@@ -49,6 +49,13 @@ export interface ObservabilityState {
49
49
  autoRepairTimer: ReturnType<typeof setInterval> | undefined;
50
50
  tempReconcileTimer: ReturnType<typeof setInterval> | undefined;
51
51
  otlpExporter: OTLPExporterInstance | undefined;
52
+ /**
53
+ * F11 (RR-018): promise of the most recent `configureObservability` run.
54
+ * `disposeObservability` settles it deliberately so teardown never races a
55
+ * still-suspended init continuation (which re-checks ownership after every
56
+ * await and self-disposes its locals when ownership was lost).
57
+ */
58
+ initPromise: Promise<void> | undefined;
52
59
  }
53
60
 
54
61
  /** Dependencies passed in by register.ts so this module stays decoupled. */
@@ -57,6 +64,13 @@ export interface ObservabilityDeps {
57
64
  getManifestCache: (cwd: string) => ReturnType<typeof import("../../runtime/manifest-cache.ts").createManifestCache>;
58
65
  notifyOperator: (notification: NotificationDescriptor) => void;
59
66
  isCleanedUp: () => boolean;
67
+ /**
68
+ * F11 (RR-018): read the RegistrationContext's `sessionGeneration`. The
69
+ * generation is bumped by cleanup AND by every session_start/before_switch,
70
+ * so unlike `isCleanedUp` it can never be reset back to "still owner" by a
71
+ * newer session. Reuse this counter — do not invent a parallel one.
72
+ */
73
+ getSessionGeneration: () => number;
60
74
  reconcileStaleRuns: (cwd: string, cache: ReturnType<ObservabilityDeps["getManifestCache"]>, currentSessionId?: string) => unknown[];
61
75
  reconcileOrphanedTempWorkspaces: (now: number, opts: { cleanupOrphanedTempDirs?: boolean }) => unknown;
62
76
  cleanupOrphanTempDirs: () => { cleaned: number; scanned: number; failed: number };
@@ -94,43 +108,77 @@ async function importOTLPExporter(): Promise<OTLPExporterCtor> {
94
108
  * - `config.reliability?.autoRecover === true` → lazy-imports crash-recovery
95
109
  * on a deferred setTimeout to avoid blocking session_start.
96
110
  */
97
- export async function configureObservability(ctx: ExtensionContext, state: ObservabilityState, deps: ObservabilityDeps): Promise<void> {
111
+ export function configureObservability(ctx: ExtensionContext, state: ObservabilityState, deps: ObservabilityDeps): Promise<void> {
112
+ // F11: track the init promise so `disposeObservability` can settle it
113
+ // deliberately instead of racing it. The internal run first awaits any
114
+ // PREVIOUS init (via disposeObservability), so this assignment can never
115
+ // self-deadlock — state.initPromise still referenced the old run when the
116
+ // internal run suspended.
117
+ const run = configureObservabilityInternal(ctx, state, deps);
118
+ state.initPromise = run;
119
+ return run;
120
+ }
121
+
122
+ async function configureObservabilityInternal(ctx: ExtensionContext, state: ObservabilityState, deps: ObservabilityDeps): Promise<void> {
123
+ // F11 (RR-018): capture the session generation at configure time. Every
124
+ // await boundary below re-verifies ownership before touching shared state;
125
+ // a continuation whose generation no longer matches can never publish —
126
+ // even when a newer session has already reset `cleanedUp` back to false
127
+ // (the real ordering: cleanup sets it true, session_start resets it false).
128
+ const ownerGeneration = deps.getSessionGeneration();
129
+ const stillOwns = (): boolean => deps.getSessionGeneration() === ownerGeneration && !deps.isCleanedUp();
130
+
98
131
  // Always start from a clean slate: dispose any prior-session state first.
99
132
  await disposeObservability(state, deps.isCleanedUp());
133
+ if (!stillOwns()) return;
100
134
 
101
135
  const config = loadConfig(ctx.cwd).config;
102
136
  if (config.observability?.enabled === false) return;
103
137
 
104
138
  // LAZY: observability stack — only paid for when observability is actually enabled
105
139
  const { createMetricRegistry } = await import("../../observability/metric-registry.ts");
140
+ if (!stillOwns()) return;
106
141
  // LAZY: event→metric bridge
107
142
  const { wireEventToMetrics } = await import("../../observability/event-to-metric.ts");
143
+ if (!stillOwns()) return;
108
144
  // LAZY: file-backed metric sink
109
145
  const { createMetricFileSink } = await import("../../observability/metric-sink.ts");
146
+ if (!stillOwns()) return;
110
147
 
111
- state.metricRegistry = createMetricRegistry();
112
- if (deps.pi.events) {
113
- state.eventMetricSub = wireEventToMetrics(deps.pi.events, state.metricRegistry);
114
- }
115
- if (config.telemetry?.enabled !== false) {
116
- state.metricSink = createMetricFileSink({
117
- crewRoot: projectCrewRoot(ctx.cwd),
118
- registry: state.metricRegistry,
119
- retentionDays: config.observability?.metricRetentionDays ?? 7,
120
- });
148
+ // F11: create into LOCALS — shared state is only touched after ownership is
149
+ // re-verified. If ownership was lost in the gap, everything just created is
150
+ // disposed right here in the continuation instead of leaking (0 orphans).
151
+ const metricRegistry = createMetricRegistry();
152
+ const eventMetricSub = deps.pi.events ? wireEventToMetrics(deps.pi.events, metricRegistry) : undefined;
153
+ const metricSink =
154
+ config.telemetry?.enabled !== false
155
+ ? createMetricFileSink({
156
+ crewRoot: projectCrewRoot(ctx.cwd),
157
+ registry: metricRegistry,
158
+ retentionDays: config.observability?.metricRetentionDays ?? 7,
159
+ })
160
+ : undefined;
161
+ if (!stillOwns()) {
162
+ eventMetricSub?.dispose();
163
+ metricSink?.dispose();
164
+ metricRegistry.dispose();
165
+ return;
121
166
  }
167
+ state.metricRegistry = metricRegistry;
168
+ state.eventMetricSub = eventMetricSub;
169
+ state.metricSink = metricSink;
122
170
 
123
171
  // OTLP export is opt-in. Lazy-loaded via dynamic import.
124
172
  if (config.otlp?.enabled === true && config.otlp.endpoint) {
125
173
  const otlpEndpoint = config.otlp.endpoint;
126
174
  const otlpHeaders = config.otlp.headers;
127
175
  const otlpInterval = config.otlp.intervalMs;
128
- const owningRegistry = state.metricRegistry;
176
+ const owningRegistry = metricRegistry;
129
177
  // LAZY: opt-in OTLP export — load the exporter module on first enable.
130
178
  void importOTLPExporter()
131
179
  .then((Ctor) => {
132
- if (deps.isCleanedUp() || state.metricRegistry !== owningRegistry || !owningRegistry) return;
133
- state.otlpExporter = new Ctor(
180
+ if (!stillOwns() || state.metricRegistry !== owningRegistry || !owningRegistry) return;
181
+ const otlpExporter = new Ctor(
134
182
  {
135
183
  endpoint: otlpEndpoint,
136
184
  headers: otlpHeaders,
@@ -138,18 +186,20 @@ export async function configureObservability(ctx: ExtensionContext, state: Obser
138
186
  },
139
187
  owningRegistry,
140
188
  );
141
- state.otlpExporter?.start();
189
+ state.otlpExporter = otlpExporter;
190
+ otlpExporter.start();
142
191
  })
143
192
  .catch((error: unknown) => logInternalError("register.otlp-lazy-import", error));
144
193
  }
145
194
 
146
195
  // LAZY: heartbeat watcher — polled per-session, wires deadletter + metric events.
147
196
  const { HeartbeatWatcher } = await import("../../runtime/heartbeat/heartbeat-watcher.ts");
148
- state.heartbeatWatcher = new HeartbeatWatcher({
197
+ if (!stillOwns()) return;
198
+ const heartbeatWatcher = new HeartbeatWatcher({
149
199
  cwd: ctx.cwd,
150
200
  pollIntervalMs: config.observability?.pollIntervalMs ?? 5000,
151
201
  manifestCache: deps.getManifestCache(ctx.cwd),
152
- registry: state.metricRegistry,
202
+ registry: metricRegistry,
153
203
  router: {
154
204
  enqueue: (notification) => {
155
205
  deps.notifyOperator(notification);
@@ -173,25 +223,24 @@ export async function configureObservability(ctx: ExtensionContext, state: Obser
173
223
  });
174
224
  },
175
225
  });
176
- state.heartbeatWatcher.start();
226
+ // F11: the watcher is created into a local and only published once
227
+ // ownership is re-verified — a lost-ownership continuation disposes it
228
+ // instead of publishing a live poller nobody will tear down.
229
+ if (!stillOwns()) {
230
+ heartbeatWatcher.dispose();
231
+ return;
232
+ }
233
+ state.heartbeatWatcher = heartbeatWatcher;
234
+ heartbeatWatcher.start();
177
235
 
178
236
  // RT-F2: opportunistic stale-run reconcile hook.
179
- // The .unref()'d setInterval below never fires while the event loop is idle
180
- // (e.g. user has the editor open but hasn't sent a turn in an hour). Pair it
181
- // with a before_agent_start hook so reconcileAllStaleRuns also fires when
182
- // the user actually drives the session — bounded by user activity.
183
- try {
184
- deps.pi.on?.("before_agent_start", () => {
185
- if (deps.isCleanedUp()) return;
186
- try {
187
- deps.reconcileStaleRuns(ctx.cwd, deps.getManifestCache(ctx.cwd), extractSessionId(ctx));
188
- } catch (error) {
189
- logInternalError("register.autoRepair.turnHook", error);
190
- }
191
- });
192
- } catch {
193
- /* older Pi without before_agent_start — rely on the interval alone */
194
- }
237
+ // F12 (RR-018): the `before_agent_start` hook is NO LONGER registered here.
238
+ // It is registered exactly ONCE per extension in `lazy-configurers.ts`
239
+ // (installTurnReconcileHook) and resolves the CURRENT session context at
240
+ // fire time — per-session registration accumulated one dead hook per
241
+ // switch (there is no pi.off), and the stale hook re-activated with the
242
+ // old session's cwd, flipping the shared manifest cache back. The
243
+ // .unref()'d setInterval below remains the idle-time safety net.
195
244
 
196
245
  // Auto-repair timers: stale-run reconcile + orphan-temp cleanup.
197
246
  // RT-F2: default raised from 60_000ms to 5 minutes (300_000ms). The previous
@@ -273,7 +322,7 @@ export async function configureObservability(ctx: ExtensionContext, state: Obser
273
322
  void deps
274
323
  .importCrashRecovery()
275
324
  .then(({ detectInterruptedRuns }) => {
276
- if (deps.isCleanedUp()) return;
325
+ if (!stillOwns()) return;
277
326
  const sid = extractSessionId(ctx);
278
327
  for (const plan of detectInterruptedRuns(cwdSnapshot, cacheSnapshot, 300_000, sid)) {
279
328
  deps.notifyOperator({
@@ -296,6 +345,20 @@ export async function configureObservability(ctx: ExtensionContext, state: Obser
296
345
  * gated by the orchestrator's overall cleanup state.
297
346
  */
298
347
  export async function disposeObservability(state: ObservabilityState, _isCleanedUp: boolean): Promise<void> {
348
+ // F11 (RR-018): settle any in-flight init BEFORE disposing shared state, so
349
+ // teardown never races a still-suspended continuation. The continuation
350
+ // itself re-checks ownership after every await and self-disposes its
351
+ // locals, so this await only bounds the race window — it never starts new
352
+ // work and cannot deadlock (the previous init completes on its own).
353
+ const initPromise = state.initPromise;
354
+ state.initPromise = undefined;
355
+ if (initPromise) {
356
+ try {
357
+ await initPromise;
358
+ } catch {
359
+ /* configureObservabilityImpl logs its own errors */
360
+ }
361
+ }
299
362
  state.heartbeatWatcher?.dispose();
300
363
  state.heartbeatWatcher = undefined;
301
364
  if (state.autoRepairTimer) {
@@ -37,12 +37,14 @@ export type AbortControllerMap = Map<AbortKey, AbortController>;
37
37
  /**
38
38
  * Single auto-recovery cooldown gate entry. Tracked by the render-tick
39
39
  * health notifier so the same warning doesn't fire repeatedly within a
40
- * 5-minute cooldown. Uses LRU-like semantics — see register.ts.
40
+ * 5-minute cooldown, and (FINDING 6) at most MAX_HEALTH_NOTIFY_FIRES times
41
+ * per unchanged fingerprint. LRU-like semantics — see register.ts and
42
+ * health-notify-policy.ts (the authoritative definition; re-exported here
43
+ * so existing imports keep working).
41
44
  */
42
- export interface AutoRecoveryEntry {
43
- insertedAt: number;
44
- lastAccessAt: number;
45
- }
45
+ export type { AutoRecoveryEntry } from "./health-notify-policy.ts";
46
+
47
+ import type { AutoRecoveryEntry } from "./health-notify-policy.ts";
46
48
 
47
49
  /**
48
50
  * Mutable context bag passed from the orchestrator to every