pi-crew 0.11.22 → 0.11.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/agents/explorer.md +3 -3
  3. package/agents/librarian.md +3 -3
  4. package/dist/index.mjs +523 -520
  5. package/docs/dynamic-workflows.md +14 -2
  6. package/package.json +4 -4
  7. package/src/agents/discover-agents.ts +8 -2
  8. package/src/config/config-validation.ts +3 -1
  9. package/src/config/env-allowlist-registry.ts +5 -0
  10. package/src/config/env-vars.ts +9 -6
  11. package/src/config/role-tools.ts +7 -6
  12. package/src/extension/management.ts +7 -0
  13. package/src/extension/register.ts +15 -0
  14. package/src/extension/run-index.ts +24 -1
  15. package/src/extension/team-tool/destructive-gate.ts +208 -0
  16. package/src/extension/team-tool/goal-wrap.ts +11 -1
  17. package/src/extension/team-tool/goal.ts +13 -1
  18. package/src/extension/team-tool/respond.ts +8 -6
  19. package/src/extension/team-tool/run.ts +4 -4
  20. package/src/extension/team-tool.ts +10 -0
  21. package/src/observability/exporters/otlp-exporter.ts +11 -1
  22. package/src/prompt/prompt-runtime.ts +102 -18
  23. package/src/prompt/surface-worker.ts +13 -1
  24. package/src/runtime/broker/crew-broker-client.ts +7 -0
  25. package/src/runtime/broker/crew-broker.ts +113 -152
  26. package/src/runtime/broker/protocol/connection-state.ts +159 -1
  27. package/src/runtime/broker/protocol/events-replay.ts +50 -10
  28. package/src/runtime/broker/protocol/request-parsers.ts +68 -0
  29. package/src/runtime/child-pi/child-pi-constants.ts +9 -0
  30. package/src/runtime/child-pi/child-pi-rpc.ts +72 -7
  31. package/src/runtime/child-pi/child-pi-streams.ts +19 -3
  32. package/src/runtime/child-pi/postmortem-registry.ts +8 -2
  33. package/src/runtime/classifier/ambient-noise-gate.ts +5 -1
  34. package/src/runtime/compaction/tool-output-pruner.ts +20 -2
  35. package/src/runtime/crew-agent-records.ts +92 -17
  36. package/src/runtime/custom-tools/submit-result-tool.ts +17 -0
  37. package/src/runtime/deadletter.ts +5 -1
  38. package/src/runtime/dispatch-batch.ts +60 -16
  39. package/src/runtime/effectiveness.ts +24 -1
  40. package/src/runtime/finalize-run.ts +12 -1
  41. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +12 -3
  42. package/src/runtime/goal-workflow/goal-evaluator.ts +30 -5
  43. package/src/runtime/goal-workflow/goal-loop-runner.ts +60 -3
  44. package/src/runtime/live-session/live-agent-control.ts +34 -14
  45. package/src/runtime/live-session/live-agent-manager.ts +20 -5
  46. package/src/runtime/live-session/live-session-runtime.ts +66 -81
  47. package/src/runtime/live-session/usage-bridge.ts +66 -2
  48. package/src/runtime/mcp-proxy.ts +10 -0
  49. package/src/runtime/model/model-fallback.ts +68 -0
  50. package/src/runtime/model/runtime-policy.ts +20 -6
  51. package/src/runtime/model/runtime-resolver.ts +5 -3
  52. package/src/runtime/orphan-worker-registry.ts +64 -24
  53. package/src/runtime/pi-spawn.ts +4 -1
  54. package/src/runtime/process/zombie-scanner.ts +6 -2
  55. package/src/runtime/recovery/crash-recovery.ts +35 -2
  56. package/src/runtime/scheduling/batch-barrier.ts +42 -5
  57. package/src/runtime/scheduling/run-coalesced-task-group.ts +9 -3
  58. package/src/runtime/scheduling/scheduler.ts +74 -29
  59. package/src/runtime/stale-reconciler.ts +45 -6
  60. package/src/runtime/surface/herdr-provider.ts +54 -1
  61. package/src/runtime/surface/launch-script.ts +7 -1
  62. package/src/runtime/surface/resolve-surface.ts +4 -1
  63. package/src/runtime/surface/tmux-provider.ts +32 -2
  64. package/src/runtime/task-runner/child-executor.ts +3 -2
  65. package/src/runtime/workspace-lock.ts +52 -4
  66. package/src/state/contracts.ts +2 -0
  67. package/src/state/coordination/process-lock.ts +50 -0
  68. package/src/state/event-log/cursor.ts +24 -3
  69. package/src/state/event-log/event-log-rotation.ts +26 -2
  70. package/src/state/event-log/worker-atomic-writer.ts +5 -0
  71. package/src/state/stores/active-run-registry.ts +56 -5
  72. package/src/state/stores/ownership-map.ts +26 -7
  73. package/src/state/stores/plan-store.ts +30 -1
  74. package/src/state/stores/sqlite-run-state.ts +91 -1
  75. package/src/state/types.ts +5 -0
  76. package/src/ui/dashboard-panes/agents-pane.ts +4 -3
  77. package/src/ui/dashboard-panes/plan-pane.ts +5 -3
  78. package/src/ui/dashboard-panes/transcript-pane.ts +2 -1
  79. package/src/ui/overlays/mailbox-detail-overlay.ts +5 -3
  80. package/src/ui/run-dashboard.ts +6 -4
  81. package/src/ui/run-snapshot-cache.ts +3 -2
  82. package/src/ui/settings-overlay.ts +68 -10
  83. package/src/ui/theme-discovery.ts +26 -7
  84. package/src/ui/transcript-viewer.ts +5 -3
  85. package/src/utils/visual.ts +84 -0
@@ -174,31 +174,46 @@ function isValidId(id: string): boolean {
174
174
  return true;
175
175
  }
176
176
 
177
- function readRegistry(): OrphanWorkerEntry[] {
177
+ // FAMILY-2 (2026-10-17): strict read variant — ENOENT still maps to [] (a
178
+ // missing registry is a legitimate empty state), but any OTHER fault
179
+ // (EACCES/EBUSY/parse error/shape mismatch) THROWS. RMW callers
180
+ // (registerWorker/unregisterWorker) must skip their write on a strict-read
181
+ // fault: persisting the resulting [] would WIPE every sibling entry and
182
+ // orphan those workers until host exit.
183
+ function readRegistryStrict(): OrphanWorkerEntry[] {
178
184
  const p = getRegistryPath();
185
+ // Atomic read: if file doesn't exist, readFileSync throws ENOENT
186
+ // which we handle explicitly. This eliminates the TOCTOU window
187
+ // between existsSync and readFileSync.
188
+ let raw: string;
179
189
  try {
180
- // Atomic read: if file doesn't exist, readFileSync throws ENOENT
181
- // which we handle explicitly. This eliminates the TOCTOU window
182
- // between existsSync and readFileSync.
183
- const raw = fs.readFileSync(p, "utf-8");
184
- const parsed = JSON.parse(raw);
185
- if (!Array.isArray(parsed)) return [];
186
- return parsed.filter(
187
- (e): e is OrphanWorkerEntry =>
188
- typeof e === "object" &&
189
- e !== null &&
190
- typeof e.pid === "number" &&
191
- typeof e.sessionId === "string" &&
192
- typeof e.runId === "string" &&
193
- typeof e.registeredAt === "number" &&
194
- typeof (e as OrphanWorkerEntry).parentPid === "number" &&
195
- typeof (e as OrphanWorkerEntry).parentPidStartTime === "number" &&
196
- typeof (e as OrphanWorkerEntry).startTime === "number",
197
- );
190
+ raw = fs.readFileSync(p, "utf-8");
191
+ } catch (error) {
192
+ if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
193
+ throw error;
194
+ }
195
+ const parsed: unknown = JSON.parse(raw); // corrupt JSON throws (strict)
196
+ if (!Array.isArray(parsed)) {
197
+ throw new Error(`orphan-worker-registry: expected array, got ${typeof parsed}`);
198
+ }
199
+ return parsed.filter(
200
+ (e): e is OrphanWorkerEntry =>
201
+ typeof e === "object" &&
202
+ e !== null &&
203
+ typeof e.pid === "number" &&
204
+ typeof e.sessionId === "string" &&
205
+ typeof e.runId === "string" &&
206
+ typeof e.registeredAt === "number" &&
207
+ typeof (e as OrphanWorkerEntry).parentPid === "number" &&
208
+ typeof (e as OrphanWorkerEntry).parentPidStartTime === "number" &&
209
+ typeof (e as OrphanWorkerEntry).startTime === "number",
210
+ );
211
+ }
212
+
213
+ function readRegistry(): OrphanWorkerEntry[] {
214
+ try {
215
+ return readRegistryStrict();
198
216
  } catch (error) {
199
- if ((error as NodeJS.ErrnoException).code === "ENOENT") {
200
- return [];
201
- }
202
217
  // Silent failure is deliberate for robustness (registry read failures
203
218
  // shouldn't crash the process), but log at warning level to aid troubleshooting.
204
219
  logInternalError("orphan-worker-registry", new Error(`readRegistry failed: ${error}`), undefined, "warn");
@@ -278,7 +293,20 @@ export function registerWorker(
278
293
  const startTime = getProcessStartTime(pid) ?? 0;
279
294
  const parentPidStartTime = Number.isFinite(parentPid) && parentPid > 0 ? (getProcessStartTime(parentPid) ?? 0) : 0;
280
295
  withFileLockSync(getRegistryPath(), () => {
281
- const entries = readRegistry();
296
+ // FAMILY-2: strict read — a read fault must SKIP the write (persisting
297
+ // the [] rebuild would wipe every sibling entry), not mask as empty.
298
+ let entries: OrphanWorkerEntry[];
299
+ try {
300
+ entries = readRegistryStrict();
301
+ } catch (error) {
302
+ logInternalError(
303
+ "orphan-worker-registry.register",
304
+ error instanceof Error ? error : new Error(String(error)),
305
+ `pid=${pid} runId=${runId} — skipping write (registry read fault)`,
306
+ "warn",
307
+ );
308
+ return;
309
+ }
282
310
  // Dedupe by PID
283
311
  const filtered = entries.filter((e) => e.pid !== pid);
284
312
  filtered.push({
@@ -301,7 +329,19 @@ export function registerWorker(
301
329
  export function unregisterWorker(pid: number): void {
302
330
  if (!Number.isFinite(pid) || pid <= 0) return;
303
331
  withFileLockSync(getRegistryPath(), () => {
304
- const entries = readRegistry();
332
+ // FAMILY-2: strict read — skip the write on a read fault (see registerWorker).
333
+ let entries: OrphanWorkerEntry[];
334
+ try {
335
+ entries = readRegistryStrict();
336
+ } catch (error) {
337
+ logInternalError(
338
+ "orphan-worker-registry.unregister",
339
+ error instanceof Error ? error : new Error(String(error)),
340
+ `pid=${pid} — skipping write (registry read fault)`,
341
+ "warn",
342
+ );
343
+ return;
344
+ }
305
345
  const filtered = entries.filter((e) => e.pid !== pid);
306
346
  if (filtered.length !== entries.length) {
307
347
  writeRegistry(filtered);
@@ -96,7 +96,10 @@ function isWithinAllowedPrefixes(resolvedPath: string): boolean {
96
96
  }
97
97
  }
98
98
 
99
- return allowedPrefixes.some((prefix) => normalized.startsWith(prefix));
99
+ // R2-L-A9 (wave-2b): a bare startsWith accepts sibling-name escapes
100
+ // (prefix "/opt/pi" also matches "/opt/pwned"). The prefix must match the
101
+ // path itself or a path-separator-bounded descendant.
102
+ return allowedPrefixes.some((prefix) => normalized === prefix || normalized.startsWith(prefix + path.sep));
100
103
  }
101
104
 
102
105
  function resolvePiPackageRoot(): string | undefined {
@@ -179,8 +179,12 @@ function isPidAlive(pid: number): boolean {
179
179
  // process.kill(pid, 0) throws if the pid is not alive (or not ours).
180
180
  process.kill(pid, 0);
181
181
  return true;
182
- } catch {
183
- return false;
182
+ } catch (error) {
183
+ // R7-A2 (2026-10-17): only ESRCH/ENOENT mean the process is gone.
184
+ // EPERM means the process is alive but protected (different security
185
+ // context); treat as alive — mirror of active-run-registry.ts fix.
186
+ const code = (error as NodeJS.ErrnoException).code;
187
+ return code !== "ESRCH" && code !== "ENOENT";
184
188
  }
185
189
  }
186
190
 
@@ -700,6 +700,7 @@ export function purgeStaleActiveRunIndex(
700
700
  // is fire-and-forget so it stays OUTSIDE the lock to avoid blocking the
701
701
  // critical section on async IPC.
702
702
  let abortedFreshLiveness = false;
703
+ let abortedIntentionalWait = false; // R2-M-A3: keep intentional-wait runs
703
704
  try {
704
705
  const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
705
706
  if (fullLoaded) {
@@ -709,6 +710,14 @@ export function purgeStaleActiveRunIndex(
709
710
  const fresh = loadRunManifestById(entry.cwd, entry.runId);
710
711
  if (!fresh) return;
711
712
  if (fresh.manifest.status !== "running") return;
713
+ // R2-M-A3 (2026-10-17): mirror :135/:329/:937 — purgeStaleActiveRunIndex
714
+ // was the only recovery path without an isIntentionalWait guard: a run
715
+ // parked on a plan approval / pending ask was purge-CANCELLED here even
716
+ // though every sibling recovery path preserves it. KEEP the entry.
717
+ if (isIntentionalWait(fresh.manifest, now)) {
718
+ abortedIntentionalWait = true;
719
+ return;
720
+ }
712
721
  // R7-6 (2026-10-16): under-lock liveness revalidation — the pre-lock
713
722
  // gate used inputs read BEFORE the lock; a concurrent `team resume`
714
723
  // adoption may have rewritten them in that window (fresh
@@ -747,6 +756,14 @@ export function purgeStaleActiveRunIndex(
747
756
  }
748
757
  }
749
758
  updateRunStatus(fresh.manifest, "cancelled", "Orphaned run: worker process dead and no recent activity");
759
+ // R2-M-A3: clear waitState on cancel (mirror cancelOrphanedRuns :431-438) —
760
+ // an orphan-cancelled run must not keep the park pointer shielding it
761
+ // from staleness repair for the full waiting TTL.
762
+ if (fresh.manifest.waitState) {
763
+ const cleared = { ...fresh.manifest, waitState: undefined, updatedAt: new Date(now).toISOString() };
764
+ saveRunManifest(cleared);
765
+ fresh.manifest = cleared;
766
+ }
750
767
  });
751
768
  // R7-6: only terminate live agents when the cancel actually landed —
752
769
  // an aborted (fresh-evidence) run is alive; terminating it would kill
@@ -767,7 +784,9 @@ export function purgeStaleActiveRunIndex(
767
784
  }
768
785
  // R7-6: a fresh-evidence abort means the run is alive again — KEEP the
769
786
  // registry entry (live-claim signal) instead of unregistering it.
770
- if (abortedFreshLiveness) {
787
+ // R2-M-A3: an intentional-wait abort keeps the entry too (same reasoning —
788
+ // the run is parked on a human decision, not crashed).
789
+ if (abortedFreshLiveness || abortedIntentionalWait) {
771
790
  kept.push(entry.runId);
772
791
  continue;
773
792
  }
@@ -794,12 +813,19 @@ export function purgeStaleActiveRunIndex(
794
813
  if (!hasRecentLifeEvidence(entry, manifest.updatedAt, now, staleThresholdMs)) {
795
814
  let cancelledUnderLock = false;
796
815
  let abortedFreshLiveness = false;
816
+ let abortedIntentionalWait = false; // R2-M-A3: keep intentional-wait runs
797
817
  try {
798
818
  const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
799
819
  if (fullLoaded && fullLoaded.manifest.status === "running") {
800
820
  withRunLockSync(fullLoaded.manifest, () => {
801
821
  const fresh = loadRunManifestById(entry.cwd, entry.runId);
802
822
  if (fresh?.manifest.status !== "running") return;
823
+ // R2-M-A3 (2026-10-17): mirror the step-5 guard (and :135/:329/:937) — a
824
+ // parked run (plan approval / pending ask) is not an orphan. KEEP it.
825
+ if (isIntentionalWait(fresh.manifest, now)) {
826
+ abortedIntentionalWait = true;
827
+ return;
828
+ }
803
829
  // R7-6 (2026-10-16): under-lock liveness revalidation — the pre-lock
804
830
  // gate used inputs read BEFORE the lock; a concurrent `team resume`
805
831
  // adoption may have rewritten them in that window (fresh
@@ -846,6 +872,12 @@ export function purgeStaleActiveRunIndex(
846
872
  Math.round(staleThresholdMs / 60000) +
847
873
  " minutes",
848
874
  );
875
+ // R2-M-A3: clear waitState on cancel (mirror :431-438 + step 5 above).
876
+ if (fresh.manifest.waitState) {
877
+ const cleared = { ...fresh.manifest, waitState: undefined, updatedAt: new Date(now).toISOString() };
878
+ saveRunManifest(cleared);
879
+ fresh.manifest = cleared;
880
+ }
849
881
  cancelledUnderLock = true;
850
882
  });
851
883
  // R7-1/R7-6: only terminate live agents when the cancel actually landed
@@ -867,7 +899,8 @@ export function purgeStaleActiveRunIndex(
867
899
  }
868
900
  // R7-1/R7-6: a fresh-evidence abort means the run is alive again — KEEP
869
901
  // the registry entry (live-claim signal) instead of unregistering it.
870
- if (abortedFreshLiveness) {
902
+ // R2-M-A3: an intentional-wait abort keeps the entry too.
903
+ if (abortedFreshLiveness || abortedIntentionalWait) {
871
904
  kept.push(entry.runId);
872
905
  continue;
873
906
  }
@@ -55,8 +55,19 @@ export class BatchBarrier {
55
55
  }
56
56
  >();
57
57
 
58
- /** Register a member at spawn time. Idempotent per (batchId, agentId). */
58
+ /** R2-L-B6 (SEV-10): consolidated-notified batches are FREED (member maps
59
+ * dropped) and remembered here as tombstones so `alreadyNotified` stays
60
+ * true and late `markTerminal`/`register` calls never re-notify. Bounded
61
+ * (evict-oldest) — previously every batchId leaked its member maps for the
62
+ * session lifetime (zero production callers of dispose()). */
63
+ private readonly notifiedTombstones = new Map<string, number>();
64
+ private static readonly TOMBSTONE_CAP = 256;
65
+
66
+ /** Register a member at spawn time. Idempotent per (batchId, agentId).
67
+ * R2-L-B6: no-op for a tombstoned (already notified) batch — the member
68
+ * map is intentionally gone and must not resurrect a re-notifiable batch. */
59
69
  register(batchId: string, agentId: string, meta?: { description?: string; type?: string }): void {
70
+ if (this.notifiedTombstones.has(batchId)) return;
60
71
  let batch = this.batches.get(batchId);
61
72
  if (!batch) {
62
73
  batch = {
@@ -83,6 +94,18 @@ export class BatchBarrier {
83
94
  * on-the-fly as a batch-of-one so its terminal state is not silently lost.
84
95
  */
85
96
  markTerminal(batchId: string, member: BatchMember): BatchSnapshot {
97
+ // R2-L-B6: the batch already fired its consolidated notification and was
98
+ // freed — report notified:true so the caller NEVER re-notifies a late
99
+ // member (documented late-spawn semantics), without resurrecting the map.
100
+ if (this.notifiedTombstones.has(batchId)) {
101
+ return {
102
+ batchId,
103
+ members: [{ ...member }],
104
+ terminal: [{ ...member }],
105
+ allDone: true,
106
+ notified: true,
107
+ };
108
+ }
86
109
  let batch = this.batches.get(batchId);
87
110
  if (!batch) {
88
111
  batch = {
@@ -117,13 +140,22 @@ export class BatchBarrier {
117
140
 
118
141
  /** Has the consolidated notification already been emitted for this batch? */
119
142
  alreadyNotified(batchId: string): boolean {
120
- return this.batches.get(batchId)?.notified ?? false;
143
+ return this.batches.get(batchId)?.notified ?? this.notifiedTombstones.has(batchId);
121
144
  }
122
145
 
123
- /** Mark the consolidated notification as emitted. No-op if already set. */
146
+ /** Mark the consolidated notification as emitted. No-op if already set.
147
+ * R2-L-B6: DISPOSE the batch here — once the consolidated notify fired, the
148
+ * member maps are dead weight; keep only a bounded tombstone so
149
+ * `alreadyNotified`/late `markTerminal` preserve their answers. */
124
150
  markNotified(batchId: string): void {
125
151
  const batch = this.batches.get(batchId);
126
152
  if (batch) batch.notified = true;
153
+ this.batches.delete(batchId);
154
+ this.notifiedTombstones.set(batchId, Date.now());
155
+ if (this.notifiedTombstones.size > BatchBarrier.TOMBSTONE_CAP) {
156
+ const oldest = this.notifiedTombstones.keys().next().value;
157
+ if (oldest !== undefined) this.notifiedTombstones.delete(oldest);
158
+ }
127
159
  }
128
160
 
129
161
  /** Read-only snapshot (for tests / debugging). */
@@ -141,7 +173,12 @@ export class BatchBarrier {
141
173
 
142
174
  /** Drop a batch (used on cleanup / test reset). */
143
175
  dispose(batchId?: string): void {
144
- if (batchId === undefined) this.batches.clear();
145
- else this.batches.delete(batchId);
176
+ if (batchId === undefined) {
177
+ this.batches.clear();
178
+ this.notifiedTombstones.clear();
179
+ } else {
180
+ this.batches.delete(batchId);
181
+ this.notifiedTombstones.delete(batchId);
182
+ }
146
183
  }
147
184
  }
@@ -61,7 +61,9 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
61
61
  }
62
62
  return t;
63
63
  });
64
- await saveRunTasksAsync(manifest, updatedTasks);
64
+ // R3-H1 (fix): fresh-read group-scoped merge — never persist the dispatch-time
65
+ // full-array snapshot over concurrent sibling state changes.
66
+ await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
65
67
  await appendEventAsync(manifest.eventsPath, {
66
68
  type: "task.coalesced_dispatch_start",
67
69
  runId: manifest.runId,
@@ -86,7 +88,9 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
86
88
  heartbeat: t.heartbeat ?? createWorkerHeartbeat(t.id),
87
89
  };
88
90
  });
89
- await saveRunTasksAsync(manifest, updatedTasks);
91
+ // R3-H1 (fix): fresh-read group-scoped merge — never persist the dispatch-time
92
+ // full-array snapshot over concurrent sibling state changes.
93
+ await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
90
94
 
91
95
  let rawOutput = "";
92
96
  let success = false;
@@ -398,7 +402,9 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
398
402
  if (drainTimeout !== undefined) clearTimeout(drainTimeout);
399
403
  }
400
404
  }
401
- await saveRunTasksAsync(manifest, updatedTasks);
405
+ // R3-H1 (fix): fresh-read group-scoped merge — never persist the dispatch-time
406
+ // full-array snapshot over concurrent sibling state changes.
407
+ await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
402
408
  let updatedManifest: TeamRunManifest = {
403
409
  ...manifest,
404
410
  artifacts: mergeArtifacts([...manifest.artifacts, ...newArtifacts]),
@@ -36,6 +36,10 @@ export interface CrewSchedulerOptions {
36
36
  * waits (e.g. a yearly cron) must be chained in hops below this ceiling. */
37
37
  const MAX_TIMER_DELAY_MS = 2_147_483_000;
38
38
 
39
+ /** R2-M-B4 (SEV-4): retention cap for job.spawnedRunIds (bounded persisted JSON;
40
+ * remove() cancels at most this most-recent window). */
41
+ const SPAWNED_RUN_IDS_CAP = 50;
42
+
39
43
  export class CrewScheduler {
40
44
  private jobs = new Map<string, ScheduledJob>();
41
45
  private timers = new Map<string, ReturnType<typeof setInterval | typeof setTimeout>>();
@@ -81,6 +85,12 @@ export class CrewScheduler {
81
85
 
82
86
  add(job: ScheduledJob): void {
83
87
  this.jobs.set(job.id, job);
88
+ // R2-M-B6 (SEV-11): a persisted nextRun in the past = occurrences missed
89
+ // while the host was down. Classic cron leaves mail; we leave a one-time
90
+ // `skipped` event + log line on RELOAD (this path). No catch-up — firing a
91
+ // stale queue at load would storm the host. Runtime re-arms go through
92
+ // update() (never add()), so a normal fire's stale nextRun cannot reach here.
93
+ this.reportMissedCronOccurrences(job);
84
94
  if (job.enabled) this.arm(job);
85
95
  this.emit?.({ type: "added", job });
86
96
  }
@@ -123,7 +133,13 @@ export class CrewScheduler {
123
133
  recordSpawnedRun(jobId: string, runId: string): void {
124
134
  const job = this.jobs.get(jobId);
125
135
  if (!job) return;
126
- const spawnedRunIds = [...(job.spawnedRunIds ?? []), runId];
136
+ // R2-M-B4 (SEV-4): cap spawnedRunIds. The unbounded append persisted the
137
+ // FULL array per fire (O(n²) writes — a 5-min job ≈ 8.6k entries/30d,
138
+ // ~300-350 KB) and remove() then cancelled ALL historical runIds. Keep the
139
+ // most recent SPAWNED_RUN_IDS_CAP; runs beyond the cap are long terminal,
140
+ // so not cancelling them is the natural no-op.
141
+ const appended = [...(job.spawnedRunIds ?? []), runId];
142
+ const spawnedRunIds = appended.length > SPAWNED_RUN_IDS_CAP ? appended.slice(-SPAWNED_RUN_IDS_CAP) : appended;
127
143
  this.jobs.set(jobId, { ...job, spawnedRunIds });
128
144
  }
129
145
 
@@ -150,6 +166,26 @@ export class CrewScheduler {
150
166
  return { ok: true };
151
167
  }
152
168
 
169
+ /** R2-M-B6: one-time missed-occurrence report for a reloaded cron job whose
170
+ * persisted nextRun is in the past (bounded occurrence walk — no catch-up). */
171
+ private reportMissedCronOccurrences(job: ScheduledJob): void {
172
+ if (job.scheduleType !== "cron" || !job.nextRun) return;
173
+ const nowMs = this.now().getTime();
174
+ const persistedMs = new Date(job.nextRun).getTime();
175
+ if (!Number.isFinite(persistedMs) || persistedMs >= nowMs) return;
176
+ let missed = 0;
177
+ let cursor = new Date(persistedMs);
178
+ for (let i = 0; i < 10_000; i++) {
179
+ const next = nextRunTime({ kind: "cron", spec: job.schedule }, cursor);
180
+ if (!(next instanceof Date) || next.getTime() > nowMs) break;
181
+ missed += 1;
182
+ cursor = next;
183
+ }
184
+ const reason = `missed ${missed} occurrence(s) while host down (persisted nextRun ${job.nextRun} < now ${new Date(nowMs).toISOString()})`;
185
+ this.emit?.({ type: "skipped", jobId: job.id, reason });
186
+ console.warn(`[pi-crew scheduler] job ${job.id} (${job.name ?? job.id}) skipped: ${reason}`);
187
+ }
188
+
153
189
  private arm(job: ScheduledJob): void {
154
190
  if (this.timers.has(job.id)) return;
155
191
  if (job.scheduleType === "interval" && job.intervalMs) {
@@ -445,46 +481,55 @@ function cronFieldCandidates(field: string, min: number, max: number, names?: Re
445
481
  return matched;
446
482
  }
447
483
 
448
- /**
449
- * Match a single cron field value against a cron field expression.
450
- * Supports the standard cron grammar: wildcard, single N, range a-b, list
451
- * a,b,c, step syntax (wildcard-step, range-step, from-step), and named tokens
452
- * (MON, JAN) when a `names` map is passed. Fixes the prior matcher that
453
- * rejected step values and named DOW (parseInt returned NaN).
454
- */
455
- function cronFieldMatches(value: number, field: string, min: number, max: number, names?: Record<string, number>): boolean {
456
- const candidates = cronFieldCandidates(field, min, max, names);
457
- return candidates?.has(value) === true;
458
- }
459
-
460
484
  function nextCronDate(spec: string, from: Date): Date | { error: string } | null {
461
485
  const parts = spec.split(/\s+/);
462
486
  if (parts.length < 5) return { error: "Invalid cron expression" };
463
487
  const [minStr, hourStr, domStr, monthStr, dowStr] = parts;
464
488
 
489
+ // R2-L-B5 (SEV-9): hoist the candidate Sets OUT of the scan loop. The old
490
+ // minute-by-minute scan rebuilt 5 Sets per step (sparse crons walked up to
491
+ // 527k iterations ≈ 309ms event-loop stall, re-run after every fire). The
492
+ // scan below keeps EXACTLY the same semantics (first matching minute in
493
+ // chronological order) but hops by hour/day when the coarse fields mismatch.
494
+ const minuteCandidates = cronFieldCandidates(minStr, 0, 59);
495
+ const hourCandidates = cronFieldCandidates(hourStr, 0, 23);
496
+ const domCandidates = cronFieldCandidates(domStr, 1, 31);
497
+ const monthCandidates = cronFieldCandidates(monthStr, 1, 12, CRON_MONTH_NAMES);
498
+ const dowCandidates = cronFieldCandidates(dowStr, 0, 6, CRON_DOW_NAMES);
499
+ if (!minuteCandidates || !hourCandidates || !domCandidates || !monthCandidates || !dowCandidates) {
500
+ return { error: "Invalid cron expression" };
501
+ }
502
+
465
503
  let cursor = new Date(from.getTime());
466
504
  cursor.setSeconds(0, 0);
467
505
  cursor = new Date(cursor.getTime() + 60_000);
468
506
 
469
- const maxIterations = 366 * 24 * 60;
470
- for (let i = 0; i < maxIterations; i++) {
471
- const min = cursor.getUTCMinutes();
472
- const hour = cursor.getUTCHours();
473
- const dom = cursor.getUTCDate();
474
- const month = cursor.getUTCMonth() + 1;
475
- const dow = cursor.getUTCDay();
476
-
507
+ // Search window: 366 days (the old 366*24*60 minute-step bound).
508
+ const deadlineMs = cursor.getTime() + 366 * 24 * 60 * 60_000;
509
+ while (cursor.getTime() <= deadlineMs) {
510
+ // Day-level filter: month/dom/dow — skip whole days on mismatch.
477
511
  if (
478
- cronFieldMatches(min, minStr, 0, 59) &&
479
- cronFieldMatches(hour, hourStr, 0, 23) &&
480
- cronFieldMatches(dom, domStr, 1, 31) &&
481
- cronFieldMatches(month, monthStr, 1, 12, CRON_MONTH_NAMES) &&
482
- cronFieldMatches(dow, dowStr, 0, 6, CRON_DOW_NAMES)
512
+ !monthCandidates.has(cursor.getUTCMonth() + 1) ||
513
+ !domCandidates.has(cursor.getUTCDate()) ||
514
+ !dowCandidates.has(cursor.getUTCDay())
483
515
  ) {
484
- return cursor;
516
+ // Hop to 00:00 of the next UTC day (Date.UTC normalizes overflow).
517
+ cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate() + 1, 0, 0));
518
+ continue;
485
519
  }
486
-
487
- cursor = new Date(cursor.getTime() + 60_000);
520
+ // Hour-level filter: skip the rest of the hour on mismatch.
521
+ if (!hourCandidates.has(cursor.getUTCHours())) {
522
+ cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate(), cursor.getUTCHours() + 1, 0));
523
+ continue;
524
+ }
525
+ // Matching day + hour: scan the remaining minutes of THIS hour only.
526
+ for (let minute = cursor.getUTCMinutes(); minute <= 59; minute++) {
527
+ if (minuteCandidates.has(minute)) {
528
+ return new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate(), cursor.getUTCHours(), minute));
529
+ }
530
+ }
531
+ // No matching minute left in this hour → next hour boundary.
532
+ cursor = new Date(Date.UTC(cursor.getUTCFullYear(), cursor.getUTCMonth(), cursor.getUTCDate(), cursor.getUTCHours() + 1, 0));
488
533
  }
489
534
 
490
535
  return { error: "No next cron occurrence found within search window" };
@@ -108,10 +108,28 @@ const WAITING_TTL_MS = 24 * 60 * 60 * 1000;
108
108
  * normal staleness repair paths apply (leak guard).
109
109
  */
110
110
  export function isWaitAnswerPending(manifest: TeamRunManifest, now = Date.now()): boolean {
111
- const askedAtMs = manifest.waitState?.askedAt ? new Date(manifest.waitState.askedAt).getTime() : Number.NaN;
112
- return Number.isFinite(askedAtMs) && now - askedAtMs <= WAITING_TTL_MS;
111
+ const waitState = manifest.waitState;
112
+ if (!waitState?.askedAt) return false;
113
+ const askedAtMs = new Date(waitState.askedAt).getTime();
114
+ if (!Number.isFinite(askedAtMs) || now - askedAtMs > WAITING_TTL_MS) return false;
115
+ // R2-M-A2 (2026-10-17): the parked-ask shield outlived the ask itself —
116
+ // WAITING_TTL_MS (24h) ≫ the server-clamped ask deadline (≤1h,
117
+ // request-parsers.ts WAIT_REQUEST_TIMEOUT_SEC_MAX), so a runner that died
118
+ // parked stayed shielded ~23h past its park deadline (dead parked runner
119
+ // blocked verdict tree/cancel/U9 fast-path). When the broker stamped a
120
+ // deadline on the park, the shield expires at deadline + grace; the 24h
121
+ // TTL above stays ONLY as the no-deadline fallback (pre-fix manifests).
122
+ if (typeof waitState.deadline === "number" && Number.isFinite(waitState.deadline)) {
123
+ return now <= waitState.deadline + PARKED_ASK_GRACE_MS;
124
+ }
125
+ return true;
113
126
  }
114
127
 
128
+ /** R2-M-A2: grace past the ask deadline before a parked run loses
129
+ * reconciliation protection (buffer for the sweep cadence / in-tool
130
+ * surfacing — the deadline-owner sweep in dispatch-batch co-owns this). */
131
+ const PARKED_ASK_GRACE_MS = 60 * 60 * 1000;
132
+
115
133
  /**
116
134
  * Generalized intentional-wait predicate (ADR-0 item 9, extended by ADR-4 §2):
117
135
  * the run is blocked on a human decision — plan approval (record-first with
@@ -415,10 +433,22 @@ export function hasLiveRunClaim(runId: string, manifest: TeamRunManifest): boole
415
433
  return false;
416
434
  }
417
435
 
418
- function repairStaleRun(manifest: TeamRunManifest, tasks: TeamTaskState[], reason: string): TeamTaskState[] {
436
+ function repairStaleRun(
437
+ manifest: TeamRunManifest,
438
+ tasks: TeamTaskState[],
439
+ reason: string,
440
+ // R2-M-A1 (2026-10-17): when set, ONLY these task ids are cancelled — used
441
+ // by the no_pid_individual_stale_task branch, whose contract (and comment)
442
+ // says "repair only the zombie subset, not the whole run": cancelling
443
+ // pid-ALIVE workers there destroyed healthy siblings in the SAME run.
444
+ onlyTaskIds?: Set<string>,
445
+ ): TeamTaskState[] {
419
446
  const now = new Date().toISOString();
420
447
  const repairedTasks = tasks.map((task) => {
421
- if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
448
+ if (
449
+ (onlyTaskIds === undefined || onlyTaskIds.has(task.id)) &&
450
+ (task.status === "running" || task.status === "queued" || task.status === "waiting")
451
+ ) {
422
452
  return {
423
453
  ...task,
424
454
  status: "cancelled" as const,
@@ -571,7 +601,13 @@ export function reconcileStaleRun(manifest: TeamRunManifest, tasks: TeamTaskStat
571
601
  // ALL tasks are zombies, while this path repairs only the zombie
572
602
  // subset when some tasks are still healthy.
573
603
  if (staleTaskIds.length > 0) {
574
- const repaired = repairStaleRun(manifest, tasks, "no_pid_individual_stale_task");
604
+ // R2-M-A1 (2026-10-17): cancel ONLY the stale subset. The pre-fix code
605
+ // called repairStaleRun unscoped, which cancelled EVERY running/queued/
606
+ // waiting task in the run — including pid-ALIVE workers the branch was
607
+ // supposed to leave alone (pinned by stale-reconciler-r2h1-retention,
608
+ // updated in the same wave: task_a now stays running).
609
+ const staleSet = new Set(staleTaskIds);
610
+ const repaired = repairStaleRun(manifest, tasks, "no_pid_individual_stale_task", staleSet);
575
611
  // Only return the individually repaired tasks in detail
576
612
  return {
577
613
  runId,
@@ -713,7 +749,10 @@ export function reconcileOrphanedTempWorkspaces(
713
749
  const result = reconcileStaleRun(manifest, tasks, now);
714
750
  if (result.repaired && result.repairedTasks) {
715
751
  // Persist repaired tasks (atomic — temp+rename to survive mid-write crash)
716
- atomicWriteJson(tasksPath, result.repairedTasks);
752
+ // R2-H1 (fix): persist the FULL array (NEW-1 persistTasks), not the filtered
753
+ // stale subset — the subset write destroyed every sibling record (mirror
754
+ // crash-recovery.ts:1066).
755
+ atomicWriteJson(tasksPath, result.persistTasks ?? result.repairedTasks);
717
756
  // Update manifest status
718
757
  const updated = {
719
758
  ...manifest,