pi-crew 0.11.1 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/CHANGELOG.md +82 -9
  2. package/README.md +161 -1036
  3. package/agents/verifier.md +18 -7
  4. package/dist/index.mjs +742 -91462
  5. package/docs/README.md +57 -46
  6. package/docs/architecture.md +87 -33
  7. package/docs/commands-reference.md +9 -5
  8. package/docs/troubleshooting.md +3 -2
  9. package/package.json +5 -4
  10. package/schema.json +29 -0
  11. package/skills/real-test-pi-crew/SKILL.md +206 -41
  12. package/src/agents/agent-config.ts +1 -1
  13. package/src/agents/discover-agents.ts +1 -1
  14. package/src/config/config-validation.ts +15 -0
  15. package/src/config/config.ts +47 -13
  16. package/src/config/env-vars.ts +35 -0
  17. package/src/config/types.ts +19 -0
  18. package/src/errors.ts +2 -2
  19. package/src/extension/async-notifier.ts +55 -8
  20. package/src/extension/crew-input-router.ts +9 -2
  21. package/src/extension/help.ts +21 -10
  22. package/src/extension/knowledge-injection.ts +2 -1
  23. package/src/extension/management.ts +8 -3
  24. package/src/extension/notification-sink.ts +17 -0
  25. package/src/extension/plan-orchestrate.ts +19 -3
  26. package/src/extension/registration/command-utils.ts +28 -2
  27. package/src/extension/registration/commands/dashboard.ts +11 -1
  28. package/src/extension/registration/commands/manage.ts +31 -15
  29. package/src/extension/registration/commands/run.ts +24 -2
  30. package/src/extension/registration/commands/shared.ts +34 -10
  31. package/src/extension/registration/commands/status.ts +25 -2
  32. package/src/extension/registration/context-builder.ts +8 -2
  33. package/src/extension/registration/foreground-run-controller.ts +2 -2
  34. package/src/extension/registration/health-notify-policy.ts +100 -0
  35. package/src/extension/registration/lazy-configurers.ts +38 -0
  36. package/src/extension/registration/lifecycle-handlers.ts +107 -40
  37. package/src/extension/registration/lifecycle.ts +77 -10
  38. package/src/extension/registration/observability.ts +125 -53
  39. package/src/extension/registration/registration-types.ts +7 -5
  40. package/src/extension/registration/runtime-cleanup.ts +9 -3
  41. package/src/extension/registration/subagent-helpers.ts +38 -0
  42. package/src/extension/registration/team-tool.ts +9 -2
  43. package/src/extension/registration/wire-cross-extension.ts +30 -0
  44. package/src/extension/run-compare.ts +220 -0
  45. package/src/extension/run-export.ts +37 -5
  46. package/src/extension/run-maintenance.ts +155 -5
  47. package/src/extension/team-manager-command.ts +3 -2
  48. package/src/extension/team-tool/dispatch/index.ts +3 -2
  49. package/src/extension/team-tool/dispatch/manage.ts +5 -2
  50. package/src/extension/team-tool/goal-wrap.ts +7 -0
  51. package/src/extension/team-tool/goal.ts +16 -1
  52. package/src/extension/team-tool/handle-settings.ts +19 -2
  53. package/src/extension/team-tool/health-monitor.ts +21 -7
  54. package/src/extension/team-tool/lifecycle-actions.ts +49 -1
  55. package/src/extension/team-tool/parallel-dispatch.ts +16 -1
  56. package/src/extension/team-tool/plan.ts +10 -0
  57. package/src/extension/team-tool/routing-hint.ts +63 -0
  58. package/src/extension/team-tool/run-intent.ts +1 -1
  59. package/src/extension/team-tool/run.ts +1 -1
  60. package/src/extension/team-tool/status.ts +4 -0
  61. package/src/extension/team-tool.ts +52 -6
  62. package/src/extension/webhook-notify.ts +382 -0
  63. package/src/observability/metric-sink.ts +12 -2
  64. package/src/prompt/prompt-runtime.ts +82 -31
  65. package/src/prompt/worker-events-channel.ts +12 -0
  66. package/src/runtime/README.md +1 -1
  67. package/src/runtime/async-runner.ts +123 -1
  68. package/src/runtime/background-runner.ts +313 -234
  69. package/src/runtime/broker/crew-broker.ts +76 -71
  70. package/src/runtime/broker/delegate/delegate-event.ts +12 -8
  71. package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
  72. package/src/runtime/broker/protocol/connection-state.ts +10 -0
  73. package/src/runtime/broker/protocol/lock-busy.ts +76 -0
  74. package/src/runtime/broker/wait-status-cache.ts +1 -1
  75. package/src/runtime/child-pi/child-pi-timers.ts +1 -1
  76. package/src/runtime/child-pi/mock-fixtures.ts +48 -0
  77. package/src/runtime/crew-agent-records.ts +337 -45
  78. package/src/runtime/deadletter.ts +43 -1
  79. package/src/runtime/delegate-spawn.ts +21 -3
  80. package/src/runtime/deterministic-ast.ts +14 -2
  81. package/src/runtime/dispatch-batch.ts +72 -5
  82. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
  83. package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
  84. package/src/runtime/goal-workflow/goal-loop-runner.ts +105 -4
  85. package/src/runtime/group-join.ts +11 -3
  86. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  87. package/src/runtime/live-session/live-session-runtime.ts +1 -1
  88. package/src/runtime/model/model-fallback.ts +21 -1
  89. package/src/runtime/model/pi-args.ts +8 -10
  90. package/src/runtime/model/runtime-warmup.ts +2 -0
  91. package/src/runtime/output/output-validator.ts +10 -5
  92. package/src/runtime/plan-approval.ts +14 -1
  93. package/src/runtime/recovery/crash-recovery.ts +96 -49
  94. package/src/runtime/run-worker.ts +12 -1
  95. package/src/runtime/scheduling/global-worker-cap.ts +13 -6
  96. package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
  97. package/src/runtime/scheduling/scheduler.ts +49 -13
  98. package/src/runtime/scheduling/semaphore.ts +148 -20
  99. package/src/runtime/scratchpad/README.md +1 -1
  100. package/src/runtime/scratchpad/protocol.ts +1 -1
  101. package/src/runtime/settings-store.ts +1 -1
  102. package/src/runtime/skill-instructions.ts +22 -0
  103. package/src/runtime/stale-reconciler.ts +85 -13
  104. package/src/runtime/subagent-manager.ts +43 -13
  105. package/src/runtime/task-runner/child-executor.ts +17 -6
  106. package/src/runtime/task-runner/post-execution.ts +44 -14
  107. package/src/runtime/task-runner/pre-execution.ts +26 -2
  108. package/src/runtime/task-runner/prompt-builder.ts +150 -45
  109. package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
  110. package/src/runtime/task-runner.ts +28 -1
  111. package/src/runtime/team-runner.ts +42 -2
  112. package/src/runtime/verification/verification-worktree.ts +3 -0
  113. package/src/runtime/workspace-lock.ts +4 -1
  114. package/src/schema/config-schema.ts +14 -0
  115. package/src/schema/team-tool-schema.ts +17 -0
  116. package/src/state/atomic-write.ts +53 -0
  117. package/src/state/contracts.ts +109 -0
  118. package/src/state/coordination/locks.ts +191 -33
  119. package/src/state/coordination/mailbox.ts +147 -17
  120. package/src/state/crew-init.ts +87 -12
  121. package/src/state/event-log/cursor.ts +37 -1
  122. package/src/state/event-log/event-log-rotation.ts +72 -7
  123. package/src/state/stores/active-run-registry.ts +13 -1
  124. package/src/state/stores/state-store.ts +120 -23
  125. package/src/state/types.ts +4 -0
  126. package/src/ui/dashboard-panes/agents-pane.ts +22 -10
  127. package/src/ui/heartbeat-aggregator.ts +34 -0
  128. package/src/ui/inline-panel/index.ts +1 -1
  129. package/src/ui/keybinding-map.ts +22 -4
  130. package/src/ui/live-conversation-overlay.ts +6 -3
  131. package/src/ui/render-diff.ts +19 -6
  132. package/src/ui/run-dashboard.ts +109 -6
  133. package/src/ui/run-snapshot-cache.ts +27 -4
  134. package/src/ui/spinner.ts +26 -2
  135. package/src/ui/tool-progress-formatter.ts +2 -1
  136. package/src/ui/tool-renderers/brief-mode.ts +2 -1
  137. package/src/ui/tool-renderers/index.ts +3 -3
  138. package/src/ui/widget/task-list.ts +8 -5
  139. package/src/ui/widget/widget-model.ts +3 -1
  140. package/src/utils/incremental-reader.ts +11 -3
  141. package/src/utils/paths.ts +94 -12
  142. package/src/utils/project-markers.ts +40 -0
  143. package/src/worktree/worktree-manager.ts +206 -26
  144. package/tsconfig.json +2 -0
  145. package/workflows/distill.workflow.md +3 -3
  146. package/workflows/fast-fix.workflow.md +1 -1
  147. package/workflows/plan-execute.workflow.md +1 -1
  148. package/workflows/review.workflow.md +1 -1
  149. package/workflows/strict-fast-fix.workflow.md +1 -1
  150. package/docs/migration-v0.4-v0.5.md +0 -208
  151. package/docs/runtime-flow.md +0 -148
  152. package/src/ui/card-colors.ts +0 -168
  153. package/src/utils/completion-dedupe.ts +0 -63
  154. package/src/utils/resolve-shell.ts +0 -36
@@ -123,10 +123,15 @@ export function validateWorkerOutput(role: string, output: string): OutputValida
123
123
  *
124
124
  * A trimmed non-empty line counts as log noise ONLY if it matches one of:
125
125
  * 1. a bracket-tag extension log line: `[oc-go] ...`, `[pi-qwen-mm] ...`,
126
- * `[pi-qwen-mm] [core] [stderr] ...` — the leading tag is a lowercase
127
- * identifier (deliberately excludes capitalized bracketed prose like
128
- * `[Note] ...`), optionally followed by known sub-tags
129
- * (core/mcp/stderr/stdout/warn/info/error/debug);
126
+ * `[pi-qwen-mm] [core] [stderr] ...`, and — Finding #2 (2026-09-29
127
+ * battery) — colon-bearing extension tags like
128
+ * `[pi-crew:crash-recovery.reconcileStaleRuns] ...` or
129
+ * `[pi-crew:crew-vibes.publish-quota-status] ...`. The leading tag
130
+ * starts with a lowercase char (deliberately excludes capitalized
131
+ * bracketed prose like `[Note] ...` / `[Note: ...]`); the rest of the tag
132
+ * may carry `:`, `-`, `_`, `.`, digits, and inner capitals (extension
133
+ * subsystem names are camelCase, e.g. `reconcileStaleRuns`), optionally
134
+ * followed by known sub-tags (core/mcp/stderr/stdout/warn/info/error/debug);
130
135
  * 2. a Python `warnings.warn(` line (deprecation-warning continuation);
131
136
  * 3. a timestamped logging line: `2026-08-15 21:49:02,986 WARNING ...`.
132
137
  *
@@ -137,7 +142,7 @@ export function validateWorkerOutput(role: string, output: string): OutputValida
137
142
  * (post-execution.ts) applies the two-gate rule (authoritative output
138
143
  * sources empty AND artifact log-noise-only) before failing a task.
139
144
  */
140
- const BRACKET_TAG_LOG_LINE = /^\[[a-z0-9][a-z0-9_.-]*\](?:\s*\[(?:core|mcp|stderr|stdout|warn|info|error|debug)\])*(?:\s.*)?$/;
145
+ const BRACKET_TAG_LOG_LINE = /^\[[a-z0-9][a-zA-Z0-9_.:\-]*\](?:\s*\[(?:core|mcp|stderr|stdout|warn|info|error|debug)\])*(?:\s.*)?$/;
141
146
  const PYTHON_WARNING_LINE = /(?:^|\s)warnings\.warn\(/;
142
147
  const TIMESTAMPED_LOG_LINE = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:[.,]\d{1,6})?(?:Z|[+-]\d{2}:?\d{2})?\s+[A-Za-z]+\b/;
143
148
 
@@ -70,7 +70,20 @@ export async function ensurePlanApprovalRequested(manifest: TeamRunManifest, tas
70
70
  const assessTask = tasks.find((task) => task.stepId === "assess" && task.status === "completed");
71
71
  // ROADMAP T1.2: for non-adaptive workflows, fall back to the most recent
72
72
  // completed read-only (planning) task as the plan reference.
73
- const planTask = assessTask ?? [...tasks].reverse().find((t) => t.status === "completed" && !isMutatingTask(t));
73
+ // RR-021 WI-4.3e: allocation-free reverse scan — the old
74
+ // [...tasks].reverse().find() copied the whole array per call.
75
+ // (findLast needs an ES2023 lib bump, which breaks tsgo's RequestInfo
76
+ // ambient resolution — see WI-4.3 evidence note.)
77
+ let planTask = assessTask;
78
+ if (!planTask) {
79
+ for (let i = tasks.length - 1; i >= 0; i--) {
80
+ const t = tasks[i];
81
+ if (t?.status === "completed" && !isMutatingTask(t)) {
82
+ planTask = t;
83
+ break;
84
+ }
85
+ }
86
+ }
74
87
  const now = new Date().toISOString();
75
88
 
76
89
  // T2/R4 (ADR-4 §6 producer 3 + §8 dual-write): make sure a PlanRecord
@@ -1,12 +1,19 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
3
  import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
4
+ import { DEFAULT_PATHS } from "../../config/defaults.ts";
4
5
  import { appendHookEvent, executeHook } from "../../hooks/registry.ts";
5
6
  import type { MetricRegistry } from "../../observability/metric-registry.ts";
6
- import { withRunLockSync } from "../../state/coordination/locks.ts";
7
+ import { discoverRunLockFiles, sweepStaleLocks, withRunLock, withRunLockSync } from "../../state/coordination/locks.ts";
7
8
  import { appendEvent, scanSequence } from "../../state/event-log/event-log.ts";
8
9
  import { readActiveRunRegistry, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
9
- import { loadRunManifestById, saveRunManifest, saveRunTasks, updateRunStatus } from "../../state/stores/state-store.ts";
10
+ import {
11
+ loadRunManifestById,
12
+ loadRunManifestByIdAsync,
13
+ saveRunManifest,
14
+ saveRunTasks,
15
+ updateRunStatus,
16
+ } from "../../state/stores/state-store.ts";
10
17
  import type { TeamTaskState } from "../../state/types.ts";
11
18
  import { logInternalError } from "../../utils/internal-error.ts";
12
19
  import { projectCrewRoot, userCrewRoot } from "../../utils/paths.ts";
@@ -17,6 +24,7 @@ import { isWorkerHeartbeatStale } from "../heartbeat/worker-heartbeat.ts";
17
24
  import { terminateLiveAgentsForRun } from "../live-session/live-agent-manager.ts";
18
25
  import type { ManifestCache } from "../manifest-cache.ts";
19
26
  import { checkProcessLiveness } from "../process-status.ts";
27
+ import { mapConcurrent } from "../scheduling/parallel-utils.ts";
20
28
  import { isIntentionalWait, isPlanApprovalPendingEffective, type ReconcileResult, reconcileStaleRun } from "../stale-reconciler.ts";
21
29
 
22
30
  export interface RecoveryPlan {
@@ -689,13 +697,12 @@ export function purgeStaleActiveRunIndex(
689
697
  return { purged, kept };
690
698
  }
691
699
 
692
- export function reconcileAllStaleRuns(
700
+ export async function reconcileAllStaleRuns(
693
701
  cwd: string,
694
702
  manifestCache: ManifestCache,
695
703
  now = Date.now(),
696
704
  currentSessionId?: string,
697
- ): ReconcileResult[] {
698
- const results: ReconcileResult[] = [];
705
+ ): Promise<ReconcileResult[]> {
699
706
  // Capture runIds to reconcile BEFORE acquiring locks — avoids TOCTOU between cache iteration and lock acquisition.
700
707
  const runIds = manifestCache
701
708
  .list(50)
@@ -707,55 +714,95 @@ export function reconcileAllStaleRuns(
707
714
  return true;
708
715
  })
709
716
  .map((m) => m.runId);
710
- for (const runId of runIds) {
711
- const cached = manifestCache.get(runId);
712
- if (!cached) continue;
713
- const loaded = loadRunManifestById(cwd, runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
714
- if (!loaded) continue;
715
- // Use lock to prevent race with cancel/status handlers modifying the same run
716
- withRunLockSync(loaded.manifest, () => {
717
- // Re-read inside lock to get freshest data
718
- const fresh = loadRunManifestById(cwd, runId); // NOTE: inside withRunLockSync - consistent read
719
- if (!fresh || (fresh.manifest.status !== "running" && fresh.manifest.status !== "blocked")) return;
720
- // Belt-and-suspenders: reconcileStaleRun itself guards this, but the run
721
- // may have flipped to blocked+plan-approval between cache-list and lock
722
- // acquisition — re-check the freshest manifest under the lock.
723
- if (isPlanApprovalPendingEffective(fresh.manifest)) {
724
- results.push({
725
- runId,
726
- verdict: "blocked_awaiting_approval",
727
- repaired: false,
728
- detail: "Plan approval is pending; stale reconciliation skipped",
729
- });
730
- return;
731
- }
732
- const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
733
- if (result.repaired || result.verdict === "result_exists") {
734
- if (result.repairedTasks) {
735
- saveRunTasks(fresh.manifest, result.repairedTasks);
736
- for (const task of result.repairedTasks) {
737
- try {
738
- upsertCrewAgent(fresh.manifest, recordFromTask(fresh.manifest, task, "scaffold"));
739
- } catch {
740
- /* non-critical */
717
+ // RR-021 WI-1.5 (audit C3): serial reconcile → mapConcurrent bound 4.
718
+ // Each runId acquires its OWN run lock file (lockPath() is per-run), so the
719
+ // acquisitions are independent; bound 4 caps fs/CPU contention. Results keep
720
+ // input (snapshot) order — mapConcurrent indexes by item — and per-run
721
+ // semantics (re-read inside lock, plan-approval re-check) are unchanged.
722
+ const perRun = await mapConcurrent(runIds, 4, async (runId): Promise<ReconcileResult[]> => {
723
+ // RR-021 review remediation: per-run error isolation — one corrupt run
724
+ // must not reject the whole sweep (serial baseline threw too, but the
725
+ // SPEC's "independent per-run" intent requires continue-on-error).
726
+ try {
727
+ const cached = manifestCache.get(runId);
728
+ if (!cached) return [];
729
+ const loaded = await loadRunManifestByIdAsync(cwd, runId); // NOTE: best-effort only; concurrent writes may cause inconsistency
730
+ if (!loaded) return [];
731
+ const out: ReconcileResult[] = [];
732
+ // Use lock to prevent race with cancel/status handlers modifying the same run.
733
+ // Same v0.9.26 lock family as the old sync acquisition — interop-safe.
734
+ await withRunLock(loaded.manifest, async () => {
735
+ // Re-read inside lock to get freshest data
736
+ const fresh = await loadRunManifestByIdAsync(cwd, runId); // NOTE: inside withRunLock - consistent read
737
+ if (!fresh || (fresh.manifest.status !== "running" && fresh.manifest.status !== "blocked")) return;
738
+ // Belt-and-suspenders: reconcileStaleRun itself guards this, but the run
739
+ // may have flipped to blocked+plan-approval between cache-list and lock
740
+ // acquisition — re-check the freshest manifest under the lock.
741
+ if (isPlanApprovalPendingEffective(fresh.manifest)) {
742
+ out.push({
743
+ runId,
744
+ verdict: "blocked_awaiting_approval",
745
+ repaired: false,
746
+ detail: "Plan approval is pending; stale reconciliation skipped",
747
+ });
748
+ return;
749
+ }
750
+ const result = reconcileStaleRun(fresh.manifest, fresh.tasks, now);
751
+ if (result.repaired || result.verdict === "result_exists") {
752
+ if (result.repairedTasks) {
753
+ saveRunTasks(fresh.manifest, result.repairedTasks);
754
+ for (const task of result.repairedTasks) {
755
+ try {
756
+ upsertCrewAgent(fresh.manifest, recordFromTask(fresh.manifest, task, "scaffold"));
757
+ } catch {
758
+ /* non-critical */
759
+ }
741
760
  }
742
761
  }
762
+ updateRunStatus(fresh.manifest, "failed", `Stale run reconciled: ${result.detail}`);
763
+ void terminateLiveAgentsForRun(fresh.manifest.runId, "failed", appendEvent, fresh.manifest.eventsPath).catch((error) =>
764
+ logInternalError("crash-recovery.reconcile.terminate", error, `runId=${fresh.manifest.runId}`, "warn"),
765
+ );
766
+ appendEvent(fresh.manifest.eventsPath, {
767
+ type: "crew.run.reconciled_stale",
768
+ runId,
769
+ message: result.detail,
770
+ data: { verdict: result.verdict },
771
+ });
772
+ }
773
+ if (result.verdict !== "healthy") {
774
+ out.push(result);
743
775
  }
744
- updateRunStatus(fresh.manifest, "failed", `Stale run reconciled: ${result.detail}`);
745
- void terminateLiveAgentsForRun(fresh.manifest.runId, "failed", appendEvent, fresh.manifest.eventsPath).catch((error) =>
746
- logInternalError("crash-recovery.reconcile.terminate", error, `runId=${fresh.manifest.runId}`, "warn"),
776
+ });
777
+ return out;
778
+ } catch (error) {
779
+ logInternalError("crash-recovery.reconcileStaleRuns", error, `runId=${runId}`, "warn");
780
+ return [];
781
+ }
782
+ });
783
+ const results = perRun.flat();
784
+ // US-002 (2026-09-22): structured stale-lock sweep. Runs after reconciliation
785
+ // so locks whose holder died (kill -9 / crash) do not linger. The sweep is
786
+ // strictly safer than acquire-time steal (stale AND holder-dead only), so a
787
+ // live holder is never disturbed. Best-effort.
788
+ try {
789
+ const runsRoots = [
790
+ path.join(projectCrewRoot(cwd), DEFAULT_PATHS.state.runsSubdir),
791
+ path.join(userCrewRoot(), DEFAULT_PATHS.state.runsSubdir),
792
+ ];
793
+ for (const runsRoot of runsRoots) {
794
+ const { removed } = sweepStaleLocks(discoverRunLockFiles(runsRoot));
795
+ if (removed.length > 0) {
796
+ logInternalError(
797
+ "crash-recovery.sweep-stale-locks",
798
+ new Error(`swept ${removed.length} stale run lock(s)`),
799
+ removed.join(", "),
800
+ "warn",
747
801
  );
748
- appendEvent(fresh.manifest.eventsPath, {
749
- type: "crew.run.reconciled_stale",
750
- runId,
751
- message: result.detail,
752
- data: { verdict: result.verdict },
753
- });
754
- }
755
- if (result.verdict !== "healthy") {
756
- results.push(result);
757
802
  }
758
- });
803
+ }
804
+ } catch (error) {
805
+ logInternalError("crash-recovery.sweep-stale-locks", error);
759
806
  }
760
807
  return results;
761
808
  }
@@ -72,7 +72,18 @@ export interface WorkerSpawnInput extends ChildPiRunInput {
72
72
  export async function runWorker(input: WorkerSpawnInput): Promise<ChildPiRunResult> {
73
73
  const { cap = true, ...childPiInput } = input;
74
74
  if (cap) {
75
- return withWorkerSlot(() => runChildPi(childPiInput));
75
+ // RR-014 / F15: thread the caller's signal into the slot WAIT itself.
76
+ // Previously the signal lived only inside childPiInput and was read by
77
+ // runChildPi AFTER the acquire resolved — a cancelled task queued on a
78
+ // full pool stayed pending until the slot holder finished (and delayed
79
+ // every drainPendingUnits awaiting it). Two consistent outcomes:
80
+ // - abort wins the race → acquire rejects (SemaphoreAbortedError), no
81
+ // slot taken, fn never runs;
82
+ // - grant wins the race → acquire resolves, runChildPi's pre-spawn
83
+ // guard (child-pi-spawn.ts B5) sees signal.aborted and returns
84
+ // kind "aborted" WITHOUT spawning, and withWorkerSlot's finally
85
+ // releases the slot within a microtask.
86
+ return withWorkerSlot(() => runChildPi(childPiInput), childPiInput.signal);
76
87
  }
77
88
  return runChildPi(childPiInput);
78
89
  }
@@ -64,11 +64,16 @@ export function __test_resetCap(testCapacity: number): void {
64
64
  * Acquire a global worker slot. Resolves immediately if under cap, else queues
65
65
  * (FIFO) until a slot frees. MUST be paired with releaseWorkerSlot().
66
66
  *
67
+ * RR-014 / F15: an optional AbortSignal makes the WAIT itself cancellable — a
68
+ * waiter whose signal fires is removed from the queue eagerly and the acquire
69
+ * REJECTS (SemaphoreAbortedError) without consuming a slot. Without a signal
70
+ * the semantics are unchanged.
71
+ *
67
72
  * Used to bound WORKER spawns only. Do NOT route the goal-judge through this
68
73
  * (see the RFC MAJ#3 rationale in the module header).
69
74
  */
70
- export async function acquireWorkerSlot(): Promise<void> {
71
- await semaphore.acquire();
75
+ export async function acquireWorkerSlot(signal?: AbortSignal): Promise<void> {
76
+ await semaphore.acquire(signal);
72
77
  }
73
78
 
74
79
  /**
@@ -82,13 +87,15 @@ export function releaseWorkerSlot(): void {
82
87
  /**
83
88
  * Convenience: acquire a worker slot, run `fn`, and release on completion OR
84
89
  * throw. The slot is ALWAYS released — including when `fn` rejects — so a
85
- * throwing worker never leaks a slot (deadlock prevention).
90
+ * throwing worker never leaks a slot (deadlock prevention). If `signal` is
91
+ * provided and fires while WAITING for a slot, the acquire rejects and `fn`
92
+ * never runs (no slot was taken, so there is nothing to release).
86
93
  *
87
94
  * Example:
88
- * const result = await withWorkerSlot(() => runChildPi(...));
95
+ * const result = await withWorkerSlot(() => runChildPi(...), input.signal);
89
96
  */
90
- export async function withWorkerSlot<T>(fn: () => Promise<T>): Promise<T> {
91
- await acquireWorkerSlot();
97
+ export async function withWorkerSlot<T>(fn: () => Promise<T>, signal?: AbortSignal): Promise<T> {
98
+ await acquireWorkerSlot(signal);
92
99
  try {
93
100
  return await fn();
94
101
  } finally {
@@ -5,6 +5,7 @@ import { writeArtifact } from "../../state/stores/artifact-store.ts";
5
5
  import { loadRunManifestById, saveRunTasksAsync, updateRunStatus } from "../../state/stores/state-store.ts";
6
6
  import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
7
7
  import type { WorkflowStep } from "../../workflows/workflow-config.ts";
8
+ import { appendDeadletter } from "../deadletter.ts";
8
9
  import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../heartbeat/worker-heartbeat.ts";
9
10
  import type { CrewRuntimeMode } from "../model/runtime-resolver.ts";
10
11
  import type { RetryPolicy } from "../recovery/retry-executor.ts";
@@ -196,7 +197,32 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
196
197
  }
197
198
  };
198
199
  try {
199
- const result = useRetry ? await executeWithRetry(runOnce, policy, { signal }) : await runOnce();
200
+ const result = useRetry
201
+ ? await executeWithRetry(runOnce, policy, {
202
+ signal,
203
+ // US-003 (2026-09-22): exhausted retries on the COALESCED path were never
204
+ // deadlettered — only the singleton path (dispatch-batch) wired
205
+ // onRetryGivenUp, so an M6 group that burned its whole retry budget left
206
+ // no post-mortem record once the run dir was pruned. The group shares ONE
207
+ // worker: the entry is recorded against the group's first task and names
208
+ // the whole group in the message. Aborts are NOT exhaustion — skip them
209
+ // (executeWithRetry also fires this hook on the cancelled exit).
210
+ onRetryGivenUp: (attempts, error) => {
211
+ if (signal?.aborted) return;
212
+ appendDeadletter(manifest, {
213
+ runId: manifest.runId,
214
+ taskId: firstTask.id,
215
+ reason: "max-retries",
216
+ attempts,
217
+ lastError: `coalesced group (${taskIds.length} task(s): ${taskIds.join(", ")}) — ${error.message}`,
218
+ timestamp: new Date().toISOString(),
219
+ agent: agent.name,
220
+ role: firstTask.role,
221
+ runStatus: manifest.status,
222
+ });
223
+ },
224
+ })
225
+ : await runOnce();
200
226
  rawOutput = result.rawFinalText ?? result.stdout ?? "";
201
227
  // RT-5 #1/#2: distinguish cancel from failure. Cancel = run-level
202
228
  // signal aborted OR the child reports cooperative cancellation.
@@ -24,6 +24,7 @@ export type ScheduleChangeEvent =
24
24
  | { type: "removed"; jobId: string; spawnedRunIds?: string[] }
25
25
  | { type: "updated"; job: ScheduledJob }
26
26
  | { type: "fired"; jobId: string; agentId: string; name: string }
27
+ | { type: "skipped"; jobId: string; reason: string }
27
28
  | { type: "error"; jobId: string; error: string };
28
29
 
29
30
  export interface CrewSchedulerOptions {
@@ -152,19 +153,29 @@ export class CrewScheduler {
152
153
  private arm(job: ScheduledJob): void {
153
154
  if (this.timers.has(job.id)) return;
154
155
  if (job.scheduleType === "interval" && job.intervalMs) {
155
- const t = setInterval(() => this.fire(job.id), job.intervalMs);
156
- t.unref();
157
- this.timers.set(job.id, t);
156
+ // 2026-09-22 spawn-storm root cause: this branch used
157
+ // setInterval(fire, intervalMs). Node timers are 32-bit — a LEGAL long
158
+ // interval (e.g. 90d = 7,776,000,000ms > 2^31-1) overflows and Node
159
+ // silently sets the delay to 1ms (TimeoutOverflowWarning): the job then
160
+ // fired every millisecond, and each
161
+ // tick dispatched another run (measured live: ~104 garbage runs + 50+
162
+ // node processes from ONE registered 90-day interval job). armCron()
163
+ // already clamps its hops below the same ceiling — the interval branch
164
+ // was missed. Reuse the identical chained-hop treatment: schedule toward
165
+ // now+intervalMs, clamped; cronTick fires on arrival (its
166
+ // advanceCronNextRun is a cron-only no-op here), and fire()'s internal
167
+ // update() → disarm→arm lifecycle re-arms the next interval.
168
+ const targetMs = this.now().getTime() + job.intervalMs;
169
+ this.setCronTimeout(job.id, targetMs);
158
170
  } else if (job.scheduleType === "once") {
159
171
  const target = new Date(job.schedule).getTime();
160
172
  const delay = target - this.now().getTime();
161
173
  if (delay > 0) {
162
- const t = setTimeout(() => {
163
- this.fire(job.id);
164
- this.update(job.id, { enabled: false });
165
- }, delay);
166
- t.unref();
167
- this.timers.set(job.id, t);
174
+ // Same 32-bit ceiling as the interval branch above: a once job armed
175
+ // > 2^31-1 ms out (e.g. "+30d" — a LEGAL relative spec — or a far ISO
176
+ // timestamp) overflowed setTimeout to a 1ms PREMATURE fire. Chained hops
177
+ // instead; cronTick self-disables once-jobs on arrival (below).
178
+ this.setCronTimeout(job.id, target);
168
179
  } else {
169
180
  this.update(job.id, { enabled: false, lastStatus: "error" });
170
181
  this.emit?.({
@@ -219,6 +230,9 @@ export class CrewScheduler {
219
230
  // hop that just fired is dead, so no timer doubles up.
220
231
  this.fire(jobId);
221
232
  this.advanceCronNextRun(jobId);
233
+ // once-semantics (was inline in arm()'s removed setTimeout): consume the
234
+ // job after its single arrival fire so a re-arm cannot fire it again.
235
+ if (job.scheduleType === "once") this.update(jobId, { enabled: false });
222
236
  }
223
237
 
224
238
  /** After a cron fire, advance the persisted nextRun to the next occurrence
@@ -265,6 +279,18 @@ export class CrewScheduler {
265
279
  const job = this.jobs.get(id);
266
280
  if (!job || !this.executor) return;
267
281
  if (!job.enabled && !force) return;
282
+ // In-flight guard (2026-09-22, storm follow-up): the executor dispatches
283
+ // asynchronously and resets lastStatus (success/error) only on completion,
284
+ // so `running` marks the in-flight window. Without backpressure here, any
285
+ // interval shorter than the run duration stacks one overlapping run per
286
+ // tick (the 1ms overflow loop was the extreme case; fixed below/above).
287
+ // Skips emit a "skipped" event (visible to pi.events consumers; the toast
288
+ // bridge stays silent — a slow executor + short interval would spam).
289
+ // `force` (run-now) is the explicit escape hatch.
290
+ if (job.lastStatus === "running" && !force) {
291
+ this.emit?.({ type: "skipped", jobId: id, reason: "previous dispatch still in flight" });
292
+ return;
293
+ }
268
294
  this.update(id, { lastStatus: "running" });
269
295
  let agentId: string;
270
296
  try {
@@ -297,10 +323,15 @@ export class CrewScheduler {
297
323
  normalized: new Date(Date.now() + ms).toISOString(),
298
324
  };
299
325
  }
300
- // Interval: 5m
301
- const ivl = trimmed.match(/^(\d+)(s|m|h|d)$/);
326
+ // Interval: 5m — "ms" MUST be accepted: handle-schedule builds
327
+ // `${params.interval}ms` from the schema-level numeric `interval` param
328
+ // (ms number), so without an ms unit here every interval schedule died in
329
+ // the parser ("Invalid schedule …") — found live 2026-09-21 when
330
+ // `team action='schedule' interval=3600000` was impossible to satisfy.
331
+ const ivl = trimmed.match(/^(\d+)(ms|s|m|h|d)$/);
302
332
  if (ivl) {
303
- const ms = parseInt(ivl[1], 10) * { s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 }[ivl[2] as "s" | "m" | "h" | "d"];
333
+ const ms =
334
+ parseInt(ivl[1], 10) * { ms: 1, s: 1000, m: 60_000, h: 3_600_000, d: 86_400_000 }[ivl[2] as "ms" | "s" | "m" | "h" | "d"];
304
335
  return { type: "interval", intervalMs: ms, normalized: trimmed };
305
336
  }
306
337
  // ISO timestamp
@@ -329,13 +360,18 @@ function parseIntervalMs(s: string): number | undefined {
329
360
  let ms = 0;
330
361
  let remaining = s;
331
362
  const unitMs: Record<string, number> = {
363
+ // "ms" MUST be first in the regex alternation below and present here:
364
+ // handle-schedule builds `${params.interval}ms` from the numeric interval
365
+ // param, so without an ms unit every interval schedule failed to parse
366
+ // (found live 2026-09-21 — `interval=3600000` was unsatisfiable).
367
+ ms: 1,
332
368
  s: 1000,
333
369
  m: 60_000,
334
370
  h: 3_600_000,
335
371
  d: 86_400_000,
336
372
  };
337
373
  while (remaining.length > 0) {
338
- const m = remaining.match(/^(\d+)(s|m|h|d)/);
374
+ const m = remaining.match(/^(\d+)(ms|s|m|h|d)/);
339
375
  if (!m) return undefined;
340
376
  ms += parseInt(m[1], 10) * unitMs[m[2]];
341
377
  remaining = remaining.slice(m[0].length);