pi-crew 0.11.24 → 0.11.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/CHANGELOG.md +72 -0
  2. package/README.md +2 -2
  3. package/agents/designer.md +23 -0
  4. package/agents/orchestrator.md +17 -0
  5. package/agents/verifier.md +1 -1
  6. package/dist/index.mjs +568 -559
  7. package/{workflows/chain.workflow.md → docs/chain-workflow.md} +8 -0
  8. package/docs/dynamic-workflows.md +23 -0
  9. package/docs/publishing.md +34 -5
  10. package/docs/troubleshooting.md +50 -2
  11. package/docs/usage.md +7 -2
  12. package/install.mjs +2 -1
  13. package/package.json +4 -2
  14. package/skills/read-only-explorer/SKILL.md +15 -5
  15. package/skills/real-test-pi-crew/SKILL.md +7 -7
  16. package/src/config/config-validation.ts +58 -0
  17. package/src/config/config.ts +90 -5
  18. package/src/config/env-vars.ts +14 -7
  19. package/src/config/role-tools.ts +7 -5
  20. package/src/config/types.ts +4 -1
  21. package/src/extension/crew-cleanup.ts +6 -0
  22. package/src/extension/project-init.ts +19 -1
  23. package/src/extension/register.ts +68 -30
  24. package/src/extension/registration/artifact-cleanup.ts +52 -11
  25. package/src/extension/registration/commands/manage.ts +9 -2
  26. package/src/extension/registration/hook-registration.ts +3 -0
  27. package/src/extension/registration/lazy-configurers.ts +2 -0
  28. package/src/extension/registration/lifecycle-handlers.ts +79 -18
  29. package/src/extension/registration/render-loop.ts +27 -2
  30. package/src/extension/registration/tool-loop-guard.ts +5 -3
  31. package/src/extension/registration/verify-gate-tool.ts +7 -3
  32. package/src/extension/team-tool/api/read.ts +12 -3
  33. package/src/extension/team-tool/cancel.ts +11 -0
  34. package/src/extension/team-tool/config-patch.ts +56 -11
  35. package/src/extension/team-tool/destructive-gate.ts +72 -12
  36. package/src/extension/team-tool/dispatch/manage.ts +9 -1
  37. package/src/extension/team-tool/handle-schedule.ts +38 -4
  38. package/src/extension/team-tool/handle-settings.ts +76 -6
  39. package/src/extension/team-tool/health-monitor.ts +14 -2
  40. package/src/extension/team-tool/lifecycle-actions.ts +28 -10
  41. package/src/extension/team-tool/respond.ts +12 -5
  42. package/src/extension/team-tool/run-deadline.ts +29 -2
  43. package/src/extension/team-tool/run.ts +11 -0
  44. package/src/extension/team-tool/status.ts +64 -8
  45. package/src/extension/team-tool/workflow-manage.ts +5 -1
  46. package/src/extension/team-tool.ts +3 -0
  47. package/src/i18n.ts +18 -14
  48. package/src/prompt/inbox-poll.ts +14 -2
  49. package/src/prompt/message-tool.ts +1 -1
  50. package/src/prompt/prompt-runtime.ts +25 -3
  51. package/src/prompt/scratchpad-lifecycle.ts +14 -13
  52. package/src/prompt/surface-worker.ts +35 -9
  53. package/src/runtime/broker/crew-broker.ts +113 -111
  54. package/src/runtime/broker/delegate/delegate-abort.ts +118 -0
  55. package/src/runtime/broker/delegate/delegate-event.ts +3 -1
  56. package/src/runtime/broker/delegate/delegate-finalize.ts +179 -0
  57. package/src/runtime/broker/protocol/wait-resolve.ts +71 -5
  58. package/src/runtime/budget-enforcement.ts +26 -6
  59. package/src/runtime/chain-parser.ts +5 -3
  60. package/src/runtime/chain-runner.ts +41 -22
  61. package/src/runtime/child-pi/mock-fixtures.ts +12 -5
  62. package/src/runtime/compaction/tool-output-pruner.ts +8 -3
  63. package/src/runtime/delegate-spawn.ts +61 -8
  64. package/src/runtime/dispatch-batch.ts +10 -4
  65. package/src/runtime/effectiveness.ts +31 -3
  66. package/src/runtime/goal-workflow/plan-templates.ts +2 -2
  67. package/src/runtime/live-session/live-agent-control.ts +89 -6
  68. package/src/runtime/live-session/live-agent-manager.ts +94 -10
  69. package/src/runtime/live-session/live-session-runtime.ts +9 -2
  70. package/src/runtime/model/model-fallback.ts +6 -9
  71. package/src/runtime/model/pi-args.ts +3 -3
  72. package/src/runtime/model/runtime-policy.ts +4 -2
  73. package/src/runtime/model/runtime-resolver.ts +6 -2
  74. package/src/runtime/process/orphan-reaper.ts +143 -0
  75. package/src/runtime/process-status.ts +38 -16
  76. package/src/runtime/resilient-edit.ts +170 -93
  77. package/src/runtime/role-permission.ts +3 -1
  78. package/src/runtime/run-tracker.ts +9 -1
  79. package/src/runtime/sanitize/worker-text.ts +98 -0
  80. package/src/runtime/scheduling/run-coalesced-task-group.ts +82 -28
  81. package/src/runtime/spawn-policy.ts +190 -8
  82. package/src/runtime/stale-reconciler.ts +36 -1
  83. package/src/runtime/subagent-manager.ts +29 -0
  84. package/src/runtime/surface/herdr-provider.ts +31 -1
  85. package/src/runtime/task-output-context.ts +58 -11
  86. package/src/runtime/task-runner/post-execution.ts +150 -0
  87. package/src/runtime/task-runner/prompt-builder.ts +41 -8
  88. package/src/runtime/task-runner.ts +16 -3
  89. package/src/runtime/verification/classifier-pre-gate.ts +27 -6
  90. package/src/runtime/verification/verify-gate.ts +153 -30
  91. package/src/state/atomic-write.ts +73 -4
  92. package/src/state/contracts.ts +7 -0
  93. package/src/state/coordination/mailbox.ts +40 -11
  94. package/src/state/event-log/event-log-rotation.ts +28 -6
  95. package/src/state/event-log/poison-marker.ts +16 -1
  96. package/src/state/stores/artifact-store.ts +181 -6
  97. package/src/state/stores/manifest-io.ts +126 -8
  98. package/src/state/stores/state-store.ts +30 -31
  99. package/src/state/types.ts +13 -0
  100. package/src/state/usage.ts +41 -0
  101. package/src/ui/run-dashboard.ts +3 -3
  102. package/src/ui/settings-overlay.ts +19 -1
  103. package/src/utils/internal-error.ts +5 -1
  104. package/src/utils/timer-limits.ts +30 -0
  105. package/src/utils/token-counter.ts +47 -0
  106. package/src/workflows/discover-workflows.ts +9 -1
  107. package/src/worktree/worktree-manager.ts +128 -18
  108. package/workflows/default.workflow.md +1 -1
  109. package/workflows/fast-fix.workflow.md +1 -1
  110. package/workflows/plan-execute.workflow.md +1 -1
  111. package/workflows/review.workflow.md +1 -1
  112. package/workflows/strict-fast-fix.workflow.md +1 -1
@@ -40,6 +40,20 @@
40
40
  * never changes the FAILED↔spawn logic; flag off keeps the gate
41
41
  * bit-for-bit identical to pre-U13-ADJ.
42
42
  *
43
+ * GROUND TRUTH (M1-7, R47-1 fix 2026-10-19): the gate NEVER derives checks
44
+ * from the task cwd's package.json at gate time — a worker-mutable file
45
+ * (shared-cwd mode) whose tampered `scripts.typecheck: "exit 0"` used to
46
+ * yield PASS + skipVerifier (worker self-grading). Ground truth is
47
+ * host-owned: `resolveHostVerifyGateSpec(cwd)` snapshots the project
48
+ * manifests + derives the check preset AT DISPATCH (host process, before
49
+ * any worker runs) and the spec is pinned on the run manifest; the spawn
50
+ * site passes it as `hostSpec`. The gate re-hashes the manifests BEFORE any
51
+ * command runs (drift ⇒ FAILED — tamper evidence, verifier spawns) and
52
+ * AFTER a PASS (drift ⇒ INCONCLUSIVE — mid-gate tamper), mirroring the P1a
53
+ * bookend (verification-integrity.ts). Absence of a spec is fail-closed:
54
+ * no spec + no explicit checks ⇒ zero checks ⇒ INCONCLUSIVE ⇒ the LLM
55
+ * verifier spawns — absence can never produce a skip.
56
+ *
43
57
  * SAFETY: a gate that cannot run (disabled by env, no checks resolved,
44
58
  * executor failure) degrades to "spawn the LLM verifier as today" — the
45
59
  * pre-gate can only SAVE a spawn on positive deterministic evidence, never
@@ -56,8 +70,11 @@ import * as fs from "node:fs";
56
70
  import { getCrewEnvBool, getCrewEnvInt } from "../../config/env-vars.ts";
57
71
  import type { MetricRegistry } from "../../observability/metric-registry.ts";
58
72
  import type { VerifierPreGateResult } from "./classifier-pre-gate.ts";
59
- import { classifyVerifierPreGate } from "./classifier-pre-gate.ts";
60
73
  import { executeCommand } from "./verification-gates.ts";
74
+ // Static edge is graph-free: verification-integrity.ts is already in the
75
+ // index.bundle value-import graph (goal.ts/goal-wrap.ts/goal-loop-runner.ts
76
+ // import it), so this adds no bundle-graph file (check-lazy-imports budget).
77
+ import { compareSnapshot, snapshotManifests } from "./verification-integrity.ts";
61
78
 
62
79
  /** One deterministic check to run. Mirrors the phase-gate shape
63
80
  * (verification-gates.ts) but kept local so the tool schema can reuse it. */
@@ -97,6 +114,10 @@ export interface VerifyGateOutcome {
97
114
  * detected flakiness or a rerun executor failure — the note is also
98
115
  * appended to evidenceLines and shapes the rendered context trailer. */
99
116
  determinismNote?: string;
117
+ /** M1-7 (R47-1): set when the dispatch-pinned manifest snapshot DRIFTED —
118
+ * the note is also appended to evidenceLines and switches the rendered
119
+ * context trailer to "results UNTRUSTED". */
120
+ integrityNote?: string;
100
121
  }
101
122
 
102
123
  /** Injected command executor. The TOOL path wraps ctx.executeTool("bash");
@@ -147,8 +168,8 @@ export function resolveVerifyDeterminismEnabled(explicit?: boolean): boolean {
147
168
  }
148
169
 
149
170
  /**
150
- * Default check preset — the spec's "check cấu hình" set, resolved from the
151
- * target repo's package.json scripts so the gate adapts instead of failing:
171
+ * Default check preset — the spec's "check cấu hình" set, resolved from a
172
+ * package.json's scripts:
152
173
  * - `typecheck` script present → `npm run --silent typecheck` (critical);
153
174
  * - `test:critical` script present → `npm run --silent test:critical`
154
175
  * (critical);
@@ -157,6 +178,11 @@ export function resolveVerifyDeterminismEnabled(explicit?: boolean): boolean {
157
178
  * `grep -c "<marker>" dist/index.mjs`) rather than defaulted.
158
179
  * Repos without either script resolve ZERO checks → INCONCLUSIVE → the
159
180
  * classifier layer (when enabled) or the LLM verifier spawn runs as today.
181
+ *
182
+ * HOST-SIDE ONLY (M1-7, R47-1): this reads a worker-MUTABLE file. It may run
183
+ * only in the host process at DISPATCH time (via resolveHostVerifyGateSpec,
184
+ * before any worker exists) — never as a gate-time fallback from a task cwd.
185
+ * Gate-time consumers must use the dispatch-pinned hostSpec instead.
160
186
  */
161
187
  export function defaultVerifyGateChecks(dir: string): VerifyGateCheck[] {
162
188
  let scripts: Record<string, unknown> = {};
@@ -177,6 +203,39 @@ export function defaultVerifyGateChecks(dir: string): VerifyGateCheck[] {
177
203
  return checks;
178
204
  }
179
205
 
206
+ /** Host-owned verify-gate spec (M1-7, R47-1 fix): captured ONCE at dispatch
207
+ * in the HOST process (pre-worker) and pinned on the run manifest.
208
+ * `snapshot` is the P1a manifest-hash set (verification-integrity.ts);
209
+ * `checks` is the preset derived from the host's own read of package.json.
210
+ * At gate time the task cwd is verified against `snapshot` instead of ever
211
+ * re-reading the worker-mutable package.json. Structural shape mirrors
212
+ * TeamRunManifest's `verifyGateSpec` field (the state layer must not import
213
+ * runtime types). */
214
+ export interface VerifyGateHostSpec {
215
+ snapshot: Record<string, string>;
216
+ checks: VerifyGateCheck[];
217
+ }
218
+
219
+ /** HOST-SIDE (dispatch) — build the verify spec for `cwd` (the project root
220
+ * as the host sees it BEFORE any worker runs). Returns undefined when the
221
+ * directory has neither manifests to hash nor a check preset (nothing to
222
+ * protect or verify); the gate then resolves no checks (fail-closed). */
223
+ export function resolveHostVerifyGateSpec(cwd: string): VerifyGateHostSpec | undefined {
224
+ const snapshot = snapshotManifests(cwd);
225
+ const checks = defaultVerifyGateChecks(cwd);
226
+ if (Object.keys(snapshot).length === 0 && checks.length === 0) return undefined;
227
+ return { snapshot, checks };
228
+ }
229
+
230
+ /** Gate-time integrity probe (M1-7): re-hash the manifests in `cwd` and diff
231
+ * against the dispatch snapshot. Non-empty result = the drifted relative
232
+ * paths — the task cwd's ground truth no longer matches what the host
233
+ * pinned at dispatch (worker tampering or legitimate churn; both fail
234
+ * closed: the gate must not grade on compromised ground truth). */
235
+ export function detectHostSpecDrift(cwd: string, spec: VerifyGateHostSpec): string[] {
236
+ return compareSnapshot(spec.snapshot, snapshotManifests(cwd));
237
+ }
238
+
180
239
  function tailOutput(output: string): string {
181
240
  if (output.length <= OUTPUT_TAIL_MAX_CHARS) return output;
182
241
  return `…${output.slice(-OUTPUT_TAIL_MAX_CHARS)}`;
@@ -414,11 +473,13 @@ export function renderVerifyGateContext(outcome: VerifyGateOutcome): string {
414
473
  // the original wording bit-for-bit; a flaky rerun (layer 3) gets its own
415
474
  // trailer because "do NOT re-run / ground truth" would be provably wrong.
416
475
  const executorDeath = outcome.checks.some((c) => !c.skipped && c.exitCode === null);
417
- const trailer = executorDeath
418
- ? "A verification command could not run (executor error — exit=err). Treat the build/test state as UNKNOWN; run the build/tests yourself before concluding."
419
- : outcome.determinismNote
420
- ? "The gate rerun detected NONDETERMINISM (a check passed once and failed on rerun). These gate results are NOT reliable — re-run the affected command yourself and treat the build/test state as UNKNOWN."
421
- : "The deterministic gate already ran these commands — do NOT re-run them; verify the WORK (diff, files, acceptance), using these results as ground truth for build/test state.";
476
+ const trailer = outcome.integrityNote
477
+ ? "Manifest integrity check FAILED: project manifest files changed since the host snapshot taken at dispatch. These gate results and any transcript claims about build/test state are UNTRUSTED — inspect what changed and re-run the verification commands yourself before concluding."
478
+ : executorDeath
479
+ ? "A verification command could not run (executor error — exit=err). Treat the build/test state as UNKNOWN; run the build/tests yourself before concluding."
480
+ : outcome.determinismNote
481
+ ? "The gate rerun detected NONDETERMINISM (a check passed once and failed on rerun). These gate results are NOT reliable — re-run the affected command yourself and treat the build/test state as UNKNOWN."
482
+ : "The deterministic gate already ran these commands — do NOT re-run them; verify the WORK (diff, files, acceptance), using these results as ground truth for build/test state.";
422
483
  lines.push("", trailer);
423
484
  return lines.join("\n");
424
485
  }
@@ -430,11 +491,23 @@ export function renderVerifyGateSummary(outcome: VerifyGateOutcome, reason: stri
430
491
 
431
492
  export interface VerifierGatePreGateInput {
432
493
  cwd: string;
433
- /** Explicit checks override the default preset (tool path / tests). */
494
+ /** Explicit checks take precedence over every other source (tool path /
495
+ * tests). Without them the gate derives NO checks on its own. */
434
496
  checks?: VerifyGateCheck[];
497
+ /** M1-7 (R47-1): host-owned spec pinned on the run manifest at dispatch.
498
+ * When provided, the gate verifies manifest integrity against the
499
+ * snapshot BEFORE running anything (drift ⇒ FAILED — tamper evidence,
500
+ * verifier spawns) and re-verifies after a PASS (drift ⇒ INCONCLUSIVE).
501
+ * When ABSENT there is NO fallback read of the worker-mutable task-cwd
502
+ * package.json: no explicit checks ⇒ zero checks ⇒ INCONCLUSIVE
503
+ * (fail-closed on absence — the LLM verifier spawns). */
504
+ hostSpec?: VerifyGateHostSpec;
435
505
  signal?: AbortSignal;
436
506
  timeoutMs?: number;
437
- /** U6B classifier layer args — consulted ONLY on INCONCLUSIVE. */
507
+ /** @deprecated M3-28 (R47-V2): NEVER consulted — the gate's structured verdict
508
+ * can never be pass-shaped, so the U6B classifier could only ever escalate
509
+ * (see the call-site note in {@link runVerifierGatePreGate}). Kept only so the
510
+ * task-runner dispatch call site still type-checks; delete together with it. */
438
511
  classifier?: {
439
512
  enabled: boolean;
440
513
  modelRegistry: unknown;
@@ -473,7 +546,42 @@ function incMetric(registry: MetricRegistry | undefined, name: string, help: str
473
546
  * to INCONCLUSIVE checks, which escalate to the LLM verifier.
474
547
  */
475
548
  export async function runVerifierGatePreGate(input: VerifierGatePreGateInput): Promise<VerifierGatePreGateResult> {
476
- const checks = input.checks ?? defaultVerifyGateChecks(input.cwd);
549
+ // ── M1-7 (R47-1): integrity BEFORE any command runs ────────────────────
550
+ // A drifted manifest means the ground truth the host pinned at dispatch is
551
+ // compromised — running ANY check now would be the self-grading exploit
552
+ // (the worker rewrote the scripts the gate was about to run). Fail FAILED
553
+ // (tamper evidence) → the verifier spawns with the drift in its prompt.
554
+ // A tampered package.json can never yield PASS + skipVerifier.
555
+ if (input.hostSpec) {
556
+ const drift = detectHostSpecDrift(input.cwd, input.hostSpec);
557
+ if (drift.length > 0) {
558
+ const note = `manifest-integrity: ${drift.join(", ")} changed since dispatch (host snapshot)`;
559
+ const outcome: VerifyGateOutcome = {
560
+ verdict: "FAILED",
561
+ checks: [],
562
+ evidenceLines: [note],
563
+ allPassed: false,
564
+ totalDurationMs: 0,
565
+ integrityNote: note,
566
+ };
567
+ const decision = resolveVerifierSpawnDecision(outcome);
568
+ incMetric(input.metricRegistry, "crew.verification.verify_gate_runs_total", "Deterministic verify_gate executions by verdict", {
569
+ verdict: outcome.verdict,
570
+ });
571
+ incMetric(
572
+ input.metricRegistry,
573
+ "crew.verification.verifier_spawns_total",
574
+ "LLM verifier spawns that still ran, labeled by the deciding layer",
575
+ { reason: decision.reason },
576
+ );
577
+ return { outcome, decision, promptContext: renderVerifyGateContext(outcome) };
578
+ }
579
+ }
580
+ // ── M1-7: NO package.json fallback ─────────────────────────────────────
581
+ // Checks come from the caller or the dispatch-pinned host spec ONLY —
582
+ // nothing here reads the worker-mutable task cwd. Zero resolvable checks
583
+ // ⇒ INCONCLUSIVE ⇒ escalate (fail-closed on absence, pre-R47-1 doctrine).
584
+ const checks = input.checks ?? input.hostSpec?.checks ?? [];
477
585
  const exec = createSubprocessVerifyGateExecutor();
478
586
  let outcome = await runVerifyGate(checks, exec, {
479
587
  cwd: input.cwd,
@@ -501,27 +609,42 @@ export async function runVerifierGatePreGate(input: VerifierGatePreGateInput): P
501
609
  { result: layer.result },
502
610
  );
503
611
  }
504
- let preGate: VerifierPreGateResult | undefined;
505
- if (outcome.verdict === "INCONCLUSIVE" && input.classifier?.enabled) {
506
- try {
507
- preGate = await classifyVerifierPreGate({
508
- enabled: true,
509
- modelRegistry: input.classifier.modelRegistry,
510
- classifierModel: input.classifier.classifierModel,
511
- verdict: {
512
- verdict: outcome.verdict,
513
- evidenceLines: outcome.evidenceLines,
514
- changedFiles: [],
515
- },
516
- eventsPath: input.classifier.eventsPath,
517
- runId: input.classifier.runId,
518
- ...(input.classifier.taskId !== undefined ? { taskId: input.classifier.taskId } : {}),
519
- metricRegistry: input.metricRegistry,
520
- });
521
- } catch {
522
- preGate = undefined; // soft: escalate to the LLM verifier
612
+ // ── M1-7 bookend: integrity AFTER a green gate (mirrors P1a) ───────────
613
+ // A PASS is only trustworthy if the manifests are STILL identical to the
614
+ // dispatch snapshot after the commands ran — a mid-gate tamper downgrades
615
+ // the verdict to INCONCLUSIVE so the LLM verifier spawns. This can only
616
+ // ever DOWNGRADE, never upgrade (the pre-check above already ruled out
617
+ // pre-existing drift, so any drift now appeared during the run).
618
+ if (input.hostSpec && outcome.verdict === "PASS") {
619
+ const drift = detectHostSpecDrift(input.cwd, input.hostSpec);
620
+ if (drift.length > 0) {
621
+ const note = `manifest-integrity: ${drift.join(", ")} changed during the gate run (host snapshot)`;
622
+ outcome = {
623
+ ...outcome,
624
+ verdict: "INCONCLUSIVE",
625
+ evidenceLines: [...outcome.evidenceLines, note],
626
+ integrityNote: note,
627
+ };
523
628
  }
524
629
  }
630
+ // ── M3-28 (R47-V2, fix-wave 6): the U6B classifier layer is NOT consulted ──
631
+ // The consult used to live here, guarded by `outcome.verdict === "INCONCLUSIVE"`
632
+ // and fed the literal gate verdict. That input can never be pass-shaped:
633
+ // `classifyVerifierPreGate` blocks only when `isPassShapedVerdict(...)` is true
634
+ // (classifier-pre-gate.ts — PASS vocabulary AND no failure vocabulary), and
635
+ // "INCONCLUSIVE" matches the failure vocabulary (…conclusive → no PASS match
636
+ // at all), so `action` was ALWAYS escalate_verdict_failed ⇒ blockVerifier
637
+ // false ⇒ the same spawn decision the no-classifier branch yields. Net effect
638
+ // of the consult: one model call, one task.verifier_pre_gate event and one
639
+ // decisions_total counter for a decision that could never differ.
640
+ // Wiring it "for real" is not possible from here: a state that says
641
+ // INCONCLUSIVE cannot be conclusive evidence of a pass, so there is nothing
642
+ // for the classifier to legitimately block on. Removed (spend without a
643
+ // decision, plus a misleading `classifier_*` reason label).
644
+ // `input.classifier` stays on the interface (deprecated) because the dispatch
645
+ // call site in task-runner.ts passes it; removing the field is a follow-up
646
+ // that must land with that call site.
647
+ const preGate: VerifierPreGateResult | undefined = undefined;
525
648
  const decision = resolveVerifierSpawnDecision(outcome, preGate);
526
649
  incMetric(input.metricRegistry, "crew.verification.verify_gate_runs_total", "Deterministic verify_gate executions by verdict", {
527
650
  verdict: outcome.verdict,
@@ -990,6 +990,62 @@ interface CoalescedAtomicWrite {
990
990
  }
991
991
  const MAX_FLUSH_RETRIES = 5;
992
992
  const pendingAtomicWrites = new Map<string, CoalescedAtomicWrite>();
993
+
994
+ /**
995
+ * R39-M1 (fix-wave 6, EP3): a coalesced write that exhausted MAX_FLUSH_RETRIES.
996
+ * Previously the flush RE-THREW from a `setTimeout` callback, which surfaced as
997
+ * an uncaughtException and killed the whole host process (exit 7) on a transient
998
+ * EACCES/EROFS/ENOSPC/EBUSY write fault — leaving the run stuck "running"
999
+ * forever. The write is now DROPPED and reported instead.
1000
+ */
1001
+ export interface CoalescedFlushFailure {
1002
+ filePath: string;
1003
+ error: string;
1004
+ retries: number;
1005
+ at: number;
1006
+ }
1007
+
1008
+ const exhaustedFlushFailures: CoalescedFlushFailure[] = [];
1009
+ const EXHAUSTED_FLUSH_FAILURES_MAX = 64;
1010
+ type CoalescedFlushExhaustedListener = (failure: CoalescedFlushFailure) => void;
1011
+ const coalescedFlushExhaustedListeners = new Set<CoalescedFlushExhaustedListener>();
1012
+
1013
+ /**
1014
+ * R39-M1: subscribe to coalesced-flush retry exhaustion (the degradation seam
1015
+ * a caller can use to mark the owning run degraded). Returns an unsubscribe fn.
1016
+ */
1017
+ export function onCoalescedFlushExhausted(listener: CoalescedFlushExhaustedListener): () => void {
1018
+ coalescedFlushExhaustedListeners.add(listener);
1019
+ return () => coalescedFlushExhaustedListeners.delete(listener);
1020
+ }
1021
+
1022
+ /** R39-M1: recently exhausted coalesced flushes (best-effort observability). */
1023
+ export function listExhaustedFlushFailures(): readonly CoalescedFlushFailure[] {
1024
+ return exhaustedFlushFailures;
1025
+ }
1026
+
1027
+ /** @internal test hook */
1028
+ export function __test__clearExhaustedFlushFailures(): void {
1029
+ exhaustedFlushFailures.length = 0;
1030
+ }
1031
+
1032
+ function recordExhaustedFlush(filePath: string, error: unknown, retries: number): void {
1033
+ const failure: CoalescedFlushFailure = {
1034
+ filePath,
1035
+ error: error instanceof Error ? error.message : String(error),
1036
+ retries,
1037
+ at: Date.now(),
1038
+ };
1039
+ exhaustedFlushFailures.push(failure);
1040
+ if (exhaustedFlushFailures.length > EXHAUSTED_FLUSH_FAILURES_MAX) exhaustedFlushFailures.shift();
1041
+ for (const listener of coalescedFlushExhaustedListeners) {
1042
+ try {
1043
+ listener(failure);
1044
+ } catch (listenerError) {
1045
+ logInternalError("atomic-write.coalesced-flush-listener", listenerError, filePath, "warn");
1046
+ }
1047
+ }
1048
+ }
993
1049
  const DEFAULT_ATOMIC_COALESCE_MS = 50;
994
1050
  /** Issue 2 fix: generation counter for coalesced writes */
995
1051
  let writeGeneration = 0;
@@ -1093,10 +1149,22 @@ function flushOnePendingAtomicWrite(filePath: string): void {
1093
1149
  const entryToRetry = current ?? entry;
1094
1150
  entryToRetry.retryCount++;
1095
1151
  if (entryToRetry.retryCount >= MAX_FLUSH_RETRIES) {
1096
- // Max retries exceeded - remove entry and propagate error to callers
1152
+ // R39-M1 (fix-wave 6, EP3): MAX retries exhausted — DEGRADE, never
1153
+ // re-throw. This code runs inside a setTimeout callback (coalesce +
1154
+ // backoff timers) as well as inside flushPendingAtomicWrites(); a
1155
+ // throw from the timer context surfaces as an uncaughtException and
1156
+ // killed the WHOLE host process (exit 7) on a transient write fault,
1157
+ // leaving the run "running" forever. Drop the buffered entry, record +
1158
+ // report the degradation, and keep the host alive.
1097
1159
  pendingAtomicWrites.delete(filePath);
1098
- // Re-throw so callers can handle the persistent failure
1099
- throw error;
1160
+ recordExhaustedFlush(filePath, error, entryToRetry.retryCount);
1161
+ logInternalError(
1162
+ "atomic-write.coalesced-flush",
1163
+ error instanceof Error ? error : new Error(String(error)),
1164
+ `${filePath} — dropped after ${entryToRetry.retryCount} retries (degraded, host stays alive)`,
1165
+ "error",
1166
+ );
1167
+ return;
1100
1168
  }
1101
1169
  // Exponential backoff: base delay * 2^(retryCount-1), capped at 30 seconds
1102
1170
  const backoffMs = Math.min(30000, entryToRetry.coalesceMs * 2 ** (entryToRetry.retryCount - 1));
@@ -1200,7 +1268,8 @@ export function peekPendingCoalescedWrite<T>(filePath: string): T | undefined {
1200
1268
  * flushed file keeps its data fsync but defers the dir fsync, and after the
1201
1269
  * serial loop completes (all renames done → R16-B1 ordering) ONE fsync is
1202
1270
  * issued per distinct pending dir. The trailing fsync lives in the `finally`
1203
- * so a mid-drain throw (a flush that exhausted MAX_FLUSH_RETRIES) still
1271
+ * so a mid-drain throw (e.g. a reporting-hook failure, or any other error the
1272
+ * flush's catch block re-raises) still
1204
1273
  * makes the already-renamed files crash-durable. Scoped flushes and
1205
1274
  * coalesce-timer flushes keep the immediate per-file dir fsync.
1206
1275
  */
@@ -151,6 +151,13 @@ export const TEAM_EVENT_TYPES = [
151
151
  "delegate.completed",
152
152
  "delegate.timed_out",
153
153
  "delegate.rolled_up",
154
+ // M3-5/R43-N3 (2026-10-19 fix-wave 6): a run/owner abort reached an
155
+ // in-flight grandchild (broker delegate abort registry → spawner signal).
156
+ "delegate.cancelled",
157
+ // M3-5/R43-N4: the parent task was already terminal when the grandchild
158
+ // settled, so the fenced result was redirected to the RUN inbox instead of
159
+ // an unreadable parent mailbox (delegate/delegate-finalize.ts).
160
+ "delegate.rerouted",
154
161
  // Budget tracking events
155
162
  "budget.initialized",
156
163
  "budget.warning",
@@ -96,12 +96,14 @@ export interface MailboxMessage {
96
96
  export interface MailboxDeliveryState {
97
97
  messages: Record<string, MailboxMessageStatus>;
98
98
  updatedAt: string;
99
- /** U5 (2026-10-10): durable (direction, requestId) delivered index —
100
- * key `${direction}:${requestId}` → id of the first message row that was
101
- * consumed under it. Written by the worker inbox pickup
102
- * (recordMailboxRequestIdDelivery) and read back after a restart so a
103
- * redelivered/resent row is dropped (exactly-once pickup). Optional so
104
- * pre-U5 delivery.json files parse unchanged. */
99
+ /** U5 (2026-10-10): durable delivered-requestId index —
100
+ * key `${direction}:${recipientTaskId}:${requestId}` → id of the message row
101
+ * that recipient consumed under it (M3-9/R33-1 recipient scoping: one group
102
+ * fan-out reuses a single requestId for every recipient, so a run-scoped key
103
+ * let the first acker starve all siblings — probe 1/0/0). Written by the
104
+ * worker inbox pickup (recordMailboxRequestIdDelivery) and read back after a
105
+ * restart so a redelivered/resent row is dropped (exactly-once pickup).
106
+ * Optional so pre-U5 delivery.json files parse unchanged. */
105
107
  requestIds?: Record<string, string>;
106
108
  }
107
109
 
@@ -738,8 +740,8 @@ function pruneDeliveryMessages(manifest: TeamRunManifest, state: MailboxDelivery
738
740
  state.messages = Object.fromEntries(remaining.filter(([id]) => keptIds.has(id)));
739
741
  }
740
742
 
741
- /** U5: bound the durable (direction, requestId) delivered index. Keys insert
742
- * chronologically (first delivery under that requestId), so eviction keeps
743
+ /** U5: bound the durable (direction, recipient, requestId) delivered index.
744
+ * Keys insert chronologically (first delivery under that key), so eviction keeps
743
745
  * the most recent MAX_DELIVERY_MESSAGES keys — dropping a key can only
744
746
  * re-enable delivery of an extremely old duplicate row, the same bounded
745
747
  * risk the messages map already accepts. */
@@ -1107,22 +1109,49 @@ export async function appendMailboxMessageIdempotentAsync(
1107
1109
  /** U5: durable receiver-side dedup record — the worker inbox pickup marks the
1108
1110
  * messages it surfaced as consumed: `messages[id] = "acknowledged"` (the F09
1109
1111
  * durable record; the inbox line itself never flips status) plus the
1110
- * `${direction}:${requestId}` index. Both survive restart, so a later poll
1111
- * (fresh process, empty seen-set) still drops them. */
1112
+ * recipient-scoped `${direction}:${taskId}:${requestId}` index. Both survive
1113
+ * restart, so a later poll (fresh process, empty seen-set) still drops them. */
1112
1114
  export interface MailboxRequestIdDeliveryRecord {
1113
1115
  direction: MailboxDirection;
1114
1116
  messageId: string;
1117
+ /** The RECIPIENT mailbox this pickup consumed (the polling task).
1118
+ * M3-9 / R33-1: part of the index key — see {@link requestIdDeliveryKey}.
1119
+ * Undefined = the run-level mailbox. */
1120
+ taskId?: string;
1115
1121
  /** Optional: rows without a requestId (pre-U5 legacy) only get the
1116
1122
  * by-id acknowledged record. */
1117
1123
  requestId?: string;
1118
1124
  }
1119
1125
 
1126
+ /**
1127
+ * M3-9 / R33-1 — THE delivery-index key: `${direction}:${recipient}:${requestId}`.
1128
+ *
1129
+ * Round 33 probe (fan-out delivered 1/0/0): a group send mints ONE requestId
1130
+ * (crew-broker handleMsgSend) and reuses it for every recipient. The pickup
1131
+ * index was run-scoped (`${direction}:${requestId}`), so the FIRST recipient to
1132
+ * ack wrote the key and every SIBLING recipient was then starved at its own
1133
+ * pickup — one delivered, the rest silently dropped (and the
1134
+ * `mailbox.pickup_deduped` audit logged it as an expected duplicate).
1135
+ * Recipient scoping restores per-recipient exactly-once: each recipient's
1136
+ * mailbox gets its own entry for the same requestId, so all N copies deliver
1137
+ * (once each).
1138
+ *
1139
+ * Migration: keys written before this change use the legacy two-part shape and
1140
+ * are no longer matched. Safe — the same-row case stays covered by the
1141
+ * per-message `messages[id]="acknowledged"` record, and the send-side
1142
+ * idempotent append prevents a resend from adding a second row at all.
1143
+ */
1144
+ export function requestIdDeliveryKey(direction: MailboxDirection, recipientTaskId: string | undefined, requestId: string): string {
1145
+ return `${direction}:${recipientTaskId ?? "-"}:${requestId}`;
1146
+ }
1147
+
1120
1148
  export function recordMailboxRequestIdDelivery(manifest: TeamRunManifest, records: MailboxRequestIdDeliveryRecord[]): MailboxDeliveryState {
1121
1149
  return withFileLockSync(deliveryFile(manifest, true), () => {
1122
1150
  const delivery = readDeliveryState(manifest);
1123
1151
  for (const record of records) {
1124
1152
  delivery.messages[record.messageId] = "acknowledged";
1125
- if (record.requestId) (delivery.requestIds ??= {})[`${record.direction}:${record.requestId}`] = record.messageId;
1153
+ if (record.requestId)
1154
+ (delivery.requestIds ??= {})[requestIdDeliveryKey(record.direction, record.taskId, record.requestId)] = record.messageId;
1126
1155
  }
1127
1156
  delivery.updatedAt = new Date().toISOString();
1128
1157
  // Pickup is terminal for that message on the worker side — full durability
@@ -167,7 +167,9 @@ export interface CompactionResult {
167
167
  * 3. Atomically write (atomicWriteFile handles temp-file + rename)
168
168
  * 4. Re-read to detect events appended during the window
169
169
  * 5. If events were lost, append them
170
- * 6. Return compaction stats
170
+ * 6. Bump the `.gen` generation sidecar (EP4/R32-M1) so byte-offset cursors
171
+ * echoing the pre-compaction generation re-read instead of resuming mid-file
172
+ * 7. Return compaction stats
171
173
  */
172
174
  export function compactEventLog(
173
175
  eventsPath: string,
@@ -404,6 +406,21 @@ export function applyCompactionUnlocked(
404
406
  logInternalError("event-log-rotation.compact", err, `eventsPath=${eventsPath}`);
405
407
  return undefined;
406
408
  }
409
+ // EP4 (R32-M1 ≡ R4-N1/FS10, fix-wave 6): compaction REWRITES the live log at
410
+ // the same path (atomicWriteFile → temp+rename, a NEW inode) and drops the
411
+ // events before the kept window. Without a generation bump, a byte-offset
412
+ // cursor reader echoing the pre-compaction `generation` (readEventsCursor's
413
+ // cache-key, cursor.ts) still matches the live sidecar and resumes from its
414
+ // old offset — which now points into the MIDDLE of the rewritten file — so
415
+ // it silently skips the kept-window head it never delivered. The `shrank`
416
+ // fallback does NOT cover this: it only fires when the file is shorter than
417
+ // the caller's offset. Bump here so EVERY compaction path (compactEventLog /
418
+ // compactPreparedEventLog / applyCompactionAppendGuarded + the inline call
419
+ // sites in event-log.ts) invalidates stale cursors exactly like rotation
420
+ // (rotateEventLogUnlocked). A no-op compaction never reaches this point —
421
+ // prepareCompaction returns undefined when originalCount <= compactToCount —
422
+ // so the generation stays stable when nothing was truncated.
423
+ bumpGenerationUnlocked(eventsPath);
407
424
  // U22 fix round: the compacted file is NEW content at the same path — every
408
425
  // old poison marker is meaningless (its byteOffset now names an arbitrary
409
426
  // line of the rewritten file). Drop the sidecar so a stale marker can never
@@ -466,9 +483,10 @@ export function applyCompactionUnlocked(
466
483
  /**
467
484
  * R-03: Path of the generation sidecar for an events file. Mirrors the
468
485
  * `.seq` sidecar pattern. The generation is bumped every time the events
469
- * file is rotated (truncated + archived), so readers holding a byte-offset
470
- * cursor can detect their offset is stale and re-read from the start of the
471
- * new file instead of missing the post-rotation events.
486
+ * file is REWRITTEN in place with different content — rotation (truncated +
487
+ * archived) or compaction (prefix dropped, kept window rewritten) — so
488
+ * readers holding a byte-offset cursor can detect their offset is stale and
489
+ * re-read from the start of the new file instead of missing events.
472
490
  */
473
491
  export function generationPath(eventsPath: string): string {
474
492
  return `${eventsPath}.gen`;
@@ -500,11 +518,15 @@ export function currentGeneration(eventsPath: string): number {
500
518
 
501
519
  /**
502
520
  * R-03: Atomically bump and persist the generation sidecar. Called by
503
- * `rotateEventLogUnlocked` after truncating the events file. Assumes the
521
+ * `rotateEventLogUnlocked` after truncating the events file, by
522
+ * `applyCompactionUnlocked` after rewriting it (EP4/R32-M1), and by
523
+ * `quarantinePoisonedEventsFile` after renaming the poisoned file away
524
+ * (EP4/FS10 residual, wave-6 M4-R2 — a rename is the same file-instance
525
+ * change at the SAME path, so byte-offset cursors must reset). Assumes the
504
526
  * caller already holds the event-log lock (or accepts the unlocked race).
505
527
  * Returns the new generation value.
506
528
  */
507
- function bumpGenerationUnlocked(eventsPath: string): number {
529
+ export function bumpGenerationUnlocked(eventsPath: string): number {
508
530
  const next = currentGeneration(eventsPath) + 1;
509
531
  try {
510
532
  atomicWriteFile(generationPath(eventsPath), String(next));
@@ -63,6 +63,7 @@
63
63
  */
64
64
  import * as fs from "node:fs";
65
65
  import type { JsonlParseFailure } from "../../utils/incremental-reader.ts";
66
+ import { bumpGenerationUnlocked } from "./event-log-rotation.ts";
66
67
 
67
68
  export type PoisonFailure = JsonlParseFailure;
68
69
 
@@ -231,7 +232,12 @@ function persistPoisonMarkers(
231
232
 
232
233
  /** Last resort — quarantineCorruptFile pattern (manifest-io.ts:22-28): rename
233
234
  * the events file to `<name>.corrupt-<ts>` (preserved for debugging) and drop
234
- * the sidecar so a future log at the same path starts clean. Best-effort. */
235
+ * the sidecar so a future log at the same path starts clean. Best-effort.
236
+ * EP4/FS10 residual (wave-6 M4-R2): the rename is a file-instance change at
237
+ * the SAME path, so bump the generation sidecar (shared helper from M2-4 /
238
+ * event-log-rotation) — a byte-offset cursor holding the old generation then
239
+ * detects the change and resets instead of resuming past the (now absent)
240
+ * file's old offsets. */
235
241
  function quarantinePoisonedEventsFile(eventsPath: string): void {
236
242
  const quarantinePath = `${eventsPath}.corrupt-${Date.now()}`;
237
243
  try {
@@ -240,6 +246,15 @@ function quarantinePoisonedEventsFile(eventsPath: string): void {
240
246
  console.error(`[poison-marker] quarantine rename failed for ${eventsPath} (file left in place):`, error);
241
247
  return;
242
248
  }
249
+ try {
250
+ // EP4/FS10: bump .gen AFTER a successful rename so readers see the new
251
+ // file instance. Upstream imported-by-us module (event-log-rotation)
252
+ // already imports poisonMarkerPath from here — a cycle, but both sides
253
+ // are hoisted function declarations, so the binding is live at call time.
254
+ bumpGenerationUnlocked(eventsPath);
255
+ } catch {
256
+ /* best-effort: a failed gen bump must not fail the quarantine */
257
+ }
243
258
  try {
244
259
  fs.rmSync(poisonMarkerPath(eventsPath), { force: true });
245
260
  } catch {