pi-crew 0.11.21 → 0.11.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/index.mjs +537 -526
  3. package/package.json +2 -1
  4. package/src/errors.ts +14 -1
  5. package/src/extension/async-notifier.ts +42 -17
  6. package/src/extension/cross-extension-rpc.ts +42 -5
  7. package/src/extension/notification-router.ts +15 -2
  8. package/src/extension/registration/lifecycle-handlers.ts +36 -13
  9. package/src/extension/registration/runtime-cleanup.ts +17 -0
  10. package/src/extension/registration/wire-cross-extension.ts +26 -0
  11. package/src/extension/rpc-hmac.ts +53 -2
  12. package/src/extension/team-tool/cancel.ts +34 -21
  13. package/src/extension/team-tool/dispatch/index.ts +6 -5
  14. package/src/extension/team-tool/dispatch/status.ts +42 -2
  15. package/src/extension/team-tool.ts +24 -2
  16. package/src/prompt/inbox-poll.ts +77 -25
  17. package/src/prompt/message-tool.ts +62 -4
  18. package/src/prompt/prompt-runtime.ts +82 -32
  19. package/src/runtime/broker/crew-broker-client.ts +74 -16
  20. package/src/runtime/broker/crew-broker.ts +111 -77
  21. package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
  22. package/src/runtime/broker/wait-status-cache.ts +14 -5
  23. package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
  24. package/src/runtime/child-pi/child-pi.ts +20 -2
  25. package/src/runtime/child-pi/mock-fixtures.ts +23 -0
  26. package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
  27. package/src/runtime/compaction/important-line-classifier.ts +85 -11
  28. package/src/runtime/compaction/tool-output-pruner.ts +35 -11
  29. package/src/runtime/crew-agent-records.ts +7 -1
  30. package/src/runtime/delivery-coordinator.ts +30 -0
  31. package/src/runtime/dwf-state-store.ts +41 -7
  32. package/src/runtime/finalize-run.ts +13 -2
  33. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
  34. package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
  35. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  36. package/src/runtime/live-session/embedded-tier.ts +3 -1
  37. package/src/runtime/live-session/live-session-runtime.ts +54 -1
  38. package/src/runtime/model/model-fallback.ts +23 -1
  39. package/src/runtime/output/pi-json-output.ts +26 -1
  40. package/src/runtime/policy-engine.ts +18 -0
  41. package/src/runtime/process/zombie-scanner.ts +78 -14
  42. package/src/runtime/process-status.ts +6 -1
  43. package/src/runtime/recovery/crash-recovery.ts +170 -28
  44. package/src/runtime/rpc/rpc-worker.ts +44 -1
  45. package/src/runtime/scheduling/run-coalesced-task-group.ts +64 -3
  46. package/src/runtime/scheduling/scheduler.ts +74 -11
  47. package/src/runtime/scratchpad/engine.ts +26 -3
  48. package/src/runtime/settings-store.ts +22 -2
  49. package/src/runtime/surface/tmux-provider.ts +24 -3
  50. package/src/runtime/task-health.ts +16 -1
  51. package/src/runtime/task-output-context.ts +143 -71
  52. package/src/runtime/task-packet.ts +70 -17
  53. package/src/runtime/task-runner/child-executor.ts +27 -0
  54. package/src/runtime/task-runner/output-splitter.ts +79 -16
  55. package/src/runtime/task-runner/post-execution.ts +75 -15
  56. package/src/runtime/task-runner/state-helpers.ts +92 -2
  57. package/src/runtime/usage-tracker.ts +11 -0
  58. package/src/runtime/verification/green-contract.ts +17 -0
  59. package/src/runtime/verification/verification-gates.ts +24 -6
  60. package/src/runtime/workspace-lock.ts +53 -0
  61. package/src/schema/team-tool-schema.ts +10 -4
  62. package/src/state/coordination/mailbox.ts +133 -38
  63. package/src/state/decision-ledger.ts +56 -8
  64. package/src/state/event-log/event-log-rotation.ts +83 -5
  65. package/src/state/event-log/event-log.ts +181 -12
  66. package/src/state/stores/active-run-registry.ts +10 -3
  67. package/src/state/types.ts +9 -1
  68. package/src/ui/agents-jobs-browser.ts +22 -8
  69. package/src/ui/dashboard-panes/agents-pane.ts +5 -2
  70. package/src/ui/events-state-source.ts +67 -28
  71. package/src/ui/inline-panel/agent-pane.ts +13 -6
  72. package/src/ui/inline-panel/agent-transcript.ts +33 -21
  73. package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
  74. package/src/ui/inline-panel/index.ts +21 -1
  75. package/src/ui/live-run-sidebar.ts +14 -3
  76. package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
  77. package/src/ui/powerbar-publisher.ts +6 -3
  78. package/src/ui/rail.ts +7 -1
  79. package/src/ui/render-scheduler.ts +53 -1
  80. package/src/ui/run-action-dispatcher.ts +68 -20
  81. package/src/ui/status-colors.ts +9 -0
  82. package/src/utils/env-filter.ts +8 -0
@@ -213,7 +213,18 @@ export async function applyRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionC
213
213
  return;
214
214
  }
215
215
 
216
- const reset = new Set(plan.resumableTasks);
216
+ // M-4 (2026-10-16): the plan was computed from a pre-hook (stale) snapshot.
217
+ // Filter the reset set to tasks whose FRESH status is still non-terminal — a
218
+ // task that completed/failed/... during the run_recovery hook gap must keep
219
+ // its terminal status and result (mirror of the requeue-eligibility filter
220
+ // in applyReconcileAtOpenFastPath). Terminal tasks pass through untouched
221
+ // below, so a stale plan can never requeue finished work.
222
+ const reset = new Set(
223
+ plan.resumableTasks.filter((id) => {
224
+ const task = fresh.tasks.find((t) => t.id === id);
225
+ return task !== undefined && !isTerminalTask(task);
226
+ }),
227
+ );
217
228
  const tasks = fresh.tasks.map((task) =>
218
229
  reset.has(task.id)
219
230
  ? {
@@ -238,10 +249,12 @@ export async function applyRecoveryPlan(plan: RecoveryPlan, ctx: Pick<ExtensionC
238
249
  event: {
239
250
  type: "crew.run.resumed",
240
251
  runId: plan.runId,
241
- message: `Recovered ${plan.resumableTasks.length} interrupted task(s).`,
252
+ message: `Recovered ${reset.size} interrupted task(s).`,
242
253
  data: {
243
254
  recoveredFromSeq: plan.lastEventSeq,
244
- resumableTasks: plan.resumableTasks,
255
+ // M-4: report what was ACTUALLY reset from the fresh snapshot
256
+ // (terminal tasks filtered out), not what the stale plan asked for.
257
+ resumableTasks: [...reset],
245
258
  },
246
259
  },
247
260
  };
@@ -349,15 +362,10 @@ export function cancelOrphanedRuns(
349
362
  const loaded = loadRunManifestById(cwd, manifest.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
350
363
  if (!loaded) continue;
351
364
 
352
- const hasRecentActivity = loaded.tasks.some((task) => {
353
- if (task.status !== "running" && task.status !== "waiting") return false;
354
- const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
355
- if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= staleThresholdMs) return true;
356
- const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
357
- return Number.isFinite(activityAt) && now - activityAt <= staleThresholdMs;
358
- });
359
-
360
- if (hasRecentActivity) {
365
+ // R7-1 (2026-10-16): the inline predicate moved to the shared
366
+ // `hasRecentTaskActivity` helper so purgeStaleActiveRunIndex step 6 can
367
+ // honor the same task-level life evidence this path has always honored.
368
+ if (hasRecentTaskActivity(loaded.tasks, now, staleThresholdMs)) {
361
369
  skipped.push(manifest.runId);
362
370
  continue;
363
371
  }
@@ -385,6 +393,27 @@ export function cancelOrphanedRuns(
385
393
  return;
386
394
  }
387
395
 
396
+ // R7-6 (2026-10-16): status is still "running", but that alone is not
397
+ // revalidation — re-derive EVERY liveness input the cancel decision was
398
+ // built on from this FRESH snapshot (manifest.updatedAt / heartbeat.json
399
+ // mtime / async pid / task heartbeats). A `team resume` that adopted the
400
+ // run between the pre-lock scan and this lock rewrote exactly those inputs;
401
+ // cancelling here would false-cancel a run that is alive again (and then
402
+ // terminate its live agents).
403
+ if (hasFreshLivenessEvidence(fresh, now, staleThresholdMs)) {
404
+ // U2 (2026-10-10): SYNC append — cancelOrphanedRuns is a sync API with no
405
+ // await boundary to flush an async append.
406
+ appendEvent(loaded.manifest.eventsPath, {
407
+ type: "crew.run.orphan_skip",
408
+ runId: manifest.runId,
409
+ message:
410
+ "Skipped orphan cancellation: fresh liveness evidence observed under the run lock (run re-adopted or still active)",
411
+ data: { reason: "fresh_liveness_under_lock" },
412
+ });
413
+ skipped.push(manifest.runId);
414
+ return;
415
+ }
416
+
388
417
  const now_iso = new Date(now).toISOString();
389
418
  const repairedTasks = fresh.tasks.map((task) => {
390
419
  if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
@@ -516,6 +545,48 @@ function hasRecentLifeEvidence(
516
545
  return false;
517
546
  }
518
547
 
548
+ /**
549
+ * R7-1 (2026-10-16): TASK-level life evidence — true when any running/waiting
550
+ * task has a live heartbeat or agent progress within `staleThresholdMs`.
551
+ * A DWF-foreground run (run.ts → runDynamicWorkflow) writes NO team-level
552
+ * heartbeat.json and never touches manifest.updatedAt while a long subagent
553
+ * step executes, so a fresh task heartbeat can be the ONLY signal proving the
554
+ * run is alive. Extracted from cancelOrphanedRuns so the purge path
555
+ * (purgeStaleActiveRunIndex step 6) honors exactly the same evidence.
556
+ */
557
+ function hasRecentTaskActivity(tasks: TeamTaskState[], now: number, staleThresholdMs: number): boolean {
558
+ return tasks.some((task) => {
559
+ if (task.status !== "running" && task.status !== "waiting") return false;
560
+ const heartbeatAt = task.heartbeat?.lastSeenAt ? new Date(task.heartbeat.lastSeenAt).getTime() : Number.NaN;
561
+ if (task.heartbeat?.alive !== false && Number.isFinite(heartbeatAt) && now - heartbeatAt <= staleThresholdMs) return true;
562
+ const activityAt = task.agentProgress?.lastActivityAt ? new Date(task.agentProgress.lastActivityAt).getTime() : Number.NaN;
563
+ return Number.isFinite(activityAt) && now - activityAt <= staleThresholdMs;
564
+ });
565
+ }
566
+
567
+ /**
568
+ * R7-6 (2026-10-16): under-lock liveness revalidation. Proving
569
+ * `status === "running"` on the in-lock re-read is NOT enough: a concurrent
570
+ * `team resume` that adopted the run between the pre-lock scan and lock
571
+ * acquisition rewrote the very liveness inputs the cancel decision was built
572
+ * on. Re-derive them from the FRESH snapshot — any ONE of these is sufficient
573
+ * evidence the run came back to life and the cancel must abort:
574
+ * - fresh on-disk `manifest.updatedAt` and/or team `heartbeat.json` mtime
575
+ * (the same corroboration `hasRecentLifeEvidence` applies pre-lock),
576
+ * - a fresh/repopulated async worker PID that is ALIVE (re-dispatch),
577
+ * - fresh task heartbeats / agent progress (DWF-foreground evidence).
578
+ */
579
+ function hasFreshLivenessEvidence(
580
+ fresh: { manifest: TeamRunManifest; tasks: TeamTaskState[] },
581
+ now: number,
582
+ staleThresholdMs: number,
583
+ ): boolean {
584
+ if (hasRecentLifeEvidence(fresh.manifest, fresh.manifest.updatedAt, now, staleThresholdMs)) return true;
585
+ const pid = fresh.manifest.async?.pid;
586
+ if (pid !== undefined && checkProcessLiveness(pid).alive) return true;
587
+ return hasRecentTaskActivity(fresh.tasks, now, staleThresholdMs);
588
+ }
589
+
519
590
  /**
520
591
  * Purge the global active-run-index of entries whose manifest is no longer active.
521
592
  *
@@ -628,6 +699,7 @@ export function purgeStaleActiveRunIndex(
628
699
  // repaired-tasks write and the manifest-status flip. terminateLiveAgentsForRun
629
700
  // is fire-and-forget so it stays OUTSIDE the lock to avoid blocking the
630
701
  // critical section on async IPC.
702
+ let abortedFreshLiveness = false;
631
703
  try {
632
704
  const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
633
705
  if (fullLoaded) {
@@ -637,6 +709,23 @@ export function purgeStaleActiveRunIndex(
637
709
  const fresh = loadRunManifestById(entry.cwd, entry.runId);
638
710
  if (!fresh) return;
639
711
  if (fresh.manifest.status !== "running") return;
712
+ // R7-6 (2026-10-16): under-lock liveness revalidation — the pre-lock
713
+ // gate used inputs read BEFORE the lock; a concurrent `team resume`
714
+ // adoption may have rewritten them in that window (fresh
715
+ // manifest.updatedAt / re-populated live pid / fresh task heartbeats).
716
+ // Fresh evidence → abort WITHOUT cancelling and KEEP the registry
717
+ // entry (it is the live-claim signal G12 relies on).
718
+ if (hasFreshLivenessEvidence(fresh, now, staleThresholdMs)) {
719
+ abortedFreshLiveness = true;
720
+ appendEvent(fresh.manifest.eventsPath, {
721
+ type: "crew.run.orphan_skip",
722
+ runId: entry.runId,
723
+ message:
724
+ "Skipped purge cancellation: fresh liveness evidence observed under the run lock (run re-adopted or still active)",
725
+ data: { reason: "fresh_liveness_under_lock" },
726
+ });
727
+ return;
728
+ }
640
729
  const now_iso = new Date(now).toISOString();
641
730
  const repairedTasks = fresh.tasks.map((task) => {
642
731
  if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
@@ -659,18 +748,29 @@ export function purgeStaleActiveRunIndex(
659
748
  }
660
749
  updateRunStatus(fresh.manifest, "cancelled", "Orphaned run: worker process dead and no recent activity");
661
750
  });
662
- void terminateLiveAgentsForRun(
663
- fullLoaded.manifest.runId,
664
- "cancelled",
665
- appendEvent,
666
- fullLoaded.manifest.eventsPath,
667
- ).catch((error) =>
668
- logInternalError("crash-recovery.pid-dead.terminate", error, `runId=${fullLoaded.manifest.runId}`, "warn"),
669
- );
751
+ // R7-6: only terminate live agents when the cancel actually landed —
752
+ // an aborted (fresh-evidence) run is alive; terminating it would kill
753
+ // the run we just decided to keep.
754
+ if (!abortedFreshLiveness) {
755
+ void terminateLiveAgentsForRun(
756
+ fullLoaded.manifest.runId,
757
+ "cancelled",
758
+ appendEvent,
759
+ fullLoaded.manifest.eventsPath,
760
+ ).catch((error) =>
761
+ logInternalError("crash-recovery.pid-dead.terminate", error, `runId=${fullLoaded.manifest.runId}`, "warn"),
762
+ );
763
+ }
670
764
  }
671
765
  } catch {
672
766
  // Best-effort manifest cleanup
673
767
  }
768
+ // R7-6: a fresh-evidence abort means the run is alive again — KEEP the
769
+ // registry entry (live-claim signal) instead of unregistering it.
770
+ if (abortedFreshLiveness) {
771
+ kept.push(entry.runId);
772
+ continue;
773
+ }
674
774
  unregisterActiveRun(entry.runId);
675
775
  purged.push(entry.runId);
676
776
  continue;
@@ -681,14 +781,44 @@ export function purgeStaleActiveRunIndex(
681
781
  // manifest was never updated to a terminal status after the worker exited.
682
782
  // Uses the same life-evidence corroboration as condition 5; the stateRoot is
683
783
  // kept on cancel so the run stays queryable/resumable with diagnostics.
784
+ // R7-1 (2026-10-16): step 6 now also honors TASK-level life evidence
785
+ // (hasRecentTaskActivity), making it symmetric with cancelOrphanedRuns.
786
+ // A DWF-foreground run (run.ts → runDynamicWorkflow) writes neither
787
+ // heartbeat.json nor manifest.updatedAt while a long subagent step
788
+ // executes — only its task heartbeats prove liveness. Without this check
789
+ // a subagent step longer than staleThresholdMs was false-cancelled
790
+ // (tasks cancelled + run flipped terminal + registry entry dropped)
791
+ // while the worker was still running; a later `team resume` then
792
+ // double-dispatched the live run.
684
793
  if (manifest?.status === "running" && manifest.async === undefined) {
685
794
  if (!hasRecentLifeEvidence(entry, manifest.updatedAt, now, staleThresholdMs)) {
795
+ let cancelledUnderLock = false;
796
+ let abortedFreshLiveness = false;
686
797
  try {
687
798
  const fullLoaded = loadRunManifestById(entry.cwd, entry.runId);
688
799
  if (fullLoaded && fullLoaded.manifest.status === "running") {
689
800
  withRunLockSync(fullLoaded.manifest, () => {
690
801
  const fresh = loadRunManifestById(entry.cwd, entry.runId);
691
802
  if (fresh?.manifest.status !== "running") return;
803
+ // R7-6 (2026-10-16): under-lock liveness revalidation — the pre-lock
804
+ // gate used inputs read BEFORE the lock; a concurrent `team resume`
805
+ // adoption may have rewritten them in that window (fresh
806
+ // manifest.updatedAt / re-populated live pid / fresh task heartbeats).
807
+ // Fresh evidence → abort WITHOUT cancelling and KEEP the registry
808
+ // entry (it is the live-claim signal G12 relies on). This ALSO covers
809
+ // R7-1’s task-heartbeat evidence: hasFreshLivenessEvidence consults
810
+ // hasRecentTaskActivity on this FRESH in-lock snapshot.
811
+ if (hasFreshLivenessEvidence(fresh, now, staleThresholdMs)) {
812
+ abortedFreshLiveness = true;
813
+ appendEvent(fresh.manifest.eventsPath, {
814
+ type: "crew.run.orphan_skip",
815
+ runId: entry.runId,
816
+ message:
817
+ "Skipped purge cancellation: fresh liveness evidence observed under the run lock (run re-adopted or still active)",
818
+ data: { reason: "fresh_liveness_under_lock" },
819
+ });
820
+ return;
821
+ }
692
822
  const now_iso = new Date(now).toISOString();
693
823
  const repairedTasks = fresh.tasks.map((task) => {
694
824
  if (task.status === "running" || task.status === "queued" || task.status === "waiting") {
@@ -716,19 +846,31 @@ export function purgeStaleActiveRunIndex(
716
846
  Math.round(staleThresholdMs / 60000) +
717
847
  " minutes",
718
848
  );
849
+ cancelledUnderLock = true;
719
850
  });
720
- void terminateLiveAgentsForRun(
721
- fullLoaded.manifest.runId,
722
- "cancelled",
723
- appendEvent,
724
- fullLoaded.manifest.eventsPath,
725
- ).catch((error) =>
726
- logInternalError("crash-recovery.pid-dead.terminate", error, `runId=${fullLoaded.manifest.runId}`, "warn"),
727
- );
851
+ // R7-1/R7-6: only terminate live agents when the cancel actually landed
852
+ // under the lock — an aborted (fresh-evidence) run is alive; terminating
853
+ // it would kill the run we just decided to keep.
854
+ if (cancelledUnderLock) {
855
+ void terminateLiveAgentsForRun(
856
+ fullLoaded.manifest.runId,
857
+ "cancelled",
858
+ appendEvent,
859
+ fullLoaded.manifest.eventsPath,
860
+ ).catch((error) =>
861
+ logInternalError("crash-recovery.pid-dead.terminate", error, `runId=${fullLoaded.manifest.runId}`, "warn"),
862
+ );
863
+ }
728
864
  }
729
865
  } catch {
730
866
  // Best-effort
731
867
  }
868
+ // R7-1/R7-6: a fresh-evidence abort means the run is alive again — KEEP
869
+ // the registry entry (live-claim signal) instead of unregistering it.
870
+ if (abortedFreshLiveness) {
871
+ kept.push(entry.runId);
872
+ continue;
873
+ }
732
874
  unregisterActiveRun(entry.runId);
733
875
  purged.push(entry.runId);
734
876
  continue;
@@ -127,6 +127,18 @@ export function resolveRpcTurnTimeoutMs(read: (name: string) => number | undefin
127
127
  interface SettleTracking {
128
128
  lastAssistantText: string | undefined;
129
129
  settled: boolean;
130
+ /** DR-01 P0 completeness: an assistant `message_end` was observed at all
131
+ * (guards the empty-settle stamp when a session produced no assistant
132
+ * turn — spawn-level failures have their own error channels). */
133
+ sawAssistantMessageEnd: boolean;
134
+ /** DR-01 P0 completeness: true when the LAST assistant `message_end` had
135
+ * non-empty text — mirrors ParsedPiJsonOutput.settledWithoutAnswer for
136
+ * the RPC transport, which has NO transcript file (stdio-only per README
137
+ * "Limitations"), so the attemptErrorFor settledWithoutAnswer branch
138
+ * cannot see this path. Upstream 429 exhaustion ends the turn with EMPTY
139
+ * assistant messages while lastAssistantText keeps an early pre-outage
140
+ * sentence — the same "finalText lies" shape as the stdio false-green. */
141
+ lastAssistantMessageEndHadText: boolean;
130
142
  /**
131
143
  * DR5 double-execution guard: true once ANY session event other than
132
144
  * `agent_settled` (message_start / message_end / extension_error / …)
@@ -165,7 +177,13 @@ function extractAssistantText(message: unknown): string | undefined {
165
177
  * (error field), never a throw from the transport itself.
166
178
  */
167
179
  export async function runRpcWorker(input: RpcWorkerInput): Promise<ChildPiRunResult> {
168
- const tracking: SettleTracking = { lastAssistantText: undefined, settled: false, agentStarted: false };
180
+ const tracking: SettleTracking = {
181
+ lastAssistantText: undefined,
182
+ settled: false,
183
+ sawAssistantMessageEnd: false,
184
+ lastAssistantMessageEndHadText: false,
185
+ agentStarted: false,
186
+ };
169
187
  let settleResolve: (() => void) | undefined;
170
188
  const settledPromise = new Promise<void>((resolve) => {
171
189
  settleResolve = resolve;
@@ -185,6 +203,14 @@ export async function runRpcWorker(input: RpcWorkerInput): Promise<ChildPiRunRes
185
203
  const record = event as { type?: unknown; message?: unknown };
186
204
  if (record?.type === "message_end") {
187
205
  const text = extractAssistantText(record.message);
206
+ const isAssistant =
207
+ typeof record.message === "object" &&
208
+ record.message !== null &&
209
+ (record.message as { role?: unknown }).role === "assistant";
210
+ if (isAssistant) {
211
+ tracking.sawAssistantMessageEnd = true;
212
+ tracking.lastAssistantMessageEndHadText = text !== undefined;
213
+ }
188
214
  if (text !== undefined) tracking.lastAssistantText = text;
189
215
  } else if (record?.type === "agent_settled") {
190
216
  tracking.settled = true;
@@ -262,6 +288,23 @@ export async function runRpcWorker(input: RpcWorkerInput): Promise<ChildPiRunRes
262
288
  if (runError) result.error = runError;
263
289
  else if (abortError && !tracking.settled) result.error = `rpc abort failed: ${abortError.message}`;
264
290
  if (tracking.lastAssistantText !== undefined) result.rawFinalText = tracking.lastAssistantText;
291
+ // DR-01 P0 completeness: empty-settle on the RPC transport. No transcript
292
+ // file exists here, so the attemptErrorFor settledWithoutAnswer branch
293
+ // cannot see this path — stamp the same retryable error from live
294
+ // tracking. Gates: no prior error (spawn/abort precedence), the turn
295
+ // actually settled, an assistant message_end was observed with EMPTY
296
+ // text, and the caller did not abort (an aborted turn must not burn a
297
+ // fallback model). rpcAgentStarted (set below) keeps
298
+ // isEarlyRpcTransportFailure from falling back to stdio for this error.
299
+ if (
300
+ !result.error &&
301
+ tracking.settled &&
302
+ tracking.sawAssistantMessageEnd &&
303
+ !tracking.lastAssistantMessageEndHadText &&
304
+ !input.signal?.aborted
305
+ ) {
306
+ result.error = "Model settled without a final answer (rpc: empty final assistant message; upstream rate-limit/429 exhaustion)";
307
+ }
265
308
  // DR5 bridge: explicit result field (NOT an error-string marker) so the
266
309
  // non-retryable contract is testable and greppable — isEarlyRpcTransportFailure
267
310
  // consults it (run-worker.ts). Set whenever the agent turn had begun.
@@ -4,10 +4,13 @@ import { appendEventAsync } from "../../state/event-log/event-log.ts";
4
4
  import { writeArtifact } from "../../state/stores/artifact-store.ts";
5
5
  import { loadRunManifestById, saveRunTasksAsync, updateRunStatus } from "../../state/stores/state-store.ts";
6
6
  import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
7
+ import { logInternalError } from "../../utils/internal-error.ts";
7
8
  import type { WorkflowStep } from "../../workflows/workflow-config.ts";
9
+ import type { ChildPiRunResult } from "../child-pi/child-pi.ts";
8
10
  import { appendDeadletter } from "../deadletter.ts";
9
11
  import { createWorkerHeartbeat, touchWorkerHeartbeat } from "../heartbeat/worker-heartbeat.ts";
10
12
  import type { CrewRuntimeMode } from "../model/runtime-resolver.ts";
13
+ import { parsePiJsonOutput } from "../output/pi-json-output.ts";
11
14
  import type { RetryPolicy } from "../recovery/retry-executor.ts";
12
15
  import { DEFAULT_RETRY_POLICY, executeWithRetry } from "../recovery/retry-executor.ts";
13
16
  import { permissionForRole } from "../role-permission.ts";
@@ -148,8 +151,17 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
148
151
  if (finalWriteStarted) {
149
152
  try {
150
153
  await repairGroupTerminalWrite(manifest, taskIds, updatedTasks);
151
- } catch {
152
- // best-effort repair — same swallow policy as the heartbeat.
154
+ } catch (error) {
155
+ // E1 (DR-02): still best-effort (must not throw into the
156
+ // interval), but a swallowed failure leaves STALE terminal
157
+ // status on disk with no trace — surface it at warn so the
158
+ // tasks.json-vs-events.jsonl divergence is diagnosable.
159
+ logInternalError(
160
+ "run-coalesced.repair-terminal-write",
161
+ error,
162
+ `runId=${manifest.runId} taskIds=${taskIds.join(",")}`,
163
+ "warn",
164
+ );
153
165
  }
154
166
  }
155
167
  }
@@ -191,7 +203,7 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
191
203
  timeoutHandle.unref?.();
192
204
  }
193
205
  try {
194
- return await runWorker({
206
+ const workerResult = await runWorker({
195
207
  cwd: firstTask.cwd,
196
208
  task: combinedPrompt,
197
209
  agent,
@@ -209,6 +221,12 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
209
221
  workerPid = pid;
210
222
  },
211
223
  });
224
+ // DR-02 WI-1 (R4-1 audit): gate the 429-empty-settle false-green
225
+ // PER ATTEMPT (inside runOnce, before the result escapes to the
226
+ // retry wrapper) so the existing retry machinery applies — see
227
+ // throwIfSettledWithoutAnswer for why this must throw, not flip.
228
+ throwIfSettledWithoutAnswer(workerResult);
229
+ return workerResult;
212
230
  } finally {
213
231
  if (timeoutHandle) clearTimeout(timeoutHandle);
214
232
  if (externalAbortListener && signal) {
@@ -461,6 +479,49 @@ export const __test__persistGroupHeartbeats = persistGroupHeartbeats;
461
479
  /** @internal 3.5 R15-3 test export — exercise the terminal repair write directly. */
462
480
  export const __test__repairGroupTerminalWrite = repairGroupTerminalWrite;
463
481
 
482
+ /**
483
+ * DR-02 WI-1 (R4-1 audit, 2026-10-15): coalesced dispatch must not
484
+ * false-green on upstream rate-limit exhaustion.
485
+ *
486
+ * Bug: this path calls runWorker directly and never parsed the pi-json event
487
+ * stream — a 429-exhausted worker (exit 0, NO stopReason error, a chain of
488
+ * EMPTY assistant message_end events ending agent_settled) sailed through
489
+ * the legacy success gate (`exitCode === 0 && !error`), while `rawFinalText`
490
+ * kept a stale opening sentence from BEFORE the outage, so EVERY task in the
491
+ * group was marked completed with garbage text. The singleton path fixed
492
+ * this in DR-01 (attemptErrorFor's settledWithoutAnswer branch,
493
+ * child-executor.ts) and the DWF ctx.agent() path mirrors it
494
+ * (dynamic-workflow-context.ts) — the coalesced path was the remaining hole.
495
+ *
496
+ * Gate (mirrors the DR-01 attemptErrorFor branch): parse the stdout event
497
+ * stream and THROW when the session settled without a final answer.
498
+ * parsePiJsonOutput is line-safe on non-JSON output (non-JSON lines are
499
+ * skipped), so plain-text and json-success outputs leave settledWithoutAnswer
500
+ * undefined and the healthy path is byte-identical to before. Patches still
501
+ * count as real work (coder roles may legitimately end patch-only) and
502
+ * cancelled attempts are excluded, exactly like the DR-01 gate. The message
503
+ * keeps the DR-01 rate-limit/429 keywords so configured retryableErrors globs
504
+ * can match it.
505
+ *
506
+ * WHY A THROW (not a post-retry success=false flip): executeWithRetry retries
507
+ * THROWN errors — a returned-but-errored result is never retried — so throwing
508
+ * per attempt inside runOnce lets the transient 429 burn the existing retry
509
+ * budget, and exhaustion fires onRetryGivenUp → the US-003 group deadletter.
510
+ * The final throw lands in the dispatch catch path: success=false, rawOutput
511
+ * carries this message, and every group task maps to "failed".
512
+ */
513
+ function throwIfSettledWithoutAnswer(result: ChildPiRunResult): void {
514
+ if (result.error) return; // a hard error already failed the attempt.
515
+ if (result.exitStatus?.cancelled) return; // an abort never fails the gate.
516
+ const parsed = parsePiJsonOutput(result.stdout);
517
+ if (parsed.settledWithoutAnswer !== true) return; // healthy/undefined: unchanged.
518
+ if ((parsed.patches?.length ?? 0) > 0) return; // patch-only work is real work.
519
+ throw new Error("Model settled without a final answer (coalesced: empty final assistant message; upstream rate-limit/429 exhaustion)");
520
+ }
521
+
522
+ /** @internal DR-02 WI-1 test export — exercise the gate directly. */
523
+ export const __test__throwIfSettledWithoutAnswer = throwIfSettledWithoutAnswer;
524
+
464
525
  async function buildCoalescedPrompt(
465
526
  manifest: TeamRunManifest,
466
527
  step: WorkflowStep,
@@ -397,13 +397,17 @@ const CRON_MONTH_NAMES: Record<string, number> = {
397
397
  };
398
398
 
399
399
  /**
400
- * Match a single cron field value against a cron field expression.
401
- * Supports the standard cron grammar: wildcard, single N, range a-b, list a,b,c,
402
- * step syntax (wildcard-step, range-step, from-step), and named tokens
403
- * (MON, JAN) when a `names` map is passed. Fixes the prior matcher that
404
- * rejected step values and named DOW (parseInt returned NaN).
400
+ * Expand a single cron field expression into the set of values it can
401
+ * match (clamped to min..max). Returns null when any part fails to parse
402
+ * (empty part, bad step, unrecognizable token) — callers treat null as
403
+ * "matches nothing / keep legacy behavior" instead of throwing.
404
+ *
405
+ * Supports the standard cron grammar: wildcard, single N, range a-b, list
406
+ * a,b,c, step syntax (wildcard-step, range-step, from-step), and named tokens
407
+ * (MON, JAN) when a `names` map is passed. Shared by the per-occurrence
408
+ * matcher below and the parse-time dom/month feasibility check (I3).
405
409
  */
406
- function cronFieldMatches(value: number, field: string, min: number, max: number, names?: Record<string, number>): boolean {
410
+ function cronFieldCandidates(field: string, min: number, max: number, names?: Record<string, number>): Set<number> | null {
407
411
  let normalized = field.trim().toUpperCase();
408
412
  if (names) {
409
413
  for (const name of Object.keys(names).sort((a, b) => b.length - a.length)) {
@@ -415,10 +419,10 @@ function cronFieldMatches(value: number, field: string, min: number, max: number
415
419
  const matched = new Set<number>();
416
420
  for (const rawPart of normalized.split(",")) {
417
421
  const part = rawPart.trim();
418
- if (part === "") return false;
422
+ if (part === "") return null;
419
423
  const stepMatch = part.match(/^(.*)\/(\d+)$/);
420
424
  const step = stepMatch ? Number.parseInt(stepMatch[2], 10) : 1;
421
- if (!Number.isFinite(step) || step < 1) return false;
425
+ if (!Number.isFinite(step) || step < 1) return null;
422
426
  const rangeStr = stepMatch ? stepMatch[1] : part;
423
427
  let lo: number;
424
428
  let hi: number;
@@ -430,7 +434,7 @@ function cronFieldMatches(value: number, field: string, min: number, max: number
430
434
  hi = stepMatch ? max : lo; // bare `N` = single value; `N/S` = N..max step S
431
435
  } else {
432
436
  const rm = rangeStr.match(/^(\d+)-(\d+)$/);
433
- if (!rm) return false;
437
+ if (!rm) return null;
434
438
  lo = Number.parseInt(rm[1], 10);
435
439
  hi = Number.parseInt(rm[2], 10);
436
440
  }
@@ -438,7 +442,19 @@ function cronFieldMatches(value: number, field: string, min: number, max: number
438
442
  if (v >= min && v <= max) matched.add(v);
439
443
  }
440
444
  }
441
- return matched.has(value);
445
+ return matched;
446
+ }
447
+
448
+ /**
449
+ * Match a single cron field value against a cron field expression.
450
+ * Supports the standard cron grammar: wildcard, single N, range a-b, list
451
+ * a,b,c, step syntax (wildcard-step, range-step, from-step), and named tokens
452
+ * (MON, JAN) when a `names` map is passed. Fixes the prior matcher that
453
+ * rejected step values and named DOW (parseInt returned NaN).
454
+ */
455
+ function cronFieldMatches(value: number, field: string, min: number, max: number, names?: Record<string, number>): boolean {
456
+ const candidates = cronFieldCandidates(field, min, max, names);
457
+ return candidates?.has(value) === true;
442
458
  }
443
459
 
444
460
  function nextCronDate(spec: string, from: Date): Date | { error: string } | null {
@@ -491,6 +507,8 @@ export function parseSchedule(spec: string): ScheduleSpec | { error: string } {
491
507
  }
492
508
  const fields = trimmed.split(/\s+/);
493
509
  if (fields.length >= 5) {
510
+ const impossible = cronDomMonthImpossible(fields[2], fields[3]);
511
+ if (impossible) return { error: `Invalid cron expression "${trimmed}": ${impossible}.` };
494
512
  return { kind: "cron", spec: trimmed };
495
513
  }
496
514
  return {
@@ -498,6 +516,47 @@ export function parseSchedule(spec: string): ScheduleSpec | { error: string } {
498
516
  };
499
517
  }
500
518
 
519
+ /** Day-of-month ceilings per month (index 1..12) for a LEAP year — the
520
+ * conservative upper bound used by the I3 parse-time feasibility check
521
+ * (February is granted 29: leap years exist inside the multi-year window,
522
+ * so Feb-29 specs stay legal at parse). */
523
+ const MONTH_DAY_CEILINGS = [0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
524
+
525
+ /**
526
+ * I3 (round-6 audit): cheap parse-time rejection of cron specs whose
527
+ * day-of-month × month combination cannot occur in ANY year — e.g.
528
+ * `0 0 31 2 *`. nextCronDate only discovers this by scanning
529
+ * minute-by-minute across its entire 366-day window (~309ms measured) after
530
+ * which arm-time disables the job; rejecting at parse surfaces the mistake
531
+ * to the user at schedule-creation time instead of a silent disable later.
532
+ *
533
+ * Conservative by design (NOT a full cron validator):
534
+ * - dow is ignored: nextCronDate uses AND semantics across fields, so an
535
+ * impossible dom/month pair can never match regardless of the dow field;
536
+ * - February is granted its leap-day ceiling of 29 (Feb-29 specs remain
537
+ * legal; the existing arm-time disableCronForUncomputableNext path owns
538
+ * the beyond-search-window case);
539
+ * - any unparseable field returns null here and keeps the legacy runtime
540
+ * behavior — this adds no new rejection classes beyond provable
541
+ * impossibility (empty expansions like inverted ranges count: an empty
542
+ * candidate set can never match under the AND-semantics matcher).
543
+ *
544
+ * Returns a human-readable reason, or null when the combo is feasible (or
545
+ * unknowable cheaply).
546
+ */
547
+ function cronDomMonthImpossible(domStr: string, monthStr: string): string | null {
548
+ const doms = cronFieldCandidates(domStr, 1, 31);
549
+ const months = cronFieldCandidates(monthStr, 1, 12, CRON_MONTH_NAMES);
550
+ if (doms === null || months === null) return null;
551
+ const feasible = [...months].some((m) => [...doms].some((d) => d >= 1 && d <= MONTH_DAY_CEILINGS[m]));
552
+ if (feasible) return null;
553
+ return `day-of-month {${[...doms]
554
+ .sort((a, b) => a - b)
555
+ .join(
556
+ ",",
557
+ )}} never occurs in month {${[...months].sort((a, b) => a - b).join(",")}} — no valid day/month combination exists in any year`;
558
+ }
559
+
501
560
  export function nextRunTime(spec: ScheduleSpec, from: Date = new Date()): Date | { error: string } {
502
561
  if (spec.kind === "once") {
503
562
  const rel = spec.spec.match(/^\+(\d+)(s|m|h|d)$/);
@@ -545,7 +604,11 @@ export function humanizeSchedule(spec: ScheduleSpec): string {
545
604
  return `every ${spec.spec}`;
546
605
  }
547
606
  if (spec.kind === "cron") {
548
- return `cron ${spec.spec}`;
607
+ // I4: cron fields are matched against UTC calendar parts (nextCronDate
608
+ // reads getUTCMinutes/getUTCHours/getUTCDate/...), so the label says so —
609
+ // a user in UTC+7 writing `0 9 * * *` otherwise gets 02:00 local with
610
+ // no hint why.
611
+ return `cron ${spec.spec} (UTC)`;
549
612
  }
550
613
  return "unknown schedule";
551
614
  }
@@ -24,6 +24,8 @@ import { dirname } from "node:path";
24
24
  import { createInterface } from "node:readline";
25
25
  import { fileURLToPath } from "node:url";
26
26
 
27
+ import { logInternalError } from "../../utils/internal-error.ts";
28
+
27
29
  /** D6 read-side cap (Checkpoint B, MINOR-2): restoreState refuses a file larger
28
30
  * than this — an independent guard from the lifecycle cap (same value, no
29
31
  * import to keep engine free of a lifecycle cycle). Bounds v8.deserialize
@@ -80,6 +82,10 @@ export interface RestoreResult {
80
82
  path: string;
81
83
  restored: string[];
82
84
  failed: { name: string; reason: string }[];
85
+ /** P-5 (DR-04 round-7): set when the snapshot was REJECTED before restore —
86
+ * e.g. `unsupported-snapshot-version:<v>`. Distinguishes "restored 0 vars
87
+ * from a valid v1 snapshot" from "refused an unknown format". */
88
+ reason?: string;
83
89
  }
84
90
 
85
91
  export interface EngineOptions {
@@ -353,7 +359,14 @@ export class EngineManager {
353
359
  */
354
360
  async dispose(snapshotPath?: string): Promise<void> {
355
361
  if (snapshotPath && this.engineState === "running") {
356
- await this.snapshotState(snapshotPath).catch(() => null);
362
+ await this.snapshotState(snapshotPath).catch((error: unknown) => {
363
+ // E2 (DR-02): the guest is SIGKILLed below regardless and dispose
364
+ // must never throw — but silently discarding the FINAL snapshot made
365
+ // crash-recovery quietly lose engine state. Surface the loss at
366
+ // warn before dropping it. Keep returning null (no rethrow).
367
+ logInternalError("scratchpad.final-snapshot-failed", error, `snapshotPath=${snapshotPath}`, "warn");
368
+ return null;
369
+ });
357
370
  }
358
371
  await this.kill();
359
372
  }
@@ -620,11 +633,21 @@ export class EngineManager {
620
633
  const st = fstatSync(fd);
621
634
  if (!st.isFile() || st.isSymbolicLink()) return null;
622
635
  if (st.size > SNAPSHOT_MAX_BYTES) return null;
623
- // file validated → lazily start the engine, then read+restore via fd.
624
- await this.start();
625
636
  const payload = JSON.parse(readFileSync(fd, { encoding: "utf8" })) as {
637
+ version?: number;
626
638
  vars?: Record<string, string>;
627
639
  };
640
+ // P-5 (DR-04 round-7): version gate BEFORE starting the engine — an
641
+ // unknown snapshot format must fail LOUDLY (warn + explicit result with
642
+ // restored:[] and a reason), not silently fail-open into an empty
643
+ // namespace (which reads as "nothing to restore" to every caller).
644
+ if (payload.version !== 1) {
645
+ const reason = `unsupported-snapshot-version:${String(payload.version)}`;
646
+ logInternalError("scratchpad.restoreState", new Error(`${reason} (expected 1) — refusing restore`), path, "warn");
647
+ return { path, restored: [], failed: [], reason };
648
+ }
649
+ // file validated → lazily start the engine, then restore.
650
+ await this.start();
628
651
  const vars = payload.vars ?? {};
629
652
  const reply = await this.request({ type: "restore", id: randomUUID(), vars }, SNAPSHOT_REQUEST_TIMEOUT_MS);
630
653
  if (reply.type !== "restore_result") return null;