pi-crew 0.11.20 → 0.11.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +43 -0
  2. package/dist/index.mjs +537 -526
  3. package/package.json +2 -1
  4. package/skills/real-test-pi-crew/SKILL.md +21 -0
  5. package/src/errors.ts +14 -1
  6. package/src/extension/async-notifier.ts +42 -17
  7. package/src/extension/cross-extension-rpc.ts +42 -5
  8. package/src/extension/notification-router.ts +15 -2
  9. package/src/extension/registration/lifecycle-handlers.ts +36 -13
  10. package/src/extension/registration/runtime-cleanup.ts +17 -0
  11. package/src/extension/registration/wire-cross-extension.ts +26 -0
  12. package/src/extension/rpc-hmac.ts +53 -2
  13. package/src/extension/team-tool/cancel.ts +34 -21
  14. package/src/extension/team-tool/dispatch/index.ts +6 -5
  15. package/src/extension/team-tool/dispatch/status.ts +42 -2
  16. package/src/extension/team-tool.ts +24 -2
  17. package/src/prompt/inbox-poll.ts +77 -25
  18. package/src/prompt/message-tool.ts +62 -4
  19. package/src/prompt/prompt-runtime.ts +82 -32
  20. package/src/runtime/broker/crew-broker-client.ts +74 -16
  21. package/src/runtime/broker/crew-broker.ts +111 -77
  22. package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
  23. package/src/runtime/broker/wait-status-cache.ts +14 -5
  24. package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
  25. package/src/runtime/child-pi/child-pi.ts +20 -2
  26. package/src/runtime/child-pi/mock-fixtures.ts +23 -0
  27. package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
  28. package/src/runtime/compaction/important-line-classifier.ts +85 -11
  29. package/src/runtime/compaction/tool-output-pruner.ts +35 -11
  30. package/src/runtime/crew-agent-records.ts +7 -1
  31. package/src/runtime/delivery-coordinator.ts +30 -0
  32. package/src/runtime/dwf-state-store.ts +41 -7
  33. package/src/runtime/finalize-run.ts +13 -2
  34. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
  35. package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
  36. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  37. package/src/runtime/live-session/embedded-tier.ts +3 -1
  38. package/src/runtime/live-session/live-session-runtime.ts +54 -1
  39. package/src/runtime/model/model-fallback.ts +23 -1
  40. package/src/runtime/output/pi-json-output.ts +26 -1
  41. package/src/runtime/policy-engine.ts +18 -0
  42. package/src/runtime/process/zombie-scanner.ts +78 -14
  43. package/src/runtime/process-status.ts +6 -1
  44. package/src/runtime/recovery/crash-recovery.ts +170 -28
  45. package/src/runtime/rpc/rpc-worker.ts +44 -1
  46. package/src/runtime/scheduling/run-coalesced-task-group.ts +64 -3
  47. package/src/runtime/scheduling/scheduler.ts +74 -11
  48. package/src/runtime/scratchpad/engine.ts +26 -3
  49. package/src/runtime/settings-store.ts +22 -2
  50. package/src/runtime/surface/tmux-provider.ts +24 -3
  51. package/src/runtime/task-health.ts +16 -1
  52. package/src/runtime/task-output-context.ts +143 -71
  53. package/src/runtime/task-packet.ts +70 -17
  54. package/src/runtime/task-runner/child-executor.ts +27 -0
  55. package/src/runtime/task-runner/output-splitter.ts +79 -16
  56. package/src/runtime/task-runner/post-execution.ts +75 -15
  57. package/src/runtime/task-runner/state-helpers.ts +92 -2
  58. package/src/runtime/usage-tracker.ts +11 -0
  59. package/src/runtime/verification/green-contract.ts +17 -0
  60. package/src/runtime/verification/verification-gates.ts +24 -6
  61. package/src/runtime/workspace-lock.ts +53 -0
  62. package/src/schema/team-tool-schema.ts +10 -4
  63. package/src/state/coordination/mailbox.ts +133 -38
  64. package/src/state/coordination/process-lock.ts +22 -2
  65. package/src/state/decision-ledger.ts +56 -8
  66. package/src/state/event-log/event-log-rotation.ts +83 -5
  67. package/src/state/event-log/event-log.ts +181 -12
  68. package/src/state/stores/active-run-registry.ts +10 -3
  69. package/src/state/types.ts +9 -1
  70. package/src/ui/agents-jobs-browser.ts +22 -8
  71. package/src/ui/dashboard-panes/agents-pane.ts +5 -2
  72. package/src/ui/events-state-source.ts +67 -28
  73. package/src/ui/inline-panel/agent-pane.ts +13 -6
  74. package/src/ui/inline-panel/agent-transcript.ts +33 -21
  75. package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
  76. package/src/ui/inline-panel/index.ts +21 -1
  77. package/src/ui/live-run-sidebar.ts +14 -3
  78. package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
  79. package/src/ui/powerbar-publisher.ts +6 -3
  80. package/src/ui/rail.ts +7 -1
  81. package/src/ui/render-scheduler.ts +53 -1
  82. package/src/ui/run-action-dispatcher.ts +68 -20
  83. package/src/ui/status-colors.ts +9 -0
  84. package/src/utils/env-filter.ts +8 -0
@@ -33,15 +33,16 @@ import type { TeamRunManifest } from "../../state/types.ts";
33
33
  import type { TeamConfig } from "../../teams/team-config.ts";
34
34
  import { logInternalError } from "../../utils/internal-error.ts";
35
35
  import { safeAbort } from "../../utils/safe-abort.ts";
36
+ import { countTokens } from "../../utils/token-counter.ts";
36
37
  import { cleanupAgentWorktreeAsync, prepareAgentWorktreeAsync } from "../../worktree/worktree-manager.ts";
37
- import type { DwfCheckpointState } from "../dwf-state-store.ts";
38
- import { parsePiJsonOutput } from "../output/pi-json-output.ts";
38
+ import type { ChildPiRunResult } from "../child-pi/child-pi.ts";
39
+ import { DWF_CHECKPOINT_VERSION, type DwfCheckpointState } from "../dwf-state-store.ts";
40
+ import { type ParsedPiJsonOutput, parsePiJsonOutput } from "../output/pi-json-output.ts";
39
41
  import { extractStructuredResult } from "../output/result-extractor.ts";
40
42
  import { executeWithRetry } from "../recovery/retry-executor.ts";
41
43
  import { runWorker } from "../run-worker.ts";
42
44
  import { mapConcurrent } from "../scheduling/parallel-utils.ts";
43
45
  import { Semaphore } from "../scheduling/semaphore.ts";
44
- import { estimateTokens } from "../task-runner/prompt-builder.ts";
45
46
  import { renderPlanTemplate } from "./plan-templates.ts";
46
47
 
47
48
  export interface AgentCallOpts {
@@ -132,6 +133,45 @@ export class DwfAgentCallCapError extends Error {
132
133
  }
133
134
  }
134
135
 
136
+ /** R6-10 (DR-03 round-6, 2026-10-16): typed bridge error thrown INSIDE ctx.retry()
137
+ * when ctx.agent() resolves ok:false. executeWithRetry retries THROWN errors
138
+ * only, so an ok:false result previously flowed straight out of ctx.retry —
139
+ * a 1-attempt "retry" for every failure class except DwfAgentCallCapError.
140
+ * Throwing this wrapper makes the failure retryable; ctx.retry catches it after
141
+ * exhaustion and returns the CARRIED AgentResult, so ctx.retry's public return
142
+ * type/shape is unchanged. DwfAgentCallCapError keeps its passthrough semantics
143
+ * (the cap must terminate the workflow, never be retried into silence). */
144
+ export class DwfRetryFailure extends Error {
145
+ /** The ok:false AgentResult the failed attempt produced. */
146
+ readonly result: AgentResult;
147
+ constructor(result: AgentResult) {
148
+ super(result.error ?? "dynamic workflow retry: agent call returned ok:false");
149
+ this.name = "DwfRetryFailure";
150
+ this.result = result;
151
+ }
152
+ }
153
+
154
+ /** R6-7 (DR-03 round-6, 2026-10-16): DWF empty-settle gate WITH the sibling
155
+ * exemptions the task-runner's attemptErrorFor (child-executor.ts:428-431) has
156
+ * but ctx.agent()'s inline gate lacked. Returns the structured 429-style error
157
+ * string when the worker settled without a final answer AND the exemptions do
158
+ * NOT apply, else undefined (gate passes):
159
+ * - patches are real work: coder roles may legitimately end patch-only
160
+ * (empty final assistant message is fine when the diff already landed);
161
+ * - cancelled/aborted attempts are excluded so an abort never surfaces as an
162
+ * upstream-rate-limit failure (and never burns fallback/retry budget on it).
163
+ * Mirrors child-executor.ts exactly — extracted so the exemptions are unit-testable
164
+ * at this seam (the mock-fixture layer cannot synthesise patch/cancelled shapes). */
165
+ export function dwfSettledWithoutAnswerFailure(
166
+ childResult: Pick<ChildPiRunResult, "exitStatus">,
167
+ parsed: Pick<ParsedPiJsonOutput, "settledWithoutAnswer" | "patches">,
168
+ ): string | undefined {
169
+ if (parsed.settledWithoutAnswer !== true) return undefined;
170
+ if ((parsed.patches?.length ?? 0) > 0) return undefined;
171
+ if (childResult.exitStatus?.cancelled) return undefined;
172
+ return "Model settled without a final answer (empty final assistant message; upstream rate-limit/429 exhaustion)";
173
+ }
174
+
135
175
  export interface WorkflowCtx {
136
176
  cwd: string;
137
177
  runId: string;
@@ -313,6 +353,14 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
313
353
  }
314
354
  : { currentPhase: undefined, phases: [] };
315
355
  let phaseCapWarned = false;
356
+ // R6-12 (DR-03 round-6): phases already recorded in the hydrated checkpoint.
357
+ // ctx.phase() does NOT re-emit phase_started/phase_completed for these on a
358
+ // resumed run — the script re-executes from the top and re-walks its phase
359
+ // history; without this guard every replayed phase title re-emitted its events
360
+ // (duplicated phase boundaries in events.jsonl). Empty on a fresh run, so the
361
+ // guard is resume-only: a fresh script may still legitimately re-enter a title
362
+ // later (e.g. phase("review") rounds) and get its events.
363
+ const hydratedPhases = new Set<string>(opts.resumedState?.phases ?? []);
316
364
  // round-14 P1-2/P1-3/P1-5: closure-scoped runtime state shared by budget/log/args.
317
365
  // Mirrors the pi-dynamic-workflows RuntimeState pattern (workflow.ts:state).
318
366
  // round-18 P2-3: hydrate spent/logs from a resumed checkpoint (backward compatible when unset).
@@ -323,9 +371,10 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
323
371
  };
324
372
  // PERS-1: per-agent-call idempotency cache. Hydrated from resumedState so a DWF
325
373
  // resume skips already-completed agent calls (avoids duplicating artifacts/mailbox/tokens).
326
- // Uses Record (not Map) for JSON serialization safety.
327
- const completedAgentCalls: Record<string, { text: string; usage?: { input: number; output: number } }> =
328
- opts.resumedState?.completedAgentCalls ?? {};
374
+ // Uses Record (not Map) for JSON serialization safety. R6-3 (DR-03 round-6):
375
+ // values are the FULL AgentResult (text + usage + structured + artifactPath +
376
+ // actual durationMs) so replay does not drop structured output / artifact linkage.
377
+ const completedAgentCalls: Record<string, AgentResult> = opts.resumedState?.completedAgentCalls ?? {};
329
378
  // round-14 P1-2: frozen budget surface. The closures read wfState.spent so the
330
379
  // object stays live after Object.freeze(ctx). total is a snapshot primitive.
331
380
  const budget = Object.freeze({
@@ -348,17 +397,19 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
348
397
  let worktreePath: string | undefined;
349
398
  let worktreeBranch: string | undefined;
350
399
  // BDG-2 + G14 (SDD-3): content-based reserve estimate — replaces the flat
351
- // ESTIMATE=4096. estimateTokens() (chars/4, the in-tree heuristic exported
352
- // from runtime/task-runner/prompt-builder.ts — the same estimator
353
- // pre-execution.ts uses) runs over the call's prompt + optional
354
- // systemPrompt override, floored at 512 tokens for the fixed per-call
355
- // overhead (schema/JSON directive, artifact headers) not present in the
356
- // prompt string. Short calls stop over-reserving; very long prompts
357
- // reserve proportionally so N concurrent calls cannot blow past the
358
- // budget between reserve and the post-run adjust. Every BDG-2 site
359
- // below (check/reserve/un-reserve/adjust/refund) uses THIS value, so
360
- // refund always matches what was reserved.
361
- const estimate = Math.max(512, estimateTokens((call.prompt?.length ?? 0) + (call.systemPrompt?.length ?? 0)));
400
+ // ESTIMATE=4096. countTokens() (R5-7, DR-02 round-5: the in-tree
401
+ // CJK-aware estimator from utils/token-counter.ts — replaces the chars/4
402
+ // estimateTokens call, which under-counted CJK/Vietnamese-diacritic
403
+ // prompts ~2-3x and could let N concurrent calls blow past the budget)
404
+ // runs over the call's prompt + optional systemPrompt override, floored
405
+ // at 512 tokens for the fixed per-call overhead (schema/JSON directive,
406
+ // artifact headers) not present in the prompt string. Short calls stop
407
+ // over-reserving; very long prompts reserve proportionally so N
408
+ // concurrent calls cannot blow past the budget between reserve and the
409
+ // post-run adjust. Every BDG-2 site below (check/reserve/un-reserve/
410
+ // adjust/refund) uses THIS value, so refund always matches what was
411
+ // reserved.
412
+ const estimate = Math.max(512, countTokens(`${call.prompt ?? ""}${call.systemPrompt ?? ""}`));
362
413
  let reserved = false;
363
414
  try {
364
415
  // SDD-3 W-C G13: agent-call cap — the structured bound for no-budget runs.
@@ -393,21 +444,20 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
393
444
  // from the call arguments and check if this call was already completed
394
445
  // (e.g. during a previous run before a crash). If so, return the cached
395
446
  // result without re-spawning the agent.
396
- const callId = JSON.stringify({
397
- role: call.role,
398
- agent: call.agent,
399
- prompt: call.prompt,
400
- model: call.model,
401
- schema: call.schema ? "yes" : "no",
402
- });
447
+ // R6-3 (DR-03 round-6, HIGH): the key now covers ALL call-shaping fields
448
+ // (inputs/systemPrompt/disableTools/maxTurns/graceTurns/worktree/skill/…
449
+ // and the schema BY VALUE) — previously only {role,agent,prompt,model,
450
+ // schema:"yes"/"no"}, so two DIFFERENT calls sharing those 5 fields hit the
451
+ // same cache entry on EVERY call (e.g. same prompt with different inputs
452
+ // returned the first call's result — inputs provably alter the dispatched
453
+ // task via composeAgentTask). See computeCallId for the normalization.
454
+ const callId = computeCallId(call);
403
455
  const cached = completedAgentCalls[callId];
404
456
  if (cached) {
405
- return {
406
- ok: true,
407
- text: cached.text,
408
- usage: cached.usage,
409
- durationMs: 0,
410
- };
457
+ // Full-result replay (R6-3): structured + artifactPath survive the
458
+ // round-trip through the checkpoint. ok is forced true — only successful
459
+ // results are cached, and legacy (pre-R6-3) hydrated records lack the field.
460
+ return { ...cached, ok: true };
411
461
  }
412
462
 
413
463
  // BDG-2: reserve-then-adjust budget. Before spawning, estimate the cost and
@@ -490,18 +540,32 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
490
540
  runId: manifest.runId,
491
541
  role: call.role ?? call.agent,
492
542
  });
493
- if (childResult.exitCode !== 0 || childResult.error) {
543
+ const parsed = parsePiJsonOutput(childResult.stdout);
544
+ // DR-01 P0 completeness (DWF): the attemptErrorFor empty-settle branch
545
+ // lives in the task-runner attempt loop, which ctx.agent() bypasses
546
+ // (runWorker is called directly here). Without this gate a
547
+ // 429-exhausted worker (exit 0, empty final assistant message) passes
548
+ // the exitCode/error check and its rawFinalText/finalText — an early
549
+ // opening sentence — becomes the step's "result": a false-green DWF
550
+ // step. See run team_20261014082741 for the live stdio shape.
551
+ // R6-7 (DR-03 round-6): the settledWithoutAnswer term now carries the
552
+ // sibling exemptions (patches = real work; cancelled ≠ 429) via
553
+ // dwfSettledWithoutAnswerFailure — mirroring child-executor.ts:428-431
554
+ // exactly, so DWF and the task-runner classify the same transcript the
555
+ // same way (a patch-only coder step or an aborted attempt no longer fails
556
+ // as rate-limit exhaustion here while passing there).
557
+ const settledFailure = dwfSettledWithoutAnswerFailure(childResult, parsed);
558
+ if (childResult.exitCode !== 0 || childResult.error || settledFailure !== undefined) {
494
559
  // BDG-2: un-reserve the estimate on spawn failure.
495
560
  wfState.spent -= estimate;
496
561
  reserved = false;
497
562
  return {
498
563
  ok: false,
499
564
  text: "",
500
- error: childResult.error ?? `exit ${childResult.exitCode}`,
565
+ error: childResult.error ?? settledFailure ?? `exit ${childResult.exitCode}`,
501
566
  durationMs: Date.now() - started,
502
567
  };
503
568
  }
504
- const parsed = parsePiJsonOutput(childResult.stdout);
505
569
  // round-14 P1-2: accumulate this run's token usage into the workflow budget.
506
570
  // BDG-2: adjust the reserve — subtract the estimate, add the actual usage.
507
571
  // This correctly reduces spent when actualUsage < estimate.
@@ -539,10 +603,16 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
539
603
  durationMs: Date.now() - started,
540
604
  };
541
605
  }
542
- // PERS-1: cache the successful result for idempotency on resume.
606
+ // PERS-1: cache the FULL successful result for idempotency on resume
607
+ // (R6-3: structured + artifactPath + actual durationMs included, so
608
+ // replay returns the same payload a live call would have).
543
609
  completedAgentCalls[callId] = {
610
+ ok: true,
544
611
  text,
545
- usage: { input: parsed.usage?.input ?? 0, output: parsed.usage?.output ?? 0 },
612
+ structured: extracted.structured ? extracted.data : undefined,
613
+ usage: parsed.usage,
614
+ artifactPath: artifact.path,
615
+ durationMs: Date.now() - started,
546
616
  };
547
617
  return {
548
618
  ok: true,
@@ -567,23 +637,17 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
567
637
  durationMs: Date.now() - started,
568
638
  };
569
639
  } finally {
570
- // round-17 P2-4: clean up the worktree after the agent completes (success
571
- // OR failure). Captures the diff as an artifact before removal. Best-effort
572
- // — a leak must never crash the workflow.
573
- if (worktreePath) {
574
- try {
575
- await cleanupAgentWorktreeAsync(manifest, worktreePath, worktreeBranch);
576
- } catch (cleanupError) {
577
- logInternalError("dynamic-workflow-context.worktree-cleanup", cleanupError, `worktreePath=${worktreePath}`);
578
- }
579
- }
580
- // round-18 P2-3: checkpoint AFTER the agent completes (success or fail) so a
581
- // crash between agent calls leaves durable state to resume from. The counter is
582
- // incremented here (after the call) so the checkpoint reflects the call that ran.
640
+ // R6-12 (DR-03 round-6): CHECKPOINT BEFORE CLEANUP. agentCount++ and
641
+ // onCheckpoint must land BEFORE the worktree-cleanup await — previously
642
+ // the checkpoint ran after it, so a crash/hang during cleanup lost the
643
+ // just-completed call from the durable checkpoint and resume re-spawned
644
+ // the same agent (duplicate work/tokens). Cleanup is best-effort and
645
+ // idempotent-ish (a leaked worktree is recoverable; a lost call is not).
583
646
  agentCount++;
584
647
  if (opts.onCheckpoint) {
585
648
  try {
586
649
  opts.onCheckpoint({
650
+ version: DWF_CHECKPOINT_VERSION,
587
651
  runId: manifest.runId,
588
652
  vars: ctx.vars,
589
653
  phases: phaseState.phases,
@@ -598,6 +662,17 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
598
662
  logInternalError("dynamic-workflow-context.checkpoint", checkpointError, `runId=${manifest.runId}`);
599
663
  }
600
664
  }
665
+ // round-17 P2-4: clean up the worktree after the agent completes (success
666
+ // OR failure). Captures the diff as an artifact before removal. Best-effort
667
+ // — a leak must never crash the workflow. R6-12: runs AFTER the checkpoint
668
+ // (see above) — ordering is the fix.
669
+ if (worktreePath) {
670
+ try {
671
+ await cleanupAgentWorktreeAsync(manifest, worktreePath, worktreeBranch);
672
+ } catch (cleanupError) {
673
+ logInternalError("dynamic-workflow-context.worktree-cleanup", cleanupError, `worktreePath=${worktreePath}`);
674
+ }
675
+ }
601
676
  semaphore.release();
602
677
  }
603
678
  },
@@ -718,19 +793,35 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
718
793
  };
719
794
  },
720
795
  async retry(taskId: string, retryOpts?: { feedback?: string }): Promise<AgentResult> {
721
- return executeWithRetry(
722
- async () =>
723
- ctx.agent({
724
- role: "executor",
725
- prompt: `Re-do task '${taskId}'.${retryOpts?.feedback ? ` Feedback: ${retryOpts.feedback}` : ""}`,
726
- }),
727
- {
728
- maxAttempts: 3,
729
- backoffMs: 0,
730
- jitterRatio: 0,
731
- exponentialFactor: 1,
732
- },
733
- );
796
+ // R6-10 (DR-03 round-6): executeWithRetry retries THROWN errors only, but
797
+ // ctx.agent() returns {ok:false} for every failure class except
798
+ // DwfAgentCallCapError — so the previous pass-through never retried an
799
+ // ok:false attempt (a 1-attempt "retry"). Bridge: throw DwfRetryFailure
800
+ // (typed, carries the AgentResult) on !ok so executeWithRetry retries it;
801
+ // after exhaustion catch it and return the CARRIED result (public return
802
+ // type/shape unchanged). DwfAgentCallCapError is not wrapped — it keeps
803
+ // its terminate-the-workflow semantics and propagates untouched.
804
+ try {
805
+ return await executeWithRetry(
806
+ async () => {
807
+ const res = await ctx.agent({
808
+ role: "executor",
809
+ prompt: `Re-do task '${taskId}'.${retryOpts?.feedback ? ` Feedback: ${retryOpts.feedback}` : ""}`,
810
+ });
811
+ if (!res.ok) throw new DwfRetryFailure(res);
812
+ return res;
813
+ },
814
+ {
815
+ maxAttempts: 3,
816
+ backoffMs: 0,
817
+ jitterRatio: 0,
818
+ exponentialFactor: 1,
819
+ },
820
+ );
821
+ } catch (error) {
822
+ if (error instanceof DwfRetryFailure) return error.result;
823
+ throw error;
824
+ }
734
825
  },
735
826
  mail(
736
827
  to: string,
@@ -778,6 +869,13 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
778
869
  }
779
870
  // Idempotency: same phase title → no event, no state change.
780
871
  if (title === phaseState.currentPhase) return;
872
+ // R6-12 (DR-03 round-6): phase no-dup on resume. A phase already recorded
873
+ // in the hydrated checkpoint is NOT re-emitted — the resumed script
874
+ // re-walks its phase history and would otherwise duplicate phase_started/
875
+ // phase_completed events in events.jsonl. Resume-only guard: hydratedPhases
876
+ // is empty on a fresh run, so legitimate same-title re-entry later in a
877
+ // fresh script still emits (see the fresh-run counter-pin test).
878
+ if (hydratedPhases.has(title)) return;
781
879
  // Close out the previous open phase BEFORE the new one opens.
782
880
  // U2 (2026-10-10): kept SYNC — ctx closures are exercised DIRECTLY by
783
881
  // unit tests that readEvents() synchronously right after the call
@@ -921,6 +1019,8 @@ export function getWorkflowCheckpoint(ctx: WorkflowCtx): DwfCheckpointState {
921
1019
  const phaseState = getWorkflowPhaseState(ctx);
922
1020
  const logs = getWorkflowLogs(ctx);
923
1021
  return {
1022
+ // R6-12: snapshot carries the format version (mirrors the onCheckpoint payload).
1023
+ version: DWF_CHECKPOINT_VERSION,
924
1024
  runId: ctx.runId,
925
1025
  vars: ctx.vars,
926
1026
  phases: phaseState?.phases ?? [],
@@ -928,13 +1028,41 @@ export function getWorkflowCheckpoint(ctx: WorkflowCtx): DwfCheckpointState {
928
1028
  logs: logs ?? [],
929
1029
  spent: ctx.budget.spent(),
930
1030
  agentCount: (ctx as unknown as { __agentCount?: number }).__agentCount ?? 0,
931
- completedAgentCalls:
932
- (ctx as unknown as { __completedAgentCalls?: Record<string, { text: string; usage?: { input: number; output: number } }> })
933
- .__completedAgentCalls ?? {},
1031
+ completedAgentCalls: (ctx as unknown as { __completedAgentCalls?: Record<string, AgentResult> }).__completedAgentCalls ?? {},
934
1032
  updatedAt: new Date().toISOString(),
935
1033
  };
936
1034
  }
937
1035
 
1036
+ /**
1037
+ * R6-3 (DR-03 round-6, HIGH): deterministic PERS-1 cache key over ALL
1038
+ * call-shaping AgentCallOpts fields — the JSON of the resolved call object
1039
+ * with stable ordering:
1040
+ * - object keys sorted (two identical calls built with different literal key
1041
+ * orders still share a cache entry);
1042
+ * - top-level array fields (inputs, skill) keyed as a SORTED copy — the same
1043
+ * set in a different order is the same call;
1044
+ * - undefined fields dropped;
1045
+ * - the schema is included BY VALUE (a structural change to the expected
1046
+ * output shape changes the dispatched task and must miss the cache — the
1047
+ * old "yes"/"no" placeholder collided different schemas).
1048
+ * Previously the key was {role, agent, prompt, model, schema:"yes"/"no"} only,
1049
+ * omitting inputs/systemPrompt/disableTools/maxTurns/graceTurns/worktree/skill
1050
+ * — e.g. the same prompt with DIFFERENT inputs (which provably alter the
1051
+ * dispatched task via composeAgentTask) hit the first call's cache entry on
1052
+ * EVERY call and returned the wrong call's result (lead-verified :398-412).
1053
+ * Note: fields that do NOT shape the call (description) are included anyway —
1054
+ * over-keying only costs a cache miss (a fresh spawn), never correctness.
1055
+ */
1056
+ function computeCallId(call: AgentCallOpts): string {
1057
+ const normalized: Record<string, unknown> = {};
1058
+ for (const key of Object.keys(call).sort()) {
1059
+ const value: unknown = call[key as keyof AgentCallOpts];
1060
+ if (value === undefined) continue;
1061
+ normalized[key] = Array.isArray(value) ? [...(value as unknown[])].sort() : value;
1062
+ }
1063
+ return JSON.stringify(normalized);
1064
+ }
1065
+
938
1066
  /** Compose the agent task: prompt + optional dependency-input context block. */
939
1067
  function composeAgentTask(call: AgentCallOpts): string {
940
1068
  let base = call.prompt;
@@ -20,6 +20,8 @@ import { createRequire } from "node:module";
20
20
  import type { AgentConfig } from "../../agents/agent-config.ts";
21
21
  import { loadConfig } from "../../config/config.ts";
22
22
  import { effectiveRunConfig } from "../../extension/team-tool/config-patch.ts";
23
+ import { isTerminalTaskStatus } from "../../state/contracts.ts";
24
+ import { withRunLockSync } from "../../state/coordination/locks.ts";
23
25
  import { appendEventBuffered } from "../../state/event-log/event-log.ts";
24
26
  import { registerActiveRun, unregisterActiveRun } from "../../state/stores/active-run-registry.ts";
25
27
  import { collectRunMetrics } from "../../state/stores/run-metrics.ts";
@@ -28,14 +30,52 @@ import type { GoalLoopState, GoalLoopStatus, GoalVerdict, TeamRunManifest, TeamT
28
30
  import type { TeamConfig } from "../../teams/team-config.ts";
29
31
  import { logInternalError } from "../../utils/internal-error.ts";
30
32
  import type { WorkflowConfig } from "../../workflows/workflow-config.ts";
33
+ import { isRunLockBusyError } from "../broker/protocol/lock-busy.ts";
31
34
  import { resolveCrewRuntime } from "../model/runtime-resolver.ts";
32
35
  import { registryFromModelContext } from "../model/session-model.ts";
33
36
  import { executeTeamRun } from "../team-runner.ts";
34
37
  import { compareSnapshot, snapshotManifests } from "../verification/verification-integrity.ts";
35
- import { acquireWorkspaceLock, type WorkspaceLockHandle } from "../workspace-lock.ts";
38
+ import { acquireWorkspaceLock, refreshWorkspaceLockHeartbeat, type WorkspaceLockHandle } from "../workspace-lock.ts";
36
39
  import { bundleEvidence, evaluateGoal } from "./goal-evaluator.ts";
37
40
  import { GoalStore } from "./goal-state-store.ts";
38
41
 
42
+ /**
43
+ * D5 (DR-02 Wave 2 Cluster C): merge the turn's FINAL task states onto a
44
+ * FRESH tasks.json read. The turn owns exactly the terminal-evidence fields
45
+ * its executor wrote (status/finishedAt/resultArtifact/error/exitCode);
46
+ * everything else on the fresh task (heartbeat/lastSeenAt, claim, delegate
47
+ * state, usage…) may be NEWER than the turn's in-memory snapshot and must
48
+ * never be regressed by the post-turn persist.
49
+ *
50
+ * Terminal-preserve: a task that is ALREADY terminal in the fresh read (an
51
+ * external decision — operator cancel, crash-recovery, the async-died
52
+ * notifier) keeps its fresh state entirely; the turn must not resurrect or
53
+ * re-decide it (same guard class as heartbeat.ts R13-S1). Turn tasks missing
54
+ * from the fresh read are appended — the turn created them and the
55
+ * budget/audit reads this persist exists for expect them on disk.
56
+ */
57
+ export function mergeTurnFinalTasksOntoFresh(freshTasks: readonly TeamTaskState[], turnTasks: readonly TeamTaskState[]): TeamTaskState[] {
58
+ const freshById = new Map(freshTasks.map((task) => [task.id, task]));
59
+ const turnById = new Map(turnTasks.map((task) => [task.id, task]));
60
+ const merged = freshTasks.map((freshTask) => {
61
+ const turnTask = turnById.get(freshTask.id);
62
+ if (!turnTask) return freshTask;
63
+ if (isTerminalTaskStatus(freshTask.status)) return freshTask;
64
+ return {
65
+ ...freshTask,
66
+ status: turnTask.status,
67
+ ...(turnTask.finishedAt !== undefined ? { finishedAt: turnTask.finishedAt } : {}),
68
+ ...(turnTask.resultArtifact !== undefined ? { resultArtifact: turnTask.resultArtifact } : {}),
69
+ ...(turnTask.error !== undefined ? { error: turnTask.error } : {}),
70
+ ...(turnTask.exitCode !== undefined ? { exitCode: turnTask.exitCode } : {}),
71
+ };
72
+ });
73
+ for (const turnTask of turnTasks) {
74
+ if (!freshById.has(turnTask.id)) merged.push(turnTask);
75
+ }
76
+ return merged;
77
+ }
78
+
39
79
  /** Required minimal shape for the worker + agents discovery (P0 uses the goal's workerAgent). */
40
80
  export interface GoalLoopRuntimeDeps {
41
81
  /** Resolve the agent configs reachable from cwd (used for executeTeamRun's agents arg). */
@@ -538,6 +578,48 @@ export function deriveTranscriptPath(artifactsRoot: string, tasks: import("../..
538
578
  return undefined;
539
579
  }
540
580
 
581
+ /** Interval (ms) between workspace-lock heartbeat refreshes while a goal loop runs (WI-2 / DR-02 R4-3). 15s = 4x margin under the 60s staleness threshold. */
582
+ export const WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS = 15_000;
583
+
584
+ export interface StartWorkspaceLockHeartbeatRefreshOptions {
585
+ /** Test injection: override the refresh interval (ms). */
586
+ refreshMs?: number;
587
+ /** Test injection: override the refresh implementation (default: refreshWorkspaceLockHeartbeat). */
588
+ refresh?: (lockPath: string, goalId: string) => boolean;
589
+ }
590
+
591
+ /**
592
+ * Start the periodic workspace-lock heartbeat refresh for a lock acquired by
593
+ * runGoalLoop (WI-2 / DR-02 R4-3).
594
+ *
595
+ * Without refresh, workspace-lock isLockStale clause (b) marks the lock stale
596
+ * after 60s REGARDLESS of holder pid liveness — a goal whose turn takes >60s
597
+ * gets its workspace lock force-stolen by a second goal on the same cwd
598
+ * (multi-goal clobber). This timer keeps the heartbeat young for the whole
599
+ * loop lifetime. It is unref()-ed (must not keep the background process
600
+ * alive) and the returned stop() MUST be called in the same finally that
601
+ * releases the lock. Refresh failures are logged as warn and swallowed — a
602
+ * heartbeat miss must never throw into the goal loop.
603
+ */
604
+ export function startWorkspaceLockHeartbeatRefresh(
605
+ handle: Pick<WorkspaceLockHandle, "lockPath" | "goalId">,
606
+ opts: StartWorkspaceLockHeartbeatRefreshOptions = {},
607
+ ): { stop: () => void } {
608
+ const refreshMs = opts.refreshMs ?? WORKSPACE_LOCK_HEARTBEAT_REFRESH_MS;
609
+ const refresh = opts.refresh ?? refreshWorkspaceLockHeartbeat;
610
+ const timer = setInterval(() => {
611
+ try {
612
+ refresh(handle.lockPath, handle.goalId);
613
+ } catch (error) {
614
+ // Never throw into the loop (interval callbacks crashing node:test aside,
615
+ // an escaping throw here would kill the background goal process).
616
+ logInternalError("goal-loop.workspaceLockHeartbeat", error, `goalId=${handle.goalId} lockPath=${handle.lockPath}`, "warn");
617
+ }
618
+ }, refreshMs);
619
+ timer.unref(); // the refresh timer must not keep the detached goal-loop process alive
620
+ return { stop: () => clearInterval(timer) };
621
+ }
622
+
541
623
  /**
542
624
  * Run the autonomous goal loop. Returns {manifest, tasks, goalState} — the OUTER
543
625
  * manifest is the synthetic goal-loop manifest (runKind:"goal-loop"); per-turn
@@ -575,10 +657,18 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
575
657
  // the background-process boundary. `goal start` / `goal resume` pre-check via isWorkspaceBusy
576
658
  // for a good error message; this acquisition is the authoritative claim.
577
659
  let workspaceLock: WorkspaceLockHandle | undefined;
660
+ // WI-2 (DR-02 R4-3): periodic heartbeat refresh for the lock above — stopped
661
+ // in the same finally that releases it (see startWorkspaceLockHeartbeatRefresh).
662
+ let lockHeartbeat: { stop: () => void } | undefined;
578
663
  try {
579
664
  workspaceLock = await acquireWorkspaceLock(goal.cwd, goal.goalId, {
580
665
  signal,
581
666
  });
667
+ // WI-2 (DR-02 R4-3): keep the heartbeat young for the whole loop lifetime —
668
+ // without this, isLockStale clause (b) force-steals the lock after 60s even
669
+ // though this pid is alive (multi-goal clobber). Started immediately after
670
+ // acquire; the timer is unref()-ed and never throws into the loop.
671
+ lockHeartbeat = startWorkspaceLockHeartbeatRefresh(workspaceLock);
582
672
  } catch (error) {
583
673
  logInternalError("goal-loop.workspaceLock", error, `goalId=${goal.goalId} cwd=${goal.cwd}`);
584
674
  goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
@@ -753,10 +843,41 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
753
843
  }
754
844
 
755
845
  // Persist final task states for budget/audit reads.
846
+ // D5 (DR-02 Wave 2 Cluster C): the bare saveRunTasks(turnResult.tasks) was
847
+ // an UNLOCKED write of the executor's in-memory snapshot (goal-loop-runner.ts
848
+ // ":757") — it clobbered any NEWER state written under the run lock between
849
+ // the executor's last persist and this save (worker heartbeats/lastSeenAt,
850
+ // task claims, delegate statuses). Now a run-locked RMW: fresh re-read inside
851
+ // the lock, then a field-level merge — the turn re-decides only the fields it
852
+ // owns and never regresses fresh heartbeat/claim/delegate data.
756
853
  try {
757
- saveRunTasks(turnResult.manifest, turnResult.tasks);
854
+ withRunLockSync(turnResult.manifest, () => {
855
+ const fresh = loadRunManifestById(turnResult.manifest.cwd, turnResult.manifest.runId);
856
+ if (!fresh) {
857
+ logInternalError(
858
+ "goal-loop.saveTurnTasks",
859
+ new Error(`turn run '${turnResult.manifest.runId}' vanished before the final task persist — skipped`),
860
+ "best-effort",
861
+ "warn",
862
+ );
863
+ return;
864
+ }
865
+ saveRunTasks(fresh.manifest, mergeTurnFinalTasksOntoFresh(fresh.tasks, turnResult.tasks));
866
+ });
758
867
  } catch (error) {
759
- logInternalError("goal-loop.saveTurnTasks", error, `turnRunId=${created.manifest.runId}`);
868
+ if (isRunLockBusyError(error)) {
869
+ // Best-effort persist: the executor already persisted terminal task
870
+ // states during the turn — this merge is belt-and-suspenders for
871
+ // budget/audit reads, so contention skips it with a warn.
872
+ logInternalError(
873
+ "goal-loop.saveTurnTasks",
874
+ error,
875
+ `turnRunId=${created.manifest.runId} run lock busy — skipped final task merge`,
876
+ "warn",
877
+ );
878
+ } else {
879
+ logInternalError("goal-loop.saveTurnTasks", error, `turnRunId=${created.manifest.runId}`);
880
+ }
760
881
  }
761
882
 
762
883
  // GL-1: if the turn ended in blocked/failed status, stop the loop without spawning the judge.
@@ -908,6 +1029,12 @@ export async function runGoalLoop(input: RunGoalLoopInput): Promise<RunGoalLoopR
908
1029
  }).catch((e) => logInternalError("goal-loop.buffered", e, "type=goal.loop_error"));
909
1030
  goal = safeSetStatus(store, goal.goalId, "blocked", goal, eventsPath);
910
1031
  } finally {
1032
+ // WI-2: stop the heartbeat refresh BEFORE releasing the lock it was keeping alive.
1033
+ try {
1034
+ lockHeartbeat?.stop();
1035
+ } catch {
1036
+ /* best-effort */
1037
+ }
911
1038
  // P1g: release the workspace lock (held since loop start).
912
1039
  try {
913
1040
  workspaceLock?.release();
@@ -231,6 +231,11 @@ export class HeartbeatWatcher {
231
231
  if (!activeKeys.has(key) && ts < cutoff) {
232
232
  this.lastLevel.delete(key);
233
233
  this.consecutiveDead.delete(key);
234
+ // M-8 (2026-10-16): also drop the deadletter cooldown marker —
235
+ // before this the map leaked one entry per vanished run:taskId key
236
+ // and a stale cooldown timestamp could suppress a legitimate
237
+ // re-trigger after the same task reappeared.
238
+ this.lastDeadletterTriggerAt.delete(key);
234
239
  this.lastSeen.delete(key);
235
240
  }
236
241
  }
@@ -242,5 +247,7 @@ export class HeartbeatWatcher {
242
247
  this.lastLevel.clear();
243
248
  this.consecutiveDead.clear();
244
249
  this.lastSeen.clear();
250
+ // M-8: clear the deadletter cooldown map too (was leaked before).
251
+ this.lastDeadletterTriggerAt.clear();
245
252
  }
246
253
  }
@@ -908,7 +908,9 @@ export function admitEmbeddedSession(input: {
908
908
  });
909
909
  // Raced against the prompt; a pre-race rejection must not surface as an
910
910
  // unhandled rejection (which would trip our own process guard).
911
- escapeSignal.catch(() => {});
911
+ escapeSignal.catch(() => {
912
+ /* intentional UR-guard — see comment above */
913
+ });
912
914
  let tornDown = false;
913
915
  registerEmbeddedSessionGuard(
914
916
  input.agentId,