pi-crew 0.11.1 → 0.11.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/CHANGELOG.md +39 -9
  2. package/README.md +161 -1036
  3. package/agents/verifier.md +18 -7
  4. package/dist/index.mjs +744 -91462
  5. package/docs/README.md +57 -46
  6. package/docs/architecture.md +87 -33
  7. package/docs/commands-reference.md +9 -5
  8. package/docs/troubleshooting.md +3 -2
  9. package/package.json +1 -3
  10. package/schema.json +29 -0
  11. package/skills/real-test-pi-crew/SKILL.md +193 -36
  12. package/src/agents/agent-config.ts +1 -1
  13. package/src/agents/discover-agents.ts +1 -1
  14. package/src/config/config-validation.ts +15 -0
  15. package/src/config/config.ts +47 -13
  16. package/src/config/env-vars.ts +35 -0
  17. package/src/config/types.ts +19 -0
  18. package/src/errors.ts +2 -2
  19. package/src/extension/async-notifier.ts +23 -0
  20. package/src/extension/help.ts +21 -10
  21. package/src/extension/knowledge-injection.ts +2 -1
  22. package/src/extension/management.ts +8 -3
  23. package/src/extension/notification-sink.ts +17 -0
  24. package/src/extension/registration/command-utils.ts +28 -2
  25. package/src/extension/registration/commands/dashboard.ts +11 -1
  26. package/src/extension/registration/commands/manage.ts +31 -15
  27. package/src/extension/registration/commands/run.ts +24 -2
  28. package/src/extension/registration/commands/shared.ts +23 -0
  29. package/src/extension/registration/commands/status.ts +25 -2
  30. package/src/extension/registration/context-builder.ts +8 -2
  31. package/src/extension/registration/health-notify-policy.ts +100 -0
  32. package/src/extension/registration/lazy-configurers.ts +35 -0
  33. package/src/extension/registration/lifecycle-handlers.ts +91 -30
  34. package/src/extension/registration/lifecycle.ts +75 -10
  35. package/src/extension/registration/observability.ts +98 -35
  36. package/src/extension/registration/registration-types.ts +7 -5
  37. package/src/extension/registration/runtime-cleanup.ts +9 -3
  38. package/src/extension/registration/subagent-helpers.ts +38 -0
  39. package/src/extension/registration/wire-cross-extension.ts +28 -0
  40. package/src/extension/run-compare.ts +220 -0
  41. package/src/extension/run-export.ts +37 -5
  42. package/src/extension/run-maintenance.ts +155 -5
  43. package/src/extension/team-tool/dispatch/index.ts +3 -2
  44. package/src/extension/team-tool/dispatch/manage.ts +5 -2
  45. package/src/extension/team-tool/goal.ts +4 -1
  46. package/src/extension/team-tool/handle-settings.ts +19 -2
  47. package/src/extension/team-tool/health-monitor.ts +21 -7
  48. package/src/extension/team-tool/lifecycle-actions.ts +49 -1
  49. package/src/extension/team-tool/plan.ts +10 -0
  50. package/src/extension/team-tool/routing-hint.ts +63 -0
  51. package/src/extension/team-tool/status.ts +4 -0
  52. package/src/extension/team-tool.ts +52 -6
  53. package/src/extension/webhook-notify.ts +382 -0
  54. package/src/observability/metric-sink.ts +12 -2
  55. package/src/prompt/prompt-runtime.ts +82 -31
  56. package/src/prompt/worker-events-channel.ts +12 -0
  57. package/src/runtime/README.md +1 -1
  58. package/src/runtime/async-runner.ts +87 -1
  59. package/src/runtime/background-runner.ts +313 -234
  60. package/src/runtime/broker/crew-broker.ts +17 -11
  61. package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
  62. package/src/runtime/broker/wait-status-cache.ts +1 -1
  63. package/src/runtime/child-pi/child-pi-timers.ts +1 -1
  64. package/src/runtime/child-pi/mock-fixtures.ts +48 -0
  65. package/src/runtime/crew-agent-records.ts +337 -45
  66. package/src/runtime/deadletter.ts +43 -1
  67. package/src/runtime/delegate-spawn.ts +5 -1
  68. package/src/runtime/dispatch-batch.ts +72 -5
  69. package/src/runtime/goal-workflow/goal-loop-runner.ts +73 -4
  70. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  71. package/src/runtime/model/model-fallback.ts +21 -1
  72. package/src/runtime/model/pi-args.ts +8 -10
  73. package/src/runtime/recovery/crash-recovery.ts +25 -1
  74. package/src/runtime/run-worker.ts +12 -1
  75. package/src/runtime/scheduling/global-worker-cap.ts +13 -6
  76. package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
  77. package/src/runtime/scheduling/scheduler.ts +49 -13
  78. package/src/runtime/scheduling/semaphore.ts +148 -20
  79. package/src/runtime/scratchpad/README.md +1 -1
  80. package/src/runtime/scratchpad/protocol.ts +1 -1
  81. package/src/runtime/settings-store.ts +1 -1
  82. package/src/runtime/skill-instructions.ts +22 -0
  83. package/src/runtime/stale-reconciler.ts +85 -13
  84. package/src/runtime/task-runner/pre-execution.ts +26 -2
  85. package/src/runtime/task-runner/prompt-builder.ts +142 -45
  86. package/src/runtime/task-runner.ts +21 -1
  87. package/src/runtime/team-runner.ts +38 -1
  88. package/src/runtime/workspace-lock.ts +4 -1
  89. package/src/schema/config-schema.ts +14 -0
  90. package/src/schema/team-tool-schema.ts +17 -0
  91. package/src/state/atomic-write.ts +53 -0
  92. package/src/state/contracts.ts +109 -0
  93. package/src/state/coordination/locks.ts +191 -33
  94. package/src/state/coordination/mailbox.ts +140 -15
  95. package/src/state/crew-init.ts +87 -12
  96. package/src/state/event-log/cursor.ts +37 -1
  97. package/src/state/event-log/event-log-rotation.ts +72 -7
  98. package/src/state/stores/active-run-registry.ts +13 -1
  99. package/src/state/stores/state-store.ts +112 -22
  100. package/src/state/types.ts +4 -0
  101. package/src/ui/dashboard-panes/agents-pane.ts +11 -2
  102. package/src/ui/heartbeat-aggregator.ts +34 -0
  103. package/src/ui/keybinding-map.ts +22 -4
  104. package/src/ui/live-conversation-overlay.ts +6 -3
  105. package/src/ui/run-dashboard.ts +98 -5
  106. package/src/ui/run-snapshot-cache.ts +18 -1
  107. package/src/ui/spinner.ts +26 -2
  108. package/src/ui/tool-progress-formatter.ts +2 -1
  109. package/src/ui/tool-renderers/brief-mode.ts +2 -1
  110. package/src/ui/tool-renderers/index.ts +3 -3
  111. package/src/utils/incremental-reader.ts +11 -3
  112. package/src/utils/paths.ts +94 -12
  113. package/src/utils/project-markers.ts +40 -0
  114. package/src/worktree/worktree-manager.ts +206 -26
  115. package/workflows/distill.workflow.md +3 -3
  116. package/workflows/fast-fix.workflow.md +1 -1
  117. package/workflows/plan-execute.workflow.md +1 -1
  118. package/workflows/review.workflow.md +1 -1
  119. package/workflows/strict-fast-fix.workflow.md +1 -1
  120. package/docs/migration-v0.4-v0.5.md +0 -208
  121. package/docs/runtime-flow.md +0 -148
@@ -114,6 +114,10 @@ export function readEvents(eventsPath: string): TeamEvent[] {
114
114
 
115
115
  export interface EventCursorOptions {
116
116
  sinceSeq?: number;
117
+ /** Delivery cap for one read. When `limit` truncates the result, `events` is
118
+ * a PREFIX of everything available — resume from the returned `nextSeq`
119
+ * (never from `nextByteOffset`; see the continuation contract on
120
+ * `readEventsCursor`). */
117
121
  limit?: number;
118
122
  fromByteOffset?: number;
119
123
  /** R-03: generation the caller captured on its previous read. When set, a
@@ -528,6 +532,30 @@ export function clearEventsCursorTailCache(eventsPath?: string): void {
528
532
  cursorTailCache.delete(eventsPath);
529
533
  }
530
534
 
535
+ /**
536
+ * Read team events with an optional incremental anchor (`sinceSeq`) plus a
537
+ * delivery cap (`limit`).
538
+ *
539
+ * Continuation / backpressure contract (BR-08):
540
+ * - `nextSeq` is the continuation token for events that CARRY `metadata.seq`
541
+ * (scheduler-emitted events): the max seq among the events DELIVERED by
542
+ * this call — never a watermark past an undelivered event. Resume with
543
+ * `{ sinceSeq: nextSeq }` (`>` comparison) to receive the rest.
544
+ * LIMITS (cold-verify correction — documented, not fixed here): events
545
+ * WITHOUT `metadata.seq` (e.g. the worker progress channel) are filtered
546
+ * out of the sinceSeq path entirely, and DUPLICATE seqs cannot be paged
547
+ * through by seq. Those cases need the `fromByteOffset` anchor.
548
+ * - `total` is the number of events AVAILABLE for this read (after the
549
+ * sinceSeq filter, before the `limit` slice) — EXCEPT that the tail cap
550
+ * (TAIL_EVENT_CAP) may drop a prefix first, in which case `total` counts
551
+ * only the retained suffix.
552
+ * - `nextByteOffset` is an OPTIONAL incremental-read accelerator, and is only
553
+ * returned when it is a SAFE resume anchor: on the `fromByteOffset` path it
554
+ * is OMITTED whenever `limit` truncated the delta, because the underlying
555
+ * offset then points past the ENTIRE delta (undelivered tail included) —
556
+ * resuming from it would silently skip events. The default (non-byte)
557
+ * path never returns it at all.
558
+ */
531
559
  export function readEventsCursor(eventsPath: string, options: EventCursorOptions = {}): EventCursorResult {
532
560
  // Incremental byte-offset path: read only new bytes since last known offset
533
561
  if (options.fromByteOffset !== undefined) {
@@ -558,11 +586,19 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
558
586
  const limit = positiveInteger(options.limit);
559
587
  const events = limit !== undefined ? merged.slice(0, limit) : merged;
560
588
  const returnedMaxSeq = events.reduce((max, event) => Math.max(max, event.metadata?.seq ?? 0), sinceSeq);
589
+ // BR-08: `newState.byteOffset` sits past the ENTIRE delta read above —
590
+ // including the events the `limit` slice just dropped. Handing that back
591
+ // as `nextByteOffset` let a caller resume PAST events it never saw (silent
592
+ // loss: the delta read then returns an empty tail). Only expose the
593
+ // accelerator when this call delivered the whole delta; under truncation
594
+ // the caller must continue from `nextSeq` (see the doc comment on
595
+ // readEventsCursor).
596
+ const deliveredWholeDelta = limit === undefined || merged.length <= limit;
561
597
  return {
562
598
  events,
563
599
  nextSeq: returnedMaxSeq,
564
600
  total: merged.length,
565
- nextByteOffset: newState.byteOffset,
601
+ ...(deliveredWholeDelta ? { nextByteOffset: newState.byteOffset } : {}),
566
602
  generation: liveGen,
567
603
  };
568
604
  }
@@ -169,18 +169,83 @@ export interface CompactionResult {
169
169
  * 6. Return compaction stats
170
170
  */
171
171
  export function compactEventLog(eventsPath: string, config?: Partial<RotationConfig>): CompactionResult | undefined {
172
- const prepared = prepareCompaction(eventsPath, config);
173
- if (!prepared) return undefined;
174
- // FIX: Wrap entire read-compact-write-recover sequence in lock to prevent
175
- // event loss during compaction. Without lock, events can be appended between
176
- // read and write, lost silently.
172
+ // US-001 (2026-09-22): lock-scope reduction. The expensive part is the READ +
173
+ // parse of the whole log (prepareCompaction). It is read-only, so it does NOT
174
+ // need the append lock: hold the lock ONLY for the write+recover phase
175
+ // (applyCompactionUnlocked), which is what must be atomic w.r.t. appends.
176
+ //
177
+ // Why this is safe: appenders take the SAME lock, so an append can only land
178
+ // before our prepareCompaction read or after our applyCompactionUnlocked
179
+ // write — never inside the write. applyCompactionUnlocked already re-reads
180
+ // after the write and splices back any events that arrived in the window
181
+ // (C2 recovery), so no event is lost; it only becomes slightly more likely to
182
+ // splice, which is exactly the case it was built for.
183
+ //
184
+ // No concurrent-rotator guard is needed: this function is fully SYNCHRONOUS
185
+ // (prepareCompaction and withEventLogLockSync both block), so two calls on the
186
+ // same path cannot interleave within a process. An in-flight flag was tried
187
+ // and removed — it was untestable dead code (mutation survived), and the
188
+ // synchronous execution already serializes rotators. Cross-process rotators
189
+ // are serialized by the on-disk lock in withEventLogLockSync.
177
190
  //
178
191
  // NOTE (Round 24 BUG 1): callers ALREADY holding the event-log lock (e.g.
179
- // appendEventInsideLock in event-log.ts) must call applyCompactionUnlocked
192
+ // appendEventInsideLock in event-log.ts) must still call applyCompactionUnlocked
180
193
  // directly — calling compactEventLog from inside the lock deadlocks (the
181
194
  // mkdir lock is not re-entrant → 5s timeout → compaction never ran → the
182
195
  // log grew unbounded until events were silently dropped past 50MB).
183
- return withEventLogLockSync(eventsPath, () => applyCompactionUnlocked(eventsPath, prepared));
196
+ const prepared = prepareCompaction(eventsPath, config);
197
+ if (!prepared) return undefined;
198
+ return compactPreparedEventLog(eventsPath, prepared);
199
+ }
200
+
201
+ /**
202
+ * US-001 phase 3 — the LOCKED write phase, extracted so the read→write race is
203
+ * deterministically testable: a caller (test) can run prepareCompaction, append
204
+ * to the file (simulating another process landing events in the window), then
205
+ * call this and assert nothing was lost.
206
+ *
207
+ * prepareCompaction reads OUTSIDE the lock, so an appender may have added events
208
+ * between that read and this write. applyCompactionUnlocked writes exactly
209
+ * `prepared.lines` (the kept window from the snapshot), which would CLOBBER those
210
+ * newer events; its C2 recovery only re-splices events that were in `kept`, not
211
+ * ones it never saw. So under the lock we read the bytes appended after the
212
+ * snapshot offset and append them to the write. Appenders hold the same lock,
213
+ * so this read is a complete tail snapshot.
214
+ */
215
+ export function compactPreparedEventLog(
216
+ eventsPath: string,
217
+ prepared: { lines: string; originalSize: number; originalCount: number; kept: TeamEvent[] },
218
+ ): CompactionResult | undefined {
219
+ return withEventLogLockSync(eventsPath, () => {
220
+ const merged = { ...prepared, lines: prepared.lines + readTailSince(eventsPath, prepared.originalSize) };
221
+ return applyCompactionUnlocked(eventsPath, merged);
222
+ });
223
+ }
224
+
225
+ /**
226
+ * US-001: return the raw text appended to `filePath` after `offset`, or "" when
227
+ * nothing was appended / the file was replaced by a smaller one (a foreign
228
+ * rotation — caller then proceeds without a splice, which is the pre-US-001
229
+ * behaviour). Must be called while holding the event-log lock.
230
+ */
231
+ function readTailSince(filePath: string, offset: number): string {
232
+ try {
233
+ const size = fs.statSync(filePath).size;
234
+ if (size <= offset) return "";
235
+ const fd = fs.openSync(filePath, "r");
236
+ try {
237
+ const length = size - offset;
238
+ const buf = Buffer.alloc(length);
239
+ const read = fs.readSync(fd, buf, 0, length, offset);
240
+ const tail = buf.subarray(0, read).toString("utf-8");
241
+ // Only splice complete lines (a partial trailing write would corrupt JSONL).
242
+ return tail.endsWith("\n") ? tail : tail.slice(0, tail.lastIndexOf("\n") + 1);
243
+ } finally {
244
+ fs.closeSync(fd);
245
+ }
246
+ } catch {
247
+ return "";
248
+ }
184
249
  }
185
250
 
186
251
  /** Round 24 (BUG 1): the lock-free pre-read for compaction. Safe to run
@@ -61,6 +61,14 @@ function removeStaleRegistryLock(lockPath: string, staleMs: number): boolean {
61
61
  }
62
62
 
63
63
  function withRegistryLock<T>(fn: () => T): T {
64
+ // US-010 (2026-09-22): sleepSync below is INTENTIONAL — this lock is sync-only
65
+ // (fn is synchronous, and its callers at :352/:378 are sync). Converting the
66
+ // backoff to `await sleep` here would repeat the documented v0.9.26
67
+ // starvation: the sync spinner's sleepSync blocks the event loop while an
68
+ // in-process async holder's `await`-based release can never run. The lock is
69
+ // held only for a sub-millisecond read-modify-write and the loop is bounded by
70
+ // a 10s deadline, so blocking is bounded and rare (contention is cross-process
71
+ // only). See src/state/event-log/sequence-cache.ts:270 for the same reasoning.
64
72
  const filePath = registryLockPath();
65
73
  const staleMs = 30_000;
66
74
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
@@ -84,7 +92,11 @@ function withRegistryLock<T>(fn: () => T): T {
84
92
  break;
85
93
  } catch (error) {
86
94
  const code = (error as NodeJS.ErrnoException).code;
87
- if (code !== "EEXIST") throw error;
95
+ // Windows lock contention surfaces as EPERM/EACCES/EBUSY (not EEXIST) while
96
+ // another handle holds the lock file open — cross-process contention in
97
+ // CI (run 36019884258). Treat as contention: bounded retry via the 10s
98
+ // deadline below, mirroring isLockContention() in coordination/locks.ts.
99
+ if (code !== "EEXIST" && code !== "EPERM" && code !== "EACCES" && code !== "EBUSY") throw error;
88
100
  if (!removeStaleRegistryLock(filePath, staleMs) && Date.now() > deadline)
89
101
  throw new Error("Active-run registry is locked by another operation.");
90
102
  sleepSync(Math.min(250, 25 * 2 ** attempt));
@@ -193,19 +193,37 @@ function resolveRunStateRoot(cwd: string, runId: string): string | undefined {
193
193
  const now = Date.now();
194
194
  const cached = runStateRootCache.get(key);
195
195
  if (cached && cached.expiresAt > now) return cached.root;
196
- const runsRoot = path.join(scopeBaseRoot(cwd), DEFAULT_PATHS.state.runsSubdir);
197
- const scopedPath = resolveContainedRelativePath(runsRoot, runId, "runId");
198
- try {
199
- resolveRealContainedPath(runsRoot, runId);
200
- } catch {
201
- return undefined;
202
- }
203
- if (runStateRootCache.size >= RUN_STATE_ROOT_CACHE_MAX) {
204
- const oldest = runStateRootCache.keys().next().value;
205
- if (oldest !== undefined) runStateRootCache.delete(oldest);
196
+ // F-L1 (2026-09-22): listing (run-index scopedRunRoots) UNIONS the user and
197
+ // project run roots, but this resolution used scopeBaseRoot's single XOR
198
+ // pick — a run living under the OTHER root (e.g. created by another cwd or
199
+ // before the repo got a marker) listed fine yet FAILED every by-ID lookup:
200
+ // team status / prune / forget / scheduler provenance all returned "not
201
+ // found". Try the primary root first (hot path unchanged — one contained
202
+ // resolve + cache hit), then fall back to the other root. The no-repo case
203
+ // never falls back to a project path, mirroring scopedRunRoots' `if
204
+ // (projectRoot)` guard — resolution sees exactly what listing sees.
205
+ const candidates = useProjectState(cwd) ? [projectCrewRoot(cwd), userCrewRoot()] : [userCrewRoot()];
206
+ for (const root of candidates) {
207
+ const runsRoot = path.join(root, DEFAULT_PATHS.state.runsSubdir);
208
+ const scopedPath = resolveContainedRelativePath(runsRoot, runId, "runId");
209
+ // resolveRealContainedPath deliberately ACCEPTS a missing target (write-path
210
+ // semantics: ENOENT is fine, only symlink/escape violations throw) — so
211
+ // existence must be probed explicitly. Without this, the primary root
212
+ // always "wins" with a phantom path and the fallback can never run.
213
+ if (!fs.existsSync(path.join(scopedPath, DEFAULT_PATHS.state.manifestFile))) continue;
214
+ try {
215
+ resolveRealContainedPath(runsRoot, runId);
216
+ } catch {
217
+ continue; // symlink/escape violation under this root — try the other
218
+ }
219
+ if (runStateRootCache.size >= RUN_STATE_ROOT_CACHE_MAX) {
220
+ const oldest = runStateRootCache.keys().next().value;
221
+ if (oldest !== undefined) runStateRootCache.delete(oldest);
222
+ }
223
+ runStateRootCache.set(key, { root: scopedPath, expiresAt: now + RUN_STATE_ROOT_TTL_MS });
224
+ return scopedPath;
206
225
  }
207
- runStateRootCache.set(key, { root: scopedPath, expiresAt: now + RUN_STATE_ROOT_TTL_MS });
208
- return scopedPath;
226
+ return undefined;
209
227
  }
210
228
 
211
229
  // PERF (2026-08-24): the artifacts containment verdict (existsSync + lstat +
@@ -239,7 +257,13 @@ function validateRunManifestPaths(cwd: string, runId: string, manifest: TeamRunM
239
257
  manifest.eventsPath !== path.join(stateRoot, "events.jsonl")
240
258
  )
241
259
  return false;
242
- const artifactsParent = path.join(scopeBaseRoot(cwd), DEFAULT_PATHS.state.artifactsSubdir);
260
+ // F-L1 (2026-09-22): derive the artifacts parent from the RESOLVED stateRoot's
261
+ // base root (…/<base>/state/runs/<runId>), not scopeBaseRoot — cross-root
262
+ // runs (state under the fallback root, artifacts written beside it at
263
+ // creation) must validate against their OWN root; scopeBaseRoot would
264
+ // reject every fallback-resolved run and keep it invisible by ID.
265
+ const baseRoot = path.resolve(stateRoot, "..", "..", "..");
266
+ const artifactsParent = path.join(baseRoot, DEFAULT_PATHS.state.artifactsSubdir);
243
267
  const expectedArtifactsRoot = resolveContainedRelativePath(artifactsParent, runId, "runId");
244
268
  if (manifest.artifactsRoot !== expectedArtifactsRoot) return false;
245
269
  // PERF (2026-08-24): memoized verdict — see artifactsVerdictCache above.
@@ -409,9 +433,14 @@ export function createRunManifest(params: {
409
433
  runKind?: "team-run" | "goal-loop" | "dynamic-workflow";
410
434
  /** round-14 P1-5: typed workflow arguments for .dwf.ts scripts (ctx.args<T>()). */
411
435
  args?: unknown;
436
+ /** Deterministic-capture support (2026-09-22): pin the run ID and/or clock
437
+ * so tools like docs/ui-samples/capture.ts produce byte-stable output.
438
+ * Defaults preserve current behavior (generated id, wall clock). */
439
+ runId?: string;
440
+ now?: () => Date;
412
441
  }): { manifest: TeamRunManifest; tasks: TeamTaskState[]; paths: RunPaths } {
413
- const paths = createRunPaths(params.cwd);
414
- const now = new Date().toISOString();
442
+ const paths = createRunPaths(params.cwd, params.runId);
443
+ const now = (params.now ? params.now() : new Date()).toISOString();
415
444
  const tasks = params.workflow ? createTasksFromWorkflow(paths.runId, params.workflow, params.team, params.cwd, params.goal) : [];
416
445
  const manifest: TeamRunManifest = {
417
446
  schemaVersion: CURRENT_SCHEMA_VERSION,
@@ -479,7 +508,12 @@ export function createRunManifest(params: {
479
508
  return { manifest, tasks, paths };
480
509
  }
481
510
 
482
- export function saveRunManifest(manifest: TeamRunManifest): void {
511
+ export function saveRunManifest(manifest: TeamRunManifest, options?: { allowTerminalExit?: boolean }): TeamRunManifest {
512
+ // Finding 8 (2026-09-23): terminal-preserve at the WRITE layer. Mid-flight
513
+ // savers (task-runner artifact/progress writes carrying a stale in-memory
514
+ // "running" manifest) previously overwrote an externally-written terminal
515
+ // status; merge/finalize had their own guards but every other save site did
516
+ // not. Guard here so NO raw save can erase a terminal disk status.
483
517
  // FIX: Capture the cached tasks array + mtime/size BEFORE we invalidate the
484
518
  // cache. The previous implementation re-read tasks.json from disk after the
485
519
  // manifest write (a JSON.parse + fs.readFileSync per call), which defeated
@@ -502,12 +536,13 @@ export function saveRunManifest(manifest: TeamRunManifest): void {
502
536
  // which is always safe.
503
537
  invalidateRunCache(manifest.stateRoot);
504
538
  const manifestPath = path.join(manifest.stateRoot, "manifest.json");
539
+ const effective = preserveDiskTerminalStatus(manifest, options?.allowTerminalExit);
505
540
  // REVIEW FIX (2026-09-10): reverted WI-2.2's coalesced conversion —
506
541
  // saveRunManifest is a SYNCHRONOUS persist by name/contract (tests assert
507
542
  // it, broker loadRunManifestById is a cross-process reader, and the
508
543
  // statSync-based cache repopulation below needs the real post-write
509
544
  // mtime/size). The 50ms coalesce window broke all three.
510
- atomicWriteJson(manifestPath, manifest);
545
+ atomicWriteJson(manifestPath, effective);
511
546
  // FIX: Re-populate cache with actual mtime/size so loadRunManifestById
512
547
  // doesn't miss the cache on next read. Without this, every load until
513
548
  // TTL expires would hit disk because cached 0 !== any real mtime.
@@ -516,16 +551,17 @@ export function saveRunManifest(manifest: TeamRunManifest): void {
516
551
  // fresh tasks should call saveRunTasks or loadRunTasks separately.
517
552
  const manifestStat = fs.statSync(manifestPath);
518
553
  setManifestCache(manifest.stateRoot, {
519
- manifest,
554
+ manifest: effective,
520
555
  tasks: cachedTasks,
521
556
  manifestMtimeMs: manifestStat.mtimeMs,
522
557
  manifestSize: manifestStat.size,
523
558
  tasksMtimeMs: cachedTasksMtimeMs,
524
559
  tasksSize: cachedTasksSize,
525
560
  });
561
+ return effective;
526
562
  }
527
563
 
528
- export async function saveRunManifestAsync(manifest: TeamRunManifest): Promise<void> {
564
+ export async function saveRunManifestAsync(manifest: TeamRunManifest, options?: { allowTerminalExit?: boolean }): Promise<TeamRunManifest> {
529
565
  // FIX: Capture cached tasks array + mtime/size BEFORE invalidating, same
530
566
  // rationale as the sync saveRunManifest above. The async path previously
531
567
  // always set tasks: [] with mtime/size 0, so any cache hit was guaranteed
@@ -538,7 +574,8 @@ export async function saveRunManifestAsync(manifest: TeamRunManifest): Promise<v
538
574
  // after a crash. See saveRunManifest for full explanation.
539
575
  invalidateRunCache(manifest.stateRoot);
540
576
  const manifestPath = path.join(manifest.stateRoot, "manifest.json");
541
- await atomicWriteJsonAsync(manifestPath, manifest);
577
+ const effective = preserveDiskTerminalStatus(manifest, options?.allowTerminalExit);
578
+ await atomicWriteJsonAsync(manifestPath, effective);
542
579
  // FIX: Re-populate cache with actual mtime/size. See saveRunManifest.
543
580
  // RACE GUARD: another concurrent async save (OPT-02) may unlink+rewrite
544
581
  // manifest.json between our atomicWriteJsonAsync and this stat. If stat
@@ -553,13 +590,14 @@ export async function saveRunManifestAsync(manifest: TeamRunManifest): Promise<v
553
590
  manifestStat = { mtimeMs: 0, size: 0 };
554
591
  }
555
592
  setManifestCache(manifest.stateRoot, {
556
- manifest,
593
+ manifest: effective,
557
594
  tasks: cachedTasks,
558
595
  manifestMtimeMs: manifestStat.mtimeMs,
559
596
  manifestSize: manifestStat.size,
560
597
  tasksMtimeMs: cachedTasksMtimeMs,
561
598
  tasksSize: cachedTasksSize,
562
599
  });
600
+ return effective;
563
601
  }
564
602
 
565
603
  /**
@@ -803,6 +841,44 @@ function saveManifestAndTasksAtomicSync(manifest: TeamRunManifest, tasks: TeamTa
803
841
  export interface UpdateRunStatusOptions {
804
842
  data?: Record<string, unknown>;
805
843
  metadata?: Parameters<typeof appendEvent>[1]["metadata"];
844
+ /** Finding 8 (2026-09-23): allow leaving a TERMINAL disk status (resume is the
845
+ * only legitimate terminal-exit flow). Defaults to false — a raw save with an
846
+ * in-memory "running" manifest must never erase an externally-written
847
+ * cancelled/failed/completed status (live race team_20260923175042: cancel
848
+ * 17:51:08 → intermediate task-runner save re-wrote "running" → finalize
849
+ * completed — the cancel was fully erased). */
850
+ allowTerminalExit?: boolean;
851
+ }
852
+
853
+ /** Finding 8: statuses a disk write must never silently leave. Mirrors
854
+ * isRunTerminalPreserved (merge-loop.ts) — kept local to avoid a runtime dep
855
+ * from the store layer to the runtime layer. */
856
+ const DISK_TERMINAL_STATUSES: ReadonlySet<TeamRunManifest["status"]> = new Set(["cancelled", "failed", "completed"]);
857
+
858
+ /** Finding 8 write-layer guard: if the DISK manifest is terminal and the
859
+ * incoming write carries a NON-terminal status (the erase class — a mid-flight
860
+ * saver with a stale in-memory "running" manifest), preserve the disk
861
+ * status/summary/updatedAt while keeping every other incoming field (artifacts,
862
+ * usage, surface…). Terminal→terminal re-decisions (e.g. cancelling a run that
863
+ * just completed — pinned by resume-cancel.test.ts) are LEGITIMATE and pass
864
+ * through; they are governed by canTransitionRunStatus at the updateRunStatus
865
+ * layer. Returns the effective manifest to persist. */
866
+ function preserveDiskTerminalStatus(manifest: TeamRunManifest, allowTerminalExit: boolean | undefined): TeamRunManifest {
867
+ if (allowTerminalExit) return manifest;
868
+ // Terminal→terminal re-decisions pass through (governed by
869
+ // canTransitionRunStatus at the updateRunStatus layer); the guard applies
870
+ // ONLY to the erase class: a NON-terminal incoming status over terminal disk.
871
+ if (DISK_TERMINAL_STATUSES.has(manifest.status)) return manifest;
872
+ try {
873
+ const manifestPath = path.join(manifest.stateRoot, "manifest.json");
874
+ // Raw read (no cache): cross-process cancel writes must be seen NOW.
875
+ const raw = fs.readFileSync(manifestPath, "utf-8");
876
+ const disk = JSON.parse(raw) as TeamRunManifest;
877
+ if (!DISK_TERMINAL_STATUSES.has(disk.status) || disk.status === manifest.status) return manifest;
878
+ return { ...manifest, status: disk.status, summary: disk.summary, updatedAt: disk.updatedAt };
879
+ } catch {
880
+ return manifest; // no disk manifest yet (create) or unreadable — normal write
881
+ }
806
882
  }
807
883
 
808
884
  export function updateRunStatus(
@@ -820,7 +896,21 @@ export function updateRunStatus(
820
896
  updatedAt: new Date().toISOString(),
821
897
  summary: summary ?? manifest.summary,
822
898
  };
823
- saveRunManifest(updated);
899
+ // Finding 8 (2026-09-23): the write-layer guard may PRESERVE a terminal disk
900
+ // status (an external cancel/reconciler write the in-memory manifest never
901
+ // saw). In that case do NOT emit run.<status>, do NOT flip the manifest —
902
+ // record the refusal and return the preserved state. Resume passes
903
+ // allowTerminalExit for its legitimate cancelled→running transition.
904
+ const saved = saveRunManifest(updated, { allowTerminalExit: options.allowTerminalExit });
905
+ if (saved.status !== status) {
906
+ appendEvent(saved.eventsPath, {
907
+ type: "run.terminal_preserved",
908
+ runId: saved.runId,
909
+ message: `Preserved terminal status '${saved.status}'; refused in-memory transition to '${status}'.`,
910
+ data: { preserved: saved.status, refused: status },
911
+ });
912
+ return saved;
913
+ }
824
914
  // Unregister from active-run-index when run reaches a terminal status.
825
915
  // Without this, stale entries accumulate (e.g. integration tests in /tmp) and
826
916
  // Pi UI shows ghost "queued" runs that are actually completed/failed/cancelled.
@@ -543,6 +543,10 @@ export interface GoalLoopState {
543
543
  nextTurnFeedback?: string;
544
544
  /** The team-run of the current in-flight turn (for cancel/steer). */
545
545
  currentRunId?: string;
546
+ /** GL-1b (2026-09-22): why the last turn (or the loop itself) failed —
547
+ * persisted at goal level because turn-run dirs are pruned (keep=10 at
548
+ * session start) and otherwise the reason becomes unrecoverable. */
549
+ lastTurnError?: string;
546
550
  verdicts: GoalVerdict[];
547
551
  history: {
548
552
  runId: string;
@@ -178,8 +178,17 @@ export function renderAgentsPane(snapshot: RunUiSnapshot | undefined, options: R
178
178
  stats.push(liveHandle.modelName);
179
179
  }
180
180
  } else if (agent.startedAt) {
181
- const ms = nowMs - new Date(agent.startedAt).getTime();
182
- if (Number.isFinite(ms)) stats.push(alignMetric(formatDuration(ms), DURATION_METRIC_WIDTH));
181
+ // Tier 13 (2026-09-21): a FINISHED agent's span must end at completedAt,
182
+ // never at the wall clock. The naive `nowMs - startedAt` made every
183
+ // completed agent of an old run inflate forever — the dashboard showed
184
+ // `01_explore … 29m39s` while the tool card showed the real `8m22s` for
185
+ // the same agent. Mirror computeLiveDurationMs: prefer a sane completedAt,
186
+ // else fall back to now; never emit a negative or absurd span.
187
+ const startedMs = new Date(agent.startedAt).getTime();
188
+ const rawCompleted = agent.completedAt ? new Date(agent.completedAt).getTime() : Number.NaN;
189
+ const completedMs = Number.isFinite(rawCompleted) && rawCompleted >= startedMs && rawCompleted <= nowMs ? rawCompleted : nowMs;
190
+ const ms = completedMs - startedMs;
191
+ if (Number.isFinite(ms) && ms >= 0) stats.push(alignMetric(formatDuration(ms), DURATION_METRIC_WIDTH));
183
192
  }
184
193
 
185
194
  const statsStr = stats.length ? ` · ${stats.join(" ")}` : "";
@@ -32,6 +32,32 @@ function isActiveTask(task: TeamTaskState): boolean {
32
32
  return task.status === "running";
33
33
  }
34
34
 
35
+ /**
36
+ * FINDING 5 (2026-09-23 battery): health ticks must count task statuses from
37
+ * DISK truth, not the (possibly lagging) snapshot cache. A worker parked on
38
+ * `ask` transitions running → waiting on disk while the cached snapshot can
39
+ * still show `running` — in that window the parked worker counted as
40
+ * active-without-heartbeat and fired a false "dead worker" notification (live:
41
+ * run team_20260923100114, 01_explore parked on ask, later answered+resumed).
42
+ *
43
+ * Overlay FRESH task statuses (and heartbeats) from disk onto the cached
44
+ * snapshot before summarizing. Returns the ORIGINAL snapshot object when
45
+ * nothing diverged (cheap identity check — callers use it to decide cache
46
+ * invalidation), or a shallow copy with the diverging tasks replaced.
47
+ */
48
+ export function overlayFreshTaskStatuses(snapshot: RunUiSnapshot, freshTasks: TeamTaskState[]): RunUiSnapshot {
49
+ if (freshTasks.length === 0) return snapshot;
50
+ const freshById = new Map(freshTasks.map((task) => [task.id, task]));
51
+ let diverged = false;
52
+ const tasks = snapshot.tasks.map((task) => {
53
+ const fresh = freshById.get(task.id);
54
+ if (!fresh || (fresh.status === task.status && fresh.heartbeat === task.heartbeat)) return task;
55
+ diverged = true;
56
+ return { ...task, status: fresh.status, heartbeat: fresh.heartbeat };
57
+ });
58
+ return diverged ? { ...snapshot, tasks } : snapshot;
59
+ }
60
+
35
61
  export function summarizeHeartbeats(snapshot: RunUiSnapshot, opts: HeartbeatSummaryOptions = {}): HeartbeatSummary {
36
62
  const staleMs = opts.staleMs ?? 60_000;
37
63
  const deadMs = opts.deadMs ?? 5 * 60_000;
@@ -55,6 +81,14 @@ export function summarizeHeartbeats(snapshot: RunUiSnapshot, opts: HeartbeatSumm
55
81
  const runTerminal = isTerminalRunStatus(snapshot.manifest.status);
56
82
  for (const task of snapshot.tasks) {
57
83
  if (runTerminal || !isActiveTask(task)) continue;
84
+ // Guest-child tasks (delegate subagents, agent === "delegate") have no
85
+ // heartbeat channel — they never write task.heartbeat and complete in
86
+ // seconds; their lifecycle is owned by delegate.requested/admitted/
87
+ // completed broker events. Counting them here fired a false "N worker(s)
88
+ // missing heartbeat" ambient for every delegate-using run (live:
89
+ // team_20260926033657_2b6c6d2610b26d9d, 2026-09-26 battery — same
90
+ // gc-blindness root shape as the heartbeat-watcher fix).
91
+ if (task.agent === "delegate") continue;
58
92
  const heartbeat = task.heartbeat;
59
93
  if (!heartbeat) {
60
94
  summary.missing += 1;
@@ -33,6 +33,7 @@ import * as fs from "node:fs";
33
33
  import * as path from "node:path";
34
34
  import { type KeyId, matchesKey } from "@earendil-works/pi-tui";
35
35
  import { getCrewEnv } from "../config/env-vars.ts";
36
+ import { projectCrewRoot } from "../utils/paths.ts";
36
37
  import { keyOf } from "./key-utils.ts";
37
38
 
38
39
  export const DASHBOARD_KEYS = {
@@ -41,6 +42,8 @@ export const DASHBOARD_KEYS = {
41
42
  help: ["?"],
42
43
  root: {
43
44
  summary: ["u"],
45
+ /** US-020: cancel the selected run (2-step confirm in the dashboard). */
46
+ cancel: ["x"],
44
47
  artifacts: ["a"],
45
48
  api: ["i"],
46
49
  agents: ["d"],
@@ -112,6 +115,7 @@ export type DashboardKeyAction =
112
115
  | "live-conversation"
113
116
  | "reload"
114
117
  | "browser"
118
+ | "cancel"
115
119
  | "pane-agents"
116
120
  | "pane-progress"
117
121
  | "pane-mailbox"
@@ -256,6 +260,9 @@ const DEFAULT_BINDINGS: readonly KeyBinding[] = [
256
260
  // safe; inside the browser overlay itself "p" is free because overlays
257
261
  // are mutually exclusive (see keybinding-map.ts header note).
258
262
  { keys: DASHBOARD_KEYS.root.browser, action: "browser" },
263
+ // US-020: x → cancel the selected run (unscoped; the schedules pane's X
264
+ // delete is pane-scoped uppercase — pass-1 exact match keeps them distinct).
265
+ { keys: DASHBOARD_KEYS.root.cancel, action: "cancel" },
259
266
  { keys: DASHBOARD_KEYS.pane.agents, action: "pane-agents" },
260
267
  { keys: DASHBOARD_KEYS.pane.progress, action: "pane-progress" },
261
268
  { keys: DASHBOARD_KEYS.pane.mailbox, action: "pane-mailbox" },
@@ -418,7 +425,10 @@ export type KeybindingOverride = Partial<Record<DashboardKeyAction | OverlayBind
418
425
  const KEYBINDINGS_ENV = "PI_CREW_KEYBINDINGS";
419
426
 
420
427
  /** Every dispatched action + every `overlay:*` binding is a valid override target. */
421
- const VALID_OVERRIDE_ACTIONS: ReadonlySet<string> = new Set<string>([...DEFAULT_BINDINGS.map((b) => b.action), ...OVERLAY_BINDING_KEYS]);
428
+ export const VALID_OVERRIDE_ACTIONS: ReadonlySet<string> = new Set<string>([
429
+ ...DEFAULT_BINDINGS.map((b) => b.action),
430
+ ...OVERLAY_BINDING_KEYS,
431
+ ]);
422
432
 
423
433
  /** Coerce an unknown parsed value into a safe {@link KeybindingOverride}. */
424
434
  function parseKeybindingOverride(raw: unknown): KeybindingOverride {
@@ -541,10 +551,17 @@ function computeEffectiveBindings(overrides: KeybindingOverride): EffectiveBindi
541
551
  return { bindings, overlayBindings, reverted: [...reverted] };
542
552
  }
543
553
 
544
- /** Read the `keybindings` section from `<cwd>/.crew/config.json`. */
554
+ /**
555
+ * Read the `keybindings` section from the project config.
556
+ *
557
+ * RR-020 Fix 4: resolve via `projectCrewRoot(cwd)` instead of a literal
558
+ * `<cwd>/.crew` — the config lives in whichever layout the project actually
559
+ * uses (`.crew/` or `.pi/teams/`), so a `.pi/teams` project's keybinding
560
+ * overrides are no longer silently ignored.
561
+ */
545
562
  function readConfigKeybindings(cwd: string): KeybindingOverride {
546
563
  try {
547
- const raw: unknown = JSON.parse(fs.readFileSync(path.join(cwd, ".crew", "config.json"), "utf-8"));
564
+ const raw: unknown = JSON.parse(fs.readFileSync(path.join(projectCrewRoot(cwd), "config.json"), "utf-8"));
548
565
  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return {};
549
566
  return parseKeybindingOverride((raw as Record<string, unknown>).keybindings);
550
567
  } catch {
@@ -565,7 +582,8 @@ function readEnvKeybindings(): KeybindingOverride {
565
582
 
566
583
  function configKeybindingsMtime(cwd: string): number | undefined {
567
584
  try {
568
- return fs.statSync(path.join(cwd, ".crew", "config.json")).mtimeMs;
585
+ // RR-020 Fix 4: same layout-resolved config path as readConfigKeybindings.
586
+ return fs.statSync(path.join(projectCrewRoot(cwd), "config.json")).mtimeMs;
569
587
  } catch {
570
588
  return undefined;
571
589
  }
@@ -62,11 +62,14 @@ export class LiveConversationOverlay {
62
62
  private handle: LiveAgentHandle;
63
63
  private theme: CrewTheme;
64
64
 
65
- constructor(handle: LiveAgentHandle, theme: CrewTheme, columns = 80, rows = 24) {
65
+ private readonly nowMs: () => number;
66
+
67
+ constructor(handle: LiveAgentHandle, theme: CrewTheme, columns = 80, rows = 24, nowMs?: () => number) {
66
68
  this.handle = handle;
67
69
  this.theme = theme;
68
70
  this.columns = columns;
69
71
  this.rows = rows;
72
+ this.nowMs = nowMs ?? Date.now;
70
73
  // R8: Subscribe to real session events if available
71
74
  const session = handle.session as Record<string, unknown>;
72
75
  if (typeof session.subscribe === "function") {
@@ -117,7 +120,7 @@ export class LiveConversationOverlay {
117
120
 
118
121
  private refreshSummary(): void {
119
122
  const act = this.handle.activity;
120
- const summary = `${LiveConversationOverlay.SUMMARY_PREFIX}[${formatCount(act.turnCount ?? 0, "turn")} · ${formatCount(act.toolUses ?? 0, "tool")} · ${(computeLiveDurationMs(act) / 1000).toFixed(1)}s]`;
123
+ const summary = `${LiveConversationOverlay.SUMMARY_PREFIX}[${formatCount(act.turnCount ?? 0, "turn")} · ${formatCount(act.toolUses ?? 0, "tool")} · ${(computeLiveDurationMs(act, this.nowMs()) / 1000).toFixed(1)}s]`;
121
124
  const lastLine = this.cachedLines[this.cachedLines.length - 1];
122
125
  if (lastLine?.startsWith(LiveConversationOverlay.SUMMARY_PREFIX)) {
123
126
  this.cachedLines[this.cachedLines.length - 1] = summary;
@@ -267,7 +270,7 @@ export class LiveConversationOverlay {
267
270
  if (act.maxTurns != null) parts.push(`turn ${act.turnCount ?? 0}/${act.maxTurns}`);
268
271
  else if ((act.turnCount ?? 0) > 0) parts.push(`turn ${act.turnCount}`);
269
272
  if ((act.toolUses ?? 0) > 0) parts.push(formatCount(act.toolUses ?? 0, "tool"));
270
- parts.push(`${(computeLiveDurationMs(act) / 1000).toFixed(1)}s`);
273
+ parts.push(`${(computeLiveDurationMs(act, this.nowMs()) / 1000).toFixed(1)}s`);
271
274
  try {
272
275
  const ctxPct = this.handle.session.getSessionStats?.()?.contextUsage?.percent;
273
276
  if (ctxPct != null) {