pi-crew 0.11.1 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/CHANGELOG.md +82 -9
  2. package/README.md +161 -1036
  3. package/agents/verifier.md +18 -7
  4. package/dist/index.mjs +742 -91462
  5. package/docs/README.md +57 -46
  6. package/docs/architecture.md +87 -33
  7. package/docs/commands-reference.md +9 -5
  8. package/docs/troubleshooting.md +3 -2
  9. package/package.json +5 -4
  10. package/schema.json +29 -0
  11. package/skills/real-test-pi-crew/SKILL.md +206 -41
  12. package/src/agents/agent-config.ts +1 -1
  13. package/src/agents/discover-agents.ts +1 -1
  14. package/src/config/config-validation.ts +15 -0
  15. package/src/config/config.ts +47 -13
  16. package/src/config/env-vars.ts +35 -0
  17. package/src/config/types.ts +19 -0
  18. package/src/errors.ts +2 -2
  19. package/src/extension/async-notifier.ts +55 -8
  20. package/src/extension/crew-input-router.ts +9 -2
  21. package/src/extension/help.ts +21 -10
  22. package/src/extension/knowledge-injection.ts +2 -1
  23. package/src/extension/management.ts +8 -3
  24. package/src/extension/notification-sink.ts +17 -0
  25. package/src/extension/plan-orchestrate.ts +19 -3
  26. package/src/extension/registration/command-utils.ts +28 -2
  27. package/src/extension/registration/commands/dashboard.ts +11 -1
  28. package/src/extension/registration/commands/manage.ts +31 -15
  29. package/src/extension/registration/commands/run.ts +24 -2
  30. package/src/extension/registration/commands/shared.ts +34 -10
  31. package/src/extension/registration/commands/status.ts +25 -2
  32. package/src/extension/registration/context-builder.ts +8 -2
  33. package/src/extension/registration/foreground-run-controller.ts +2 -2
  34. package/src/extension/registration/health-notify-policy.ts +100 -0
  35. package/src/extension/registration/lazy-configurers.ts +38 -0
  36. package/src/extension/registration/lifecycle-handlers.ts +107 -40
  37. package/src/extension/registration/lifecycle.ts +77 -10
  38. package/src/extension/registration/observability.ts +125 -53
  39. package/src/extension/registration/registration-types.ts +7 -5
  40. package/src/extension/registration/runtime-cleanup.ts +9 -3
  41. package/src/extension/registration/subagent-helpers.ts +38 -0
  42. package/src/extension/registration/team-tool.ts +9 -2
  43. package/src/extension/registration/wire-cross-extension.ts +30 -0
  44. package/src/extension/run-compare.ts +220 -0
  45. package/src/extension/run-export.ts +37 -5
  46. package/src/extension/run-maintenance.ts +155 -5
  47. package/src/extension/team-manager-command.ts +3 -2
  48. package/src/extension/team-tool/dispatch/index.ts +3 -2
  49. package/src/extension/team-tool/dispatch/manage.ts +5 -2
  50. package/src/extension/team-tool/goal-wrap.ts +7 -0
  51. package/src/extension/team-tool/goal.ts +16 -1
  52. package/src/extension/team-tool/handle-settings.ts +19 -2
  53. package/src/extension/team-tool/health-monitor.ts +21 -7
  54. package/src/extension/team-tool/lifecycle-actions.ts +49 -1
  55. package/src/extension/team-tool/parallel-dispatch.ts +16 -1
  56. package/src/extension/team-tool/plan.ts +10 -0
  57. package/src/extension/team-tool/routing-hint.ts +63 -0
  58. package/src/extension/team-tool/run-intent.ts +1 -1
  59. package/src/extension/team-tool/run.ts +1 -1
  60. package/src/extension/team-tool/status.ts +4 -0
  61. package/src/extension/team-tool.ts +52 -6
  62. package/src/extension/webhook-notify.ts +382 -0
  63. package/src/observability/metric-sink.ts +12 -2
  64. package/src/prompt/prompt-runtime.ts +82 -31
  65. package/src/prompt/worker-events-channel.ts +12 -0
  66. package/src/runtime/README.md +1 -1
  67. package/src/runtime/async-runner.ts +123 -1
  68. package/src/runtime/background-runner.ts +313 -234
  69. package/src/runtime/broker/crew-broker.ts +76 -71
  70. package/src/runtime/broker/delegate/delegate-event.ts +12 -8
  71. package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
  72. package/src/runtime/broker/protocol/connection-state.ts +10 -0
  73. package/src/runtime/broker/protocol/lock-busy.ts +76 -0
  74. package/src/runtime/broker/wait-status-cache.ts +1 -1
  75. package/src/runtime/child-pi/child-pi-timers.ts +1 -1
  76. package/src/runtime/child-pi/mock-fixtures.ts +48 -0
  77. package/src/runtime/crew-agent-records.ts +337 -45
  78. package/src/runtime/deadletter.ts +43 -1
  79. package/src/runtime/delegate-spawn.ts +21 -3
  80. package/src/runtime/deterministic-ast.ts +14 -2
  81. package/src/runtime/dispatch-batch.ts +72 -5
  82. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
  83. package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
  84. package/src/runtime/goal-workflow/goal-loop-runner.ts +105 -4
  85. package/src/runtime/group-join.ts +11 -3
  86. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  87. package/src/runtime/live-session/live-session-runtime.ts +1 -1
  88. package/src/runtime/model/model-fallback.ts +21 -1
  89. package/src/runtime/model/pi-args.ts +8 -10
  90. package/src/runtime/model/runtime-warmup.ts +2 -0
  91. package/src/runtime/output/output-validator.ts +10 -5
  92. package/src/runtime/plan-approval.ts +14 -1
  93. package/src/runtime/recovery/crash-recovery.ts +96 -49
  94. package/src/runtime/run-worker.ts +12 -1
  95. package/src/runtime/scheduling/global-worker-cap.ts +13 -6
  96. package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
  97. package/src/runtime/scheduling/scheduler.ts +49 -13
  98. package/src/runtime/scheduling/semaphore.ts +148 -20
  99. package/src/runtime/scratchpad/README.md +1 -1
  100. package/src/runtime/scratchpad/protocol.ts +1 -1
  101. package/src/runtime/settings-store.ts +1 -1
  102. package/src/runtime/skill-instructions.ts +22 -0
  103. package/src/runtime/stale-reconciler.ts +85 -13
  104. package/src/runtime/subagent-manager.ts +43 -13
  105. package/src/runtime/task-runner/child-executor.ts +17 -6
  106. package/src/runtime/task-runner/post-execution.ts +44 -14
  107. package/src/runtime/task-runner/pre-execution.ts +26 -2
  108. package/src/runtime/task-runner/prompt-builder.ts +150 -45
  109. package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
  110. package/src/runtime/task-runner.ts +28 -1
  111. package/src/runtime/team-runner.ts +42 -2
  112. package/src/runtime/verification/verification-worktree.ts +3 -0
  113. package/src/runtime/workspace-lock.ts +4 -1
  114. package/src/schema/config-schema.ts +14 -0
  115. package/src/schema/team-tool-schema.ts +17 -0
  116. package/src/state/atomic-write.ts +53 -0
  117. package/src/state/contracts.ts +109 -0
  118. package/src/state/coordination/locks.ts +191 -33
  119. package/src/state/coordination/mailbox.ts +147 -17
  120. package/src/state/crew-init.ts +87 -12
  121. package/src/state/event-log/cursor.ts +37 -1
  122. package/src/state/event-log/event-log-rotation.ts +72 -7
  123. package/src/state/stores/active-run-registry.ts +13 -1
  124. package/src/state/stores/state-store.ts +120 -23
  125. package/src/state/types.ts +4 -0
  126. package/src/ui/dashboard-panes/agents-pane.ts +22 -10
  127. package/src/ui/heartbeat-aggregator.ts +34 -0
  128. package/src/ui/inline-panel/index.ts +1 -1
  129. package/src/ui/keybinding-map.ts +22 -4
  130. package/src/ui/live-conversation-overlay.ts +6 -3
  131. package/src/ui/render-diff.ts +19 -6
  132. package/src/ui/run-dashboard.ts +109 -6
  133. package/src/ui/run-snapshot-cache.ts +27 -4
  134. package/src/ui/spinner.ts +26 -2
  135. package/src/ui/tool-progress-formatter.ts +2 -1
  136. package/src/ui/tool-renderers/brief-mode.ts +2 -1
  137. package/src/ui/tool-renderers/index.ts +3 -3
  138. package/src/ui/widget/task-list.ts +8 -5
  139. package/src/ui/widget/widget-model.ts +3 -1
  140. package/src/utils/incremental-reader.ts +11 -3
  141. package/src/utils/paths.ts +94 -12
  142. package/src/utils/project-markers.ts +40 -0
  143. package/src/worktree/worktree-manager.ts +206 -26
  144. package/tsconfig.json +2 -0
  145. package/workflows/distill.workflow.md +3 -3
  146. package/workflows/fast-fix.workflow.md +1 -1
  147. package/workflows/plan-execute.workflow.md +1 -1
  148. package/workflows/review.workflow.md +1 -1
  149. package/workflows/strict-fast-fix.workflow.md +1 -1
  150. package/docs/migration-v0.4-v0.5.md +0 -208
  151. package/docs/runtime-flow.md +0 -148
  152. package/src/ui/card-colors.ts +0 -168
  153. package/src/utils/completion-dedupe.ts +0 -63
  154. package/src/utils/resolve-shell.ts +0 -36
@@ -66,6 +66,7 @@ export function checkWorktreeSandboxAvailable(cwd: string): { available: true; c
66
66
  cwd,
67
67
  stdio: ["ignore", "pipe", "pipe"],
68
68
  encoding: "utf-8",
69
+ timeout: 5000,
69
70
  }).trim();
70
71
  if (!toplevel)
71
72
  return {
@@ -77,6 +78,7 @@ export function checkWorktreeSandboxAvailable(cwd: string): { available: true; c
77
78
  cwd,
78
79
  stdio: ["ignore", "pipe", "pipe"],
79
80
  encoding: "utf-8",
81
+ timeout: 5000,
80
82
  }).trim();
81
83
  if (!commitSha)
82
84
  return {
@@ -92,6 +94,7 @@ export function checkWorktreeSandboxAvailable(cwd: string): { available: true; c
92
94
  cwd,
93
95
  stdio: ["ignore", "pipe", "pipe"],
94
96
  encoding: "utf-8",
97
+ timeout: 5000,
95
98
  }).trim();
96
99
  if (status.length > 0)
97
100
  return {
@@ -187,7 +187,10 @@ function claimLock(lockPath: string, contents: WorkspaceLockContents, staleRecla
187
187
  return true;
188
188
  } catch (error) {
189
189
  const code = (error as NodeJS.ErrnoException).code;
190
- if (code !== "EEXIST") throw error;
190
+ // EEXIST → held. Windows contention can surface as EPERM/EACCES/EBUSY
191
+ // while another handle has the lock open — same meaning here: not acquired
192
+ // (caller backs off / queues), never an abort.
193
+ if (code !== "EEXIST" && code !== "EPERM" && code !== "EACCES" && code !== "EBUSY") throw error;
191
194
  return false;
192
195
  }
193
196
  };
@@ -231,6 +231,20 @@ export const PiTeamsNotificationsConfigSchema = Type.Object(
231
231
  batchWindowMs: Type.Optional(Type.Integer({ minimum: 0 })),
232
232
  quietHours: Type.Optional(Type.String({ pattern: "^\\d{2}:\\d{2}-\\d{2}:\\d{2}$" })),
233
233
  sinkRetentionDays: Type.Optional(Type.Integer({ minimum: 1, maximum: 90 })),
234
+ // US-030: whole block is `sensitive` (terminal mark) — project config drops
235
+ // it, so an untrusted repo cannot set a webhook URL or bypass the SSRF
236
+ // guard via `allowLocalhost`. User config only.
237
+ webhook: Type.Optional(
238
+ Type.Object(
239
+ {
240
+ url: Type.String({ minLength: 1, pattern: "^https?://" }),
241
+ enabled: Type.Optional(Type.Boolean()),
242
+ secret: Type.Optional(Type.String({ minLength: 1 })),
243
+ allowLocalhost: Type.Optional(Type.Boolean()),
244
+ },
245
+ { additionalProperties: false, sensitive: true },
246
+ ),
247
+ ),
234
248
  },
235
249
  { additionalProperties: false },
236
250
  );
@@ -148,6 +148,13 @@ const sharedFields = {
148
148
  }),
149
149
  ),
150
150
  taskId: Type.Optional(Type.String({ description: "Task ID for respond action." })),
151
+ // US-021: the two runs to diff. Items allow "" so model callers that emit
152
+ // unset strings are filtered at the handler, same policy as runId.
153
+ runIds: Type.Optional(
154
+ Type.Array(Type.String({ pattern: "^$|^[A-Za-z0-9_-]+$" }), {
155
+ description: "The two run IDs to compare: { action: 'compare', runIds: ['team_a', 'team_b'] }.",
156
+ }),
157
+ ),
151
158
  message: Type.Optional(Type.String({ description: "Message for respond action." })),
152
159
  async: Type.Optional(
153
160
  Type.Boolean({
@@ -277,6 +284,12 @@ const sharedFields = {
277
284
  // Empty-string unset marker accepted (Tier-9: models emit "" when unset).
278
285
  // 0 accepted as "unset/disabled" (models emit 0 for off); still rejects 1-999
279
286
  // as the MISCONFIGURATION GUARD against typo'd silent-abort configs.
287
+ // Stringified numbers accepted (same pi-ai coercion the sibling budget
288
+ // params handle): this Union has a Literal("") branch, so pi-ai stringifies
289
+ // numeric arguments (budgetTotal: 100000 → "100000") and the call died at
290
+ // schema validation before any coercion could run — found live 2026-09-21
291
+ // when `team action='goal' budgetTotal=100000` was rejected. Coerced back
292
+ // to a number by normalizeLooseNumericFields before handlers run.
280
293
  Type.Union(
281
294
  [
282
295
  Type.Literal(""),
@@ -284,6 +297,7 @@ const sharedFields = {
284
297
  Type.Number({
285
298
  minimum: 1000,
286
299
  }),
300
+ Type.String({ pattern: NUMERIC_STRING_RE }),
287
301
  ],
288
302
  {
289
303
  description:
@@ -431,6 +445,7 @@ const MANAGE_ACTIONS = [
431
445
  "import",
432
446
  "imports",
433
447
  "export",
448
+ "compare",
434
449
  ] as const;
435
450
  const manageActions = Type.Optional(buildStringEnum(MANAGE_ACTIONS, ACTION_DESCRIPTION));
436
451
 
@@ -515,6 +530,8 @@ export interface TeamToolParamsValue {
515
530
  task?: string;
516
531
  singleAgent?: boolean;
517
532
  runId?: string;
533
+ /** (compare) The two runs to diff: { action: 'compare', runIds: ['team_a', 'team_b'] } (US-021). */
534
+ runIds?: string[];
518
535
  taskId?: string;
519
536
  message?: string;
520
537
  async?: boolean;
@@ -1122,6 +1122,59 @@ function cancelPendingCoalescedWrite(filePath: string): void {
1122
1122
  }
1123
1123
  }
1124
1124
 
1125
+ /**
1126
+ * F10 / RR-016: path-scoped PUBLIC cancel for callers that DELETE a file whose
1127
+ * coalesced write is still buffered. Without it the pending timer fires after
1128
+ * the unlink and RE-CREATES the file with stale content — the measured
1129
+ * `removeCrewAgent` defect (`agents/<task>/status.json` reappearing with
1130
+ * `status:"running"` after the exit drain, while `agents.json` stayed empty).
1131
+ *
1132
+ * `cancelPendingCoalescedWrite` above already has exactly this semantics for
1133
+ * immediate-write supersession; this export only widens the seam so a deleter
1134
+ * can use it too. Returns true when an entry was pending (and was cancelled).
1135
+ */
1136
+ export function cancelPendingCoalescedWriteForPath(filePath: string): boolean {
1137
+ const pending = pendingAtomicWrites.has(filePath);
1138
+ cancelPendingCoalescedWrite(filePath);
1139
+ return pending;
1140
+ }
1141
+
1142
+ /** @internal Test/diagnostic hook: is a coalesced write pending for this exact path? */
1143
+ export function hasPendingCoalescedWrite(filePath: string): boolean {
1144
+ // (kept adjacent for discoverability; pendingCoalescedWriteCount below serves teardown drains)
1145
+ return pendingAtomicWrites.has(filePath);
1146
+ }
1147
+
1148
+ /** Number of coalesced writes currently pending (test teardown quiesce loops). */
1149
+ export function pendingCoalescedWriteCount(): number {
1150
+ return pendingAtomicWrites.size;
1151
+ }
1152
+
1153
+ /**
1154
+ * Read the buffered (not-yet-flushed) value for `filePath`, if any.
1155
+ *
1156
+ * F06 / RR-017: gives callers a read-after-write view WITHOUT forcing the
1157
+ * pending coalesced write to disk. `readCrewAgents()` previously called
1158
+ * `flushPendingAtomicWrites(agentsPath)` before every read, and since
1159
+ * `upsertCrewAgent` always reads first, EVERY non-terminal upsert destroyed the
1160
+ * coalescing window it had just created (20 progress upserts → 19 agents.json
1161
+ * renames). Overlaying this snapshot on the on-disk content preserves
1162
+ * read-after-write semantics with zero I/O.
1163
+ *
1164
+ * The value is returned by reference, exactly like the flush path stringifies
1165
+ * `entry.value` at flush time — callers must treat it as read-only.
1166
+ */
1167
+ export function peekPendingCoalescedWrite<T>(filePath: string): T | undefined {
1168
+ const value = pendingAtomicWrites.get(filePath)?.value;
1169
+ if (value === undefined) return undefined;
1170
+ // RM-01 (2026-09-22): return a DEEP COPY, not a reference. The flush path
1171
+ // stringifies `entry.value` at flush time, so a caller mutating the returned
1172
+ // object would corrupt the not-yet-flushed buffer (a write-then-flush
1173
+ // alias bug). All in-tree payloads are JSON-shaped, so structuredClone is
1174
+ // exact; the only caller is readCrewAgents (bounded per-run record list).
1175
+ return structuredClone(value) as T;
1176
+ }
1177
+
1125
1178
  /**
1126
1179
  * Flush every queued coalesced write synchronously. Safe to call any time.
1127
1180
  *
@@ -142,6 +142,7 @@ export const TEAM_EVENT_TYPES = [
142
142
  "goal.turn_evaluated",
143
143
  "goal.budget_warning",
144
144
  "goal.loop_end",
145
+ "goal.loop_error",
145
146
  "goal.feedback_steered",
146
147
  "goal.state_changed",
147
148
  // Dynamic workflow events (P2) — script-driven orchestration.
@@ -155,6 +156,114 @@ export const TEAM_EVENT_TYPES = [
155
156
  // RLM/scratchpad adoption metrics (plan I5)
156
157
  "scratchpad.cell",
157
158
  "scratchpad.restored",
159
+ // ─── 2026-09-17 drift closure ─────────────────────────────────────────────
160
+ // The remaining types below were ALREADY EMITTED in production but never
161
+ // registered — they were silent to consumers of TEAM_EVENT_TYPES. The
162
+ // check:event-types gate also had a detection bug (conditional `type:`
163
+ // expressions like `type: error ? "task.failed" : ...` were invisible),
164
+ // which is why this drift accumulated unnoticed. Registered here grouped
165
+ // by prefix; verified against literal emit sites (see
166
+ // scripts/check-event-types-registry.mjs and the 2026-09-17 review
167
+ // verification, §6.3). The gate now runs with --enforce in CI.
168
+ // Adaptive planning (goal-workflow/adaptive-plan.ts)
169
+ "adaptive.plan_injected",
170
+ "adaptive.plan_missing",
171
+ "adaptive.plan_repaired",
172
+ "adaptive.plan_repair_failed",
173
+ // Agent control / group-join / nudge
174
+ "agent.control.queued",
175
+ "agent.group_join.acknowledged",
176
+ "agent.group_join.ack_timeout",
177
+ "agent.group_join.delivery_reused",
178
+ "agent.group_join.partial",
179
+ "agent.group_join.completed",
180
+ "agent.nudged",
181
+ // Background-runner lifecycle forensics (async sidecar/runner death, signals)
182
+ "async.died",
183
+ "async.exit",
184
+ "async.interrupt_detected",
185
+ "async.kill_requested",
186
+ "async.sigterm_received_graceful_shutdown",
187
+ "async.watchdog_fired",
188
+ "background.unregister_worker_failed",
189
+ // Chain runner
190
+ "chain.step_completed",
191
+ // Config
192
+ "config.warning",
193
+ // Stale-run reconciliation (stale-reconciler.ts)
194
+ "crew.run.reconciled_stale",
195
+ "crew.run.orphan_cancelled",
196
+ "crew.run.orphan_skip",
197
+ "crew.run.recovery_blocked",
198
+ "crew.run.recovery_declined",
199
+ "crew.run.recovery_skipped",
200
+ "crew.run.resumed",
201
+ "crew.task.heartbeat_dead",
202
+ "crew.task.retry_attempt",
203
+ // Dynamic workflow resume
204
+ "dwf.resumed",
205
+ // Foreground interrupt
206
+ "foreground.interrupt_requested",
207
+ // Goal loop (P0/P1) additional outcomes
208
+ "goal.resumed",
209
+ "goal.resume_spawn_failed",
210
+ "goal.stuck",
211
+ "goal.turn_terminal_status",
212
+ "goal.verification_compromised",
213
+ "goal.workspace_lock_failed",
214
+ // Hook execution trace
215
+ "hook.executed",
216
+ // Limits
217
+ "limits.unbounded",
218
+ // Mailbox delivery (ack/replay/timeout observable surface)
219
+ "mailbox.acknowledged",
220
+ "mailbox.message",
221
+ "mailbox.replayed",
222
+ // Recovery
223
+ "recovery.rerun_task",
224
+ // Run-level budget/effectiveness/export/lifecycle bookkeeping
225
+ "run.started",
226
+ "run.budget_warning",
227
+ "run.budget_abort",
228
+ "run.deliverable_warning",
229
+ "run.effectiveness",
230
+ "run.exported",
231
+ "run.forget_requested",
232
+ "run.goal_achievement",
233
+ "run.resume_requested",
234
+ // Runtime/surface resolution
235
+ "runtime.resolved",
236
+ "surface.degraded",
237
+ "surface.requeued",
238
+ // Task scheduling/steer/budget/fairness bookkeeping
239
+ "task.attention",
240
+ "task.claimed",
241
+ "task.claim_released",
242
+ "task.coalesced",
243
+ "task.coalesced_dispatch_start",
244
+ "task.coalesced_dispatch_end",
245
+ "task.parallel_start",
246
+ "task.status_transitioned",
247
+ "task.reconciled_from_disk",
248
+ "task.checkpoint_recovered",
249
+ "task.retry_attempt",
250
+ "task.budget_fair_share",
251
+ "task.model_dropped",
252
+ "task.output_validation",
253
+ "task.steer_queued",
254
+ "task.steer_dropped",
255
+ "task.surface_lost",
256
+ // Worker lifecycle (surface runtime / broker-side)
257
+ "worker.heartbeat",
258
+ "worker.cancelled",
259
+ "worker.kill_stale",
260
+ "worker.message",
261
+ // Workflow phase advance (note: supersedes the legacy `phase.*` names
262
+ // above, which are kept registered for backward compatibility)
263
+ "workflow.phase_completed",
264
+ "workflow.phase_failed",
265
+ "workflow.phase_guard_blocked",
266
+ "workflow.preconditions",
158
267
  ] as const;
159
268
  export type TeamEventType = (typeof TEAM_EVENT_TYPES)[number];
160
269
 
@@ -80,7 +80,11 @@ function isLockHolderAlive(filePath: string): boolean {
80
80
  *
81
81
  * Returns `{ canSteal: true }` if the lock is stale OR the holder is dead
82
82
  * (safe to forcibly remove); `{ canSteal: false }` if it is fresh AND held by
83
- * a live process (must keep waiting).
83
+ * a live process (must keep waiting). RR-011 additionally reports
84
+ * `heldByLiveInProcess: true` when the stored token belongs to a run-lock
85
+ * acquisition of THIS process that is still inside its critical section — the
86
+ * async acquire loop then WAITS (timer-based retry) instead of throwing or
87
+ * stealing.
84
88
  *
85
89
  * ## EPERM Handling (Accepted Risk)
86
90
  *
@@ -105,7 +109,11 @@ function isLockHolderAlive(filePath: string): boolean {
105
109
  *
106
110
  * See also: SECURITY-ISSUES.md SEC-008 for documented acceptance.
107
111
  */
108
- function readLockSnapshot(filePath: string, staleMs: number, options?: { treatOwnPidAsStealable?: boolean }): { canSteal: boolean } {
112
+ function readLockSnapshot(
113
+ filePath: string,
114
+ staleMs: number,
115
+ options?: { treatOwnPidAsStealable?: boolean; activeHolderTokens?: ReadonlySet<string> },
116
+ ): { canSteal: boolean; heldByLiveInProcess: boolean } {
109
117
  const treatOwnPidAsStealable = options?.treatOwnPidAsStealable === true;
110
118
  let stat: fs.Stats | undefined;
111
119
  let raw: string | undefined;
@@ -119,21 +127,23 @@ function readLockSnapshot(filePath: string, staleMs: number, options?: { treatOw
119
127
  // "locked" error.
120
128
  const code = (error as NodeJS.ErrnoException).code;
121
129
  if (code === "ENOENT") {
122
- return { canSteal: true };
130
+ return { canSteal: true, heldByLiveInProcess: false };
123
131
  }
124
132
  // Transient I/O error — be conservative (don't steal, retry the read on
125
133
  // next attempt).
126
- return { canSteal: false };
134
+ return { canSteal: false, heldByLiveInProcess: false };
127
135
  }
128
136
  // Staleness from a single snapshot.
129
137
  let createdAt = parseCreatedAtFromLock(raw);
130
138
  if (createdAt === undefined) createdAt = stat.mtimeMs;
131
139
  const isStale = Date.now() - createdAt > staleMs;
132
140
  let holderPid: number | undefined;
141
+ let holderToken: string | undefined;
133
142
  let isAlive = true;
134
143
  try {
135
- const parsed = JSON.parse(raw) as { pid?: unknown };
144
+ const parsed = JSON.parse(raw) as { pid?: unknown; token?: unknown };
136
145
  holderPid = typeof parsed.pid === "number" ? parsed.pid : undefined;
146
+ holderToken = typeof parsed.token === "string" ? parsed.token : undefined;
137
147
  } catch {
138
148
  /* malformed payload — keep isAlive=true */
139
149
  }
@@ -151,9 +161,28 @@ function readLockSnapshot(filePath: string, staleMs: number, options?: { treatOw
151
161
  // (used by withRunLock* which is single-process — a fresh lock with our pid
152
162
  // between acquisitions is just a leftover from a previous call that
153
163
  // releaseOwnLock didn't get to delete yet; safe to steal).
164
+ //
165
+ // RR-011 (F02): "our own pid" alone can no longer authorize a steal for the
166
+ // async run-lock path — a lock CURRENTLY HELD by another async context of
167
+ // this process also carries our pid while its holder is merely awaiting
168
+ // inside the critical section. The on-disk token disambiguates: a token in
169
+ // `activeHolderTokens` (runLockHeldTokens — every live acquisition of this
170
+ // process registers its token) means the holder is ALIVE in-process → the
171
+ // own-pid steal branch is suppressed (see acquireLockWithRetryAsync: such a
172
+ // holder WAITS instead of stealing/throwing). A token-less payload (legacy
173
+ // lock file) or a token not in the set is a leftover corpse → still stealable,
174
+ // preserving the anti-CI-flake behaviour the flag was added for.
175
+ // NOTE (trap, design.md §4.2): comparing the stored token to the CURRENT
176
+ // acquisition's token would be WRONG — each acquisition mints a fresh
177
+ // randomUUID, so a live holder's token always differs from ours and the
178
+ // predicate would steal anyway. Only a lookup into the LIVE-token set is
179
+ // correct.
154
180
  const isOurOwnHolder = holderPid === process.pid;
181
+ const holderIsLiveInProcess = holderToken !== undefined && (options?.activeHolderTokens?.has(holderToken) ?? false);
182
+ const ownPidStealable = treatOwnPidAsStealable && isOurOwnHolder && !holderIsLiveInProcess;
155
183
  return {
156
- canSteal: isStale || !isAlive || (treatOwnPidAsStealable && isOurOwnHolder),
184
+ canSteal: isStale || !isAlive || ownPidStealable,
185
+ heldByLiveInProcess: holderIsLiveInProcess,
157
186
  };
158
187
  }
159
188
 
@@ -239,25 +268,82 @@ function timingSafeTokenMatch(a: string, b: string): boolean {
239
268
  }
240
269
 
241
270
  /**
242
- * Release the lock we (this process) just acquired. Unlike releaseLock, this is
243
- * used in the `finally` blocks of withRunLock and withRunLockSync so the file was
244
- * created earlier in the SAME call. Within the same process the new acquire uses
245
- * a fresh randomUUID token, so token matching would falsely fail and leak the
246
- * file across releases.
271
+ * Release the lock we (this process) just acquired. Used in the `finally` blocks
272
+ * of withRunLock and withRunLockSync, so the file was created earlier in the SAME
273
+ * call and carries OUR token.
247
274
  *
248
275
  * LOCK-1 (Round 2): PID-guarded release. Previously this deleted
249
276
  * UNCONDITIONALLY — if our critical section exceeded staleMs, another process
250
277
  * could steal our lock (overwrite the file with its own pid); our finally would
251
- * then DELETE THE STEALER's lock, breaking mutual exclusion. We now verify the
252
- * lock file still records OUR pid before removing; if it records a different pid
253
- * the lock was stolen and the current holder owns it. Same-process re-acquire is
254
- * preserved (pid matches → delete). Mirrors the proven pattern in event-log.ts
255
- * (Round 26, BUG 5).
278
+ * then DELETE THE STEALER's lock, breaking mutual exclusion. We therefore verify
279
+ * the lock file still records OUR pid before removing; if it records a different
280
+ * pid the lock was stolen and the current holder owns it.
281
+ *
282
+ * RR-011 (F02): within one process, PID equality no longer identifies "our" lock
283
+ * — two same-process async holders are distinguished ONLY by token. A finishing
284
+ * context must not delete another acquisition's lock (verification probe 2: the
285
+ * lock file was ENOENT while the second holder was still inside its critical
286
+ * section). So after the pid check, the stored token must also match ours. A
287
+ * token-less payload (legacy lock file from an older release) keeps the
288
+ * PID-only behaviour so old files are still cleaned up.
256
289
  *
257
290
  * Symlink guard is preserved: if a symlink appeared since our writeLockFile, we
258
291
  * don't rm it (defense against attacker-planted symlinks).
259
292
  */
260
- export function releaseOwnLock(filePath: string, _token: string): void {
293
+ /**
294
+ * US-002 (2026-09-22): structured, safe, idempotent sweep of stale run locks.
295
+ *
296
+ * Locks whose holder died without releasing (kill -9, crash) previously lingered
297
+ * until some future acquirer hit the stale-steal path. This gives the
298
+ * stale-reconciler an explicit pass.
299
+ *
300
+ * IMPORTANT — the sweep predicate is STRICTER than acquire-time `canSteal`:
301
+ * acquire steals on `stale OR holder-dead` (deliberate: a stale lock must not
302
+ * block a live run forever), but a SWEEP must only remove a lock whose holder is
303
+ * provably DEAD. Removing a stale-but-live holder's lock would let two processes
304
+ * enter the critical section. So: stale AND (no pid OR pid dead). A live pid is
305
+ * never swept, no matter how old.
306
+ *
307
+ * Returns the lock files it removed. Idempotent: a second sweep finds nothing.
308
+ */
309
+ export function sweepStaleLocks(lockFiles: readonly string[], options: RunLockOptions = {}): { removed: string[] } {
310
+ const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
311
+ const removed: string[] = [];
312
+ for (const filePath of lockFiles) {
313
+ try {
314
+ if (!fs.existsSync(filePath)) continue;
315
+ if (!isLockStale(filePath, staleMs)) continue; // fresh → leave alone
316
+ // Stale. Remove ONLY when no live holder owns it.
317
+ if (isLockHolderAlive(filePath)) continue;
318
+ fs.rmSync(filePath, { force: true });
319
+ removed.push(filePath);
320
+ } catch (error) {
321
+ logInternalError("locks.sweep-stale", error, `path=${filePath}`);
322
+ }
323
+ }
324
+ return { removed };
325
+ }
326
+
327
+ /**
328
+ * US-002: discover every `run.lock` under a runs root
329
+ * (`<runsRoot>/<runId>/run.lock`), bounded, for sweepStaleLocks. Missing → empty.
330
+ */
331
+ export function discoverRunLockFiles(runsRoot: string, maxDirs = 500): string[] {
332
+ try {
333
+ if (!fs.existsSync(runsRoot)) return [];
334
+ return fs
335
+ .readdirSync(runsRoot, { withFileTypes: true })
336
+ .filter((e) => e.isDirectory())
337
+ .slice(0, maxDirs)
338
+ .map((e) => path.join(runsRoot, e.name, "run.lock"))
339
+ .filter((p) => fs.existsSync(p));
340
+ } catch (error) {
341
+ logInternalError("locks.discover-run-locks", error, `runsRoot=${runsRoot}`);
342
+ return [];
343
+ }
344
+ }
345
+
346
+ export function releaseOwnLock(filePath: string, token: string): void {
261
347
  try {
262
348
  const stat = fs.lstatSync(filePath);
263
349
  if (stat.isSymbolicLink()) return;
@@ -266,11 +352,20 @@ export function releaseOwnLock(filePath: string, _token: string): void {
266
352
  }
267
353
  try {
268
354
  const raw = fs.readFileSync(filePath, "utf-8");
269
- const holderPid = (JSON.parse(raw) as { pid?: unknown })?.pid;
270
- if (holderPid === process.pid) {
271
- fs.rmSync(filePath, { force: true });
355
+ const parsed = JSON.parse(raw) as { pid?: unknown; token?: unknown };
356
+ const holderPid = parsed.pid;
357
+ const storedToken = typeof parsed.token === "string" ? parsed.token : undefined;
358
+ if (holderPid !== process.pid) {
359
+ // holderPid !== process.pid → lock stolen by another process; do NOT touch.
360
+ return;
272
361
  }
273
- // holderPid !== process.pid → lock stolen by another process; do NOT touch.
362
+ if (storedToken !== undefined && storedToken !== token) {
363
+ // RR-011 (F02): same process, different token → the lock now belongs to
364
+ // another acquisition of THIS process (steal window / superseded holder).
365
+ // Do not delete it — the current holder owns it.
366
+ return;
367
+ }
368
+ fs.rmSync(filePath, { force: true });
274
369
  } catch (error) {
275
370
  const code = (error as NodeJS.ErrnoException).code;
276
371
  if (code !== "ENOENT") {
@@ -322,6 +417,25 @@ function isLockContention(code: string | undefined): boolean {
322
417
  return code === "EEXIST" || code === "EPERM" || code === "EBUSY";
323
418
  }
324
419
 
420
+ // RR-011 (F02): tokens of run-lock acquisitions that are CURRENTLY HELD in this
421
+ // process. It answers exactly one question — "is this holder still alive
422
+ // in-process?" — and NOTHING else:
423
+ // - Re-entrance (bypass) decisions stay in `lockCtx` (per async context). This
424
+ // set must NEVER be consulted for bypass — that is the H-1 bug class.
425
+ // - The async steal predicate consults it (via readLockSnapshot's
426
+ // activeHolderTokens) so a SECOND independent async context can no longer
427
+ // steal a lock whose holder is merely awaiting inside its critical section.
428
+ //
429
+ // INVARIANT: a token is added synchronously right after a successful acquire
430
+ // (no await between acquire returning and the add) and deleted in the
431
+ // acquisition's `finally` BEFORE releaseOwnLock runs — so a token in this set
432
+ // always corresponds to a live critical section, and a leftover lock file from
433
+ // a finished acquisition (token no longer in the set, or a legacy token-less
434
+ // payload) remains stealable, preserving the CI-flake fix that
435
+ // treatOwnPidAsStealable was added for. If a `finally` were ever skipped, the
436
+ // leaked entry is inert: future lock files always mint a fresh randomUUID.
437
+ const runLockHeldTokens = new Set<string>();
438
+
325
439
  function acquireLockWithRetry(filePath: string, staleMs: number, kind: LockKind = "file"): string {
326
440
  let attempt = 0;
327
441
  const deadline = Date.now() + staleMs * 2;
@@ -381,10 +495,48 @@ async function acquireLockWithRetryAsync(filePath: string, staleMs: number, kind
381
495
  // file still exists with our own pid. Stealing it avoids spurious 'locked'
382
496
  // errors. The sync file-lock path (withFileLockSync) above uses false to
383
497
  // preserve the multi-process safety guarantee.
384
- const { canSteal } = readLockSnapshot(filePath, staleMs, { treatOwnPidAsStealable: true });
385
- if (!canSteal) {
498
+ // RR-011 (F02): the steal now consults runLockHeldTokens — a lock whose
499
+ // stored token belongs to a LIVE acquisition of this process is NOT a
500
+ // leftover corpse and must not be stolen (that broke async↔async mutual
501
+ // exclusion: two independent async contexts entered the critical section
502
+ // together). Token-less legacy files are still stolen (CI-flake fix).
503
+ const verdict = readLockSnapshot(filePath, staleMs, {
504
+ treatOwnPidAsStealable: true,
505
+ activeHolderTokens: runLockHeldTokens,
506
+ });
507
+ if (!verdict.canSteal && verdict.heldByLiveInProcess) {
508
+ // RR-011 (F02): the holder is a LIVE acquisition in THIS process (another
509
+ // async context awaiting inside its critical section). It releases in its
510
+ // `finally` on this same event loop — WAIT via a timer (never sleepSync:
511
+ // blocking the event loop would starve the very holder we are waiting
512
+ // for, the v0.9.26 deadlock class) and retry the create. The loop deadline
513
+ // above plus the staleMs check in readLockSnapshot bound the wait, so a
514
+ // hung holder eventually becomes stale-stealable and this can never
515
+ // block indefinitely.
516
+ const delay = Math.min(250, 25 * 2 ** attempt);
517
+ await sleep(delay);
518
+ attempt++;
519
+ continue;
520
+ }
521
+ if (!verdict.canSteal) {
386
522
  throw new Error(`Run '${path.basename(filePath)}' is locked by another operation.`);
387
523
  }
524
+ if (verdict.heldByLiveInProcess) {
525
+ // Review MAJOR 2: the staleMs backstop is stealing a lock whose holder
526
+ // is STILL a live acquisition of this process (its critical section
527
+ // legitimately exceeded staleMs — fsync stalls, loaded machine). The
528
+ // steal is the documented design (ADR 2026-09-17-run-lock-async-
529
+ // ownership) but mutual exclusion is about to be violated knowingly,
530
+ // so it must be observable, never silent — production holders are
531
+ // ms-scale; seeing this warn means the invariant "critical section <
532
+ // staleMs" was broken and the budget should be revisited.
533
+ logInternalError(
534
+ "locks.steal-live-holder",
535
+ new Error(`staleMs steal of a lock held by a LIVE in-process acquisition (staleMs=${staleMs})`),
536
+ filePath,
537
+ "warn",
538
+ );
539
+ }
388
540
  // Stale or dead holder — forcibly remove the lock.
389
541
  try {
390
542
  fs.rmSync(filePath, { force: true });
@@ -622,6 +774,10 @@ export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, optio
622
774
  }
623
775
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
624
776
  const token = acquireLockWithRetry(filePath, staleMs, "run");
777
+ // RR-011 (F02): register the hold so async contenders of this process see a
778
+ // LIVE holder (wait) instead of a stealable corpse. Sync here too, keeping the
779
+ // invariant uniform for every run-lock acquisition of this process.
780
+ runLockHeldTokens.add(token);
625
781
  const prevHeld = lockCtx.getStore() ?? new Set<string>();
626
782
  const newHeld = new Set(prevHeld);
627
783
  newHeld.add(filePath);
@@ -632,14 +788,11 @@ export function withRunLockSync<T>(manifest: TeamRunManifest, fn: () => T, optio
632
788
  try {
633
789
  return fn();
634
790
  } finally {
635
- // FIX (CI flake): releaseLock uses token matching to prevent the
636
- // "losing contender wipes winner's lock" race in multi-process scenarios.
637
- // But within withRunLockSync/withRunLock (same process), the new acquire
638
- // uses a FRESH random token — the previous stored token doesn't match the
639
- // current token — and the lock file is never removed. Repeat acquisitions
640
- // then keep failing with EEXIST because the file lingers.
641
- // Since withRunLock* is always single-process, unconditionally delete
642
- // (symlink guard is still applied for safety).
791
+ // RR-011 (F02): unregister BEFORE releasing (see runLockHeldTokens
792
+ // invariant) — the token must not linger as "live" once the hold ends.
793
+ // releaseOwnLock is token-guarded: if our lock was superseded in the
794
+ // meantime, the current holder's file is left untouched.
795
+ runLockHeldTokens.delete(token);
643
796
  releaseOwnLock(filePath, token);
644
797
  }
645
798
  });
@@ -654,6 +807,10 @@ export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promis
654
807
  }
655
808
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
656
809
  const token = await acquireLockWithRetryAsync(filePath, staleMs, "run");
810
+ // RR-011 (F02): register the hold IMMEDIATELY (no await between the acquire
811
+ // returning and this add — otherwise a concurrent contender could observe the
812
+ // fresh on-disk token as a stealable corpse). Deleted in the finally below.
813
+ runLockHeldTokens.add(token);
657
814
  const prevHeld = lockCtx.getStore() ?? new Set<string>();
658
815
  const newHeld = new Set(prevHeld);
659
816
  newHeld.add(filePath);
@@ -664,8 +821,9 @@ export async function withRunLock<T>(manifest: TeamRunManifest, fn: () => Promis
664
821
  try {
665
822
  return await fn();
666
823
  } finally {
667
- // FIX (CI flake): see withRunLockSync above — use releaseOwnLock for
668
- // unconditional deletion of the lock file we created (same process).
824
+ // RR-011 (F02): see withRunLockSync above — unregister the live token
825
+ // BEFORE the token-guarded releaseOwnLock (runLockHeldTokens invariant).
826
+ runLockHeldTokens.delete(token);
669
827
  releaseOwnLock(filePath, token);
670
828
  }
671
829
  });