pi-crew 0.11.22 → 0.11.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/agents/explorer.md +3 -3
  3. package/agents/librarian.md +3 -3
  4. package/dist/index.mjs +523 -520
  5. package/docs/dynamic-workflows.md +14 -2
  6. package/package.json +4 -4
  7. package/src/agents/discover-agents.ts +8 -2
  8. package/src/config/config-validation.ts +3 -1
  9. package/src/config/env-allowlist-registry.ts +5 -0
  10. package/src/config/env-vars.ts +9 -6
  11. package/src/config/role-tools.ts +7 -6
  12. package/src/extension/management.ts +7 -0
  13. package/src/extension/register.ts +15 -0
  14. package/src/extension/run-index.ts +24 -1
  15. package/src/extension/team-tool/destructive-gate.ts +208 -0
  16. package/src/extension/team-tool/goal-wrap.ts +11 -1
  17. package/src/extension/team-tool/goal.ts +13 -1
  18. package/src/extension/team-tool/respond.ts +8 -6
  19. package/src/extension/team-tool/run.ts +4 -4
  20. package/src/extension/team-tool.ts +10 -0
  21. package/src/observability/exporters/otlp-exporter.ts +11 -1
  22. package/src/prompt/prompt-runtime.ts +102 -18
  23. package/src/prompt/surface-worker.ts +13 -1
  24. package/src/runtime/broker/crew-broker-client.ts +7 -0
  25. package/src/runtime/broker/crew-broker.ts +113 -152
  26. package/src/runtime/broker/protocol/connection-state.ts +159 -1
  27. package/src/runtime/broker/protocol/events-replay.ts +50 -10
  28. package/src/runtime/broker/protocol/request-parsers.ts +68 -0
  29. package/src/runtime/child-pi/child-pi-constants.ts +9 -0
  30. package/src/runtime/child-pi/child-pi-rpc.ts +72 -7
  31. package/src/runtime/child-pi/child-pi-streams.ts +19 -3
  32. package/src/runtime/child-pi/postmortem-registry.ts +8 -2
  33. package/src/runtime/classifier/ambient-noise-gate.ts +5 -1
  34. package/src/runtime/compaction/tool-output-pruner.ts +20 -2
  35. package/src/runtime/crew-agent-records.ts +92 -17
  36. package/src/runtime/custom-tools/submit-result-tool.ts +17 -0
  37. package/src/runtime/deadletter.ts +5 -1
  38. package/src/runtime/dispatch-batch.ts +60 -16
  39. package/src/runtime/effectiveness.ts +24 -1
  40. package/src/runtime/finalize-run.ts +12 -1
  41. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +12 -3
  42. package/src/runtime/goal-workflow/goal-evaluator.ts +30 -5
  43. package/src/runtime/goal-workflow/goal-loop-runner.ts +60 -3
  44. package/src/runtime/live-session/live-agent-control.ts +34 -14
  45. package/src/runtime/live-session/live-agent-manager.ts +20 -5
  46. package/src/runtime/live-session/live-session-runtime.ts +66 -81
  47. package/src/runtime/live-session/usage-bridge.ts +66 -2
  48. package/src/runtime/mcp-proxy.ts +10 -0
  49. package/src/runtime/model/model-fallback.ts +68 -0
  50. package/src/runtime/model/runtime-policy.ts +20 -6
  51. package/src/runtime/model/runtime-resolver.ts +5 -3
  52. package/src/runtime/orphan-worker-registry.ts +64 -24
  53. package/src/runtime/pi-spawn.ts +4 -1
  54. package/src/runtime/process/zombie-scanner.ts +6 -2
  55. package/src/runtime/recovery/crash-recovery.ts +35 -2
  56. package/src/runtime/scheduling/batch-barrier.ts +42 -5
  57. package/src/runtime/scheduling/run-coalesced-task-group.ts +9 -3
  58. package/src/runtime/scheduling/scheduler.ts +74 -29
  59. package/src/runtime/stale-reconciler.ts +45 -6
  60. package/src/runtime/surface/herdr-provider.ts +54 -1
  61. package/src/runtime/surface/launch-script.ts +7 -1
  62. package/src/runtime/surface/resolve-surface.ts +4 -1
  63. package/src/runtime/surface/tmux-provider.ts +32 -2
  64. package/src/runtime/task-runner/child-executor.ts +3 -2
  65. package/src/runtime/workspace-lock.ts +52 -4
  66. package/src/state/contracts.ts +2 -0
  67. package/src/state/coordination/process-lock.ts +50 -0
  68. package/src/state/event-log/cursor.ts +24 -3
  69. package/src/state/event-log/event-log-rotation.ts +26 -2
  70. package/src/state/event-log/worker-atomic-writer.ts +5 -0
  71. package/src/state/stores/active-run-registry.ts +56 -5
  72. package/src/state/stores/ownership-map.ts +26 -7
  73. package/src/state/stores/plan-store.ts +30 -1
  74. package/src/state/stores/sqlite-run-state.ts +91 -1
  75. package/src/state/types.ts +5 -0
  76. package/src/ui/dashboard-panes/agents-pane.ts +4 -3
  77. package/src/ui/dashboard-panes/plan-pane.ts +5 -3
  78. package/src/ui/dashboard-panes/transcript-pane.ts +2 -1
  79. package/src/ui/overlays/mailbox-detail-overlay.ts +5 -3
  80. package/src/ui/run-dashboard.ts +6 -4
  81. package/src/ui/run-snapshot-cache.ts +3 -2
  82. package/src/ui/settings-overlay.ts +68 -10
  83. package/src/ui/theme-discovery.ts +26 -7
  84. package/src/ui/transcript-viewer.ts +5 -3
  85. package/src/utils/visual.ts +84 -0
@@ -36,7 +36,7 @@ import * as fs from "node:fs";
36
36
  import * as net from "node:net";
37
37
  import * as os from "node:os";
38
38
  import * as path from "node:path";
39
-
39
+ import { logInternalError } from "../../utils/internal-error.ts";
40
40
  import { killProcessTree } from "../child-pi/child-pi-kill.ts";
41
41
  import type { SurfaceDetection, SurfaceExitReason, SurfaceHandle, SurfaceProvider, SurfaceSpawnOpts } from "./surface-provider.ts";
42
42
  import { MAX_PANES_PER_TAB, splitDirectionFor } from "./surface-provider.ts";
@@ -44,6 +44,11 @@ import { MAX_PANES_PER_TAB, splitDirectionFor } from "./surface-provider.ts";
44
44
  /** Grace window giữa SIGTERM worker pid và pane.close fallback (đối chiều tmux provider). */
45
45
  const GRACEFUL_TERM_WAIT_MS = 3000;
46
46
 
47
+ /** R3-M-B3: per-request deadline on the one-connection-per-request `call()` —
48
+ * a wedged herdr server must fail the caller instead of hanging the host
49
+ * until some outer 600s deadline (gap caught in the E2E herdr battery). */
50
+ const HERDR_REQUEST_TIMEOUT_MS = 10_000;
51
+
47
52
  /**
48
53
  * Socket abstraction injectable. Quy ước EOF/error: onLine được gọi đúng một
49
54
  * lần với chuỗi rỗng khi server đóng connection (đối chiếu NDJSON — dòng
@@ -137,6 +142,11 @@ interface WireResponse {
137
142
  interface WireEvent {
138
143
  event?: string;
139
144
  data?: { pane_id?: string };
145
+ /** R3-M-B3: an error ACK carries the request id and an error object — no
146
+ * `event` envelope (previously silently ignored, leaving watchers hung on
147
+ * a subscription the server actually refused). */
148
+ id?: string;
149
+ error?: { code?: string; message?: string };
140
150
  }
141
151
 
142
152
  /** Watcher theo pane id — callback list + trạng thái exit cho replay. */
@@ -157,6 +167,10 @@ export function createHerdrProvider(deps: HerdrProviderDeps = {}): SurfaceProvid
157
167
  let reqSeq = 0;
158
168
  let subscription: HerdrSocket | null = null;
159
169
  let subscriptionCb: ((line: string) => void) | null = null;
170
+ // R3-M-B3: id of the in-flight events.subscribe request — lets
171
+ // onSubscriptionLine recognize an error ACK (frame with id, no event) and
172
+ // treat the subscription as dead instead of waiting forever.
173
+ let subscriptionRequestId: string | null = null;
160
174
 
161
175
  // Tab-layout (spec 2026-08-27-surface-tab-layout): tabKey(run) → {tabIds
162
176
  // (MỌI tab của run), rootPaneId (tab đang nhận split), paneCount}. Tab mở
@@ -208,11 +222,22 @@ export function createHerdrProvider(deps: HerdrProviderDeps = {}): SurfaceProvid
208
222
  return;
209
223
  }
210
224
  let settled = false;
225
+ // R3-M-B3: per-request deadline — see HERDR_REQUEST_TIMEOUT_MS. A wedged
226
+ // herdr server must fail the caller instead of hanging until an outer
227
+ // 600s deadline (gap caught in the E2E herdr battery).
228
+ const requestTimer = setTimeout(() => {
229
+ if (settled) return;
230
+ settled = true;
231
+ socket.close();
232
+ reject(new Error(`herdr request timeout: no response to ${id} (${method}) within ${HERDR_REQUEST_TIMEOUT_MS}ms`));
233
+ }, HERDR_REQUEST_TIMEOUT_MS);
234
+ requestTimer.unref?.();
211
235
  socket.onLine((line) => {
212
236
  if (settled) return;
213
237
  if (!line) {
214
238
  // EOF trước response — server chết giữa chừng.
215
239
  settled = true;
240
+ clearTimeout(requestTimer);
216
241
  socket.close();
217
242
  reject(new Error(`herdr socket closed before response to ${id} (${method})`));
218
243
  return;
@@ -225,6 +250,7 @@ export function createHerdrProvider(deps: HerdrProviderDeps = {}): SurfaceProvid
225
250
  }
226
251
  if (msg.id !== id) return;
227
252
  settled = true;
253
+ clearTimeout(requestTimer);
228
254
  socket.close();
229
255
  if (msg.error) {
230
256
  reject(new Error(`${msg.error.code ?? "herdr_error"}: ${msg.error.message ?? "unknown error"}`));
@@ -246,6 +272,7 @@ export function createHerdrProvider(deps: HerdrProviderDeps = {}): SurfaceProvid
246
272
  // Server đóng subscription socket — mọi handle còn sống thành mux-dead.
247
273
  subscription = null;
248
274
  subscriptionCb = null;
275
+ subscriptionRequestId = null;
249
276
  for (const paneId of [...watchers.keys()]) fire(paneId, "mux-dead");
250
277
  return;
251
278
  }
@@ -255,6 +282,31 @@ export function createHerdrProvider(deps: HerdrProviderDeps = {}): SurfaceProvid
255
282
  } catch {
256
283
  return;
257
284
  }
285
+ // R3-M-B3 (2026-10-17): error ACK for events.subscribe — a frame carrying
286
+ // the request id and an error object (NO event envelope) means the server
287
+ // refused the subscription (bad params / protocol drift). Previously this
288
+ // frame fell through the event filter below and was silently ignored,
289
+ // leaving every watcher hung until the 600s outer deadline. Treat it as
290
+ // mux-dead (same as EOF) so handles fail fast + the reason is logged.
291
+ if (msg.id !== undefined && msg.error !== undefined && msg.id === subscriptionRequestId) {
292
+ logInternalError(
293
+ "herdr-provider",
294
+ new Error(`events.subscribe rejected: ${msg.error.code ?? "herdr_error"}: ${msg.error.message ?? "unknown error"}`),
295
+ herdrSocketPath(env),
296
+ "warn",
297
+ );
298
+ const deadSubscription = subscription;
299
+ subscription = null;
300
+ subscriptionCb = null;
301
+ subscriptionRequestId = null;
302
+ try {
303
+ deadSubscription?.close();
304
+ } catch {
305
+ /* already closed */
306
+ }
307
+ for (const paneId of [...watchers.keys()]) fire(paneId, "mux-dead");
308
+ return;
309
+ }
258
310
  // CẢ HAI loại event đều là "pane biến mất" cho pi-crew (verified live
259
311
  // herdr 0.8.2, 2026-08-27): `pane_closed` chỉ bắn khi đóng qua API
260
312
  // pane.close; process exit tự nhiên (worker xong việc → shell `exit`)
@@ -285,6 +337,7 @@ export function createHerdrProvider(deps: HerdrProviderDeps = {}): SurfaceProvid
285
337
  }
286
338
  subscription = socket;
287
339
  subscriptionCb = onSubscriptionLine;
340
+ subscriptionRequestId = id;
288
341
  socket.onLine((line) => subscriptionCb?.(line));
289
342
  // Frame nối \n bởi defaultConnect's write() wrapper — KHÔNG thêm \n ở
290
343
  // đây (frame `\n\n` → server đóng subscription, xem comment trong call()).
@@ -99,13 +99,19 @@ export function buildLaunchScript(input: BuildLaunchScriptInput): string {
99
99
  assertSafeTaskId(input.taskId);
100
100
  const depth = currentCrewDepth(input.callerEnv ?? input.env);
101
101
  if (depth > 0) throw new SurfaceDepthGuardError(depth);
102
-
103
102
  // path.resolve (không phải path.join): script path phải ABSOLUTE — `rm -f
104
103
  // -- "$0"` chạy trong pane SAU dòng `cd <cwd>`, nên $0 relative sẽ trỏ sai
105
104
  // chỗ và self-delete thành no-op (T7 obs, fixed ở T14).
106
105
  const scriptPath = path.resolve(input.baseDir, `pi-crew-launch-${input.taskId}-${process.pid}.sh`);
107
106
  const lines: string[] = ["#!/bin/bash"];
108
107
  for (const [key, value] of Object.entries(input.env)) {
108
+ // R2-L-B2 (wave-2b): env KEYS interpolate into the script UNQUOTED
109
+ // (`export KEY=...`) — only VALUES were shellEscape'd. A task-sourced
110
+ // key like `FOO; curl evil` or `A=b` would inject script lines. Keys must
111
+ // be valid POSIX identifiers.
112
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) {
113
+ throw new Error(`buildLaunchScript: invalid env key ${JSON.stringify(key)} (must match /^[A-Za-z_][A-Za-z0-9_]*$/)`);
114
+ }
109
115
  lines.push(`export ${key}=${shellEscape(value)}`);
110
116
  }
111
117
  lines.push(`cd ${shellEscape(input.cwd)}`);
@@ -58,7 +58,10 @@ function hasBinary(bin: string): boolean {
58
58
  if (cached !== undefined) return cached;
59
59
  let available = false;
60
60
  try {
61
- execFileSync("sh", ["-c", `command -v ${bin}`], { stdio: "ignore" });
61
+ // R2-L-B3 (wave-2b): `bin` interpolates into a shell command — pass it
62
+ // as a POSITIONAL ARG to `command -v "$1"` instead of splicing it into
63
+ // the command string (shell-injection-proof probe).
64
+ execFileSync("sh", ["-c", 'command -v "$1"', "sh", bin], { stdio: "ignore" });
62
65
  available = true;
63
66
  } catch {
64
67
  available = false;
@@ -58,7 +58,9 @@ function defaultHasCommand(bin: string): boolean {
58
58
  if (cached !== undefined) return cached;
59
59
  let available = false;
60
60
  try {
61
- execFileSync("sh", ["-c", `command -v ${bin}`], { stdio: "ignore" });
61
+ // R2-L-B3 (wave-2b): positional-arg probe — see resolve-surface.ts
62
+ // hasBinary; `bin` must never splice into the shell command string.
63
+ execFileSync("sh", ["-c", 'command -v "$1"', "sh", bin], { stdio: "ignore" });
62
64
  available = true;
63
65
  } catch {
64
66
  available = false;
@@ -143,6 +145,14 @@ export function createTmuxProvider(deps: TmuxProviderDeps = {}): SurfaceProvider
143
145
  timer = null;
144
146
  }
145
147
 
148
+ // R2-L-B4 (SEV-8): exited watchers were retained forever — no production
149
+ // caller of handle.dispose(). fire() now MOVES the entry into a bounded
150
+ // recently-exited ring (late subscribers still get their reason replayed,
151
+ // once) instead of leaking callbacks + reason per pane for the host
152
+ // session lifetime.
153
+ const exitedRecent = new Map<string, SurfaceExitReason>();
154
+ const EXITED_RECENT_CAP = 64;
155
+
146
156
  /** Bắn reason MỘT lần cho mọi callback của pane (copy list — cb có thể dispose). */
147
157
  function fire(paneId: string, reason: SurfaceExitReason): void {
148
158
  const watcher = watchers.get(paneId);
@@ -150,6 +160,14 @@ export function createTmuxProvider(deps: TmuxProviderDeps = {}): SurfaceProvider
150
160
  watcher.exited = true;
151
161
  watcher.reason = reason;
152
162
  for (const cb of [...watcher.callbacks]) cb(reason);
163
+ // R2-L-B4: free the watcher entry after the callbacks fired — keep only a
164
+ // bounded reason ring for late-subscriber replay (onExit below).
165
+ watchers.delete(paneId);
166
+ exitedRecent.set(paneId, reason);
167
+ if (exitedRecent.size > EXITED_RECENT_CAP) {
168
+ const oldest = exitedRecent.keys().next().value;
169
+ if (oldest !== undefined) exitedRecent.delete(oldest);
170
+ }
153
171
  maybeStopTimer();
154
172
  }
155
173
 
@@ -192,6 +210,14 @@ export function createTmuxProvider(deps: TmuxProviderDeps = {}): SurfaceProvider
192
210
  kind: "tmux",
193
211
  ...(tabId ? { tabId } : {}),
194
212
  onExit(cb) {
213
+ // R2-L-B4: late subscriber on an already-fired pane — replay the exit
214
+ // reason from the bounded ring, then serve it (single replay).
215
+ const exitedReason = exitedRecent.get(paneId);
216
+ if (exitedReason !== undefined) {
217
+ exitedRecent.delete(paneId);
218
+ cb(exitedReason);
219
+ return;
220
+ }
195
221
  let watcher = watchers.get(paneId);
196
222
  if (!watcher) {
197
223
  watcher = { callbacks: [], exited: false };
@@ -207,10 +233,14 @@ export function createTmuxProvider(deps: TmuxProviderDeps = {}): SurfaceProvider
207
233
  },
208
234
  dispose() {
209
235
  const watcher = watchers.get(paneId);
210
- if (!watcher) return;
236
+ if (!watcher) {
237
+ exitedRecent.delete(paneId);
238
+ return;
239
+ }
211
240
  // Host chủ động dispose khi pane còn sống → "detached" cho listener.
212
241
  if (!watcher.exited) fire(paneId, "detached");
213
242
  watchers.delete(paneId);
243
+ exitedRecent.delete(paneId);
214
244
  maybeStopTimer();
215
245
  },
216
246
  };
@@ -57,6 +57,7 @@ import {
57
57
  buildConfiguredModelRouting,
58
58
  formatModelAttemptNote,
59
59
  isRetryableModelFailure,
60
+ isRetryableModelFailureFrom,
60
61
  type ModelAttemptSummary,
61
62
  type ModelFallbackPolicy,
62
63
  resolveDefaultSubagentModel,
@@ -1217,7 +1218,7 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
1217
1218
  // after the precompute, or the precompute ran before the parent
1218
1219
  // model was known). If a different candidate is found, use it as
1219
1220
  // nextModel; otherwise fall through to the existing break.
1220
- if (!nextModel && !reResolveUsed && isRetryableModelFailure(error)) {
1221
+ if (!nextModel && !reResolveUsed && isRetryableModelFailureFrom(childResult, error)) {
1221
1222
  reResolveUsed = true;
1222
1223
  const reResolved = buildConfiguredModelRouting({
1223
1224
  overrideModel: undefined,
@@ -1252,7 +1253,7 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
1252
1253
  if (input.spawnBudget) input.spawnBudget.max += 1;
1253
1254
  }
1254
1255
  }
1255
- if (!nextModel || !isRetryableModelFailure(error)) break;
1256
+ if (!nextModel || !isRetryableModelFailureFrom(childResult, error)) break;
1256
1257
  // P2-1 spike: before burning the queued (expensive) re-attempt, ask the
1257
1258
  // host classifier whether the failure looks transient. Dormant unless
1258
1259
  // classifierTriageEnabled — the disabled path returns without touching
@@ -149,7 +149,14 @@ function readLock(lockPath: string): WorkspaceLockContents | undefined {
149
149
  try {
150
150
  const parsed = JSON.parse(readFileSync(lockPath, "utf-8"));
151
151
  if (!parsed || typeof parsed !== "object") return undefined;
152
- return parsed as WorkspaceLockContents;
152
+ const lock = parsed as WorkspaceLockContents;
153
+ // R7-C2 (wave-2b): a non-numeric `heartbeat` (corrupt / hand-edited
154
+ // lockfile) made `now - lock.heartbeat` NaN and BOTH staleness
155
+ // comparisons false — an IMMORTAL lock nothing could steal. Treat a
156
+ // non-finite heartbeat as a corrupt lockfile (undefined → the steal
157
+ // path), same as unparseable JSON.
158
+ if (typeof lock.heartbeat !== "number" || !Number.isFinite(lock.heartbeat)) return undefined;
159
+ return lock;
153
160
  } catch {
154
161
  return undefined;
155
162
  }
@@ -196,6 +203,14 @@ export function refreshWorkspaceLockHeartbeat(lockPath: string, goalId: string,
196
203
  if (current.goalId !== goalId || current.pid !== pid) return false;
197
204
  const now = opts.now ?? Date.now;
198
205
  writeLock(lockPath, { ...current, heartbeat: now() });
206
+ // R3-M-B1/FS4 (Wave-3): writeLock is temp+rename (NOT atomic against a
207
+ // concurrent stale-steal's unlink→O_EXCL) — a stealer may have replaced
208
+ // the lock between our read and our rename, and our rename then clobbered
209
+ // THEIR file. Re-read and verify: if the file no longer holds our
210
+ // goalId+pid we lost ownership — return false so the heartbeat loop blocks
211
+ // the goal (never silently keep refreshing a lock another goal owns).
212
+ const reread = readLock(lockPath);
213
+ if (!reread || reread.goalId !== goalId || reread.pid !== pid) return false;
199
214
  return true;
200
215
  } catch {
201
216
  return false;
@@ -229,6 +244,13 @@ export function refreshWorkspaceLockHeartbeat(lockPath: string, goalId: string,
229
244
  function claimLock(lockPath: string, contents: WorkspaceLockContents, staleReclaimMs: number, forceOverwrite = false): boolean {
230
245
  mkdirSync(path.dirname(lockPath), { recursive: true });
231
246
  const json = JSON.stringify(contents);
247
+ // R3-M-B1/FS4 (Wave-3): after a stale-steal claim (unlink→O_EXCL), re-read
248
+ // and VERIFY the lockfile still holds OUR goalId+pid. O_EXCL guarantees we
249
+ // created the file, but a concurrent reclaimer's refresh (temp+rename
250
+ // overwrite) can race in between; without verification two goals could both
251
+ // believe they own the lock. On mismatch: release best-effort (unlink ONLY
252
+ // the foreign-current file we observed — its owner's next refresh re-verifies
253
+ // and fails closed) and report NOT acquired.
232
254
  const tryCreate = (): boolean => {
233
255
  try {
234
256
  const fd = openSync(lockPath, "wx"); // O_EXCL — throws EEXIST if it exists.
@@ -247,6 +269,19 @@ function claimLock(lockPath: string, contents: WorkspaceLockContents, staleRecla
247
269
  return false;
248
270
  }
249
271
  };
272
+ const tryCreateVerified = (): boolean => {
273
+ if (!tryCreate()) return false;
274
+ const reread = readLock(lockPath);
275
+ if (reread && reread.goalId === contents.goalId && reread.pid === contents.pid) return true;
276
+ // A concurrent owner raced in (or the FS reordered the write) — release
277
+ // best-effort and fail the claim; never claim ownership we cannot prove.
278
+ try {
279
+ unlinkSync(lockPath);
280
+ } catch {
281
+ /* best-effort — the foreign owner's own refresh/verify owns the rest */
282
+ }
283
+ return false;
284
+ };
250
285
  if (forceOverwrite) {
251
286
  // Caller verified the existing lock is logically stale; remove it and claim. A concurrent
252
287
  // reclaimer might re-create between our unlink and our open — that's fine, we lose the race
@@ -256,9 +291,22 @@ function claimLock(lockPath: string, contents: WorkspaceLockContents, staleRecla
256
291
  } catch {
257
292
  /* best-effort */
258
293
  }
259
- return tryCreate();
294
+ return tryCreateVerified();
295
+ }
296
+ if (tryCreate()) {
297
+ // Fresh create won the O_EXCL race — same post-claim verification applies
298
+ // (cheap: one read) for uniform ownership proof.
299
+ const reread = readLock(lockPath);
300
+ if (!(reread && reread.goalId === contents.goalId && reread.pid === contents.pid)) {
301
+ try {
302
+ unlinkSync(lockPath);
303
+ } catch {
304
+ /* best-effort */
305
+ }
306
+ return false;
307
+ }
308
+ return true;
260
309
  }
261
- if (tryCreate()) return true;
262
310
  // Stale recovery by mtime age: if the lockfile is older than staleReclaimMs, force-delete + retry.
263
311
  try {
264
312
  const stat = statSync(lockPath);
@@ -268,7 +316,7 @@ function claimLock(lockPath: string, contents: WorkspaceLockContents, staleRecla
268
316
  } catch {
269
317
  /* fall through */
270
318
  }
271
- return tryCreate();
319
+ return tryCreateVerified();
272
320
  }
273
321
  } catch {
274
322
  /* fall through to false */
@@ -166,6 +166,7 @@ export const TEAM_EVENT_TYPES = [
166
166
  "goal.turn_evaluated",
167
167
  "goal.budget_warning",
168
168
  "goal.loop_end",
169
+ "goal.workspace_lock_lost", // Wave-3 R3-M-B1/FS4: ownership lost → blocked
169
170
  "goal.loop_error",
170
171
  "goal.feedback_steered",
171
172
  "goal.state_changed",
@@ -289,6 +290,7 @@ export const TEAM_EVENT_TYPES = [
289
290
  "task.budget_fair_share",
290
291
  "task.model_dropped",
291
292
  "task.mcp_enforcement_degraded",
293
+ "task.mcp_enforcement_fail_closed", // Wave-3 R1-M2: fail-closed abort
292
294
  "task.output_validation",
293
295
  // U13/U6B (upgrade wave 3, 2026-10-10): deterministic verifier pre-gate
294
296
  // decision — emitted by verify-gate/classifier-pre-gate when a task's
@@ -105,6 +105,52 @@ function socketAddress(name: string): SocketAddress {
105
105
  return { address: fileSocketAddress(key), file: true };
106
106
  }
107
107
 
108
+ /** R9-OPS2 (2026-10-17): age threshold for the stale-socket sweep (mtime). */
109
+ export const STALE_SOCKET_SWEEP_AGE_MS = 24 * 60 * 60 * 1000;
110
+
111
+ let staleSocketSweepDone = false;
112
+
113
+ /** R9-OPS2: sweep stale pi-crew/pc socket FILES from a directory (default
114
+ * os.tmpdir()). Only entries that lstat as actual unix socket files, older
115
+ * than STALE_SOCKET_SWEEP_AGE_MS, and not the excluded current path are
116
+ * unlinked. Best-effort throughout — a sweep failure never breaks the lock
117
+ * path. Returns the number of files removed (tests). */
118
+ export function sweepStaleSocketFiles(options?: { now?: number; tmpDir?: string; excludePath?: string }): number {
119
+ const tmpDir = options?.tmpDir ?? os.tmpdir();
120
+ const now = options?.now ?? Date.now();
121
+ let removed = 0;
122
+ try {
123
+ for (const name of fs.readdirSync(tmpDir)) {
124
+ if (!name.endsWith(".sock")) continue;
125
+ if (!name.startsWith("pi-crew-") && !name.startsWith("pc-")) continue;
126
+ const full = path.join(tmpDir, name);
127
+ if (options?.excludePath !== undefined && full === options.excludePath) continue;
128
+ try {
129
+ const stat = fs.lstatSync(full);
130
+ if (!stat.isSocket()) continue; // socket FILES only — never touch anything else
131
+ if (now - stat.mtimeMs < STALE_SOCKET_SWEEP_AGE_MS) continue; // fresh — keep
132
+ fs.unlinkSync(full);
133
+ removed += 1;
134
+ } catch {
135
+ /* best-effort per file */
136
+ }
137
+ }
138
+ } catch {
139
+ /* best-effort: unreadable dir etc. */
140
+ }
141
+ if (removed > 0) {
142
+ // ONE line — R9-OPS2 explicitly asks for a single log line, not a flood.
143
+ console.warn(`[pi-crew] swept ${removed} stale socket file(s) older than 24h from ${tmpDir}`);
144
+ }
145
+ return removed;
146
+ }
147
+
148
+ function ensureStaleSocketSweep(currentSocketPath?: string): void {
149
+ if (staleSocketSweepDone) return;
150
+ staleSocketSweepDone = true;
151
+ sweepStaleSocketFiles(currentSocketPath !== undefined ? { excludePath: currentSocketPath } : undefined);
152
+ }
153
+
108
154
  /**
109
155
  * Probe whether a live server accepts connections on `address` (leftover-corpse
110
156
  * check). A single refusal is NOT trusted on file-socket platforms: macOS can
@@ -149,6 +195,10 @@ async function accepting(address: string): Promise<boolean> {
149
195
  */
150
196
  export async function acquireProcessLock(name: string): Promise<ProcessLockHandle> {
151
197
  const { address, file } = socketAddress(name);
198
+ // R9-OPS2 (2026-10-17): once per process, sweep stale socket FILES left
199
+ // behind by crashed sessions (observed: 299 stale pc-*.sock in /tmp). The
200
+ // CURRENT lock's own path is excluded; abstract sockets leave no file.
201
+ ensureStaleSocketSweep(file ? address : undefined);
152
202
  const server = net.createServer((socket) => socket.end());
153
203
  for (;;) {
154
204
  try {
@@ -220,6 +220,12 @@ export interface EventCursorResult {
220
220
  nextSeq: number;
221
221
  total: number;
222
222
  nextByteOffset?: number;
223
+ /** R9-N2 (2026-10-17): true when the WIDE (default) path dropped a prefix
224
+ * because the log exceeds the 4 MB tail window — callers paging by seq
225
+ * alone cannot reach the dropped prefix; pass `fromByteOffset` to resume
226
+ * scanning from a byte position instead. Additive/optional: callers that
227
+ * ignore it keep the exact previous behavior. */
228
+ truncated?: boolean;
223
229
  /** R-03: live generation of the events file at read time. Callers doing
224
230
  * streaming byte-offset reads should echo this back as `generation` on the
225
231
  * next call so rotation is detected and the cursor resets. */
@@ -522,7 +528,15 @@ function cursorResultFromEvents(
522
528
  // streaming consumers. No-rotation case: no archives exist → behavior is
523
529
  // byte-identical to before (mergeArchiveTailEvents returns liveEvents).
524
530
  const merged = mergeArchiveTailEvents(readArchiveTailEvents(eventsPath, sinceSeq), filtered);
525
- const events = limit !== undefined ? merged.slice(0, limit) : merged;
531
+ // FS3 (fix): seq-sort before the limit slice. File-order slicing on a
532
+ // documented non-monotonic log can deliver an early high seq inside the
533
+ // first `limit` events; nextSeq=max-delivered then permanently skips every
534
+ // undelivered lower-seq event for paging consumers resuming from nextSeq.
535
+ // Sort ONLY under a limit: the no-limit path delivers every event (nextSeq
536
+ // = true max ⇒ nothing skippable) and keeps its historical FILE-order
537
+ // contract (pinned by event-log-torn-poison.test.ts AC7, U22 Phase 1).
538
+ const events =
539
+ limit !== undefined ? [...merged].sort((a, b) => (a.metadata?.seq ?? 0) - (b.metadata?.seq ?? 0)).slice(0, limit) : merged;
526
540
  const returnedMaxSeq = events.reduce((max, event) => Math.max(max, event.metadata?.seq ?? 0), sinceSeq);
527
541
  return {
528
542
  events,
@@ -782,7 +796,11 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
782
796
  const filtered = safeItems.filter((event) => (event.metadata?.seq ?? 0) > sinceSeq);
783
797
  const merged = mergeArchiveTailEvents(drainedArchiveEvents, filtered);
784
798
  const limit = positiveInteger(options.limit);
785
- const events = limit !== undefined ? merged.slice(0, limit) : merged;
799
+ // FS3 (fix): seq-sort before the limit slice — see cursorResultFromEvents.
800
+ // Sort ONLY under a limit (no-limit keeps the historical file-order
801
+ // contract; nothing is skippable when everything is delivered).
802
+ const events =
803
+ limit !== undefined ? [...merged].sort((a, b) => (a.metadata?.seq ?? 0) - (b.metadata?.seq ?? 0)).slice(0, limit) : merged;
786
804
  const returnedMaxSeq = events.reduce((max, event) => Math.max(max, event.metadata?.seq ?? 0), sinceSeq);
787
805
  // BR-08: `read.state.byteOffset` sits past the ENTIRE delta read above —
788
806
  // including the events the `limit` slice just dropped. Handing that back
@@ -849,9 +867,12 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
849
867
  tailBytes: TAIL_BYTES,
850
868
  });
851
869
  }
852
- return cursorResultFromEvents(tail.events, eventsPath, sinceSeq, limit, {
870
+ const result = cursorResultFromEvents(tail.events, eventsPath, sinceSeq, limit, {
853
871
  tornTail: tail.tornTail,
854
872
  poisonSkipped: tail.poisonSkipped,
855
873
  parseFailures: tail.parseFailures,
856
874
  });
875
+ // R9-N2: surface the dropped-prefix condition so replay consumers can
876
+ // switch to the fromByteOffset anchor instead of silently missing the head.
877
+ return tail.truncated ? { ...result, truncated: true } : result;
857
878
  }
@@ -307,10 +307,31 @@ export function applyCompactionAppendGuarded(
307
307
  * rotation — caller then proceeds without a splice, which is the pre-US-001
308
308
  * behaviour). Must be called while holding the event-log lock.
309
309
  */
310
+ /** R4-N1: typed abort — distinguishes "foreign rotation, skip this compaction
311
+ * cycle" from the benign transient read errors readTailSince degrades on. */
312
+ class ForeignRotationAbortError extends Error {
313
+ constructor(size: number, offset: number) {
314
+ super(
315
+ `event-log.compaction-foreign-rotation: live file (${size}B) smaller than compaction snapshot offset (${offset}B) — aborting cycle`,
316
+ );
317
+ this.name = "ForeignRotationAbortError";
318
+ }
319
+ }
320
+
310
321
  function readTailSince(filePath: string, offset: number): string {
311
322
  try {
312
323
  const size = fs.statSync(filePath).size;
313
- if (size <= offset) return "";
324
+ // R4-N1 (2026-10-17): a file SMALLER than the snapshot offset means a
325
+ // foreign rotation replaced the live file with a smaller one. The old
326
+ // `size <= offset` proceed-no-splice let the caller's atomicWrite then
327
+ // CLOBBER the new live file (post-rotation appends lost, ~1000 stale
328
+ // events resurrected, C2 reported SUCCESS). size === offset is the
329
+ // benign "nothing appended" case; size < offset ABORTS the compaction
330
+ // cycle (typed throw — every caller of applyCompactionAppendGuarded
331
+ // wraps in try/catch+log and skips this cycle; the next cycle
332
+ // re-evaluates against the new file).
333
+ if (size < offset) throw new ForeignRotationAbortError(size, offset);
334
+ if (size === offset) return "";
314
335
  const fd = fs.openSync(filePath, "r");
315
336
  try {
316
337
  const length = size - offset;
@@ -322,7 +343,10 @@ function readTailSince(filePath: string, offset: number): string {
322
343
  } finally {
323
344
  fs.closeSync(fd);
324
345
  }
325
- } catch {
346
+ } catch (error) {
347
+ // R4-N1: the foreign-rotation abort must ESCAPE the benign-error degrade
348
+ // below — the whole point is that the caller must not proceed.
349
+ if (error instanceof ForeignRotationAbortError) throw error;
326
350
  return "";
327
351
  }
328
352
  }
@@ -275,6 +275,11 @@ export class WorkerRequestBookkeeper {
275
275
  entry.timer = setTimeout(() => {
276
276
  this.pending.delete(id);
277
277
  entry.reject(new Error(`worker-atomic-writer: request timed out after ${this.timeoutMs}ms (id=${id})`));
278
+ // FS8 (2026-10-17): a timed-out request means the writer thread is
279
+ // WEDGED — every other pending/future request would time out too.
280
+ // Terminate the worker (existing kill util — drains the rest, frees
281
+ // the temp dir; the next write re-spawns a fresh worker).
282
+ terminateWorkerAtomicWriter();
278
283
  }, this.timeoutMs);
279
284
  entry.timer.unref();
280
285
  }
@@ -134,7 +134,15 @@ function normalizeEntry(value: unknown): ActiveRunRegistryEntry | undefined {
134
134
  return { runId, cwd, stateRoot, manifestPath, updatedAt };
135
135
  }
136
136
 
137
- export function readActiveRunRegistry(maxEntries = DEFAULT_CACHE.manifestMaxEntries): ActiveRunRegistryEntry[] {
137
+ /** FAMILY-2 (2026-10-17): fault-aware read — `readFault` is true when BOTH
138
+ * mirrors failed while at least one file EXISTS (existing-but-unreadable /
139
+ * corrupt). [] from a both-missing state is a legitimate fresh install;
140
+ * [] from a read fault must NOT be persisted back by RMW callers (that
141
+ * wipes every sibling entry). */
142
+ export function readActiveRunRegistryWithFault(maxEntries = DEFAULT_CACHE.manifestMaxEntries): {
143
+ entries: ActiveRunRegistryEntry[];
144
+ readFault: boolean;
145
+ } {
138
146
  let parsed: unknown;
139
147
  // 2.4 — prefer the binary mirror (JSON after header for schema-safe deserialization).
140
148
  // Fall back to JSON when the binary is missing or corrupt; this lets a 2-release
@@ -159,7 +167,19 @@ export function readActiveRunRegistry(maxEntries = DEFAULT_CACHE.manifestMaxEntr
159
167
  try {
160
168
  parsed = JSON.parse(fs.readFileSync(registryPath(), "utf-8"));
161
169
  } catch {
162
- return [];
170
+ // FAMILY-2: BOTH mirrors failed. [] is legitimate ONLY when neither
171
+ // file exists; an existing-but-unreadable mirror is a READ FAULT — log
172
+ // loudly so RMW callers know to skip the rebuild persist.
173
+ const anyMirrorPresent = fs.existsSync(registryBinaryPath()) || fs.existsSync(registryPath());
174
+ if (anyMirrorPresent) {
175
+ logInternalError(
176
+ "active-run-registry",
177
+ new Error("both mirrors failed to read — returning [] with readFault=true; RMW callers must SKIP the rebuild persist"),
178
+ undefined,
179
+ "warn",
180
+ );
181
+ }
182
+ return { entries: [], readFault: anyMirrorPresent };
163
183
  }
164
184
  }
165
185
  const entries = Array.isArray(parsed)
@@ -169,7 +189,11 @@ export function readActiveRunRegistry(maxEntries = DEFAULT_CACHE.manifestMaxEntr
169
189
  for (const entry of entries.sort((a, b) => (b.updatedAt ?? "").localeCompare(a.updatedAt ?? ""))) {
170
190
  if (!byId.has(entry.runId)) byId.set(entry.runId, entry);
171
191
  }
172
- return [...byId.values()].slice(0, Math.max(0, maxEntries));
192
+ return { entries: [...byId.values()].slice(0, Math.max(0, maxEntries)), readFault: false };
193
+ }
194
+
195
+ export function readActiveRunRegistry(maxEntries = DEFAULT_CACHE.manifestMaxEntries): ActiveRunRegistryEntry[] {
196
+ return readActiveRunRegistryWithFault(maxEntries).entries;
173
197
  }
174
198
 
175
199
  /**
@@ -372,7 +396,21 @@ export function registerActiveRun(manifest: TeamRunManifest): void {
372
396
  updatedAt: manifest.updatedAt,
373
397
  };
374
398
  withRegistryLock(() => {
375
- const existing = readActiveRunRegistry().filter((item) => item.runId !== manifest.runId);
399
+ // FAMILY-2: on a both-mirror read fault, SKIP the persist — registering
400
+ // into a wiped [] would drop every sibling entry (orphaned live-claims).
401
+ const readOutcome = readActiveRunRegistryWithFault();
402
+ if (readOutcome.readFault && readOutcome.entries.length === 0) {
403
+ logInternalError(
404
+ "active-run-registry",
405
+ new Error(
406
+ `registerActiveRun: registry unreadable — SKIPPING persist for runId=${manifest.runId} (rebuild would wipe siblings)`,
407
+ ),
408
+ undefined,
409
+ "warn",
410
+ );
411
+ return;
412
+ }
413
+ const existing = readOutcome.entries.filter((item) => item.runId !== manifest.runId);
376
414
  // Inline cleanup: remove terminal-status and stale entries before writing.
377
415
  // This prevents unbounded growth between sessions.
378
416
  const { alive, terminalRunIds } = filterAliveEntries(existing);
@@ -398,9 +436,22 @@ export function unregisterActiveRun(runId: string): void {
398
436
  return;
399
437
  }
400
438
  withRegistryLock(() => {
439
+ // FAMILY-2: on a both-mirror read fault, SKIP the persist — the []
440
+ // rebuild would wipe every sibling entry. The entry simply stays
441
+ // registered and a later purge retries.
442
+ const readOutcome = readActiveRunRegistryWithFault();
443
+ if (readOutcome.readFault && readOutcome.entries.length === 0) {
444
+ logInternalError(
445
+ "active-run-registry",
446
+ new Error(`unregisterActiveRun: registry unreadable — SKIPPING persist for runId=${runId} (rebuild would wipe siblings)`),
447
+ undefined,
448
+ "warn",
449
+ );
450
+ return;
451
+ }
401
452
  // FIX Issue 1: Filter alive entries and pass terminal runIds to writeEntries,
402
453
  // closing the TOCTOU race between status check and write (same pattern as registerActiveRun).
403
- const { alive, terminalRunIds } = filterAliveEntries(readActiveRunRegistry().filter((entry) => entry.runId !== runId));
454
+ const { alive, terminalRunIds } = filterAliveEntries(readOutcome.entries.filter((entry) => entry.runId !== runId));
404
455
  writeEntries(alive, terminalRunIds);
405
456
  });
406
457
  }