pi-crew 0.11.1 → 0.11.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (154) hide show
  1. package/CHANGELOG.md +82 -9
  2. package/README.md +161 -1036
  3. package/agents/verifier.md +18 -7
  4. package/dist/index.mjs +742 -91462
  5. package/docs/README.md +57 -46
  6. package/docs/architecture.md +87 -33
  7. package/docs/commands-reference.md +9 -5
  8. package/docs/troubleshooting.md +3 -2
  9. package/package.json +5 -4
  10. package/schema.json +29 -0
  11. package/skills/real-test-pi-crew/SKILL.md +206 -41
  12. package/src/agents/agent-config.ts +1 -1
  13. package/src/agents/discover-agents.ts +1 -1
  14. package/src/config/config-validation.ts +15 -0
  15. package/src/config/config.ts +47 -13
  16. package/src/config/env-vars.ts +35 -0
  17. package/src/config/types.ts +19 -0
  18. package/src/errors.ts +2 -2
  19. package/src/extension/async-notifier.ts +55 -8
  20. package/src/extension/crew-input-router.ts +9 -2
  21. package/src/extension/help.ts +21 -10
  22. package/src/extension/knowledge-injection.ts +2 -1
  23. package/src/extension/management.ts +8 -3
  24. package/src/extension/notification-sink.ts +17 -0
  25. package/src/extension/plan-orchestrate.ts +19 -3
  26. package/src/extension/registration/command-utils.ts +28 -2
  27. package/src/extension/registration/commands/dashboard.ts +11 -1
  28. package/src/extension/registration/commands/manage.ts +31 -15
  29. package/src/extension/registration/commands/run.ts +24 -2
  30. package/src/extension/registration/commands/shared.ts +34 -10
  31. package/src/extension/registration/commands/status.ts +25 -2
  32. package/src/extension/registration/context-builder.ts +8 -2
  33. package/src/extension/registration/foreground-run-controller.ts +2 -2
  34. package/src/extension/registration/health-notify-policy.ts +100 -0
  35. package/src/extension/registration/lazy-configurers.ts +38 -0
  36. package/src/extension/registration/lifecycle-handlers.ts +107 -40
  37. package/src/extension/registration/lifecycle.ts +77 -10
  38. package/src/extension/registration/observability.ts +125 -53
  39. package/src/extension/registration/registration-types.ts +7 -5
  40. package/src/extension/registration/runtime-cleanup.ts +9 -3
  41. package/src/extension/registration/subagent-helpers.ts +38 -0
  42. package/src/extension/registration/team-tool.ts +9 -2
  43. package/src/extension/registration/wire-cross-extension.ts +30 -0
  44. package/src/extension/run-compare.ts +220 -0
  45. package/src/extension/run-export.ts +37 -5
  46. package/src/extension/run-maintenance.ts +155 -5
  47. package/src/extension/team-manager-command.ts +3 -2
  48. package/src/extension/team-tool/dispatch/index.ts +3 -2
  49. package/src/extension/team-tool/dispatch/manage.ts +5 -2
  50. package/src/extension/team-tool/goal-wrap.ts +7 -0
  51. package/src/extension/team-tool/goal.ts +16 -1
  52. package/src/extension/team-tool/handle-settings.ts +19 -2
  53. package/src/extension/team-tool/health-monitor.ts +21 -7
  54. package/src/extension/team-tool/lifecycle-actions.ts +49 -1
  55. package/src/extension/team-tool/parallel-dispatch.ts +16 -1
  56. package/src/extension/team-tool/plan.ts +10 -0
  57. package/src/extension/team-tool/routing-hint.ts +63 -0
  58. package/src/extension/team-tool/run-intent.ts +1 -1
  59. package/src/extension/team-tool/run.ts +1 -1
  60. package/src/extension/team-tool/status.ts +4 -0
  61. package/src/extension/team-tool.ts +52 -6
  62. package/src/extension/webhook-notify.ts +382 -0
  63. package/src/observability/metric-sink.ts +12 -2
  64. package/src/prompt/prompt-runtime.ts +82 -31
  65. package/src/prompt/worker-events-channel.ts +12 -0
  66. package/src/runtime/README.md +1 -1
  67. package/src/runtime/async-runner.ts +123 -1
  68. package/src/runtime/background-runner.ts +313 -234
  69. package/src/runtime/broker/crew-broker.ts +76 -71
  70. package/src/runtime/broker/delegate/delegate-event.ts +12 -8
  71. package/src/runtime/broker/delegate/shadow-lifecycle.ts +92 -0
  72. package/src/runtime/broker/protocol/connection-state.ts +10 -0
  73. package/src/runtime/broker/protocol/lock-busy.ts +76 -0
  74. package/src/runtime/broker/wait-status-cache.ts +1 -1
  75. package/src/runtime/child-pi/child-pi-timers.ts +1 -1
  76. package/src/runtime/child-pi/mock-fixtures.ts +48 -0
  77. package/src/runtime/crew-agent-records.ts +337 -45
  78. package/src/runtime/deadletter.ts +43 -1
  79. package/src/runtime/delegate-spawn.ts +21 -3
  80. package/src/runtime/deterministic-ast.ts +14 -2
  81. package/src/runtime/dispatch-batch.ts +72 -5
  82. package/src/runtime/goal-workflow/dynamic-workflow-context.ts +78 -58
  83. package/src/runtime/goal-workflow/dynamic-workflow-runner.ts +14 -2
  84. package/src/runtime/goal-workflow/goal-loop-runner.ts +105 -4
  85. package/src/runtime/group-join.ts +11 -3
  86. package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
  87. package/src/runtime/live-session/live-session-runtime.ts +1 -1
  88. package/src/runtime/model/model-fallback.ts +21 -1
  89. package/src/runtime/model/pi-args.ts +8 -10
  90. package/src/runtime/model/runtime-warmup.ts +2 -0
  91. package/src/runtime/output/output-validator.ts +10 -5
  92. package/src/runtime/plan-approval.ts +14 -1
  93. package/src/runtime/recovery/crash-recovery.ts +96 -49
  94. package/src/runtime/run-worker.ts +12 -1
  95. package/src/runtime/scheduling/global-worker-cap.ts +13 -6
  96. package/src/runtime/scheduling/run-coalesced-task-group.ts +27 -1
  97. package/src/runtime/scheduling/scheduler.ts +49 -13
  98. package/src/runtime/scheduling/semaphore.ts +148 -20
  99. package/src/runtime/scratchpad/README.md +1 -1
  100. package/src/runtime/scratchpad/protocol.ts +1 -1
  101. package/src/runtime/settings-store.ts +1 -1
  102. package/src/runtime/skill-instructions.ts +22 -0
  103. package/src/runtime/stale-reconciler.ts +85 -13
  104. package/src/runtime/subagent-manager.ts +43 -13
  105. package/src/runtime/task-runner/child-executor.ts +17 -6
  106. package/src/runtime/task-runner/post-execution.ts +44 -14
  107. package/src/runtime/task-runner/pre-execution.ts +26 -2
  108. package/src/runtime/task-runner/prompt-builder.ts +150 -45
  109. package/src/runtime/task-runner/retrieval-orchestrator.ts +36 -14
  110. package/src/runtime/task-runner.ts +28 -1
  111. package/src/runtime/team-runner.ts +42 -2
  112. package/src/runtime/verification/verification-worktree.ts +3 -0
  113. package/src/runtime/workspace-lock.ts +4 -1
  114. package/src/schema/config-schema.ts +14 -0
  115. package/src/schema/team-tool-schema.ts +17 -0
  116. package/src/state/atomic-write.ts +53 -0
  117. package/src/state/contracts.ts +109 -0
  118. package/src/state/coordination/locks.ts +191 -33
  119. package/src/state/coordination/mailbox.ts +147 -17
  120. package/src/state/crew-init.ts +87 -12
  121. package/src/state/event-log/cursor.ts +37 -1
  122. package/src/state/event-log/event-log-rotation.ts +72 -7
  123. package/src/state/stores/active-run-registry.ts +13 -1
  124. package/src/state/stores/state-store.ts +120 -23
  125. package/src/state/types.ts +4 -0
  126. package/src/ui/dashboard-panes/agents-pane.ts +22 -10
  127. package/src/ui/heartbeat-aggregator.ts +34 -0
  128. package/src/ui/inline-panel/index.ts +1 -1
  129. package/src/ui/keybinding-map.ts +22 -4
  130. package/src/ui/live-conversation-overlay.ts +6 -3
  131. package/src/ui/render-diff.ts +19 -6
  132. package/src/ui/run-dashboard.ts +109 -6
  133. package/src/ui/run-snapshot-cache.ts +27 -4
  134. package/src/ui/spinner.ts +26 -2
  135. package/src/ui/tool-progress-formatter.ts +2 -1
  136. package/src/ui/tool-renderers/brief-mode.ts +2 -1
  137. package/src/ui/tool-renderers/index.ts +3 -3
  138. package/src/ui/widget/task-list.ts +8 -5
  139. package/src/ui/widget/widget-model.ts +3 -1
  140. package/src/utils/incremental-reader.ts +11 -3
  141. package/src/utils/paths.ts +94 -12
  142. package/src/utils/project-markers.ts +40 -0
  143. package/src/worktree/worktree-manager.ts +206 -26
  144. package/tsconfig.json +2 -0
  145. package/workflows/distill.workflow.md +3 -3
  146. package/workflows/fast-fix.workflow.md +1 -1
  147. package/workflows/plan-execute.workflow.md +1 -1
  148. package/workflows/review.workflow.md +1 -1
  149. package/workflows/strict-fast-fix.workflow.md +1 -1
  150. package/docs/migration-v0.4-v0.5.md +0 -208
  151. package/docs/runtime-flow.md +0 -148
  152. package/src/ui/card-colors.ts +0 -168
  153. package/src/utils/completion-dedupe.ts +0 -63
  154. package/src/utils/resolve-shell.ts +0 -36
@@ -3,11 +3,92 @@
3
3
  *
4
4
  * Adapted from oh-my-pi's `parallel.ts` Semaphore class and
5
5
  * `mapWithConcurrencyLimit` implementation. Provides:
6
- * - Explicit acquire/release Semaphore for concurrency control
7
- * - Fail-fast on first error (via Promise.race)
8
- * - AbortSignal support for graceful cancellation
9
- * - Partial results on abort
6
+ * - Explicit acquire/release Semaphore for concurrency control, with an
7
+ * OPTIONAL AbortSignal on acquire() (RR-014 / F15: a queued waiter settles
8
+ * promptly when its signal fires — it never waits out the current slot
9
+ * holder, and never consumes a slot once cancelled)
10
+ * - Fail-fast on first error (via Promise.race) — mapWithFailFast below
11
+ * - AbortSignal support for graceful cancellation — mapWithFailFast below
12
+ * - Partial results on abort — mapWithFailFast below
13
+ *
14
+ * RR-014 / F15 defect history: acquire() previously took NO AbortSignal, so a
15
+ * cancelled waiter stayed pending until the slot holder released, and then
16
+ * "received" a slot it could never use (verified probe: acquire() returned
17
+ * with signal.aborted === true). The signal-aware acquire below closes that:
18
+ * aborted waiters are eagerly removed from the queue and REJECT with
19
+ * SemaphoreAbortedError; #current never changes for an aborted acquire.
20
+ */
21
+
22
+ /**
23
+ * Rejection reason for {@link Semaphore.acquire} when the caller's AbortSignal
24
+ * fires before a slot is granted (RR-014 / F15). Distinguishable via
25
+ * `name === "SemaphoreAbortedError"` — an aborted acquire never bare-resolves.
26
+ */
27
+ export class SemaphoreAbortedError extends Error {
28
+ constructor() {
29
+ super("Semaphore acquire aborted: the caller's AbortSignal fired before a slot was granted");
30
+ this.name = "SemaphoreAbortedError";
31
+ }
32
+ }
33
+
34
+ /** Lifecycle outcome of a queued waiter. */
35
+ type WaiterOutcome = "granted" | "aborted";
36
+
37
+ /**
38
+ * Queue entry WITH identity (RR-014 / F15). The pre-fix queue stored bare
39
+ * resolve functions: an aborted waiter could not be referenced or removed, so
40
+ * it stayed pending until a release() handed it a slot it could never use.
41
+ * A record with an idempotent settle() can detach itself from its abort
42
+ * listener exactly once no matter how the abort↔handoff race resolves.
43
+ */
44
+ interface Waiter {
45
+ /** Resolves on "granted"; rejects with SemaphoreAbortedError on "aborted". */
46
+ readonly promise: Promise<void>;
47
+ /** Whether settle() already ran (first call wins). */
48
+ readonly settled: boolean;
49
+ /**
50
+ * Settle this waiter exactly once and detach its abort listener.
51
+ * - "granted" → resolve: the waiter now OWNS a slot and must release().
52
+ * - "aborted" → reject: the waiter never owned a slot.
53
+ */
54
+ settle(kind: WaiterOutcome): void;
55
+ }
56
+
57
+ /**
58
+ * Create a waiter and attach its abort listener on the CALLER'S signal
59
+ * (never on a derived signal — a listener on AbortSignal.any() cannot be
60
+ * removed from the original). `onAbort` runs synchronously when the signal
61
+ * fires; the semaphore uses it to remove the waiter from the queue eagerly.
10
62
  */
63
+ function createWaiter(signal: AbortSignal | undefined, onAbort: () => void): Waiter {
64
+ let resolve!: () => void;
65
+ let reject!: (error: Error) => void;
66
+ const promise = new Promise<void>((res, rej) => {
67
+ resolve = res;
68
+ reject = rej;
69
+ });
70
+ let isSettled = false;
71
+ let detachListener: (() => void) | undefined;
72
+ if (signal) {
73
+ const listener = () => onAbort();
74
+ signal.addEventListener("abort", listener, { once: true });
75
+ detachListener = () => signal.removeEventListener("abort", listener);
76
+ }
77
+ return {
78
+ promise,
79
+ get settled() {
80
+ return isSettled;
81
+ },
82
+ settle(kind: WaiterOutcome) {
83
+ if (isSettled) return;
84
+ isSettled = true;
85
+ detachListener?.();
86
+ detachListener = undefined;
87
+ if (kind === "granted") resolve();
88
+ else reject(new SemaphoreAbortedError());
89
+ },
90
+ };
91
+ }
11
92
 
12
93
  /**
13
94
  * Simple counting semaphore for limiting concurrency across independently-scheduled async work.
@@ -15,7 +96,7 @@
15
96
  export class Semaphore {
16
97
  #max: number;
17
98
  #current = 0;
18
- #queue: Array<() => void> = [];
99
+ #queue: Waiter[] = [];
19
100
  // FIX (Round 15): Cap the waiter queue to prevent unbounded memory growth
20
101
  // if the semaphore is held for a long period and many tasks accumulate.
21
102
  static readonly MAX_QUEUE = 10_000;
@@ -24,9 +105,44 @@ export class Semaphore {
24
105
  this.#max = Math.max(1, max);
25
106
  }
26
107
 
27
- async acquire(): Promise<void> {
108
+ /**
109
+ * Acquire a slot, optionally abortable (RR-014 / F15).
110
+ *
111
+ * - No signal → unchanged semantics: blocks until a slot frees.
112
+ * - Signal already aborted at entry → rejects immediately (fail-closed):
113
+ * no slot taken, nothing enqueued. Abort listeners never re-fire for an
114
+ * already-aborted signal, so such a waiter could never be removed later.
115
+ * - Signal fires while WAITING → the waiter is removed from the queue
116
+ * EAGERLY (synchronously inside the abort listener) and rejects with
117
+ * SemaphoreAbortedError. It never consumes a slot: #current is untouched.
118
+ *
119
+ * Abort↔handoff race resolves to exactly one consistent state because
120
+ * every #queue/#current mutation is synchronous (no await between them)
121
+ * and settle() is idempotent (first call wins, and it detaches the abort
122
+ * listener):
123
+ * - abort wins → the waiter was spliced out of the queue, so release()
124
+ * cannot hand it a slot; the freed slot reaches the next ALIVE waiter
125
+ * (or decrements #current). Capacity intact.
126
+ * - grant wins → settle("granted") detached the listener, so a later
127
+ * abort is a no-op at semaphore level; the caller observes
128
+ * signal.aborted and releases normally (runChildPi's pre-spawn guard
129
+ * returns kind "aborted" without spawning, so withWorkerSlot's finally
130
+ * releases the slot within a microtask of the grant).
131
+ */
132
+ async acquire(signal?: AbortSignal): Promise<void> {
133
+ if (signal?.aborted) {
134
+ return Promise.reject(new SemaphoreAbortedError());
135
+ }
28
136
  if (this.#current < this.#max) {
29
137
  this.#current++;
138
+ // Re-check the abort↔grant window before resolving. The path above
139
+ // is synchronous so the flag cannot flip today; this guards against
140
+ // a future await being inserted here (belt-and-braces — concurrency
141
+ // primitive). Never resolve a granted acquire for an aborted signal.
142
+ if (signal?.aborted) {
143
+ this.#current--;
144
+ return Promise.reject(new SemaphoreAbortedError());
145
+ }
30
146
  return;
31
147
  }
32
148
  // FIX (Round 15): Reject when the waiter queue is full. The previous
@@ -38,23 +154,35 @@ export class Semaphore {
38
154
  new Error(`Semaphore queue full: ${this.#queue.length} waiters (max ${Semaphore.MAX_QUEUE}); cannot acquire slot`),
39
155
  );
40
156
  }
41
- const { promise, resolve } = (() => {
42
- let res: () => void;
43
- const p = new Promise<void>((r) => {
44
- res = r;
45
- });
46
- return { promise: p, resolve: res! };
47
- })();
48
- this.#queue.push(resolve);
49
- return promise;
157
+ const waiter = createWaiter(signal, () => {
158
+ // Eager removal, synchronously inside the abort listener: the waiter
159
+ // leaves the queue BEFORE settling. A release() that already
160
+ // shift()ed it (grant won the race) finds settle() idempotent; a
161
+ // release() that has not cannot hand a slot to a dead waiter.
162
+ // #current is untouched — this waiter never owned a slot.
163
+ const index = this.#queue.indexOf(waiter);
164
+ if (index >= 0) this.#queue.splice(index, 1);
165
+ waiter.settle("aborted");
166
+ });
167
+ this.#queue.push(waiter);
168
+ return waiter.promise;
50
169
  }
51
170
 
52
171
  release(): void {
53
- const next = this.#queue.shift();
54
- if (next) {
55
- next();
56
- } else if (this.#current > 0) {
57
- this.#current--;
172
+ // Hand the freed slot to the first ALIVE waiter (FIFO). A settled entry
173
+ // still sitting in the queue is impossible with eager removal but is
174
+ // skipped defensively — a slot must never burn on a dead waiter. Handoff
175
+ // TRANSFERS ownership: #current is NOT decremented; only when no alive
176
+ // waiter remains does it decrement.
177
+ while (true) {
178
+ const next = this.#queue.shift();
179
+ if (next === undefined) {
180
+ if (this.#current > 0) this.#current--;
181
+ return;
182
+ }
183
+ if (next.settled) continue;
184
+ next.settle("granted");
185
+ return;
58
186
  }
59
187
  // Guard: over-release is a no-op to prevent #current going negative
60
188
  }
@@ -125,7 +125,7 @@ attempt N+1 worker (scratchpad-lifecycle.ts)
125
125
  is a Phase 2.5/3 hardening if artifacts ever land in a shared location.
126
126
 
127
127
  > **REMOVED (2026-08-12):** `src/runtime/scratchpad/snapshot-hmac.ts` and its
128
- > test were deleted. Decision (P3 of `docs/rlm-fixes-implementation-plan.md`):
128
+ > test were deleted. Decision (P3 of `docs/archive/rlm-fixes-implementation-plan.md`):
129
129
  > the HMAC threat is double-conditional — it only materializes when scratchpad
130
130
  > has adoption (>0 cells; today: 0/83 runs) AND snapshots move to a
131
131
  > shared/networked store (today: same-uid dev-machine). Hardening a
@@ -46,7 +46,7 @@ export interface GuestToHost {
46
46
  };
47
47
  pong: { type: "pong"; id: string };
48
48
  host_request: {
49
- // RESERVED for future host bridge (rlm-deep-review-2026-08-12.md §5.2F /
49
+ // RESERVED for future host bridge (docs/archive/rlm-deep-review-2026-08-12.md §5.2F /
50
50
  // J2): guest-side cells would request host services (tools.read,
51
51
  // tools.grep) so data enters the namespace WITHOUT crossing the
52
52
  // transcript — pi-rlm's core token-saving value proposition. Declared
@@ -112,7 +112,7 @@ function readSettingsFile(filePath: string): CrewSettings {
112
112
  * API — never this function. Kept exported for back-compat reads/tests;
113
113
  * adding a new consumer that writes into loadedConfig re-introduces the
114
114
  * fixed P1 bypass (see docs/decisions/2026-08-15-schema-driven-sanitize.md
115
- * and the Wave 2B execution log in docs/refactor-plan.md).
115
+ * and the Wave 2B execution log in docs/archive/refactor-plan.md).
116
116
  */
117
117
  export function loadCrewSettings(cwd: string = process.cwd(), globalFile: string = globalPath()): CrewSettings {
118
118
  return {
@@ -7,6 +7,7 @@ import { packageRoot } from "../utils/paths.ts";
7
7
  import { isSafePathId, resolveRealContainedPath } from "../utils/safe-paths.ts";
8
8
  import type { WorkflowStep } from "../workflows/workflow-config.ts";
9
9
  import { CONFIDENCE_THRESHOLDS, getWeightedSkillsForRole, registerSkillEffectivenessHooks } from "./skill-effectiveness.ts";
10
+ import { promptSkillMode } from "./task-runner/prompt-builder.ts";
10
11
 
11
12
  const PACKAGE_SKILLS_DIR = path.join(packageRoot(), "skills");
12
13
 
@@ -324,6 +325,10 @@ export function renderSkillInstructions(
324
325
  const names = allNames.slice(0, MAX_SELECTED_SKILLS);
325
326
  const overflowCount = Math.max(0, allNames.length - names.length);
326
327
  if (names.length === 0) return { names, paths: [], block: "" };
328
+ // SR-02 phase 2: "index" mode (default) injects compact entries instead of
329
+ // full skill bodies — skills were 40-50% of measured worker prompts while a
330
+ // Path pointer + read tool gives the worker the SAME information on demand.
331
+ const mode = promptSkillMode();
327
332
  const sections: string[] = [];
328
333
  const skillPaths: string[] = [];
329
334
  let total = 0;
@@ -379,6 +384,23 @@ export function renderSkillInstructions(
379
384
  ]
380
385
  .filter(Boolean)
381
386
  .join("\n");
387
+ if (mode === "index") {
388
+ // Index entry: the description (frontmatter) says WHEN the skill
389
+ // applies; the Path says WHERE to read it. The worker loads the full
390
+ // SKILL.md only when the task matches — same information, on demand.
391
+ const entry = [
392
+ `## ${safeName}`,
393
+ description ? `Description: ${description}${confidenceNote}` : undefined,
394
+ `Source: ${source}`,
395
+ `Path: ${path.dirname(loaded.path)}`,
396
+ // biome-ignore lint/suspicious/noTemplateCurlyInString: ${Path} refers to the Path field printed above, not a JS interpolation
397
+ "When this skill matches your task, FIRST read ${Path}/SKILL.md and follow it.",
398
+ ]
399
+ .filter(Boolean)
400
+ .join("\n");
401
+ if (!pushSection(entry)) omittedCount += 1;
402
+ continue;
403
+ }
382
404
  const rawContent = loaded.compacted;
383
405
  // Wrap skill content with provenance markers to help LLMs distinguish skill instructions
384
406
  const wrappedContent = `<!-- skill: ${safeName} -->\n${rawContent}\n<!-- end-skill: ${safeName} -->`;
@@ -1,17 +1,29 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as os from "node:os";
3
3
  import * as path from "node:path";
4
+ import { getCrewEnv } from "../config/env-vars.ts";
4
5
  import { errors } from "../errors.ts";
5
6
  import { atomicWriteFile, atomicWriteJson } from "../state/atomic-write.ts";
6
7
  import { getCurrentPlanRecord } from "../state/stores/plan-store.ts";
7
8
  import { loadManifestWithRecovery, loadTasksWithRecovery, saveRunManifest } from "../state/stores/state-store.ts";
8
9
  import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
9
10
  import { logInternalError } from "../utils/internal-error.ts";
11
+ import { findRunStateDir } from "../utils/paths.ts";
10
12
  import { recordFromTask, upsertCrewAgent } from "./crew-agent-records.ts";
11
13
  import { checkProcessLiveness } from "./process-status.ts";
12
14
 
13
15
  /** Age threshold for orphaned temp directory cleanup: 1 hour. */
14
16
  const ORPHAN_TEMP_DIR_AGE_THRESHOLD_MS = 60 * 60 * 1000;
17
+ /** A `.cleanup-in-progress` sentinel older than this is abandoned — its owner
18
+ * died mid-cleanup (session kill between sentinel-create and dir-delete) and
19
+ * without reclamation the workspace is frozen out of the cleanup cycle
20
+ * FOREVER (EEXIST → skip, no TTL). Live 2026-09-26: 42 frozen workspaces. */
21
+ const SENTINEL_STALE_RECLAIM_MS = 10 * 60 * 1000;
22
+ /** Assumed production cadence of the temp-workspace reconcile tick (60s —
23
+ * see observability.ts tempReconcileTimer). Only used to derive the
24
+ * stateless round-robin batch index; a different real cadence still cycles
25
+ * through every batch, just with different pacing. */
26
+ const ORPHAN_RECONCILE_TICK_MS = 60_000;
15
27
  /** Defense-in-depth: cap the number of /tmp/pi-crew-* entries processed per
16
28
  * reconcile tick. With a few thousand accumulated dirs, processing them all
17
29
  * synchronously can block the main thread for many seconds, causing the
@@ -284,7 +296,7 @@ function isTaskHeartbeatStale(task: TeamTaskState, now: number): boolean {
284
296
  const taskPid = task.heartbeat?.pid ?? task.checkpoint?.childPid;
285
297
  const pidAlive = taskPid ? checkProcessLiveness(taskPid).alive : false;
286
298
  if (taskPid && pidAlive) return false;
287
- if (process.env.PI_CREW_DEBUG_STALE === "1") {
299
+ if (getCrewEnv("PI_CREW_DEBUG_STALE") === "1") {
288
300
  // F1 forensic (battery 2026-09-10): sidecar log of every STALE verdict so
289
301
  // live repros can show exactly what the reconciler saw (pid present? alive?
290
302
  // elapsed?) — the reconciler may run in ANY host process, hence a fixed
@@ -568,18 +580,38 @@ export function reconcileOrphanedTempWorkspaces(
568
580
  const scanBatch = options?.scanBatchSize ?? ORPHAN_TEMP_SCAN_BATCH_SIZE;
569
581
  const candidates = entries
570
582
  .filter((e) => e.isDirectory() && e.name.startsWith("pi-crew-"))
571
- .sort((a, b) => a.name.localeCompare(b.name))
572
- .slice(0, scanBatch);
573
- for (const entry of candidates) {
583
+ .sort((a, b) => a.name.localeCompare(b.name));
584
+ // STATELESS ROUND-ROBIN across ticks (Bug B, live 2026-09-26): the old
585
+ // `slice(0, scanBatch)` visited only the alphabetically-first batch, so a
586
+ // cluster of uncleanable dirs (frozen sentinels / fresh sentinels /
587
+ // waiting runs) starved everything behind it — 42 frozen sentinels in the
588
+ // `agent-stale-wakeup-test-*` cluster (alphabetically first) blocked ~3.4k
589
+ // dirs from EVER being scanned. Each tick now takes the NEXT slice,
590
+ // derived deterministically from `now` (production cadence is 60s/tick —
591
+ // see observability.ts tempReconcileTimer), so every batch is visited
592
+ // within `batches` ticks without any persisted state. Deterministic under
593
+ // the injected `now` that tests already use.
594
+ const batches = Math.max(1, Math.ceil(candidates.length / scanBatch));
595
+ const batchIdx = Math.floor(now / ORPHAN_RECONCILE_TICK_MS) % batches;
596
+ const selected = candidates.slice(batchIdx * scanBatch, batchIdx * scanBatch + scanBatch);
597
+ for (const entry of selected) {
574
598
  if (!entry.isDirectory() || !entry.name.startsWith("pi-crew-")) continue;
575
599
  const workspaceDir = path.join(tmpDir, entry.name);
576
- const crewDir = path.join(workspaceDir, ".crew");
577
- if (!fs.existsSync(crewDir)) continue;
578
- const stateRunsDir = path.join(crewDir, "state", "runs");
579
- if (!fs.existsSync(stateRunsDir)) continue;
600
+ // RR-020 Fix 3: accept BOTH supported run-state layouts — `<dir>/.crew/
601
+ // state/runs` and the `.pi`-based `<dir>/.pi/teams/state/runs`. Only the
602
+ // `.crew` layout used to be seen here, so a temp workspace with live
603
+ // `.pi/teams` run state was invisible to the reconciler (and then
604
+ // deleted as debris by the legacy cleanup sweep).
605
+ const stateRunsDir = findRunStateDir(workspaceDir);
606
+ if (!stateRunsDir) continue;
580
607
  let hasRunning = false;
581
608
  try {
582
- for (const runDir of fs.readdirSync(stateRunsDir)) {
609
+ // Cold-verify F-2: Dirent.isDirectory() is FALSE for a symlink-to-dir
610
+ // (withFileTypes does not follow), so a planted `runs/<runId>` symlink
611
+ // is skipped instead of being read/written out-of-tree.
612
+ for (const entry of fs.readdirSync(stateRunsDir, { withFileTypes: true })) {
613
+ if (!entry.isDirectory()) continue;
614
+ const runDir = entry.name;
583
615
  const manifestPath = path.join(stateRunsDir, runDir, "manifest.json");
584
616
  const tasksPath = path.join(stateRunsDir, runDir, "tasks.json");
585
617
  if (!fs.existsSync(manifestPath) || !fs.existsSync(tasksPath)) continue;
@@ -700,14 +732,50 @@ export function reconcileOrphanedTempWorkspaces(
700
732
  fs.closeSync(sentinelFd);
701
733
  atomicWriteFile(sentinelPath, JSON.stringify({ startedAt: now }));
702
734
  } catch {
703
- // Sentinel already exists (another cleanup in progress) — skip
704
- canCleanup = false;
735
+ // Sentinel already exists. Either another cleanup is genuinely in
736
+ // progress (it holds the sentinel for seconds at most) OR a previous
737
+ // process died mid-cleanup and ABANDONED it — the frozen-workspace
738
+ // bug (no TTL on the skip). Reclaim sentinels older than
739
+ // SENTINEL_STALE_RECLAIM_MS and retry the exclusive create once;
740
+ // losing the retry race means someone else reclaimed first — skip.
741
+ let sentinelAgeMs = Number.NaN;
742
+ try {
743
+ sentinelAgeMs = now - fs.statSync(sentinelPath).mtimeMs;
744
+ } catch {
745
+ /* sentinel vanished — retry create below */
746
+ }
747
+ if (Number.isFinite(sentinelAgeMs) && sentinelAgeMs <= SENTINEL_STALE_RECLAIM_MS) {
748
+ canCleanup = false; // fresh sentinel — a live cleanup owns it
749
+ } else {
750
+ try {
751
+ fs.unlinkSync(sentinelPath); // stale/absent — reclaim
752
+ } catch {
753
+ /* already gone */
754
+ }
755
+ try {
756
+ const retryFd = fs.openSync(sentinelPath, "wx");
757
+ fs.closeSync(retryFd);
758
+ atomicWriteFile(sentinelPath, JSON.stringify({ startedAt: now, reclaimedFromStale: true }));
759
+ } catch {
760
+ canCleanup = false; // lost the reclaim race — skip
761
+ }
762
+ }
705
763
  }
706
764
  }
707
765
  if (canCleanup) {
708
766
  if (fs.existsSync(stateRunsDir)) {
709
767
  try {
710
- for (const runDir of fs.readdirSync(stateRunsDir)) {
768
+ // Cold-verify round 5 (E1): a string readdir FOLLOWS a symlinked
769
+ // `<runId>` entry, and loadManifestWithRecovery on a corrupt manifest
770
+ // then QUARANTINES it — renameSync resolves through the symlinked
771
+ // dir and renamed a file OUTSIDE the scanned tree (reproduced even
772
+ // with cleanupOrphanedTempDirs:false, fired by the production
773
+ // tempReconcileTimer every session). Dirent.isDirectory() is false
774
+ // for a symlink-to-dir, so planted entries are skipped. Deletion
775
+ // itself was already safe (rmSync unlinks symlinks, never follows).
776
+ for (const entry of fs.readdirSync(stateRunsDir, { withFileTypes: true })) {
777
+ if (!entry.isDirectory()) continue;
778
+ const runDir = entry.name;
711
779
  const manifestPath = path.join(stateRunsDir, runDir, "manifest.json");
712
780
  if (!fs.existsSync(manifestPath)) continue;
713
781
  const manifest = loadManifestWithRecovery(manifestPath, runDir);
@@ -733,7 +801,11 @@ export function reconcileOrphanedTempWorkspaces(
733
801
  let stillClean = true;
734
802
  if (fs.existsSync(stateRunsDir)) {
735
803
  try {
736
- for (const runDir of fs.readdirSync(stateRunsDir)) {
804
+ // Cold-verify round 5 (E1): same Dirent guard as the first gate —
805
+ // quarantine-rename through a symlinked `<runId>` wrote outside.
806
+ for (const entry of fs.readdirSync(stateRunsDir, { withFileTypes: true })) {
807
+ if (!entry.isDirectory()) continue;
808
+ const runDir = entry.name;
737
809
  const manifestPath = path.join(stateRunsDir, runDir, "manifest.json");
738
810
  if (!fs.existsSync(manifestPath)) continue;
739
811
  const manifest = loadManifestWithRecovery(manifestPath, runDir);
@@ -342,16 +342,43 @@ export class SubagentManager {
342
342
  }
343
343
  }
344
344
 
345
- async waitForRecord(id: string): Promise<SubagentRecord | undefined> {
345
+ async waitForRecord(id: string, timeoutMs = 300_000): Promise<SubagentRecord | undefined> {
346
+ // RR-021 WI-4.3i: bounded wait. A record with no promise whose status
347
+ // stays running/blocked used to spin forever (100ms sleep loop). On
348
+ // deadline expiry we return the CURRENT record — never undefined, which
349
+ // would be ambiguous with 'no such record'. Callers can inspect
350
+ // record.status to see the wait timed out on a still-active record.
351
+ const deadline = Date.now() + timeoutMs;
346
352
  while (true) {
347
353
  const record = this.records.get(id);
348
354
  if (!record) return undefined;
349
355
  if (record.status !== "running" && record.status !== "queued") return record;
350
- if (record.promise)
351
- await record.promise.catch((error) => {
352
- logInternalError("subagent-manager.waitForRecord", error, `id=${id}`);
353
- });
354
- else await new Promise((resolve) => setTimeout(resolve, 100));
356
+ if (Date.now() > deadline) return record;
357
+ if (record.promise) {
358
+ // Race the run promise against the remaining deadline — a wedged
359
+ // promise must not block the deadline check.
360
+ const remaining = deadline - Date.now();
361
+ let timer: NodeJS.Timeout | undefined;
362
+ try {
363
+ await Promise.race([
364
+ record.promise.catch((error) => {
365
+ logInternalError("subagent-manager.waitForRecord", error, `id=${id}`);
366
+ }),
367
+ // RR-021 review remediation: this timer MUST stay ref'd. unref() let the
368
+ // event loop drain before the deadline fired (node:test then cancels the
369
+ // awaiting test — "Promise resolution is still pending but the event loop
370
+ // has already resolved"). Always clearTimeout'd in the finally below, so
371
+ // ref'ing cannot leak (knowledge.md OwnedProcess timer guidance).
372
+ new Promise<void>((resolve) => {
373
+ timer = setTimeout(resolve, remaining);
374
+ }),
375
+ ]);
376
+ } finally {
377
+ if (timer !== undefined) clearTimeout(timer);
378
+ }
379
+ } else {
380
+ await new Promise((resolve) => setTimeout(resolve, Math.min(100, Math.max(1, deadline - Date.now()))));
381
+ }
355
382
  }
356
383
  }
357
384
 
@@ -477,19 +504,22 @@ export class SubagentManager {
477
504
  }
478
505
 
479
506
  private async pollRunToTerminal(cwd: string, record: SubagentRecord): Promise<void> {
480
- // Safety: max 30 minutes (1800 polls at 1s interval) to prevent infinite
481
- // polling if manifest file is deleted or run state becomes unrecoverable.
482
- const MAX_POLL_COUNT = 1800;
483
- let pollCount = 0;
507
+ // Safety: max 30 minutes of WALL-CLOCK polling to prevent infinite
508
+ // polling if the manifest file is deleted or run state becomes unrecoverable.
509
+ // (RR-021 WI-4.3h: was MAX_POLL_COUNT=1800, which silently assumed a 1s
510
+ // pollIntervalMs — but the interval is configurable, so a 5s interval
511
+ // polled for 2.5h while a 200ms interval timed out after 6 minutes.)
512
+ const POLL_DEADLINE_MS = 30 * 60 * 1000;
513
+ const pollDeadline = Date.now() + POLL_DEADLINE_MS;
484
514
  while (record.runId && (record.status === "running" || record.status === "blocked")) {
485
- if (++pollCount > MAX_POLL_COUNT) {
515
+ if (Date.now() > pollDeadline) {
486
516
  logInternalError(
487
517
  "subagent-manager.poll-timeout",
488
- new Error(`pollRunToTerminal exceeded ${MAX_POLL_COUNT} polls for runId=${record.runId}`),
518
+ new Error(`pollRunToTerminal exceeded ${POLL_DEADLINE_MS}ms wall-clock for runId=${record.runId}`),
489
519
  `id=${record.id}`,
490
520
  );
491
521
  record.status = "error";
492
- record.error = `Poll timeout: run did not reach terminal state after ${MAX_POLL_COUNT} seconds`;
522
+ record.error = `Poll timeout: run did not reach terminal state after ${POLL_DEADLINE_MS}ms (wall-clock)`;
493
523
  record.completedAt = Date.now();
494
524
  savePersistedSubagentRecord(cwd, record);
495
525
  return;
@@ -68,7 +68,7 @@ import { DEFAULT_RETRY_POLICY } from "../recovery/retry-executor.ts";
68
68
  import { runWorker } from "../run-worker.ts";
69
69
  import { parseSessionUsage } from "../session-usage.ts";
70
70
  import { recordSupervisorContact, supervisorContactFromEvent } from "../supervisor-contact.ts";
71
- import type { TaskExecutionResult } from "./post-execution.ts";
71
+ import type { ResultSource, TaskExecutionResult } from "./post-execution.ts";
72
72
  import type { StreamBridgeHandle, TaskExecutionContext } from "./pre-execution.ts";
73
73
  import { applyAgentProgressEvent, applyUsageToProgress, progressEventSummary, shouldFlushProgressEvent } from "./progress.ts";
74
74
  import { cleanResultText, isFinalChildEvent } from "./result-utils.ts";
@@ -987,6 +987,20 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
987
987
  });
988
988
  }
989
989
  }
990
+ // Finding #2 (2026-09-29 battery): tag WHICH fallback branch produced the
991
+ // artifact so post-execution can fail stderr/none-sourced results
992
+ // deterministically — the noise classifier alone is not a verdict oracle.
993
+ // Order/semantics are identical to the previous `??` chain: first branch
994
+ // whose cleaned text is non-empty wins; otherwise the placeholder ("none").
995
+ const resultCandidates: ReadonlyArray<{ source: ResultSource; text: string | undefined }> = [
996
+ { source: "rawFinalText", text: cleanResultText(rawFinalText) },
997
+ { source: "finalText", text: cleanResultText(parsedOutput?.finalText) },
998
+ { source: "stdout", text: cleanResultText(finalStdout) },
999
+ { source: "stderr", text: cleanResultText(finalStderr) },
1000
+ { source: "findings", text: cleanResultText(intermediateFindings) },
1001
+ ];
1002
+ const usedResult = resultCandidates.find((candidate) => candidate.text);
1003
+ const resultSource: ResultSource = usedResult?.source ?? "none";
990
1004
  const resultArtifact = writeArtifact(manifest.artifactsRoot, {
991
1005
  kind: "result",
992
1006
  relativePath: `results/${task.id}.txt`,
@@ -995,14 +1009,10 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
995
1009
  // transcript's 16K compaction — this is the authoritative worker output.
996
1010
  // Fall back to transcript-derived finalText, then stdout/stderr, so a
997
1011
  // missing raw capture (mock/error path) never yields empty/garbage.
998
- cleanResultText(rawFinalText) ??
999
- cleanResultText(parsedOutput?.finalText) ??
1000
- cleanResultText(finalStdout) ??
1001
- cleanResultText(finalStderr) ??
1012
+ usedResult?.text ??
1002
1013
  // #7 hardening: if all real output paths are empty (worker exhausted
1003
1014
  // budget on tool calls, no assistant text), use intermediate findings.
1004
1015
  // intermediateFindings captures the last N tool-result display lines.
1005
- cleanResultText(intermediateFindings) ??
1006
1016
  "(no output)",
1007
1017
  producer: task.id,
1008
1018
  });
@@ -1094,6 +1104,7 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
1094
1104
  parsedOutput,
1095
1105
  finalStdout,
1096
1106
  rawFinalText,
1107
+ resultSource,
1097
1108
  transcriptPath,
1098
1109
  terminalEvidence,
1099
1110
  startupEvidence,