pi-crew 0.9.44 → 0.9.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +136 -0
  2. package/README.md +38 -3
  3. package/dist/build-meta.json +349 -203
  4. package/dist/index.mjs +2229 -2968
  5. package/dist/index.mjs.map +4 -4
  6. package/docs/decisions/2026-07-21-broker-phase4-default-on.md +77 -0
  7. package/docs/decisions/2026-07-21-broker-windows-perms.md +91 -0
  8. package/docs/decisions/2026-07-22-broker-phase4-gated-on.md +99 -0
  9. package/docs/decisions/README.md +3 -0
  10. package/docs/publishing.md +26 -0
  11. package/package.json +3 -1
  12. package/scripts/build-bundle.mjs +7 -0
  13. package/scripts/postinstall.mjs +35 -1
  14. package/scripts/pty_probe.py +174 -0
  15. package/skills/real-test-pi-crew/SKILL.md +659 -0
  16. package/src/agents/discover-agents.ts +1 -1
  17. package/src/config/config.ts +42 -1
  18. package/src/config/defaults.ts +45 -1
  19. package/src/config/types.ts +19 -0
  20. package/src/extension/register.ts +6 -1
  21. package/src/extension/registration/context-builder.ts +4 -0
  22. package/src/extension/registration/lifecycle-handlers.ts +200 -6
  23. package/src/extension/registration/registration-types.ts +9 -0
  24. package/src/extension/registration/subagent-manager-setup.ts +178 -59
  25. package/src/extension/run-import.ts +21 -1
  26. package/src/extension/team-tool/api.ts +4 -2
  27. package/src/prompt/prompt-runtime.ts +108 -0
  28. package/src/runtime/async-runner.ts +9 -1
  29. package/src/runtime/broker-issuer.ts +37 -0
  30. package/src/runtime/child-pi-spawn.ts +53 -0
  31. package/src/runtime/child-pi.ts +42 -11
  32. package/src/runtime/crew-broker-child.ts +88 -0
  33. package/src/runtime/crew-broker-client.ts +673 -0
  34. package/src/runtime/crew-broker-tokens.ts +84 -0
  35. package/src/runtime/crew-broker.ts +1276 -0
  36. package/src/runtime/dynamic-workflow-context.ts +7 -3
  37. package/src/runtime/dynamic-workflow-runner.ts +1 -1
  38. package/src/runtime/manifest-cache.ts +30 -0
  39. package/src/runtime/plan-templates.ts +8 -6
  40. package/src/runtime/resilient-edit.ts +16 -15
  41. package/src/runtime/role-permission.ts +27 -2
  42. package/src/runtime/run-coalesced-task-group.ts +72 -15
  43. package/src/runtime/task-packet.ts +1 -1
  44. package/src/schema/config-schema.ts +14 -0
  45. package/src/state/event-log.ts +88 -34
  46. package/src/state/locks.ts +53 -0
  47. package/src/state/mailbox.ts +208 -4
  48. package/src/state/run-metrics.ts +40 -12
  49. package/src/ui/key-utils.ts +42 -0
  50. package/src/ui/keybinding-map.ts +29 -3
  51. package/src/ui/live-run-sidebar.ts +1 -9
  52. package/src/ui/run-dashboard.ts +29 -9
  53. package/src/ui/settings-overlay.ts +42 -22
  54. package/src/utils/incremental-reader.ts +105 -0
  55. package/src/utils/ndjson.ts +115 -0
  56. package/src/utils/session-utils.ts +30 -0
  57. package/src/utils/socket-path.ts +127 -0
  58. package/src/utils/visual.ts +27 -91
  59. package/workflows/default.workflow.md +1 -1
  60. package/workflows/fast-fix.workflow.md +1 -1
  61. package/workflows/plan-execute.workflow.md +1 -1
  62. package/workflows/review.workflow.md +1 -1
  63. package/src/runtime/auto-resume.ts +0 -100
  64. package/src/runtime/notebook-helpers.ts +0 -88
  65. package/src/runtime/orphan-sentinel.ts +0 -7
@@ -236,9 +236,13 @@ export function synthesizeAgentConfig(name: string, model?: string): AgentConfig
236
236
  description: `Synthesized agent for dynamic workflow (${name}).`,
237
237
  source: "dynamic",
238
238
  filePath: `<dynamic-workflow>`,
239
- systemPrompt: `You are ${name}.`,
239
+ systemPrompt: `You are ${name}, an agent in a dynamic pi-crew workflow. Use the provided tools (read, grep, find, ls, bash) to investigate the target and produce concrete written findings. Do not return an empty response — always output substantive content for your task.`,
240
240
  model,
241
- tools: [],
241
+ // Round-N fix: give synthesized agents the STANDARD file-investigation toolkit
242
+ // (matches real built-in agents). Previously tools:[] relied on pi-args default
243
+ // behavior, which left most synthesized agents (explorer/analyst/critic/executor)
244
+ // unable to read the codebase → 10/11 agents returned empty/!ok in distill-dwf runs.
245
+ tools: ["read", "grep", "find", "ls", "bash"],
242
246
  inheritProjectContext: false,
243
247
  inheritSkills: false,
244
248
  };
@@ -420,7 +424,7 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
420
424
  // This correctly reduces spent when actualUsage < ESTIMATE.
421
425
  wfState.spent += (parsed.usage?.input ?? 0) + (parsed.usage?.output ?? 0) - ESTIMATE;
422
426
  reserved = false;
423
- let text = parsed.finalText ?? "";
427
+ let text = childResult.rawFinalText || parsed.finalText || "";
424
428
  // Round-11 test fix: parsePiJsonOutput only extracts text from pi event stream
425
429
  // ({type:"message_end", message:{role:"assistant", content:[...]}}). When the
426
430
  // agent emits plain JSON, plain text, or a different format, finalText is empty.
@@ -208,7 +208,7 @@ export async function runDynamicWorkflow(input: RunDynamicWorkflowInput): Promis
208
208
  // it does NOT kill the script. Promise.race with a hard timeout at least returns an
209
209
  // error so the runner doesn't hang. The spawned child process is leaked, but the
210
210
  // dynamic-workflow returns failure promptly. (v1.5: use Worker threads to actually kill.)
211
- const SCRIPT_TIMEOUT_MS = Number.parseInt(process.env.PI_CREW_DWF_SCRIPT_TIMEOUT_MS ?? "", 10) || 600_000; // 10 min default
211
+ const SCRIPT_TIMEOUT_MS = Number.parseInt(process.env.PI_CREW_DWF_SCRIPT_TIMEOUT_MS ?? "", 10) || 1_800_000; // 30 min default (was 10 min; raised for distill-dwf + similar long pipelines)
212
212
  let timeoutHandle: NodeJS.Timeout | undefined;
213
213
  const timeoutPromise = new Promise<never>((_, reject) => {
214
214
  timeoutHandle = setTimeout(() => {
@@ -164,6 +164,7 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
164
164
  manifestIndex.clear();
165
165
  }
166
166
  listCache.clear();
167
+ invalidateListActive();
167
168
  }
168
169
 
169
170
  function scheduleListRefresh(): void {
@@ -174,12 +175,17 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
174
175
  const timer = listTimer;
175
176
  listTimer = undefined;
176
177
  listCache.clear();
178
+ invalidateListActive();
177
179
  timer?.unref();
178
180
  }, ttlMs);
179
181
  // Unref immediately so the timer never blocks process exit (defense in
180
182
  // depth: the in-callback unref above may not run if shutdown happens
181
183
  // before the timer fires).
182
184
  listTimer.unref();
185
+ // FIND-03: invalidate the listActive() cache eagerly on every watcher
186
+ // tick. The TTL is the fallback for missed events; the watcher-driven
187
+ // path gives the tightest possible invalidation.
188
+ invalidateListActive();
183
189
  }
184
190
 
185
191
  function loadManifest(runId: string, rootsToCheck: string[]): CachedManifest | undefined {
@@ -266,6 +272,18 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
266
272
  return undefined;
267
273
  }
268
274
 
275
+ // FIND-03: short-TTL cache for listActive(). Mirrors the listCache pattern
276
+ // used by list(): we cache the un-capped running set and apply the caller's
277
+ // `limit` post-hoc on every return. Storing the full set (not a
278
+ // limit-sliced array) is what preserves the RT-F3 contract — callers with
279
+ // different `limit` values all see the same underlying "every running run"
280
+ // result, never a top-N createdAt-filtered view.
281
+ let listActiveCache: { result: TeamRunManifest[] | null; expiresAt: number } = { result: null, expiresAt: 0 };
282
+
283
+ function invalidateListActive(): void {
284
+ listActiveCache = { result: null, expiresAt: 0 };
285
+ }
286
+
269
287
  /**
270
288
  * RT-F3: filter to `status === "running"` BEFORE applying the limit so an
271
289
  * orphaned run that has been pushed past the top-N by recent successful
@@ -277,9 +295,19 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
277
295
  * silently drop "running" runs that fell past the top-N createdAt cutoff.
278
296
  * Still goes through parseManifestIfChanged for stat+size memoization, so
279
297
  * the per-run I/O cost is the same as list().
298
+ *
299
+ * FIND-03 perf: the full scan is memoized behind a 500ms TTL (same TTL as
300
+ * list()). fs.watch-driven scheduleListRefresh() invalidates the cache
301
+ * immediately so the next call re-scans. The cap is applied AFTER the
302
+ * cache lookup so a cached scan result can be sliced to ANY limit without
303
+ * re-scanning.
280
304
  */
281
305
  function listActive(limit: number): TeamRunManifest[] {
282
306
  const cap = Math.max(0, limit);
307
+ const now = Date.now();
308
+ if (listActiveCache.result !== null && listActiveCache.expiresAt > now) {
309
+ return listActiveCache.result.slice(0, cap);
310
+ }
283
311
  const parsedEntries = [
284
312
  ...roots.flatMap((root) => collectRoots(root)),
285
313
  ...activeRunEntries().map((entry) => ({
@@ -303,6 +331,7 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
303
331
  .filter((value): value is CachedManifest => value !== undefined)
304
332
  .map((value) => value.manifest)
305
333
  .filter((manifest) => manifest.status === "running");
334
+ listActiveCache = { result: running, expiresAt: now + ttlMs };
306
335
  return running.slice(0, cap);
307
336
  }
308
337
 
@@ -340,6 +369,7 @@ export function createManifestCache(cwd: string, options: ManifestCacheOptions =
340
369
  watchers = [];
341
370
  manifestIndex.clear();
342
371
  listCache.clear();
372
+ invalidateListActive();
343
373
  },
344
374
  };
345
375
  }
@@ -140,14 +140,15 @@ registerPlanTemplate({
140
140
  {
141
141
  name: "verify",
142
142
  role: "verifier",
143
- taskTemplate: "Verify that all review findings are addressed. Run tests if applicable. Confirm: {{goal}} is achieved.",
143
+ taskTemplate:
144
+ "Verify that all review findings are addressed. Run FAST checks (completes in <2 min): `npm run test:critical && npx tsc --noEmit`. Do NOT run `npm run test:unit` or `npm test` — too slow (642 files, >4 min). Confirm: {{goal}} is achieved.",
144
145
  maxTasks: 1,
145
146
  dependsOn: ["review"],
146
- verificationCommand: "npm test",
147
+ verificationCommand: "npm run test:critical && npx tsc --noEmit",
147
148
  },
148
149
  ],
149
150
  verificationCommands: {
150
- verify: "npm test",
151
+ verify: "npm run test:critical && npx tsc --noEmit",
151
152
  },
152
153
  });
153
154
 
@@ -187,13 +188,14 @@ registerPlanTemplate({
187
188
  {
188
189
  name: "verify",
189
190
  role: "verifier",
190
- taskTemplate: "Verify the complete implementation of: {{goal}}. Run tests, check types, validate all acceptance criteria.",
191
+ taskTemplate:
192
+ "Verify the complete implementation of: {{goal}}. Run FAST checks (`npm run test:critical && npx tsc --noEmit`, completes in <2 min). Do NOT run `npm run test:unit` or `npm test` — too slow for in-loop verification (642 files, >4 min). Validate all acceptance criteria.",
191
193
  maxTasks: 1,
192
194
  dependsOn: ["review"],
193
- verificationCommand: "npm test && npx tsc --noEmit",
195
+ verificationCommand: "npm run test:critical && npx tsc --noEmit",
194
196
  },
195
197
  ],
196
198
  verificationCommands: {
197
- verify: "npm test && npx tsc --noEmit",
199
+ verify: "npm run test:critical && npx tsc --noEmit",
198
200
  },
199
201
  });
@@ -25,13 +25,11 @@
25
25
  import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
26
26
  import { type ReplaceResult, replace } from "../runtime/replace.ts";
27
27
 
28
- interface ToolLike {
28
+ interface EditToolLike {
29
29
  name: string;
30
30
  description: string;
31
31
  parameters: unknown;
32
- execute: (toolCallId: string, params: any, signal: any, onUpdate: any) => Promise<unknown>;
33
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
34
- [key: string]: any;
32
+ execute: (toolCallId: string, params: unknown, signal: unknown, onUpdate: unknown) => Promise<unknown>;
35
33
  }
36
34
 
37
35
  interface EditParams {
@@ -62,12 +60,11 @@ function isNotFoundResult(result: unknown): boolean {
62
60
  /** Detect whether pi-diff is loaded (to avoid double-wrapping edit). */
63
61
  function isPiDiffLoaded(pi: ExtensionAPI): boolean {
64
62
  try {
65
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
66
- const piAny = pi as any;
67
- const extensions = piAny?.extensions ?? piAny?._extensions ?? [];
63
+ const piExt = pi as unknown as { extensions?: unknown[] | Record<string, unknown>; _extensions?: unknown[] };
64
+ const extensions = piExt?.extensions ?? piExt?._extensions ?? [];
68
65
  const names = Array.isArray(extensions)
69
66
  ? extensions.map((e: unknown) => (typeof e === "string" ? e : ((e as { name?: string })?.name ?? "")))
70
- : Object.keys(extensions);
67
+ : Object.keys(extensions as Record<string, unknown>);
71
68
  return names.some((n: string) => typeof n === "string" && n.includes("pi-diff"));
72
69
  } catch {
73
70
  return false;
@@ -82,28 +79,32 @@ function isPiDiffLoaded(pi: ExtensionAPI): boolean {
82
79
  * @param tools optional injected tool registry (for testing)
83
80
  * @returns true if the wrapper was applied, false if skipped
84
81
  */
85
- export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: ToolLike }): boolean {
82
+ export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: EditToolLike }): boolean {
86
83
  // Auto-disable if pi-diff is present (it has its own replace integration).
87
84
  if (isPiDiffLoaded(pi)) {
88
85
  return false;
89
86
  }
90
87
 
91
- const t = tools ?? (pi as unknown as { tools?: { edit?: ToolLike } }).tools;
88
+ const t = tools ?? (pi as unknown as { tools?: { edit?: EditToolLike } }).tools;
92
89
  if (!t?.edit?.execute) return false;
93
90
 
94
91
  const nativeExecute = t.edit.execute.bind(t.edit);
95
92
 
96
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
97
- t.edit.execute = async function resilientExecute(toolCallId: string, params: any, signal: any, onUpdate: any): Promise<unknown> {
93
+ t.edit.execute = async function resilientExecute(
94
+ toolCallId: string,
95
+ params: unknown,
96
+ signal: unknown,
97
+ onUpdate: unknown,
98
+ ): Promise<unknown> {
98
99
  try {
99
100
  const result = await nativeExecute(toolCallId, params, signal, onUpdate);
100
101
  if (!isNotFoundResult(result)) return result;
101
102
  // Fall through to resilient retry.
102
- return await retryWithReplace(params, toolCallId, signal, onUpdate);
103
+ return await retryWithReplace(params as EditParams, toolCallId, signal, onUpdate);
103
104
  } catch (err) {
104
105
  const msg = err instanceof Error ? err.message : String(err);
105
106
  if (NOT_FOUND_PATTERNS.some((re) => re.test(msg))) {
106
- return await retryWithReplace(params, toolCallId, signal, onUpdate);
107
+ return await retryWithReplace(params as EditParams, toolCallId, signal, onUpdate);
107
108
  }
108
109
  throw err;
109
110
  }
@@ -111,7 +112,7 @@ export function wrapEditWithResilientReplace(pi: ExtensionAPI, tools?: { edit: T
111
112
 
112
113
  return true;
113
114
 
114
- async function retryWithReplace(params: EditParams, toolCallId: string, signal: any, onUpdate: any): Promise<unknown> {
115
+ async function retryWithReplace(params: EditParams, toolCallId: string, signal: unknown, onUpdate: unknown): Promise<unknown> {
115
116
  const filePath = params.path ?? params.filePath;
116
117
  const oldStr = params.oldString ?? params.old_string;
117
118
  const newStr = params.newString ?? params.new_string;
@@ -1,20 +1,45 @@
1
1
  export type RolePermissionMode = "read_only" | "workspace_write" | "danger_full_access" | "explicit_confirm";
2
2
 
3
+ // (FIND-12 R1 review: the AgentPermissionOptions opt-in was dropped — no caller
4
+ // passed `options` and AgentConfig has no `permissions` field, so it was dead
5
+ // code. Custom write-capable roles must instead be added to WRITE_ROLES below.)
6
+
3
7
  // Read-only roles: cannot mutate files/source. `verifier` is NOT here — it runs
4
8
  // tests (bash + cache writes) so it is a WRITE role (F4). `planner` stays
5
9
  // read-only to preserve the plan-approval gate boundary (F3).
6
10
  const READ_ONLY_ROLES = new Set(["explorer", "reviewer", "security-reviewer", "analyst", "critic", "planner"]);
7
- const WRITE_ROLES = new Set(["executor", "test-engineer", "writer", "verifier"]);
11
+ // Write-capable roles: can mutate files/source within the workspace.
12
+ // FIND-12 (R1 review): this is an EXPLICIT allowlist — every shipped write-capable
13
+ // role must be listed here, otherwise default-deny demotes it to read-only.
14
+ // `agent` is the default direct-agent role (run.ts); `cold-verifier` runs bash/
15
+ // cache writes; `chain-executor` is the chain-workflow executor variant;
16
+ // `worker` is the autonomous goal-loop + dynamic-workflow executor role
17
+ // (goal-loop-runner.ts, run.ts dynamic workflow synthesis).
18
+ const WRITE_ROLES = new Set(["executor", "test-engineer", "writer", "verifier", "agent", "cold-verifier", "chain-executor", "worker"]);
8
19
  export interface PermissionCheckResult {
9
20
  allowed: boolean;
10
21
  mode: RolePermissionMode;
11
22
  reason?: string;
12
23
  }
13
24
 
25
+ /**
26
+ * Resolve the permission mode for a given role.
27
+ *
28
+ * FIND-12 (2026-07-20): **default-deny** — unknown/unrecognized roles now
29
+ * receive `"read_only"` instead of the previous permissive `"workspace_write"`.
30
+ * This prevents privilege escalation via typo'd or custom agent names that
31
+ * accidentally inherit write capabilities. Write-capable roles are an EXPLICIT
32
+ * allowlist (`WRITE_ROLES`); to add a custom write-capable role, add it there.
33
+ * (R1 review: a per-agent-config `permissions.workspaceWrite` opt-in was
34
+ * considered but dropped — it was unreachable from all 8 call sites.)
35
+ *
36
+ * @param role the role name (case-sensitive, must match exactly)
37
+ */
14
38
  export function permissionForRole(role: string): RolePermissionMode {
15
39
  if (READ_ONLY_ROLES.has(role)) return "read_only";
16
40
  if (WRITE_ROLES.has(role)) return "workspace_write";
17
- return "workspace_write";
41
+ // FIND-12: default-deny for unknown/unrecognized roles.
42
+ return "read_only";
18
43
  }
19
44
 
20
45
  export function currentCrewRole(env: NodeJS.ProcessEnv = process.env): string | undefined {
@@ -80,6 +80,18 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
80
80
 
81
81
  let rawOutput = "";
82
82
  let success = false;
83
+ // FIND-06: serialize heartbeat saves and retain the active save so terminal
84
+ // results can drain it before their final write.
85
+ let heartbeatTimer: ReturnType<typeof setInterval> | null = null;
86
+ let heartbeatInFlight = false;
87
+ let heartbeatPromise: Promise<void> | null = null;
88
+ // FIND-06 P1 fix (R1 review): a heartbeat save that exceeds the 5s drain
89
+ // timeout continues in the background and can rename its temp file AFTER
90
+ // the terminal write, clobbering terminal state with the pre-terminal
91
+ // snapshot. finalWriteStarted lets the IIFE repair that by re-writing the
92
+ // (now-terminal) updatedTasks AFTER its own save resolves, so the terminal
93
+ // state always lands last regardless of disk timing.
94
+ let finalWriteStarted = false;
83
95
  if (!executeWorkers) {
84
96
  rawOutput = buildScaffoldOutput(groupTasks);
85
97
  success = true;
@@ -87,20 +99,41 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
87
99
  // Heartbeat refresher: touch every task's heartbeat every 15s while the
88
100
  // worker is alive. Set `alive: true` explicitly so post-completion
89
101
  // staleness checks immediately recognize liveness.
90
- const heartbeatTimer = setInterval(async () => {
91
- const now = new Date().toISOString();
92
- updatedTasks = updatedTasks.map((t) => {
93
- if (!taskIds.includes(t.id)) return t;
94
- return {
95
- ...t,
96
- heartbeat: touchWorkerHeartbeat(t.heartbeat ?? createWorkerHeartbeat(t.id), { alive: true }),
97
- };
98
- });
99
- try {
100
- await saveRunTasksAsync(manifest, updatedTasks);
101
- } catch {
102
- // Run may have been pruned mid-dispatch — best-effort only.
103
- }
102
+ heartbeatTimer = setInterval(() => {
103
+ // FIND-06: setInterval does not await async callbacks. Do not start a
104
+ // second read/modify/write while the preceding heartbeat save is active.
105
+ if (heartbeatInFlight) return;
106
+ heartbeatInFlight = true;
107
+ heartbeatPromise = (async () => {
108
+ try {
109
+ updatedTasks = updatedTasks.map((t) => {
110
+ if (!taskIds.includes(t.id)) return t;
111
+ // FIND-06 belt-and-suspenders: never replace terminal state or its
112
+ // resultArtifact with a heartbeat-only snapshot.
113
+ if (t.status === "completed" || t.status === "failed") return t;
114
+ return {
115
+ ...t,
116
+ heartbeat: touchWorkerHeartbeat(t.heartbeat ?? createWorkerHeartbeat(t.id), { alive: true }),
117
+ };
118
+ });
119
+ await saveRunTasksAsync(manifest, updatedTasks);
120
+ } catch {
121
+ // Run may have been pruned mid-dispatch — best-effort only.
122
+ } finally {
123
+ heartbeatInFlight = false;
124
+ // FIND-06 P1 fix: if the terminal write started while our heartbeat
125
+ // save was in flight, re-write the (now-terminal) updatedTasks so
126
+ // our late snapshot cannot leave stale pre-terminal state on disk.
127
+ // Runs after our own save resolves, so it always lands last.
128
+ if (finalWriteStarted) {
129
+ try {
130
+ await saveRunTasksAsync(manifest, updatedTasks);
131
+ } catch {
132
+ // best-effort repair — same swallow policy as the heartbeat.
133
+ }
134
+ }
135
+ }
136
+ })();
104
137
  }, 15_000);
105
138
  try {
106
139
  const result = await executeWithRetry(
@@ -124,7 +157,7 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
124
157
  rawOutput = `Worker dispatch failed: ${err instanceof Error ? err.message : String(err)}`;
125
158
  success = false;
126
159
  } finally {
127
- clearInterval(heartbeatTimer);
160
+ if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
128
161
  }
129
162
  }
130
163
 
@@ -163,6 +196,30 @@ export async function runCoalescedTaskGroup(input: CoalescedTaskGroupInput): Pro
163
196
  resultArtifact,
164
197
  };
165
198
  });
199
+
200
+ // FIND-06: stop ticks and drain the snapshot captured by any active
201
+ // heartbeat before persisting terminal results. Bound the wait so a stuck
202
+ // filesystem operation cannot block dispatch completion indefinitely.
203
+ // P1 fix (R1 review): set finalWriteStarted BEFORE the drain so any pending
204
+ // heartbeat IIFE observes it and re-writes the terminal state after its
205
+ // own (possibly late) save resolves — preventing a late heartbeat snapshot
206
+ // from clobbering the terminal write.
207
+ finalWriteStarted = true;
208
+ if (heartbeatTimer !== null) clearInterval(heartbeatTimer);
209
+ const pendingHeartbeat = heartbeatPromise;
210
+ if (pendingHeartbeat) {
211
+ let drainTimeout: ReturnType<typeof setTimeout> | undefined;
212
+ try {
213
+ await Promise.race([
214
+ pendingHeartbeat,
215
+ new Promise<void>((resolve) => {
216
+ drainTimeout = setTimeout(resolve, 5_000);
217
+ }),
218
+ ]);
219
+ } finally {
220
+ if (drainTimeout !== undefined) clearTimeout(drainTimeout);
221
+ }
222
+ }
166
223
  await saveRunTasksAsync(manifest, updatedTasks);
167
224
  let updatedManifest: TeamRunManifest = {
168
225
  ...manifest,
@@ -85,7 +85,7 @@ export function buildTaskPacket(input: BuildTaskPacketInput): TaskPacket {
85
85
 
86
86
  // Generate a deterministic hash-based task ID for traceability and logging.
87
87
  // Uses goal + step ID + run ID as content parts.
88
- // TODO: Once TaskPacket type gains a hashId field, include this in the packet.
88
+ // TODO(hashId): tracked — add hashId to TaskPacket when the schema supports it.
89
89
  const _taskHashId = generateTaskHashId([input.manifest.goal, input.step.id, input.manifest.runId]);
90
90
 
91
91
  return {
@@ -275,6 +275,19 @@ export const PiTeamsUiConfigSchema = Type.Object(
275
275
  { additionalProperties: false },
276
276
  );
277
277
 
278
+ /** Phase 0 inter-pi broker config schema. Numeric limits are bounded per
279
+ * the plan: pathHashLen 4..32, maxFrameBytes 1024..1048576 (default 256 KiB),
280
+ * outboundQueueCap 32..4096 (default 256). */
281
+ export const CrewBrokerConfigSchema = Type.Object(
282
+ {
283
+ enabled: Type.Optional(Type.Boolean()),
284
+ pathHashLen: Type.Optional(Type.Integer({ minimum: 4, maximum: 32 })),
285
+ maxFrameBytes: Type.Optional(Type.Integer({ minimum: 1024, maximum: 1_048_576 })),
286
+ outboundQueueCap: Type.Optional(Type.Integer({ minimum: 32, maximum: 4096 })),
287
+ },
288
+ { additionalProperties: false },
289
+ );
290
+
278
291
  export const PiTeamsConfigSchema = Type.Object(
279
292
  {
280
293
  asyncByDefault: Type.Optional(Type.Boolean()),
@@ -297,6 +310,7 @@ export const PiTeamsConfigSchema = Type.Object(
297
310
  reliability: Type.Optional(PiTeamsReliabilityConfigSchema),
298
311
  otlp: Type.Optional(PiTeamsOtlpConfigSchema),
299
312
  ui: Type.Optional(PiTeamsUiConfigSchema),
313
+ broker: Type.Optional(CrewBrokerConfigSchema),
300
314
  },
301
315
  { additionalProperties: false },
302
316
  );
@@ -4,7 +4,7 @@ import * as path from "node:path";
4
4
  import { DEFAULT_EVENT_LOG } from "../config/defaults.ts";
5
5
  import { errors } from "../errors.ts";
6
6
  import { emitFromTeamEvent } from "../ui/run-event-bus.ts";
7
- import { type IncrementalReadState, readJsonlSince } from "../utils/incremental-reader.ts";
7
+ import { type IncrementalReadState, readJsonlSince, readJsonlTail } from "../utils/incremental-reader.ts";
8
8
  import { logInternalError } from "../utils/internal-error.ts";
9
9
  import { redactSecrets } from "../utils/redaction.ts";
10
10
  import { sleepSync } from "../utils/sleep.ts";
@@ -543,9 +543,13 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
543
543
  } catch {
544
544
  /* file does not exist */
545
545
  }
546
+ // FIND-10: track whether overflow handling modified the file so we can
547
+ // reuse fileStat for the post-overflow size check (avoids redundant stat).
548
+ let overflowHandled = false;
546
549
  if (!isTerminal && fileStat) {
547
550
  const stat = fileStat;
548
551
  if (stat.size > MAX_EVENTS_BYTES) {
552
+ overflowHandled = true;
549
553
  try {
550
554
  compactEventLog(eventsPath);
551
555
  } catch (error) {
@@ -564,11 +568,17 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
564
568
  }
565
569
  }
566
570
  }
571
+ // FIND-10: collapse redundant stat. If no overflow handling occurred,
572
+ // the file hasn't changed since fileStat — reuse it instead of re-stat'ing.
567
573
  let sizeCheckStat: fs.Stats | undefined;
568
- try {
569
- sizeCheckStat = await fs.promises.stat(eventsPath).catch(() => undefined);
570
- } catch {
571
- /* file does not exist */
574
+ if (overflowHandled) {
575
+ try {
576
+ sizeCheckStat = await fs.promises.stat(eventsPath).catch(() => undefined);
577
+ } catch {
578
+ /* file does not exist */
579
+ }
580
+ } else {
581
+ sizeCheckStat = fileStat;
572
582
  }
573
583
  try {
574
584
  if (sizeCheckStat && sizeCheckStat.size > MAX_EVENTS_BYTES) {
@@ -583,25 +593,44 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
583
593
  logInternalError("event-log.size-check", error, `eventsPath=${eventsPath}`);
584
594
  }
585
595
 
596
+ // FIND-10: post-append stat captured from the same fd (non-worker path)
597
+ // for reuse in the cache update below, avoiding a redundant path stat.
598
+ let postAppendStat: fs.Stats | undefined;
586
599
  if (!skippedDueToSize) {
587
600
  const line = JSON.stringify(redactSecrets(fullEvent)) + "\n";
588
601
  // Phase 1.5: when worker atomic writer is enabled, append via worker.
589
602
  if (isWorkerAtomicWriterEnabled()) {
590
603
  await appendFileViaWorker(eventsPath, line);
604
+ // Worker path: fsync via a separate open (worker manages its own fd).
605
+ const fd = await fs.promises.open(eventsPath, "r+");
606
+ try {
607
+ await fd.sync();
608
+ } finally {
609
+ await fd.close();
610
+ }
591
611
  } else {
592
- await fs.promises.appendFile(eventsPath, line, {
593
- encoding: "utf-8",
594
- flag: "a",
595
- });
596
- }
597
- // FIX: fsync to ensure event content is flushed to disk before persisting
598
- // the sequence number. This closes the crash window between appendFile and
599
- // persistSequence where sequence reuse could occur on restart.
600
- const fd = await fs.promises.open(eventsPath, "r+");
601
- try {
602
- await fd.sync();
603
- } finally {
604
- await fd.close();
612
+ // FIND-10: single-fd append+fsync. Opens in append mode, writes,
613
+ // fsyncs on the SAME fd, then closes — eliminating the separate
614
+ // open("r+") + sync that previously doubled the fd count. The
615
+ // fsync (seq-integrity protection) is preserved exactly: it still
616
+ // closes the crash window between append and persistSequence.
617
+ const fd = await fs.promises.open(eventsPath, "a");
618
+ try {
619
+ await fd.appendFile(line, "utf-8");
620
+ await fd.sync();
621
+ // FIND-10 R1 fix: the cache-optimization fd.stat() must NOT sit in the
622
+ // seq-durability critical path. If it threw (rare — fd invalidated),
623
+ // it would skip persistSequence below and reopen the seq-reuse
624
+ // window the fsync just closed. Guard it; fall back to undefined
625
+ // (the later cache-update takes a path stat instead).
626
+ try {
627
+ postAppendStat = await fd.stat();
628
+ } catch {
629
+ postAppendStat = undefined;
630
+ }
631
+ } finally {
632
+ await fd.close();
633
+ }
605
634
  }
606
635
  // FIX: Persist sequence AFTER successful appendFile to ensure sidecar
607
636
  // is only updated when the event is definitively written. If appendFile
@@ -609,7 +638,11 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
609
638
  // preventing sequence reuse on restart.
610
639
  persistSequence(eventsPath, seq);
611
640
  }
641
+ // FIND-10: track whether compaction happened after the append so the
642
+ // cache-update stat can safely reuse postAppendStat (file unchanged).
643
+ let compactedAfterAppend = false;
612
644
  if (appendCounter % 100 === 0 && needsRotation(eventsPath)) {
645
+ compactedAfterAppend = true;
613
646
  try {
614
647
  compactEventLog(eventsPath);
615
648
  } catch (error) {
@@ -626,11 +659,18 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
626
659
  // Only update the cache here (the sidecar persist is already done).
627
660
  const finalSeq = fullEvent.metadata?.seq ?? 0;
628
661
  try {
662
+ // FIND-10: reuse post-append fd stat when available and no compaction
663
+ // happened after the append (file unchanged). Falls back to path stat
664
+ // for the worker path, skipped events, or post-compaction cases.
629
665
  let statResult: fs.Stats | undefined;
630
- try {
631
- statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
632
- } catch {
633
- /* file may not exist */
666
+ if (postAppendStat && !compactedAfterAppend) {
667
+ statResult = postAppendStat;
668
+ } else {
669
+ try {
670
+ statResult = await fs.promises.stat(eventsPath).catch(() => undefined);
671
+ } catch {
672
+ /* file may not exist */
673
+ }
634
674
  }
635
675
  if (statResult) {
636
676
  if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
@@ -1217,24 +1257,38 @@ export function readEventsCursor(eventsPath: string, options: EventCursorOptions
1217
1257
  };
1218
1258
  }
1219
1259
 
1220
- // Original behavior: read entire file.
1221
- // FIX (Round 14, H7): When called WITHOUT fromByteOffset on a large file,
1222
- // fall back to reading only the tail (last 1MB) plus metadata about the
1223
- // dropped prefix. This avoids O(n) memory load on hot UI paths while
1224
- // preserving a sensible default.
1260
+ // FIND-05 default path: byte-level tail read (last 4MB) instead of
1261
+ // full-file read. Bounds CPU to O(tail bytes) instead of O(total
1262
+ // events). The legacy readEvents() full parse path is preserved for
1263
+ // callers that explicitly need the full history (e.g. tests that
1264
+ // assert exact contents) and as a small-file fallback.
1265
+ //
1266
+ // The 5000-event tail cap and the "event-log.cursor-full-read"
1267
+ // warning are preserved. A separate cursor-tail-truncated warning is
1268
+ // emitted whenever the file exceeds the 4MB tail budget, signalling
1269
+ // that a prefix was dropped and callers should pass fromByteOffset for
1270
+ // streaming reads.
1271
+ const TAIL_BYTES = 4 * 1024 * 1024; // 4 MB
1272
+ const TAIL_EVENT_CAP = 5000;
1225
1273
  const sinceSeq = positiveInteger(options.sinceSeq) ?? 0;
1226
1274
  const limit = positiveInteger(options.limit);
1227
- let all = readEvents(eventsPath);
1228
- const totalAll = all.length;
1229
- if (totalAll > 5000 && options.fromByteOffset === undefined) {
1230
- // TAIL READ: keep the most recent 5000 events to bound memory.
1231
- // Callers that need full history should pass fromByteOffset to stream.
1275
+
1276
+ const tail = readJsonlTail<TeamEvent>(eventsPath, TAIL_BYTES);
1277
+ let all = tail.items;
1278
+ if (tail.truncated) {
1279
+ logInternalError("event-log.cursor-tail-truncated", {
1280
+ eventsPath,
1281
+ returned: all.length,
1282
+ tailBytes: TAIL_BYTES,
1283
+ });
1284
+ }
1285
+ if (all.length > TAIL_EVENT_CAP) {
1232
1286
  logInternalError(
1233
1287
  "event-log.cursor-full-read",
1234
- new Error(`readEventsCursor read entire ${totalAll}-event log; pass fromByteOffset for incremental reads`),
1288
+ new Error(`readEventsCursor tail read dropped events from a larger log; pass fromByteOffset for incremental reads`),
1235
1289
  `eventsPath=${eventsPath}`,
1236
1290
  );
1237
- all = all.slice(-5000);
1291
+ all = all.slice(-TAIL_EVENT_CAP);
1238
1292
  }
1239
1293
  const filtered = all.filter((event) => (event.metadata?.seq ?? 0) > sinceSeq);
1240
1294
  const events = limit !== undefined ? filtered.slice(0, limit) : filtered;