pi-crew 0.11.11 → 0.11.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +102 -0
  2. package/dist/index.mjs +551 -519
  3. package/package.json +1 -1
  4. package/src/config/config-validation.ts +9 -0
  5. package/src/config/defaults.ts +7 -0
  6. package/src/config/env-vars.ts +24 -0
  7. package/src/config/types.ts +11 -0
  8. package/src/extension/registration/tool-registration.ts +7 -0
  9. package/src/extension/registration/verify-gate-tool.ts +263 -0
  10. package/src/extension/team-tool/run-deadline.ts +22 -1
  11. package/src/extension/team-tool/run.ts +38 -2
  12. package/src/extension/team-tool-types.ts +4 -0
  13. package/src/extension/team-tool.ts +31 -3
  14. package/src/prompt/inbox-poll.ts +141 -3
  15. package/src/runtime/broker/crew-broker.ts +48 -2
  16. package/src/runtime/broker/protocol/lock-busy.ts +47 -1
  17. package/src/runtime/broker/protocol/request-parsers.ts +8 -1
  18. package/src/runtime/budget-enforcement.ts +36 -20
  19. package/src/runtime/child-pi/child-pi-constants.ts +24 -0
  20. package/src/runtime/child-pi/child-pi-kill.ts +59 -2
  21. package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
  22. package/src/runtime/child-pi/child-pi-steering.ts +10 -8
  23. package/src/runtime/child-pi/child-pi-streams.ts +69 -19
  24. package/src/runtime/child-pi/child-pi.ts +29 -30
  25. package/src/runtime/child-pi/owned-process.ts +434 -0
  26. package/src/runtime/child-pi/postmortem-registry.ts +134 -0
  27. package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
  28. package/src/runtime/classifier/classifier-service.ts +277 -61
  29. package/src/runtime/dispatch-batch.ts +32 -0
  30. package/src/runtime/live-session/live-agent-control.ts +2 -0
  31. package/src/runtime/live-session/live-agent-manager.ts +72 -2
  32. package/src/runtime/live-session/live-session-runtime.ts +435 -40
  33. package/src/runtime/merge-loop.ts +63 -35
  34. package/src/runtime/recovery/crash-recovery.ts +141 -3
  35. package/src/runtime/recovery/retry-executor.ts +29 -0
  36. package/src/runtime/run-tracker.ts +28 -0
  37. package/src/runtime/stale-reconciler.ts +36 -1
  38. package/src/runtime/task-runner.ts +103 -0
  39. package/src/runtime/verification/classifier-pre-gate.ts +262 -0
  40. package/src/runtime/verification/verification-gates.ts +5 -1
  41. package/src/runtime/verification/verify-gate.ts +394 -0
  42. package/src/schema/config-schema.ts +13 -0
  43. package/src/state/contracts.ts +18 -0
  44. package/src/state/coordination/mailbox.ts +203 -55
  45. package/src/state/stores/sqlite-run-state.ts +378 -0
  46. package/src/state/stores/state-store.ts +134 -6
  47. package/src/ui/agents-jobs-browser.ts +154 -8
  48. package/src/ui/events-state-source.ts +358 -0
  49. package/src/ui/run-snapshot-cache.ts +87 -3
  50. package/src/ui/widget/widget-model.ts +13 -2
@@ -22,10 +22,25 @@
22
22
  * authenticated taskId, so a group broadcast returns to its sender and
23
23
  * must be dropped at consume time);
24
24
  * - dedup by message id across polls via a caller-owned seen-set and/or a
25
- * `sinceTs` watermark — one message delivers once.
25
+ * `sinceTs` watermark — one message delivers once;
26
+ * - U5 (2026-10-10): dedup is ALSO durable across restarts — every picked
27
+ * row is recorded in delivery.json (`messages[id]="acknowledged"` +
28
+ * `${direction}:${requestId}` index) and re-read on the next poll, so a
29
+ * worker that restarts mid-run (fresh process, EMPTY seen-set) never
30
+ * re-delivers. A resend carrying the same requestId is dropped by the
31
+ * same index; duplicate rows under one requestId deliver exactly once.
26
32
  */
27
- import { type MailboxMessage, readAllMailboxMessages } from "../state/coordination/mailbox.ts";
33
+ import * as path from "node:path";
34
+ import {
35
+ type MailboxDeliveryState,
36
+ type MailboxMessage,
37
+ readAllMailboxMessages,
38
+ readDeliveryState,
39
+ recordMailboxRequestIdDelivery,
40
+ } from "../state/coordination/mailbox.ts";
41
+ import { appendEventFireAndForget } from "../state/event-log/event-log.ts";
28
42
  import type { TeamRunManifest } from "../state/types.ts";
43
+ import { logInternalError } from "../utils/internal-error.ts";
29
44
 
30
45
  export interface WorkerInboxPickup {
31
46
  stateRoot: string;
@@ -37,23 +52,99 @@ export interface WorkerInboxPickup {
37
52
  seenIds?: Set<string>;
38
53
  }
39
54
 
55
+ /** U5: requestId of a row — the U5 top-level field with the legacy
56
+ * `data.requestId` shape (group_join) as fallback. */
57
+ function messageIdempotencyKey(message: MailboxMessage): string | undefined {
58
+ if (message.requestId) return message.requestId;
59
+ const fromData = message.data?.requestId;
60
+ return typeof fromData === "string" ? fromData : undefined;
61
+ }
62
+
63
+ // U5: in-process throttle for `mailbox.pickup_deduped` audit events — a
64
+ // restarted worker re-scans the same durable rows on EVERY 500ms tick, so an
65
+ // un-throttled per-skip event would spam the log (2/s/row). One event per
66
+ // (run, task, key) per process lifetime is the audit signal ("redelivery was
67
+ // prevented by the durable record"); bounded FIFO like the parse caches.
68
+ const DEDUP_EVENT_KEYS_MAX = 512;
69
+ const dedupEventKeys = new Map<string, true>();
70
+
71
+ function shouldEmitDedupEvent(runId: string, taskId: string, key: string): boolean {
72
+ const mapKey = `${runId}:${taskId}:${key}`;
73
+ if (dedupEventKeys.has(mapKey)) return false;
74
+ dedupEventKeys.set(mapKey, true);
75
+ while (dedupEventKeys.size > DEDUP_EVENT_KEYS_MAX) {
76
+ const oldest = dedupEventKeys.keys().next().value;
77
+ if (oldest === undefined) break;
78
+ dedupEventKeys.delete(oldest);
79
+ }
80
+ return true;
81
+ }
82
+
83
+ /** U5: audit event for a redelivery that the DURABLE record prevented (the
84
+ * in-memory seen-set alone could not — this is exactly the restart window). */
85
+ function emitPickupDedupedEvent(stateRoot: string, runId: string, taskId: string, message: MailboxMessage, layer: string): void {
86
+ if (!shouldEmitDedupEvent(runId, taskId, `${layer}:${message.id}`)) return;
87
+ try {
88
+ appendEventFireAndForget(path.join(stateRoot, "events.jsonl"), {
89
+ type: "mailbox.pickup_deduped",
90
+ runId,
91
+ taskId,
92
+ message: `Dropped duplicate inbox delivery (${layer}) for task ${taskId}.`,
93
+ data: {
94
+ layer,
95
+ messageId: message.id,
96
+ requestId: messageIdempotencyKey(message),
97
+ from: message.from,
98
+ kind: message.kind,
99
+ },
100
+ });
101
+ } catch (error) {
102
+ logInternalError("inbox-poll.pickup-deduped-event", error, `runId=${runId} taskId=${taskId}`);
103
+ }
104
+ }
105
+
106
+ /** U5: audit event for a successful pickup — with the requestId, the event
107
+ * log is sufficient to audit the dedup chain end-to-end (broker
108
+ * `mailbox.send` → worker `mailbox.pickup` → `mailbox.pickup_deduped`). */
109
+ function emitPickupEvent(stateRoot: string, runId: string, taskId: string, picked: MailboxMessage[]): void {
110
+ try {
111
+ appendEventFireAndForget(path.join(stateRoot, "events.jsonl"), {
112
+ type: "mailbox.pickup",
113
+ runId,
114
+ taskId,
115
+ message: `Picked up ${picked.length} inbox message(s) for task ${taskId}.`,
116
+ data: {
117
+ messageIds: picked.map((m) => m.id),
118
+ requestIds: picked.map((m) => messageIdempotencyKey(m) ?? null),
119
+ },
120
+ });
121
+ } catch (error) {
122
+ logInternalError("inbox-poll.pickup-event", error, `runId=${runId} taskId=${taskId}`);
123
+ }
124
+ }
125
+
40
126
  /**
41
127
  * Read the worker's inbox mailbox and return the messages that should be
42
128
  * surfaced as fenced context on the next turn.
43
129
  *
44
130
  * Stateless apart from the optional caller-owned `seenIds`/`sinceTs` — safe
45
- * to call on every 500ms poll tick.
131
+ * to call on every 500ms poll tick. U5: picked rows are ALSO recorded in the
132
+ * durable delivery state (delivery.json), so dedup survives a restart.
46
133
  */
47
134
  export function pollWorkerInbox(pickup: WorkerInboxPickup): MailboxMessage[] {
48
135
  const { stateRoot, runId, taskId } = pickup;
49
136
  if (!stateRoot || !runId || !taskId) return [];
50
137
  const manifest = { stateRoot, runId } as unknown as TeamRunManifest;
51
138
  let messages: MailboxMessage[];
139
+ let delivery: MailboxDeliveryState;
52
140
  try {
53
141
  // readAllMailboxMessages already merges run-level inbox + every task
54
142
  // mailbox; we route TO this worker below (never trust a caller-scoped
55
143
  // file path).
56
144
  messages = readAllMailboxMessages(manifest, "inbox");
145
+ // U5: the durable delivery record (mtime-cached read — cheap on the
146
+ // 500ms tick) is the restart-safe dedup source of truth.
147
+ delivery = readDeliveryState(manifest);
57
148
  } catch {
58
149
  // Transient read error (lock contention, rotated file) — never throw
59
150
  // out of a poll tick; the next 500ms tick retries.
@@ -61,8 +152,10 @@ export function pollWorkerInbox(pickup: WorkerInboxPickup): MailboxMessage[] {
61
152
  }
62
153
 
63
154
  const seen = pickup.seenIds;
155
+ const deliveredRequestIds = delivery.requestIds ?? {};
64
156
  const picked: MailboxMessage[] = [];
65
157
  const byId = new Set<string>();
158
+ const byRequestId = new Set<string>();
66
159
  for (const m of messages) {
67
160
  // Kind gate: only durable `message`s (per §15.2). Notify is
68
161
  // fire-and-forget (its own channel), steer/response/follow-up are
@@ -81,10 +174,55 @@ export function pollWorkerInbox(pickup: WorkerInboxPickup): MailboxMessage[] {
81
174
  if (seen?.has(m.id)) continue;
82
175
  // Within-call id dedup (duplicate file rows → one delivery).
83
176
  if (byId.has(m.id)) continue;
177
+ // ── U5 durable dedup (2026-10-10) ──────────────────────────────────
178
+ // These two layers survive a worker restart (delivery.json); the
179
+ // in-memory gates above cannot. Every skip here is a redelivery that
180
+ // was PREVENTED — audited via mailbox.pickup_deduped (once per key
181
+ // per process so the 500ms re-scan cannot spam the event log).
182
+ const requestId = messageIdempotencyKey(m);
183
+ const dedupKey = requestId !== undefined ? `${m.direction}:${requestId}` : undefined;
184
+ if (dedupKey !== undefined) {
185
+ // (direction, requestId) index — catches resends carrying the same
186
+ // requestId AND duplicate rows under one requestId.
187
+ if (deliveredRequestIds[dedupKey] !== undefined) {
188
+ emitPickupDedupedEvent(stateRoot, runId, taskId, m, "requestId");
189
+ continue;
190
+ }
191
+ // Within-call requestId dedup (two rows, one requestId → first wins).
192
+ if (byRequestId.has(dedupKey)) continue;
193
+ }
194
+ // Acknowledged delivery entry = the F09 durable "handled" record; the
195
+ // inbox line itself keeps `status:"queued"` forever, so WITHOUT this
196
+ // check a restarted worker re-delivers every already-consumed row
197
+ // (legacy rows without a requestId land here too).
198
+ if (delivery.messages[m.id] === "acknowledged") {
199
+ emitPickupDedupedEvent(stateRoot, runId, taskId, m, "acknowledged");
200
+ continue;
201
+ }
84
202
  byId.add(m.id);
203
+ if (dedupKey !== undefined) byRequestId.add(dedupKey);
85
204
  seen?.add(m.id);
86
205
  picked.push(m);
87
206
  }
207
+ if (picked.length > 0) {
208
+ // U5: persist the pickup — the durable record that powers the two
209
+ // layers above after a restart. Best-effort on failure: the in-memory
210
+ // seen-set still dedups within THIS process, and logInternalError
211
+ // makes the lost-durability window visible.
212
+ try {
213
+ recordMailboxRequestIdDelivery(
214
+ manifest,
215
+ picked.map((m) => ({
216
+ direction: m.direction,
217
+ messageId: m.id,
218
+ requestId: messageIdempotencyKey(m),
219
+ })),
220
+ );
221
+ emitPickupEvent(stateRoot, runId, taskId, picked);
222
+ } catch (error) {
223
+ logInternalError("inbox-poll.record-delivery", error, `runId=${runId} taskId=${taskId}`);
224
+ }
225
+ }
88
226
  // Deterministic delivery order (same sort readAllMailboxMessages applies).
89
227
  return picked;
90
228
  }
@@ -25,7 +25,12 @@ import { randomUUID } from "node:crypto";
25
25
  import * as fsp from "node:fs/promises";
26
26
  import * as net from "node:net";
27
27
  import { withRunLockSync } from "../../state/coordination/locks.ts";
28
- import { appendMailboxMessageAsync, type MailboxMessage, registerMailboxAppendObserver } from "../../state/coordination/mailbox.ts";
28
+ import {
29
+ appendMailboxMessageAsync,
30
+ appendMailboxMessageIdempotentAsync,
31
+ type MailboxMessage,
32
+ registerMailboxAppendObserver,
33
+ } from "../../state/coordination/mailbox.ts";
29
34
  import { appendEventAsync } from "../../state/event-log/event-log.ts";
30
35
  import { loadRunManifestById, saveRunManifest, saveRunTasks } from "../../state/stores/state-store.ts";
31
36
  import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
@@ -936,6 +941,13 @@ export class CrewBroker {
936
941
  }
937
942
  const messageId = `msg_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
938
943
  const fromField = isWorker ? conn.taskId! : (conn.taskId ?? conn.runId);
944
+ // U5 (2026-10-10): every send carries a unique requestId — client-supplied
945
+ // (idempotent retries reuse the SAME one) or broker-generated
946
+ // `{runId}:{sender}:{uuid}`. The durable (direction, requestId) index in
947
+ // delivery.json + the idempotent append below make resend-after-resume a
948
+ // no-op instead of a duplicate delivery.
949
+ const requestId = parsed.requestId ?? `${manifest.runId}:${fromField}:${randomUUID()}`;
950
+ let dedupedRecipients = 0;
939
951
  let durable = false;
940
952
  try {
941
953
  // PERF (2026-08-24): to:"all" with 50 tasks used to run 50 sequential
@@ -947,7 +959,7 @@ export class CrewBroker {
947
959
  for (let i = 0; i < targets.length; i += CHUNK) {
948
960
  const results = await Promise.allSettled(
949
961
  targets.slice(i, i + CHUNK).map((target) =>
950
- appendMailboxMessageAsync(manifest, {
962
+ appendMailboxMessageIdempotentAsync(manifest, {
951
963
  id: `${messageId}_${target.label}`,
952
964
  direction: "inbox",
953
965
  from: fromField,
@@ -958,9 +970,13 @@ export class CrewBroker {
958
970
  priority: parsed.priority ?? "normal",
959
971
  deliveryMode: "next_turn",
960
972
  replyTo: parsed.replyTo,
973
+ requestId,
961
974
  }),
962
975
  ),
963
976
  );
977
+ for (const r of results) {
978
+ if (r.status === "fulfilled" && r.value.deduped) dedupedRecipients++;
979
+ }
964
980
  const failure = results.find((r) => r.status === "rejected") as PromiseRejectedResult | undefined;
965
981
  if (failure) throw failure.reason;
966
982
  }
@@ -969,6 +985,32 @@ export class CrewBroker {
969
985
  this.sendError(conn, id, "durable-failed", (err as Error).message);
970
986
  return;
971
987
  }
988
+ // U5: one bounded audit event per send — requestId + dedup count make the
989
+ // exactly-once chain auditable end-to-end (mailbox.send → worker
990
+ // mailbox.pickup → mailbox.pickup_deduped). Never the body. Failure is
991
+ // non-fatal: the mailbox rows are the source of truth.
992
+ try {
993
+ await appendEventAsync(manifest.eventsPath, {
994
+ type: "mailbox.send",
995
+ runId: manifest.runId,
996
+ taskId: fromField,
997
+ message: `Mailbox send ${dedupedRecipients > 0 ? "(resend, deduped) " : ""}to ${targets.length} recipient(s).`,
998
+ data: {
999
+ requestId,
1000
+ messageId,
1001
+ recipients: targets.length,
1002
+ deduped: dedupedRecipients,
1003
+ kind: parsed.kind ?? "message",
1004
+ to: isWorker ? (typeof parsed.to === "string" ? parsed.to : "group") : "orchestrator",
1005
+ },
1006
+ });
1007
+ } catch (err) {
1008
+ logInternalError(
1009
+ "crew-broker.msg.mailbox-send-event",
1010
+ err instanceof Error ? err : new Error(String(err)),
1011
+ `runId=${conn.runId}`,
1012
+ );
1013
+ }
972
1014
  // Task 5b (spec §15.2 wake): a worker message addressed to the parent
973
1015
  // appends a bounded `worker.message` run event so the host-side event
974
1016
  // bus (sidebar/widget refresh) and any live orchestrator connection
@@ -986,6 +1028,8 @@ export class CrewBroker {
986
1028
  to: "parent",
987
1029
  kind: parsed.kind ?? "message",
988
1030
  ...(parsed.subject !== undefined ? { subject: parsed.subject } : {}),
1031
+ // U5: correlate the wake event with the durable send.
1032
+ requestId,
989
1033
  },
990
1034
  });
991
1035
  } catch (err) {
@@ -998,7 +1042,9 @@ export class CrewBroker {
998
1042
  }
999
1043
  this.sendResult(conn, id, {
1000
1044
  messageId,
1045
+ requestId,
1001
1046
  recipientCount: targets.length,
1047
+ dedupedRecipients,
1002
1048
  durableStatus: durable ? "ok" : "failed",
1003
1049
  liveDeliveryStatus: "ok",
1004
1050
  });
@@ -16,7 +16,7 @@
16
16
  * Precedent: protocol/request-parsers.ts (M4/WI-4.1 extraction discipline).
17
17
  */
18
18
 
19
- import { withRunLockSync } from "../../../state/coordination/locks.ts";
19
+ import { withRunLock, withRunLockSync } from "../../../state/coordination/locks.ts";
20
20
  import type { TeamRunManifest } from "../../../state/types.ts";
21
21
 
22
22
  /** Default bounded retry schedule for run.lock contention (ms). Five
@@ -24,6 +24,19 @@ import type { TeamRunManifest } from "../../../state/types.ts";
24
24
  * detached runner's state flush — without parking the request. */
25
25
  export const DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS: readonly number[] = [50, 100, 200, 400, 800];
26
26
 
27
+ /** Merge-site busy-retry schedule (ms) — RELIABILITY FIX 2026-10-10
28
+ * (run team_20261010100956 incident: background-runner killed by an
29
+ * unhandled rejection when mergeUnitResult's withRunLock lost a run.lock
30
+ * contention race against a concurrent writer). Longer than the broker
31
+ * default above: a merge under contention sits on the scheduler's critical
32
+ * path — the run is mid-flight and the settled unit is held in memory — so
33
+ * it is worth ~7.9s of back-off (6 attempts) before the busy error is
34
+ * allowed to propagate to the run-level error handler. Each attempt itself
35
+ * already waits up to staleMs*2 inside acquireLockWithRetryAsync for
36
+ * in-process holders; this schedule covers the LIVE FOREIGN holder case,
37
+ * which throws immediately. */
38
+ export const MERGE_LOCK_BUSY_RETRY_DELAYS_MS: readonly number[] = [100, 250, 500, 1000, 2000, 4000];
39
+
27
40
  /** Match the exact identity locks.ts throws for a LIVE holder on the
28
41
  * never-steal path (sync acquireLockWithRetry, its async twin, and the
29
42
  * lockCtx fallthrough): `Run '<basename>' is locked by another operation.`
@@ -74,3 +87,36 @@ export async function withRunLockBusyRetry<T>(
74
87
  function sleep(ms: number): Promise<void> {
75
88
  return new Promise((resolve) => setTimeout(resolve, ms));
76
89
  }
90
+
91
+ /** Async twin of withRunLockBusyRetry for the scheduler's merge sites
92
+ * (merge-loop.ts mergeUnitResult, budget-enforcement.ts
93
+ * terminaliseRunWithDrain): run `fn` under the ASYNC withRunLock with a
94
+ * bounded busy-retry, notifying `onRetry` before each back-off sleep so the
95
+ * call site can log a lock_retry event.
96
+ *
97
+ * Retry safety mirrors the sync twin: the busy throw is raised by the lock
98
+ * ACQUIRE before `fn` runs, and every nested state write inside the merge
99
+ * bodies is lockCtx-reentrant (same-context nesting never re-acquires), so a
100
+ * retry never re-executes a partially-run body. Any NON-busy error
101
+ * propagates unchanged — real faults (ENOSPC, EACCES, corruption) keep their
102
+ * current fatal path; only the transient contention identity is retried.
103
+ * When the schedule is exhausted the busy error is rethrown for the run-level
104
+ * handler to classify (bounded — never an infinite loop). */
105
+ export async function withRunLockBusyRetryAsync<T>(
106
+ manifest: TeamRunManifest,
107
+ delaysMs: readonly number[],
108
+ fn: () => Promise<T>,
109
+ onRetry?: (attempt: number, delayMs: number, error: Error) => void,
110
+ ): Promise<T> {
111
+ for (let attempt = 0; ; attempt++) {
112
+ try {
113
+ return await withRunLock(manifest, fn);
114
+ } catch (error) {
115
+ if (!isRunLockBusyError(error)) throw error;
116
+ const delay = delaysMs[attempt];
117
+ if (delay === undefined) throw error;
118
+ onRetry?.(attempt + 1, delay, error as Error);
119
+ await sleep(delay);
120
+ }
121
+ }
122
+ }
@@ -61,6 +61,11 @@ export interface MsgSendParams {
61
61
  /** Task 5b (§15.2): short subject echoed into the worker.message wake
62
62
  * event (bounded like the tool-side MSG_SUBJECT_MAX_CHARS). */
63
63
  subject?: string;
64
+ /** U5 (2026-10-10): client-supplied idempotency key — a msg.send retried
65
+ * with the SAME requestId is deduped per target mailbox (no duplicate
66
+ * rows; the broker returns the existing message). Charset-bounded like
67
+ * cross-extension-rpc's requestId to keep it a safe log/mailbox token. */
68
+ requestId?: string;
64
69
  }
65
70
 
66
71
  export function parseMsgSendParams(value: unknown): MsgSendParams | undefined {
@@ -81,7 +86,9 @@ export function parseMsgSendParams(value: unknown): MsgSendParams | undefined {
81
86
  }
82
87
  const replyTo = typeof v.replyTo === "string" ? v.replyTo : undefined;
83
88
  const subject = typeof v.subject === "string" && v.subject.length > 0 && v.subject.length <= 256 ? v.subject : undefined;
84
- return { to: to as string | string[] | "all", body: v.body, kind, priority, replyTo, subject };
89
+ const rawRequestId = v.requestId;
90
+ const requestId = typeof rawRequestId === "string" && /^[\w:.-]{1,256}$/.test(rawRequestId) ? rawRequestId : undefined;
91
+ return { to: to as string | string[] | "all", body: v.body, kind, priority, replyTo, subject, requestId };
85
92
  }
86
93
 
87
94
  export interface MsgInboxParams {
@@ -16,12 +16,12 @@
16
16
  * team-runner.ts would create a cycle).
17
17
  */
18
18
  import { flushPendingAtomicWrites } from "../state/atomic-write.ts";
19
- import { withRunLock } from "../state/coordination/locks.ts";
20
19
  import { appendEventAsync } from "../state/event-log/event-log.ts";
21
20
  import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
22
21
  import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
23
22
  import { aggregateUsage, formatTokens } from "../state/usage.ts";
24
23
  import { logInternalError } from "../utils/internal-error.ts";
24
+ import { MERGE_LOCK_BUSY_RETRY_DELAYS_MS, withRunLockBusyRetryAsync } from "./broker/protocol/lock-busy.ts";
25
25
  import { saveCrewAgents } from "./crew-agent-records.ts";
26
26
  import { cancelNonTerminalTasks, markBlocked } from "./dispatch-batch.ts";
27
27
  import { isNonTerminalTaskStatus, mergeTaskUpdatesPreservingTerminal } from "./merge-gate.ts";
@@ -136,25 +136,41 @@ export async function terminaliseRunWithDrain(
136
136
  if (validResults.length > 0) {
137
137
  // Merge under the run lock — same pattern as mergeUnitResult:
138
138
  // flush pending writes, load disk state, merge artifacts + tasks,
139
- // save atomically.
140
- const mergeResult = await withRunLock(ctx.manifest, async () => {
141
- flushPendingAtomicWrites();
142
- const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
143
- const diskManifest = disk?.manifest ?? ctx.manifest;
144
- const reconciledArtifacts = mergeArtifacts([
145
- ...diskManifest.artifacts,
146
- ...validResults.flatMap((item) => item.manifest.artifacts),
147
- ]);
148
- const resultManifest = updateRunStatus(
149
- { ...diskManifest, artifacts: reconciledArtifacts },
150
- "running",
151
- "Merged in-flight results during failed-task abort.",
152
- );
153
- const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
154
- await saveRunManifestAsync(resultManifest);
155
- await saveRunTasksAsync(resultManifest, resultTasks);
156
- return { resultManifest, resultTasks };
157
- });
139
+ // save atomically. RELIABILITY FIX 2026-10-10: bare withRunLock here had
140
+ // the same contention-fatal flaw as mergeUnitResult (see merge-loop.ts) —
141
+ // a busy run.lock on this abort path would surface as an unhandled
142
+ // rejection downstream. Bounded busy-retry + run.lock_retry event now.
143
+ const mergeResult = await withRunLockBusyRetryAsync(
144
+ ctx.manifest,
145
+ MERGE_LOCK_BUSY_RETRY_DELAYS_MS,
146
+ async () => {
147
+ flushPendingAtomicWrites();
148
+ const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
149
+ const diskManifest = disk?.manifest ?? ctx.manifest;
150
+ const reconciledArtifacts = mergeArtifacts([
151
+ ...diskManifest.artifacts,
152
+ ...validResults.flatMap((item) => item.manifest.artifacts),
153
+ ]);
154
+ const resultManifest = updateRunStatus(
155
+ { ...diskManifest, artifacts: reconciledArtifacts },
156
+ "running",
157
+ "Merged in-flight results during failed-task abort.",
158
+ );
159
+ const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
160
+ await saveRunManifestAsync(resultManifest);
161
+ await saveRunTasksAsync(resultManifest, resultTasks);
162
+ return { resultManifest, resultTasks };
163
+ },
164
+ (attempt, delayMs, error) => {
165
+ // Best-effort observability only — a failed event append must not break the retry.
166
+ appendEventAsync(ctx.manifest.eventsPath, {
167
+ type: "run.lock_retry",
168
+ runId: ctx.manifest.runId,
169
+ message: `run.lock contention during abort drain merge (attempt ${attempt}); retrying in ${delayMs}ms: ${error.message}`,
170
+ data: { attempt, delayMs, site: "terminaliseRunWithDrain" },
171
+ }).catch(() => undefined);
172
+ },
173
+ );
158
174
  ctx.manifest = mergeResult.resultManifest;
159
175
  ctx.tasks = mergeResult.resultTasks;
160
176
  }
@@ -9,6 +9,8 @@
9
9
  */
10
10
 
11
11
  import { DEFAULT_CHILD_PI } from "../../config/defaults.ts";
12
+ import { getCrewEnv } from "../../config/env-vars.ts";
13
+ import type { ChildPiRunInput } from "./child-pi.ts";
12
14
 
13
15
  /** Post-exit window during which stdio is guarded against late writes. */
14
16
  export const POST_EXIT_STDIO_GUARD_MS = DEFAULT_CHILD_PI.postExitStdioGuardMs;
@@ -48,3 +50,25 @@ export const MAX_TOOL_INPUT_CHARS = DEFAULT_CHILD_PI.maxToolInputChars;
48
50
 
49
51
  /** Maximum characters for general compactable content (used by TruncationStage). */
50
52
  export const MAX_COMPACT_CONTENT_CHARS = DEFAULT_CHILD_PI.maxCompactContentChars;
53
+
54
+ /**
55
+ * Effective no-response timeout for a child run: env
56
+ * PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS (bounded to [1s, 1h] — FIX Round 14: a
57
+ * hostile/accidental value like 1 or 999_999_999 must neither disable the
58
+ * timeout nor cause instant kills) beats ChildPiRunInput.responseTimeoutMs
59
+ * beats the DEFAULT_CHILD_PI default.
60
+ *
61
+ * U14: extracted from the json branch's inline copy so the rpc transport
62
+ * (child-pi-rpc.ts) resolves the EXACT same budget — the watchdog timing is
63
+ * part of the bảng KHÔNG ĐỔI contract and must not drift between transports.
64
+ */
65
+ export function resolveResponseTimeoutMs(input: Pick<ChildPiRunInput, "responseTimeoutMs">): number {
66
+ const RESPONSE_TIMEOUT_MIN_MS = 1_000;
67
+ const RESPONSE_TIMEOUT_MAX_MS = 3_600_000;
68
+ const responseTimeoutEnv = Number.parseInt(getCrewEnv("PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS") ?? "", 10);
69
+ const envInRange =
70
+ Number.isFinite(responseTimeoutEnv) &&
71
+ responseTimeoutEnv >= RESPONSE_TIMEOUT_MIN_MS &&
72
+ responseTimeoutEnv <= RESPONSE_TIMEOUT_MAX_MS;
73
+ return envInRange ? responseTimeoutEnv : (input.responseTimeoutMs ?? RESPONSE_TIMEOUT_MS);
74
+ }
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * child-pi-kill.ts — Kill/escape path for child Pi worker processes.
3
3
  *
4
- * Extracted from child-pi.ts (H-7 decomposition, step 2). Zero behavior change.
4
+ * Extracted from child-pi.ts (H-7 decomposition, step 2).
5
5
  *
6
6
  * Responsibilities:
7
7
  * - appendBoundedTail(): bounded concatenation with truncation marker.
@@ -13,6 +13,15 @@
13
13
  *
14
14
  * The activeChildrenMap and hardKillTimers Map are kept here (not re-exported)
15
15
  * because they are tightly coupled to the kill lifecycle.
16
+ *
17
+ * U15 (upgrade-spec 2026-10-09): every child registered via registerActiveChild
18
+ * is ALSO adopted into the OwnedProcess registry (owned-process.ts) — kills then
19
+ * route through the escalating group dispose (SIGTERM → poll → SIGKILL → cap →
20
+ * warn give-up) instead of a one-shot fire-and-forget signal pair, and the
21
+ * root-exit drain-reconcile reaps backgrounded descendants. The background
22
+ * zombie scanner and the child-side PI_CREW_PARENT_PID guard are KEPT as the
23
+ * complementary layers (owner-side catchable exits vs child-side leader-death
24
+ * backstop — see spec U15 "Complementary" note).
16
25
  */
17
26
 
18
27
  import type { ChildProcess } from "node:child_process";
@@ -21,6 +30,7 @@ import { DEFAULT_CHILD_PI } from "../../config/defaults.ts";
21
30
  import { logInternalError } from "../../utils/internal-error.ts";
22
31
  import { TailCaptureStage } from "../compaction/compact-stages/tail-capture-stage.ts";
23
32
  import { HARD_KILL_MS } from "./child-pi-constants.ts";
33
+ import { adoptOwnedProcess, disposeAllOwnedProcesses, type OwnedProcess } from "./owned-process.ts";
24
34
 
25
35
  const MAX_CAPTURE_BYTES = DEFAULT_CHILD_PI.maxCaptureBytes;
26
36
 
@@ -28,6 +38,12 @@ const MAX_CAPTURE_BYTES = DEFAULT_CHILD_PI.maxCaptureBytes;
28
38
  const activeChildProcesses = new Map<number, ChildProcess>();
29
39
  const childHardKillTimers = new Map<number, NodeJS.Timeout>();
30
40
 
41
+ // U15: adopted OwnedProcess handles by pid. Filled by registerActiveChild;
42
+ // entries survive root exit on purpose (the owned process owns the drain-
43
+ // reconcile of backgrounded descendants) and are pruned by the scanner below
44
+ // once the owner reaches its terminal state.
45
+ const ownedByPid = new Map<number, OwnedProcess>();
46
+
31
47
  // Periodic cleanup of dead child process entries to prevent memory leaks.
32
48
  // If a child process never emits exit/close (zombie), the entry would leak.
33
49
  setInterval(() => {
@@ -38,14 +54,30 @@ setInterval(() => {
38
54
  activeChildProcesses.delete(pid);
39
55
  }
40
56
  }
57
+ // U15: prune adopted owners that reached terminal state (group confirmed
58
+ // gone via dispose or drain-reconcile). Non-terminal owners stay routable.
59
+ for (const [pid, owner] of ownedByPid) {
60
+ if (owner.terminated) ownedByPid.delete(pid);
61
+ }
41
62
  }, 60_000).unref();
42
63
 
43
64
  /** Register a newly-spawned child so it can be tracked + killed on shutdown. */
44
65
  export function registerActiveChild(pid: number, child: ChildProcess): void {
45
66
  activeChildProcesses.set(pid, child);
67
+ // U15: adopt into the owned-process registry so kill paths escalate through
68
+ // group dispose. Group leadership is resolved inside adoptOwnedProcess
69
+ // (authoritative /proc check on Linux — the rpc branch child, spawned
70
+ // without detached, lands on the single-process fallback automatically).
71
+ ownedByPid.set(pid, adoptOwnedProcess(child, { name: `child-pi:${pid}` }));
46
72
  }
47
73
 
48
- /** Remove a child from the active set once it has exited (or before kill). */
74
+ /**
75
+ * Remove a child from the active set once it has exited (or before kill).
76
+ *
77
+ * NOTE (U15): this does NOT drop the adopted OwnedProcess handle — the owner
78
+ * still owes a drain-reconcile for backgrounded descendants after the root
79
+ * exit. The owned entry is pruned by the scanner once terminal.
80
+ */
49
81
  export function unregisterActiveChild(pid: number): void {
50
82
  activeChildProcesses.delete(pid);
51
83
  }
@@ -94,6 +126,16 @@ function spawnTaskkillSafe(pid: number): void {
94
126
 
95
127
  export function killProcessPid(pid: number): void {
96
128
  if (!Number.isInteger(pid) || pid <= 0) return;
129
+ // U15: adopted children escalate through the owned dispose ladder
130
+ // (SIGTERM group → poll → SIGKILL group → cap → warn give-up) instead of
131
+ // the one-shot signal pair below. External pid-only callers that never
132
+ // registered (live-session agents, cancel paths on stale pids) keep the
133
+ // legacy one-shot behavior unchanged.
134
+ const owned = ownedByPid.get(pid);
135
+ if (owned) {
136
+ void owned.dispose();
137
+ return;
138
+ }
97
139
  try {
98
140
  if (process.platform === "win32") {
99
141
  // 3.8: Windows path uses taskkill /T /F (force kill the entire tree).
@@ -168,6 +210,16 @@ export function killProcessTree(pid: number | undefined, child?: ChildProcess):
168
210
  /* diagnostic best-effort */
169
211
  }
170
212
  if (!pid || !Number.isInteger(pid) || pid <= 0) return;
213
+ // U15: route adopted children through the escalating owned dispose. This
214
+ // deliberately BYPASSES the `child.exitCode !== null` early-out below: a
215
+ // dead root can still have live backgrounded descendants in its group, and
216
+ // dispose() is the component that knows (groupAlive is its source of truth;
217
+ // a terminal owner settles as a no-op via the PID-recycle guard).
218
+ const owned = ownedByPid.get(pid);
219
+ if (owned) {
220
+ void owned.dispose();
221
+ return;
222
+ }
171
223
  if (child && child.exitCode !== null) return;
172
224
  killProcessPid(pid);
173
225
  child?.once("exit", () => clearHardKillTimer(pid));
@@ -176,5 +228,10 @@ export function killProcessTree(pid: number | undefined, child?: ChildProcess):
176
228
  export function terminateActiveChildPiProcesses(): number {
177
229
  const entries = [...activeChildProcesses.entries()];
178
230
  for (const [pid, child] of entries) killProcessTree(pid, child);
231
+ // U15: also reap owners whose active-set entry is already gone (root exited,
232
+ // backgrounded descendants mid-drain). Fire-and-forget by design — the sync
233
+ // SIGTERM prefix is delivered immediately; the ladder completes while the
234
+ // shutdown path drains, and the postmortem registry covers a hard exit.
235
+ void disposeAllOwnedProcesses();
179
236
  return entries.length;
180
237
  }