pi-crew 0.11.11 → 0.11.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +102 -0
- package/dist/index.mjs +551 -519
- package/package.json +1 -1
- package/src/config/config-validation.ts +9 -0
- package/src/config/defaults.ts +7 -0
- package/src/config/env-vars.ts +24 -0
- package/src/config/types.ts +11 -0
- package/src/extension/registration/tool-registration.ts +7 -0
- package/src/extension/registration/verify-gate-tool.ts +263 -0
- package/src/extension/team-tool/run-deadline.ts +22 -1
- package/src/extension/team-tool/run.ts +38 -2
- package/src/extension/team-tool-types.ts +4 -0
- package/src/extension/team-tool.ts +31 -3
- package/src/prompt/inbox-poll.ts +141 -3
- package/src/runtime/broker/crew-broker.ts +48 -2
- package/src/runtime/broker/protocol/lock-busy.ts +47 -1
- package/src/runtime/broker/protocol/request-parsers.ts +8 -1
- package/src/runtime/budget-enforcement.ts +36 -20
- package/src/runtime/child-pi/child-pi-constants.ts +24 -0
- package/src/runtime/child-pi/child-pi-kill.ts +59 -2
- package/src/runtime/child-pi/child-pi-rpc.ts +1021 -0
- package/src/runtime/child-pi/child-pi-steering.ts +10 -8
- package/src/runtime/child-pi/child-pi-streams.ts +69 -19
- package/src/runtime/child-pi/child-pi.ts +29 -30
- package/src/runtime/child-pi/owned-process.ts +434 -0
- package/src/runtime/child-pi/postmortem-registry.ts +134 -0
- package/src/runtime/classifier/ambient-noise-gate.ts +126 -0
- package/src/runtime/classifier/classifier-service.ts +277 -61
- package/src/runtime/dispatch-batch.ts +32 -0
- package/src/runtime/live-session/live-agent-control.ts +2 -0
- package/src/runtime/live-session/live-agent-manager.ts +72 -2
- package/src/runtime/live-session/live-session-runtime.ts +435 -40
- package/src/runtime/merge-loop.ts +63 -35
- package/src/runtime/recovery/crash-recovery.ts +141 -3
- package/src/runtime/recovery/retry-executor.ts +29 -0
- package/src/runtime/run-tracker.ts +28 -0
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/task-runner.ts +103 -0
- package/src/runtime/verification/classifier-pre-gate.ts +262 -0
- package/src/runtime/verification/verification-gates.ts +5 -1
- package/src/runtime/verification/verify-gate.ts +394 -0
- package/src/schema/config-schema.ts +13 -0
- package/src/state/contracts.ts +18 -0
- package/src/state/coordination/mailbox.ts +203 -55
- package/src/state/stores/sqlite-run-state.ts +378 -0
- package/src/state/stores/state-store.ts +134 -6
- package/src/ui/agents-jobs-browser.ts +154 -8
- package/src/ui/events-state-source.ts +358 -0
- package/src/ui/run-snapshot-cache.ts +87 -3
- package/src/ui/widget/widget-model.ts +13 -2
package/src/prompt/inbox-poll.ts
CHANGED
|
@@ -22,10 +22,25 @@
|
|
|
22
22
|
* authenticated taskId, so a group broadcast returns to its sender and
|
|
23
23
|
* must be dropped at consume time);
|
|
24
24
|
* - dedup by message id across polls via a caller-owned seen-set and/or a
|
|
25
|
-
* `sinceTs` watermark — one message delivers once
|
|
25
|
+
* `sinceTs` watermark — one message delivers once;
|
|
26
|
+
* - U5 (2026-10-10): dedup is ALSO durable across restarts — every picked
|
|
27
|
+
* row is recorded in delivery.json (`messages[id]="acknowledged"` +
|
|
28
|
+
* `${direction}:${requestId}` index) and re-read on the next poll, so a
|
|
29
|
+
* worker that restarts mid-run (fresh process, EMPTY seen-set) never
|
|
30
|
+
* re-delivers. A resend carrying the same requestId is dropped by the
|
|
31
|
+
* same index; duplicate rows under one requestId deliver exactly once.
|
|
26
32
|
*/
|
|
27
|
-
import
|
|
33
|
+
import * as path from "node:path";
|
|
34
|
+
import {
|
|
35
|
+
type MailboxDeliveryState,
|
|
36
|
+
type MailboxMessage,
|
|
37
|
+
readAllMailboxMessages,
|
|
38
|
+
readDeliveryState,
|
|
39
|
+
recordMailboxRequestIdDelivery,
|
|
40
|
+
} from "../state/coordination/mailbox.ts";
|
|
41
|
+
import { appendEventFireAndForget } from "../state/event-log/event-log.ts";
|
|
28
42
|
import type { TeamRunManifest } from "../state/types.ts";
|
|
43
|
+
import { logInternalError } from "../utils/internal-error.ts";
|
|
29
44
|
|
|
30
45
|
export interface WorkerInboxPickup {
|
|
31
46
|
stateRoot: string;
|
|
@@ -37,23 +52,99 @@ export interface WorkerInboxPickup {
|
|
|
37
52
|
seenIds?: Set<string>;
|
|
38
53
|
}
|
|
39
54
|
|
|
55
|
+
/** U5: requestId of a row — the U5 top-level field with the legacy
|
|
56
|
+
* `data.requestId` shape (group_join) as fallback. */
|
|
57
|
+
function messageIdempotencyKey(message: MailboxMessage): string | undefined {
|
|
58
|
+
if (message.requestId) return message.requestId;
|
|
59
|
+
const fromData = message.data?.requestId;
|
|
60
|
+
return typeof fromData === "string" ? fromData : undefined;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// U5: in-process throttle for `mailbox.pickup_deduped` audit events — a
|
|
64
|
+
// restarted worker re-scans the same durable rows on EVERY 500ms tick, so an
|
|
65
|
+
// un-throttled per-skip event would spam the log (2/s/row). One event per
|
|
66
|
+
// (run, task, key) per process lifetime is the audit signal ("redelivery was
|
|
67
|
+
// prevented by the durable record"); bounded FIFO like the parse caches.
|
|
68
|
+
const DEDUP_EVENT_KEYS_MAX = 512;
|
|
69
|
+
const dedupEventKeys = new Map<string, true>();
|
|
70
|
+
|
|
71
|
+
function shouldEmitDedupEvent(runId: string, taskId: string, key: string): boolean {
|
|
72
|
+
const mapKey = `${runId}:${taskId}:${key}`;
|
|
73
|
+
if (dedupEventKeys.has(mapKey)) return false;
|
|
74
|
+
dedupEventKeys.set(mapKey, true);
|
|
75
|
+
while (dedupEventKeys.size > DEDUP_EVENT_KEYS_MAX) {
|
|
76
|
+
const oldest = dedupEventKeys.keys().next().value;
|
|
77
|
+
if (oldest === undefined) break;
|
|
78
|
+
dedupEventKeys.delete(oldest);
|
|
79
|
+
}
|
|
80
|
+
return true;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** U5: audit event for a redelivery that the DURABLE record prevented (the
|
|
84
|
+
* in-memory seen-set alone could not — this is exactly the restart window). */
|
|
85
|
+
function emitPickupDedupedEvent(stateRoot: string, runId: string, taskId: string, message: MailboxMessage, layer: string): void {
|
|
86
|
+
if (!shouldEmitDedupEvent(runId, taskId, `${layer}:${message.id}`)) return;
|
|
87
|
+
try {
|
|
88
|
+
appendEventFireAndForget(path.join(stateRoot, "events.jsonl"), {
|
|
89
|
+
type: "mailbox.pickup_deduped",
|
|
90
|
+
runId,
|
|
91
|
+
taskId,
|
|
92
|
+
message: `Dropped duplicate inbox delivery (${layer}) for task ${taskId}.`,
|
|
93
|
+
data: {
|
|
94
|
+
layer,
|
|
95
|
+
messageId: message.id,
|
|
96
|
+
requestId: messageIdempotencyKey(message),
|
|
97
|
+
from: message.from,
|
|
98
|
+
kind: message.kind,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
} catch (error) {
|
|
102
|
+
logInternalError("inbox-poll.pickup-deduped-event", error, `runId=${runId} taskId=${taskId}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/** U5: audit event for a successful pickup — with the requestId, the event
|
|
107
|
+
* log is sufficient to audit the dedup chain end-to-end (broker
|
|
108
|
+
* `mailbox.send` → worker `mailbox.pickup` → `mailbox.pickup_deduped`). */
|
|
109
|
+
function emitPickupEvent(stateRoot: string, runId: string, taskId: string, picked: MailboxMessage[]): void {
|
|
110
|
+
try {
|
|
111
|
+
appendEventFireAndForget(path.join(stateRoot, "events.jsonl"), {
|
|
112
|
+
type: "mailbox.pickup",
|
|
113
|
+
runId,
|
|
114
|
+
taskId,
|
|
115
|
+
message: `Picked up ${picked.length} inbox message(s) for task ${taskId}.`,
|
|
116
|
+
data: {
|
|
117
|
+
messageIds: picked.map((m) => m.id),
|
|
118
|
+
requestIds: picked.map((m) => messageIdempotencyKey(m) ?? null),
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
} catch (error) {
|
|
122
|
+
logInternalError("inbox-poll.pickup-event", error, `runId=${runId} taskId=${taskId}`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
40
126
|
/**
|
|
41
127
|
* Read the worker's inbox mailbox and return the messages that should be
|
|
42
128
|
* surfaced as fenced context on the next turn.
|
|
43
129
|
*
|
|
44
130
|
* Stateless apart from the optional caller-owned `seenIds`/`sinceTs` — safe
|
|
45
|
-
* to call on every 500ms poll tick.
|
|
131
|
+
* to call on every 500ms poll tick. U5: picked rows are ALSO recorded in the
|
|
132
|
+
* durable delivery state (delivery.json), so dedup survives a restart.
|
|
46
133
|
*/
|
|
47
134
|
export function pollWorkerInbox(pickup: WorkerInboxPickup): MailboxMessage[] {
|
|
48
135
|
const { stateRoot, runId, taskId } = pickup;
|
|
49
136
|
if (!stateRoot || !runId || !taskId) return [];
|
|
50
137
|
const manifest = { stateRoot, runId } as unknown as TeamRunManifest;
|
|
51
138
|
let messages: MailboxMessage[];
|
|
139
|
+
let delivery: MailboxDeliveryState;
|
|
52
140
|
try {
|
|
53
141
|
// readAllMailboxMessages already merges run-level inbox + every task
|
|
54
142
|
// mailbox; we route TO this worker below (never trust a caller-scoped
|
|
55
143
|
// file path).
|
|
56
144
|
messages = readAllMailboxMessages(manifest, "inbox");
|
|
145
|
+
// U5: the durable delivery record (mtime-cached read — cheap on the
|
|
146
|
+
// 500ms tick) is the restart-safe dedup source of truth.
|
|
147
|
+
delivery = readDeliveryState(manifest);
|
|
57
148
|
} catch {
|
|
58
149
|
// Transient read error (lock contention, rotated file) — never throw
|
|
59
150
|
// out of a poll tick; the next 500ms tick retries.
|
|
@@ -61,8 +152,10 @@ export function pollWorkerInbox(pickup: WorkerInboxPickup): MailboxMessage[] {
|
|
|
61
152
|
}
|
|
62
153
|
|
|
63
154
|
const seen = pickup.seenIds;
|
|
155
|
+
const deliveredRequestIds = delivery.requestIds ?? {};
|
|
64
156
|
const picked: MailboxMessage[] = [];
|
|
65
157
|
const byId = new Set<string>();
|
|
158
|
+
const byRequestId = new Set<string>();
|
|
66
159
|
for (const m of messages) {
|
|
67
160
|
// Kind gate: only durable `message`s (per §15.2). Notify is
|
|
68
161
|
// fire-and-forget (its own channel), steer/response/follow-up are
|
|
@@ -81,10 +174,55 @@ export function pollWorkerInbox(pickup: WorkerInboxPickup): MailboxMessage[] {
|
|
|
81
174
|
if (seen?.has(m.id)) continue;
|
|
82
175
|
// Within-call id dedup (duplicate file rows → one delivery).
|
|
83
176
|
if (byId.has(m.id)) continue;
|
|
177
|
+
// ── U5 durable dedup (2026-10-10) ──────────────────────────────────
|
|
178
|
+
// These two layers survive a worker restart (delivery.json); the
|
|
179
|
+
// in-memory gates above cannot. Every skip here is a redelivery that
|
|
180
|
+
// was PREVENTED — audited via mailbox.pickup_deduped (once per key
|
|
181
|
+
// per process so the 500ms re-scan cannot spam the event log).
|
|
182
|
+
const requestId = messageIdempotencyKey(m);
|
|
183
|
+
const dedupKey = requestId !== undefined ? `${m.direction}:${requestId}` : undefined;
|
|
184
|
+
if (dedupKey !== undefined) {
|
|
185
|
+
// (direction, requestId) index — catches resends carrying the same
|
|
186
|
+
// requestId AND duplicate rows under one requestId.
|
|
187
|
+
if (deliveredRequestIds[dedupKey] !== undefined) {
|
|
188
|
+
emitPickupDedupedEvent(stateRoot, runId, taskId, m, "requestId");
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
// Within-call requestId dedup (two rows, one requestId → first wins).
|
|
192
|
+
if (byRequestId.has(dedupKey)) continue;
|
|
193
|
+
}
|
|
194
|
+
// Acknowledged delivery entry = the F09 durable "handled" record; the
|
|
195
|
+
// inbox line itself keeps `status:"queued"` forever, so WITHOUT this
|
|
196
|
+
// check a restarted worker re-delivers every already-consumed row
|
|
197
|
+
// (legacy rows without a requestId land here too).
|
|
198
|
+
if (delivery.messages[m.id] === "acknowledged") {
|
|
199
|
+
emitPickupDedupedEvent(stateRoot, runId, taskId, m, "acknowledged");
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
84
202
|
byId.add(m.id);
|
|
203
|
+
if (dedupKey !== undefined) byRequestId.add(dedupKey);
|
|
85
204
|
seen?.add(m.id);
|
|
86
205
|
picked.push(m);
|
|
87
206
|
}
|
|
207
|
+
if (picked.length > 0) {
|
|
208
|
+
// U5: persist the pickup — the durable record that powers the two
|
|
209
|
+
// layers above after a restart. Best-effort on failure: the in-memory
|
|
210
|
+
// seen-set still dedups within THIS process, and logInternalError
|
|
211
|
+
// makes the lost-durability window visible.
|
|
212
|
+
try {
|
|
213
|
+
recordMailboxRequestIdDelivery(
|
|
214
|
+
manifest,
|
|
215
|
+
picked.map((m) => ({
|
|
216
|
+
direction: m.direction,
|
|
217
|
+
messageId: m.id,
|
|
218
|
+
requestId: messageIdempotencyKey(m),
|
|
219
|
+
})),
|
|
220
|
+
);
|
|
221
|
+
emitPickupEvent(stateRoot, runId, taskId, picked);
|
|
222
|
+
} catch (error) {
|
|
223
|
+
logInternalError("inbox-poll.record-delivery", error, `runId=${runId} taskId=${taskId}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
88
226
|
// Deterministic delivery order (same sort readAllMailboxMessages applies).
|
|
89
227
|
return picked;
|
|
90
228
|
}
|
|
@@ -25,7 +25,12 @@ import { randomUUID } from "node:crypto";
|
|
|
25
25
|
import * as fsp from "node:fs/promises";
|
|
26
26
|
import * as net from "node:net";
|
|
27
27
|
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
28
|
-
import {
|
|
28
|
+
import {
|
|
29
|
+
appendMailboxMessageAsync,
|
|
30
|
+
appendMailboxMessageIdempotentAsync,
|
|
31
|
+
type MailboxMessage,
|
|
32
|
+
registerMailboxAppendObserver,
|
|
33
|
+
} from "../../state/coordination/mailbox.ts";
|
|
29
34
|
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
30
35
|
import { loadRunManifestById, saveRunManifest, saveRunTasks } from "../../state/stores/state-store.ts";
|
|
31
36
|
import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
@@ -936,6 +941,13 @@ export class CrewBroker {
|
|
|
936
941
|
}
|
|
937
942
|
const messageId = `msg_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
938
943
|
const fromField = isWorker ? conn.taskId! : (conn.taskId ?? conn.runId);
|
|
944
|
+
// U5 (2026-10-10): every send carries a unique requestId — client-supplied
|
|
945
|
+
// (idempotent retries reuse the SAME one) or broker-generated
|
|
946
|
+
// `{runId}:{sender}:{uuid}`. The durable (direction, requestId) index in
|
|
947
|
+
// delivery.json + the idempotent append below make resend-after-resume a
|
|
948
|
+
// no-op instead of a duplicate delivery.
|
|
949
|
+
const requestId = parsed.requestId ?? `${manifest.runId}:${fromField}:${randomUUID()}`;
|
|
950
|
+
let dedupedRecipients = 0;
|
|
939
951
|
let durable = false;
|
|
940
952
|
try {
|
|
941
953
|
// PERF (2026-08-24): to:"all" with 50 tasks used to run 50 sequential
|
|
@@ -947,7 +959,7 @@ export class CrewBroker {
|
|
|
947
959
|
for (let i = 0; i < targets.length; i += CHUNK) {
|
|
948
960
|
const results = await Promise.allSettled(
|
|
949
961
|
targets.slice(i, i + CHUNK).map((target) =>
|
|
950
|
-
|
|
962
|
+
appendMailboxMessageIdempotentAsync(manifest, {
|
|
951
963
|
id: `${messageId}_${target.label}`,
|
|
952
964
|
direction: "inbox",
|
|
953
965
|
from: fromField,
|
|
@@ -958,9 +970,13 @@ export class CrewBroker {
|
|
|
958
970
|
priority: parsed.priority ?? "normal",
|
|
959
971
|
deliveryMode: "next_turn",
|
|
960
972
|
replyTo: parsed.replyTo,
|
|
973
|
+
requestId,
|
|
961
974
|
}),
|
|
962
975
|
),
|
|
963
976
|
);
|
|
977
|
+
for (const r of results) {
|
|
978
|
+
if (r.status === "fulfilled" && r.value.deduped) dedupedRecipients++;
|
|
979
|
+
}
|
|
964
980
|
const failure = results.find((r) => r.status === "rejected") as PromiseRejectedResult | undefined;
|
|
965
981
|
if (failure) throw failure.reason;
|
|
966
982
|
}
|
|
@@ -969,6 +985,32 @@ export class CrewBroker {
|
|
|
969
985
|
this.sendError(conn, id, "durable-failed", (err as Error).message);
|
|
970
986
|
return;
|
|
971
987
|
}
|
|
988
|
+
// U5: one bounded audit event per send — requestId + dedup count make the
|
|
989
|
+
// exactly-once chain auditable end-to-end (mailbox.send → worker
|
|
990
|
+
// mailbox.pickup → mailbox.pickup_deduped). Never the body. Failure is
|
|
991
|
+
// non-fatal: the mailbox rows are the source of truth.
|
|
992
|
+
try {
|
|
993
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
994
|
+
type: "mailbox.send",
|
|
995
|
+
runId: manifest.runId,
|
|
996
|
+
taskId: fromField,
|
|
997
|
+
message: `Mailbox send ${dedupedRecipients > 0 ? "(resend, deduped) " : ""}to ${targets.length} recipient(s).`,
|
|
998
|
+
data: {
|
|
999
|
+
requestId,
|
|
1000
|
+
messageId,
|
|
1001
|
+
recipients: targets.length,
|
|
1002
|
+
deduped: dedupedRecipients,
|
|
1003
|
+
kind: parsed.kind ?? "message",
|
|
1004
|
+
to: isWorker ? (typeof parsed.to === "string" ? parsed.to : "group") : "orchestrator",
|
|
1005
|
+
},
|
|
1006
|
+
});
|
|
1007
|
+
} catch (err) {
|
|
1008
|
+
logInternalError(
|
|
1009
|
+
"crew-broker.msg.mailbox-send-event",
|
|
1010
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
1011
|
+
`runId=${conn.runId}`,
|
|
1012
|
+
);
|
|
1013
|
+
}
|
|
972
1014
|
// Task 5b (spec §15.2 wake): a worker message addressed to the parent
|
|
973
1015
|
// appends a bounded `worker.message` run event so the host-side event
|
|
974
1016
|
// bus (sidebar/widget refresh) and any live orchestrator connection
|
|
@@ -986,6 +1028,8 @@ export class CrewBroker {
|
|
|
986
1028
|
to: "parent",
|
|
987
1029
|
kind: parsed.kind ?? "message",
|
|
988
1030
|
...(parsed.subject !== undefined ? { subject: parsed.subject } : {}),
|
|
1031
|
+
// U5: correlate the wake event with the durable send.
|
|
1032
|
+
requestId,
|
|
989
1033
|
},
|
|
990
1034
|
});
|
|
991
1035
|
} catch (err) {
|
|
@@ -998,7 +1042,9 @@ export class CrewBroker {
|
|
|
998
1042
|
}
|
|
999
1043
|
this.sendResult(conn, id, {
|
|
1000
1044
|
messageId,
|
|
1045
|
+
requestId,
|
|
1001
1046
|
recipientCount: targets.length,
|
|
1047
|
+
dedupedRecipients,
|
|
1002
1048
|
durableStatus: durable ? "ok" : "failed",
|
|
1003
1049
|
liveDeliveryStatus: "ok",
|
|
1004
1050
|
});
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* Precedent: protocol/request-parsers.ts (M4/WI-4.1 extraction discipline).
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import { withRunLockSync } from "../../../state/coordination/locks.ts";
|
|
19
|
+
import { withRunLock, withRunLockSync } from "../../../state/coordination/locks.ts";
|
|
20
20
|
import type { TeamRunManifest } from "../../../state/types.ts";
|
|
21
21
|
|
|
22
22
|
/** Default bounded retry schedule for run.lock contention (ms). Five
|
|
@@ -24,6 +24,19 @@ import type { TeamRunManifest } from "../../../state/types.ts";
|
|
|
24
24
|
* detached runner's state flush — without parking the request. */
|
|
25
25
|
export const DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS: readonly number[] = [50, 100, 200, 400, 800];
|
|
26
26
|
|
|
27
|
+
/** Merge-site busy-retry schedule (ms) — RELIABILITY FIX 2026-10-10
|
|
28
|
+
* (run team_20261010100956 incident: background-runner killed by an
|
|
29
|
+
* unhandled rejection when mergeUnitResult's withRunLock lost a run.lock
|
|
30
|
+
* contention race against a concurrent writer). Longer than the broker
|
|
31
|
+
* default above: a merge under contention sits on the scheduler's critical
|
|
32
|
+
* path — the run is mid-flight and the settled unit is held in memory — so
|
|
33
|
+
* it is worth ~7.9s of back-off (6 attempts) before the busy error is
|
|
34
|
+
* allowed to propagate to the run-level error handler. Each attempt itself
|
|
35
|
+
* already waits up to staleMs*2 inside acquireLockWithRetryAsync for
|
|
36
|
+
* in-process holders; this schedule covers the LIVE FOREIGN holder case,
|
|
37
|
+
* which throws immediately. */
|
|
38
|
+
export const MERGE_LOCK_BUSY_RETRY_DELAYS_MS: readonly number[] = [100, 250, 500, 1000, 2000, 4000];
|
|
39
|
+
|
|
27
40
|
/** Match the exact identity locks.ts throws for a LIVE holder on the
|
|
28
41
|
* never-steal path (sync acquireLockWithRetry, its async twin, and the
|
|
29
42
|
* lockCtx fallthrough): `Run '<basename>' is locked by another operation.`
|
|
@@ -74,3 +87,36 @@ export async function withRunLockBusyRetry<T>(
|
|
|
74
87
|
function sleep(ms: number): Promise<void> {
|
|
75
88
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
76
89
|
}
|
|
90
|
+
|
|
91
|
+
/** Async twin of withRunLockBusyRetry for the scheduler's merge sites
|
|
92
|
+
* (merge-loop.ts mergeUnitResult, budget-enforcement.ts
|
|
93
|
+
* terminaliseRunWithDrain): run `fn` under the ASYNC withRunLock with a
|
|
94
|
+
* bounded busy-retry, notifying `onRetry` before each back-off sleep so the
|
|
95
|
+
* call site can log a lock_retry event.
|
|
96
|
+
*
|
|
97
|
+
* Retry safety mirrors the sync twin: the busy throw is raised by the lock
|
|
98
|
+
* ACQUIRE before `fn` runs, and every nested state write inside the merge
|
|
99
|
+
* bodies is lockCtx-reentrant (same-context nesting never re-acquires), so a
|
|
100
|
+
* retry never re-executes a partially-run body. Any NON-busy error
|
|
101
|
+
* propagates unchanged — real faults (ENOSPC, EACCES, corruption) keep their
|
|
102
|
+
* current fatal path; only the transient contention identity is retried.
|
|
103
|
+
* When the schedule is exhausted the busy error is rethrown for the run-level
|
|
104
|
+
* handler to classify (bounded — never an infinite loop). */
|
|
105
|
+
export async function withRunLockBusyRetryAsync<T>(
|
|
106
|
+
manifest: TeamRunManifest,
|
|
107
|
+
delaysMs: readonly number[],
|
|
108
|
+
fn: () => Promise<T>,
|
|
109
|
+
onRetry?: (attempt: number, delayMs: number, error: Error) => void,
|
|
110
|
+
): Promise<T> {
|
|
111
|
+
for (let attempt = 0; ; attempt++) {
|
|
112
|
+
try {
|
|
113
|
+
return await withRunLock(manifest, fn);
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (!isRunLockBusyError(error)) throw error;
|
|
116
|
+
const delay = delaysMs[attempt];
|
|
117
|
+
if (delay === undefined) throw error;
|
|
118
|
+
onRetry?.(attempt + 1, delay, error as Error);
|
|
119
|
+
await sleep(delay);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
@@ -61,6 +61,11 @@ export interface MsgSendParams {
|
|
|
61
61
|
/** Task 5b (§15.2): short subject echoed into the worker.message wake
|
|
62
62
|
* event (bounded like the tool-side MSG_SUBJECT_MAX_CHARS). */
|
|
63
63
|
subject?: string;
|
|
64
|
+
/** U5 (2026-10-10): client-supplied idempotency key — a msg.send retried
|
|
65
|
+
* with the SAME requestId is deduped per target mailbox (no duplicate
|
|
66
|
+
* rows; the broker returns the existing message). Charset-bounded like
|
|
67
|
+
* cross-extension-rpc's requestId to keep it a safe log/mailbox token. */
|
|
68
|
+
requestId?: string;
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
export function parseMsgSendParams(value: unknown): MsgSendParams | undefined {
|
|
@@ -81,7 +86,9 @@ export function parseMsgSendParams(value: unknown): MsgSendParams | undefined {
|
|
|
81
86
|
}
|
|
82
87
|
const replyTo = typeof v.replyTo === "string" ? v.replyTo : undefined;
|
|
83
88
|
const subject = typeof v.subject === "string" && v.subject.length > 0 && v.subject.length <= 256 ? v.subject : undefined;
|
|
84
|
-
|
|
89
|
+
const rawRequestId = v.requestId;
|
|
90
|
+
const requestId = typeof rawRequestId === "string" && /^[\w:.-]{1,256}$/.test(rawRequestId) ? rawRequestId : undefined;
|
|
91
|
+
return { to: to as string | string[] | "all", body: v.body, kind, priority, replyTo, subject, requestId };
|
|
85
92
|
}
|
|
86
93
|
|
|
87
94
|
export interface MsgInboxParams {
|
|
@@ -16,12 +16,12 @@
|
|
|
16
16
|
* team-runner.ts would create a cycle).
|
|
17
17
|
*/
|
|
18
18
|
import { flushPendingAtomicWrites } from "../state/atomic-write.ts";
|
|
19
|
-
import { withRunLock } from "../state/coordination/locks.ts";
|
|
20
19
|
import { appendEventAsync } from "../state/event-log/event-log.ts";
|
|
21
20
|
import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
|
|
22
21
|
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
23
22
|
import { aggregateUsage, formatTokens } from "../state/usage.ts";
|
|
24
23
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
24
|
+
import { MERGE_LOCK_BUSY_RETRY_DELAYS_MS, withRunLockBusyRetryAsync } from "./broker/protocol/lock-busy.ts";
|
|
25
25
|
import { saveCrewAgents } from "./crew-agent-records.ts";
|
|
26
26
|
import { cancelNonTerminalTasks, markBlocked } from "./dispatch-batch.ts";
|
|
27
27
|
import { isNonTerminalTaskStatus, mergeTaskUpdatesPreservingTerminal } from "./merge-gate.ts";
|
|
@@ -136,25 +136,41 @@ export async function terminaliseRunWithDrain(
|
|
|
136
136
|
if (validResults.length > 0) {
|
|
137
137
|
// Merge under the run lock — same pattern as mergeUnitResult:
|
|
138
138
|
// flush pending writes, load disk state, merge artifacts + tasks,
|
|
139
|
-
// save atomically.
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
139
|
+
// save atomically. RELIABILITY FIX 2026-10-10: bare withRunLock here had
|
|
140
|
+
// the same contention-fatal flaw as mergeUnitResult (see merge-loop.ts) —
|
|
141
|
+
// a busy run.lock on this abort path would surface as an unhandled
|
|
142
|
+
// rejection downstream. Bounded busy-retry + run.lock_retry event now.
|
|
143
|
+
const mergeResult = await withRunLockBusyRetryAsync(
|
|
144
|
+
ctx.manifest,
|
|
145
|
+
MERGE_LOCK_BUSY_RETRY_DELAYS_MS,
|
|
146
|
+
async () => {
|
|
147
|
+
flushPendingAtomicWrites();
|
|
148
|
+
const disk = loadRunManifestById(ctx.manifest.cwd, ctx.manifest.runId);
|
|
149
|
+
const diskManifest = disk?.manifest ?? ctx.manifest;
|
|
150
|
+
const reconciledArtifacts = mergeArtifacts([
|
|
151
|
+
...diskManifest.artifacts,
|
|
152
|
+
...validResults.flatMap((item) => item.manifest.artifacts),
|
|
153
|
+
]);
|
|
154
|
+
const resultManifest = updateRunStatus(
|
|
155
|
+
{ ...diskManifest, artifacts: reconciledArtifacts },
|
|
156
|
+
"running",
|
|
157
|
+
"Merged in-flight results during failed-task abort.",
|
|
158
|
+
);
|
|
159
|
+
const resultTasks = mergeTaskUpdatesPreservingTerminal(disk?.tasks ?? ctx.tasks, validResults);
|
|
160
|
+
await saveRunManifestAsync(resultManifest);
|
|
161
|
+
await saveRunTasksAsync(resultManifest, resultTasks);
|
|
162
|
+
return { resultManifest, resultTasks };
|
|
163
|
+
},
|
|
164
|
+
(attempt, delayMs, error) => {
|
|
165
|
+
// Best-effort observability only — a failed event append must not break the retry.
|
|
166
|
+
appendEventAsync(ctx.manifest.eventsPath, {
|
|
167
|
+
type: "run.lock_retry",
|
|
168
|
+
runId: ctx.manifest.runId,
|
|
169
|
+
message: `run.lock contention during abort drain merge (attempt ${attempt}); retrying in ${delayMs}ms: ${error.message}`,
|
|
170
|
+
data: { attempt, delayMs, site: "terminaliseRunWithDrain" },
|
|
171
|
+
}).catch(() => undefined);
|
|
172
|
+
},
|
|
173
|
+
);
|
|
158
174
|
ctx.manifest = mergeResult.resultManifest;
|
|
159
175
|
ctx.tasks = mergeResult.resultTasks;
|
|
160
176
|
}
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { DEFAULT_CHILD_PI } from "../../config/defaults.ts";
|
|
12
|
+
import { getCrewEnv } from "../../config/env-vars.ts";
|
|
13
|
+
import type { ChildPiRunInput } from "./child-pi.ts";
|
|
12
14
|
|
|
13
15
|
/** Post-exit window during which stdio is guarded against late writes. */
|
|
14
16
|
export const POST_EXIT_STDIO_GUARD_MS = DEFAULT_CHILD_PI.postExitStdioGuardMs;
|
|
@@ -48,3 +50,25 @@ export const MAX_TOOL_INPUT_CHARS = DEFAULT_CHILD_PI.maxToolInputChars;
|
|
|
48
50
|
|
|
49
51
|
/** Maximum characters for general compactable content (used by TruncationStage). */
|
|
50
52
|
export const MAX_COMPACT_CONTENT_CHARS = DEFAULT_CHILD_PI.maxCompactContentChars;
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Effective no-response timeout for a child run: env
|
|
56
|
+
* PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS (bounded to [1s, 1h] — FIX Round 14: a
|
|
57
|
+
* hostile/accidental value like 1 or 999_999_999 must neither disable the
|
|
58
|
+
* timeout nor cause instant kills) beats ChildPiRunInput.responseTimeoutMs
|
|
59
|
+
* beats the DEFAULT_CHILD_PI default.
|
|
60
|
+
*
|
|
61
|
+
* U14: extracted from the json branch's inline copy so the rpc transport
|
|
62
|
+
* (child-pi-rpc.ts) resolves the EXACT same budget — the watchdog timing is
|
|
63
|
+
* part of the bảng KHÔNG ĐỔI contract and must not drift between transports.
|
|
64
|
+
*/
|
|
65
|
+
export function resolveResponseTimeoutMs(input: Pick<ChildPiRunInput, "responseTimeoutMs">): number {
|
|
66
|
+
const RESPONSE_TIMEOUT_MIN_MS = 1_000;
|
|
67
|
+
const RESPONSE_TIMEOUT_MAX_MS = 3_600_000;
|
|
68
|
+
const responseTimeoutEnv = Number.parseInt(getCrewEnv("PI_TEAMS_CHILD_RESPONSE_TIMEOUT_MS") ?? "", 10);
|
|
69
|
+
const envInRange =
|
|
70
|
+
Number.isFinite(responseTimeoutEnv) &&
|
|
71
|
+
responseTimeoutEnv >= RESPONSE_TIMEOUT_MIN_MS &&
|
|
72
|
+
responseTimeoutEnv <= RESPONSE_TIMEOUT_MAX_MS;
|
|
73
|
+
return envInRange ? responseTimeoutEnv : (input.responseTimeoutMs ?? RESPONSE_TIMEOUT_MS);
|
|
74
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* child-pi-kill.ts — Kill/escape path for child Pi worker processes.
|
|
3
3
|
*
|
|
4
|
-
* Extracted from child-pi.ts (H-7 decomposition, step 2).
|
|
4
|
+
* Extracted from child-pi.ts (H-7 decomposition, step 2).
|
|
5
5
|
*
|
|
6
6
|
* Responsibilities:
|
|
7
7
|
* - appendBoundedTail(): bounded concatenation with truncation marker.
|
|
@@ -13,6 +13,15 @@
|
|
|
13
13
|
*
|
|
14
14
|
* The activeChildrenMap and hardKillTimers Map are kept here (not re-exported)
|
|
15
15
|
* because they are tightly coupled to the kill lifecycle.
|
|
16
|
+
*
|
|
17
|
+
* U15 (upgrade-spec 2026-10-09): every child registered via registerActiveChild
|
|
18
|
+
* is ALSO adopted into the OwnedProcess registry (owned-process.ts) — kills then
|
|
19
|
+
* route through the escalating group dispose (SIGTERM → poll → SIGKILL → cap →
|
|
20
|
+
* warn give-up) instead of a one-shot fire-and-forget signal pair, and the
|
|
21
|
+
* root-exit drain-reconcile reaps backgrounded descendants. The background
|
|
22
|
+
* zombie scanner and the child-side PI_CREW_PARENT_PID guard are KEPT as the
|
|
23
|
+
* complementary layers (owner-side catchable exits vs child-side leader-death
|
|
24
|
+
* backstop — see spec U15 "Complementary" note).
|
|
16
25
|
*/
|
|
17
26
|
|
|
18
27
|
import type { ChildProcess } from "node:child_process";
|
|
@@ -21,6 +30,7 @@ import { DEFAULT_CHILD_PI } from "../../config/defaults.ts";
|
|
|
21
30
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
22
31
|
import { TailCaptureStage } from "../compaction/compact-stages/tail-capture-stage.ts";
|
|
23
32
|
import { HARD_KILL_MS } from "./child-pi-constants.ts";
|
|
33
|
+
import { adoptOwnedProcess, disposeAllOwnedProcesses, type OwnedProcess } from "./owned-process.ts";
|
|
24
34
|
|
|
25
35
|
const MAX_CAPTURE_BYTES = DEFAULT_CHILD_PI.maxCaptureBytes;
|
|
26
36
|
|
|
@@ -28,6 +38,12 @@ const MAX_CAPTURE_BYTES = DEFAULT_CHILD_PI.maxCaptureBytes;
|
|
|
28
38
|
const activeChildProcesses = new Map<number, ChildProcess>();
|
|
29
39
|
const childHardKillTimers = new Map<number, NodeJS.Timeout>();
|
|
30
40
|
|
|
41
|
+
// U15: adopted OwnedProcess handles by pid. Filled by registerActiveChild;
|
|
42
|
+
// entries survive root exit on purpose (the owned process owns the drain-
|
|
43
|
+
// reconcile of backgrounded descendants) and are pruned by the scanner below
|
|
44
|
+
// once the owner reaches its terminal state.
|
|
45
|
+
const ownedByPid = new Map<number, OwnedProcess>();
|
|
46
|
+
|
|
31
47
|
// Periodic cleanup of dead child process entries to prevent memory leaks.
|
|
32
48
|
// If a child process never emits exit/close (zombie), the entry would leak.
|
|
33
49
|
setInterval(() => {
|
|
@@ -38,14 +54,30 @@ setInterval(() => {
|
|
|
38
54
|
activeChildProcesses.delete(pid);
|
|
39
55
|
}
|
|
40
56
|
}
|
|
57
|
+
// U15: prune adopted owners that reached terminal state (group confirmed
|
|
58
|
+
// gone via dispose or drain-reconcile). Non-terminal owners stay routable.
|
|
59
|
+
for (const [pid, owner] of ownedByPid) {
|
|
60
|
+
if (owner.terminated) ownedByPid.delete(pid);
|
|
61
|
+
}
|
|
41
62
|
}, 60_000).unref();
|
|
42
63
|
|
|
43
64
|
/** Register a newly-spawned child so it can be tracked + killed on shutdown. */
|
|
44
65
|
export function registerActiveChild(pid: number, child: ChildProcess): void {
|
|
45
66
|
activeChildProcesses.set(pid, child);
|
|
67
|
+
// U15: adopt into the owned-process registry so kill paths escalate through
|
|
68
|
+
// group dispose. Group leadership is resolved inside adoptOwnedProcess
|
|
69
|
+
// (authoritative /proc check on Linux — the rpc branch child, spawned
|
|
70
|
+
// without detached, lands on the single-process fallback automatically).
|
|
71
|
+
ownedByPid.set(pid, adoptOwnedProcess(child, { name: `child-pi:${pid}` }));
|
|
46
72
|
}
|
|
47
73
|
|
|
48
|
-
/**
|
|
74
|
+
/**
|
|
75
|
+
* Remove a child from the active set once it has exited (or before kill).
|
|
76
|
+
*
|
|
77
|
+
* NOTE (U15): this does NOT drop the adopted OwnedProcess handle — the owner
|
|
78
|
+
* still owes a drain-reconcile for backgrounded descendants after the root
|
|
79
|
+
* exit. The owned entry is pruned by the scanner once terminal.
|
|
80
|
+
*/
|
|
49
81
|
export function unregisterActiveChild(pid: number): void {
|
|
50
82
|
activeChildProcesses.delete(pid);
|
|
51
83
|
}
|
|
@@ -94,6 +126,16 @@ function spawnTaskkillSafe(pid: number): void {
|
|
|
94
126
|
|
|
95
127
|
export function killProcessPid(pid: number): void {
|
|
96
128
|
if (!Number.isInteger(pid) || pid <= 0) return;
|
|
129
|
+
// U15: adopted children escalate through the owned dispose ladder
|
|
130
|
+
// (SIGTERM group → poll → SIGKILL group → cap → warn give-up) instead of
|
|
131
|
+
// the one-shot signal pair below. External pid-only callers that never
|
|
132
|
+
// registered (live-session agents, cancel paths on stale pids) keep the
|
|
133
|
+
// legacy one-shot behavior unchanged.
|
|
134
|
+
const owned = ownedByPid.get(pid);
|
|
135
|
+
if (owned) {
|
|
136
|
+
void owned.dispose();
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
97
139
|
try {
|
|
98
140
|
if (process.platform === "win32") {
|
|
99
141
|
// 3.8: Windows path uses taskkill /T /F (force kill the entire tree).
|
|
@@ -168,6 +210,16 @@ export function killProcessTree(pid: number | undefined, child?: ChildProcess):
|
|
|
168
210
|
/* diagnostic best-effort */
|
|
169
211
|
}
|
|
170
212
|
if (!pid || !Number.isInteger(pid) || pid <= 0) return;
|
|
213
|
+
// U15: route adopted children through the escalating owned dispose. This
|
|
214
|
+
// deliberately BYPASSES the `child.exitCode !== null` early-out below: a
|
|
215
|
+
// dead root can still have live backgrounded descendants in its group, and
|
|
216
|
+
// dispose() is the component that knows (groupAlive is its source of truth;
|
|
217
|
+
// a terminal owner settles as a no-op via the PID-recycle guard).
|
|
218
|
+
const owned = ownedByPid.get(pid);
|
|
219
|
+
if (owned) {
|
|
220
|
+
void owned.dispose();
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
171
223
|
if (child && child.exitCode !== null) return;
|
|
172
224
|
killProcessPid(pid);
|
|
173
225
|
child?.once("exit", () => clearHardKillTimer(pid));
|
|
@@ -176,5 +228,10 @@ export function killProcessTree(pid: number | undefined, child?: ChildProcess):
|
|
|
176
228
|
export function terminateActiveChildPiProcesses(): number {
|
|
177
229
|
const entries = [...activeChildProcesses.entries()];
|
|
178
230
|
for (const [pid, child] of entries) killProcessTree(pid, child);
|
|
231
|
+
// U15: also reap owners whose active-set entry is already gone (root exited,
|
|
232
|
+
// backgrounded descendants mid-drain). Fire-and-forget by design — the sync
|
|
233
|
+
// SIGTERM prefix is delivered immediately; the ladder completes while the
|
|
234
|
+
// shutdown path drains, and the postmortem registry covers a hard exit.
|
|
235
|
+
void disposeAllOwnedProcesses();
|
|
179
236
|
return entries.length;
|
|
180
237
|
}
|