pi-crew 0.11.55 → 0.11.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -0
- package/dist/index.mjs +530 -529
- package/package.json +1 -1
- package/src/extension/registration/lifecycle-handlers.ts +10 -1
- package/src/extension/team-tool/respond-delivery.ts +429 -0
- package/src/extension/team-tool/respond.ts +135 -50
- package/src/extension/team-tool.ts +41 -1
- package/src/runtime/async-runner.ts +10 -1
- package/src/runtime/background-runner.ts +93 -17
- package/src/runtime/broker/crew-broker.ts +146 -1
- package/src/runtime/broker/dispatcher-key-registry.ts +39 -0
- package/src/runtime/broker/protocol/broker-respond.ts +365 -0
- package/src/runtime/broker/protocol/broker-steer.ts +151 -0
- package/src/runtime/broker/protocol/connection-state.ts +10 -0
- package/src/runtime/broker/protocol/request-parsers.ts +137 -1
- package/src/runtime/broker/runner-broker.ts +189 -0
- package/src/runtime/broker/stdin-handshake.ts +39 -4
- package/src/runtime/run-worker.ts +17 -2
- package/src/runtime/spawn-phase-trace.ts +53 -0
- package/src/runtime/task-runner/child-executor.ts +14 -0
- package/src/runtime/task-runner/state-helpers.ts +32 -0
- package/src/state/contracts.ts +15 -0
- package/src/state/coordination/mailbox.ts +15 -0
- package/src/state/types.ts +11 -0
|
@@ -8,12 +8,13 @@ import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
|
8
8
|
import { appendMailboxMessage, readMailbox, updateMailboxMessageReply } from "../../state/coordination/mailbox.ts";
|
|
9
9
|
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
10
10
|
import { loadRunManifestById, saveRunManifest, saveRunTasks, updateRunStatus } from "../../state/stores/state-store.ts";
|
|
11
|
-
import type { TeamTaskState } from "../../state/types.ts";
|
|
11
|
+
import type { TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
12
12
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
13
13
|
import { locateRunCwd } from "../team-tool.ts";
|
|
14
14
|
import type { PiTeamsToolResult } from "../tool-result.ts";
|
|
15
15
|
import { result, type TeamContext } from "./context.ts";
|
|
16
16
|
import { paramRequired } from "./param-error.ts";
|
|
17
|
+
import { deliverResponseRow, type RespondDeliveryRow } from "./respond-delivery.ts";
|
|
17
18
|
import { RUN_NOT_FOUND_HINT } from "./run-not-found.ts";
|
|
18
19
|
|
|
19
20
|
/** WP-2/R2 (ADR-0 item 8/security P1-4): fence the injected ask answer as
|
|
@@ -47,6 +48,27 @@ function requeueWaitingTask(tasks: TeamTaskState[], taskId: string, message: str
|
|
|
47
48
|
});
|
|
48
49
|
}
|
|
49
50
|
|
|
51
|
+
/** Lock-section product for handleRespond: either an early error result, or
|
|
52
|
+
* the full post-lock context. HƯỚNG A (2026-11-01): ALIVE-park response rows
|
|
53
|
+
* are STAGED inside the run lock (atomically with the exactly-one-dispatch
|
|
54
|
+
* scan) but DELIVERED after it releases — deliverResponseRow's broker RPC
|
|
55
|
+
* window must never sit under withRunLockSync (the broker handler takes its
|
|
56
|
+
* own run lock via withRunLockBusyRetry; an in-lock RPC window would
|
|
57
|
+
* livelock cross-process). */
|
|
58
|
+
type LockedRespondOutcome =
|
|
59
|
+
| { ok: false; early: PiTeamsToolResult }
|
|
60
|
+
| {
|
|
61
|
+
ok: true;
|
|
62
|
+
manifest: TeamRunManifest;
|
|
63
|
+
targetTaskIds: string[];
|
|
64
|
+
answeredIds: string[];
|
|
65
|
+
requeuedIds: string[];
|
|
66
|
+
noopIds: string[];
|
|
67
|
+
mailboxIds: string[];
|
|
68
|
+
pendingDeliveries: RespondDeliveryRow[];
|
|
69
|
+
message: string;
|
|
70
|
+
};
|
|
71
|
+
|
|
50
72
|
/**
|
|
51
73
|
* Handle `respond` action: answer a waiting (parked) task's question.
|
|
52
74
|
*
|
|
@@ -54,25 +76,33 @@ function requeueWaitingTask(tasks: TeamTaskState[], taskId: string, message: str
|
|
|
54
76
|
* liveness discrimination replaces the old blanket re-queue of ALL waiting
|
|
55
77
|
* tasks.
|
|
56
78
|
* - ALIVE (heartbeat last-beat within the 60s gradient-stale window, or a
|
|
57
|
-
* live in-memory session handle) →
|
|
79
|
+
* live in-memory session handle) → stage a mailbox `kind:"response"`
|
|
58
80
|
* entry carrying the task's `waiting.questionId` (from leader, to the task)
|
|
59
81
|
* and LEAVE `task.waiting` in place: the parked `ask` tool polls the task
|
|
60
82
|
* mailbox stream, picks the response up and flips waiting→running via its
|
|
61
|
-
* own terminal report.
|
|
83
|
+
* own terminal report. HƯỚNG A: the staged row is delivered AFTER the run
|
|
84
|
+
* lock releases via deliverResponseRow (broker-first CR path → a
|
|
85
|
+
* BROKER-SIGNED row that verifies in the worker even when this responder
|
|
86
|
+
* is not the dispatch process; local signed append fallback = hướng C).
|
|
62
87
|
* - DEAD → re-queue the task AND inject the fenced answer into the next
|
|
63
88
|
* dispatch prompt (pendingSteers cross-attempt channel). No questionId-
|
|
64
89
|
* tagged mailbox response is written.
|
|
65
90
|
* - EXACTLY-ONE-DISPATCH GUARD: a response for a given questionId dispatches
|
|
66
91
|
* via mailbox-write OR requeue+inject — never both — and is idempotent per
|
|
67
|
-
* questionId. The branch is atomic under the run lock (check +
|
|
68
|
-
* one withRunLockSync + fresh-reload critical section); the
|
|
69
|
-
* clears `waiting` so a second respond finds no waiting task
|
|
92
|
+
* questionId. The branch decision is atomic under the run lock (check +
|
|
93
|
+
* staging inside one withRunLockSync + fresh-reload critical section); the
|
|
94
|
+
* requeue path clears `waiting` so a second respond finds no waiting task
|
|
95
|
+
* (no-op), and the out-of-lock row write is exactly-once via the durable
|
|
96
|
+
* requestId = questionId idempotency key (broker append AND local fallback
|
|
97
|
+
* both dedupe on it — a concurrent second respond that missed the in-lock
|
|
98
|
+
* scan reuses the first row instead of duplicating it).
|
|
70
99
|
* - Legacy waiting parks without `task.waiting` (pre-v2 records) keep the old
|
|
71
100
|
* mailbox + re-queue resume behavior — there is no parked tool to pick a
|
|
72
|
-
* questionId-tagged response up
|
|
101
|
+
* questionId-tagged response up (no questionId → no broker contract row;
|
|
102
|
+
* the legacy append stays inline/local, verbatim).
|
|
73
103
|
* - `ask.answered` is emitted on both discrimination paths.
|
|
74
104
|
*/
|
|
75
|
-
export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): PiTeamsToolResult {
|
|
105
|
+
export async function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Promise<PiTeamsToolResult> {
|
|
76
106
|
if (!params.runId)
|
|
77
107
|
return result(
|
|
78
108
|
paramRequired("respond", "runId", "{ action: 'respond', runId: 'team_...', message: '...' }"),
|
|
@@ -95,20 +125,27 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
95
125
|
const loaded = loadRunManifestById(runCwd, params.runId); // NOTE: no withRunLock - best-effort only; concurrent writes may cause inconsistency
|
|
96
126
|
if (!loaded) return result(`Run '${params.runId}' not found.${RUN_NOT_FOUND_HINT}`, { action: "respond", status: "error" }, true);
|
|
97
127
|
|
|
98
|
-
|
|
128
|
+
const locked = withRunLockSync(loaded.manifest, (): LockedRespondOutcome => {
|
|
99
129
|
const fresh = loadRunManifestById(loaded.manifest.cwd, params.runId!); // NOTE: inside withRunLockSync - consistent read
|
|
100
|
-
if (!fresh)
|
|
130
|
+
if (!fresh)
|
|
131
|
+
return {
|
|
132
|
+
ok: false,
|
|
133
|
+
early: result(`Run '${params.runId}' not found.${RUN_NOT_FOUND_HINT}`, { action: "respond", status: "error" }, true),
|
|
134
|
+
};
|
|
101
135
|
const foreignRun = typeof fresh.manifest.ownerSessionId === "string" && fresh.manifest.ownerSessionId !== ctx.sessionId;
|
|
102
136
|
if (foreignRun && !params.force)
|
|
103
|
-
return
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
137
|
+
return {
|
|
138
|
+
ok: false,
|
|
139
|
+
early: result(
|
|
140
|
+
`Run ${fresh.manifest.runId} belongs to another session. Use force: true to override.`,
|
|
141
|
+
{
|
|
142
|
+
action: "respond",
|
|
143
|
+
status: "error",
|
|
144
|
+
runId: fresh.manifest.runId,
|
|
145
|
+
},
|
|
146
|
+
true,
|
|
147
|
+
),
|
|
148
|
+
};
|
|
112
149
|
|
|
113
150
|
const taskId = params.taskId;
|
|
114
151
|
const message = params.message ?? "";
|
|
@@ -184,19 +221,22 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
184
221
|
const existing = taskId ? fresh.tasks.find((t) => t.id === taskId) : undefined;
|
|
185
222
|
const hint =
|
|
186
223
|
" Use api operation=follow-up-agent for continuation prompts or api operation=steer-agent to interrupt active work.";
|
|
187
|
-
return
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
224
|
+
return {
|
|
225
|
+
ok: false,
|
|
226
|
+
early: result(
|
|
227
|
+
(taskId
|
|
228
|
+
? existing
|
|
229
|
+
? `Task '${taskId}' is ${existing.status}, not waiting.`
|
|
230
|
+
: `Task '${taskId}' not found.`
|
|
231
|
+
: `No waiting tasks in run ${fresh.manifest.runId}.`) + hint,
|
|
232
|
+
{
|
|
233
|
+
action: "respond",
|
|
234
|
+
status: "error",
|
|
235
|
+
runId: fresh.manifest.runId,
|
|
236
|
+
},
|
|
237
|
+
true,
|
|
238
|
+
),
|
|
239
|
+
};
|
|
200
240
|
}
|
|
201
241
|
|
|
202
242
|
const mailboxIds: string[] = [];
|
|
@@ -206,6 +246,12 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
206
246
|
const requeuedIds: string[] = [];
|
|
207
247
|
/** Exactly-once no-ops: a mailbox response already exists for the questionId. */
|
|
208
248
|
const noopIds: string[] = [];
|
|
249
|
+
/** HƯỚNG A (2026-11-01): ALIVE-park response rows staged IN-LOCK (atomic
|
|
250
|
+
* with the exactly-one-dispatch scan above) for post-lock delivery via
|
|
251
|
+
* deliverResponseRow — broker-first CR path / local signed fallback. The
|
|
252
|
+
* durable requestId = questionId key keeps the out-of-lock write
|
|
253
|
+
* exactly-once. */
|
|
254
|
+
const pendingDeliveries: RespondDeliveryRow[] = [];
|
|
209
255
|
const answeredEvents: Array<{ taskId: string; questionId: string; delivery: "mailbox" | "requeue" }> = [];
|
|
210
256
|
/** G5 (deep-review 2026-10-01): DEAD parks — the questionId-tagged response
|
|
211
257
|
* can never be picked up (the parked poller is gone); recorded with the
|
|
@@ -256,7 +302,16 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
256
302
|
// ALIVE: deliver via the task mailbox stream the parked ask tool
|
|
257
303
|
// polls (≤500ms). The park stays — the tool's terminal report
|
|
258
304
|
// flips waiting→running; root-side status must NOT change here.
|
|
259
|
-
|
|
305
|
+
// HƯỚNG A (2026-11-01): the row is no longer appended inline. It is
|
|
306
|
+
// staged here — inside the run lock, atomically with the
|
|
307
|
+
// exactly-one-dispatch scan — and delivered AFTER the lock releases
|
|
308
|
+
// via deliverResponseRow (broker-first CR path → a BROKER-SIGNED row
|
|
309
|
+
// that verifies in the worker even when this responder is not the
|
|
310
|
+
// dispatch process; local signed append fallback = hướng C behavior).
|
|
311
|
+
// Holding withRunLockSync across the RPC window would livelock
|
|
312
|
+
// against the broker handler's own run lock (withRunLockBusyRetry);
|
|
313
|
+
// exactly-once moves to the durable requestId = questionId key.
|
|
314
|
+
pendingDeliveries.push({
|
|
260
315
|
direction: "inbox",
|
|
261
316
|
from: "leader",
|
|
262
317
|
to: task.id,
|
|
@@ -266,12 +321,12 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
266
321
|
priority: "normal",
|
|
267
322
|
deliveryMode: "next_turn",
|
|
268
323
|
questionId,
|
|
324
|
+
requestId: questionId,
|
|
269
325
|
data: { action: "respond", kind: "response", questionId },
|
|
270
326
|
replyTo: params.replyTo,
|
|
271
327
|
replyFrom: params.replyFrom,
|
|
272
328
|
replyDeadline: params.replyDeadline,
|
|
273
329
|
});
|
|
274
|
-
mailboxIds.push(mailbox.id);
|
|
275
330
|
answeredIds.push(task.id);
|
|
276
331
|
answeredEvents.push({ taskId: task.id, questionId, delivery: "mailbox" });
|
|
277
332
|
continue;
|
|
@@ -410,21 +465,51 @@ export function handleRespond(params: TeamToolParamsValue, ctx: TeamContext): Pi
|
|
|
410
465
|
}
|
|
411
466
|
}
|
|
412
467
|
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
468
|
+
// HƯỚNG A (2026-11-01): ALIVE rows are staged, not appended — the lock
|
|
469
|
+
// section returns the context and delivery happens after the release
|
|
470
|
+
// (see deliverResponseRow). task.resumed events above snapshot
|
|
471
|
+
// `mailboxIds` at fire time (in-lock), so their data lists the legacy
|
|
472
|
+
// inline ids only — informational field, the ALIVE ids land in the tool
|
|
473
|
+
// result below.
|
|
474
|
+
return {
|
|
475
|
+
ok: true,
|
|
476
|
+
manifest,
|
|
477
|
+
targetTaskIds: targetTasks.map((t) => t.id),
|
|
478
|
+
answeredIds,
|
|
479
|
+
requeuedIds,
|
|
480
|
+
noopIds,
|
|
481
|
+
mailboxIds,
|
|
482
|
+
pendingDeliveries,
|
|
483
|
+
message,
|
|
484
|
+
};
|
|
429
485
|
});
|
|
486
|
+
if (!locked.ok) return locked.early;
|
|
487
|
+
// Deliver the staged ALIVE rows OUTSIDE the run lock — broker-first CR
|
|
488
|
+
// path (broker-SIGNED row, verifies in the worker regardless of who the
|
|
489
|
+
// responder process is) with the local signed idempotent append as
|
|
490
|
+
// fallback (hướng C). deliverResponseRow never throws on broker failure;
|
|
491
|
+
// a local append failure propagates exactly like the pre-hướng-A inline
|
|
492
|
+
// append did.
|
|
493
|
+
for (const row of locked.pendingDeliveries) {
|
|
494
|
+
const delivered = await deliverResponseRow(locked.manifest, row);
|
|
495
|
+
locked.mailboxIds.push(delivered.messageId);
|
|
496
|
+
}
|
|
497
|
+
const summaryParts: string[] = [];
|
|
498
|
+
if (locked.answeredIds.length > 0) summaryParts.push(`answered via mailbox (worker alive): ${locked.answeredIds.join(", ")}`);
|
|
499
|
+
if (locked.requeuedIds.length > 0) summaryParts.push(`requeued with injected answer (worker dead): ${locked.requeuedIds.join(", ")}`);
|
|
500
|
+
if (locked.noopIds.length > 0) summaryParts.push(`already answered, no-op: ${locked.noopIds.join(", ")}`);
|
|
501
|
+
return result(
|
|
502
|
+
`Responded to ${locked.targetTaskIds.length} waiting task(s). ${summaryParts.join("; ")}. Message: ${
|
|
503
|
+
locked.message || "(no message)"
|
|
504
|
+
}`,
|
|
505
|
+
{
|
|
506
|
+
action: "respond",
|
|
507
|
+
status: "ok",
|
|
508
|
+
runId: locked.manifest.runId,
|
|
509
|
+
resumedIds: locked.requeuedIds,
|
|
510
|
+
mailboxIds: locked.mailboxIds,
|
|
511
|
+
intent: `responding to ${locked.targetTaskIds.join(", ")} in ${locked.manifest.runId}`,
|
|
512
|
+
data: { answeredIds: locked.answeredIds, noopIds: locked.noopIds },
|
|
513
|
+
},
|
|
514
|
+
);
|
|
430
515
|
}
|
|
@@ -11,7 +11,7 @@ import type { executeTeamRun as _executeTeamRunFn } from "../runtime/team-runner
|
|
|
11
11
|
import type { TeamToolParamsValue } from "../schema/team-tool-schema.ts";
|
|
12
12
|
import { TEAM_TERMINAL_TASK_STATUSES } from "../state/contracts.ts";
|
|
13
13
|
import { withRunLock } from "../state/coordination/locks.ts";
|
|
14
|
-
import { replayPendingMailboxMessages, signSteeringEntry } from "../state/coordination/mailbox.ts";
|
|
14
|
+
import { getMailboxVerifyKeyPublic, replayPendingMailboxMessages, signSteeringEntry } from "../state/coordination/mailbox.ts";
|
|
15
15
|
import { isTaskClaimExpired } from "../state/coordination/task-claims.ts";
|
|
16
16
|
import { appendEventAsync, appendEventFireAndForget } from "../state/event-log/event-log.ts";
|
|
17
17
|
import { ACTIVE_RUN_STALE_MS, activeRunEntries, registerActiveRun, unregisterActiveRun } from "../state/stores/active-run-registry.ts";
|
|
@@ -1069,6 +1069,27 @@ function clampSteerBody(message: string): string {
|
|
|
1069
1069
|
return `${message.slice(0, TEAM_STEER_MAX_MESSAGE_LENGTH - TEAM_STEER_TRUNCATION_MARKER.length)}${TEAM_STEER_TRUNCATION_MARKER}`;
|
|
1070
1070
|
}
|
|
1071
1071
|
|
|
1072
|
+
/** HƯỚNG A follow-up (probe19): token-less steer.force client — single
|
|
1073
|
+
* NDJSON frame to the run's broker (stamped socket preferred, derivation
|
|
1074
|
+
* fallback). The broker (detached runner host) performs the trusted write
|
|
1075
|
+
* pair. Mirrors respond-delivery's force tier; LAZY imports keep the
|
|
1076
|
+
* extension registration cold-start free of the broker-client graph. */
|
|
1077
|
+
async function attemptSteerForceViaBroker(manifest: TeamRunManifest, taskId: string, message: string): Promise<void> {
|
|
1078
|
+
const { brokerRoundTrip } = await import("./team-tool/respond-delivery.ts"); // LAZY: broker-client graph out of extension registration cold-start
|
|
1079
|
+
const { getBrokerSocketPath } = await import("../utils/socket-path.ts"); // LAZY: same — keep cold-start lean
|
|
1080
|
+
const stamped = manifest.brokerSocketPath;
|
|
1081
|
+
const socketPath = typeof stamped === "string" && stamped.length > 0 ? stamped : getBrokerSocketPath(manifest.sessionId ?? "");
|
|
1082
|
+
let sent = false;
|
|
1083
|
+
await brokerRoundTrip(socketPath, () => {
|
|
1084
|
+
if (sent) return undefined; // force ack received — exchange done
|
|
1085
|
+
sent = true;
|
|
1086
|
+
return {
|
|
1087
|
+
method: "steer.force",
|
|
1088
|
+
params: { runId: manifest.runId, taskId, body: message },
|
|
1089
|
+
};
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
|
|
1072
1093
|
export function handleSteer(params: TeamToolParamsValue, ctx: TeamContext): PiTeamsToolResult {
|
|
1073
1094
|
const { runId, taskId } = params;
|
|
1074
1095
|
const rawMessage = params.message;
|
|
@@ -1125,6 +1146,25 @@ export function handleSteer(params: TeamToolParamsValue, ctx: TeamContext): PiTe
|
|
|
1125
1146
|
} catch {
|
|
1126
1147
|
// Best-effort: file write failure doesn't block the steer from pending array
|
|
1127
1148
|
}
|
|
1149
|
+
// HƯỚNG A follow-up (probe19, 2026-11-01): a RESTARTED/foreign session's
|
|
1150
|
+
// steering-file row is signed with a key the live worker does NOT hold
|
|
1151
|
+
// (its verify set = dispatcher + spawner) — pollSteering drops it, so the
|
|
1152
|
+
// fs row is dead for this incarnation and pendingSteers only delivers at
|
|
1153
|
+
// the NEXT spawn. When we are not the trusted signer, ALSO route the
|
|
1154
|
+
// message through the run's broker via token-less steer.force: the
|
|
1155
|
+
// broker-host (detached runner) signs with a key every worker trusts and
|
|
1156
|
+
// appends mailbox row + steering line itself. Fire-and-forget (handleSteer
|
|
1157
|
+
// is sync); the steer.broker_force audit event is the success trace.
|
|
1158
|
+
if (loaded.manifest.dispatcherVerifyKey !== getMailboxVerifyKeyPublic()) {
|
|
1159
|
+
void attemptSteerForceViaBroker(loaded.manifest, taskId, message).catch((error) =>
|
|
1160
|
+
logInternalError(
|
|
1161
|
+
"steer.force-delivery-fallback",
|
|
1162
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1163
|
+
`runId=${runId} taskId=${taskId}`,
|
|
1164
|
+
"warn",
|
|
1165
|
+
),
|
|
1166
|
+
);
|
|
1167
|
+
}
|
|
1128
1168
|
// H1 (2026-08-10): handleSteer is a SYNC function — fire-and-forget async.
|
|
1129
1169
|
void appendEventAsync(loaded.manifest.eventsPath, {
|
|
1130
1170
|
type: "task.steer_queued",
|
|
@@ -5,6 +5,7 @@ import * as path from "node:path";
|
|
|
5
5
|
import { pathToFileURL } from "node:url";
|
|
6
6
|
import { backgroundRunnerEnvAllowlist } from "../config/env-allowlist-registry.ts";
|
|
7
7
|
import { getCrewEnv } from "../config/env-vars.ts";
|
|
8
|
+
import { ensureCoordinationPublicVerifyKey } from "../state/coordination/mailbox.ts";
|
|
8
9
|
import { appendEventAsync } from "../state/event-log/event-log.ts";
|
|
9
10
|
import type { TeamRunManifest } from "../state/types.ts";
|
|
10
11
|
import { sanitizeEnvSecrets } from "../utils/env-filter.ts";
|
|
@@ -416,7 +417,15 @@ export async function spawnBackgroundTeamRun(manifest: TeamRunManifest): Promise
|
|
|
416
417
|
}
|
|
417
418
|
}
|
|
418
419
|
if (socketPath && Object.keys(tasks).length > 0) {
|
|
419
|
-
|
|
420
|
+
// HƯỚNG A (ADR Option 3): thread THIS (spawning/dispatcher) process's
|
|
421
|
+
// PUBLIC coordination verify key over the start-options pipe so a
|
|
422
|
+
// broker constructed inside the runner binds the DISPATCHER's key for
|
|
423
|
+
// respond.append challenge-response auth. Derived in THIS heap at
|
|
424
|
+
// spawn time — deliberately NOT read from manifest.dispatcherVerifyKey
|
|
425
|
+
// (the manifest file is unauthenticated/worker-writable: a w1a NO-GO
|
|
426
|
+
// for auth bindings). The spawner IS the effective dispatcher (it will
|
|
427
|
+
// author future respond rows for this run). PUBLIC material only.
|
|
428
|
+
line = buildBrokerStdinLine(manifest.runId, socketPath, tasks, ensureCoordinationPublicVerifyKey());
|
|
420
429
|
child.stdin?.write(line);
|
|
421
430
|
} else {
|
|
422
431
|
child.stdin?.write("\n");
|
|
@@ -45,6 +45,8 @@ import { writeAsyncStartMarker } from "./async-marker.ts";
|
|
|
45
45
|
// F4: broker creds handshake helpers live in broker/stdin-handshake.ts
|
|
46
46
|
// (pure module — background-runner runs await main() at module scope, so
|
|
47
47
|
// importing THIS file from tests would boot the runner).
|
|
48
|
+
import { setActiveDispatcherVerifyKey } from "./broker/dispatcher-key-registry.ts";
|
|
49
|
+
import type { RunnerHostedBrokerHandle } from "./broker/runner-broker.ts";
|
|
48
50
|
import { parseStdinBrokerPayload, readStdinFirstLine } from "./broker/stdin-handshake.ts";
|
|
49
51
|
import { terminateActiveChildPiProcesses } from "./child-pi/child-pi.ts";
|
|
50
52
|
import { directTeamAndWorkflowFromRun } from "./direct-run.ts";
|
|
@@ -321,7 +323,7 @@ function setupUnhandledRejectionGuard(
|
|
|
321
323
|
* and error handlers. This ensures consistent cleanup regardless of how
|
|
322
324
|
* the process exits (normal flow, unhandled rejection, or main() exception).
|
|
323
325
|
*/
|
|
324
|
-
function runCleanup(
|
|
326
|
+
async function runCleanup(
|
|
325
327
|
stopInterruptGuard: () => void,
|
|
326
328
|
stopParentGuard: () => void,
|
|
327
329
|
stopHeartbeat: () => void,
|
|
@@ -329,7 +331,7 @@ function runCleanup(
|
|
|
329
331
|
watchdogTimer: NodeJS.Timeout,
|
|
330
332
|
exitDueToRejection: boolean,
|
|
331
333
|
eventsPath?: string,
|
|
332
|
-
): void {
|
|
334
|
+
): Promise<void> {
|
|
333
335
|
console.log(`[background-runner] runCleanup, exitDueToRejection=${exitDueToRejection}`);
|
|
334
336
|
stopInterruptGuard();
|
|
335
337
|
stopParentGuard();
|
|
@@ -350,6 +352,19 @@ function runCleanup(
|
|
|
350
352
|
console.log(`[background-runner] runCleanup: terminateActiveChildPiProcesses error: ${errorMessage(error)}`);
|
|
351
353
|
}
|
|
352
354
|
console.log(`[background-runner] runCleanup: killed ${killed} child processes`);
|
|
355
|
+
// FINDING D: stop the runner-hosted broker AFTER the children (they may
|
|
356
|
+
// still flush final frames) — ownership-gated unlink of its socket. Never
|
|
357
|
+
// re-throws: cleanup must keep going.
|
|
358
|
+
if (activeRunnerBroker) {
|
|
359
|
+
const brokerToStop = activeRunnerBroker;
|
|
360
|
+
activeRunnerBroker = undefined;
|
|
361
|
+
try {
|
|
362
|
+
await brokerToStop.stop();
|
|
363
|
+
console.log("[background-runner] runCleanup: runner-hosted broker stopped");
|
|
364
|
+
} catch (error) {
|
|
365
|
+
console.log(`[background-runner] runCleanup: broker stop error: ${errorMessage(error)}`);
|
|
366
|
+
}
|
|
367
|
+
}
|
|
353
368
|
// FIX Issue #5: Unregister this worker from the orphan registry on exit.
|
|
354
369
|
// Previously this was only cleaned up on the next session_start cleanup cycle,
|
|
355
370
|
// causing unnecessary delay in removing stale registrations.
|
|
@@ -381,6 +396,9 @@ function runCleanup(
|
|
|
381
396
|
// Module-level flag: set by unhandled rejection guard and main() catch.
|
|
382
397
|
// Used by the module-level catch to signal that finally should call process.exit(1).
|
|
383
398
|
let exitDueToRejection = false;
|
|
399
|
+
/** FINDING D: the runner-hosted broker for THIS run (undefined = legacy
|
|
400
|
+
* dispatcher-socket path / hosting failed). Stopped in runCleanup(). */
|
|
401
|
+
let activeRunnerBroker: RunnerHostedBrokerHandle | undefined;
|
|
384
402
|
|
|
385
403
|
export interface ExecuteBackgroundRunOptions {
|
|
386
404
|
/**
|
|
@@ -810,20 +828,78 @@ async function main(): Promise<void> {
|
|
|
810
828
|
const { setActiveBrokerIssuer } = await import("./broker/broker-issuer.ts");
|
|
811
829
|
// LAZY: pi-args pulls the model registry chain.
|
|
812
830
|
const { resolveCrewMaxDepth } = await import("./model/pi-args.ts");
|
|
813
|
-
//
|
|
814
|
-
//
|
|
815
|
-
//
|
|
816
|
-
//
|
|
817
|
-
//
|
|
818
|
-
//
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
831
|
+
// HƯỚNG A (ADR Option 3): bind the dispatcher's PUBLIC verify key from
|
|
832
|
+
// the same start-options payload (heap → pipe → heap). A broker
|
|
833
|
+
// constructed in THIS process (lifecycle-handlers.ts construct site)
|
|
834
|
+
// then binds the DISPATCHER's key for respond.append CR auth instead
|
|
835
|
+
// of this runner's own keypair. Absent (older spawner) = no binding —
|
|
836
|
+
// the construct site falls back to its own key (sync-path semantics).
|
|
837
|
+
// The registry is a pure module (static import, stdin-handshake
|
|
838
|
+
// discipline — no lazy boundary needed).
|
|
839
|
+
if (payload.dispatcherVerifyKey !== undefined) {
|
|
840
|
+
setActiveDispatcherVerifyKey(payload.dispatcherVerifyKey);
|
|
841
|
+
debugLog(`[broker] dispatcher verify key bound from start options (run ${runId})`);
|
|
842
|
+
}
|
|
843
|
+
// FINDING D (2026-11-01, probes 8/9): payload.socketPath is the
|
|
844
|
+
// DISPATCHER's broker — a `pi -p` dispatcher exits seconds after
|
|
845
|
+
// async dispatch and session_shutdown unlinks that socket while the
|
|
846
|
+
// run lives on (every later worker broker op hit ENOENT). Host the
|
|
847
|
+
// run's broker HERE instead: the runner outlives the whole run, mints
|
|
848
|
+
// its own per-task tokens, and stamps ITS socket on the manifest so
|
|
849
|
+
// cross-process responders (respond-delivery RPC / respond.force)
|
|
850
|
+
// reach a listener that exists for the run's full lifetime. Soft-fail:
|
|
851
|
+
// when hosting fails (disabled / bind error), keep the legacy payload
|
|
852
|
+
// issuer + dispatcher socket below — exact pre-Finding-D behavior,
|
|
853
|
+
// correct while the dispatcher still lives.
|
|
854
|
+
let hostedBroker: RunnerHostedBrokerHandle | undefined;
|
|
855
|
+
try {
|
|
856
|
+
const { startRunnerHostedBroker } = await import("./broker/runner-broker.ts"); // LAZY: broker module has process-level side effects at load
|
|
857
|
+
hostedBroker = await startRunnerHostedBroker({
|
|
858
|
+
runId,
|
|
859
|
+
cwd,
|
|
860
|
+
dispatcherVerifyKey: payload.dispatcherVerifyKey,
|
|
861
|
+
resolveMaxDepth: () => resolveCrewMaxDepth(undefined),
|
|
862
|
+
});
|
|
863
|
+
} catch (error) {
|
|
864
|
+
logInternalError("background-runner.runner-broker-boot", error, `runId=${runId}`, "warn");
|
|
865
|
+
}
|
|
866
|
+
if (hostedBroker) {
|
|
867
|
+
activeRunnerBroker = hostedBroker;
|
|
868
|
+
setActiveBrokerIssuer(hostedBroker.issueForChild);
|
|
869
|
+
debugLog(`[broker] runner-hosted broker listening (${hostedBroker.socketPath})`);
|
|
870
|
+
} else {
|
|
871
|
+
// Legacy path: static issuer scoped to THIS run, serving PRE-MINTED
|
|
872
|
+
// per-task COMPOUND tokens — wait.* rejects bare-runId tokens
|
|
873
|
+
// (ADR-0 item 6). Unknown taskIds get NO creds — follow-up:
|
|
874
|
+
// broker-side mint RPC. Depth-cap parity with the parent-side
|
|
875
|
+
// issueForChild gate (lifecycle-handlers.ts:1130-1136).
|
|
876
|
+
setActiveBrokerIssuer(async (rid, taskId, childDepth) => {
|
|
877
|
+
if (rid !== payload.runId) return undefined;
|
|
878
|
+
if (childDepth !== undefined && childDepth >= resolveCrewMaxDepth(undefined)) return undefined;
|
|
879
|
+
if (!taskId) return undefined;
|
|
880
|
+
const token = payload.tasks[taskId];
|
|
881
|
+
if (!token) return undefined;
|
|
882
|
+
return { socketPath: payload.socketPath, token };
|
|
883
|
+
});
|
|
884
|
+
}
|
|
885
|
+
// HƯỚNG A + FINDING D: persist the run's ACTUAL broker socket path onto
|
|
886
|
+
// the manifest so cross-process responders can find it — the
|
|
887
|
+
// runner-hosted socket when hosting succeeded, else the dispatcher's
|
|
888
|
+
// (pre-Finding-D semantics; derivation from manifest.sessionId was
|
|
889
|
+
// never correct in async topology — probe8: derived 7e4df036 vs actual
|
|
890
|
+
// 9529a08a). Best-effort: a stamp failure never blocks the run
|
|
891
|
+
// (responders fall back to derivation, then to the local signed
|
|
892
|
+
// append — hướng C).
|
|
893
|
+
const stampPath = hostedBroker?.socketPath ?? payload.socketPath;
|
|
894
|
+
try {
|
|
895
|
+
const loaded = loadRunManifestById(cwd, runId);
|
|
896
|
+
if (loaded && loaded.manifest.brokerSocketPath !== stampPath) {
|
|
897
|
+
await saveRunManifestAsync({ ...loaded.manifest, brokerSocketPath: stampPath });
|
|
898
|
+
debugLog(`[broker] manifest.brokerSocketPath stamped (${stampPath})`);
|
|
899
|
+
}
|
|
900
|
+
} catch (error) {
|
|
901
|
+
logInternalError("background-runner.broker-socket-stamp", error, `runId=${runId}`, "warn");
|
|
902
|
+
}
|
|
827
903
|
debugLog(
|
|
828
904
|
`[broker] stdin handshake accepted for run ${runId} (${Object.keys(payload.tasks).length} task tokens, socket ${payload.socketPath})`,
|
|
829
905
|
);
|
|
@@ -1073,7 +1149,7 @@ async function main(): Promise<void> {
|
|
|
1073
1149
|
// FIX Issue #1: Wrap runCleanup in try/catch to ensure process.exit(1)
|
|
1074
1150
|
// is called even if runCleanup throws unexpectedly.
|
|
1075
1151
|
try {
|
|
1076
|
-
runCleanup(
|
|
1152
|
+
await runCleanup(
|
|
1077
1153
|
stopInterruptGuard,
|
|
1078
1154
|
stopParentGuard,
|
|
1079
1155
|
stopHeartbeat,
|