pi-crew 0.11.55 → 0.11.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -0
- package/dist/index.mjs +530 -529
- package/package.json +1 -1
- package/src/extension/registration/lifecycle-handlers.ts +10 -1
- package/src/extension/team-tool/respond-delivery.ts +429 -0
- package/src/extension/team-tool/respond.ts +135 -50
- package/src/extension/team-tool.ts +41 -1
- package/src/runtime/async-runner.ts +10 -1
- package/src/runtime/background-runner.ts +93 -17
- package/src/runtime/broker/crew-broker.ts +146 -1
- package/src/runtime/broker/dispatcher-key-registry.ts +39 -0
- package/src/runtime/broker/protocol/broker-respond.ts +365 -0
- package/src/runtime/broker/protocol/broker-steer.ts +151 -0
- package/src/runtime/broker/protocol/connection-state.ts +10 -0
- package/src/runtime/broker/protocol/request-parsers.ts +137 -1
- package/src/runtime/broker/runner-broker.ts +189 -0
- package/src/runtime/broker/stdin-handshake.ts +39 -4
- package/src/runtime/run-worker.ts +17 -2
- package/src/runtime/spawn-phase-trace.ts +53 -0
- package/src/runtime/task-runner/child-executor.ts +14 -0
- package/src/runtime/task-runner/state-helpers.ts +32 -0
- package/src/state/contracts.ts +15 -0
- package/src/state/coordination/mailbox.ts +15 -0
- package/src/state/types.ts +11 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* broker-steer.ts — HƯỚNG A follow-up sibling (2026-11-01, probe19): the
|
|
3
|
+
* steer.dead-end. A RESTARTED/foreign session's `team steer` writes a
|
|
4
|
+
* pendingSteers entry + a steering-file row signed with ITS OWN keypair — but
|
|
5
|
+
* the live worker's pollSteering only accepts rows signed by a key in its
|
|
6
|
+
* PI_CREW_MAILBOX_VERIFY_KEYS set (dispatcher + spawner/runner), so the row is
|
|
7
|
+
* dropped as unauthenticated noise and the in-context injection NEVER happens
|
|
8
|
+
* (probe19: file row present, worker conversation received NOTHING). The
|
|
9
|
+
* pendingSteers entry only delivers at the NEXT spawn — useless for a live
|
|
10
|
+
* worker. respond.force closed the same class for asks; steer.force closes it
|
|
11
|
+
* for steering.
|
|
12
|
+
*
|
|
13
|
+
* `steer.force` is a PRE-hello, token-less broker method (mirrors
|
|
14
|
+
* respond.force's lane): the RUN's broker — hosted by the detached runner,
|
|
15
|
+
* whose keypair IS in every worker's verify set — performs the delivery
|
|
16
|
+
* itself: a mailbox row (kind:"steer", deliveryMode:"interrupt") + a
|
|
17
|
+
* BROKER-SIGNED steering-file line the live worker accepts.
|
|
18
|
+
*
|
|
19
|
+
* Trust: same-uid domain (the socket dir is 0700; any local process could
|
|
20
|
+
* already write the fs rows — this surface only adds VERIFIED delivery, never
|
|
21
|
+
* new authority). Gates: worker-role connections rejected (defense-in-depth),
|
|
22
|
+
* task must exist and be live (running|waiting), requestId dedup mirrors
|
|
23
|
+
* steer.push (R2-L-A4), audit event steer.broker_force lands in events.jsonl.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import { isTerminalTaskStatus } from "../../../state/contracts.ts";
|
|
27
|
+
import { appendEventAsync } from "../../../state/event-log/event-log.ts";
|
|
28
|
+
import { loadRunManifestById } from "../../../state/stores/state-store.ts";
|
|
29
|
+
import type { TeamRunManifest } from "../../../state/types.ts";
|
|
30
|
+
import { logInternalError } from "../../../utils/internal-error.ts";
|
|
31
|
+
import type { ServerConnection } from "./connection-state.ts";
|
|
32
|
+
|
|
33
|
+
export interface SteerForceParams {
|
|
34
|
+
runId: string;
|
|
35
|
+
taskId: string;
|
|
36
|
+
body: string;
|
|
37
|
+
/** Optional idempotency key — resend-after-ack-loss dedupes (R2-L-A4). */
|
|
38
|
+
requestId?: string;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function parseSteerForceParams(value: unknown): SteerForceParams | undefined {
|
|
42
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
43
|
+
const v = value as Record<string, unknown>;
|
|
44
|
+
const runId = typeof v.runId === "string" ? v.runId : undefined;
|
|
45
|
+
const taskId = typeof v.taskId === "string" ? v.taskId : undefined;
|
|
46
|
+
const body = typeof v.body === "string" ? v.body : undefined;
|
|
47
|
+
if (!runId || !taskId || body === undefined) return undefined;
|
|
48
|
+
const requestId = typeof v.requestId === "string" && v.requestId.length > 0 ? v.requestId.slice(0, 128) : undefined;
|
|
49
|
+
return requestId === undefined ? { runId, taskId, body } : { runId, taskId, body, requestId };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface SteerForceDeps {
|
|
53
|
+
sendError: (conn: ServerConnection, id: string, code: string, message: string) => void;
|
|
54
|
+
sendResult: (conn: ServerConnection, id: string, result: unknown) => void;
|
|
55
|
+
/** Mailbox append (idempotent when requestId supplied). */
|
|
56
|
+
appendSteerRow: (manifest: TeamRunManifest, params: SteerForceParams, messageId: string) => Promise<{ deduped: boolean }>;
|
|
57
|
+
/** Signed steering-file append (the delivery the live worker accepts). */
|
|
58
|
+
appendSignedSteeringLine: (manifest: TeamRunManifest, taskId: string, body: string, messageId: string) => Promise<void>;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** handleSteerForce — see module header. Never throws (typed errors only). */
|
|
62
|
+
export async function handleSteerForce(
|
|
63
|
+
deps: SteerForceDeps,
|
|
64
|
+
conn: ServerConnection,
|
|
65
|
+
id: string,
|
|
66
|
+
params: unknown,
|
|
67
|
+
options: { cwd?: string },
|
|
68
|
+
): Promise<void> {
|
|
69
|
+
// Defense-in-depth (mirrors respond.force): a worker-token connection must
|
|
70
|
+
// not steer a sibling through this lane.
|
|
71
|
+
if (conn.role === "worker") {
|
|
72
|
+
deps.sendError(conn, id, "forbidden", "steer.force is not available to worker tokens");
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
const parsed = parseSteerForceParams(params);
|
|
76
|
+
if (!parsed) {
|
|
77
|
+
deps.sendError(conn, id, "bad-params", "steer.force: runId, taskId and body are required");
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
if (conn.authed && conn.runId !== undefined && conn.runId !== parsed.runId) {
|
|
81
|
+
deps.sendError(conn, id, "forbidden", "steer.force: runId does not match the authenticated run");
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
if (!options.cwd) {
|
|
85
|
+
deps.sendError(conn, id, "no-manifest", "broker has no cwd configured");
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
let loaded: NonNullable<ReturnType<typeof loadRunManifestById>>;
|
|
89
|
+
try {
|
|
90
|
+
const l = loadRunManifestById(options.cwd, parsed.runId);
|
|
91
|
+
if (!l) {
|
|
92
|
+
deps.sendError(conn, id, "no-manifest", `run '${parsed.runId}' not found`);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
loaded = l;
|
|
96
|
+
} catch (err) {
|
|
97
|
+
deps.sendError(conn, id, "no-manifest", (err as Error).message);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
const task = loaded.tasks.find((t) => t.id === parsed.taskId);
|
|
101
|
+
if (!task) {
|
|
102
|
+
deps.sendError(conn, id, "no-task", `steer.force: task '${parsed.taskId}' not found`);
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (isTerminalTaskStatus(task.status)) {
|
|
106
|
+
deps.sendError(conn, id, "no-task", `steer.force: task '${parsed.taskId}' is ${task.status}`);
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
const messageId = `steerf_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
110
|
+
let deduped = false;
|
|
111
|
+
try {
|
|
112
|
+
const outcome = await deps.appendSteerRow(loaded.manifest, parsed, messageId);
|
|
113
|
+
deduped = outcome.deduped;
|
|
114
|
+
} catch (err) {
|
|
115
|
+
logInternalError(
|
|
116
|
+
"crew-broker.steer.force-mailbox",
|
|
117
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
118
|
+
`runId=${parsed.runId} taskId=${parsed.taskId}`,
|
|
119
|
+
);
|
|
120
|
+
deps.sendError(conn, id, "append-failed", "steer.force: mailbox append failed");
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
// Best-effort signed steering line — a failure here must NOT fail the call
|
|
124
|
+
// (the mailbox row's live fanout already delivered); mirrors handleSteerPush
|
|
125
|
+
// write-2 discipline. Skip on dedup (R2-L-A4).
|
|
126
|
+
if (!deduped) {
|
|
127
|
+
try {
|
|
128
|
+
await deps.appendSignedSteeringLine(loaded.manifest, parsed.taskId, parsed.body, messageId);
|
|
129
|
+
} catch (err) {
|
|
130
|
+
logInternalError(
|
|
131
|
+
"crew-broker.steer.force-file",
|
|
132
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
133
|
+
`runId=${parsed.runId} taskId=${parsed.taskId}`,
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
void appendEventAsync(loaded.manifest.eventsPath, {
|
|
138
|
+
type: "steer.broker_force",
|
|
139
|
+
runId: parsed.runId,
|
|
140
|
+
taskId: parsed.taskId,
|
|
141
|
+
message: `Foreign/restarted session steer delivered via broker (auth:forced).`,
|
|
142
|
+
data: { taskId: parsed.taskId, auth: "forced" },
|
|
143
|
+
}).catch((err) =>
|
|
144
|
+
logInternalError(
|
|
145
|
+
"crew-broker.steer.force-event",
|
|
146
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
147
|
+
`runId=${parsed.runId} taskId=${parsed.taskId}`,
|
|
148
|
+
),
|
|
149
|
+
);
|
|
150
|
+
deps.sendResult(conn, id, { ok: true, taskId: parsed.taskId, messageId, deduped, auth: "forced" });
|
|
151
|
+
}
|
|
@@ -80,6 +80,16 @@ export interface CrewBrokerOptions {
|
|
|
80
80
|
/** ADR-5 §9: mirrors config limits.serializeOnPathOverlap for the workspace
|
|
81
81
|
* admission gate. Default false. */
|
|
82
82
|
serializeOnPathOverlap?: boolean;
|
|
83
|
+
/** HƯỚNG A (ADR Option 3, 2026-11-01): the DISPATCHER's public coordination
|
|
84
|
+
* verify key (base64 SPKI DER Ed25519) bound HEAP-ONLY at construct time —
|
|
85
|
+
* the trust anchor for `respond.append` challenge-response auth. Sourced by
|
|
86
|
+
* the lifecycle controller from the stdin-threaded dispatcher-key-registry
|
|
87
|
+
* (detached-runner start options) or, absent that, from THIS process's own
|
|
88
|
+
* keypair (broker host == dispatcher). NEVER from the manifest file
|
|
89
|
+
* (unauthenticated, worker-writable — w1a NO-GO). Invalid material fails
|
|
90
|
+
* closed to undefined: respond.append answers "no binding" instead of
|
|
91
|
+
* trusting a malformed key. */
|
|
92
|
+
dispatcherVerifyKey?: string;
|
|
83
93
|
}
|
|
84
94
|
|
|
85
95
|
/** Per-connection server-side state. */
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* guards against control chars / oversized names reaching the dispatcher.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
-
import type { MailboxMessageKind, MailboxMessagePriority } from "../../../state/coordination/mailbox.ts";
|
|
11
|
+
import type { MailboxDeliveryMode, MailboxMessageKind, MailboxMessagePriority } from "../../../state/coordination/mailbox.ts";
|
|
12
12
|
|
|
13
13
|
/** Protocol version negotiated at `hello` time. Bump on breaking change. */
|
|
14
14
|
export const BROKER_PROTOCOL = 1;
|
|
@@ -264,3 +264,139 @@ export function parseWaitResolveParams(value: unknown): WaitResolveParams | unde
|
|
|
264
264
|
if (v.outcome !== undefined && v.outcome !== "answered" && v.outcome !== "timed-out" && v.outcome !== "aborted") return undefined;
|
|
265
265
|
return { to: v.to, questionId: v.questionId, ...(v.outcome !== undefined ? { outcome: v.outcome } : {}) };
|
|
266
266
|
}
|
|
267
|
+
|
|
268
|
+
// ============================================================================
|
|
269
|
+
// HƯỚNG A (ADR Option 3, 2026-11-01) — broker respond.* parameter parsers +
|
|
270
|
+
// the challenge-response canonical payload CONTRACT.
|
|
271
|
+
//
|
|
272
|
+
// `respond.challenge` / `respond.append` / `respond.force` are deliberately
|
|
273
|
+
// reachable PRE-hello (see CrewBroker.dispatchFrame): the whole point is that
|
|
274
|
+
// the responder may hold NO broker token at all — a dispatcher proving key
|
|
275
|
+
// possession (challenge-response) or a foreign session (force, gated by a
|
|
276
|
+
// live park). Because a pre-hello connection carries no runId, every params
|
|
277
|
+
// shape carries `runId` explicitly.
|
|
278
|
+
// ============================================================================
|
|
279
|
+
|
|
280
|
+
/** Bounded respond body (defense-in-depth under the 256 KiB frame cap). */
|
|
281
|
+
export const RESPOND_BODY_MAX_CHARS = 65_536;
|
|
282
|
+
/** Nonce shape: the broker issues 32 random bytes base64url (43 chars); the
|
|
283
|
+
* parser accepts a bounded envelope so future issuers stay compatible. */
|
|
284
|
+
export const RESPOND_NONCE_MIN_CHARS = 16;
|
|
285
|
+
export const RESPOND_NONCE_MAX_CHARS = 128;
|
|
286
|
+
/** Ed25519 signature, base64 (64-byte sig = 88 chars); bounded envelope. */
|
|
287
|
+
export const RESPOND_SIG_MAX_CHARS = 256;
|
|
288
|
+
|
|
289
|
+
/** Fields shared by respond.append and respond.force params. */
|
|
290
|
+
export interface RespondCommonParams {
|
|
291
|
+
runId: string;
|
|
292
|
+
taskId: string;
|
|
293
|
+
questionId: string;
|
|
294
|
+
body: string;
|
|
295
|
+
replyTo?: string;
|
|
296
|
+
priority?: MailboxMessagePriority;
|
|
297
|
+
deliveryMode?: MailboxDeliveryMode;
|
|
298
|
+
/** Optional client idempotency key; the broker defaults it to the
|
|
299
|
+
* questionId (exactly-one response row per question on the broker paths). */
|
|
300
|
+
requestId?: string;
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
export interface RespondChallengeParams {
|
|
304
|
+
runId: string;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
export interface RespondAppendParams extends RespondCommonParams {
|
|
308
|
+
nonce: string;
|
|
309
|
+
sig: string;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export type RespondForceParams = RespondCommonParams;
|
|
313
|
+
|
|
314
|
+
function parseRespondCommonFields(v: Record<string, unknown>): RespondCommonParams | undefined {
|
|
315
|
+
if (typeof v.runId !== "string" || v.runId.length === 0 || v.runId.length > 256) return undefined;
|
|
316
|
+
if (typeof v.taskId !== "string" || v.taskId.length === 0 || v.taskId.length > 256) return undefined;
|
|
317
|
+
if (typeof v.questionId !== "string" || v.questionId.length === 0 || v.questionId.length > 128) return undefined;
|
|
318
|
+
if (typeof v.body !== "string" || v.body.length === 0 || v.body.length > RESPOND_BODY_MAX_CHARS) return undefined;
|
|
319
|
+
const replyTo = v.replyTo;
|
|
320
|
+
if (replyTo !== undefined && (typeof replyTo !== "string" || replyTo.length === 0 || replyTo.length > 256)) return undefined;
|
|
321
|
+
const priority = v.priority as MailboxMessagePriority | undefined;
|
|
322
|
+
if (priority !== undefined && !["urgent", "normal", "low"].includes(priority)) return undefined;
|
|
323
|
+
const deliveryMode = v.deliveryMode as MailboxDeliveryMode | undefined;
|
|
324
|
+
if (deliveryMode !== undefined && deliveryMode !== "interrupt" && deliveryMode !== "next_turn") return undefined;
|
|
325
|
+
const rawRequestId = v.requestId;
|
|
326
|
+
if (rawRequestId !== undefined && (typeof rawRequestId !== "string" || !/^[\w:.-]{1,256}$/.test(rawRequestId))) return undefined;
|
|
327
|
+
return {
|
|
328
|
+
runId: v.runId,
|
|
329
|
+
taskId: v.taskId,
|
|
330
|
+
questionId: v.questionId,
|
|
331
|
+
body: v.body,
|
|
332
|
+
...(replyTo !== undefined ? { replyTo } : {}),
|
|
333
|
+
...(priority !== undefined ? { priority } : {}),
|
|
334
|
+
...(deliveryMode !== undefined ? { deliveryMode } : {}),
|
|
335
|
+
...(rawRequestId !== undefined ? { requestId: rawRequestId } : {}),
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
export function parseRespondChallengeParams(value: unknown): RespondChallengeParams | undefined {
|
|
340
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
341
|
+
const v = value as Record<string, unknown>;
|
|
342
|
+
if (typeof v.runId !== "string" || v.runId.length === 0 || v.runId.length > 256) return undefined;
|
|
343
|
+
return { runId: v.runId };
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
export function parseRespondAppendParams(value: unknown): RespondAppendParams | undefined {
|
|
347
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
348
|
+
const v = value as Record<string, unknown>;
|
|
349
|
+
const common = parseRespondCommonFields(v);
|
|
350
|
+
if (!common) return undefined;
|
|
351
|
+
if (typeof v.nonce !== "string" || v.nonce.length < RESPOND_NONCE_MIN_CHARS || v.nonce.length > RESPOND_NONCE_MAX_CHARS) {
|
|
352
|
+
return undefined;
|
|
353
|
+
}
|
|
354
|
+
if (typeof v.sig !== "string" || v.sig.length === 0 || v.sig.length > RESPOND_SIG_MAX_CHARS) return undefined;
|
|
355
|
+
return { ...common, nonce: v.nonce, sig: v.sig };
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
export function parseRespondForceParams(value: unknown): RespondForceParams | undefined {
|
|
359
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
360
|
+
return parseRespondCommonFields(value as Record<string, unknown>);
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** HƯỚNG A CR CONTRACT — canonical payload string over the client-supplied
|
|
364
|
+
* respond fields, in the STYLE of mailbox.ts's canonicalMailboxPayload
|
|
365
|
+
* (fixed field order, undefined → null; that helper is module-private, so
|
|
366
|
+
* the respond surface mirrors its discipline here rather than importing
|
|
367
|
+
* it). The dispatcher signs THIS string (prefixed by the nonce — see
|
|
368
|
+
* respondAppendSignInput) and the broker verifies the exact same bytes, so
|
|
369
|
+
* writer and verifier can never drift.
|
|
370
|
+
*
|
|
371
|
+
* Field order: taskId, questionId, body, replyTo, priority, deliveryMode,
|
|
372
|
+
* requestId. CONVERGED (2026-11-01, cross-slice alignment): NO `runId` and
|
|
373
|
+
* NO separator — the client implementation (respond-delivery.ts) follows
|
|
374
|
+
* the ADR's literal `sign(nonce ∥ canonical payload)`; run binding is
|
|
375
|
+
* enforced by the runId-bound single-use nonce record (respond.challenge
|
|
376
|
+
* binds runId; respond.append requires record.runId === params.runId), and
|
|
377
|
+
* the appended row's runId comes from the server-side manifest load, never
|
|
378
|
+
* from the client. Client signers SHOULD build this via the exported
|
|
379
|
+
* respondAppendSignInput rather than re-implementing it. */
|
|
380
|
+
export function canonicalRespondAppendPayload(params: RespondCommonParams): string {
|
|
381
|
+
const pick = (value: unknown): unknown => (value === undefined ? null : value);
|
|
382
|
+
return JSON.stringify([
|
|
383
|
+
params.taskId,
|
|
384
|
+
params.questionId,
|
|
385
|
+
params.body,
|
|
386
|
+
pick(params.replyTo),
|
|
387
|
+
pick(params.priority),
|
|
388
|
+
pick(params.deliveryMode),
|
|
389
|
+
pick(params.requestId),
|
|
390
|
+
]);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/** HƯỚNG A CR CONTRACT — the exact bytes the dispatcher's Ed25519 private key
|
|
394
|
+
* signs: `${nonce}${canonicalRespondAppendPayload(params)}` (UTF-8, plain
|
|
395
|
+
* concatenation — the ADR's `nonce ∥ canonical payload`; unambiguous since
|
|
396
|
+
* the nonce is base64url and never contains the canonical's leading `[`).
|
|
397
|
+
* The broker's respond.append handler verifies against the bound public
|
|
398
|
+
* key over this same string. Signers on the client side MUST build it via
|
|
399
|
+
* this export (never re-implement) so the two ends cannot drift. */
|
|
400
|
+
export function respondAppendSignInput(nonce: string, params: RespondCommonParams): string {
|
|
401
|
+
return `${nonce}${canonicalRespondAppendPayload(params)}`;
|
|
402
|
+
}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* runner-broker.ts — FINDING D (2026-11-01): async runs previously pointed
|
|
3
|
+
* every worker at the DISPATCHER's broker socket (stdin payload
|
|
4
|
+
* `payload.socketPath`). A `pi -p` dispatcher exits seconds after async
|
|
5
|
+
* dispatch; session_shutdown stops that broker and unlinks its socket — every
|
|
6
|
+
* later broker op from the still-running run (ask parks, msg.notify, respond
|
|
7
|
+
* RPC, delegate nesting) then hit connect ENOENT. Live probes (8/9) caught
|
|
8
|
+
* the vanish via the manifest stamp: stamped path served the park, then
|
|
9
|
+
* vanished while the runner kept heartbeating — the broker host was never the
|
|
10
|
+
* runner, it was the departed dispatcher.
|
|
11
|
+
*
|
|
12
|
+
* The detached runner is the only process guaranteed to outlive the run, so
|
|
13
|
+
* IT hosts the run's broker from now on:
|
|
14
|
+
* - fresh `runner-${runId}` sessionId → fresh socket (no collision with any
|
|
15
|
+
* live session; a leftover from a crashed runner is stale-removed at bind)
|
|
16
|
+
* - per-task COMPOUND tokens minted in the runner heap — the same trust
|
|
17
|
+
* domain as the stdin-payload pre-mints they replace (creds crossed one
|
|
18
|
+
* pipe either way); fail-closed on missing taskId (w1a
|
|
19
|
+
* broker-bare-run-token-forged-task-identity, mirroring both prior issuers)
|
|
20
|
+
* - `dispatcherVerifyKey` from the stdin payload stays the respond.append CR
|
|
21
|
+
* trust anchor: signatures still verify against the DISPATCHER's keypair,
|
|
22
|
+
* so the alive-dispatcher path is unchanged; respond.force keeps its
|
|
23
|
+
* token-less one-shot for a live park
|
|
24
|
+
* - construction mirrors the lifecycle host (lifecycle-handlers.ts getOrStart
|
|
25
|
+
* Broker) config-for-config — waitMethods/nesting/limits/catalog — so the
|
|
26
|
+
* capability surface is identical on either host
|
|
27
|
+
* - the runner's finally → runCleanup() stops it → ownership-gated unlink
|
|
28
|
+
*
|
|
29
|
+
* Failure to start fails SOFT and returns undefined: the caller keeps the
|
|
30
|
+
* legacy payload issuer (dispatcher socket + pre-minted tokens), which is
|
|
31
|
+
* exactly the pre-Finding-D behavior (correct while the dispatcher lives).
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { loadConfig } from "../../config/config.ts";
|
|
35
|
+
import { getCrewEnv } from "../../config/env-vars.ts";
|
|
36
|
+
import { ensureCoordinationPublicVerifyKey } from "../../state/coordination/mailbox.ts";
|
|
37
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
38
|
+
import { getBrokerSocketPath } from "../../utils/socket-path.ts";
|
|
39
|
+
import { configuredModelInfosFromPiConfig } from "../model/model-fallback.ts";
|
|
40
|
+
import type { BrokerSpawnCredentials } from "./broker-issuer.ts";
|
|
41
|
+
import { CrewBroker } from "./crew-broker.ts";
|
|
42
|
+
import { getActiveDispatcherVerifyKey, setActiveDispatcherVerifyKey } from "./dispatcher-key-registry.ts";
|
|
43
|
+
|
|
44
|
+
/** Handle on the runner-hosted broker handed back to background-runner. */
|
|
45
|
+
export interface RunnerHostedBrokerHandle {
|
|
46
|
+
/** The live broker (already started). */
|
|
47
|
+
broker: CrewBroker;
|
|
48
|
+
/** Resolved listening socket path (safe to stamp into the manifest). */
|
|
49
|
+
socketPath: string;
|
|
50
|
+
/** BrokerSpawnCredentials issuer — SAME contract as the stdin-payload
|
|
51
|
+
* issuer it replaces (fail-closed on missing taskId, depth-capped). */
|
|
52
|
+
issueForChild: (runId: string, taskId?: string, childDepth?: number) => Promise<BrokerSpawnCredentials | undefined>;
|
|
53
|
+
/** Ownership-gated stop (idempotent). */
|
|
54
|
+
stop: () => Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Mirror of the lifecycle host's effectiveEnabled gate (env wins, config
|
|
58
|
+
* default-on since v0.9.47). Never throws. */
|
|
59
|
+
function brokerEnabled(cwd: string): boolean {
|
|
60
|
+
const envOverride = getCrewEnv("PI_CREW_BROKER");
|
|
61
|
+
if (envOverride === "0") return false;
|
|
62
|
+
try {
|
|
63
|
+
const cfg = loadConfig(cwd).config.broker;
|
|
64
|
+
if (envOverride === "1") return cfg !== undefined ? cfg.enabled !== false : true;
|
|
65
|
+
return cfg?.enabled !== false;
|
|
66
|
+
} catch {
|
|
67
|
+
// Fail-safe: config load failed → keep the broker off (legacy issuer).
|
|
68
|
+
return false;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Start a broker hosted by THIS detached-runner process for `runId`.
|
|
74
|
+
* Returns undefined (soft-fail) when the broker is disabled or fails to
|
|
75
|
+
* start — callers must then keep the legacy stdin-payload issuer.
|
|
76
|
+
*/
|
|
77
|
+
export async function startRunnerHostedBroker(input: {
|
|
78
|
+
runId: string;
|
|
79
|
+
cwd: string;
|
|
80
|
+
/** Optional socket-path override (test seam / explicit placement). */
|
|
81
|
+
socketPath?: string;
|
|
82
|
+
/** The DISPATCHER's public verify key from the stdin payload (CR anchor).
|
|
83
|
+
* Falls back to this process's own keypair so a manifest-stamped
|
|
84
|
+
* responder always has SOME binding (same fallback as the lifecycle
|
|
85
|
+
* host; public material only — w1a-safe). */
|
|
86
|
+
dispatcherVerifyKey?: string;
|
|
87
|
+
/** Depth-cap resolver (PI_CREW_MAX_DEPTH semantics) — injected so this
|
|
88
|
+
* module stays free of the model-registry import chain at load time. */
|
|
89
|
+
resolveMaxDepth: () => number;
|
|
90
|
+
}): Promise<RunnerHostedBrokerHandle | undefined> {
|
|
91
|
+
const { runId, cwd } = input;
|
|
92
|
+
if (typeof runId !== "string" || runId.length === 0) return undefined;
|
|
93
|
+
if (!brokerEnabled(cwd)) return undefined;
|
|
94
|
+
|
|
95
|
+
// Config reads mirror lifecycle-handlers.ts getOrStartBroker (same cwd
|
|
96
|
+
// discipline — B1 battery 2026-08-18: loadConfig WITHOUT cwd silently
|
|
97
|
+
// drops the workspace layer where broker.waitMethodsEnabled lives).
|
|
98
|
+
const cfg = (() => {
|
|
99
|
+
try {
|
|
100
|
+
return loadConfig(cwd).config.broker;
|
|
101
|
+
} catch {
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
})();
|
|
105
|
+
const nestingCfg = (() => {
|
|
106
|
+
try {
|
|
107
|
+
const c = loadConfig(cwd).config;
|
|
108
|
+
return { nesting: c.nesting, limits: c.limits };
|
|
109
|
+
} catch {
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
})();
|
|
113
|
+
|
|
114
|
+
const sessionId = `runner-${runId}`;
|
|
115
|
+
const socketPath = input.socketPath ?? getBrokerSocketPath(sessionId);
|
|
116
|
+
|
|
117
|
+
// HƯỚNG A CR anchor: the payload key wins (the true dispatcher); a run
|
|
118
|
+
// dispatched without one still gets a self-consistent binding. Bind it in
|
|
119
|
+
// the registry too so broker-side CR verification and any in-process
|
|
120
|
+
// responders agree on ONE anchor.
|
|
121
|
+
const verifyKey = input.dispatcherVerifyKey ?? getActiveDispatcherVerifyKey() ?? ensureCoordinationPublicVerifyKey();
|
|
122
|
+
setActiveDispatcherVerifyKey(verifyKey);
|
|
123
|
+
|
|
124
|
+
const broker = new CrewBroker({
|
|
125
|
+
sessionId,
|
|
126
|
+
socketPath,
|
|
127
|
+
maxFrameBytes: cfg?.maxFrameBytes ?? 262144,
|
|
128
|
+
outboundQueueCap: cfg?.outboundQueueCap ?? 256,
|
|
129
|
+
waitMethodsEnabled: cfg?.waitMethodsEnabled ?? false,
|
|
130
|
+
nestingEnabled: nestingCfg?.nesting?.enabled ?? false,
|
|
131
|
+
...(nestingCfg?.nesting?.maxSlots !== undefined ? { nestingMaxSlots: nestingCfg.nesting.maxSlots } : {}),
|
|
132
|
+
...(nestingCfg?.nesting?.maxDepth !== undefined ? { nestingMaxDepth: nestingCfg.nesting.maxDepth } : {}),
|
|
133
|
+
nestingTrustedEscalation: nestingCfg?.nesting?.enabled === true,
|
|
134
|
+
...(nestingCfg?.limits?.maxConcurrentWorkers !== undefined
|
|
135
|
+
? { globalWorkerSemaphore: nestingCfg.limits.maxConcurrentWorkers }
|
|
136
|
+
: {}),
|
|
137
|
+
serializeOnPathOverlap: nestingCfg?.limits?.serializeOnPathOverlap ?? false,
|
|
138
|
+
dispatcherVerifyKey: verifyKey,
|
|
139
|
+
modelCatalog: () => {
|
|
140
|
+
try {
|
|
141
|
+
return configuredModelInfosFromPiConfig(cwd).map((info) => info.fullId);
|
|
142
|
+
} catch {
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
enabled: true,
|
|
147
|
+
cwd,
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
try {
|
|
151
|
+
await broker.start();
|
|
152
|
+
} catch (error) {
|
|
153
|
+
logInternalError(
|
|
154
|
+
"runner-broker.start.failed",
|
|
155
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
156
|
+
`runId=${runId} socket=${socketPath}`,
|
|
157
|
+
);
|
|
158
|
+
// Soft-fail: best-effort stop to release anything partially bound.
|
|
159
|
+
await broker.stop().catch(() => {});
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Payload-parity token cache: repeated spawns of the SAME task reuse the
|
|
164
|
+
// SAME compound token (the stdin issuer served one pre-mint per task). A
|
|
165
|
+
// revokeTaskToken (surface-degrade T10) + re-ask cycle mints a fresh one.
|
|
166
|
+
const mintedTokens = new Map<string, string>();
|
|
167
|
+
|
|
168
|
+
const issueForChild = async (rid: string, taskId?: string, childDepth?: number): Promise<BrokerSpawnCredentials | undefined> => {
|
|
169
|
+
if (rid !== runId) return undefined;
|
|
170
|
+
if (childDepth !== undefined && childDepth >= input.resolveMaxDepth()) return undefined;
|
|
171
|
+
// w1a fail-closed: no taskId → no creds (mirrors both prior issuers).
|
|
172
|
+
if (!taskId) return undefined;
|
|
173
|
+
const cached = mintedTokens.get(taskId);
|
|
174
|
+
if (cached !== undefined) return { socketPath, token: cached };
|
|
175
|
+
const token = broker.issueRunToken(rid, taskId);
|
|
176
|
+
mintedTokens.set(taskId, token);
|
|
177
|
+
return { socketPath, token };
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
return {
|
|
181
|
+
broker,
|
|
182
|
+
socketPath,
|
|
183
|
+
issueForChild,
|
|
184
|
+
stop: async () => {
|
|
185
|
+
mintedTokens.clear();
|
|
186
|
+
await broker.stop();
|
|
187
|
+
},
|
|
188
|
+
};
|
|
189
|
+
}
|
|
@@ -41,12 +41,21 @@ import { logInternalError } from "../../utils/internal-error.ts";
|
|
|
41
41
|
* authenticates the connection but waitAuthError rejects its parks with
|
|
42
42
|
* `forbidden`. The dispatching session pre-mints a compound token for every
|
|
43
43
|
* task in the manifest (they exist before dispatch); dynamic-workflow tasks
|
|
44
|
-
* planned inside the runner get NO creds (follow-up: broker mint RPC).
|
|
44
|
+
* planned inside the runner get NO creds (follow-up: broker mint RPC).
|
|
45
|
+
*
|
|
46
|
+
* HƯỚNG A (ADR Option 3, 2026-11-01): optional `dispatcherVerifyKey` — the
|
|
47
|
+
* SPAWNING (dispatcher) process's PUBLIC coordination verify key, derived in
|
|
48
|
+
* its heap at spawn time. This is the runner START-OPTIONS channel the broker
|
|
49
|
+
* CR binding is threaded through (env allowlist is closed by design; the
|
|
50
|
+
* manifest file is a w1a NO-GO for auth bindings — worker-writable). PUBLIC
|
|
51
|
+
* material only; additive optional field, so v2 stays the wire version (old
|
|
52
|
+
* writer/reader pairs ignore it safely). */
|
|
45
53
|
export interface StdinBrokerPayload {
|
|
46
54
|
v: 2;
|
|
47
55
|
runId: string;
|
|
48
56
|
socketPath: string;
|
|
49
57
|
tasks: Record<string, string>;
|
|
58
|
+
dispatcherVerifyKey?: string;
|
|
50
59
|
}
|
|
51
60
|
|
|
52
61
|
/** Parse + validate one handshake line. Rejects wrong version, wrong run
|
|
@@ -64,7 +73,22 @@ export function parseStdinBrokerPayload(raw: string, expectedRunId: string): Std
|
|
|
64
73
|
for (const [id, tok] of Object.entries(tasks)) {
|
|
65
74
|
if (id.length === 0 || typeof tok !== "string" || tok.length === 0) return undefined;
|
|
66
75
|
}
|
|
67
|
-
|
|
76
|
+
// HƯỚNG A: optional dispatcher public verify key (base64 SPKI DER envelope).
|
|
77
|
+
// Invalid material drops the FIELD (creds still parse) — the key is
|
|
78
|
+
// authority-bearing, so a mangled value must fail closed at the later
|
|
79
|
+
// bind site rather than half-trust it here.
|
|
80
|
+
const rawKey = o.dispatcherVerifyKey;
|
|
81
|
+
const dispatcherVerifyKey =
|
|
82
|
+
typeof rawKey === "string" && rawKey.length >= 32 && rawKey.length <= 128 && /^[A-Za-z0-9+/=]+$/.test(rawKey)
|
|
83
|
+
? rawKey
|
|
84
|
+
: undefined;
|
|
85
|
+
return {
|
|
86
|
+
v: 2,
|
|
87
|
+
runId: o.runId,
|
|
88
|
+
socketPath: o.socketPath,
|
|
89
|
+
tasks: tasks as Record<string, string>,
|
|
90
|
+
...(dispatcherVerifyKey !== undefined ? { dispatcherVerifyKey } : {}),
|
|
91
|
+
};
|
|
68
92
|
} catch {
|
|
69
93
|
return undefined;
|
|
70
94
|
}
|
|
@@ -89,8 +113,19 @@ export function parseStdinBrokerPayload(raw: string, expectedRunId: string): Std
|
|
|
89
113
|
* returned (and written by the caller) — never blocked. */
|
|
90
114
|
export const BROKER_STDIN_LINE_WARN_BYTES = 60 * 1024;
|
|
91
115
|
|
|
92
|
-
export function buildBrokerStdinLine(
|
|
93
|
-
|
|
116
|
+
export function buildBrokerStdinLine(
|
|
117
|
+
runId: string,
|
|
118
|
+
socketPath: string,
|
|
119
|
+
tasks: Record<string, string>,
|
|
120
|
+
dispatcherVerifyKey?: string,
|
|
121
|
+
): string {
|
|
122
|
+
const line = `${JSON.stringify({
|
|
123
|
+
v: 2,
|
|
124
|
+
runId,
|
|
125
|
+
socketPath,
|
|
126
|
+
tasks,
|
|
127
|
+
...(dispatcherVerifyKey ? { dispatcherVerifyKey } : {}),
|
|
128
|
+
})}\n`;
|
|
94
129
|
if (line.length > BROKER_STDIN_LINE_WARN_BYTES) {
|
|
95
130
|
logInternalError(
|
|
96
131
|
"stdin-handshake.oversize-payload",
|
|
@@ -161,7 +161,7 @@ export async function runWorker(input: WorkerSpawnInput): Promise<ChildPiRunResu
|
|
|
161
161
|
}
|
|
162
162
|
if (cap) {
|
|
163
163
|
// RR-014 / F15: thread the caller's signal into the slot WAIT itself.
|
|
164
|
-
//
|
|
164
|
+
// previously the signal lived only inside childPiInput and was read by
|
|
165
165
|
// runChildPi AFTER the acquire resolved — a cancelled task queued on a
|
|
166
166
|
// full pool stayed pending until the slot holder finished (and delayed
|
|
167
167
|
// every drainPendingUnits awaiting it). Two consistent outcomes:
|
|
@@ -171,7 +171,22 @@ export async function runWorker(input: WorkerSpawnInput): Promise<ChildPiRunResu
|
|
|
171
171
|
// guard (child-pi-spawn.ts B5) sees signal.aborted and returns
|
|
172
172
|
// kind "aborted" WITHOUT spawning, and withWorkerSlot's finally
|
|
173
173
|
// releases the slot within a microtask.
|
|
174
|
-
|
|
174
|
+
//
|
|
175
|
+
// spawn.trace (2026-11-01, run team_20261101074334 forensics): the slot
|
|
176
|
+
// WAIT is a prime log-silent hang candidate (unbounded on a full pool).
|
|
177
|
+
// Measure acquire → fn-start; >5s gets a breadcrumb so the next hang is
|
|
178
|
+
// attributable (slot starvation vs in-child phases) from background.log.
|
|
179
|
+
const slotWaitStartedAt = Date.now();
|
|
180
|
+
let slotAcquiredAt: number | undefined;
|
|
181
|
+
const waitTrace = (label: string): void => {
|
|
182
|
+
if (slotAcquiredAt === undefined) return;
|
|
183
|
+
const waitedMs = slotAcquiredAt - slotWaitStartedAt;
|
|
184
|
+
if (waitedMs > 5_000) console.error(`[spawn.trace] worker-slot wait=${waitedMs}ms then ${label}`);
|
|
185
|
+
};
|
|
186
|
+
return withWorkerSlot(() => {
|
|
187
|
+
slotAcquiredAt = Date.now();
|
|
188
|
+
return runChildPi(childPiInput).finally(() => waitTrace("runChildPi-settled"));
|
|
189
|
+
}, childPiInput.signal);
|
|
175
190
|
}
|
|
176
191
|
return runChildPi(childPiInput);
|
|
177
192
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* spawn-phase-trace.ts — Observability for pre-worker-spawn hangs.
|
|
3
|
+
*
|
|
4
|
+
* LIVE INCIDENT (2026-11-01, run team_20261101074334, probe19): 02_execute's
|
|
5
|
+
* first worker spawn hung ~7 MINUTES between task.started (07:50:05) and the
|
|
6
|
+
* retry respawn (07:57:05) with ZERO trace — no worker.spawned, no
|
|
7
|
+
* checkpoint childPid, no background.log line in the window. The candidate
|
|
8
|
+
* blocking spans are all log-silent today: the worker-slot semaphore wait
|
|
9
|
+
* (runWorker → withWorkerSlot — waits unboundedly on a full pool), the
|
|
10
|
+
* model-registry resolution chain, and the child spawn env build. The
|
|
11
|
+
* heartbeat watcher correctly flagged "stuck worker" 5 minutes in, but
|
|
12
|
+
* forensics could not bisect WHERE it hung.
|
|
13
|
+
*
|
|
14
|
+
* This tracer closes that gap: a 30s unref'd interval that prints one line
|
|
15
|
+
* per interval to the runner's background.log naming the CURRENT phase and
|
|
16
|
+
* elapsed time. A healthy spawn (<30s) logs NOTHING — zero noise; a hang
|
|
17
|
+
* leaves a breadcrumb every 30s until it resolves.
|
|
18
|
+
*
|
|
19
|
+
* Ticks print via console.error (the background runner's stderr →
|
|
20
|
+
* background.log — same channel pi-spawn ENOENT warnings already use).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
export interface SpawnPhaseTracer {
|
|
24
|
+
/** Advance to the next phase label (idempotent end() clears the timer). */
|
|
25
|
+
phase(label: string): void;
|
|
26
|
+
/** Stop ticking (call when the worker spawned / the attempt settled). */
|
|
27
|
+
end(): void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const TICK_MS = 30_000;
|
|
31
|
+
|
|
32
|
+
export function createSpawnPhaseTracer(taskId: string, attempt: number, tickMs: number = TICK_MS): SpawnPhaseTracer {
|
|
33
|
+
const startedAt = Date.now();
|
|
34
|
+
let currentPhase = "init";
|
|
35
|
+
let timer: NodeJS.Timeout | undefined;
|
|
36
|
+
const tick = (): void => {
|
|
37
|
+
console.error(`[spawn.trace] task=${taskId} attempt=${attempt} phase=${currentPhase} elapsedMs=${Date.now() - startedAt}`);
|
|
38
|
+
};
|
|
39
|
+
// First tick fires only after tickMs — sub-tick spawns stay silent.
|
|
40
|
+
timer = setInterval(tick, tickMs);
|
|
41
|
+
timer.unref?.();
|
|
42
|
+
return {
|
|
43
|
+
phase(label: string): void {
|
|
44
|
+
currentPhase = label;
|
|
45
|
+
},
|
|
46
|
+
end(): void {
|
|
47
|
+
if (timer !== undefined) {
|
|
48
|
+
clearInterval(timer);
|
|
49
|
+
timer = undefined;
|
|
50
|
+
}
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|