pi-crew 0.11.55 → 0.11.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +136 -0
- package/dist/index.mjs +530 -529
- package/package.json +1 -1
- package/src/extension/registration/lifecycle-handlers.ts +10 -1
- package/src/extension/team-tool/respond-delivery.ts +429 -0
- package/src/extension/team-tool/respond.ts +135 -50
- package/src/extension/team-tool.ts +41 -1
- package/src/runtime/async-runner.ts +10 -1
- package/src/runtime/background-runner.ts +93 -17
- package/src/runtime/broker/crew-broker.ts +146 -1
- package/src/runtime/broker/dispatcher-key-registry.ts +39 -0
- package/src/runtime/broker/protocol/broker-respond.ts +365 -0
- package/src/runtime/broker/protocol/broker-steer.ts +151 -0
- package/src/runtime/broker/protocol/connection-state.ts +10 -0
- package/src/runtime/broker/protocol/request-parsers.ts +137 -1
- package/src/runtime/broker/runner-broker.ts +189 -0
- package/src/runtime/broker/stdin-handshake.ts +39 -4
- package/src/runtime/run-worker.ts +17 -2
- package/src/runtime/spawn-phase-trace.ts +53 -0
- package/src/runtime/task-runner/child-executor.ts +14 -0
- package/src/runtime/task-runner/state-helpers.ts +32 -0
- package/src/state/contracts.ts +15 -0
- package/src/state/coordination/mailbox.ts +15 -0
- package/src/state/types.ts +11 -0
|
@@ -19,6 +19,11 @@
|
|
|
19
19
|
* `delegate.request` (governed-nesting grandchild spawn through
|
|
20
20
|
* `delegate-spawn.ts`, capability-gated via `options.nestingEnabled`,
|
|
21
21
|
* fail-closed). Unknown methods return a typed `not-implemented`.
|
|
22
|
+
* HƯỚNG A (ADR Option 3, 2026-11-01): `respond.challenge` /
|
|
23
|
+
* `respond.append` / `respond.force` are additionally reachable
|
|
24
|
+
* PRE-hello — their auth is the dispatcher challenge-response (bound
|
|
25
|
+
* heap-only public key + single-use nonce) or the live-park one-shot
|
|
26
|
+
* force gate, never a hello token.
|
|
22
27
|
* - Mailbox/event persistence flows through the `src/state/` stores
|
|
23
28
|
* (`appendMailboxMessage*`, event-log) — the broker itself keeps no
|
|
24
29
|
* durable state of its own.
|
|
@@ -36,7 +41,7 @@
|
|
|
36
41
|
* described.
|
|
37
42
|
*/
|
|
38
43
|
|
|
39
|
-
import { randomUUID } from "node:crypto";
|
|
44
|
+
import { createPublicKey, randomUUID } from "node:crypto";
|
|
40
45
|
import * as fsp from "node:fs/promises";
|
|
41
46
|
import * as net from "node:net";
|
|
42
47
|
import {
|
|
@@ -61,6 +66,8 @@ import { type BrokerToken, BrokerTokenRegistry } from "./crew-broker-tokens.ts";
|
|
|
61
66
|
import { DelegateAbortRegistry } from "./delegate/delegate-abort.ts";
|
|
62
67
|
import { type DelegateEventTarget, type DelegateEventType, recordDelegateEvent } from "./delegate/delegate-event.ts";
|
|
63
68
|
import { fanoutMailboxMessage } from "./mailbox-observer/mailbox-fanout.ts";
|
|
69
|
+
import { type BrokerRespondDeps, handleRespondMethod, type RespondChallengeRecord } from "./protocol/broker-respond.ts";
|
|
70
|
+
import { handleSteerForce } from "./protocol/broker-steer.ts";
|
|
64
71
|
import type { CrewBrokerOptions, ServerConnection } from "./protocol/connection-state.ts";
|
|
65
72
|
// Outbound-queue management + caps (wc-gate move — bodies in protocol/connection-state.ts)
|
|
66
73
|
import {
|
|
@@ -131,6 +138,7 @@ export class CrewBroker {
|
|
|
131
138
|
| "modelCatalog"
|
|
132
139
|
| "serializeOnPathOverlap"
|
|
133
140
|
| "lockBusyRetryDelaysMs"
|
|
141
|
+
| "dispatcherVerifyKey"
|
|
134
142
|
>;
|
|
135
143
|
private readonly tokens = new BrokerTokenRegistry();
|
|
136
144
|
/** Task 10 (mux-surface A1 §5.2): taskId → the compound token most
|
|
@@ -139,6 +147,21 @@ export class CrewBroker {
|
|
|
139
147
|
* runs; a colliding taskId in another run revokes both, which is the
|
|
140
148
|
* conservative direction). Heap-only like the registry. */
|
|
141
149
|
private readonly taskTokens = new Map<string, BrokerToken>();
|
|
150
|
+
/** HƯỚNG A (ADR Option 3): the DISPATCHER's public verify key (base64 SPKI
|
|
151
|
+
* DER Ed25519) bound HEAP-ONLY at construct time — the trust anchor for
|
|
152
|
+
* respond.append challenge-response auth. Never manifest/env/disk; see
|
|
153
|
+
* dispatcher-key-registry.ts for the threading. */
|
|
154
|
+
private readonly dispatcherVerifyKey: string | undefined;
|
|
155
|
+
/** HƯỚNG A: outstanding respond.append challenge per connection.
|
|
156
|
+
* WeakMap — the challenge dies with the connection. HEAP ONLY. */
|
|
157
|
+
private readonly respondChallenges = new WeakMap<ServerConnection, RespondChallengeRecord>();
|
|
158
|
+
/** HƯỚNG A: challenge issuance counter per connection (rate bound). */
|
|
159
|
+
private readonly respondChallengeCounts = new WeakMap<ServerConnection, number>();
|
|
160
|
+
/** HƯỚNG A: broker-answered questionIds (`${runId}:${questionId}`) — the
|
|
161
|
+
* heap-side fast-fail of the one-shot-per-park gate. The DURABLE
|
|
162
|
+
* exactly-one guarantee is the idempotent append (requestId=questionId);
|
|
163
|
+
* this set only cheaply rejects a second broker respond after a success. */
|
|
164
|
+
private readonly respondedQuestionIds = new Set<string>();
|
|
142
165
|
private server: net.Server | null = null;
|
|
143
166
|
private resolvedSocketPath: string | null = null;
|
|
144
167
|
/** [3] stop-unlink race: dev+ino of the leaf we created, captured right
|
|
@@ -193,6 +216,22 @@ export class CrewBroker {
|
|
|
193
216
|
serializeOnPathOverlap: options.serializeOnPathOverlap === true,
|
|
194
217
|
};
|
|
195
218
|
this.waitStatusCache = options.waitStatusCache ?? new WaitStatusCache();
|
|
219
|
+
// HƯỚNG A: validate + bind the dispatcher public key. Invalid material
|
|
220
|
+
// fails CLOSED to undefined (respond.append then answers "no binding")
|
|
221
|
+
// rather than half-trusting a mangled key.
|
|
222
|
+
this.dispatcherVerifyKey = CrewBroker.validateDispatcherVerifyKey(options.dispatcherVerifyKey);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/** HƯỚNG A: accept only parseable base64 SPKI DER key material. */
|
|
226
|
+
private static validateDispatcherVerifyKey(key: string | undefined): string | undefined {
|
|
227
|
+
if (key === undefined) return undefined;
|
|
228
|
+
if (typeof key !== "string" || key.length === 0 || key.length > 256) return undefined;
|
|
229
|
+
try {
|
|
230
|
+
createPublicKey({ key: Buffer.from(key, "base64"), format: "der", type: "spki" });
|
|
231
|
+
return key;
|
|
232
|
+
} catch {
|
|
233
|
+
return undefined;
|
|
234
|
+
}
|
|
196
235
|
}
|
|
197
236
|
|
|
198
237
|
/** Read the resolved socket path. Available after start() resolves. */
|
|
@@ -480,6 +519,9 @@ export class CrewBroker {
|
|
|
480
519
|
this.tokens.clear();
|
|
481
520
|
// Task 10: drop the taskId → token index with it.
|
|
482
521
|
this.taskTokens.clear();
|
|
522
|
+
// HƯỚNG A: drop the respond one-shot marks (challenges are WeakMap-bound
|
|
523
|
+
// to connections and die with them; the binding itself is readonly).
|
|
524
|
+
this.respondedQuestionIds.clear();
|
|
483
525
|
}
|
|
484
526
|
|
|
485
527
|
// Connection lifecycle
|
|
@@ -623,7 +665,26 @@ export class CrewBroker {
|
|
|
623
665
|
|
|
624
666
|
// Hello MUST be the first method. Any other method before hello
|
|
625
667
|
// returns a generic protocol error and closes.
|
|
668
|
+
// EXCEPTION — HƯỚNG A (ADR Option 3): the respond.* surface is
|
|
669
|
+
// deliberately reachable PRE-hello: its auth is the dispatcher
|
|
670
|
+
// challenge-response (respond.append) or the live-park one-shot gate
|
|
671
|
+
// (respond.force), NEVER a hello token — the callers it exists for (a
|
|
672
|
+
// dispatcher re-attaching without a token, a foreign session forcing a
|
|
673
|
+
// stuck park) hold no token at all.
|
|
626
674
|
if (!conn.authed) {
|
|
675
|
+
if (method === "respond.challenge" || method === "respond.append" || method === "respond.force") {
|
|
676
|
+
await this.dispatchRespondMethod(conn, id, method, params);
|
|
677
|
+
return;
|
|
678
|
+
}
|
|
679
|
+
// steer.force (HƯỚNG A follow-up, probe19): token-less live-worker
|
|
680
|
+
// steer for a restarted/foreign session — the broker performs the
|
|
681
|
+
// delivery itself (mailbox row + BROKER-SIGNED steering line),
|
|
682
|
+
// mirroring respond.force's lane. Auth is the same-uid socket domain
|
|
683
|
+
// + the live-task gate + the steer.broker_force audit event.
|
|
684
|
+
if (method === "steer.force") {
|
|
685
|
+
await this.dispatchSteerForce(conn, id, params);
|
|
686
|
+
return;
|
|
687
|
+
}
|
|
627
688
|
if (method !== "hello") {
|
|
628
689
|
this.sendErrorAndClose(conn, id, "protocol", "hello required");
|
|
629
690
|
return;
|
|
@@ -695,6 +756,18 @@ export class CrewBroker {
|
|
|
695
756
|
case "delegate.request":
|
|
696
757
|
await this.handleDelegateRequest(conn, id, params);
|
|
697
758
|
return;
|
|
759
|
+
// HƯỚNG A (ADR Option 3): broker-signed respond delivery — token-less
|
|
760
|
+
// auth paths, also reachable pre-hello (see the dispatchFrame gate
|
|
761
|
+
// above). Hello'd connections hit the worker-role reject inside the
|
|
762
|
+
// handlers (defense-in-depth).
|
|
763
|
+
case "respond.challenge":
|
|
764
|
+
case "respond.append":
|
|
765
|
+
case "respond.force":
|
|
766
|
+
await this.dispatchRespondMethod(conn, id, method, params);
|
|
767
|
+
return;
|
|
768
|
+
case "steer.force":
|
|
769
|
+
await this.dispatchSteerForce(conn, id, params);
|
|
770
|
+
return;
|
|
698
771
|
default:
|
|
699
772
|
// Unhandled method → typed not-implemented.
|
|
700
773
|
this.sendError(conn, id, "not-implemented", `method '${method}' is not implemented`);
|
|
@@ -1425,6 +1498,78 @@ export class CrewBroker {
|
|
|
1425
1498
|
}
|
|
1426
1499
|
}
|
|
1427
1500
|
|
|
1501
|
+
// HƯỚNG A (ADR Option 3, 2026-11-01) — broker-side respond delivery.
|
|
1502
|
+
// Handler bodies live in ./protocol/broker-respond.ts (wc-gate, wait-resolve
|
|
1503
|
+
// pattern); this class owns the heap state + thin delegation. Auth summary:
|
|
1504
|
+
// respond.append = dispatcher challenge-response against the construct-time
|
|
1505
|
+
// bound public key (pre-hello reachable); respond.force = no-credential
|
|
1506
|
+
// one-shot for a LIVE park (worker-token connections rejected).
|
|
1507
|
+
|
|
1508
|
+
/** HƯỚNG A: deps bundle for the protocol handlers (heap state owned here). */
|
|
1509
|
+
private respondDeps(): BrokerRespondDeps {
|
|
1510
|
+
return {
|
|
1511
|
+
sendError: (c, i, code, msg) => this.sendError(c, i, code, msg),
|
|
1512
|
+
sendResult: (c, i, r) => this.sendResult(c, i, r),
|
|
1513
|
+
cwd: this.options.cwd,
|
|
1514
|
+
dispatcherVerifyKey: this.dispatcherVerifyKey,
|
|
1515
|
+
respondChallenges: this.respondChallenges,
|
|
1516
|
+
respondChallengeCounts: this.respondChallengeCounts,
|
|
1517
|
+
respondedQuestionIds: this.respondedQuestionIds,
|
|
1518
|
+
runLockBusyRetry: (manifest, fn) => this.runLockBusyRetry(manifest, fn),
|
|
1519
|
+
socketPath: this.socketPath,
|
|
1520
|
+
};
|
|
1521
|
+
}
|
|
1522
|
+
|
|
1523
|
+
/** HƯỚNG A: method router (pre- and post-hello dispatch sites). */
|
|
1524
|
+
private async dispatchRespondMethod(conn: ServerConnection, id: string, method: string, params: unknown): Promise<void> {
|
|
1525
|
+
await handleRespondMethod(this.respondDeps(), conn, id, method, params);
|
|
1526
|
+
}
|
|
1527
|
+
|
|
1528
|
+
/** HƯỚNG A follow-up (probe19): token-less steer.force for a
|
|
1529
|
+
* restarted/foreign session — wires the broker-side delivery (mailbox
|
|
1530
|
+
* row + BROKER-SIGNED steering line, the exact write pair
|
|
1531
|
+
* handleSteerPush performs for orchestrators). */
|
|
1532
|
+
private async dispatchSteerForce(conn: ServerConnection, id: string, params: unknown): Promise<void> {
|
|
1533
|
+
await handleSteerForce(
|
|
1534
|
+
{
|
|
1535
|
+
sendError: (c, i, code, message) => this.sendError(c, i, code, message),
|
|
1536
|
+
sendResult: (c, i, result) => this.sendResult(c, i, result),
|
|
1537
|
+
appendSteerRow: async (manifest, parsed, messageId) => {
|
|
1538
|
+
const base = {
|
|
1539
|
+
id: messageId,
|
|
1540
|
+
direction: "inbox" as const,
|
|
1541
|
+
from: parsed.runId,
|
|
1542
|
+
to: parsed.taskId,
|
|
1543
|
+
taskId: parsed.taskId,
|
|
1544
|
+
body: parsed.body,
|
|
1545
|
+
kind: "steer" as const,
|
|
1546
|
+
priority: "urgent" as const,
|
|
1547
|
+
deliveryMode: "interrupt" as const,
|
|
1548
|
+
};
|
|
1549
|
+
if (parsed.requestId === undefined) {
|
|
1550
|
+
await appendMailboxMessageAsync(manifest, base);
|
|
1551
|
+
return { deduped: false };
|
|
1552
|
+
}
|
|
1553
|
+
const outcome = await appendMailboxMessageIdempotentAsync(manifest, { ...base, requestId: parsed.requestId });
|
|
1554
|
+
return { deduped: outcome.deduped };
|
|
1555
|
+
},
|
|
1556
|
+
appendSignedSteeringLine: async (manifest, taskId, body, messageId) => {
|
|
1557
|
+
const steeringDir = `${manifest.artifactsRoot}/steering`;
|
|
1558
|
+
const steeringPath = resolveRealContainedPath(manifest.artifactsRoot, `steering/${taskId}.jsonl`);
|
|
1559
|
+
const line =
|
|
1560
|
+
JSON.stringify(signSteeringEntry({ type: "steer", message: body, id: messageId, ts: new Date().toISOString() })) +
|
|
1561
|
+
"\n";
|
|
1562
|
+
await fsp.mkdir(steeringDir, { recursive: true });
|
|
1563
|
+
await fsp.appendFile(steeringPath, line, { encoding: "utf8" });
|
|
1564
|
+
},
|
|
1565
|
+
},
|
|
1566
|
+
conn,
|
|
1567
|
+
id,
|
|
1568
|
+
params,
|
|
1569
|
+
{ cwd: this.options.cwd },
|
|
1570
|
+
);
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1428
1573
|
// WP-2/R2: wait.request / wait.resolve (ADR-0 2026-08-17-waiting-producer-ask)
|
|
1429
1574
|
// waitAuthError: imported directly from protocol/wait-auth.ts (M4/WI-4.1).
|
|
1430
1575
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dispatcher-key-registry.ts — HƯỚNG A (ADR Option 3, 2026-11-01): process-local
|
|
3
|
+
* registry holding the DISPATCHER's PUBLIC coordination verify key for
|
|
4
|
+
* broker-side `respond.append` challenge-response auth.
|
|
5
|
+
*
|
|
6
|
+
* Who writes:
|
|
7
|
+
* - The detached background runner bootstrap (background-runner.ts) when the
|
|
8
|
+
* spawning (dispatcher) process threaded its public key over the stdin
|
|
9
|
+
* start-options handshake (async-runner.ts → buildBrokerStdinLine). The key
|
|
10
|
+
* is derived in the SPAWNER's heap via ensureCoordinationPublicVerifyKey()
|
|
11
|
+
* — it is NEVER sourced from the manifest file (unauthenticated,
|
|
12
|
+
* worker-writable; w1a NO-GO) and never from env.
|
|
13
|
+
* - Nothing else. No broker method can write it.
|
|
14
|
+
*
|
|
15
|
+
* Who reads:
|
|
16
|
+
* - The broker lifecycle controller (lifecycle-handlers.ts) at CrewBroker
|
|
17
|
+
* CONSTRUCT time, binding the value heap-only on the broker instance
|
|
18
|
+
* (CrewBrokerOptions.dispatcherVerifyKey). When no value is registered the
|
|
19
|
+
* constructing process IS the dispatcher (broker host == dispatcher in
|
|
20
|
+
* both sync and async dispatch today) and binds its own public key.
|
|
21
|
+
*
|
|
22
|
+
* Mirrors broker-issuer.ts: module-level singleton, PUBLIC material only (a
|
|
23
|
+
* base64 SPKI DER Ed25519 verify key — non-secret, but its AUTHORITY is the
|
|
24
|
+
* trust anchor for respond.append, so it still never touches disk/env/logs).
|
|
25
|
+
* Cleared only by process exit (binding is lifetime-of-process by design).
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
let activeDispatcherVerifyKey: string | undefined;
|
|
29
|
+
|
|
30
|
+
/** Register the dispatcher public verify key (base64 SPKI DER). Undefined clears. */
|
|
31
|
+
export function setActiveDispatcherVerifyKey(key: string | undefined): void {
|
|
32
|
+
activeDispatcherVerifyKey = typeof key === "string" && key.length > 0 ? key : undefined;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Read the registered dispatcher public verify key, if any. Returns undefined
|
|
36
|
+
* when this process was never handed one (root dispatcher host — bind own key). */
|
|
37
|
+
export function getActiveDispatcherVerifyKey(): string | undefined {
|
|
38
|
+
return activeDispatcherVerifyKey;
|
|
39
|
+
}
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* broker-respond.ts — HƯỚNG A (ADR Option 3, 2026-11-01): the broker's
|
|
3
|
+
* `respond.challenge` / `respond.append` / `respond.force` handlers.
|
|
4
|
+
*
|
|
5
|
+
* Extracted from crew-broker.ts on the wait-resolve.ts / msg-inbox.ts
|
|
6
|
+
* pattern (wc-gate ≤2000): top-level handler functions; the class keeps thin
|
|
7
|
+
* delegations and owns the heap state (threaded via BrokerRespondDeps).
|
|
8
|
+
*
|
|
9
|
+
* WHY these methods are reachable PRE-hello (see CrewBroker.dispatchFrame):
|
|
10
|
+
* their auth is NEVER a hello token —
|
|
11
|
+
* - `respond.append`: challenge-response. The broker binds the DISPATCHER's
|
|
12
|
+
* public verify key HEAP-ONLY at construct time (CrewBrokerOptions.
|
|
13
|
+
* dispatcherVerifyKey ← dispatcher-key-registry.ts ← detached-runner
|
|
14
|
+
* stdin start options; root dispatcher hosts bind their own key). The
|
|
15
|
+
* client proves possession of the dispatcher PRIVATE key by signing
|
|
16
|
+
* nonce ∥ canonical params (respondAppendSignInput — the frozen contract
|
|
17
|
+
* shared with the client seam). A worker holds only public material, so
|
|
18
|
+
* it can never pass; a same-uid forger without the key cannot either.
|
|
19
|
+
* - `respond.force`: no credential — the human-responsibility one-shot for
|
|
20
|
+
* a LIVE park (ADR residual iii: a same-uid process CAN force-answer
|
|
21
|
+
* exactly one live park; the row is tagged `forced` + audited).
|
|
22
|
+
*
|
|
23
|
+
* The appended row is written by THIS broker process's mailbox append
|
|
24
|
+
* (appendMailboxMessageIdempotentAsync signs with the broker host's
|
|
25
|
+
* keypair), so it verifies in every worker's delivery gate (w1a) regardless
|
|
26
|
+
* of which process requested the append.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { createPublicKey, verify as ed25519Verify, randomBytes } from "node:crypto";
|
|
30
|
+
import { appendMailboxMessageIdempotentAsync } from "../../../state/coordination/mailbox.ts";
|
|
31
|
+
import { appendEventAsync } from "../../../state/event-log/event-log.ts";
|
|
32
|
+
import { loadRunManifestById } from "../../../state/stores/state-store.ts";
|
|
33
|
+
import type { TeamRunManifest } from "../../../state/types.ts";
|
|
34
|
+
import { logInternalError } from "../../../utils/internal-error.ts";
|
|
35
|
+
import type { ServerConnection } from "./connection-state.ts";
|
|
36
|
+
import type { LockBusyOutcome } from "./lock-busy.ts";
|
|
37
|
+
import {
|
|
38
|
+
parseRespondAppendParams,
|
|
39
|
+
parseRespondChallengeParams,
|
|
40
|
+
parseRespondForceParams,
|
|
41
|
+
type RespondAppendParams,
|
|
42
|
+
type RespondCommonParams,
|
|
43
|
+
respondAppendSignInput,
|
|
44
|
+
} from "./request-parsers.ts";
|
|
45
|
+
|
|
46
|
+
/** Nominal single-use nonce lifetime. The EFFECTIVE lifetime is also bounded
|
|
47
|
+
* by the connection: an un-authed connection still dies at the 1s hello
|
|
48
|
+
* deadline, and the challenge/append round trip is a local Ed25519 sign
|
|
49
|
+
* (~µs) plus two socket writes, so the TTL below is an upper bound, never a
|
|
50
|
+
* deadline extension (no re-arm path exists — nothing can keep a connection
|
|
51
|
+
* alive by looping respond.challenge). */
|
|
52
|
+
export const RESPOND_CHALLENGE_TTL_MS = 30_000;
|
|
53
|
+
/** Challenge issuance bound per connection (nonce-flood defense). */
|
|
54
|
+
export const RESPOND_CHALLENGE_MAX_PER_CONN = 16;
|
|
55
|
+
|
|
56
|
+
/** Outstanding respond.append challenge: single-use nonce bound to
|
|
57
|
+
* (connection, runId). HEAP ONLY — dies with the connection (WeakMap). */
|
|
58
|
+
export interface RespondChallengeRecord {
|
|
59
|
+
nonce: string;
|
|
60
|
+
runId: string;
|
|
61
|
+
expiresAt: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Heap state + IO seams the handlers run against (owned by CrewBroker). */
|
|
65
|
+
export interface BrokerRespondDeps {
|
|
66
|
+
sendError(conn: ServerConnection, id: string, code: string, message: string): void;
|
|
67
|
+
sendResult(conn: ServerConnection, id: string, result: unknown): void;
|
|
68
|
+
/** Broker cwd (run state root). Undefined → no-manifest error. */
|
|
69
|
+
cwd?: string;
|
|
70
|
+
/** Construct-time bound dispatcher public verify key (base64 SPKI DER). */
|
|
71
|
+
dispatcherVerifyKey: string | undefined;
|
|
72
|
+
respondChallenges: WeakMap<ServerConnection, RespondChallengeRecord>;
|
|
73
|
+
respondChallengeCounts: WeakMap<ServerConnection, number>;
|
|
74
|
+
/** Broker-answered `${runId}:${questionId}` fast-fail set. */
|
|
75
|
+
respondedQuestionIds: Set<string>;
|
|
76
|
+
runLockBusyRetry<T>(manifest: TeamRunManifest, fn: () => T): Promise<LockBusyOutcome<T>>;
|
|
77
|
+
/** Broker socket path (audit identity on the forced path). */
|
|
78
|
+
socketPath: string;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Method router for the respond.* surface (pre- and post-hello). */
|
|
82
|
+
export async function handleRespondMethod(
|
|
83
|
+
deps: BrokerRespondDeps,
|
|
84
|
+
conn: ServerConnection,
|
|
85
|
+
id: string,
|
|
86
|
+
method: string,
|
|
87
|
+
params: unknown,
|
|
88
|
+
): Promise<void> {
|
|
89
|
+
switch (method) {
|
|
90
|
+
case "respond.challenge":
|
|
91
|
+
handleRespondChallenge(deps, conn, id, params);
|
|
92
|
+
return;
|
|
93
|
+
case "respond.append":
|
|
94
|
+
await handleRespondAppend(deps, conn, id, params);
|
|
95
|
+
return;
|
|
96
|
+
case "respond.force":
|
|
97
|
+
await handleRespondForce(deps, conn, id, params);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** CR step 1 — issue a single-use nonce bound to (connection, runId).
|
|
103
|
+
* Issuing a nonce grants NOTHING by itself (the append gate needs a valid
|
|
104
|
+
* signature over it); the connection stays un-authed and still dies at the
|
|
105
|
+
* 1s hello deadline. Rate-bounded per connection. */
|
|
106
|
+
function handleRespondChallenge(deps: BrokerRespondDeps, conn: ServerConnection, id: string, params: unknown): void {
|
|
107
|
+
const parsed = parseRespondChallengeParams(params);
|
|
108
|
+
if (!parsed) {
|
|
109
|
+
deps.sendError(conn, id, "bad-params", "respond.challenge: invalid params");
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const issued = (deps.respondChallengeCounts.get(conn) ?? 0) + 1;
|
|
113
|
+
deps.respondChallengeCounts.set(conn, issued);
|
|
114
|
+
if (issued > RESPOND_CHALLENGE_MAX_PER_CONN) {
|
|
115
|
+
deps.sendError(conn, id, "rate-limited", "respond.challenge: too many challenges on this connection");
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
const nonce = randomBytes(32).toString("base64url");
|
|
119
|
+
deps.respondChallenges.set(conn, { nonce, runId: parsed.runId, expiresAt: Date.now() + RESPOND_CHALLENGE_TTL_MS });
|
|
120
|
+
deps.sendResult(conn, id, { nonce, expiresInMs: RESPOND_CHALLENGE_TTL_MS, runId: parsed.runId });
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** CR step 2 — verify dispatcher possession of the private key over
|
|
124
|
+
* `${nonce}${canonical}` (respondAppendSignInput — frozen contract shared
|
|
125
|
+
* with the client seam) against the bound public key. Never throws. */
|
|
126
|
+
function verifyDispatcherSignature(deps: BrokerRespondDeps, parsed: RespondAppendParams): boolean {
|
|
127
|
+
if (!deps.dispatcherVerifyKey) return false;
|
|
128
|
+
try {
|
|
129
|
+
const pub = createPublicKey({ key: Buffer.from(deps.dispatcherVerifyKey, "base64"), format: "der", type: "spki" });
|
|
130
|
+
const sigBuf = Buffer.from(parsed.sig, "base64");
|
|
131
|
+
return ed25519Verify(null, Buffer.from(respondAppendSignInput(parsed.nonce, parsed), "utf-8"), pub, sigBuf);
|
|
132
|
+
} catch {
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** Shared durable write for BOTH respond paths: idempotent append
|
|
138
|
+
* (requestId = client requestId ?? questionId → exactly-one response row
|
|
139
|
+
* per question across append+force on the broker surface) + the
|
|
140
|
+
* `respond.broker_append` audit event (registered in state/contracts.ts
|
|
141
|
+
* TEAM_EVENT_TYPES). Row shape mirrors respond.ts's alive-path response row
|
|
142
|
+
* (from "leader"/"leader-forced", kind "response", questionId-tagged). */
|
|
143
|
+
async function appendBrokerResponseRow(
|
|
144
|
+
deps: BrokerRespondDeps,
|
|
145
|
+
manifest: TeamRunManifest,
|
|
146
|
+
params: RespondCommonParams & { auth: "challenge-response" | "forced" },
|
|
147
|
+
): Promise<{ ok: true; messageId: string; requestId: string; deduped: boolean } | { ok: false; message: string }> {
|
|
148
|
+
const requestId = params.requestId ?? params.questionId;
|
|
149
|
+
const forced = params.auth === "forced";
|
|
150
|
+
const messageId = `${forced ? "respf" : "resp"}_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
151
|
+
try {
|
|
152
|
+
const outcome = await appendMailboxMessageIdempotentAsync(manifest, {
|
|
153
|
+
id: messageId,
|
|
154
|
+
direction: "inbox",
|
|
155
|
+
from: forced ? "leader-forced" : "leader",
|
|
156
|
+
to: params.taskId,
|
|
157
|
+
taskId: params.taskId,
|
|
158
|
+
body: params.body,
|
|
159
|
+
kind: "response",
|
|
160
|
+
priority: params.priority ?? "normal",
|
|
161
|
+
deliveryMode: params.deliveryMode ?? "next_turn",
|
|
162
|
+
questionId: params.questionId,
|
|
163
|
+
...(params.replyTo !== undefined ? { replyTo: params.replyTo } : {}),
|
|
164
|
+
data: {
|
|
165
|
+
action: "respond",
|
|
166
|
+
kind: "response",
|
|
167
|
+
questionId: params.questionId,
|
|
168
|
+
via: "broker",
|
|
169
|
+
...(forced ? { tag: "forced" } : {}),
|
|
170
|
+
},
|
|
171
|
+
requestId,
|
|
172
|
+
});
|
|
173
|
+
// Heap-side one-shot mark (the durable guarantee is the requestId dedup).
|
|
174
|
+
deps.respondedQuestionIds.add(`${manifest.runId}:${params.questionId}`);
|
|
175
|
+
try {
|
|
176
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
177
|
+
type: "respond.broker_append",
|
|
178
|
+
runId: manifest.runId,
|
|
179
|
+
taskId: params.taskId,
|
|
180
|
+
message: `Broker appended ${forced ? "FORCED " : ""}leader response for question ${params.questionId}.`,
|
|
181
|
+
data: {
|
|
182
|
+
taskId: params.taskId,
|
|
183
|
+
questionId: params.questionId,
|
|
184
|
+
auth: params.auth,
|
|
185
|
+
requestId,
|
|
186
|
+
messageId: outcome.message.id,
|
|
187
|
+
deduped: outcome.deduped,
|
|
188
|
+
...(forced ? { tag: "forced", brokerSocket: deps.socketPath } : {}),
|
|
189
|
+
},
|
|
190
|
+
});
|
|
191
|
+
} catch (err) {
|
|
192
|
+
logInternalError(
|
|
193
|
+
"crew-broker.respond.append-event",
|
|
194
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
195
|
+
`runId=${manifest.runId}`,
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
return { ok: true, messageId: outcome.message.id, requestId, deduped: outcome.deduped };
|
|
199
|
+
} catch (err) {
|
|
200
|
+
return { ok: false, message: (err as Error).message };
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** CR — dispatcher-authenticated respond append. Auth order: worker-role
|
|
205
|
+
* reject → parse → binding → single-use nonce (burned on FIRST attempt: a
|
|
206
|
+
* failed verify also consumes it, one guess per challenge) → Ed25519 verify
|
|
207
|
+
* → run/task resolution → durable append. */
|
|
208
|
+
async function handleRespondAppend(deps: BrokerRespondDeps, conn: ServerConnection, id: string, params: unknown): Promise<void> {
|
|
209
|
+
// Defense-in-depth: role derives from token TYPE (F-06) — a worker
|
|
210
|
+
// connection is never the dispatcher, reject before touching challenge
|
|
211
|
+
// state (same gate as force).
|
|
212
|
+
if (conn.role === "worker") {
|
|
213
|
+
deps.sendError(conn, id, "forbidden", "respond.append requires the dispatcher identity, not a worker token");
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
const parsed = parseRespondAppendParams(params);
|
|
217
|
+
if (!parsed) {
|
|
218
|
+
deps.sendError(conn, id, "bad-params", "respond.append: invalid params");
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
// Hello'd orchestrator connections must target their authenticated run.
|
|
222
|
+
if (conn.authed && conn.runId !== undefined && conn.runId !== parsed.runId) {
|
|
223
|
+
deps.sendError(conn, id, "forbidden", "respond.append: runId does not match the authenticated run");
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
if (!deps.dispatcherVerifyKey) {
|
|
227
|
+
deps.sendError(conn, id, "auth", "respond.append: no dispatcher key bound on this broker");
|
|
228
|
+
return;
|
|
229
|
+
}
|
|
230
|
+
const record = deps.respondChallenges.get(conn);
|
|
231
|
+
if (!record) {
|
|
232
|
+
deps.sendError(conn, id, "auth", "respond.append: no active challenge (call respond.challenge first)");
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
// Single-use: burn on attempt regardless of outcome (anti-brute-force).
|
|
236
|
+
deps.respondChallenges.delete(conn);
|
|
237
|
+
if (record.expiresAt < Date.now()) {
|
|
238
|
+
deps.sendError(conn, id, "auth", "respond.append: challenge expired");
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
if (record.nonce !== parsed.nonce || record.runId !== parsed.runId) {
|
|
242
|
+
deps.sendError(conn, id, "auth", "respond.append: challenge mismatch");
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
if (!verifyDispatcherSignature(deps, parsed)) {
|
|
246
|
+
deps.sendError(conn, id, "auth", "respond.append: signature verification failed");
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
const loaded = loadRunForRespond(deps, conn, id, parsed.runId);
|
|
250
|
+
if (!loaded) return;
|
|
251
|
+
if (!(loaded.tasks ?? []).some((t) => t.id === parsed.taskId)) {
|
|
252
|
+
deps.sendError(conn, id, "no-task", `respond.append: task '${parsed.taskId}' not found`);
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
const outcome = await appendBrokerResponseRow(deps, loaded.manifest, { ...parsed, auth: "challenge-response" });
|
|
256
|
+
if (!outcome.ok) {
|
|
257
|
+
deps.sendError(conn, id, "respond-failed", outcome.message);
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
deps.sendResult(conn, id, {
|
|
261
|
+
messageId: outcome.messageId,
|
|
262
|
+
requestId: outcome.requestId,
|
|
263
|
+
deduped: outcome.deduped,
|
|
264
|
+
auth: "challenge-response",
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** Force — token-less, human-responsibility one-shot per live park. Gates:
|
|
269
|
+
* (a) questionId MUST match a park that is live RIGHT NOW (fresh reload
|
|
270
|
+
* under the bounded busy-retry run lock — RR-023 F4: a cross-process holder,
|
|
271
|
+
* e.g. the respond caller, yields a typed `busy`, never a deadlock);
|
|
272
|
+
* (b) one-shot (heap fast-fail + durable requestId dedup); (c) worker-token
|
|
273
|
+
* connections rejected (defense-in-depth). The append runs OUTSIDE the run
|
|
274
|
+
* lock — an async mailbox append under withRunLockSync would re-order the
|
|
275
|
+
* lock families; exactly-one is guaranteed by the idempotent append, not by
|
|
276
|
+
* holding the lock. */
|
|
277
|
+
async function handleRespondForce(deps: BrokerRespondDeps, conn: ServerConnection, id: string, params: unknown): Promise<void> {
|
|
278
|
+
if (conn.role === "worker") {
|
|
279
|
+
deps.sendError(conn, id, "forbidden", "respond.force is not available to worker tokens");
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
const parsed = parseRespondForceParams(params);
|
|
283
|
+
if (!parsed) {
|
|
284
|
+
deps.sendError(conn, id, "bad-params", "respond.force: invalid params");
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
if (conn.authed && conn.runId !== undefined && conn.runId !== parsed.runId) {
|
|
288
|
+
deps.sendError(conn, id, "forbidden", "respond.force: runId does not match the authenticated run");
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
if (deps.respondedQuestionIds.has(`${parsed.runId}:${parsed.questionId}`)) {
|
|
292
|
+
deps.sendError(conn, id, "already-answered", `respond.force: question '${parsed.questionId}' already has a broker response`);
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
const loaded = loadRunForRespond(deps, conn, id, parsed.runId);
|
|
296
|
+
if (!loaded) return;
|
|
297
|
+
const runId = parsed.runId;
|
|
298
|
+
const taskId = parsed.taskId;
|
|
299
|
+
const questionId = parsed.questionId;
|
|
300
|
+
const gate = await deps.runLockBusyRetry(loaded.manifest, () => {
|
|
301
|
+
// Fresh reload INSIDE the lock (respond.ts:42-43 discipline).
|
|
302
|
+
const fresh = loadRunManifestById(loaded.manifest.cwd, runId);
|
|
303
|
+
if (!fresh) return { code: "no-manifest" as const, message: `run '${runId}' not found`, workerPid: undefined };
|
|
304
|
+
const task = fresh.tasks.find((t) => t.id === taskId);
|
|
305
|
+
if (!task) {
|
|
306
|
+
return { code: "no-task" as const, message: `respond.force: task '${taskId}' not found`, workerPid: undefined };
|
|
307
|
+
}
|
|
308
|
+
if (task.status !== "waiting" || task.waiting?.questionId !== questionId) {
|
|
309
|
+
return {
|
|
310
|
+
code: "no-park" as const,
|
|
311
|
+
message: `respond.force: no live park for question '${questionId}' on task '${taskId}'`,
|
|
312
|
+
workerPid: undefined,
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
return {
|
|
316
|
+
code: "ok" as const,
|
|
317
|
+
message: "",
|
|
318
|
+
workerPid: task.heartbeat?.pid ?? task.checkpoint?.childPid,
|
|
319
|
+
};
|
|
320
|
+
});
|
|
321
|
+
if (!gate.ok) {
|
|
322
|
+
deps.sendError(conn, id, "busy", gate.message);
|
|
323
|
+
return;
|
|
324
|
+
}
|
|
325
|
+
if (gate.value.code !== "ok") {
|
|
326
|
+
deps.sendError(conn, id, gate.value.code, gate.value.message);
|
|
327
|
+
return;
|
|
328
|
+
}
|
|
329
|
+
const outcome = await appendBrokerResponseRow(deps, loaded.manifest, { ...parsed, auth: "forced" });
|
|
330
|
+
if (!outcome.ok) {
|
|
331
|
+
deps.sendError(conn, id, "respond-failed", outcome.message);
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
deps.sendResult(conn, id, {
|
|
335
|
+
messageId: outcome.messageId,
|
|
336
|
+
requestId: outcome.requestId,
|
|
337
|
+
deduped: outcome.deduped,
|
|
338
|
+
auth: "forced",
|
|
339
|
+
...(gate.value.workerPid !== undefined ? { workerPid: gate.value.workerPid } : {}),
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
/** Shared cwd + manifest resolution (typed error frames on failure). */
|
|
344
|
+
function loadRunForRespond(
|
|
345
|
+
deps: BrokerRespondDeps,
|
|
346
|
+
conn: ServerConnection,
|
|
347
|
+
id: string,
|
|
348
|
+
runId: string,
|
|
349
|
+
): NonNullable<ReturnType<typeof loadRunManifestById>> | undefined {
|
|
350
|
+
if (!deps.cwd) {
|
|
351
|
+
deps.sendError(conn, id, "no-manifest", "broker has no cwd configured");
|
|
352
|
+
return undefined;
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
const l = loadRunManifestById(deps.cwd, runId);
|
|
356
|
+
if (!l) {
|
|
357
|
+
deps.sendError(conn, id, "no-manifest", `run '${runId}' not found`);
|
|
358
|
+
return undefined;
|
|
359
|
+
}
|
|
360
|
+
return l;
|
|
361
|
+
} catch (err) {
|
|
362
|
+
deps.sendError(conn, id, "no-manifest", (err as Error).message);
|
|
363
|
+
return undefined;
|
|
364
|
+
}
|
|
365
|
+
}
|