pi-crew 0.11.21 → 0.11.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/dist/index.mjs +542 -531
- package/package.json +2 -1
- package/src/errors.ts +14 -1
- package/src/extension/async-notifier.ts +42 -17
- package/src/extension/cross-extension-rpc.ts +42 -5
- package/src/extension/notification-router.ts +15 -2
- package/src/extension/registration/lifecycle-handlers.ts +36 -13
- package/src/extension/registration/runtime-cleanup.ts +17 -0
- package/src/extension/registration/wire-cross-extension.ts +26 -0
- package/src/extension/rpc-hmac.ts +53 -2
- package/src/extension/team-tool/cancel.ts +34 -21
- package/src/extension/team-tool/dispatch/index.ts +6 -5
- package/src/extension/team-tool/dispatch/status.ts +42 -2
- package/src/extension/team-tool/respond.ts +8 -6
- package/src/extension/team-tool/run.ts +4 -4
- package/src/extension/team-tool.ts +24 -2
- package/src/prompt/inbox-poll.ts +77 -25
- package/src/prompt/message-tool.ts +62 -4
- package/src/prompt/prompt-runtime.ts +82 -32
- package/src/prompt/surface-worker.ts +13 -1
- package/src/runtime/broker/crew-broker-client.ts +74 -16
- package/src/runtime/broker/crew-broker.ts +120 -97
- package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
- package/src/runtime/broker/protocol/request-parsers.ts +68 -0
- package/src/runtime/broker/wait-status-cache.ts +14 -5
- package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
- package/src/runtime/child-pi/child-pi.ts +20 -2
- package/src/runtime/child-pi/mock-fixtures.ts +23 -0
- package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
- package/src/runtime/compaction/important-line-classifier.ts +85 -11
- package/src/runtime/compaction/tool-output-pruner.ts +35 -11
- package/src/runtime/crew-agent-records.ts +32 -14
- package/src/runtime/delivery-coordinator.ts +30 -0
- package/src/runtime/dispatch-batch.ts +9 -6
- package/src/runtime/dwf-state-store.ts +41 -7
- package/src/runtime/finalize-run.ts +13 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
- package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
- package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
- package/src/runtime/live-session/embedded-tier.ts +3 -1
- package/src/runtime/live-session/live-session-runtime.ts +54 -1
- package/src/runtime/model/model-fallback.ts +23 -1
- package/src/runtime/output/pi-json-output.ts +26 -1
- package/src/runtime/policy-engine.ts +18 -0
- package/src/runtime/process/zombie-scanner.ts +84 -16
- package/src/runtime/process-status.ts +6 -1
- package/src/runtime/recovery/crash-recovery.ts +170 -28
- package/src/runtime/rpc/rpc-worker.ts +44 -1
- package/src/runtime/scheduling/run-coalesced-task-group.ts +73 -6
- package/src/runtime/scheduling/scheduler.ts +74 -11
- package/src/runtime/scratchpad/engine.ts +26 -3
- package/src/runtime/settings-store.ts +22 -2
- package/src/runtime/stale-reconciler.ts +4 -1
- package/src/runtime/surface/tmux-provider.ts +24 -3
- package/src/runtime/task-health.ts +16 -1
- package/src/runtime/task-output-context.ts +143 -71
- package/src/runtime/task-packet.ts +70 -17
- package/src/runtime/task-runner/child-executor.ts +27 -0
- package/src/runtime/task-runner/output-splitter.ts +79 -16
- package/src/runtime/task-runner/post-execution.ts +75 -15
- package/src/runtime/task-runner/state-helpers.ts +92 -2
- package/src/runtime/usage-tracker.ts +11 -0
- package/src/runtime/verification/green-contract.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +24 -6
- package/src/runtime/workspace-lock.ts +53 -0
- package/src/schema/team-tool-schema.ts +10 -4
- package/src/state/coordination/mailbox.ts +133 -38
- package/src/state/decision-ledger.ts +56 -8
- package/src/state/event-log/cursor.ts +14 -2
- package/src/state/event-log/event-log-rotation.ts +83 -5
- package/src/state/event-log/event-log.ts +181 -12
- package/src/state/stores/active-run-registry.ts +10 -3
- package/src/state/types.ts +9 -1
- package/src/ui/agents-jobs-browser.ts +22 -8
- package/src/ui/dashboard-panes/agents-pane.ts +5 -2
- package/src/ui/events-state-source.ts +67 -28
- package/src/ui/inline-panel/agent-pane.ts +13 -6
- package/src/ui/inline-panel/agent-transcript.ts +33 -21
- package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
- package/src/ui/inline-panel/index.ts +21 -1
- package/src/ui/live-run-sidebar.ts +14 -3
- package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
- package/src/ui/powerbar-publisher.ts +6 -3
- package/src/ui/rail.ts +7 -1
- package/src/ui/render-scheduler.ts +53 -1
- package/src/ui/run-action-dispatcher.ts +68 -20
- package/src/ui/status-colors.ts +9 -0
- package/src/utils/env-filter.ts +8 -0
|
@@ -2,16 +2,25 @@
|
|
|
2
2
|
* crew-broker-client.ts — Child-side connector for the local broker.
|
|
3
3
|
* PHASE 0 skeleton (sub-task 0.3).
|
|
4
4
|
*
|
|
5
|
-
* - Connects only on the first request/subscribe.
|
|
5
|
+
* - Connects only on the first request/subscribe. The connect is
|
|
6
|
+
* SINGLE-FLIGHT (H7, DR-03 round-6 2026-10-16): concurrent request()s
|
|
7
|
+
* on a fresh client share one in-flight connectAndHello() instead of
|
|
8
|
+
* racing N sockets into existence.
|
|
6
9
|
* - Sends `hello` first; any other method before ack is a protocol error.
|
|
7
10
|
* - Bounded backoff (50/100/200/400/800 ms + jitter, max 4 attempts).
|
|
8
11
|
* After exhaustion, no hot reconnect loop is scheduled. The only way back
|
|
9
12
|
* to the live path is an explicit `reconnect()` (or a new worker lifecycle).
|
|
10
13
|
* - On ANY connect/auth/timeout/close error, transitions to `fallback` mode
|
|
11
|
-
* and never lets an exception escape `request()
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
14
|
+
* and never lets an exception escape `request()`. The caller can read
|
|
15
|
+
* `client.mode` or the `{ok:false, fallback:true}` result and continue
|
|
16
|
+
* using today's file-based paths. (`subscribe()` is the ONE deliberate
|
|
17
|
+
* exception — it throws "not supported" (H8) because a silent no-op
|
|
18
|
+
* unsubscribe let callers believe they were subscribed; it has zero
|
|
19
|
+
* production callers.)
|
|
20
|
+
* - `close()` removes all listeners and pending requests. Any teardown of
|
|
21
|
+
* the current socket (close/reconnect) rejects stranded pending entries
|
|
22
|
+
* IMMEDIATELY with a typed `connection-closed` error (H8) instead of
|
|
23
|
+
* leaving them to their request timers.
|
|
15
24
|
* - All diagnostic strings pass through `redactSecretString` from
|
|
16
25
|
* `src/utils/redaction.ts`. Token and payload bytes are never logged.
|
|
17
26
|
*
|
|
@@ -112,6 +121,13 @@ export class CrewBrokerClient {
|
|
|
112
121
|
}> = [];
|
|
113
122
|
/** Set in close(); gates reconnect attempts and backoff loops. */
|
|
114
123
|
private closed = false;
|
|
124
|
+
/** H7: in-flight connect (single-flight). Concurrent request()s on an
|
|
125
|
+
* unstarted client await the SAME connectAndHello() promise instead of
|
|
126
|
+
* each opening its own socket (previously two concurrent first-requests
|
|
127
|
+
* raced two sockets; the loser's socket leaked until its hello deadline).
|
|
128
|
+
* Cleared on settle — success moves `mode` to "connected" and failure to
|
|
129
|
+
* "fallback", so a null value never masks a needed reconnect. */
|
|
130
|
+
private connecting: Promise<BrokerClientResult<true>> | null = null;
|
|
115
131
|
/** Handle to the in-flight backoff timer so close() can cancel it. */
|
|
116
132
|
private backoffTimer: NodeJS.Timeout | null = null;
|
|
117
133
|
/** Resolver for the in-flight backoff await so close() can unblock it.
|
|
@@ -171,9 +187,10 @@ export class CrewBrokerClient {
|
|
|
171
187
|
return { ok: false, fallback: true, errorCode: "bad-method" };
|
|
172
188
|
}
|
|
173
189
|
|
|
174
|
-
// Lazy connect + hello on first use.
|
|
190
|
+
// Lazy connect + hello on first use. H7: single-flight — concurrent
|
|
191
|
+
// requests share one connectAndHello() (one socket), not N.
|
|
175
192
|
if (this._mode === "unstarted" || !this.socket) {
|
|
176
|
-
const connectResult = await this.
|
|
193
|
+
const connectResult = await this.ensureConnected();
|
|
177
194
|
if (!connectResult.ok) {
|
|
178
195
|
return connectResult;
|
|
179
196
|
}
|
|
@@ -246,18 +263,17 @@ export class CrewBrokerClient {
|
|
|
246
263
|
}
|
|
247
264
|
|
|
248
265
|
/**
|
|
249
|
-
* Subscribe to a per-run event stream.
|
|
250
|
-
*
|
|
251
|
-
*
|
|
252
|
-
*
|
|
266
|
+
* Subscribe to a per-run event stream. NOT SUPPORTED (H8, DR-03
|
|
267
|
+
* round-6 2026-10-16): Phase 0 never implemented the event stream, and
|
|
268
|
+
* the old silent no-op unsubscribe let a caller believe it was
|
|
269
|
+
* subscribed while frames went nowhere. Zero production callers exist,
|
|
270
|
+
* so this now throws instead of silently misbehaving. Live frames are
|
|
271
|
+
* available via the `onEvent` constructor option; durable resync via
|
|
272
|
+
* `msg.inbox` / `events.since` polling.
|
|
253
273
|
*/
|
|
254
274
|
subscribe(options: { runId: string; sinceSeq: number; onEvent: (event: unknown) => void }): () => void {
|
|
255
|
-
// Phase 0: subscription is a typed not-implemented. We register a
|
|
256
|
-
// no-op unsubscribe so the caller can call it without errors.
|
|
257
275
|
void options;
|
|
258
|
-
|
|
259
|
-
/* no-op */
|
|
260
|
-
};
|
|
276
|
+
throw new BrokerError("not-implemented", "subscribe is not supported; use onEvent / events.since polling");
|
|
261
277
|
}
|
|
262
278
|
|
|
263
279
|
/**
|
|
@@ -331,6 +347,19 @@ export class CrewBrokerClient {
|
|
|
331
347
|
// Connect + hello with bounded backoff
|
|
332
348
|
// ------------------------------------------------------------------------
|
|
333
349
|
|
|
350
|
+
/** H7: single-flight wrapper around connectAndHello. The first caller
|
|
351
|
+
* creates the in-flight promise; concurrent callers await the same one.
|
|
352
|
+
* Settled promises are dropped so a later reconnect() (explicit
|
|
353
|
+
* `unstarted` reset) starts a genuinely fresh connect. */
|
|
354
|
+
private ensureConnected(): Promise<BrokerClientResult<true>> {
|
|
355
|
+
if (!this.connecting) {
|
|
356
|
+
this.connecting = this.connectAndHello().finally(() => {
|
|
357
|
+
this.connecting = null;
|
|
358
|
+
});
|
|
359
|
+
}
|
|
360
|
+
return this.connecting;
|
|
361
|
+
}
|
|
362
|
+
|
|
334
363
|
private async connectAndHello(): Promise<BrokerClientResult<true>> {
|
|
335
364
|
if (!this.options.socketPath || !this.options.token) {
|
|
336
365
|
this.enterFallbackOnce("missing-credentials", new Error("socket or token not provided"));
|
|
@@ -642,6 +671,35 @@ export class CrewBrokerClient {
|
|
|
642
671
|
}
|
|
643
672
|
|
|
644
673
|
private teardownSocket(): void {
|
|
674
|
+
// H8: reject stranded pending entries IMMEDIATELY. A teardown means
|
|
675
|
+
// the socket they were written to is going away (close() semantics, a
|
|
676
|
+
// reconnect() replacing it, or the closed-race in connectAndHello) —
|
|
677
|
+
// their responses can never arrive on it. Leaving them to their
|
|
678
|
+
// request timers kept callers (e.g. an await inside reconnect())
|
|
679
|
+
// hanging for up to the full request timeout on a dead connection.
|
|
680
|
+
// The typed `connection-closed` error funnels into request()'s catch →
|
|
681
|
+
// {ok:false, fallback:true, errorCode:"connection-closed"}.
|
|
682
|
+
if (this.pending.size > 0) {
|
|
683
|
+
const clearTimeoutFn = this.options.clearTimeoutFn ?? ((t: NodeJS.Timeout) => clearTimeout(t));
|
|
684
|
+
for (const [, p] of this.pending) {
|
|
685
|
+
if (p.timer) {
|
|
686
|
+
try {
|
|
687
|
+
clearTimeoutFn(p.timer);
|
|
688
|
+
} catch {
|
|
689
|
+
/* best-effort */
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
p.reject(
|
|
693
|
+
// Code outside the closed BrokerErrorCode union (which lives in
|
|
694
|
+
// ndjson.ts and gates protocol responses); same cast precedent as
|
|
695
|
+
// the wire-error path in attemptHello. Surfaces to callers as
|
|
696
|
+
// errorCode "connection-closed" (BrokerClientResult.errorCode is a
|
|
697
|
+
// plain string).
|
|
698
|
+
new BrokerError("connection-closed" as never, "connection torn down before response"),
|
|
699
|
+
);
|
|
700
|
+
}
|
|
701
|
+
this.pending.clear();
|
|
702
|
+
}
|
|
645
703
|
if (!this.socket) {
|
|
646
704
|
this.socket = null;
|
|
647
705
|
this.decoder = null;
|
|
@@ -1,24 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* crew-broker.ts — Root-only local socket server for cross-process pi-worker
|
|
3
|
-
* message transport
|
|
3
|
+
* message transport (grew out of the Phase 0 skeleton,
|
|
4
|
+
* sub-task 0.4 — see history note below).
|
|
4
5
|
*
|
|
5
6
|
* - Bound to a `node:net` Unix domain socket or Windows named pipe.
|
|
6
7
|
* - One broker per root session; per-run token auth on first `hello`.
|
|
7
8
|
* - NDJSON framing, 256 KiB UTF-8 cap, 1s hello deadline.
|
|
8
|
-
* - Per-connection outbound queue cap (default 256)
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
9
|
+
* - Per-connection outbound queue cap (default 256). Fanout/event frames
|
|
10
|
+
* are drop-newest at cap, and the drop now CLOSES the connection with a
|
|
11
|
+
* typed `protocol.close` reason ("outbound-overflow-resync") so the
|
|
12
|
+
* client reconnects and replays (R6-5). Request RESPONSE frames
|
|
13
|
+
* (sendResult/sendError acks) bypass the cap — bounded by in-flight
|
|
14
|
+
* request count, never the fanout storm.
|
|
15
|
+
* - Dispatches 12 methods post-hello: `ping`, `hello`, `msg.send`,
|
|
16
|
+
* `msg.inbox`, `events.since`, `events.subscribe`, `task.waitStatus`,
|
|
17
|
+
* `steer.push`, `escalate`, `wait.request`/`wait.resolve`
|
|
18
|
+
* (capability-gated via `options.waitMethodsEnabled`, fail-closed), and
|
|
19
|
+
* `delegate.request` (governed-nesting grandchild spawn through
|
|
20
|
+
* `delegate-spawn.ts`, capability-gated via `options.nestingEnabled`,
|
|
21
|
+
* fail-closed). Unknown methods return a typed `not-implemented`.
|
|
22
|
+
* - Mailbox/event persistence flows through the `src/state/` stores
|
|
23
|
+
* (`appendMailboxMessage*`, event-log) — the broker itself keeps no
|
|
24
|
+
* durable state of its own.
|
|
13
25
|
* - Token map is HEAP ONLY; cleared on `stop()`. Never serialized.
|
|
14
26
|
* - `stop()` is idempotent. Never calls `process.kill`.
|
|
15
27
|
* - All log/error scopes use `crew-broker.*` prefix; every diagnostic
|
|
16
28
|
* passes through `redactSecretString` from `src/utils/redaction.ts`.
|
|
17
29
|
*
|
|
18
|
-
* NO outbound TCP. NO
|
|
30
|
+
* NO outbound TCP. NO process killing.
|
|
19
31
|
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
32
|
+
* History: the Phase 0 contract (hello/ping only, no persistence, no
|
|
33
|
+
* children) lives in `reports/inter-pi-broker-impl-plan-2026-07-21.md` §0.4;
|
|
34
|
+
* the 12-method surface above supersedes it. DR-01 (2026-10-14) rewrote
|
|
35
|
+
* this header after it had drifted ~1900 LOC past the skeleton it
|
|
36
|
+
* described.
|
|
22
37
|
*/
|
|
23
38
|
|
|
24
39
|
import { randomUUID } from "node:crypto";
|
|
@@ -53,10 +68,12 @@ import { handleEventsSince } from "./protocol/events-replay.ts";
|
|
|
53
68
|
import { DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, type LockBusyOutcome, withRunLockBusyRetry } from "./protocol/lock-busy.ts";
|
|
54
69
|
import { loadRunForHello } from "./protocol/manifest-loader.ts";
|
|
55
70
|
import { handleMsgInbox } from "./protocol/msg-inbox.ts";
|
|
71
|
+
import { resolveMsgSendTargets } from "./protocol/msg-send-targets.ts";
|
|
56
72
|
import {
|
|
57
73
|
BROKER_PROTOCOL,
|
|
58
74
|
isHelloParams,
|
|
59
75
|
isRequestObject,
|
|
76
|
+
parseDelegateRequestParams,
|
|
60
77
|
parseMsgSendParams,
|
|
61
78
|
parseWaitRequestParams,
|
|
62
79
|
safeStringify,
|
|
@@ -93,6 +110,26 @@ const STALE_RUN_STATUSES: ReadonlySet<string> = new Set(["completed", "failed",
|
|
|
93
110
|
/** Default per-connection outbound queue cap (events). */
|
|
94
111
|
const DEFAULT_OUTBOUND_QUEUE_CAP = 256;
|
|
95
112
|
|
|
113
|
+
/** R6-5 (2026-10-16): typed protocol close-reason sent as a best-effort
|
|
114
|
+
* `protocol.close` event frame (direct write, jumps the saturated queue)
|
|
115
|
+
* before the connection is closed after a fanout/event frame was dropped
|
|
116
|
+
* at the queue cap. Clients observe the reason, reconnect, and replay via
|
|
117
|
+
* `events.since` (Phase 1.5). */
|
|
118
|
+
export const OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON = "outbound-overflow-resync";
|
|
119
|
+
|
|
120
|
+
/** R6-5: admission class for an outbound frame.
|
|
121
|
+
* - `false` — fanout/event class: drop-newest at cap → resync close.
|
|
122
|
+
* - `true` — close-reason class: bypass the cap, attempt an immediate
|
|
123
|
+
* direct write (existing `sendErrorAndClose` behavior).
|
|
124
|
+
* - `"response"` — request-response class (sendResult/sendError acks):
|
|
125
|
+
* bypass the drop-newest cap but stay IN the queue (ordering preserved,
|
|
126
|
+
* backpressure honored). Bounded by the client's in-flight request count
|
|
127
|
+
* — one response frame per request — so it can never storm the way a
|
|
128
|
+
* fanout burst can. Kept a `boolean | "response"` union (not a fresh
|
|
129
|
+
* enum) so the external FanoutWriters interface (mailbox-fanout.ts) keeps
|
|
130
|
+
* its `(conn, buf, force: boolean)` shape. */
|
|
131
|
+
type OutboundAdmission = boolean | "response";
|
|
132
|
+
|
|
96
133
|
export class CrewBroker {
|
|
97
134
|
private readonly options: Required<Pick<CrewBrokerOptions, "sessionId" | "enabled" | "waitMethodsEnabled" | "nestingEnabled">> &
|
|
98
135
|
Pick<
|
|
@@ -750,7 +787,7 @@ export class CrewBroker {
|
|
|
750
787
|
* truth for authentication). */
|
|
751
788
|
// loadRunForHello: inlined at the 2 call sites (was a 5-line method; M4/WI-4.1).
|
|
752
789
|
|
|
753
|
-
// Outbound queue + drop-newest +
|
|
790
|
+
// Outbound queue + drop-newest + resync-close (R6-5)
|
|
754
791
|
|
|
755
792
|
private sendResult(conn: ServerConnection, id: string, result: unknown): void {
|
|
756
793
|
this.enqueueFrame(conn, { id, result });
|
|
@@ -787,13 +824,27 @@ export class CrewBroker {
|
|
|
787
824
|
);
|
|
788
825
|
return;
|
|
789
826
|
}
|
|
790
|
-
|
|
827
|
+
// R6-5: request responses (the acks for ping/msg.send/wait.request/…)
|
|
828
|
+
// are response-class — they bypass the drop-newest cap so a saturated
|
|
829
|
+
// fanout queue can no longer swallow an ack and leave the caller to its
|
|
830
|
+
// request-timeout (a delivered send reported "NOT sent").
|
|
831
|
+
this.writeOrQueue(conn, buf, "response");
|
|
791
832
|
}
|
|
792
833
|
|
|
793
|
-
private writeOrQueue(conn: ServerConnection, buf: Buffer, force:
|
|
834
|
+
private writeOrQueue(conn: ServerConnection, buf: Buffer, force: OutboundAdmission): void {
|
|
794
835
|
if (conn.closed) return;
|
|
795
836
|
const cap = this.options.outboundQueueCap ?? DEFAULT_OUTBOUND_QUEUE_CAP;
|
|
796
837
|
if (conn.outbound.length >= cap) {
|
|
838
|
+
if (force === "response") {
|
|
839
|
+
// R6-5(a): response frames queue PAST the cap instead of being
|
|
840
|
+
// drop-newest'd. Bounded: at most one response per in-flight
|
|
841
|
+
// request, and the client's request pipeline is serial — this can
|
|
842
|
+
// never grow unbounded the way a fanout burst can. Ordering is
|
|
843
|
+
// preserved (pushed at the tail; drainOutbound honors backpressure).
|
|
844
|
+
conn.outbound.push(buf);
|
|
845
|
+
this.drainOutbound(conn);
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
797
848
|
if (force) {
|
|
798
849
|
// Forced sends (e.g. close-reason) bypass the cap and attempt
|
|
799
850
|
// to flush directly; if the socket is busy they may still drop.
|
|
@@ -804,13 +855,13 @@ export class CrewBroker {
|
|
|
804
855
|
}
|
|
805
856
|
return;
|
|
806
857
|
}
|
|
807
|
-
//
|
|
808
|
-
//
|
|
809
|
-
//
|
|
810
|
-
|
|
811
|
-
//
|
|
812
|
-
//
|
|
813
|
-
|
|
858
|
+
// R6-5(b): fanout/event frame dropped at cap (drop-newest). The old
|
|
859
|
+
// `needsResync` marker was write-only (set, never read) — the claimed
|
|
860
|
+
// "pause + client notices" recovery did not exist, so live frames
|
|
861
|
+
// were silently dropped for the connection's remaining lifetime.
|
|
862
|
+
// Now the drop CLOSES the connection with a typed close-reason so
|
|
863
|
+
// the client reconnects and replays (events.since / msg.inbox).
|
|
864
|
+
this.handleOutboundOverflow(conn);
|
|
814
865
|
return;
|
|
815
866
|
}
|
|
816
867
|
conn.outbound.push(buf);
|
|
@@ -841,6 +892,33 @@ export class CrewBroker {
|
|
|
841
892
|
}
|
|
842
893
|
}
|
|
843
894
|
|
|
895
|
+
/** R6-5(b): a fanout/event frame was dropped at the outbound queue cap.
|
|
896
|
+
* Close the connection with a typed protocol close-reason so the client
|
|
897
|
+
* can reconnect + replay instead of silently missing live frames
|
|
898
|
+
* forever. The reason frame is written DIRECTLY (jumps the saturated
|
|
899
|
+
* queue — those frames are already doomed once we close) and delivery
|
|
900
|
+
* is best-effort, exactly like `sendErrorAndClose`. */
|
|
901
|
+
private handleOutboundOverflow(conn: ServerConnection): void {
|
|
902
|
+
// Single-flight: fanoutMailboxMessage iterates a snapshot of the run's
|
|
903
|
+
// connections, so one burst can re-enter here for an already-overflowing
|
|
904
|
+
// conn before `closeConnection`'s `closed` guard becomes visible. The
|
|
905
|
+
// flag is the "close already initiated" bit (its only read).
|
|
906
|
+
if (conn.needsResync) return;
|
|
907
|
+
conn.needsResync = true;
|
|
908
|
+
logInternalError(
|
|
909
|
+
"crew-broker.outbound.overflow",
|
|
910
|
+
new Error("outbound queue overflow — closing connection for resync"),
|
|
911
|
+
`sessionId=${this.options.sessionId} runId=${conn.runId ?? "?"} queued=${conn.outbound.length} seq=${conn.outboundSeq}`,
|
|
912
|
+
);
|
|
913
|
+
try {
|
|
914
|
+
const buf = encodeBrokerFrame({ event: "protocol.close", data: { reason: OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON } });
|
|
915
|
+
conn.socket.write(buf);
|
|
916
|
+
} catch {
|
|
917
|
+
/* socket may be dead; closeConnection below sweeps regardless */
|
|
918
|
+
}
|
|
919
|
+
this.closeConnection(conn);
|
|
920
|
+
}
|
|
921
|
+
|
|
844
922
|
// Phase 1: msg.send + msg.inbox handlers
|
|
845
923
|
|
|
846
924
|
/** Phase 1.1: direct or broadcast mailbox write via the durable append path. */
|
|
@@ -894,67 +972,25 @@ export class CrewBroker {
|
|
|
894
972
|
this.sendError(conn, id, "no-manifest", (err as Error).message);
|
|
895
973
|
return;
|
|
896
974
|
}
|
|
897
|
-
// ── Recipient resolution
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
// Task 5b (spec §15.2 wake): set when a worker addresses the parent —
|
|
903
|
-
// the durable write alone would sit unread in the run-level inbox.
|
|
904
|
-
let sentToParent = false;
|
|
905
|
-
if (isWorker) {
|
|
906
|
-
// Worker constraint (1): from is ALWAYS the authenticated taskId.
|
|
907
|
-
// Worker constraint (2): to is limited to parent | valid sibling
|
|
908
|
-
// taskId | group.
|
|
909
|
-
if (!conn.taskId) {
|
|
910
|
-
this.sendError(conn, id, "forbidden", "msg.send worker requires a task-scoped identity");
|
|
911
|
-
return;
|
|
912
|
-
}
|
|
913
|
-
const to = typeof parsed.to === "string" ? parsed.to : undefined;
|
|
914
|
-
if (to === "parent") {
|
|
915
|
-
// Run-level inbox (taskId undefined) → the orchestrator session.
|
|
916
|
-
targets = [{ label: "parent", mailboxTaskId: undefined }];
|
|
917
|
-
sentToParent = true;
|
|
918
|
-
} else if (to === "group") {
|
|
919
|
-
targets = taskIds.map((t) => ({ label: t, mailboxTaskId: t }));
|
|
920
|
-
} else if (to !== undefined && taskIds.includes(to)) {
|
|
921
|
-
targets = [{ label: to, mailboxTaskId: to }];
|
|
922
|
-
} else {
|
|
923
|
-
this.sendError(conn, id, "forbidden", `msg.send: worker cannot target '${to}'`);
|
|
924
|
-
return;
|
|
925
|
-
}
|
|
926
|
-
if (targets.length === 0 || targets.length > 64) {
|
|
927
|
-
this.sendError(conn, id, "bad-params", "msg.send: recipient count out of range");
|
|
928
|
-
return;
|
|
929
|
-
}
|
|
930
|
-
} else {
|
|
931
|
-
const recipients: string[] = Array.isArray(parsed.to)
|
|
932
|
-
? (parsed.to as string[])
|
|
933
|
-
: parsed.to === "all"
|
|
934
|
-
? taskIds
|
|
935
|
-
: [parsed.to as string];
|
|
936
|
-
if (recipients.length === 0 || recipients.length > 64) {
|
|
937
|
-
this.sendError(conn, id, "bad-params", "msg.send: recipient count out of range");
|
|
938
|
-
return;
|
|
939
|
-
}
|
|
940
|
-
targets = recipients.map((recipient) => ({ label: recipient, mailboxTaskId: recipient }));
|
|
975
|
+
// ── Recipient resolution: pure D9/§15.2 matrix in protocol/msg-send-targets.ts (wc-gate).
|
|
976
|
+
const resolved = resolveMsgSendTargets({ isWorker, connTaskId: conn.taskId, to: parsed.to, taskIds });
|
|
977
|
+
if (!resolved.ok) {
|
|
978
|
+
this.sendError(conn, id, resolved.code, resolved.message);
|
|
979
|
+
return;
|
|
941
980
|
}
|
|
981
|
+
const { targets, sentToParent } = resolved;
|
|
942
982
|
const messageId = `msg_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
943
983
|
const fromField = isWorker ? conn.taskId! : (conn.taskId ?? conn.runId);
|
|
944
|
-
// U5
|
|
945
|
-
//
|
|
946
|
-
//
|
|
947
|
-
// delivery.json + the idempotent append below make resend-after-resume a
|
|
948
|
-
// no-op instead of a duplicate delivery.
|
|
984
|
+
// U5: every send carries a unique requestId — client-supplied (idempotent
|
|
985
|
+
// retries reuse the SAME one) or broker-generated. The durable
|
|
986
|
+
// (direction, requestId) index makes resend-after-resume a no-op.
|
|
949
987
|
const requestId = parsed.requestId ?? `${manifest.runId}:${fromField}:${randomUUID()}`;
|
|
950
988
|
let dedupedRecipients = 0;
|
|
951
989
|
let durable = false;
|
|
952
990
|
try {
|
|
953
|
-
// PERF
|
|
954
|
-
//
|
|
955
|
-
//
|
|
956
|
-
// mailbox files append concurrently; delivery.json stays serialized by
|
|
957
|
-
// its own lock.
|
|
991
|
+
// PERF: to:"all" with 50 tasks used to run 50 sequential awaited locked
|
|
992
|
+
// appends while the connection's frames queued behind it. Chunked
|
|
993
|
+
// fan-out — independent mailbox files append concurrently.
|
|
958
994
|
const CHUNK = 8;
|
|
959
995
|
for (let i = 0; i < targets.length; i += CHUNK) {
|
|
960
996
|
const results = await Promise.allSettled(
|
|
@@ -985,10 +1021,9 @@ export class CrewBroker {
|
|
|
985
1021
|
this.sendError(conn, id, "durable-failed", (err as Error).message);
|
|
986
1022
|
return;
|
|
987
1023
|
}
|
|
988
|
-
// U5: one bounded audit event per send — requestId + dedup count make
|
|
989
|
-
// exactly-once chain auditable
|
|
990
|
-
//
|
|
991
|
-
// non-fatal: the mailbox rows are the source of truth.
|
|
1024
|
+
// U5: one bounded audit event per send — requestId + dedup count make
|
|
1025
|
+
// the exactly-once chain auditable. Never the body; mailbox rows are
|
|
1026
|
+
// the source of truth.
|
|
992
1027
|
try {
|
|
993
1028
|
await appendEventAsync(manifest.eventsPath, {
|
|
994
1029
|
type: "mailbox.send",
|
|
@@ -1421,28 +1456,16 @@ export class CrewBroker {
|
|
|
1421
1456
|
this.sendError(conn, id, "forbidden", "delegate requires a task-scoped token; re-dispatch with PI_CREW_BROKER_TASK_ID");
|
|
1422
1457
|
return;
|
|
1423
1458
|
}
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
timeoutSec?: unknown;
|
|
1432
|
-
};
|
|
1433
|
-
if (typeof p.prompt !== "string" || p.prompt.trim().length === 0) {
|
|
1434
|
-
this.sendError(conn, id, "bad-params", "delegate.request: 'prompt' (non-empty string) is required");
|
|
1459
|
+
// R7-H1 fix: parse params through the protocol-level parser (mirrors
|
|
1460
|
+
// parseWaitRequestParams discipline) — non-integer/non-finite/negative/
|
|
1461
|
+
// over-ceiling maxTurns/budgetTokens never reach admission or the
|
|
1462
|
+
// budget reservation (durable-negative tokensSpent escape).
|
|
1463
|
+
const parsed = parseDelegateRequestParams(params);
|
|
1464
|
+
if (!parsed) {
|
|
1465
|
+
this.sendError(conn, id, "bad-params", "delegate.request: invalid params");
|
|
1435
1466
|
return;
|
|
1436
1467
|
}
|
|
1437
|
-
const requested =
|
|
1438
|
-
prompt: p.prompt,
|
|
1439
|
-
...(typeof p.description === "string" ? { description: p.description } : {}),
|
|
1440
|
-
...(typeof p.role === "string" ? { role: p.role } : {}),
|
|
1441
|
-
...(typeof p.model === "string" ? { model: p.model } : {}),
|
|
1442
|
-
...(typeof p.maxTurns === "number" ? { maxTurns: p.maxTurns } : {}),
|
|
1443
|
-
...(typeof p.budgetTokens === "number" ? { budgetTokens: p.budgetTokens } : {}),
|
|
1444
|
-
...(typeof p.timeoutSec === "number" ? { timeoutSec: p.timeoutSec } : {}),
|
|
1445
|
-
};
|
|
1468
|
+
const requested = parsed;
|
|
1446
1469
|
const cwd = this.options.cwd;
|
|
1447
1470
|
if (!cwd) {
|
|
1448
1471
|
this.sendError(conn, id, "no-manifest", "broker has no cwd configured");
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure recipient resolution for the broker's `msg.send` handler.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from crew-broker.ts (DR-03 wave, wc-gate: no module over 2000
|
|
5
|
+
* lines — same precedent as live-session/usage-bridge.ts): the worker/
|
|
6
|
+
* orchestrator recipient matrices are pure functions of
|
|
7
|
+
* {role, taskId, parsed.to, taskIds} — no broker state involved.
|
|
8
|
+
*
|
|
9
|
+
* Worker constraints (D9/§15.2): from is always the authenticated taskId;
|
|
10
|
+
* to is limited to `parent` | valid sibling taskId | `group`; kind limited
|
|
11
|
+
* to notify|message (checked by the caller BEFORE resolution).
|
|
12
|
+
* Orchestrator keeps the full prior surface (arrays / "all" / arbitrary
|
|
13
|
+
* recipient sets).
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export type MsgSendTarget = { label: string; mailboxTaskId: string | undefined };
|
|
17
|
+
|
|
18
|
+
export type MsgSendTargetResolution =
|
|
19
|
+
| { ok: true; targets: MsgSendTarget[]; sentToParent: boolean }
|
|
20
|
+
| { ok: false; code: "forbidden" | "bad-params"; message: string };
|
|
21
|
+
|
|
22
|
+
/** Max recipients per send (both roles) — matches the historical inline check. */
|
|
23
|
+
export const MSG_SEND_MAX_RECIPIENTS = 64;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Resolve `parsed.to` into mailbox targets.
|
|
27
|
+
*
|
|
28
|
+
* Each target is `{label, mailboxTaskId}`: `label` is echoed in the ack and
|
|
29
|
+
* message id; `mailboxTaskId` is the mailbox file the append lands in
|
|
30
|
+
* (undefined = run-level inbox, which the orchestrator consumes).
|
|
31
|
+
* `sentToParent` is set when a worker addresses the parent — the durable
|
|
32
|
+
* write alone would sit unread in the run-level inbox (spec §15.2 wake).
|
|
33
|
+
*/
|
|
34
|
+
export function resolveMsgSendTargets(input: {
|
|
35
|
+
isWorker: boolean;
|
|
36
|
+
connTaskId: string | undefined;
|
|
37
|
+
to: unknown;
|
|
38
|
+
taskIds: string[];
|
|
39
|
+
}): MsgSendTargetResolution {
|
|
40
|
+
const { isWorker, connTaskId, taskIds } = input;
|
|
41
|
+
|
|
42
|
+
if (isWorker) {
|
|
43
|
+
// Worker constraint (1): from is ALWAYS the authenticated taskId.
|
|
44
|
+
if (!connTaskId) {
|
|
45
|
+
return { ok: false, code: "forbidden", message: "msg.send worker requires a task-scoped identity" };
|
|
46
|
+
}
|
|
47
|
+
// Worker constraint (2): to is limited to parent | valid sibling
|
|
48
|
+
// taskId | group.
|
|
49
|
+
const to = typeof input.to === "string" ? input.to : undefined;
|
|
50
|
+
let targets: MsgSendTarget[];
|
|
51
|
+
let sentToParent = false;
|
|
52
|
+
if (to === "parent") {
|
|
53
|
+
// Run-level inbox (taskId undefined) → the orchestrator session.
|
|
54
|
+
targets = [{ label: "parent", mailboxTaskId: undefined }];
|
|
55
|
+
sentToParent = true;
|
|
56
|
+
} else if (to === "group") {
|
|
57
|
+
targets = taskIds.map((t) => ({ label: t, mailboxTaskId: t }));
|
|
58
|
+
} else if (to !== undefined && taskIds.includes(to)) {
|
|
59
|
+
targets = [{ label: to, mailboxTaskId: to }];
|
|
60
|
+
} else {
|
|
61
|
+
return { ok: false, code: "forbidden", message: `msg.send: worker cannot target '${to}'` };
|
|
62
|
+
}
|
|
63
|
+
if (targets.length === 0 || targets.length > MSG_SEND_MAX_RECIPIENTS) {
|
|
64
|
+
return { ok: false, code: "bad-params", message: "msg.send: recipient count out of range" };
|
|
65
|
+
}
|
|
66
|
+
return { ok: true, targets, sentToParent };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const recipients: string[] = Array.isArray(input.to) ? (input.to as string[]) : input.to === "all" ? taskIds : [input.to as string];
|
|
70
|
+
if (recipients.length === 0 || recipients.length > MSG_SEND_MAX_RECIPIENTS) {
|
|
71
|
+
return { ok: false, code: "bad-params", message: "msg.send: recipient count out of range" };
|
|
72
|
+
}
|
|
73
|
+
return { ok: true, targets: recipients.map((recipient) => ({ label: recipient, mailboxTaskId: recipient })), sentToParent: false };
|
|
74
|
+
}
|
|
@@ -176,6 +176,74 @@ export function parseWaitRequestParams(value: unknown): WaitRequestParams | unde
|
|
|
176
176
|
};
|
|
177
177
|
}
|
|
178
178
|
|
|
179
|
+
// ============================================================================
|
|
180
|
+
// R7-H1 fix (deep-review round 8): delegate.request parameter parser.
|
|
181
|
+
// Mirrors parseWaitRequestParams discipline — the handler previously spread
|
|
182
|
+
// `typeof x === "number"` values straight into `requested`, so non-integer,
|
|
183
|
+
// non-finite, negative and over-ceiling maxTurns/budgetTokens reached
|
|
184
|
+
// admission (e.g. raw-wire -1e9 budget → durable negative tokensSpent;
|
|
185
|
+
// 1e999 → Infinity maxTurns → dead child-pi turn caps).
|
|
186
|
+
// ============================================================================
|
|
187
|
+
|
|
188
|
+
/** Server-side ceiling for grandchild turn caps (client tool schema: 1..200). */
|
|
189
|
+
export const DELEGATE_MAX_TURNS_CEILING = 200;
|
|
190
|
+
/** Server-side ceiling for grandchild token budgets (client schema: <=100M). */
|
|
191
|
+
export const DELEGATE_BUDGET_TOKENS_CEILING = 100_000_000;
|
|
192
|
+
/** Bounded delegate prompt (defense-in-depth under the frame cap). */
|
|
193
|
+
export const DELEGATE_PROMPT_MAX_CHARS = 100_000;
|
|
194
|
+
|
|
195
|
+
export interface DelegateRequestParams {
|
|
196
|
+
prompt: string;
|
|
197
|
+
description?: string;
|
|
198
|
+
role?: string;
|
|
199
|
+
model?: string;
|
|
200
|
+
maxTurns?: number;
|
|
201
|
+
budgetTokens?: number;
|
|
202
|
+
timeoutSec?: number;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export function parseDelegateRequestParams(value: unknown): DelegateRequestParams | undefined {
|
|
206
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
207
|
+
const v = value as Record<string, unknown>;
|
|
208
|
+
if (typeof v.prompt !== "string" || v.prompt.trim().length === 0 || v.prompt.length > DELEGATE_PROMPT_MAX_CHARS) {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
if (v.description !== undefined && typeof v.description !== "string") return undefined;
|
|
212
|
+
if (v.role !== undefined && !(typeof v.role === "string" && ["explorer", "analyst", "executor"].includes(v.role))) {
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
if (v.model !== undefined && typeof v.model !== "string") return undefined;
|
|
216
|
+
if (v.maxTurns !== undefined) {
|
|
217
|
+
if (typeof v.maxTurns !== "number" || !Number.isInteger(v.maxTurns) || v.maxTurns < 1 || v.maxTurns > DELEGATE_MAX_TURNS_CEILING) {
|
|
218
|
+
return undefined;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
if (v.budgetTokens !== undefined) {
|
|
222
|
+
if (
|
|
223
|
+
typeof v.budgetTokens !== "number" ||
|
|
224
|
+
!Number.isInteger(v.budgetTokens) || // R8 lane-B correction: client Type.Integer parity (0.5/3.5 pass isFinite)
|
|
225
|
+
v.budgetTokens < 0 ||
|
|
226
|
+
v.budgetTokens > DELEGATE_BUDGET_TOKENS_CEILING
|
|
227
|
+
) {
|
|
228
|
+
return undefined;
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
// timeoutSec stays finite-only here; the 1..MAX range enforcement remains
|
|
232
|
+
// in spawn-policy §7 (evaluateDelegateAdmission).
|
|
233
|
+
if (v.timeoutSec !== undefined && (typeof v.timeoutSec !== "number" || !Number.isFinite(v.timeoutSec))) {
|
|
234
|
+
return undefined;
|
|
235
|
+
}
|
|
236
|
+
return {
|
|
237
|
+
prompt: v.prompt,
|
|
238
|
+
...(typeof v.description === "string" ? { description: v.description } : {}),
|
|
239
|
+
...(typeof v.role === "string" ? { role: v.role } : {}),
|
|
240
|
+
...(typeof v.model === "string" ? { model: v.model } : {}),
|
|
241
|
+
...(v.maxTurns !== undefined ? { maxTurns: v.maxTurns as number } : {}),
|
|
242
|
+
...(v.budgetTokens !== undefined ? { budgetTokens: v.budgetTokens as number } : {}),
|
|
243
|
+
...(v.timeoutSec !== undefined ? { timeoutSec: v.timeoutSec as number } : {}),
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
179
247
|
export interface WaitResolveParams {
|
|
180
248
|
to: string;
|
|
181
249
|
questionId: string;
|
|
@@ -119,9 +119,18 @@ export class WaitStatusCache {
|
|
|
119
119
|
/**
|
|
120
120
|
* Stat-gated load. Fast path (stamp unchanged): 2 statSync calls, zero
|
|
121
121
|
* parses. Slow path (first sight or changed mtime/size): delegate to the
|
|
122
|
-
* loader, then record the
|
|
123
|
-
*
|
|
124
|
-
*
|
|
122
|
+
* loader, then record the PRE-load stamp (H5, DR-03 round-6 2026-10-16).
|
|
123
|
+
*
|
|
124
|
+
* Why PRE-load: keying the entry on the POST-load stamp had a
|
|
125
|
+
* parse→stat TOCTOU bug — a writer landing between the parse and the
|
|
126
|
+
* after-stat produced an entry whose stamp was FRESH while its cached
|
|
127
|
+
* value was STALE, and every later tick with an unchanged disk served
|
|
128
|
+
* the stale value forever (stale-under-fresh pinning). With the PRE-load
|
|
129
|
+
* stamp, a writer that lands anywhere between the pre-stat and the parse
|
|
130
|
+
* leaves the entry keyed on the OLDER stamp, so the very next tick sees
|
|
131
|
+
* the stamp change and reloads. Worst case is one spurious reload
|
|
132
|
+
* (writer landed before the parse — the loaded value was already fresh),
|
|
133
|
+
* never a missed one.
|
|
125
134
|
*/
|
|
126
135
|
load(cwd: string, runId: string): LoadedRun {
|
|
127
136
|
// Same path derivation as resolveRunStateRoot inside
|
|
@@ -135,13 +144,13 @@ export class WaitStatusCache {
|
|
|
135
144
|
}
|
|
136
145
|
const loaded = this.loader(cwd, runId);
|
|
137
146
|
this.reloadsValue += 1;
|
|
138
|
-
const after = statStamp(paths.manifestPath, paths.tasksPath);
|
|
139
147
|
if (this.entries.size >= WAIT_STATUS_CACHE_MAX && !this.entries.has(runId)) {
|
|
140
148
|
// Map preserves insertion order — evict the oldest entry.
|
|
141
149
|
const oldest = this.entries.keys().next().value;
|
|
142
150
|
if (oldest !== undefined) this.entries.delete(oldest);
|
|
143
151
|
}
|
|
144
|
-
|
|
152
|
+
// H5: key on the PRE-load stamp (`before`), NOT a post-load re-stat.
|
|
153
|
+
this.entries.set(runId, { ...before, loaded });
|
|
145
154
|
return loaded;
|
|
146
155
|
}
|
|
147
156
|
|