pi-crew 0.11.23 → 0.11.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/agents/explorer.md +3 -3
- package/agents/librarian.md +3 -3
- package/dist/index.mjs +528 -525
- package/docs/dynamic-workflows.md +14 -2
- package/package.json +4 -4
- package/src/agents/discover-agents.ts +8 -2
- package/src/config/config-validation.ts +3 -1
- package/src/config/env-allowlist-registry.ts +5 -0
- package/src/config/env-vars.ts +9 -6
- package/src/config/role-tools.ts +7 -6
- package/src/extension/management.ts +7 -0
- package/src/extension/register.ts +15 -0
- package/src/extension/run-index.ts +24 -1
- package/src/extension/team-tool/destructive-gate.ts +208 -0
- package/src/extension/team-tool/goal-wrap.ts +11 -1
- package/src/extension/team-tool/goal.ts +13 -1
- package/src/extension/team-tool.ts +10 -0
- package/src/observability/exporters/otlp-exporter.ts +11 -1
- package/src/prompt/prompt-runtime.ts +102 -18
- package/src/runtime/broker/crew-broker-client.ts +7 -0
- package/src/runtime/broker/crew-broker.ts +104 -132
- package/src/runtime/broker/protocol/connection-state.ts +159 -1
- package/src/runtime/broker/protocol/events-replay.ts +50 -10
- package/src/runtime/child-pi/child-pi-constants.ts +9 -0
- package/src/runtime/child-pi/child-pi-rpc.ts +72 -7
- package/src/runtime/child-pi/child-pi-streams.ts +19 -3
- package/src/runtime/child-pi/postmortem-registry.ts +8 -2
- package/src/runtime/classifier/ambient-noise-gate.ts +5 -1
- package/src/runtime/compaction/tool-output-pruner.ts +20 -2
- package/src/runtime/crew-agent-records.ts +67 -4
- package/src/runtime/custom-tools/submit-result-tool.ts +17 -0
- package/src/runtime/deadletter.ts +5 -1
- package/src/runtime/dispatch-batch.ts +51 -10
- package/src/runtime/effectiveness.ts +24 -1
- package/src/runtime/finalize-run.ts +12 -1
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +12 -3
- package/src/runtime/goal-workflow/goal-evaluator.ts +30 -5
- package/src/runtime/goal-workflow/goal-loop-runner.ts +60 -3
- package/src/runtime/live-session/live-agent-control.ts +34 -14
- package/src/runtime/live-session/live-agent-manager.ts +20 -5
- package/src/runtime/live-session/live-session-runtime.ts +66 -81
- package/src/runtime/live-session/usage-bridge.ts +66 -2
- package/src/runtime/mcp-proxy.ts +10 -0
- package/src/runtime/model/model-fallback.ts +68 -0
- package/src/runtime/model/runtime-policy.ts +20 -6
- package/src/runtime/model/runtime-resolver.ts +5 -3
- package/src/runtime/orphan-worker-registry.ts +64 -24
- package/src/runtime/pi-spawn.ts +4 -1
- package/src/runtime/recovery/crash-recovery.ts +35 -2
- package/src/runtime/scheduling/batch-barrier.ts +42 -5
- package/src/runtime/scheduling/scheduler.ts +74 -29
- package/src/runtime/stale-reconciler.ts +41 -5
- package/src/runtime/surface/herdr-provider.ts +54 -1
- package/src/runtime/surface/launch-script.ts +7 -1
- package/src/runtime/surface/resolve-surface.ts +4 -1
- package/src/runtime/surface/tmux-provider.ts +32 -2
- package/src/runtime/task-runner/child-executor.ts +3 -2
- package/src/runtime/workspace-lock.ts +52 -4
- package/src/state/contracts.ts +2 -0
- package/src/state/coordination/process-lock.ts +50 -0
- package/src/state/event-log/cursor.ts +10 -1
- package/src/state/event-log/event-log-rotation.ts +26 -2
- package/src/state/event-log/worker-atomic-writer.ts +5 -0
- package/src/state/stores/active-run-registry.ts +56 -5
- package/src/state/stores/ownership-map.ts +26 -7
- package/src/state/stores/plan-store.ts +30 -1
- package/src/state/stores/sqlite-run-state.ts +91 -1
- package/src/state/types.ts +5 -0
- package/src/ui/dashboard-panes/agents-pane.ts +4 -3
- package/src/ui/dashboard-panes/plan-pane.ts +5 -3
- package/src/ui/dashboard-panes/transcript-pane.ts +2 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +5 -3
- package/src/ui/run-dashboard.ts +6 -4
- package/src/ui/run-snapshot-cache.ts +3 -2
- package/src/ui/settings-overlay.ts +68 -10
- package/src/ui/theme-discovery.ts +26 -7
- package/src/ui/transcript-viewer.ts +5 -3
- package/src/utils/visual.ts +84 -0
|
@@ -408,6 +408,65 @@ export function shouldRegisterAskTool(env: NodeJS.ProcessEnv = process.env): boo
|
|
|
408
408
|
|
|
409
409
|
const PI_CREW_DELEGATE_ENABLED_ENV = "PI_CREW_DELEGATE_ENABLED";
|
|
410
410
|
export const DELEGATE_TIMED_OUT_RESULT = "[delegate timed out]";
|
|
411
|
+
|
|
412
|
+
// ── R9-F1 (2026-10-17): delegate-result recovery across parent turn-restarts ──
|
|
413
|
+
// A parent session can restart mid-turn (pi-core seam): the delegate poll loop
|
|
414
|
+
// dies with it, the grandchild later COMPLETES durably into the task inbox,
|
|
415
|
+
// and nothing ever re-reads that message — the report is silently lost
|
|
416
|
+
// (runtime-proven: run event seq 209/216/281/431, mailbox msg_b0297f83).
|
|
417
|
+
// Conservative, seam-level mitigation (the pi-core trigger is external):
|
|
418
|
+
// - every in-flight subId is tracked in `inFlightDelegateSubIds`;
|
|
419
|
+
// - whenever a delegate poll loop (re)starts, the inbox is scanned ONCE for
|
|
420
|
+
// results of still-pending subIds that have no delivered result yet and
|
|
421
|
+
// they are re-injected into `recoveredDelegateResults` (logged as
|
|
422
|
+
// `delegate.result.recovered`) so the owning loop's next iteration — or a
|
|
423
|
+
// RESTARTED loop for the same subId — surfaces it without re-deriving it;
|
|
424
|
+
// - the timed-out notice points at the durable copy in the task inbox.
|
|
425
|
+
// `no double-count`: a recovered message is consumed exactly once (deleted
|
|
426
|
+
// from the map on delivery) and the inbox stays append-only ground truth.
|
|
427
|
+
const inFlightDelegateSubIds = new Set<string>();
|
|
428
|
+
const recoveredDelegateResults = new Map<string, MailboxMessage>();
|
|
429
|
+
const RECOVERED_DELEGATE_RESULTS_MAX = 64;
|
|
430
|
+
|
|
431
|
+
/** R9-F1: scan the task inbox for completed results of still-pending subIds.
|
|
432
|
+
* Pure w.r.t. its inputs (injectable reader for tests); mutates the module
|
|
433
|
+
* recovery map + logs one `delegate.result.recovered` internal event per
|
|
434
|
+
* recovered subId. Returns the recovered messages keyed by subId. */
|
|
435
|
+
export function recoverPendingDelegateResults(
|
|
436
|
+
manifest: TeamRunManifest,
|
|
437
|
+
pendingSubIds: ReadonlySet<string>,
|
|
438
|
+
readInbox: (m: TeamRunManifest) => MailboxMessage[] = (m) => readAllMailboxMessages(m, "inbox"),
|
|
439
|
+
): Map<string, MailboxMessage> {
|
|
440
|
+
const recovered = new Map<string, MailboxMessage>();
|
|
441
|
+
if (pendingSubIds.size === 0) return recovered;
|
|
442
|
+
let messages: MailboxMessage[];
|
|
443
|
+
try {
|
|
444
|
+
messages = readInbox(manifest);
|
|
445
|
+
} catch (error) {
|
|
446
|
+
logInternalError("prompt-runtime.delegate-recovery-inbox-read", error instanceof Error ? error : new Error(String(error)), "warn");
|
|
447
|
+
return recovered;
|
|
448
|
+
}
|
|
449
|
+
for (const message of messages) {
|
|
450
|
+
const from = message.from ?? "";
|
|
451
|
+
if (!from.startsWith("delegate:")) continue;
|
|
452
|
+
const subId = from.slice("delegate:".length);
|
|
453
|
+
if (!pendingSubIds.has(subId)) continue;
|
|
454
|
+
if (recoveredDelegateResults.has(subId)) continue; // already recovered — no double-count
|
|
455
|
+
recoveredDelegateResults.set(subId, message);
|
|
456
|
+
while (recoveredDelegateResults.size > RECOVERED_DELEGATE_RESULTS_MAX) {
|
|
457
|
+
const oldest = recoveredDelegateResults.keys().next().value;
|
|
458
|
+
if (oldest === undefined) break;
|
|
459
|
+
recoveredDelegateResults.delete(oldest);
|
|
460
|
+
}
|
|
461
|
+
recovered.set(subId, message);
|
|
462
|
+
logInternalError(
|
|
463
|
+
"delegate.result.recovered",
|
|
464
|
+
new Error(`recovered completed delegate result from task inbox after a parent turn-restart (subId=${subId})`),
|
|
465
|
+
"warn",
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
return recovered;
|
|
469
|
+
}
|
|
411
470
|
/** P3-11: poll slack past the server deadline — the server timer fires first
|
|
412
471
|
* and writes the fenced (timed out) result; the client checks a bit longer
|
|
413
472
|
* so the outcome lands in-tool instead of lingering unread in the inbox. */
|
|
@@ -540,24 +599,40 @@ export function createDelegateTool(deps: DelegateToolDeps = {}): DelegateToolDef
|
|
|
540
599
|
const fromTag = `delegate:${subId}`;
|
|
541
600
|
let terminal: "completed" | "timed-out" | "aborted" = "timed-out";
|
|
542
601
|
let result: MailboxMessage | undefined;
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
602
|
+
// R9-F1: register the subId as in-flight and, at every loop (re)start,
|
|
603
|
+
// re-inject results of still-pending subIds that a previous (restarted)
|
|
604
|
+
// turn never delivered — they become visible to THIS loop's reads.
|
|
605
|
+
inFlightDelegateSubIds.add(subId);
|
|
606
|
+
try {
|
|
607
|
+
recoverPendingDelegateResults(manifest, inFlightDelegateSubIds);
|
|
608
|
+
while (true) {
|
|
609
|
+
if (signal?.aborted) {
|
|
610
|
+
terminal = "aborted";
|
|
611
|
+
break;
|
|
612
|
+
}
|
|
613
|
+
// Consume a recovered result FIRST (exactly once — map delete), then
|
|
614
|
+
// fall through to the durable inbox read (append-only ground truth).
|
|
615
|
+
result = recoveredDelegateResults.get(subId);
|
|
616
|
+
if (result) recoveredDelegateResults.delete(subId);
|
|
617
|
+
if (!result) {
|
|
618
|
+
try {
|
|
619
|
+
result = readAllMailboxMessages(manifest, "inbox").find(
|
|
620
|
+
(m) => m.from === fromTag && (m.taskId === undefined || m.taskId === taskId),
|
|
621
|
+
);
|
|
622
|
+
} catch {
|
|
623
|
+
/* transient read error — keep polling */
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
if (result) {
|
|
627
|
+
terminal = "completed";
|
|
628
|
+
break;
|
|
629
|
+
}
|
|
630
|
+
if (now() >= deadline) break;
|
|
631
|
+
await sleep(effectiveSteeringInterval(hasLiveControlRealtimeListeners()));
|
|
558
632
|
}
|
|
559
|
-
|
|
560
|
-
|
|
633
|
+
} finally {
|
|
634
|
+
inFlightDelegateSubIds.delete(subId);
|
|
635
|
+
recoveredDelegateResults.delete(subId);
|
|
561
636
|
}
|
|
562
637
|
const waitedMs = now() - startedAt;
|
|
563
638
|
if (terminal === "completed" && result) {
|
|
@@ -574,8 +649,17 @@ export function createDelegateTool(deps: DelegateToolDeps = {}): DelegateToolDef
|
|
|
574
649
|
details: { status: "aborted", grandchildTaskRef: subId, waitedMs },
|
|
575
650
|
};
|
|
576
651
|
}
|
|
652
|
+
// R9-F1: recovery hint — the grandchild can still complete AFTER this
|
|
653
|
+
// timeout (and after a parent turn-restart the tool result itself can be
|
|
654
|
+
// lost); the durable copy lands in the task inbox. Point the model at it
|
|
655
|
+
// instead of silently redoing the delegated work.
|
|
577
656
|
return {
|
|
578
|
-
content: [
|
|
657
|
+
content: [
|
|
658
|
+
{
|
|
659
|
+
type: "text" as const,
|
|
660
|
+
text: `${DELEGATE_TIMED_OUT_RESULT} — the grandchild may still complete; its result lands durably in the task inbox (from=delegate:${subId}) — check the task inbox / lastResult before redoing the work, or ask the leader to relay it.`,
|
|
661
|
+
},
|
|
662
|
+
],
|
|
579
663
|
details: { status: "timed-out", grandchildTaskRef: subId, waitedMs },
|
|
580
664
|
};
|
|
581
665
|
} finally {
|
|
@@ -223,6 +223,13 @@ export class CrewBrokerClient {
|
|
|
223
223
|
Math.max(1, Math.floor(timeoutMs)),
|
|
224
224
|
);
|
|
225
225
|
if (entry) entry.timer = requestTimer;
|
|
226
|
+
// R2-L-A3 (2026-10-17): unref the per-request timer (mirror the hello/
|
|
227
|
+
// backoff timers at :466-474) — a stranded in-flight request must not
|
|
228
|
+
// hold a finishing child's event loop open for up to timeoutMs. The
|
|
229
|
+
// typeof guard keeps injected test timer fakes (no unref) working.
|
|
230
|
+
if (requestTimer && typeof (requestTimer as { unref?: () => void }).unref === "function") {
|
|
231
|
+
(requestTimer as { unref?: () => void }).unref?.();
|
|
232
|
+
}
|
|
226
233
|
try {
|
|
227
234
|
const frame = encodeBrokerFrame({ id, method, params });
|
|
228
235
|
// Write may emit EPIPE etc. We don't await drain here — the response
|
|
@@ -64,6 +64,13 @@ import { type DelegateEventTarget, type DelegateEventType, recordDelegateEvent }
|
|
|
64
64
|
import { promoteShadowToRunning } from "./delegate/shadow-lifecycle.ts";
|
|
65
65
|
import { fanoutMailboxMessage } from "./mailbox-observer/mailbox-fanout.ts";
|
|
66
66
|
import type { CrewBrokerOptions, ServerConnection } from "./protocol/connection-state.ts";
|
|
67
|
+
// Outbound-queue management + caps (wc-gate move — bodies in protocol/connection-state.ts)
|
|
68
|
+
import {
|
|
69
|
+
handleOutboundOverflowFrame,
|
|
70
|
+
type OutboundAdmission,
|
|
71
|
+
type OutboundQueueHooks,
|
|
72
|
+
writeOrQueueFrame,
|
|
73
|
+
} from "./protocol/connection-state.ts";
|
|
67
74
|
import { handleEventsSince } from "./protocol/events-replay.ts";
|
|
68
75
|
import { DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, type LockBusyOutcome, withRunLockBusyRetry } from "./protocol/lock-busy.ts";
|
|
69
76
|
import { loadRunForHello } from "./protocol/manifest-loader.ts";
|
|
@@ -107,29 +114,6 @@ const HELLO_DEADLINE_MS = 1_000;
|
|
|
107
114
|
* blocked run still authenticates normally. */
|
|
108
115
|
const STALE_RUN_STATUSES: ReadonlySet<string> = new Set(["completed", "failed", "cancelled"]);
|
|
109
116
|
|
|
110
|
-
/** Default per-connection outbound queue cap (events). */
|
|
111
|
-
const DEFAULT_OUTBOUND_QUEUE_CAP = 256;
|
|
112
|
-
|
|
113
|
-
/** R6-5 (2026-10-16): typed protocol close-reason sent as a best-effort
|
|
114
|
-
* `protocol.close` event frame (direct write, jumps the saturated queue)
|
|
115
|
-
* before the connection is closed after a fanout/event frame was dropped
|
|
116
|
-
* at the queue cap. Clients observe the reason, reconnect, and replay via
|
|
117
|
-
* `events.since` (Phase 1.5). */
|
|
118
|
-
export const OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON = "outbound-overflow-resync";
|
|
119
|
-
|
|
120
|
-
/** R6-5: admission class for an outbound frame.
|
|
121
|
-
* - `false` — fanout/event class: drop-newest at cap → resync close.
|
|
122
|
-
* - `true` — close-reason class: bypass the cap, attempt an immediate
|
|
123
|
-
* direct write (existing `sendErrorAndClose` behavior).
|
|
124
|
-
* - `"response"` — request-response class (sendResult/sendError acks):
|
|
125
|
-
* bypass the drop-newest cap but stay IN the queue (ordering preserved,
|
|
126
|
-
* backpressure honored). Bounded by the client's in-flight request count
|
|
127
|
-
* — one response frame per request — so it can never storm the way a
|
|
128
|
-
* fanout burst can. Kept a `boolean | "response"` union (not a fresh
|
|
129
|
-
* enum) so the external FanoutWriters interface (mailbox-fanout.ts) keeps
|
|
130
|
-
* its `(conn, buf, force: boolean)` shape. */
|
|
131
|
-
type OutboundAdmission = boolean | "response";
|
|
132
|
-
|
|
133
117
|
export class CrewBroker {
|
|
134
118
|
private readonly options: Required<Pick<CrewBrokerOptions, "sessionId" | "enabled" | "waitMethodsEnabled" | "nestingEnabled">> &
|
|
135
119
|
Pick<
|
|
@@ -137,6 +121,7 @@ export class CrewBroker {
|
|
|
137
121
|
| "socketPath"
|
|
138
122
|
| "maxFrameBytes"
|
|
139
123
|
| "outboundQueueCap"
|
|
124
|
+
| "outboundQueueBytesCap"
|
|
140
125
|
| "cwd"
|
|
141
126
|
| "netModule"
|
|
142
127
|
| "waitStatusCache"
|
|
@@ -189,6 +174,7 @@ export class CrewBroker {
|
|
|
189
174
|
socketPath: options.socketPath,
|
|
190
175
|
maxFrameBytes: options.maxFrameBytes,
|
|
191
176
|
outboundQueueCap: options.outboundQueueCap,
|
|
177
|
+
outboundQueueBytesCap: options.outboundQueueBytesCap,
|
|
192
178
|
cwd: options.cwd,
|
|
193
179
|
netModule: options.netModule,
|
|
194
180
|
waitMethodsEnabled: options.waitMethodsEnabled === true,
|
|
@@ -473,6 +459,7 @@ export class CrewBroker {
|
|
|
473
459
|
role: undefined,
|
|
474
460
|
authMatchKind: undefined,
|
|
475
461
|
outbound: [],
|
|
462
|
+
queuedBytes: 0,
|
|
476
463
|
needsResync: false,
|
|
477
464
|
closed: false,
|
|
478
465
|
helloTimer: null,
|
|
@@ -822,6 +809,27 @@ export class CrewBroker {
|
|
|
822
809
|
err instanceof Error ? err : new Error(String(err)),
|
|
823
810
|
`sessionId=${this.options.sessionId}`,
|
|
824
811
|
);
|
|
812
|
+
// R9-N1 (2026-10-17): silently dropping a REQUEST-RESPONSE frame here
|
|
813
|
+
// (e.g. an events.since replay page larger than MAX_BROKER_FRAME_BYTES)
|
|
814
|
+
// left the client waiting on its id until its request timeout — a
|
|
815
|
+
// hang with no diagnostic. When the payload carries an id, answer it
|
|
816
|
+
// with a small typed error frame on the SAME id; if even that cannot
|
|
817
|
+
// encode, close (the client's reconnect+replay is the only durable
|
|
818
|
+
// answer left).
|
|
819
|
+
const id = (payload as { id?: unknown } | null)?.id;
|
|
820
|
+
if (typeof id === "number" || typeof id === "string") {
|
|
821
|
+
try {
|
|
822
|
+
const errBuf = encodeBrokerFrame({
|
|
823
|
+
id,
|
|
824
|
+
error: { code: "response-oversize", message: "response frame exceeds the broker frame limit" },
|
|
825
|
+
});
|
|
826
|
+
this.writeOrQueue(conn, errBuf, /*force*/ true);
|
|
827
|
+
return;
|
|
828
|
+
} catch {
|
|
829
|
+
/* fall through to close */
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
this.closeConnection(conn);
|
|
825
833
|
return;
|
|
826
834
|
}
|
|
827
835
|
// R6-5: request responses (the acks for ping/msg.send/wait.request/…)
|
|
@@ -832,91 +840,19 @@ export class CrewBroker {
|
|
|
832
840
|
}
|
|
833
841
|
|
|
834
842
|
private writeOrQueue(conn: ServerConnection, buf: Buffer, force: OutboundAdmission): void {
|
|
835
|
-
|
|
836
|
-
const cap = this.options.outboundQueueCap ?? DEFAULT_OUTBOUND_QUEUE_CAP;
|
|
837
|
-
if (conn.outbound.length >= cap) {
|
|
838
|
-
if (force === "response") {
|
|
839
|
-
// R6-5(a): response frames queue PAST the cap instead of being
|
|
840
|
-
// drop-newest'd. Bounded: at most one response per in-flight
|
|
841
|
-
// request, and the client's request pipeline is serial — this can
|
|
842
|
-
// never grow unbounded the way a fanout burst can. Ordering is
|
|
843
|
-
// preserved (pushed at the tail; drainOutbound honors backpressure).
|
|
844
|
-
conn.outbound.push(buf);
|
|
845
|
-
this.drainOutbound(conn);
|
|
846
|
-
return;
|
|
847
|
-
}
|
|
848
|
-
if (force) {
|
|
849
|
-
// Forced sends (e.g. close-reason) bypass the cap and attempt
|
|
850
|
-
// to flush directly; if the socket is busy they may still drop.
|
|
851
|
-
try {
|
|
852
|
-
conn.socket.write(buf);
|
|
853
|
-
} catch {
|
|
854
|
-
/* socket may have closed; the close handler will sweep. */
|
|
855
|
-
}
|
|
856
|
-
return;
|
|
857
|
-
}
|
|
858
|
-
// R6-5(b): fanout/event frame dropped at cap (drop-newest). The old
|
|
859
|
-
// `needsResync` marker was write-only (set, never read) — the claimed
|
|
860
|
-
// "pause + client notices" recovery did not exist, so live frames
|
|
861
|
-
// were silently dropped for the connection's remaining lifetime.
|
|
862
|
-
// Now the drop CLOSES the connection with a typed close-reason so
|
|
863
|
-
// the client reconnects and replays (events.since / msg.inbox).
|
|
864
|
-
this.handleOutboundOverflow(conn);
|
|
865
|
-
return;
|
|
866
|
-
}
|
|
867
|
-
conn.outbound.push(buf);
|
|
868
|
-
this.drainOutbound(conn);
|
|
843
|
+
writeOrQueueFrame(conn, buf, force, this.options, this.outboundHooks());
|
|
869
844
|
}
|
|
870
845
|
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
if (buf === undefined) break;
|
|
875
|
-
// Try the write; if it returns false, wait for drain before pushing more.
|
|
876
|
-
try {
|
|
877
|
-
const ok = conn.socket.write(buf);
|
|
878
|
-
if (!ok) {
|
|
879
|
-
// Backpressure — re-arm on drain event.
|
|
880
|
-
conn.socket.once("drain", () => {
|
|
881
|
-
if (!conn.closed) this.drainOutbound(conn);
|
|
882
|
-
});
|
|
883
|
-
return;
|
|
884
|
-
}
|
|
885
|
-
} catch {
|
|
886
|
-
// Write failed — close the connection (we already had it open).
|
|
887
|
-
this.closeConnection(conn);
|
|
888
|
-
return;
|
|
889
|
-
}
|
|
890
|
-
conn.outbound.shift();
|
|
891
|
-
conn.outboundSeq += 1;
|
|
892
|
-
}
|
|
846
|
+
/** R6-5(b): fanout/event frame dropped at cap → typed resync close (moved body in connection-state.ts). */
|
|
847
|
+
private handleOutboundOverflow(conn: ServerConnection): void {
|
|
848
|
+
handleOutboundOverflowFrame(conn, this.options, this.outboundHooks());
|
|
893
849
|
}
|
|
894
850
|
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
* is best-effort, exactly like `sendErrorAndClose`. */
|
|
901
|
-
private handleOutboundOverflow(conn: ServerConnection): void {
|
|
902
|
-
// Single-flight: fanoutMailboxMessage iterates a snapshot of the run's
|
|
903
|
-
// connections, so one burst can re-enter here for an already-overflowing
|
|
904
|
-
// conn before `closeConnection`'s `closed` guard becomes visible. The
|
|
905
|
-
// flag is the "close already initiated" bit (its only read).
|
|
906
|
-
if (conn.needsResync) return;
|
|
907
|
-
conn.needsResync = true;
|
|
908
|
-
logInternalError(
|
|
909
|
-
"crew-broker.outbound.overflow",
|
|
910
|
-
new Error("outbound queue overflow — closing connection for resync"),
|
|
911
|
-
`sessionId=${this.options.sessionId} runId=${conn.runId ?? "?"} queued=${conn.outbound.length} seq=${conn.outboundSeq}`,
|
|
912
|
-
);
|
|
913
|
-
try {
|
|
914
|
-
const buf = encodeBrokerFrame({ event: "protocol.close", data: { reason: OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON } });
|
|
915
|
-
conn.socket.write(buf);
|
|
916
|
-
} catch {
|
|
917
|
-
/* socket may be dead; closeConnection below sweeps regardless */
|
|
918
|
-
}
|
|
919
|
-
this.closeConnection(conn);
|
|
851
|
+
private outboundHooks(): OutboundQueueHooks {
|
|
852
|
+
return {
|
|
853
|
+
onOverflow: (conn) => this.handleOutboundOverflow(conn),
|
|
854
|
+
onClose: (conn) => this.closeConnection(conn),
|
|
855
|
+
};
|
|
920
856
|
}
|
|
921
857
|
|
|
922
858
|
// Phase 1: msg.send + msg.inbox handlers
|
|
@@ -1298,6 +1234,11 @@ export class CrewBroker {
|
|
|
1298
1234
|
const v = params && typeof params === "object" && !Array.isArray(params) ? (params as Record<string, unknown>) : {};
|
|
1299
1235
|
const targetTaskId = typeof v.taskId === "string" ? v.taskId : undefined;
|
|
1300
1236
|
const body = typeof v.body === "string" ? v.body : undefined;
|
|
1237
|
+
// R2-L-A4 (2026-10-17): optional client requestId — mirrors msg.send's U5
|
|
1238
|
+
// dedup. An orchestrator retry after ack-loss used to DUPLICATE the steer
|
|
1239
|
+
// (mailbox append + steering file); a stable requestId makes the resend a
|
|
1240
|
+
// no-op.
|
|
1241
|
+
const requestId = typeof v.requestId === "string" && v.requestId.length > 0 ? v.requestId.slice(0, 128) : undefined;
|
|
1301
1242
|
if (!targetTaskId || body === undefined) {
|
|
1302
1243
|
this.sendError(conn, id, "bad-params", "steer.push: taskId and body are required");
|
|
1303
1244
|
return;
|
|
@@ -1320,39 +1261,61 @@ export class CrewBroker {
|
|
|
1320
1261
|
|
|
1321
1262
|
const messageId = `steer_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
1322
1263
|
// Write 1: mailbox — live broker fanout + persistent inbox read.
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1264
|
+
// R2-L-A4: idempotent append when the client supplied a requestId —
|
|
1265
|
+
// resend-after-ack-loss dedupes instead of duplicating the steer.
|
|
1266
|
+
let deduped = false;
|
|
1267
|
+
if (requestId) {
|
|
1268
|
+
const outcome = await appendMailboxMessageIdempotentAsync(loaded.manifest, {
|
|
1269
|
+
id: messageId,
|
|
1270
|
+
direction: "inbox",
|
|
1271
|
+
from: conn.taskId ?? conn.runId,
|
|
1272
|
+
to: targetTaskId,
|
|
1273
|
+
taskId: targetTaskId,
|
|
1274
|
+
body,
|
|
1275
|
+
kind: "steer",
|
|
1276
|
+
priority: (v.priority as "urgent" | "normal" | "low" | undefined) ?? "urgent",
|
|
1277
|
+
deliveryMode: "interrupt",
|
|
1278
|
+
requestId,
|
|
1279
|
+
});
|
|
1280
|
+
deduped = outcome.deduped;
|
|
1281
|
+
} else {
|
|
1282
|
+
await appendMailboxMessageAsync(loaded.manifest, {
|
|
1283
|
+
id: messageId,
|
|
1284
|
+
direction: "inbox",
|
|
1285
|
+
from: conn.taskId ?? conn.runId,
|
|
1286
|
+
to: targetTaskId,
|
|
1287
|
+
taskId: targetTaskId,
|
|
1288
|
+
body,
|
|
1289
|
+
kind: "steer",
|
|
1290
|
+
priority: (v.priority as "urgent" | "normal" | "low" | undefined) ?? "urgent",
|
|
1291
|
+
deliveryMode: "interrupt",
|
|
1292
|
+
});
|
|
1293
|
+
}
|
|
1334
1294
|
// Write 2: steering file — durable fallback so pollSteering() picks
|
|
1335
1295
|
// up the steer even when the recipient child's broker connection is down. Matches the
|
|
1336
1296
|
// JSONL format of appendSteeringAsync in task-runner.ts.
|
|
1337
1297
|
// Best-effort: a failure here must NOT fail the push (mailbox write
|
|
1338
1298
|
// already succeeded).
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1299
|
+
// R2-L-A4: a deduped resend must NOT re-append the steering file either.
|
|
1300
|
+
if (!deduped) {
|
|
1301
|
+
try {
|
|
1302
|
+
const steeringDir = `${loaded.manifest.artifactsRoot}/steering`;
|
|
1303
|
+
const steeringPath = resolveRealContainedPath(loaded.manifest.artifactsRoot, `steering/${targetTaskId}.jsonl`);
|
|
1304
|
+
const line =
|
|
1305
|
+
JSON.stringify({
|
|
1306
|
+
type: "steer",
|
|
1307
|
+
message: body,
|
|
1308
|
+
id: messageId,
|
|
1309
|
+
ts: new Date().toISOString(),
|
|
1310
|
+
}) + "\n";
|
|
1311
|
+
await fsp.mkdir(steeringDir, { recursive: true });
|
|
1312
|
+
await fsp.appendFile(steeringPath, line, "utf-8");
|
|
1313
|
+
} catch (fileErr) {
|
|
1314
|
+
const safeMessage = fileErr instanceof Error ? redactSecretString(fileErr.message) : "";
|
|
1315
|
+
logInternalError("crew-broker.steer-file-write-failed", new Error(safeMessage), `taskId=${targetTaskId}`);
|
|
1316
|
+
}
|
|
1354
1317
|
}
|
|
1355
|
-
this.sendResult(conn, id, { messageId, taskId: targetTaskId, durable: true });
|
|
1318
|
+
this.sendResult(conn, id, { messageId, taskId: targetTaskId, durable: true, ...(requestId ? { requestId, deduped } : {}) });
|
|
1356
1319
|
} catch (err) {
|
|
1357
1320
|
this.sendError(conn, id, "steer-failed", (err as Error).message);
|
|
1358
1321
|
}
|
|
@@ -1389,6 +1352,15 @@ export class CrewBroker {
|
|
|
1389
1352
|
}
|
|
1390
1353
|
// Default recipient: the sender's taskId (the orchestrator that
|
|
1391
1354
|
// spawned this worker). If 'to' is provided, use it instead.
|
|
1355
|
+
// R2-L-A6 (wave-2b): 'to' is SENDER-CHOSEN and lands in a mailbox
|
|
1356
|
+
// record (taskId field) — an arbitrary value would let a worker
|
|
1357
|
+
// address any task in the run (spoofed follow-up routing). Restrict
|
|
1358
|
+
// to the two identities the sender legitimately owns: its run and
|
|
1359
|
+
// its own (parent) task.
|
|
1360
|
+
if (to !== undefined && to !== conn.runId && to !== conn.taskId) {
|
|
1361
|
+
this.sendError(conn, id, "bad-params", `escalate: 'to' must be this connection's run or task`);
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1392
1364
|
const target = to ?? conn.taskId ?? conn.runId;
|
|
1393
1365
|
try {
|
|
1394
1366
|
const loaded = loadRunManifestById(cwd, conn.runId);
|
|
@@ -1884,7 +1856,7 @@ export class CrewBroker {
|
|
|
1884
1856
|
const updatedManifest = {
|
|
1885
1857
|
...fresh.manifest,
|
|
1886
1858
|
// manifest.status stays "running" — park NEVER flips run status (ADR item 3).
|
|
1887
|
-
waitState: { taskId, questionId, askedAt },
|
|
1859
|
+
waitState: { taskId, questionId, askedAt, deadline }, // R2-M-A2: mirror task.waiting.deadline for shield expiry
|
|
1888
1860
|
updatedAt: askedAt,
|
|
1889
1861
|
};
|
|
1890
1862
|
saveRunTasks(updatedManifest, updatedTasks);
|
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import type * as net from "node:net";
|
|
12
|
-
import
|
|
12
|
+
import { logInternalError } from "../../../utils/internal-error.ts";
|
|
13
|
+
import { encodeBrokerFrame, type NdjsonDecoder } from "../../../utils/ndjson.ts";
|
|
13
14
|
import type { GrandchildSpawnInput, GrandchildSpawnResult } from "../../delegate-spawn.ts";
|
|
14
15
|
import type { WaitStatusCache } from "../wait-status-cache.ts";
|
|
15
16
|
|
|
@@ -22,6 +23,10 @@ export interface CrewBrokerOptions {
|
|
|
22
23
|
maxFrameBytes?: number;
|
|
23
24
|
/** Per-connection outbound queue cap. Default 256. */
|
|
24
25
|
outboundQueueCap?: number;
|
|
26
|
+
/** R8-M1: per-connection queued-BYTES cap (frames can be up to
|
|
27
|
+
* MAX_BROKER_FRAME_BYTES each, so the frame cap alone bounds the queue
|
|
28
|
+
* at cap × 256 KiB ≈ 64 MiB in-app). Default 4 MiB. */
|
|
29
|
+
outboundQueueBytesCap?: number;
|
|
25
30
|
/** Required: when false, start() is a no-op and the server never binds.
|
|
26
31
|
* Lets the lifecycle controller install the broker unconditionally and
|
|
27
32
|
* have a single kill switch. */
|
|
@@ -102,6 +107,9 @@ export interface ServerConnection {
|
|
|
102
107
|
authedSecretHash?: string;
|
|
103
108
|
/** Outbound queue of encoded frames awaiting drain. */
|
|
104
109
|
outbound: Buffer[];
|
|
110
|
+
/** R8-M1: sum of `outbound[i].length` — maintained by writeOrQueue/
|
|
111
|
+
* drainOutbound so the queued-BYTES cap does not rescan the queue. */
|
|
112
|
+
queuedBytes: number;
|
|
105
113
|
/** Set when the queue has hit the cap and a frame was dropped. */
|
|
106
114
|
needsResync: boolean;
|
|
107
115
|
/** Set when the connection is closing (idempotent). */
|
|
@@ -111,3 +119,153 @@ export interface ServerConnection {
|
|
|
111
119
|
/** Monotonic seq counter for outbound events (diagnostic). */
|
|
112
120
|
outboundSeq: number;
|
|
113
121
|
}
|
|
122
|
+
|
|
123
|
+
// ---------------------------------------------------------------------------
|
|
124
|
+
// Outbound queue management (moved from crew-broker.ts — M4 wc-gate ≤2000;
|
|
125
|
+
// pure move, no behavior change). The broker keeps thin delegating methods.
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
/** Default per-connection outbound queue cap (events). */
|
|
129
|
+
export const DEFAULT_OUTBOUND_QUEUE_CAP = 256;
|
|
130
|
+
|
|
131
|
+
/** R8-M1 (2026-10-17): default per-connection queued-BYTES cap. The frame cap
|
|
132
|
+
* alone lets a subscribed-but-never-reading client buffer 256 × 256 KiB ≈
|
|
133
|
+
* 64 MiB in-app before the R6-5 typed close engages (kernel backpressure
|
|
134
|
+
* shifts frames off the queue, so the count rarely grows — the BYTES the
|
|
135
|
+
* process holds do). ~4 MiB bounds a stuck connection's memory while a
|
|
136
|
+
* healthy LAN client drains comfortably inside it. */
|
|
137
|
+
export const DEFAULT_OUTBOUND_QUEUE_BYTES = 4 * 1024 * 1024;
|
|
138
|
+
|
|
139
|
+
/** R6-5 (2026-10-16): typed protocol close-reason sent as a best-effort
|
|
140
|
+
* `protocol.close` event frame (direct write, jumps the saturated queue)
|
|
141
|
+
* before the connection is closed after a fanout/event frame was dropped
|
|
142
|
+
* at the queue cap. Clients observe the reason, reconnect, and replay via
|
|
143
|
+
* `events.since` (Phase 1.5). */
|
|
144
|
+
export const OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON = "outbound-overflow-resync";
|
|
145
|
+
|
|
146
|
+
/** R6-5: admission class for an outbound frame.
|
|
147
|
+
* - `false` — fanout/event class: drop-newest at cap → resync close.
|
|
148
|
+
* - `true` — close-reason class: bypass the cap, attempt an immediate
|
|
149
|
+
* direct write (existing `sendErrorAndClose` behavior).
|
|
150
|
+
* - `"response"` — request-response class (sendResult/sendError acks):
|
|
151
|
+
* bypass the drop-newest cap but stay IN the queue (ordering preserved,
|
|
152
|
+
* backpressure honored). Bounded by the client's in-flight request count
|
|
153
|
+
* — one response frame per request — so it can never storm the way a
|
|
154
|
+
* fanout burst can. Kept a `boolean | "response"` union (not a fresh
|
|
155
|
+
* enum) so the external FanoutWriters interface (mailbox-fanout.ts) keeps
|
|
156
|
+
* its `(conn, buf, force: boolean)` shape. */
|
|
157
|
+
export type OutboundAdmission = boolean | "response";
|
|
158
|
+
|
|
159
|
+
export interface OutboundQueueOpts {
|
|
160
|
+
outboundQueueCap?: number;
|
|
161
|
+
outboundQueueBytesCap?: number;
|
|
162
|
+
sessionId: string;
|
|
163
|
+
}
|
|
164
|
+
export interface OutboundQueueHooks {
|
|
165
|
+
onOverflow(conn: ServerConnection): void;
|
|
166
|
+
onClose(conn: ServerConnection): void;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function writeOrQueueFrame(
|
|
170
|
+
conn: ServerConnection,
|
|
171
|
+
buf: Buffer,
|
|
172
|
+
force: OutboundAdmission,
|
|
173
|
+
opts: OutboundQueueOpts,
|
|
174
|
+
hooks: OutboundQueueHooks,
|
|
175
|
+
): void {
|
|
176
|
+
if (conn.closed) return;
|
|
177
|
+
const cap = opts.outboundQueueCap ?? DEFAULT_OUTBOUND_QUEUE_CAP;
|
|
178
|
+
// R8-M1: queued-BYTES cap alongside the frame cap. Few-but-large frames
|
|
179
|
+
// (a slow reader behind a 256 KiB fanout) never trip a 256-frame cap;
|
|
180
|
+
// without this the process buffers up to ~64 MiB per stuck connection.
|
|
181
|
+
// Overflow takes the same typed-close path as the frame cap (resync +
|
|
182
|
+
// reconnect + replay) — applied to ALL admission classes because the
|
|
183
|
+
// connection is about to close anyway and the alternative is unbounded
|
|
184
|
+
// memory, not a delivered response.
|
|
185
|
+
const bytesCap = opts.outboundQueueBytesCap ?? DEFAULT_OUTBOUND_QUEUE_BYTES;
|
|
186
|
+
if (conn.queuedBytes + buf.length > bytesCap) {
|
|
187
|
+
hooks.onOverflow(conn);
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
if (conn.outbound.length >= cap) {
|
|
191
|
+
if (force === "response") {
|
|
192
|
+
// R6-5(a): response frames queue PAST the cap instead of being
|
|
193
|
+
// drop-newest'd. Bounded: at most one response per in-flight
|
|
194
|
+
// request, and the client's request pipeline is serial — this can
|
|
195
|
+
// never grow unbounded the way a fanout burst can. Ordering is
|
|
196
|
+
// preserved (pushed at the tail; drainOutbound honors backpressure).
|
|
197
|
+
conn.outbound.push(buf);
|
|
198
|
+
conn.queuedBytes += buf.length;
|
|
199
|
+
drainOutboundFrames(conn, hooks);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
if (force) {
|
|
203
|
+
// Forced sends (e.g. close-reason) bypass the cap and attempt
|
|
204
|
+
// to flush directly; if the socket is busy they may still drop.
|
|
205
|
+
try {
|
|
206
|
+
conn.socket.write(buf);
|
|
207
|
+
} catch {
|
|
208
|
+
/* socket may have closed; the close handler will sweep. */
|
|
209
|
+
}
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
// R6-5(b): fanout/event frame dropped at cap (drop-newest). The old
|
|
213
|
+
// `needsResync` marker was write-only (set, never read) — the claimed
|
|
214
|
+
// "pause + client notices" recovery did not exist, so live frames
|
|
215
|
+
// were silently dropped for the connection's remaining lifetime.
|
|
216
|
+
// Now the drop CLOSES the connection with a typed close-reason so
|
|
217
|
+
// the client reconnects and replays (events.since / msg.inbox).
|
|
218
|
+
hooks.onOverflow(conn);
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
conn.outbound.push(buf);
|
|
222
|
+
conn.queuedBytes += buf.length;
|
|
223
|
+
drainOutboundFrames(conn, hooks);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
export function drainOutboundFrames(conn: ServerConnection, hooks: OutboundQueueHooks): void {
|
|
227
|
+
while (conn.outbound.length > 0) {
|
|
228
|
+
const buf = conn.outbound[0];
|
|
229
|
+
if (buf === undefined) break;
|
|
230
|
+
// Try the write; if it returns false, wait for drain before pushing more.
|
|
231
|
+
try {
|
|
232
|
+
const ok = conn.socket.write(buf);
|
|
233
|
+
if (!ok) {
|
|
234
|
+
// Backpressure — re-arm on drain event.
|
|
235
|
+
conn.socket.once("drain", () => {
|
|
236
|
+
if (!conn.closed) drainOutboundFrames(conn, hooks);
|
|
237
|
+
});
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
} catch {
|
|
241
|
+
// Write failed — close the connection (we already had it open).
|
|
242
|
+
hooks.onClose(conn);
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
conn.outbound.shift();
|
|
246
|
+
conn.queuedBytes -= buf.length;
|
|
247
|
+
conn.outboundSeq += 1;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
export function handleOutboundOverflowFrame(conn: ServerConnection, opts: OutboundQueueOpts, hooks: OutboundQueueHooks): void {
|
|
252
|
+
// Single-flight: fanoutMailboxMessage iterates a snapshot of the run's
|
|
253
|
+
// connections, so one burst can re-enter here for an already-overflowing
|
|
254
|
+
// conn before `closeConnection`'s `closed` guard becomes visible. The
|
|
255
|
+
// flag is the "close already initiated" bit (its only read).
|
|
256
|
+
if (conn.needsResync) return;
|
|
257
|
+
conn.needsResync = true;
|
|
258
|
+
logInternalError(
|
|
259
|
+
"crew-broker.outbound.overflow",
|
|
260
|
+
new Error("outbound queue overflow — closing connection for resync"),
|
|
261
|
+
`sessionId=${opts.sessionId}
|
|
262
|
+
runId=${conn.runId ?? "?"} queued=${conn.outbound.length} seq=${conn.outboundSeq}`,
|
|
263
|
+
);
|
|
264
|
+
try {
|
|
265
|
+
const buf = encodeBrokerFrame({ event: "protocol.close", data: { reason: OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON } });
|
|
266
|
+
conn.socket.write(buf);
|
|
267
|
+
} catch {
|
|
268
|
+
/* socket may be dead; closeConnection below sweeps regardless */
|
|
269
|
+
}
|
|
270
|
+
hooks.onClose(conn);
|
|
271
|
+
}
|