pi-crew 0.11.22 → 0.11.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/agents/explorer.md +3 -3
- package/agents/librarian.md +3 -3
- package/dist/index.mjs +523 -520
- package/docs/dynamic-workflows.md +14 -2
- package/package.json +4 -4
- package/src/agents/discover-agents.ts +8 -2
- package/src/config/config-validation.ts +3 -1
- package/src/config/env-allowlist-registry.ts +5 -0
- package/src/config/env-vars.ts +9 -6
- package/src/config/role-tools.ts +7 -6
- package/src/extension/management.ts +7 -0
- package/src/extension/register.ts +15 -0
- package/src/extension/run-index.ts +24 -1
- package/src/extension/team-tool/destructive-gate.ts +208 -0
- package/src/extension/team-tool/goal-wrap.ts +11 -1
- package/src/extension/team-tool/goal.ts +13 -1
- package/src/extension/team-tool/respond.ts +8 -6
- package/src/extension/team-tool/run.ts +4 -4
- package/src/extension/team-tool.ts +10 -0
- package/src/observability/exporters/otlp-exporter.ts +11 -1
- package/src/prompt/prompt-runtime.ts +102 -18
- package/src/prompt/surface-worker.ts +13 -1
- package/src/runtime/broker/crew-broker-client.ts +7 -0
- package/src/runtime/broker/crew-broker.ts +113 -152
- package/src/runtime/broker/protocol/connection-state.ts +159 -1
- package/src/runtime/broker/protocol/events-replay.ts +50 -10
- package/src/runtime/broker/protocol/request-parsers.ts +68 -0
- package/src/runtime/child-pi/child-pi-constants.ts +9 -0
- package/src/runtime/child-pi/child-pi-rpc.ts +72 -7
- package/src/runtime/child-pi/child-pi-streams.ts +19 -3
- package/src/runtime/child-pi/postmortem-registry.ts +8 -2
- package/src/runtime/classifier/ambient-noise-gate.ts +5 -1
- package/src/runtime/compaction/tool-output-pruner.ts +20 -2
- package/src/runtime/crew-agent-records.ts +92 -17
- package/src/runtime/custom-tools/submit-result-tool.ts +17 -0
- package/src/runtime/deadletter.ts +5 -1
- package/src/runtime/dispatch-batch.ts +60 -16
- package/src/runtime/effectiveness.ts +24 -1
- package/src/runtime/finalize-run.ts +12 -1
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +12 -3
- package/src/runtime/goal-workflow/goal-evaluator.ts +30 -5
- package/src/runtime/goal-workflow/goal-loop-runner.ts +60 -3
- package/src/runtime/live-session/live-agent-control.ts +34 -14
- package/src/runtime/live-session/live-agent-manager.ts +20 -5
- package/src/runtime/live-session/live-session-runtime.ts +66 -81
- package/src/runtime/live-session/usage-bridge.ts +66 -2
- package/src/runtime/mcp-proxy.ts +10 -0
- package/src/runtime/model/model-fallback.ts +68 -0
- package/src/runtime/model/runtime-policy.ts +20 -6
- package/src/runtime/model/runtime-resolver.ts +5 -3
- package/src/runtime/orphan-worker-registry.ts +64 -24
- package/src/runtime/pi-spawn.ts +4 -1
- package/src/runtime/process/zombie-scanner.ts +6 -2
- package/src/runtime/recovery/crash-recovery.ts +35 -2
- package/src/runtime/scheduling/batch-barrier.ts +42 -5
- package/src/runtime/scheduling/run-coalesced-task-group.ts +9 -3
- package/src/runtime/scheduling/scheduler.ts +74 -29
- package/src/runtime/stale-reconciler.ts +45 -6
- package/src/runtime/surface/herdr-provider.ts +54 -1
- package/src/runtime/surface/launch-script.ts +7 -1
- package/src/runtime/surface/resolve-surface.ts +4 -1
- package/src/runtime/surface/tmux-provider.ts +32 -2
- package/src/runtime/task-runner/child-executor.ts +3 -2
- package/src/runtime/workspace-lock.ts +52 -4
- package/src/state/contracts.ts +2 -0
- package/src/state/coordination/process-lock.ts +50 -0
- package/src/state/event-log/cursor.ts +24 -3
- package/src/state/event-log/event-log-rotation.ts +26 -2
- package/src/state/event-log/worker-atomic-writer.ts +5 -0
- package/src/state/stores/active-run-registry.ts +56 -5
- package/src/state/stores/ownership-map.ts +26 -7
- package/src/state/stores/plan-store.ts +30 -1
- package/src/state/stores/sqlite-run-state.ts +91 -1
- package/src/state/types.ts +5 -0
- package/src/ui/dashboard-panes/agents-pane.ts +4 -3
- package/src/ui/dashboard-panes/plan-pane.ts +5 -3
- package/src/ui/dashboard-panes/transcript-pane.ts +2 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +5 -3
- package/src/ui/run-dashboard.ts +6 -4
- package/src/ui/run-snapshot-cache.ts +3 -2
- package/src/ui/settings-overlay.ts +68 -10
- package/src/ui/theme-discovery.ts +26 -7
- package/src/ui/transcript-viewer.ts +5 -3
- package/src/utils/visual.ts +84 -0
|
@@ -319,6 +319,12 @@ export function convertToOTLP(snapshots: MetricSnapshot[]): unknown {
|
|
|
319
319
|
name: snapshot.name,
|
|
320
320
|
description: snapshot.description,
|
|
321
321
|
[snapshot.type === "histogram" ? "histogram" : snapshot.type === "gauge" ? "gauge" : "sum"]: {
|
|
322
|
+
// wave-2b (OTLP): counters exported as Sum MUST carry
|
|
323
|
+
// aggregationTemporality or OTLP-compliant backends reject /
|
|
324
|
+
// mis-aggregate the series (proto enum AggregationTemporality:
|
|
325
|
+
// 0=UNSPECIFIED, 1=DELTA, 2=CUMULATIVE). Registry counters are
|
|
326
|
+
// cumulative monotonic — declare CUMULATIVE.
|
|
327
|
+
...(snapshot.type === "counter" ? { aggregationTemporality: 2 } : {}),
|
|
322
328
|
dataPoints: pointValues(snapshot),
|
|
323
329
|
},
|
|
324
330
|
})),
|
|
@@ -394,10 +400,14 @@ export class OTLPExporter implements MetricExporter {
|
|
|
394
400
|
const body = await gzipAsync(Buffer.from(json));
|
|
395
401
|
const response = await fetch(this.opts.endpoint, {
|
|
396
402
|
method: "POST",
|
|
403
|
+
// wave-2b (OTLP): user headers spread FIRST so OUR wire-critical
|
|
404
|
+
// content-type/encoding win — a user header of
|
|
405
|
+
// content-type: text/plain previously replaced ours and collectors
|
|
406
|
+
// 415'd the push (the old spread-last overrode us).
|
|
397
407
|
headers: {
|
|
408
|
+
...(this.opts.headers ?? {}),
|
|
398
409
|
"content-type": "application/json",
|
|
399
410
|
"content-encoding": "gzip",
|
|
400
|
-
...(this.opts.headers ?? {}),
|
|
401
411
|
},
|
|
402
412
|
body,
|
|
403
413
|
signal: controller.signal,
|
|
@@ -408,6 +408,65 @@ export function shouldRegisterAskTool(env: NodeJS.ProcessEnv = process.env): boo
|
|
|
408
408
|
|
|
409
409
|
const PI_CREW_DELEGATE_ENABLED_ENV = "PI_CREW_DELEGATE_ENABLED";
|
|
410
410
|
export const DELEGATE_TIMED_OUT_RESULT = "[delegate timed out]";
|
|
411
|
+
|
|
412
|
+
// ── R9-F1 (2026-10-17): delegate-result recovery across parent turn-restarts ──
|
|
413
|
+
// A parent session can restart mid-turn (pi-core seam): the delegate poll loop
|
|
414
|
+
// dies with it, the grandchild later COMPLETES durably into the task inbox,
|
|
415
|
+
// and nothing ever re-reads that message — the report is silently lost
|
|
416
|
+
// (runtime-proven: run event seq 209/216/281/431, mailbox msg_b0297f83).
|
|
417
|
+
// Conservative, seam-level mitigation (the pi-core trigger is external):
|
|
418
|
+
// - every in-flight subId is tracked in `inFlightDelegateSubIds`;
|
|
419
|
+
// - whenever a delegate poll loop (re)starts, the inbox is scanned ONCE for
|
|
420
|
+
// results of still-pending subIds that have no delivered result yet and
|
|
421
|
+
// they are re-injected into `recoveredDelegateResults` (logged as
|
|
422
|
+
// `delegate.result.recovered`) so the owning loop's next iteration — or a
|
|
423
|
+
// RESTARTED loop for the same subId — surfaces it without re-deriving it;
|
|
424
|
+
// - the timed-out notice points at the durable copy in the task inbox.
|
|
425
|
+
// `no double-count`: a recovered message is consumed exactly once (deleted
|
|
426
|
+
// from the map on delivery) and the inbox stays append-only ground truth.
|
|
427
|
+
const inFlightDelegateSubIds = new Set<string>();
|
|
428
|
+
const recoveredDelegateResults = new Map<string, MailboxMessage>();
|
|
429
|
+
const RECOVERED_DELEGATE_RESULTS_MAX = 64;
|
|
430
|
+
|
|
431
|
+
/** R9-F1: scan the task inbox for completed results of still-pending subIds.
|
|
432
|
+
* Pure w.r.t. its inputs (injectable reader for tests); mutates the module
|
|
433
|
+
* recovery map + logs one `delegate.result.recovered` internal event per
|
|
434
|
+
* recovered subId. Returns the recovered messages keyed by subId. */
|
|
435
|
+
export function recoverPendingDelegateResults(
|
|
436
|
+
manifest: TeamRunManifest,
|
|
437
|
+
pendingSubIds: ReadonlySet<string>,
|
|
438
|
+
readInbox: (m: TeamRunManifest) => MailboxMessage[] = (m) => readAllMailboxMessages(m, "inbox"),
|
|
439
|
+
): Map<string, MailboxMessage> {
|
|
440
|
+
const recovered = new Map<string, MailboxMessage>();
|
|
441
|
+
if (pendingSubIds.size === 0) return recovered;
|
|
442
|
+
let messages: MailboxMessage[];
|
|
443
|
+
try {
|
|
444
|
+
messages = readInbox(manifest);
|
|
445
|
+
} catch (error) {
|
|
446
|
+
logInternalError("prompt-runtime.delegate-recovery-inbox-read", error instanceof Error ? error : new Error(String(error)), "warn");
|
|
447
|
+
return recovered;
|
|
448
|
+
}
|
|
449
|
+
for (const message of messages) {
|
|
450
|
+
const from = message.from ?? "";
|
|
451
|
+
if (!from.startsWith("delegate:")) continue;
|
|
452
|
+
const subId = from.slice("delegate:".length);
|
|
453
|
+
if (!pendingSubIds.has(subId)) continue;
|
|
454
|
+
if (recoveredDelegateResults.has(subId)) continue; // already recovered — no double-count
|
|
455
|
+
recoveredDelegateResults.set(subId, message);
|
|
456
|
+
while (recoveredDelegateResults.size > RECOVERED_DELEGATE_RESULTS_MAX) {
|
|
457
|
+
const oldest = recoveredDelegateResults.keys().next().value;
|
|
458
|
+
if (oldest === undefined) break;
|
|
459
|
+
recoveredDelegateResults.delete(oldest);
|
|
460
|
+
}
|
|
461
|
+
recovered.set(subId, message);
|
|
462
|
+
logInternalError(
|
|
463
|
+
"delegate.result.recovered",
|
|
464
|
+
new Error(`recovered completed delegate result from task inbox after a parent turn-restart (subId=${subId})`),
|
|
465
|
+
"warn",
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
return recovered;
|
|
469
|
+
}
|
|
411
470
|
/** P3-11: poll slack past the server deadline — the server timer fires first
|
|
412
471
|
* and writes the fenced (timed out) result; the client checks a bit longer
|
|
413
472
|
* so the outcome lands in-tool instead of lingering unread in the inbox. */
|
|
@@ -540,24 +599,40 @@ export function createDelegateTool(deps: DelegateToolDeps = {}): DelegateToolDef
|
|
|
540
599
|
const fromTag = `delegate:${subId}`;
|
|
541
600
|
let terminal: "completed" | "timed-out" | "aborted" = "timed-out";
|
|
542
601
|
let result: MailboxMessage | undefined;
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
602
|
+
// R9-F1: register the subId as in-flight and, at every loop (re)start,
|
|
603
|
+
// re-inject results of still-pending subIds that a previous (restarted)
|
|
604
|
+
// turn never delivered — they become visible to THIS loop's reads.
|
|
605
|
+
inFlightDelegateSubIds.add(subId);
|
|
606
|
+
try {
|
|
607
|
+
recoverPendingDelegateResults(manifest, inFlightDelegateSubIds);
|
|
608
|
+
while (true) {
|
|
609
|
+
if (signal?.aborted) {
|
|
610
|
+
terminal = "aborted";
|
|
611
|
+
break;
|
|
612
|
+
}
|
|
613
|
+
// Consume a recovered result FIRST (exactly once — map delete), then
|
|
614
|
+
// fall through to the durable inbox read (append-only ground truth).
|
|
615
|
+
result = recoveredDelegateResults.get(subId);
|
|
616
|
+
if (result) recoveredDelegateResults.delete(subId);
|
|
617
|
+
if (!result) {
|
|
618
|
+
try {
|
|
619
|
+
result = readAllMailboxMessages(manifest, "inbox").find(
|
|
620
|
+
(m) => m.from === fromTag && (m.taskId === undefined || m.taskId === taskId),
|
|
621
|
+
);
|
|
622
|
+
} catch {
|
|
623
|
+
/* transient read error — keep polling */
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
if (result) {
|
|
627
|
+
terminal = "completed";
|
|
628
|
+
break;
|
|
629
|
+
}
|
|
630
|
+
if (now() >= deadline) break;
|
|
631
|
+
await sleep(effectiveSteeringInterval(hasLiveControlRealtimeListeners()));
|
|
558
632
|
}
|
|
559
|
-
|
|
560
|
-
|
|
633
|
+
} finally {
|
|
634
|
+
inFlightDelegateSubIds.delete(subId);
|
|
635
|
+
recoveredDelegateResults.delete(subId);
|
|
561
636
|
}
|
|
562
637
|
const waitedMs = now() - startedAt;
|
|
563
638
|
if (terminal === "completed" && result) {
|
|
@@ -574,8 +649,17 @@ export function createDelegateTool(deps: DelegateToolDeps = {}): DelegateToolDef
|
|
|
574
649
|
details: { status: "aborted", grandchildTaskRef: subId, waitedMs },
|
|
575
650
|
};
|
|
576
651
|
}
|
|
652
|
+
// R9-F1: recovery hint — the grandchild can still complete AFTER this
|
|
653
|
+
// timeout (and after a parent turn-restart the tool result itself can be
|
|
654
|
+
// lost); the durable copy lands in the task inbox. Point the model at it
|
|
655
|
+
// instead of silently redoing the delegated work.
|
|
577
656
|
return {
|
|
578
|
-
content: [
|
|
657
|
+
content: [
|
|
658
|
+
{
|
|
659
|
+
type: "text" as const,
|
|
660
|
+
text: `${DELEGATE_TIMED_OUT_RESULT} — the grandchild may still complete; its result lands durably in the task inbox (from=delegate:${subId}) — check the task inbox / lastResult before redoing the work, or ask the leader to relay it.`,
|
|
661
|
+
},
|
|
662
|
+
],
|
|
579
663
|
details: { status: "timed-out", grandchildTaskRef: subId, waitedMs },
|
|
580
664
|
};
|
|
581
665
|
} finally {
|
|
@@ -142,11 +142,23 @@ export function parentAlive(pid: number, expectedStartTime: string | undefined,
|
|
|
142
142
|
}
|
|
143
143
|
});
|
|
144
144
|
let stat: string | undefined;
|
|
145
|
+
let readerThrew = false;
|
|
145
146
|
try {
|
|
146
147
|
stat = reader(pid);
|
|
147
148
|
} catch {
|
|
148
|
-
|
|
149
|
+
// R7-P1 guard-gate fix (deep-review round 7/9): a THROWING/unreadable
|
|
150
|
+
// stat reader (macOS-shaped: /proc absent → read error) with NO
|
|
151
|
+
// recorded expectedStartTime cannot PROVE the parent died. The doc
|
|
152
|
+
// contract above ("Never kills on … an unreadable clock field — a
|
|
153
|
+
// false 'alive' falls back to the host watchdog") demands the
|
|
154
|
+
// alive-with-caveat tolerance here; the pre-fix code fell through to
|
|
155
|
+
// `return false` BEFORE the tolerance check and killed healthy
|
|
156
|
+
// workers at the first 5s tick.
|
|
157
|
+
readerThrew = true;
|
|
149
158
|
}
|
|
159
|
+
if (readerThrew) return expectedStartTime ? false : true;
|
|
160
|
+
// Reader returned "no entry" (default /proc reader ENOENT = dead on
|
|
161
|
+
// Linux) — dead-parent detection stays intact.
|
|
150
162
|
if (stat === undefined) return false;
|
|
151
163
|
if (!expectedStartTime) return true;
|
|
152
164
|
const observed = procStartTimeTicks(stat);
|
|
@@ -223,6 +223,13 @@ export class CrewBrokerClient {
|
|
|
223
223
|
Math.max(1, Math.floor(timeoutMs)),
|
|
224
224
|
);
|
|
225
225
|
if (entry) entry.timer = requestTimer;
|
|
226
|
+
// R2-L-A3 (2026-10-17): unref the per-request timer (mirror the hello/
|
|
227
|
+
// backoff timers at :466-474) — a stranded in-flight request must not
|
|
228
|
+
// hold a finishing child's event loop open for up to timeoutMs. The
|
|
229
|
+
// typeof guard keeps injected test timer fakes (no unref) working.
|
|
230
|
+
if (requestTimer && typeof (requestTimer as { unref?: () => void }).unref === "function") {
|
|
231
|
+
(requestTimer as { unref?: () => void }).unref?.();
|
|
232
|
+
}
|
|
226
233
|
try {
|
|
227
234
|
const frame = encodeBrokerFrame({ id, method, params });
|
|
228
235
|
// Write may emit EPIPE etc. We don't await drain here — the response
|
|
@@ -64,6 +64,13 @@ import { type DelegateEventTarget, type DelegateEventType, recordDelegateEvent }
|
|
|
64
64
|
import { promoteShadowToRunning } from "./delegate/shadow-lifecycle.ts";
|
|
65
65
|
import { fanoutMailboxMessage } from "./mailbox-observer/mailbox-fanout.ts";
|
|
66
66
|
import type { CrewBrokerOptions, ServerConnection } from "./protocol/connection-state.ts";
|
|
67
|
+
// Outbound-queue management + caps (wc-gate move — bodies in protocol/connection-state.ts)
|
|
68
|
+
import {
|
|
69
|
+
handleOutboundOverflowFrame,
|
|
70
|
+
type OutboundAdmission,
|
|
71
|
+
type OutboundQueueHooks,
|
|
72
|
+
writeOrQueueFrame,
|
|
73
|
+
} from "./protocol/connection-state.ts";
|
|
67
74
|
import { handleEventsSince } from "./protocol/events-replay.ts";
|
|
68
75
|
import { DEFAULT_LOCK_BUSY_RETRY_DELAYS_MS, type LockBusyOutcome, withRunLockBusyRetry } from "./protocol/lock-busy.ts";
|
|
69
76
|
import { loadRunForHello } from "./protocol/manifest-loader.ts";
|
|
@@ -73,6 +80,7 @@ import {
|
|
|
73
80
|
BROKER_PROTOCOL,
|
|
74
81
|
isHelloParams,
|
|
75
82
|
isRequestObject,
|
|
83
|
+
parseDelegateRequestParams,
|
|
76
84
|
parseMsgSendParams,
|
|
77
85
|
parseWaitRequestParams,
|
|
78
86
|
safeStringify,
|
|
@@ -106,29 +114,6 @@ const HELLO_DEADLINE_MS = 1_000;
|
|
|
106
114
|
* blocked run still authenticates normally. */
|
|
107
115
|
const STALE_RUN_STATUSES: ReadonlySet<string> = new Set(["completed", "failed", "cancelled"]);
|
|
108
116
|
|
|
109
|
-
/** Default per-connection outbound queue cap (events). */
|
|
110
|
-
const DEFAULT_OUTBOUND_QUEUE_CAP = 256;
|
|
111
|
-
|
|
112
|
-
/** R6-5 (2026-10-16): typed protocol close-reason sent as a best-effort
|
|
113
|
-
* `protocol.close` event frame (direct write, jumps the saturated queue)
|
|
114
|
-
* before the connection is closed after a fanout/event frame was dropped
|
|
115
|
-
* at the queue cap. Clients observe the reason, reconnect, and replay via
|
|
116
|
-
* `events.since` (Phase 1.5). */
|
|
117
|
-
export const OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON = "outbound-overflow-resync";
|
|
118
|
-
|
|
119
|
-
/** R6-5: admission class for an outbound frame.
|
|
120
|
-
* - `false` — fanout/event class: drop-newest at cap → resync close.
|
|
121
|
-
* - `true` — close-reason class: bypass the cap, attempt an immediate
|
|
122
|
-
* direct write (existing `sendErrorAndClose` behavior).
|
|
123
|
-
* - `"response"` — request-response class (sendResult/sendError acks):
|
|
124
|
-
* bypass the drop-newest cap but stay IN the queue (ordering preserved,
|
|
125
|
-
* backpressure honored). Bounded by the client's in-flight request count
|
|
126
|
-
* — one response frame per request — so it can never storm the way a
|
|
127
|
-
* fanout burst can. Kept a `boolean | "response"` union (not a fresh
|
|
128
|
-
* enum) so the external FanoutWriters interface (mailbox-fanout.ts) keeps
|
|
129
|
-
* its `(conn, buf, force: boolean)` shape. */
|
|
130
|
-
type OutboundAdmission = boolean | "response";
|
|
131
|
-
|
|
132
117
|
export class CrewBroker {
|
|
133
118
|
private readonly options: Required<Pick<CrewBrokerOptions, "sessionId" | "enabled" | "waitMethodsEnabled" | "nestingEnabled">> &
|
|
134
119
|
Pick<
|
|
@@ -136,6 +121,7 @@ export class CrewBroker {
|
|
|
136
121
|
| "socketPath"
|
|
137
122
|
| "maxFrameBytes"
|
|
138
123
|
| "outboundQueueCap"
|
|
124
|
+
| "outboundQueueBytesCap"
|
|
139
125
|
| "cwd"
|
|
140
126
|
| "netModule"
|
|
141
127
|
| "waitStatusCache"
|
|
@@ -188,6 +174,7 @@ export class CrewBroker {
|
|
|
188
174
|
socketPath: options.socketPath,
|
|
189
175
|
maxFrameBytes: options.maxFrameBytes,
|
|
190
176
|
outboundQueueCap: options.outboundQueueCap,
|
|
177
|
+
outboundQueueBytesCap: options.outboundQueueBytesCap,
|
|
191
178
|
cwd: options.cwd,
|
|
192
179
|
netModule: options.netModule,
|
|
193
180
|
waitMethodsEnabled: options.waitMethodsEnabled === true,
|
|
@@ -472,6 +459,7 @@ export class CrewBroker {
|
|
|
472
459
|
role: undefined,
|
|
473
460
|
authMatchKind: undefined,
|
|
474
461
|
outbound: [],
|
|
462
|
+
queuedBytes: 0,
|
|
475
463
|
needsResync: false,
|
|
476
464
|
closed: false,
|
|
477
465
|
helloTimer: null,
|
|
@@ -821,6 +809,27 @@ export class CrewBroker {
|
|
|
821
809
|
err instanceof Error ? err : new Error(String(err)),
|
|
822
810
|
`sessionId=${this.options.sessionId}`,
|
|
823
811
|
);
|
|
812
|
+
// R9-N1 (2026-10-17): silently dropping a REQUEST-RESPONSE frame here
|
|
813
|
+
// (e.g. an events.since replay page larger than MAX_BROKER_FRAME_BYTES)
|
|
814
|
+
// left the client waiting on its id until its request timeout — a
|
|
815
|
+
// hang with no diagnostic. When the payload carries an id, answer it
|
|
816
|
+
// with a small typed error frame on the SAME id; if even that cannot
|
|
817
|
+
// encode, close (the client's reconnect+replay is the only durable
|
|
818
|
+
// answer left).
|
|
819
|
+
const id = (payload as { id?: unknown } | null)?.id;
|
|
820
|
+
if (typeof id === "number" || typeof id === "string") {
|
|
821
|
+
try {
|
|
822
|
+
const errBuf = encodeBrokerFrame({
|
|
823
|
+
id,
|
|
824
|
+
error: { code: "response-oversize", message: "response frame exceeds the broker frame limit" },
|
|
825
|
+
});
|
|
826
|
+
this.writeOrQueue(conn, errBuf, /*force*/ true);
|
|
827
|
+
return;
|
|
828
|
+
} catch {
|
|
829
|
+
/* fall through to close */
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
this.closeConnection(conn);
|
|
824
833
|
return;
|
|
825
834
|
}
|
|
826
835
|
// R6-5: request responses (the acks for ping/msg.send/wait.request/…)
|
|
@@ -831,91 +840,19 @@ export class CrewBroker {
|
|
|
831
840
|
}
|
|
832
841
|
|
|
833
842
|
private writeOrQueue(conn: ServerConnection, buf: Buffer, force: OutboundAdmission): void {
|
|
834
|
-
|
|
835
|
-
const cap = this.options.outboundQueueCap ?? DEFAULT_OUTBOUND_QUEUE_CAP;
|
|
836
|
-
if (conn.outbound.length >= cap) {
|
|
837
|
-
if (force === "response") {
|
|
838
|
-
// R6-5(a): response frames queue PAST the cap instead of being
|
|
839
|
-
// drop-newest'd. Bounded: at most one response per in-flight
|
|
840
|
-
// request, and the client's request pipeline is serial — this can
|
|
841
|
-
// never grow unbounded the way a fanout burst can. Ordering is
|
|
842
|
-
// preserved (pushed at the tail; drainOutbound honors backpressure).
|
|
843
|
-
conn.outbound.push(buf);
|
|
844
|
-
this.drainOutbound(conn);
|
|
845
|
-
return;
|
|
846
|
-
}
|
|
847
|
-
if (force) {
|
|
848
|
-
// Forced sends (e.g. close-reason) bypass the cap and attempt
|
|
849
|
-
// to flush directly; if the socket is busy they may still drop.
|
|
850
|
-
try {
|
|
851
|
-
conn.socket.write(buf);
|
|
852
|
-
} catch {
|
|
853
|
-
/* socket may have closed; the close handler will sweep. */
|
|
854
|
-
}
|
|
855
|
-
return;
|
|
856
|
-
}
|
|
857
|
-
// R6-5(b): fanout/event frame dropped at cap (drop-newest). The old
|
|
858
|
-
// `needsResync` marker was write-only (set, never read) — the claimed
|
|
859
|
-
// "pause + client notices" recovery did not exist, so live frames
|
|
860
|
-
// were silently dropped for the connection's remaining lifetime.
|
|
861
|
-
// Now the drop CLOSES the connection with a typed close-reason so
|
|
862
|
-
// the client reconnects and replays (events.since / msg.inbox).
|
|
863
|
-
this.handleOutboundOverflow(conn);
|
|
864
|
-
return;
|
|
865
|
-
}
|
|
866
|
-
conn.outbound.push(buf);
|
|
867
|
-
this.drainOutbound(conn);
|
|
843
|
+
writeOrQueueFrame(conn, buf, force, this.options, this.outboundHooks());
|
|
868
844
|
}
|
|
869
845
|
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
if (buf === undefined) break;
|
|
874
|
-
// Try the write; if it returns false, wait for drain before pushing more.
|
|
875
|
-
try {
|
|
876
|
-
const ok = conn.socket.write(buf);
|
|
877
|
-
if (!ok) {
|
|
878
|
-
// Backpressure — re-arm on drain event.
|
|
879
|
-
conn.socket.once("drain", () => {
|
|
880
|
-
if (!conn.closed) this.drainOutbound(conn);
|
|
881
|
-
});
|
|
882
|
-
return;
|
|
883
|
-
}
|
|
884
|
-
} catch {
|
|
885
|
-
// Write failed — close the connection (we already had it open).
|
|
886
|
-
this.closeConnection(conn);
|
|
887
|
-
return;
|
|
888
|
-
}
|
|
889
|
-
conn.outbound.shift();
|
|
890
|
-
conn.outboundSeq += 1;
|
|
891
|
-
}
|
|
846
|
+
/** R6-5(b): fanout/event frame dropped at cap → typed resync close (moved body in connection-state.ts). */
|
|
847
|
+
private handleOutboundOverflow(conn: ServerConnection): void {
|
|
848
|
+
handleOutboundOverflowFrame(conn, this.options, this.outboundHooks());
|
|
892
849
|
}
|
|
893
850
|
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
* is best-effort, exactly like `sendErrorAndClose`. */
|
|
900
|
-
private handleOutboundOverflow(conn: ServerConnection): void {
|
|
901
|
-
// Single-flight: fanoutMailboxMessage iterates a snapshot of the run's
|
|
902
|
-
// connections, so one burst can re-enter here for an already-overflowing
|
|
903
|
-
// conn before `closeConnection`'s `closed` guard becomes visible. The
|
|
904
|
-
// flag is the "close already initiated" bit (its only read).
|
|
905
|
-
if (conn.needsResync) return;
|
|
906
|
-
conn.needsResync = true;
|
|
907
|
-
logInternalError(
|
|
908
|
-
"crew-broker.outbound.overflow",
|
|
909
|
-
new Error("outbound queue overflow — closing connection for resync"),
|
|
910
|
-
`sessionId=${this.options.sessionId} runId=${conn.runId ?? "?"} queued=${conn.outbound.length} seq=${conn.outboundSeq}`,
|
|
911
|
-
);
|
|
912
|
-
try {
|
|
913
|
-
const buf = encodeBrokerFrame({ event: "protocol.close", data: { reason: OUTBOUND_OVERFLOW_RESYNC_CLOSE_REASON } });
|
|
914
|
-
conn.socket.write(buf);
|
|
915
|
-
} catch {
|
|
916
|
-
/* socket may be dead; closeConnection below sweeps regardless */
|
|
917
|
-
}
|
|
918
|
-
this.closeConnection(conn);
|
|
851
|
+
private outboundHooks(): OutboundQueueHooks {
|
|
852
|
+
return {
|
|
853
|
+
onOverflow: (conn) => this.handleOutboundOverflow(conn),
|
|
854
|
+
onClose: (conn) => this.closeConnection(conn),
|
|
855
|
+
};
|
|
919
856
|
}
|
|
920
857
|
|
|
921
858
|
// Phase 1: msg.send + msg.inbox handlers
|
|
@@ -1297,6 +1234,11 @@ export class CrewBroker {
|
|
|
1297
1234
|
const v = params && typeof params === "object" && !Array.isArray(params) ? (params as Record<string, unknown>) : {};
|
|
1298
1235
|
const targetTaskId = typeof v.taskId === "string" ? v.taskId : undefined;
|
|
1299
1236
|
const body = typeof v.body === "string" ? v.body : undefined;
|
|
1237
|
+
// R2-L-A4 (2026-10-17): optional client requestId — mirrors msg.send's U5
|
|
1238
|
+
// dedup. An orchestrator retry after ack-loss used to DUPLICATE the steer
|
|
1239
|
+
// (mailbox append + steering file); a stable requestId makes the resend a
|
|
1240
|
+
// no-op.
|
|
1241
|
+
const requestId = typeof v.requestId === "string" && v.requestId.length > 0 ? v.requestId.slice(0, 128) : undefined;
|
|
1300
1242
|
if (!targetTaskId || body === undefined) {
|
|
1301
1243
|
this.sendError(conn, id, "bad-params", "steer.push: taskId and body are required");
|
|
1302
1244
|
return;
|
|
@@ -1319,39 +1261,61 @@ export class CrewBroker {
|
|
|
1319
1261
|
|
|
1320
1262
|
const messageId = `steer_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`;
|
|
1321
1263
|
// Write 1: mailbox — live broker fanout + persistent inbox read.
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1264
|
+
// R2-L-A4: idempotent append when the client supplied a requestId —
|
|
1265
|
+
// resend-after-ack-loss dedupes instead of duplicating the steer.
|
|
1266
|
+
let deduped = false;
|
|
1267
|
+
if (requestId) {
|
|
1268
|
+
const outcome = await appendMailboxMessageIdempotentAsync(loaded.manifest, {
|
|
1269
|
+
id: messageId,
|
|
1270
|
+
direction: "inbox",
|
|
1271
|
+
from: conn.taskId ?? conn.runId,
|
|
1272
|
+
to: targetTaskId,
|
|
1273
|
+
taskId: targetTaskId,
|
|
1274
|
+
body,
|
|
1275
|
+
kind: "steer",
|
|
1276
|
+
priority: (v.priority as "urgent" | "normal" | "low" | undefined) ?? "urgent",
|
|
1277
|
+
deliveryMode: "interrupt",
|
|
1278
|
+
requestId,
|
|
1279
|
+
});
|
|
1280
|
+
deduped = outcome.deduped;
|
|
1281
|
+
} else {
|
|
1282
|
+
await appendMailboxMessageAsync(loaded.manifest, {
|
|
1283
|
+
id: messageId,
|
|
1284
|
+
direction: "inbox",
|
|
1285
|
+
from: conn.taskId ?? conn.runId,
|
|
1286
|
+
to: targetTaskId,
|
|
1287
|
+
taskId: targetTaskId,
|
|
1288
|
+
body,
|
|
1289
|
+
kind: "steer",
|
|
1290
|
+
priority: (v.priority as "urgent" | "normal" | "low" | undefined) ?? "urgent",
|
|
1291
|
+
deliveryMode: "interrupt",
|
|
1292
|
+
});
|
|
1293
|
+
}
|
|
1333
1294
|
// Write 2: steering file — durable fallback so pollSteering() picks
|
|
1334
1295
|
// up the steer even when the recipient child's broker connection is down. Matches the
|
|
1335
1296
|
// JSONL format of appendSteeringAsync in task-runner.ts.
|
|
1336
1297
|
// Best-effort: a failure here must NOT fail the push (mailbox write
|
|
1337
1298
|
// already succeeded).
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1299
|
+
// R2-L-A4: a deduped resend must NOT re-append the steering file either.
|
|
1300
|
+
if (!deduped) {
|
|
1301
|
+
try {
|
|
1302
|
+
const steeringDir = `${loaded.manifest.artifactsRoot}/steering`;
|
|
1303
|
+
const steeringPath = resolveRealContainedPath(loaded.manifest.artifactsRoot, `steering/${targetTaskId}.jsonl`);
|
|
1304
|
+
const line =
|
|
1305
|
+
JSON.stringify({
|
|
1306
|
+
type: "steer",
|
|
1307
|
+
message: body,
|
|
1308
|
+
id: messageId,
|
|
1309
|
+
ts: new Date().toISOString(),
|
|
1310
|
+
}) + "\n";
|
|
1311
|
+
await fsp.mkdir(steeringDir, { recursive: true });
|
|
1312
|
+
await fsp.appendFile(steeringPath, line, "utf-8");
|
|
1313
|
+
} catch (fileErr) {
|
|
1314
|
+
const safeMessage = fileErr instanceof Error ? redactSecretString(fileErr.message) : "";
|
|
1315
|
+
logInternalError("crew-broker.steer-file-write-failed", new Error(safeMessage), `taskId=${targetTaskId}`);
|
|
1316
|
+
}
|
|
1353
1317
|
}
|
|
1354
|
-
this.sendResult(conn, id, { messageId, taskId: targetTaskId, durable: true });
|
|
1318
|
+
this.sendResult(conn, id, { messageId, taskId: targetTaskId, durable: true, ...(requestId ? { requestId, deduped } : {}) });
|
|
1355
1319
|
} catch (err) {
|
|
1356
1320
|
this.sendError(conn, id, "steer-failed", (err as Error).message);
|
|
1357
1321
|
}
|
|
@@ -1388,6 +1352,15 @@ export class CrewBroker {
|
|
|
1388
1352
|
}
|
|
1389
1353
|
// Default recipient: the sender's taskId (the orchestrator that
|
|
1390
1354
|
// spawned this worker). If 'to' is provided, use it instead.
|
|
1355
|
+
// R2-L-A6 (wave-2b): 'to' is SENDER-CHOSEN and lands in a mailbox
|
|
1356
|
+
// record (taskId field) — an arbitrary value would let a worker
|
|
1357
|
+
// address any task in the run (spoofed follow-up routing). Restrict
|
|
1358
|
+
// to the two identities the sender legitimately owns: its run and
|
|
1359
|
+
// its own (parent) task.
|
|
1360
|
+
if (to !== undefined && to !== conn.runId && to !== conn.taskId) {
|
|
1361
|
+
this.sendError(conn, id, "bad-params", `escalate: 'to' must be this connection's run or task`);
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1391
1364
|
const target = to ?? conn.taskId ?? conn.runId;
|
|
1392
1365
|
try {
|
|
1393
1366
|
const loaded = loadRunManifestById(cwd, conn.runId);
|
|
@@ -1455,28 +1428,16 @@ export class CrewBroker {
|
|
|
1455
1428
|
this.sendError(conn, id, "forbidden", "delegate requires a task-scoped token; re-dispatch with PI_CREW_BROKER_TASK_ID");
|
|
1456
1429
|
return;
|
|
1457
1430
|
}
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
timeoutSec?: unknown;
|
|
1466
|
-
};
|
|
1467
|
-
if (typeof p.prompt !== "string" || p.prompt.trim().length === 0) {
|
|
1468
|
-
this.sendError(conn, id, "bad-params", "delegate.request: 'prompt' (non-empty string) is required");
|
|
1431
|
+
// R7-H1 fix: parse params through the protocol-level parser (mirrors
|
|
1432
|
+
// parseWaitRequestParams discipline) — non-integer/non-finite/negative/
|
|
1433
|
+
// over-ceiling maxTurns/budgetTokens never reach admission or the
|
|
1434
|
+
// budget reservation (durable-negative tokensSpent escape).
|
|
1435
|
+
const parsed = parseDelegateRequestParams(params);
|
|
1436
|
+
if (!parsed) {
|
|
1437
|
+
this.sendError(conn, id, "bad-params", "delegate.request: invalid params");
|
|
1469
1438
|
return;
|
|
1470
1439
|
}
|
|
1471
|
-
const requested =
|
|
1472
|
-
prompt: p.prompt,
|
|
1473
|
-
...(typeof p.description === "string" ? { description: p.description } : {}),
|
|
1474
|
-
...(typeof p.role === "string" ? { role: p.role } : {}),
|
|
1475
|
-
...(typeof p.model === "string" ? { model: p.model } : {}),
|
|
1476
|
-
...(typeof p.maxTurns === "number" ? { maxTurns: p.maxTurns } : {}),
|
|
1477
|
-
...(typeof p.budgetTokens === "number" ? { budgetTokens: p.budgetTokens } : {}),
|
|
1478
|
-
...(typeof p.timeoutSec === "number" ? { timeoutSec: p.timeoutSec } : {}),
|
|
1479
|
-
};
|
|
1440
|
+
const requested = parsed;
|
|
1480
1441
|
const cwd = this.options.cwd;
|
|
1481
1442
|
if (!cwd) {
|
|
1482
1443
|
this.sendError(conn, id, "no-manifest", "broker has no cwd configured");
|
|
@@ -1895,7 +1856,7 @@ export class CrewBroker {
|
|
|
1895
1856
|
const updatedManifest = {
|
|
1896
1857
|
...fresh.manifest,
|
|
1897
1858
|
// manifest.status stays "running" — park NEVER flips run status (ADR item 3).
|
|
1898
|
-
waitState: { taskId, questionId, askedAt },
|
|
1859
|
+
waitState: { taskId, questionId, askedAt, deadline }, // R2-M-A2: mirror task.waiting.deadline for shield expiry
|
|
1899
1860
|
updatedAt: askedAt,
|
|
1900
1861
|
};
|
|
1901
1862
|
saveRunTasks(updatedManifest, updatedTasks);
|