pi-crew 0.11.23 → 0.11.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/agents/explorer.md +3 -3
- package/agents/librarian.md +3 -3
- package/dist/index.mjs +528 -525
- package/docs/dynamic-workflows.md +14 -2
- package/package.json +4 -4
- package/src/agents/discover-agents.ts +8 -2
- package/src/config/config-validation.ts +3 -1
- package/src/config/env-allowlist-registry.ts +5 -0
- package/src/config/env-vars.ts +9 -6
- package/src/config/role-tools.ts +7 -6
- package/src/extension/management.ts +7 -0
- package/src/extension/register.ts +15 -0
- package/src/extension/run-index.ts +24 -1
- package/src/extension/team-tool/destructive-gate.ts +208 -0
- package/src/extension/team-tool/goal-wrap.ts +11 -1
- package/src/extension/team-tool/goal.ts +13 -1
- package/src/extension/team-tool.ts +10 -0
- package/src/observability/exporters/otlp-exporter.ts +11 -1
- package/src/prompt/prompt-runtime.ts +102 -18
- package/src/runtime/broker/crew-broker-client.ts +7 -0
- package/src/runtime/broker/crew-broker.ts +104 -132
- package/src/runtime/broker/protocol/connection-state.ts +159 -1
- package/src/runtime/broker/protocol/events-replay.ts +50 -10
- package/src/runtime/child-pi/child-pi-constants.ts +9 -0
- package/src/runtime/child-pi/child-pi-rpc.ts +72 -7
- package/src/runtime/child-pi/child-pi-streams.ts +19 -3
- package/src/runtime/child-pi/postmortem-registry.ts +8 -2
- package/src/runtime/classifier/ambient-noise-gate.ts +5 -1
- package/src/runtime/compaction/tool-output-pruner.ts +20 -2
- package/src/runtime/crew-agent-records.ts +67 -4
- package/src/runtime/custom-tools/submit-result-tool.ts +17 -0
- package/src/runtime/deadletter.ts +5 -1
- package/src/runtime/dispatch-batch.ts +51 -10
- package/src/runtime/effectiveness.ts +24 -1
- package/src/runtime/finalize-run.ts +12 -1
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +12 -3
- package/src/runtime/goal-workflow/goal-evaluator.ts +30 -5
- package/src/runtime/goal-workflow/goal-loop-runner.ts +60 -3
- package/src/runtime/live-session/live-agent-control.ts +34 -14
- package/src/runtime/live-session/live-agent-manager.ts +20 -5
- package/src/runtime/live-session/live-session-runtime.ts +66 -81
- package/src/runtime/live-session/usage-bridge.ts +66 -2
- package/src/runtime/mcp-proxy.ts +10 -0
- package/src/runtime/model/model-fallback.ts +68 -0
- package/src/runtime/model/runtime-policy.ts +20 -6
- package/src/runtime/model/runtime-resolver.ts +5 -3
- package/src/runtime/orphan-worker-registry.ts +64 -24
- package/src/runtime/pi-spawn.ts +4 -1
- package/src/runtime/recovery/crash-recovery.ts +35 -2
- package/src/runtime/scheduling/batch-barrier.ts +42 -5
- package/src/runtime/scheduling/scheduler.ts +74 -29
- package/src/runtime/stale-reconciler.ts +41 -5
- package/src/runtime/surface/herdr-provider.ts +54 -1
- package/src/runtime/surface/launch-script.ts +7 -1
- package/src/runtime/surface/resolve-surface.ts +4 -1
- package/src/runtime/surface/tmux-provider.ts +32 -2
- package/src/runtime/task-runner/child-executor.ts +3 -2
- package/src/runtime/workspace-lock.ts +52 -4
- package/src/state/contracts.ts +2 -0
- package/src/state/coordination/process-lock.ts +50 -0
- package/src/state/event-log/cursor.ts +10 -1
- package/src/state/event-log/event-log-rotation.ts +26 -2
- package/src/state/event-log/worker-atomic-writer.ts +5 -0
- package/src/state/stores/active-run-registry.ts +56 -5
- package/src/state/stores/ownership-map.ts +26 -7
- package/src/state/stores/plan-store.ts +30 -1
- package/src/state/stores/sqlite-run-state.ts +91 -1
- package/src/state/types.ts +5 -0
- package/src/ui/dashboard-panes/agents-pane.ts +4 -3
- package/src/ui/dashboard-panes/plan-pane.ts +5 -3
- package/src/ui/dashboard-panes/transcript-pane.ts +2 -1
- package/src/ui/overlays/mailbox-detail-overlay.ts +5 -3
- package/src/ui/run-dashboard.ts +6 -4
- package/src/ui/run-snapshot-cache.ts +3 -2
- package/src/ui/settings-overlay.ts +68 -10
- package/src/ui/theme-discovery.ts +26 -7
- package/src/ui/transcript-viewer.ts +5 -3
- package/src/utils/visual.ts +84 -0
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { readEventsCursor } from "../../../state/event-log/event-log.ts";
|
|
13
13
|
import { loadRunManifestById } from "../../../state/stores/state-store.ts";
|
|
14
|
+
import { encodeBrokerFrame } from "../../../utils/ndjson.ts";
|
|
14
15
|
import type { ServerConnection } from "./connection-state.ts";
|
|
15
16
|
|
|
16
17
|
export interface EventWriterHelpers {
|
|
@@ -50,18 +51,57 @@ export async function handleEventsSince(
|
|
|
50
51
|
const v = params && typeof params === "object" && !Array.isArray(params) ? (params as Record<string, unknown>) : {};
|
|
51
52
|
const sinceSeq = typeof v.sinceSeq === "number" && Number.isFinite(v.sinceSeq) ? Math.max(0, Math.floor(v.sinceSeq)) : 0;
|
|
52
53
|
const limit = typeof v.limit === "number" && Number.isFinite(v.limit) ? Math.min(Math.max(1, Math.floor(v.limit)), 1000) : 1000;
|
|
54
|
+
// R9-N2 (2026-10-17): optional byte-offset anchor + generation echo. The
|
|
55
|
+
// default (non-offset) read only sees the last TAIL_BYTES (4 MB) of the
|
|
56
|
+
// log, so the PREFIX of a larger log is unreachable by seq alone; a client
|
|
57
|
+
// that passes fromByteOffset pages it (the cursor resumes scanning from
|
|
58
|
+
// that byte position). Omitted → exact previous behavior.
|
|
59
|
+
const fromByteOffset =
|
|
60
|
+
typeof v.fromByteOffset === "number" && Number.isFinite(v.fromByteOffset) && v.fromByteOffset >= 0
|
|
61
|
+
? Math.floor(v.fromByteOffset)
|
|
62
|
+
: undefined;
|
|
63
|
+
const generation = typeof v.generation === "number" && Number.isFinite(v.generation) ? Math.floor(v.generation) : undefined;
|
|
53
64
|
try {
|
|
54
|
-
const result = readEventsCursor(eventsPath, { sinceSeq, limit });
|
|
55
|
-
//
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
65
|
+
const result = readEventsCursor(eventsPath, { sinceSeq, limit, fromByteOffset, generation });
|
|
66
|
+
// R9-N1 (2026-10-17): a page whose encoded frame exceeds the broker
|
|
67
|
+
// frame limit used to make sendResult → enqueueFrame → encodeBrokerFrame
|
|
68
|
+
// throw into a debug-only log — the client hung on its id with no
|
|
69
|
+
// response (runtime-proven on >256 KiB replays). Pre-check the EXACT
|
|
70
|
+
// response frame here and binary-halve the batch until it encodes;
|
|
71
|
+
// `truncated` + `nextSinceSeq` tell the client to page the rest. Never
|
|
72
|
+
// silence — if even ONE event cannot encode, answer with a typed error.
|
|
73
|
+
let events = result.events;
|
|
74
|
+
let frameTruncated = false;
|
|
75
|
+
let response: Record<string, unknown>;
|
|
76
|
+
const buildResponse = (): Record<string, unknown> => ({
|
|
77
|
+
events,
|
|
78
|
+
nextSeq: events.length ? events.reduce((max, event) => Math.max(max, event.metadata?.seq ?? 0), sinceSeq) : result.nextSeq,
|
|
79
|
+
hasMore: result.total > events.length,
|
|
80
|
+
// Continuation contract: nextSinceSeq is the seq to pass as sinceSeq on
|
|
81
|
+
// the next call (safe resume anchor — never past an undelivered event).
|
|
82
|
+
nextSinceSeq: events.length ? events.reduce((max, event) => Math.max(max, event.metadata?.seq ?? 0), sinceSeq) : result.nextSeq,
|
|
83
|
+
truncated: frameTruncated || result.truncated === true,
|
|
84
|
+
...(result.nextByteOffset !== undefined ? { nextByteOffset: result.nextByteOffset } : {}),
|
|
85
|
+
...(result.generation !== undefined ? { generation: result.generation } : {}),
|
|
64
86
|
});
|
|
87
|
+
while (true) {
|
|
88
|
+
response = buildResponse();
|
|
89
|
+
try {
|
|
90
|
+
encodeBrokerFrame(response);
|
|
91
|
+
break;
|
|
92
|
+
} catch {
|
|
93
|
+
if (events.length <= 1) {
|
|
94
|
+
// A single event over the frame limit can never page — typed error
|
|
95
|
+
// (enqueueFrame's R9-N1 fallback would also catch this, but failing
|
|
96
|
+
// here keeps the client's paging loop coherent).
|
|
97
|
+
helpers.sendError(conn, id, "response-oversize", "events.since: single event exceeds the broker frame limit");
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
events = events.slice(0, Math.ceil(events.length / 2));
|
|
101
|
+
frameTruncated = true;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
helpers.sendResult(conn, id, response);
|
|
65
105
|
} catch (err) {
|
|
66
106
|
helpers.sendError(conn, id, "replay-failed", (err as Error).message);
|
|
67
107
|
}
|
|
@@ -31,6 +31,15 @@ export const RESPONSE_TIMEOUT_MS = DEFAULT_CHILD_PI.responseTimeoutMs;
|
|
|
31
31
|
*/
|
|
32
32
|
export const MAX_LINE_BUFFER_BYTES = 1024 * 1024; // 1 MB
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* R3A-M3 (wave-2b): byte cap for a single persisted transcript line. The
|
|
36
|
+
* buffer force-flush can hand the pipeline a raw "line" of up to 1 MB, and
|
|
37
|
+
* non-JSON lines are persisted VERBATIM (no compact stage) — a worker that
|
|
38
|
+
* base64-dumps a binary into its stdout would write it whole into the
|
|
39
|
+
* transcript. Lines beyond this cap are truncated with an explicit marker.
|
|
40
|
+
*/
|
|
41
|
+
export const MAX_PERSISTED_LINE_BYTES = 64 * 1024; // 64 KiB
|
|
42
|
+
|
|
34
43
|
/** Maximum characters for assistant text fragments in compacted events. */
|
|
35
44
|
export const MAX_ASSISTANT_TEXT_CHARS = DEFAULT_CHILD_PI.maxAssistantTextChars;
|
|
36
45
|
|
|
@@ -169,12 +169,29 @@ interface RpcPoolEntry {
|
|
|
169
169
|
proc: ChildProcess;
|
|
170
170
|
/** True while the pooled process is idle between runs (reusable). */
|
|
171
171
|
idle: boolean;
|
|
172
|
+
/** R2-M-A5 (Wave-3): the run that pinned this process. Reuse re-validates
|
|
173
|
+
* the requesting run matches — a pooled worker (its env, session dir,
|
|
174
|
+
* transcript paths) is run-scoped and must NEVER cross run boundaries. */
|
|
175
|
+
pinnedRunId: string;
|
|
172
176
|
}
|
|
173
177
|
|
|
174
178
|
const rpcPool = new Map<string, RpcPoolEntry>();
|
|
175
179
|
|
|
176
|
-
|
|
177
|
-
|
|
180
|
+
/** @internal — exported for the R2-M-A5 run-identity pinning test. */
|
|
181
|
+
export function poolKeyFor(input: ChildPiRunInput): string {
|
|
182
|
+
// R2-M-A5 (Wave-3): the pool key is RUN-SCOPED. The old key (agentId only)
|
|
183
|
+
// let two DIFFERENT runs with the same agent id reuse one process — a
|
|
184
|
+
// worker's env/session/transcript state leaking across runs (and stale
|
|
185
|
+
// run-scoped registrations pointing at the wrong run).
|
|
186
|
+
const runId = input.runId ?? "untracked";
|
|
187
|
+
const agentId = input.agentId ?? "-";
|
|
188
|
+
return `${runId}:${agentId}:${input.agent.name}`;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** R2-M-A5 (Wave-3): the run id a pool lookup must match for reuse. */
|
|
192
|
+
/** @internal — exported for the R2-M-A5 run-identity pinning test. */
|
|
193
|
+
export function requestingRunIdFor(input: ChildPiRunInput): string {
|
|
194
|
+
return input.runId ?? "untracked";
|
|
178
195
|
}
|
|
179
196
|
|
|
180
197
|
/**
|
|
@@ -187,9 +204,19 @@ function poolKeyFor(input: ChildPiRunInput): string {
|
|
|
187
204
|
* evicted (the watchdog owns their failure); busy entries are left in place
|
|
188
205
|
* and the caller falls back to a fresh spawn.
|
|
189
206
|
*/
|
|
190
|
-
function acquirePooledRpcClient(key: string): RpcPoolEntry | undefined {
|
|
207
|
+
function acquirePooledRpcClient(key: string, requestingRunId?: string): RpcPoolEntry | undefined {
|
|
191
208
|
const entry = rpcPool.get(key);
|
|
192
209
|
if (!entry) return undefined;
|
|
210
|
+
// R2-M-A5 (Wave-3): belt-and-braces identity re-validation — even if the
|
|
211
|
+
// key were ever reused across runs, a pinned-runId mismatch evicts the
|
|
212
|
+
// entry (stop best-effort) and forces a fresh spawn.
|
|
213
|
+
if (requestingRunId !== undefined && entry.pinnedRunId !== requestingRunId) {
|
|
214
|
+
rpcPool.delete(key);
|
|
215
|
+
void entry.client.stop().catch(() => {
|
|
216
|
+
/* identity eviction best-effort */
|
|
217
|
+
});
|
|
218
|
+
return undefined;
|
|
219
|
+
}
|
|
193
220
|
const stdin = entry.proc.stdin;
|
|
194
221
|
const procAlive = entry.proc.exitCode === null && entry.proc.signalCode === null && !!stdin && !stdin.destroyed;
|
|
195
222
|
if (!procAlive) {
|
|
@@ -319,7 +346,10 @@ export async function tryChildPiRpcRun(plan: ChildPiRpcRunPlan): Promise<ChildPi
|
|
|
319
346
|
// resolves in ~0ms; only a cold standalone test pays module load.
|
|
320
347
|
// LAZY: registered with scripts/check-lazy-imports.mjs (runtime dynamic import)
|
|
321
348
|
const { RpcClient } = await import("@earendil-works/pi-coding-agent");
|
|
322
|
-
|
|
349
|
+
// R2-M-A5 (Wave-3): pass the requesting run id so reuse re-validates the pinned run.
|
|
350
|
+
const pooled = resolveChildPiRpcPoolEnabled()
|
|
351
|
+
? acquirePooledRpcClient(poolKeyFor(plan.input), requestingRunIdFor(plan.input))
|
|
352
|
+
: undefined;
|
|
323
353
|
const Ctor = RpcClient as unknown as new (options: unknown) => RpcClientInstance;
|
|
324
354
|
return runRpcSession(Ctor, plan, cliPath, pooled);
|
|
325
355
|
}
|
|
@@ -357,6 +387,20 @@ async function runRpcSession(
|
|
|
357
387
|
}
|
|
358
388
|
|
|
359
389
|
// ── start (fresh spawns only) ─────────────────────────────────────────
|
|
390
|
+
// R2-L-A5 rider (wave-2b): every early-failure return below (spawn-error,
|
|
391
|
+
// no-process, canary-kill) bypasses the settle/finally tempDir cleanup at
|
|
392
|
+
// the bottom of this function — a leaked per-run scratch dir each time.
|
|
393
|
+
// finally-equivalent cleanup right before each return (fresh spawns only:
|
|
394
|
+
// a pooled reuse must not delete another task's scratch dir).
|
|
395
|
+
const cleanupFreshTempDir = (): void => {
|
|
396
|
+
if (!pooled && ctx.tempDir) {
|
|
397
|
+
try {
|
|
398
|
+
cleanupTempDir(ctx.tempDir);
|
|
399
|
+
} catch {
|
|
400
|
+
/* cleanup never masks the start failure */
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
};
|
|
360
404
|
if (!pooled) {
|
|
361
405
|
// Env-splice: RpcClient spawns with `{...process.env, ...options.env}`
|
|
362
406
|
// (rpc-client.js start()) — WITHOUT this splice the child would inherit
|
|
@@ -397,6 +441,7 @@ async function runRpcSession(
|
|
|
397
441
|
} catch {
|
|
398
442
|
/* listener errors never mask the start failure */
|
|
399
443
|
}
|
|
444
|
+
cleanupFreshTempDir();
|
|
400
445
|
return {
|
|
401
446
|
exitCode: null,
|
|
402
447
|
stdout: "",
|
|
@@ -425,6 +470,7 @@ async function runRpcSession(
|
|
|
425
470
|
// runtime (rpc-client.js field) — hence the structural cast.
|
|
426
471
|
const child = (client as unknown as { process: ChildProcess | null }).process;
|
|
427
472
|
if (!child) {
|
|
473
|
+
cleanupFreshTempDir();
|
|
428
474
|
return {
|
|
429
475
|
exitCode: null,
|
|
430
476
|
stdout: "",
|
|
@@ -458,6 +504,7 @@ async function runRpcSession(
|
|
|
458
504
|
} catch {
|
|
459
505
|
/* listener errors never mask the leak failure */
|
|
460
506
|
}
|
|
507
|
+
cleanupFreshTempDir();
|
|
461
508
|
return {
|
|
462
509
|
exitCode: null,
|
|
463
510
|
stdout: "",
|
|
@@ -764,9 +811,27 @@ async function runRpcSession(
|
|
|
764
811
|
child.stdin &&
|
|
765
812
|
!child.stdin.destroyed
|
|
766
813
|
) {
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
814
|
+
// R2-M-A5 (Wave-3): pin the registering run — reuse re-validates.
|
|
815
|
+
poolEntry = { client, proc: child, idle: false, pinnedRunId: requestingRunIdFor(input) };
|
|
816
|
+
// R2-M-A4 (2026-10-17): NEVER overwrite a live same-key entry — the
|
|
817
|
+
// previous unconditional rpcPool.set replaced (and orphaned) a live
|
|
818
|
+
// process: stdin open, unreachable, dead only at host shutdown. Skip
|
|
819
|
+
// registration instead (the run settles through the normal close path).
|
|
820
|
+
const existing = rpcPool.get(poolKey);
|
|
821
|
+
if (existing && existing !== poolEntry && existing.proc.exitCode === null) {
|
|
822
|
+
logInternalError(
|
|
823
|
+
"child-pi.rpc-pool-overwrite-guard",
|
|
824
|
+
new Error(
|
|
825
|
+
`rpcPool registration skipped: live entry already holds pool key ${poolKey} (pinnedRunId=${existing.pinnedRunId ?? "unknown"})`,
|
|
826
|
+
),
|
|
827
|
+
`pid=${existing.proc.pid}`,
|
|
828
|
+
"warn",
|
|
829
|
+
);
|
|
830
|
+
poolEntry = undefined;
|
|
831
|
+
} else {
|
|
832
|
+
rpcPool.set(poolKey, poolEntry);
|
|
833
|
+
attachPoolExitHook(poolKey, child);
|
|
834
|
+
}
|
|
770
835
|
}
|
|
771
836
|
if (poolEntry) {
|
|
772
837
|
// Warm path (acquired OR just registered): the process STAYS alive
|
|
@@ -14,6 +14,7 @@ import type { ChildPiRunInput } from "./child-pi.ts";
|
|
|
14
14
|
import {
|
|
15
15
|
MAX_ASSISTANT_TEXT_CHARS,
|
|
16
16
|
MAX_LINE_BUFFER_BYTES,
|
|
17
|
+
MAX_PERSISTED_LINE_BYTES,
|
|
17
18
|
MAX_THINKING_CHARS,
|
|
18
19
|
MAX_TOOL_INPUT_CHARS,
|
|
19
20
|
MAX_TOOL_RESULT_CHARS,
|
|
@@ -85,7 +86,11 @@ export function compactChildPiEvent(event: unknown): unknown | undefined {
|
|
|
85
86
|
return {
|
|
86
87
|
type: record.type,
|
|
87
88
|
toolName: record.toolName,
|
|
88
|
-
|
|
89
|
+
// R3A-M3 (wave-2b): tool args were passed through RAW — a single
|
|
90
|
+
// tool_execution carrying a 1 MB arg (e.g. an edit diff or base64
|
|
91
|
+
// blob) landed whole in the agent event log. Route through the same
|
|
92
|
+
// compaction as message content parts.
|
|
93
|
+
args: compactValue(record.args),
|
|
89
94
|
};
|
|
90
95
|
}
|
|
91
96
|
if (record.type === "tool_result_end" || record.type === "message_end" || record.type === "message") {
|
|
@@ -124,7 +129,10 @@ export function compactChildPiEvent(event: unknown): unknown | undefined {
|
|
|
124
129
|
taskId: record.taskId,
|
|
125
130
|
reason: record.reason,
|
|
126
131
|
message: record.message,
|
|
127
|
-
|
|
132
|
+
// R3A-M3 (wave-2b): the "full payload" pass-through kept `data` raw —
|
|
133
|
+
// compact it too (the P2-25 intent was feature fidelity, not unbounded
|
|
134
|
+
// payload retention).
|
|
135
|
+
data: compactValue(record.data),
|
|
128
136
|
};
|
|
129
137
|
}
|
|
130
138
|
return record.type ? { type: record.type } : undefined;
|
|
@@ -178,7 +186,15 @@ function nonJsonLineResult(line: string): {
|
|
|
178
186
|
displayLine?: string;
|
|
179
187
|
json: boolean;
|
|
180
188
|
} {
|
|
181
|
-
|
|
189
|
+
// R3A-M3 (wave-2b): non-JSON lines persist VERBATIM — cap them so a
|
|
190
|
+
// force-flushed mega-line (up to MAX_LINE_BUFFER_BYTES) or a binary dump
|
|
191
|
+
// cannot balloon the transcript. JSON lines go through compactChildPiEvent
|
|
192
|
+
// (bounded per-field); this is the unbounded escape hatch.
|
|
193
|
+
const capped =
|
|
194
|
+
line.length > MAX_PERSISTED_LINE_BYTES
|
|
195
|
+
? `${line.slice(0, MAX_PERSISTED_LINE_BYTES)}[pi-crew truncated ${line.length - MAX_PERSISTED_LINE_BYTES} chars]`
|
|
196
|
+
: line;
|
|
197
|
+
return { json: false, persistedLine: capped, displayLine: capped };
|
|
182
198
|
}
|
|
183
199
|
|
|
184
200
|
function compactChildPiLine(
|
|
@@ -31,8 +31,14 @@
|
|
|
31
31
|
* event-log.ts) so sync flush handlers run first.
|
|
32
32
|
* - uncaughtException/unhandledRejection → setImmediate kick, never swallow:
|
|
33
33
|
* we add no output and no exit; existing pi-crew
|
|
34
|
-
* fatal handlers
|
|
35
|
-
*
|
|
34
|
+
* fatal handlers own crash reporting. NOTE
|
|
35
|
+
* (R3A-M1, 2026-10-17): they do NOT suppress —
|
|
36
|
+
* event-log's unhandledRejection handler re-throw
|
|
37
|
+
* keeps the process-fatal default, so a stray
|
|
38
|
+
* rejection still kills the host. Call sites must
|
|
39
|
+
* not rely on suppression existing; guard their
|
|
40
|
+
* own fire-and-forget promises (see
|
|
41
|
+
* live-agent-control.ts / live-session-runtime.ts).
|
|
36
42
|
*/
|
|
37
43
|
|
|
38
44
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
35
35
|
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
36
36
|
import { logInternalError } from "../../utils/internal-error.ts";
|
|
37
|
+
import { redactSecrets } from "../../utils/redaction.ts";
|
|
37
38
|
import { classifyBool } from "./classifier-service.ts";
|
|
38
39
|
|
|
39
40
|
export interface AmbientNoiseGateArgs {
|
|
@@ -88,7 +89,10 @@ export async function classifyAmbientNoiseGate(args: AmbientNoiseGateArgs): Prom
|
|
|
88
89
|
state: {
|
|
89
90
|
// Cap the summary: classifiers reason over small JSON state — the raw
|
|
90
91
|
// error can carry multi-KB stderr tails (same cap as retry-triage).
|
|
91
|
-
|
|
92
|
+
// R2-L-B8 (wave-2b): redact BEFORE the summary (and its slice) reaches
|
|
93
|
+
// classifier state — provider/API-key echoes in an auth failure would
|
|
94
|
+
// otherwise ride into the model prompt and the observed telemetry.
|
|
95
|
+
failureSummary: String(redactSecrets(args.failureSummary)).slice(0, 2000),
|
|
92
96
|
attempt: args.attempt,
|
|
93
97
|
maxAttempts: args.maxAttempts,
|
|
94
98
|
},
|
|
@@ -168,9 +168,13 @@ export function resultDigest(toolName: string, content: string, isError?: boolea
|
|
|
168
168
|
return undefined;
|
|
169
169
|
}
|
|
170
170
|
|
|
171
|
-
function createPrunedNotice(tokens: number, entry: ToolResultEntry): string {
|
|
171
|
+
export function createPrunedNotice(tokens: number, entry: ToolResultEntry): string {
|
|
172
172
|
const generic = `[Output pruned — ${tokens} tokens]`;
|
|
173
|
-
|
|
173
|
+
// R2-L-B7 (SEV-12): tools without a known digest format (edit/write/read…)
|
|
174
|
+
// previously fell back to the content-free generic notice — dependency-context
|
|
175
|
+
// consumers lost ALL evidence the tool did anything. Generic head-2/tail-2
|
|
176
|
+
// digest keeps a bounded content signal for every tool.
|
|
177
|
+
const digest = resultDigest(entry.toolName, entry.content, entry.isError, entry.exitCode) ?? genericLineDigest(entry.content);
|
|
174
178
|
if (!digest) return generic;
|
|
175
179
|
const genericTokens = countTokens(generic);
|
|
176
180
|
const maxTokens = Math.max(genericTokens, Math.floor(genericTokens * DIGEST_NOTICE_TOKEN_CAP_MULTIPLIER));
|
|
@@ -180,6 +184,20 @@ function createPrunedNotice(tokens: number, entry: ToolResultEntry): string {
|
|
|
180
184
|
return `${prefix}${truncateField(digest, maxChars)}${suffix}`;
|
|
181
185
|
}
|
|
182
186
|
|
|
187
|
+
/** R2-L-B7: generic digest for tools without a dedicated format — first two
|
|
188
|
+
* and last two non-empty lines, deduped when the content is short. Never
|
|
189
|
+
* empty (falls back to a line count) so the pruned notice always carries a
|
|
190
|
+
* content signal. */
|
|
191
|
+
function genericLineDigest(content: string): string {
|
|
192
|
+
const lines = (content ?? "")
|
|
193
|
+
.split(/\r?\n/)
|
|
194
|
+
.map((line) => line.trim())
|
|
195
|
+
.filter(Boolean);
|
|
196
|
+
if (lines.length === 0) return "empty output";
|
|
197
|
+
if (lines.length <= 4) return `${lines.length} line(s): ${lines.join(" ⏎ ")}`;
|
|
198
|
+
return `head=${lines.slice(0, 2).join(" ⏎ ")}; tail=${lines.slice(-2).join(" ⏎ ")}`;
|
|
199
|
+
}
|
|
200
|
+
|
|
183
201
|
// ---------------------------------------------------------------------------
|
|
184
202
|
// Target key resolution
|
|
185
203
|
// ---------------------------------------------------------------------------
|
|
@@ -292,6 +292,43 @@ function setAsyncAgentReaderCache(
|
|
|
292
292
|
}
|
|
293
293
|
}
|
|
294
294
|
|
|
295
|
+
/** FS2 (2026-10-17): sentinel returned by `readCrewAgents` when agents.json
|
|
296
|
+
* EXISTS but could not be read or parsed. Empty array by identity — every
|
|
297
|
+
* read-only caller keeps its "no agents" degrade behavior — but persisting
|
|
298
|
+
* callers compare `=== CREW_AGENTS_READ_FAILED` and SKIP the save so an
|
|
299
|
+
* unreadable agents.json is never overwritten with []. */
|
|
300
|
+
export const CREW_AGENTS_READ_FAILED: CrewAgentRecord[] = [];
|
|
301
|
+
|
|
302
|
+
/** FS2: strict disk read for the sync agents path. ENOENT is a legitimate
|
|
303
|
+
* first-write [] — every other read/parse failure returns the sentinel after
|
|
304
|
+
* logging (atomic-write.ts readJsonFile conflates both into undefined, so the
|
|
305
|
+
* errno/JSON.parse classification happens here). */
|
|
306
|
+
function readAgentsRecordsStrict(agentsFilePath: string): CrewAgentRecord[] {
|
|
307
|
+
let raw: string;
|
|
308
|
+
try {
|
|
309
|
+
raw = fs.readFileSync(agentsFilePath, "utf-8");
|
|
310
|
+
} catch (error) {
|
|
311
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return [];
|
|
312
|
+
logInternalError(
|
|
313
|
+
"crew-agents.read-failed",
|
|
314
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
315
|
+
`agentsPath=${agentsFilePath}`,
|
|
316
|
+
);
|
|
317
|
+
return CREW_AGENTS_READ_FAILED;
|
|
318
|
+
}
|
|
319
|
+
try {
|
|
320
|
+
const parsed = JSON.parse(raw) as unknown;
|
|
321
|
+
return Array.isArray(parsed) ? (parsed as CrewAgentRecord[]) : [];
|
|
322
|
+
} catch (error) {
|
|
323
|
+
logInternalError(
|
|
324
|
+
"crew-agents.parse-failed",
|
|
325
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
326
|
+
`agentsPath=${agentsFilePath}`,
|
|
327
|
+
);
|
|
328
|
+
return CREW_AGENTS_READ_FAILED;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
295
332
|
export function readCrewAgents(manifest: TeamRunManifest): CrewAgentRecord[] {
|
|
296
333
|
// F06 / RR-017: read-after-write WITHOUT forcing the pending coalesced write
|
|
297
334
|
// to disk. `upsertCrewAgent` always reads first, so the previous scoped
|
|
@@ -303,12 +340,20 @@ export function readCrewAgents(manifest: TeamRunManifest): CrewAgentRecord[] {
|
|
|
303
340
|
// written, so overlaying it preserves the "intra-process readers see the
|
|
304
341
|
// latest data" contract (and cross-process readers still see the file within
|
|
305
342
|
// one coalesce window) with zero I/O.
|
|
343
|
+
//
|
|
344
|
+
// FS2 (2026-10-17): a MISSING file (ENOENT) is a legitimate [] — first
|
|
345
|
+
// write. A file that EXISTS but cannot be read (EACCES/EIO) or parsed
|
|
346
|
+
// (corrupt JSON) must NOT degrade to [] (atomic-write.ts readJsonFile
|
|
347
|
+
// returns undefined for BOTH cases, so we classify ourselves): the next
|
|
348
|
+
// upsert would persist that [] and WIPE agents.json. Such reads return the
|
|
349
|
+
// CREW_AGENTS_READ_FAILED sentinel (identity ===, iterable-empty) and
|
|
350
|
+
// persisting callers (upsertCrewAgent / removeCrewAgent) skip the save.
|
|
306
351
|
const agentsFilePath = agentsPath(manifest);
|
|
307
352
|
const pending = peekPendingCoalescedWrite<CrewAgentRecord[]>(agentsFilePath);
|
|
308
353
|
try {
|
|
309
354
|
const records =
|
|
310
|
-
pending ??
|
|
311
|
-
|
|
355
|
+
pending ?? readJsonFileCoalesced(agentsFilePath, AGENT_READER_TTL_MS, () => readAgentsRecordsStrict(agentsFilePath));
|
|
356
|
+
if (records === CREW_AGENTS_READ_FAILED) return CREW_AGENTS_READ_FAILED;
|
|
312
357
|
// Validate schema and deduplicate by id to handle concurrent write conflicts
|
|
313
358
|
const seen = new Set<string>();
|
|
314
359
|
const deduped = records.filter((r) => {
|
|
@@ -581,8 +626,20 @@ export function upsertCrewAgent(manifest: TeamRunManifest, record: CrewAgentReco
|
|
|
581
626
|
asyncAgentReaderCache.delete(agentsPath(manifest));
|
|
582
627
|
return;
|
|
583
628
|
}
|
|
584
|
-
// Read current state
|
|
629
|
+
// Read current state. FS2 (2026-10-17): when agents.json exists but is
|
|
630
|
+
// unreadable/unparseable, readCrewAgents returns the CREW_AGENTS_READ_FAILED
|
|
631
|
+
// sentinel — persisting the merged (empty) array would WIPE the authoritative
|
|
632
|
+
// index over state we could not read. Skip the save entirely (log already
|
|
633
|
+
// emitted by the reader) and let the next healthy read repair it.
|
|
585
634
|
const existing = readCrewAgents(manifest);
|
|
635
|
+
if (existing === CREW_AGENTS_READ_FAILED) {
|
|
636
|
+
logInternalError(
|
|
637
|
+
"crew-agents.upsert-skipped-unreadable",
|
|
638
|
+
new Error("skipping agents.json persist — current records unreadable (read/parse failure)"),
|
|
639
|
+
`agentsPath=${agentsPath(manifest)} taskId=${record.taskId}`,
|
|
640
|
+
);
|
|
641
|
+
return;
|
|
642
|
+
}
|
|
586
643
|
// Deduplicate by id: keep newer record when same id appears
|
|
587
644
|
const idIndex = new Map(existing.map((item, i) => [item.id, i]));
|
|
588
645
|
const merged: CrewAgentRecord[] = existing.map((item) => (item.id === record.id ? record : item));
|
|
@@ -1000,7 +1057,13 @@ export function readCrewAgentEventsCursor(
|
|
|
1000
1057
|
const limit = typeof options.limit === "number" && Number.isInteger(options.limit) && options.limit >= 0 ? options.limit : undefined;
|
|
1001
1058
|
|
|
1002
1059
|
const filtered = readAgentEventMatches(filePath, sinceSeq);
|
|
1003
|
-
|
|
1060
|
+
// R8-L2 (wave-2b, FS3 twin): seq-sort BEFORE the limit slice. File-order
|
|
1061
|
+
// slicing on a non-monotonic file can deliver an early high seq inside the
|
|
1062
|
+
// first `limit` events; nextSeq=max-delivered then permanently skips every
|
|
1063
|
+
// undelivered lower-seq event for paging consumers. Sort ONLY under a
|
|
1064
|
+
// limit — the no-limit path delivers everything (nextSeq = true max) and
|
|
1065
|
+
// keeps its historical FILE-order contract (mirrors cursor.ts FS3).
|
|
1066
|
+
const events = limit !== undefined ? [...filtered].sort((a, b) => (a.seq as number) - (b.seq as number)).slice(0, limit) : filtered;
|
|
1004
1067
|
const returnedMaxSeq = events.reduce((max, event) => (typeof event.seq === "number" ? Math.max(max, event.seq) : max), sinceSeq);
|
|
1005
1068
|
return {
|
|
1006
1069
|
path: filePath,
|
|
@@ -91,10 +91,27 @@ export function createSubmitResultTool(
|
|
|
91
91
|
structuredData: params.structuredData,
|
|
92
92
|
},
|
|
93
93
|
};
|
|
94
|
+
// R2-L-B1 (wave-2b): a THROWING onYield previously still returned the
|
|
95
|
+
// success text — the model believed its result was recorded and ended
|
|
96
|
+
// the turn, silently dropping it. The error tool-result makes the
|
|
97
|
+
// failure visible so the worker retries or falls back to plain text.
|
|
94
98
|
try {
|
|
95
99
|
onYield(result);
|
|
96
100
|
} catch (error) {
|
|
97
101
|
logInternalError("submit-result-tool.yield", error, toolCallId);
|
|
102
|
+
return {
|
|
103
|
+
content: [
|
|
104
|
+
{
|
|
105
|
+
type: "text",
|
|
106
|
+
text: `Result submission FAILED: ${error instanceof Error ? error.message : String(error)} — the result was NOT recorded. Try again or state the result in your final message.`,
|
|
107
|
+
},
|
|
108
|
+
],
|
|
109
|
+
details: {
|
|
110
|
+
summary: params.summary,
|
|
111
|
+
...(params.artifacts ? { artifacts: params.artifacts } : {}),
|
|
112
|
+
...(params.structuredData ? { structuredData: params.structuredData } : {}),
|
|
113
|
+
},
|
|
114
|
+
};
|
|
98
115
|
}
|
|
99
116
|
return response;
|
|
100
117
|
},
|
|
@@ -3,8 +3,9 @@ import * as path from "node:path";
|
|
|
3
3
|
import type { TeamRunManifest } from "../state/types.ts";
|
|
4
4
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
5
5
|
import { projectCrewRoot } from "../utils/paths.ts";
|
|
6
|
+
import { assertSafePathId } from "../utils/safe-paths.ts";
|
|
6
7
|
|
|
7
|
-
export type DeadletterReason = "max-retries" | "heartbeat-dead" | "manual";
|
|
8
|
+
export type DeadletterReason = "max-retries" | "heartbeat-dead" | "manual" | "dispatch-fallback"; // FS7: catch-fallback failure (dispatch-batch)
|
|
8
9
|
|
|
9
10
|
export interface DeadletterEntry {
|
|
10
11
|
taskId: string;
|
|
@@ -35,6 +36,9 @@ export function deadletterPath(manifest: TeamRunManifest): string {
|
|
|
35
36
|
* this is the durable out-of-band record for post-mortem analysis.
|
|
36
37
|
*/
|
|
37
38
|
export function deadletterIndexPath(manifest: TeamRunManifest): string {
|
|
39
|
+
// R2-L-A8 (wave-2b): runId is interpolated into a project-level path —
|
|
40
|
+
// a manifest with ".."-bearing runId would escape the deadletter dir.
|
|
41
|
+
assertSafePathId("runId", manifest.runId);
|
|
38
42
|
return path.join(projectCrewRoot(manifest.cwd), "state", "deadletter", `${manifest.runId}.jsonl`);
|
|
39
43
|
}
|
|
40
44
|
|
|
@@ -147,13 +147,20 @@ export async function sweepExpiredWaitingTasks(
|
|
|
147
147
|
if (task.status !== "waiting" || task.waiting === undefined) continue;
|
|
148
148
|
if (task.waiting.deadline > now) continue;
|
|
149
149
|
const questionId = task.waiting.questionId;
|
|
150
|
-
|
|
150
|
+
// FS6 (2026-10-17): liveness BEFORE the emittedAskTimedout exactly-once
|
|
151
|
+
// guard. The guard used to mark the questionId on the FIRST sweep
|
|
152
|
+
// regardless of liveness — a worker ALIVE at sweep 1 (event emitted, task
|
|
153
|
+
// left waiting) that DIED before sweep 2 was skipped forever by the guard
|
|
154
|
+
// and never requeued. Now only the alive-worker "surfaced in-tool"
|
|
155
|
+
// event is exactly-once; a dead worker always proceeds to the requeue
|
|
156
|
+
// (the status transition makes the next sweep a natural no-op).
|
|
157
|
+
const workerAlive = isWaitingWorkerAlive(task, now);
|
|
158
|
+
if (workerAlive && emittedAskTimedoutQuestionIds.has(questionId)) continue; // exactly-once per process (alive-worker events only)
|
|
151
159
|
if (emittedAskTimedoutQuestionIds.size >= MAX_EMITTED_ASK_TIMEDOUT) {
|
|
152
160
|
const oldest = emittedAskTimedoutQuestionIds.keys().next().value;
|
|
153
161
|
if (oldest !== undefined) emittedAskTimedoutQuestionIds.delete(oldest);
|
|
154
162
|
}
|
|
155
163
|
emittedAskTimedoutQuestionIds.add(questionId);
|
|
156
|
-
const workerAlive = isWaitingWorkerAlive(task, now);
|
|
157
164
|
timedOutQuestionIds.push(questionId);
|
|
158
165
|
pendingEvents.push({ taskId: task.id, questionId, workerAlive });
|
|
159
166
|
if (!workerAlive) {
|
|
@@ -1018,14 +1025,48 @@ export async function dispatchBatch(ctx: SchedulerContext, decision: DispatchBat
|
|
|
1018
1025
|
const freshTasks = fresh?.tasks ?? ctx.tasks;
|
|
1019
1026
|
const freshTask = freshTasks.find((item) => item.id === task.id) ?? task;
|
|
1020
1027
|
if (freshTask.status !== "queued" && freshTask.status !== "running") return { manifest: freshManifest, tasks: freshTasks };
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1028
|
+
// FS7 (2026-10-17): the final fallback attempt previously ran OUTSIDE the
|
|
1029
|
+
// accounting/deadletter bookkeeping the retry-loop attempts get
|
|
1030
|
+
// (onRetryGivenUp above) — a throw here escaped with NO deadletter, no
|
|
1031
|
+
// metric, no event. Wrap it so a fallback failure lands in the SAME
|
|
1032
|
+
// bookkeeping as the loop attempts.
|
|
1033
|
+
try {
|
|
1034
|
+
return await withCorrelation(childCorrelation(freshManifest.runId, task.id), () =>
|
|
1035
|
+
runTeamTask({
|
|
1036
|
+
...baseInput,
|
|
1037
|
+
manifest: freshManifest,
|
|
1038
|
+
tasks: freshTasks,
|
|
1039
|
+
task: freshTask,
|
|
1040
|
+
}),
|
|
1041
|
+
);
|
|
1042
|
+
} catch (fallbackError) {
|
|
1043
|
+
const fallbackMessage = fallbackError instanceof Error ? fallbackError.message : String(fallbackError);
|
|
1044
|
+
appendDeadletter(freshManifest, {
|
|
1045
|
+
runId: freshManifest.runId,
|
|
1046
|
+
taskId: task.id,
|
|
1047
|
+
reason: "dispatch-fallback",
|
|
1048
|
+
attempts: 1,
|
|
1049
|
+
lastError: fallbackMessage,
|
|
1050
|
+
timestamp: new Date().toISOString(),
|
|
1051
|
+
agent: freshTask.agent,
|
|
1052
|
+
role: freshTask.role,
|
|
1053
|
+
modelAttempts: freshTask.modelAttempts?.length,
|
|
1054
|
+
runStatus: freshManifest.status,
|
|
1055
|
+
});
|
|
1056
|
+
input.metricRegistry
|
|
1057
|
+
?.counter("crew.task.deadletter_total", "Deadletter triggers by reason")
|
|
1058
|
+
.inc({ reason: "dispatch-fallback" });
|
|
1059
|
+
const nowIso = new Date().toISOString();
|
|
1060
|
+
const failedTasks = freshTasks.map((item) =>
|
|
1061
|
+
item.id === task.id && (item.status === "queued" || item.status === "running")
|
|
1062
|
+
? { ...item, status: "failed" as const, finishedAt: nowIso, error: fallbackMessage }
|
|
1063
|
+
: item,
|
|
1064
|
+
);
|
|
1065
|
+
return {
|
|
1066
|
+
manifest: updateRunStatus(freshManifest, "failed", `dispatch fallback failed: ${fallbackMessage}`),
|
|
1067
|
+
tasks: failedTasks,
|
|
1068
|
+
};
|
|
1069
|
+
}
|
|
1029
1070
|
}
|
|
1030
1071
|
};
|
|
1031
1072
|
// ── OPT-01 streaming dispatch: dispatch units into ctx.pendingUnits ──
|
|
@@ -16,6 +16,24 @@ export interface RunEffectivenessSummary {
|
|
|
16
16
|
severity: RunEffectivenessSeverity;
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
/** R9-F3 (2026-10-17): task ids with a `delegate.completed ok:true` event
|
|
20
|
+
* (data.subId). A completed grandchild delegate task carries no
|
|
21
|
+
* usage/transcript of its own — its real output flowed through the broker
|
|
22
|
+
* mailbox to the parent — so the guard previously misflagged it as
|
|
23
|
+
* "no observable worker activity" and escalated effective runs to blocked
|
|
24
|
+
* (R8-OPS M3 contributor). Callers pass this set in when events are
|
|
25
|
+
* available (finalize-run reads the log anyway). */
|
|
26
|
+
export function delegateCompletedTaskIdsFromEvents(events: ReadonlyArray<{ type?: string; data?: unknown }>): Set<string> {
|
|
27
|
+
const ids = new Set<string>();
|
|
28
|
+
for (const event of events) {
|
|
29
|
+
if (event.type !== "delegate.completed") continue;
|
|
30
|
+
const data = (event.data ?? {}) as { subId?: unknown; ok?: unknown };
|
|
31
|
+
if (data.ok !== true) continue;
|
|
32
|
+
if (typeof data.subId === "string") ids.add(data.subId);
|
|
33
|
+
}
|
|
34
|
+
return ids;
|
|
35
|
+
}
|
|
36
|
+
|
|
19
37
|
export function taskHasObservableWorkerActivity(task: TeamTaskState): boolean {
|
|
20
38
|
return Boolean(
|
|
21
39
|
(task.agentProgress?.toolCount ?? 0) > 0 ||
|
|
@@ -68,13 +86,18 @@ export function evaluateRunEffectiveness(input: {
|
|
|
68
86
|
tasks: TeamTaskState[];
|
|
69
87
|
executeWorkers: boolean;
|
|
70
88
|
runtimeConfig?: CrewRuntimeConfig;
|
|
89
|
+
/** R9-F3: task ids with delegate.completed ok:true events — counted as
|
|
90
|
+
* observable work (see delegateCompletedTaskIdsFromEvents). */
|
|
91
|
+
delegateCompleted?: ReadonlySet<string>;
|
|
71
92
|
}): RunEffectivenessSummary {
|
|
72
93
|
const completedTasks = input.tasks.filter((task) => task.status === "completed");
|
|
73
94
|
// F4: a completed task with an EMPTY result artifact (0 bytes) is
|
|
74
95
|
// treated exactly like no-observed-work — the worker may have spun up
|
|
75
96
|
// and streamed transcript events, but it produced no real output
|
|
76
97
|
// (e.g. a 429 rate-limit absorbed by the fallback chain).
|
|
77
|
-
const noObservedWorkTasks = completedTasks.filter(
|
|
98
|
+
const noObservedWorkTasks = completedTasks.filter(
|
|
99
|
+
(task) => (!input.delegateCompleted?.has(task.id) && !taskHasObservableWorkerActivity(task)) || taskHasEmptyResult(task),
|
|
100
|
+
);
|
|
78
101
|
const needsAttentionTasks = input.tasks.filter((task) => task.agentProgress?.activityState === "needs_attention");
|
|
79
102
|
const workerExecution: WorkerExecutionState = input.executeWorkers ? "enabled" : "disabled/scaffold";
|
|
80
103
|
const guardMode = resolveEffectivenessGuardMode(input.runtimeConfig, input.manifest);
|