pi-crew 0.11.21 → 0.11.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/index.mjs +537 -526
- package/package.json +2 -1
- package/src/errors.ts +14 -1
- package/src/extension/async-notifier.ts +42 -17
- package/src/extension/cross-extension-rpc.ts +42 -5
- package/src/extension/notification-router.ts +15 -2
- package/src/extension/registration/lifecycle-handlers.ts +36 -13
- package/src/extension/registration/runtime-cleanup.ts +17 -0
- package/src/extension/registration/wire-cross-extension.ts +26 -0
- package/src/extension/rpc-hmac.ts +53 -2
- package/src/extension/team-tool/cancel.ts +34 -21
- package/src/extension/team-tool/dispatch/index.ts +6 -5
- package/src/extension/team-tool/dispatch/status.ts +42 -2
- package/src/extension/team-tool.ts +24 -2
- package/src/prompt/inbox-poll.ts +77 -25
- package/src/prompt/message-tool.ts +62 -4
- package/src/prompt/prompt-runtime.ts +82 -32
- package/src/runtime/broker/crew-broker-client.ts +74 -16
- package/src/runtime/broker/crew-broker.ts +111 -77
- package/src/runtime/broker/protocol/msg-send-targets.ts +74 -0
- package/src/runtime/broker/wait-status-cache.ts +14 -5
- package/src/runtime/child-pi/child-pi-spawn.ts +47 -9
- package/src/runtime/child-pi/child-pi.ts +20 -2
- package/src/runtime/child-pi/mock-fixtures.ts +23 -0
- package/src/runtime/compaction/compact-stages/truncation-stage.ts +45 -3
- package/src/runtime/compaction/important-line-classifier.ts +85 -11
- package/src/runtime/compaction/tool-output-pruner.ts +35 -11
- package/src/runtime/crew-agent-records.ts +7 -1
- package/src/runtime/delivery-coordinator.ts +30 -0
- package/src/runtime/dwf-state-store.ts +41 -7
- package/src/runtime/finalize-run.ts +13 -2
- package/src/runtime/goal-workflow/dynamic-workflow-context.ts +192 -64
- package/src/runtime/goal-workflow/goal-loop-runner.ts +130 -3
- package/src/runtime/heartbeat/heartbeat-watcher.ts +7 -0
- package/src/runtime/live-session/embedded-tier.ts +3 -1
- package/src/runtime/live-session/live-session-runtime.ts +54 -1
- package/src/runtime/model/model-fallback.ts +23 -1
- package/src/runtime/output/pi-json-output.ts +26 -1
- package/src/runtime/policy-engine.ts +18 -0
- package/src/runtime/process/zombie-scanner.ts +78 -14
- package/src/runtime/process-status.ts +6 -1
- package/src/runtime/recovery/crash-recovery.ts +170 -28
- package/src/runtime/rpc/rpc-worker.ts +44 -1
- package/src/runtime/scheduling/run-coalesced-task-group.ts +64 -3
- package/src/runtime/scheduling/scheduler.ts +74 -11
- package/src/runtime/scratchpad/engine.ts +26 -3
- package/src/runtime/settings-store.ts +22 -2
- package/src/runtime/surface/tmux-provider.ts +24 -3
- package/src/runtime/task-health.ts +16 -1
- package/src/runtime/task-output-context.ts +143 -71
- package/src/runtime/task-packet.ts +70 -17
- package/src/runtime/task-runner/child-executor.ts +27 -0
- package/src/runtime/task-runner/output-splitter.ts +79 -16
- package/src/runtime/task-runner/post-execution.ts +75 -15
- package/src/runtime/task-runner/state-helpers.ts +92 -2
- package/src/runtime/usage-tracker.ts +11 -0
- package/src/runtime/verification/green-contract.ts +17 -0
- package/src/runtime/verification/verification-gates.ts +24 -6
- package/src/runtime/workspace-lock.ts +53 -0
- package/src/schema/team-tool-schema.ts +10 -4
- package/src/state/coordination/mailbox.ts +133 -38
- package/src/state/decision-ledger.ts +56 -8
- package/src/state/event-log/event-log-rotation.ts +83 -5
- package/src/state/event-log/event-log.ts +181 -12
- package/src/state/stores/active-run-registry.ts +10 -3
- package/src/state/types.ts +9 -1
- package/src/ui/agents-jobs-browser.ts +22 -8
- package/src/ui/dashboard-panes/agents-pane.ts +5 -2
- package/src/ui/events-state-source.ts +67 -28
- package/src/ui/inline-panel/agent-pane.ts +13 -6
- package/src/ui/inline-panel/agent-transcript.ts +33 -21
- package/src/ui/inline-panel/agent-view-overlay.ts +18 -3
- package/src/ui/inline-panel/index.ts +21 -1
- package/src/ui/live-run-sidebar.ts +14 -3
- package/src/ui/overlays/mailbox-compose-overlay.ts +10 -4
- package/src/ui/powerbar-publisher.ts +6 -3
- package/src/ui/rail.ts +7 -1
- package/src/ui/render-scheduler.ts +53 -1
- package/src/ui/run-action-dispatcher.ts +68 -20
- package/src/ui/status-colors.ts +9 -0
- package/src/utils/env-filter.ts +8 -0
|
@@ -414,6 +414,21 @@ export function attemptErrorFor(
|
|
|
414
414
|
const rateLimitErr = detectRetryableModelFailureFromOutput(parsedOutput);
|
|
415
415
|
if (rateLimitErr) err = rateLimitErr;
|
|
416
416
|
}
|
|
417
|
+
// DR-01 (deep review 2026-10-14, P0 false-green): upstream rate-limit
|
|
418
|
+
// exhaustion can end a session WITHOUT stopReason:"error" — the transcript
|
|
419
|
+
// is a chain of EMPTY assistant message_end events (content [], usage 0)
|
|
420
|
+
// between auto-retry cycles, ending in agent_settled. The detector above
|
|
421
|
+
// misses that shape (no errorMessage is ever emitted), and finalText lies
|
|
422
|
+
// (it keeps the LAST NON-EMPTY text — an early opening sentence from
|
|
423
|
+
// before the outage, which is exactly what made failed audits look
|
|
424
|
+
// green). settledWithoutAnswer tracks the actual FINAL assistant message;
|
|
425
|
+
// patches still count as real work (coder roles may legitimately end
|
|
426
|
+
// patch-only); cancelled attempts are excluded so an abort never burns a
|
|
427
|
+
// fallback model. Message includes rate-limit/429 keywords so
|
|
428
|
+
// isRetryableModelFailure lets the model-fallback chain advance.
|
|
429
|
+
if (!err && !childResult.exitStatus?.cancelled && parsedOutput?.settledWithoutAnswer === true && !(parsedOutput.patches?.length ?? 0)) {
|
|
430
|
+
err = "Model settled without a final answer (empty final assistant message; upstream rate-limit/429 exhaustion)";
|
|
431
|
+
}
|
|
417
432
|
return err;
|
|
418
433
|
}
|
|
419
434
|
|
|
@@ -797,6 +812,13 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
797
812
|
workerTransport: input.runtimeConfig?.workerTransport,
|
|
798
813
|
agent: input.agent,
|
|
799
814
|
model,
|
|
815
|
+
// R6-8 (DR-03 round-6): thread the full candidate chain (primary +
|
|
816
|
+
// fallbacks) so buildChildPiSpawnOptions scopes provider env keys to
|
|
817
|
+
// the UNION of the chain's providers — a bare-id primary would
|
|
818
|
+
// otherwise spawn with ZERO provider keys and break the chain on a
|
|
819
|
+
// non-retryable auth failure while qualified fallbacks sat queued.
|
|
820
|
+
// Parameter threading only — the attempt loop logic is untouched.
|
|
821
|
+
modelChain: attemptModels,
|
|
800
822
|
signal: timeoutController.signal,
|
|
801
823
|
transcriptPath,
|
|
802
824
|
maxDepth: input.limits?.maxTaskDepth,
|
|
@@ -1265,9 +1287,14 @@ export async function runChildProcessTask(ctx: TaskExecutionContext): Promise<Ta
|
|
|
1265
1287
|
if (error && modelAttempts.length > 1) {
|
|
1266
1288
|
// E2/E1 (Round 15): structured CrewError (E008). Build via the factory so
|
|
1267
1289
|
// the error carries a code + help hint; keep its .message as the task error.
|
|
1290
|
+
// J4 (DR-03 round-6): attemptModels may still hold UNTRIED candidates when
|
|
1291
|
+
// the loop broke early (non-retryable failure, classifier veto, or
|
|
1292
|
+
// spawn-budget stop) — pass the planned chain length so E008 distinguishes
|
|
1293
|
+
// "Model chain failed early (tried X of N)" from genuine exhaustion.
|
|
1268
1294
|
error = errors.modelExhausted(
|
|
1269
1295
|
modelAttempts.map((a) => a.model),
|
|
1270
1296
|
error,
|
|
1297
|
+
attemptModels.length,
|
|
1271
1298
|
).message;
|
|
1272
1299
|
}
|
|
1273
1300
|
// bug-026 sub-issue B: reclassify after the E008 modelExhausted rewrite —
|
|
@@ -28,15 +28,36 @@
|
|
|
28
28
|
* raw output. When the deadletter strategy is returned, all entries have
|
|
29
29
|
* empty text — the caller decides what marker to surface per task.
|
|
30
30
|
*
|
|
31
|
-
* Note:
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
31
|
+
* Note: deterministic — no I/O, no LLM calls. (R6-11 exception: duplicate
|
|
32
|
+
* delimiter hits emit a structured `logInternalError` warn to the console
|
|
33
|
+
* audit trail; parsing itself stays deterministic.) This is deliberately
|
|
34
|
+
* the simplest possible splitter; we don't try fuzzy matching, NL
|
|
35
|
+
* extraction, or retry-the-LLM. The MVP trusts workers to follow
|
|
36
|
+
* delimiters 99% of the time; a zero-marker miss degrades to broadcast
|
|
37
|
+
* (nothing task-specific exists to leak), and a partial violation now
|
|
38
|
+
* fails closed (G20) instead of leaking cross-task delimited bodies.
|
|
39
|
+
*
|
|
40
|
+
* R6-11 (DR-03 round-6): duplicate-delimiter handling. A worker that QUOTES
|
|
41
|
+
* an earlier task's delimited block (in prose, code fences, or a transcript)
|
|
42
|
+
* and then emits its own genuine block produces two `<<<TASK_RESULT:id>>>`
|
|
43
|
+
* hits for the same id. Previously first-occurrence-wins let the quoted /
|
|
44
|
+
* forged block steal attribution with full `strategy:"delimiter"` trust.
|
|
45
|
+
* Now the LAST occurrence wins (the worker's own genuine emission follows
|
|
46
|
+
* any quote of earlier material), and every duplicate hit raises a
|
|
47
|
+
* structured warn via `logInternalError` naming the task and hit count so
|
|
48
|
+
* operators can audit the group.
|
|
49
|
+
*
|
|
50
|
+
* R6-11 residual (documented, accepted): emit-then-quote inversion. A worker
|
|
51
|
+
* that emits its genuine block FIRST and quotes a forged `<<<TASK_RESULT:…>>>`
|
|
52
|
+
* block LAST still misattributes under last-wins — as would first-wins under
|
|
53
|
+
* the mirror-image attack. Without content signing (cryptographic proof that
|
|
54
|
+
* a block was emitted by its owning worker, not merely quoted) neither
|
|
55
|
+
* ordering heuristic can distinguish genuine from quoted. Signing is out of
|
|
56
|
+
* scope; the duplicate-hit warn is the audit signal for exactly this case.
|
|
38
57
|
*/
|
|
39
58
|
|
|
59
|
+
import { logInternalError } from "../../utils/internal-error.ts";
|
|
60
|
+
|
|
40
61
|
export interface SplitResult {
|
|
41
62
|
taskId: string;
|
|
42
63
|
text: string;
|
|
@@ -60,6 +81,7 @@ export function splitCoalescedOutput(rawOutput: string, taskIds: string[]): Spli
|
|
|
60
81
|
|
|
61
82
|
const byId = new Map<string, string>();
|
|
62
83
|
const delimiterHits = new Set<string>();
|
|
84
|
+
const delimiterHitCounts = new Map<string, number>();
|
|
63
85
|
|
|
64
86
|
// Strategy 1: delimiter parse.
|
|
65
87
|
// Pattern: <<<TASK_RESULT:{id}>>> ... <<<END_TASK_RESULT>>>
|
|
@@ -69,10 +91,28 @@ export function splitCoalescedOutput(rawOutput: string, taskIds: string[]): Spli
|
|
|
69
91
|
while ((match = delimiterRegex.exec(rawOutput)) !== null) {
|
|
70
92
|
const id = match[1]!;
|
|
71
93
|
const body = match[2]!.trim();
|
|
72
|
-
if (taskIds.includes(id)
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
94
|
+
if (!taskIds.includes(id)) continue;
|
|
95
|
+
delimiterHitCounts.set(id, (delimiterHitCounts.get(id) ?? 0) + 1);
|
|
96
|
+
// R6-11: LAST-wins. First-occurrence-wins previously let a quoted /
|
|
97
|
+
// forged block steal attribution (a worker quoting an earlier task's
|
|
98
|
+
// delimited output before emitting its own genuine block). With
|
|
99
|
+
// last-wins the worker's own emission wins; duplicates raise a
|
|
100
|
+
// structured warn below. Residual: emit-then-quote inversion — see
|
|
101
|
+
// the header note; unsolvable without content signing.
|
|
102
|
+
byId.set(id, body);
|
|
103
|
+
delimiterHits.add(id);
|
|
104
|
+
}
|
|
105
|
+
// R6-11 (b): duplicate-hit detection → structured warn naming the task
|
|
106
|
+
// and hit count, so misattribution attempts (and accidental self-quotes)
|
|
107
|
+
// surface in the audit trail instead of silently shifting attribution.
|
|
108
|
+
for (const [id, hits] of delimiterHitCounts) {
|
|
109
|
+
if (hits <= 1) continue;
|
|
110
|
+
logInternalError(
|
|
111
|
+
"output-splitter.duplicate-delimiter",
|
|
112
|
+
new Error(`task "${id}" produced ${hits} TASK_RESULT delimiter blocks; LAST occurrence wins attribution`),
|
|
113
|
+
`taskId=${id} hits=${hits} resolution=last-wins`,
|
|
114
|
+
"warn",
|
|
115
|
+
);
|
|
76
116
|
}
|
|
77
117
|
|
|
78
118
|
if (delimiterHits.size === taskIds.length) {
|
|
@@ -135,12 +175,32 @@ export function splitCoalescedOutput(rawOutput: string, taskIds: string[]): Spli
|
|
|
135
175
|
}));
|
|
136
176
|
}
|
|
137
177
|
|
|
178
|
+
/**
|
|
179
|
+
* True iff `headerText` attributes the section to `id` by exact or
|
|
180
|
+
* word-boundary match. R6-11: plain substring matching let a shorter id
|
|
181
|
+
* claim a longer id's header ("task-a" matching "### Task task-alpha").
|
|
182
|
+
* `id` is escaped before regex construction; the surrounding `\b` anchors
|
|
183
|
+
* require a non-word char (or string edge) on both sides, so "task-a" no
|
|
184
|
+
* longer matches "task-alpha" while "task-a", "task-a:" and
|
|
185
|
+
* "task-a (done)" all still match "task-a".
|
|
186
|
+
*/
|
|
187
|
+
function headerMatchesId(headerText: string, id: string): boolean {
|
|
188
|
+
if (headerText === id) return true;
|
|
189
|
+
const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
190
|
+
return new RegExp(`\\b${escaped}\\b`).test(headerText);
|
|
191
|
+
}
|
|
192
|
+
|
|
138
193
|
/**
|
|
139
194
|
* Strategy 2 implementation: parse sections by heading patterns.
|
|
140
195
|
*
|
|
141
196
|
* Matches `### Task N of M` (N=1..M) or `### Task N` headers, then
|
|
142
197
|
* captures text up to the next header or end-of-input.
|
|
143
198
|
*
|
|
199
|
+
* R6-11: numbered sections map to taskIds by their declared `num`
|
|
200
|
+
* (`### Task 2 of 2` → taskIds[1]), NOT by scan position — a worker that
|
|
201
|
+
* emits its sections out of order previously had them assigned positionally
|
|
202
|
+
* to the wrong tasks. Out-of-range `num` values are ignored (no task to map).
|
|
203
|
+
*
|
|
144
204
|
* Falls back to "first heading wins all text" if no N-of-M numbering is
|
|
145
205
|
* present, which is common in worker output that just labels sections.
|
|
146
206
|
*/
|
|
@@ -155,12 +215,15 @@ function parseBySectionHeadings(rawOutput: string, taskIds: string[]): Map<strin
|
|
|
155
215
|
}
|
|
156
216
|
if (sections.length >= taskIds.length) {
|
|
157
217
|
sections.sort((a, b) => a.start - b.start);
|
|
158
|
-
for (let i = 0; i < sections.length
|
|
218
|
+
for (let i = 0; i < sections.length; i += 1) {
|
|
159
219
|
const section = sections[i]!;
|
|
160
220
|
const end = sections[i + 1]?.start ?? rawOutput.length;
|
|
161
221
|
const text = rawOutput.slice(section.start, end).trim();
|
|
162
|
-
|
|
163
|
-
|
|
222
|
+
// R6-11: honor the declared `num` — the section's own identity —
|
|
223
|
+
// instead of mapping by scan position.
|
|
224
|
+
const taskId = taskIds[section.num - 1];
|
|
225
|
+
if (taskId === undefined) continue; // num out of range — no task to map
|
|
226
|
+
if (!result.has(taskId)) result.set(taskId, text); // first occurrence wins
|
|
164
227
|
}
|
|
165
228
|
return result;
|
|
166
229
|
}
|
|
@@ -168,7 +231,7 @@ function parseBySectionHeadings(rawOutput: string, taskIds: string[]): Map<strin
|
|
|
168
231
|
const idHeaderRegex = /^(#{2,3})\s+Task\s+([^\s].*?)$/gm;
|
|
169
232
|
while ((m = idHeaderRegex.exec(rawOutput)) !== null) {
|
|
170
233
|
const headerText = m[2]!.trim();
|
|
171
|
-
const matchedId = taskIds.find((id) => headerText
|
|
234
|
+
const matchedId = taskIds.find((id) => headerMatchesId(headerText, id));
|
|
172
235
|
if (matchedId && !result.has(matchedId)) {
|
|
173
236
|
result.set(matchedId, ""); // placeholder; filled below
|
|
174
237
|
}
|
|
@@ -179,7 +242,7 @@ function parseBySectionHeadings(rawOutput: string, taskIds: string[]): Map<strin
|
|
|
179
242
|
const idHeaderRegex2 = /^(#{2,3})\s+Task\s+([^\s].*?)$/gm;
|
|
180
243
|
while ((m = idHeaderRegex2.exec(rawOutput)) !== null) {
|
|
181
244
|
const headerText = m[2]!.trim();
|
|
182
|
-
const matchedId = taskIds.find((id) => headerText
|
|
245
|
+
const matchedId = taskIds.find((id) => headerMatchesId(headerText, id));
|
|
183
246
|
if (matchedId) headerPositions.push({ taskId: matchedId, start: m.index + m[0].length });
|
|
184
247
|
}
|
|
185
248
|
headerPositions.sort((a, b) => a.start - b.start);
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { readFileSync } from "node:fs";
|
|
18
18
|
import { appendHookEvent, executeHook } from "../../hooks/registry.ts";
|
|
19
|
+
import type { TeamEventType } from "../../state/contracts.ts";
|
|
19
20
|
import { withRunLock } from "../../state/coordination/locks.ts";
|
|
20
21
|
import { appendEventAsync } from "../../state/event-log/event-log.ts";
|
|
21
22
|
import { writeArtifact } from "../../state/stores/artifact-store.ts";
|
|
@@ -25,6 +26,7 @@ import type {
|
|
|
25
26
|
OperationTerminalEvidence,
|
|
26
27
|
TeamRunManifest,
|
|
27
28
|
TeamTaskState,
|
|
29
|
+
TeamTaskStatus,
|
|
28
30
|
VerificationEvidence,
|
|
29
31
|
} from "../../state/types.ts";
|
|
30
32
|
import { captureWorktreeDiffAsync, captureWorktreeDiffStatAsync } from "../../worktree/worktree-manager.ts";
|
|
@@ -47,6 +49,24 @@ import { buildWorkerPromptPipeline } from "./prompt-pipeline.ts";
|
|
|
47
49
|
import { computeSpecGate } from "./spec-evidence.ts";
|
|
48
50
|
import { persistSingleTaskUpdate, updateTask } from "./state-helpers.ts";
|
|
49
51
|
|
|
52
|
+
/**
|
|
53
|
+
* R6-1 (DR-03, 2026-10-16): terminal status → singleton-path terminal event
|
|
54
|
+
* type. When persistSingleTaskUpdate PRESERVED a disk terminal status (e.g.
|
|
55
|
+
* `cancelled` written by crash-recovery while this worker was settling), the
|
|
56
|
+
* emitted event must match the PRESERVED record — emitting task.completed
|
|
57
|
+
* over a cancelled tasks.json entry recreates the event-log vs tasks.json
|
|
58
|
+
* divergence crash-recovery reconstructs from. Every terminal task status
|
|
59
|
+
* has a matching event type in TEAM_EVENT_TYPES; the lookup is total in
|
|
60
|
+
* practice, the undefined guard is defensive only.
|
|
61
|
+
*/
|
|
62
|
+
const TERMINAL_TASK_EVENT_TYPES: Readonly<Partial<Record<TeamTaskStatus, TeamEventType>>> = {
|
|
63
|
+
completed: "task.completed",
|
|
64
|
+
failed: "task.failed",
|
|
65
|
+
cancelled: "task.cancelled",
|
|
66
|
+
skipped: "task.skipped",
|
|
67
|
+
needs_attention: "task.needs_attention",
|
|
68
|
+
};
|
|
69
|
+
|
|
50
70
|
/**
|
|
51
71
|
* Branch execution output — the set of variables produced by the
|
|
52
72
|
* child-process / live-session / scaffold branch dispatch. These flow
|
|
@@ -424,12 +444,23 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
424
444
|
? "Safe scaffold mode; verification commands were not executed."
|
|
425
445
|
: `${runtimeKind} worker finished without reporting a verification failure.`,
|
|
426
446
|
);
|
|
447
|
+
// R4-5 (DR-02 wave 2): decorative contract — required green level but zero
|
|
448
|
+
// verification commands — means satisfaction is purely self-attested. The
|
|
449
|
+
// `decorative` flag is stamped by createVerificationEvidence; surface it in
|
|
450
|
+
// the notes too so the evidence artifact reads honestly WITHOUT changing
|
|
451
|
+
// the satisfied logic or defaultVerificationContract defaults.
|
|
452
|
+
const seededEvidence: VerificationEvidence = baseEvidence.decorative
|
|
453
|
+
? {
|
|
454
|
+
...baseEvidence,
|
|
455
|
+
notes: `${baseEvidence.notes ?? ""} — self-attested (no verification commands configured)`,
|
|
456
|
+
}
|
|
457
|
+
: baseEvidence;
|
|
427
458
|
|
|
428
459
|
// Only execute verification commands when:
|
|
429
460
|
// 1. Task completed successfully (no error)
|
|
430
461
|
// 2. Verification contract has commands
|
|
431
462
|
// 3. Not in scaffold mode (scaffold mode intentionally skips execution)
|
|
432
|
-
let verificationEvidence: VerificationEvidence =
|
|
463
|
+
let verificationEvidence: VerificationEvidence = seededEvidence;
|
|
433
464
|
if (runtimeKind !== "scaffold" && taskPacket.verification?.commands?.length) {
|
|
434
465
|
try {
|
|
435
466
|
const commandResults = await executeVerificationCommands(
|
|
@@ -470,11 +501,20 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
470
501
|
: `${failedCount}/${commandResults.length} verification commands failed`,
|
|
471
502
|
};
|
|
472
503
|
} catch (execError) {
|
|
473
|
-
//
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
504
|
+
// B4 (DR-02 wave 2): verification execution THREW — fail CLOSED instead of
|
|
505
|
+
// keeping the self-attested baseEvidence (satisfied=true via manual
|
|
506
|
+
// evidence), which used to green-light a task whose verification never
|
|
507
|
+
// ran. Only contracts that require a green level flip to unsatisfied;
|
|
508
|
+
// level "none" has nothing to gate, so baseEvidence stays untouched.
|
|
509
|
+
if (taskPacket.verification && taskPacket.verification.requiredGreenLevel !== "none") {
|
|
510
|
+
const message = execError instanceof Error ? execError.message : String(execError);
|
|
511
|
+
verificationEvidence = {
|
|
512
|
+
...seededEvidence,
|
|
513
|
+
satisfied: false,
|
|
514
|
+
observedGreenLevel: "none",
|
|
515
|
+
notes: `Verification execution failed (fail-closed): ${message}`,
|
|
516
|
+
};
|
|
517
|
+
}
|
|
478
518
|
}
|
|
479
519
|
}
|
|
480
520
|
|
|
@@ -659,7 +699,15 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
659
699
|
await saveRunManifestAsync(manifest);
|
|
660
700
|
return persistSingleTaskUpdate(manifest, tasks, task, undefined, true);
|
|
661
701
|
});
|
|
662
|
-
|
|
702
|
+
// R6-1 (DR-03): persistSingleTaskUpdate may have PRESERVED a disk terminal
|
|
703
|
+
// status (e.g. cancelled by crash-recovery / stale-reconciler) instead of
|
|
704
|
+
// this worker's settle status. Everything downstream of the persist — the
|
|
705
|
+
// agents.json record, the terminal event, and the after_task_complete hook
|
|
706
|
+
// status — must match the PRESERVED record, never the in-memory verdict.
|
|
707
|
+
const persistedTask = tasks.find((t) => t.id === task.id) ?? task;
|
|
708
|
+
const preservedStatus: TeamTaskStatus | undefined = persistedTask.status !== task.status ? persistedTask.status : undefined;
|
|
709
|
+
const effectiveStatus: TeamTaskStatus = preservedStatus ?? task.status;
|
|
710
|
+
upsertCrewAgent(manifest, recordFromTask(manifest, persistedTask, runtimeKind));
|
|
663
711
|
// Execute task_result hook before emitting terminal event
|
|
664
712
|
const hookReport = await executeHook("task_result", {
|
|
665
713
|
runId: manifest.runId,
|
|
@@ -667,23 +715,35 @@ export async function finalizeTaskResult(ctx: TaskExecutionContext, execResult:
|
|
|
667
715
|
cwd: manifest.cwd,
|
|
668
716
|
});
|
|
669
717
|
appendHookEvent(manifest, hookReport);
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
718
|
+
// R6-1: emit the terminal event for the PRESERVED status when one was
|
|
719
|
+
// preserved (task.cancelled over a cancelled record — never task.completed);
|
|
720
|
+
// otherwise the inferred settle event exactly as before.
|
|
721
|
+
const terminalEventType = TERMINAL_TASK_EVENT_TYPES[effectiveStatus];
|
|
722
|
+
if (terminalEventType !== undefined) {
|
|
675
723
|
// bug-026 sub-issue B: surface the classified fatal-fs cause (enospc/
|
|
676
724
|
// edquot/emfile/enfile) on the failure event so operators see "disk
|
|
677
725
|
// full" instead of a generic timeout diagnostic.
|
|
678
|
-
|
|
679
|
-
|
|
726
|
+
const eventData: Record<string, unknown> = {
|
|
727
|
+
...(task.failureCause ? { failureCause: task.failureCause } : {}),
|
|
728
|
+
...(preservedStatus ? { preservedFrom: task.status } : {}),
|
|
729
|
+
};
|
|
730
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
731
|
+
type: terminalEventType,
|
|
732
|
+
runId: manifest.runId,
|
|
733
|
+
taskId: task.id,
|
|
734
|
+
// For a preserved terminal the disk record's error (the cancel
|
|
735
|
+
// reason) is the truthful message; otherwise the settle error.
|
|
736
|
+
message: preservedStatus ? (persistedTask.error ?? error) : error,
|
|
737
|
+
...(Object.keys(eventData).length > 0 ? { data: eventData } : {}),
|
|
738
|
+
});
|
|
739
|
+
}
|
|
680
740
|
|
|
681
741
|
// Execute after_task_complete lifecycle hook (non-blocking)
|
|
682
742
|
const afterTaskReport = await executeHook("after_task_complete", {
|
|
683
743
|
runId: manifest.runId,
|
|
684
744
|
taskId: task.id,
|
|
685
745
|
cwd: manifest.cwd,
|
|
686
|
-
status:
|
|
746
|
+
status: effectiveStatus,
|
|
687
747
|
});
|
|
688
748
|
appendHookEvent(manifest, afterTaskReport);
|
|
689
749
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as fs from "node:fs";
|
|
2
2
|
import { loadConfig } from "../../config/config.ts";
|
|
3
3
|
import { flushPendingAtomicWrites } from "../../state/atomic-write.ts";
|
|
4
|
+
import { isTerminalTaskStatus } from "../../state/contracts.ts";
|
|
4
5
|
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
5
6
|
import { loadRunManifestById, saveRunTasksCoalesced } from "../../state/stores/state-store.ts";
|
|
6
7
|
import type { TaskCheckpointState, TeamRunManifest, TeamTaskState } from "../../state/types.ts";
|
|
@@ -11,6 +12,87 @@ export function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): Team
|
|
|
11
12
|
return tasks.map((task) => (task.id === updated.id ? updated : task));
|
|
12
13
|
}
|
|
13
14
|
|
|
15
|
+
/**
|
|
16
|
+
* R6-1 + R6-4 (DR-03, 2026-10-16): field-level merge of the INCOMING task
|
|
17
|
+
* onto the DISK task at the persistSingleTaskUpdate merge point. BUG-028
|
|
18
|
+
* made DISK authoritative for SIBLING tasks; this extends disk authority to
|
|
19
|
+
* two fields of THIS task where the incoming runner snapshot is provably
|
|
20
|
+
* stale relative to another writer:
|
|
21
|
+
*
|
|
22
|
+
* R6-1 terminal-preserve — when the incoming status is TERMINAL and disk
|
|
23
|
+
* already holds a DIFFERENT terminal status (e.g. `cancelled` written by
|
|
24
|
+
* crash-recovery / stale-reconciler / external cancel while a detached
|
|
25
|
+
* worker was still settling), the disk terminal status + error are
|
|
26
|
+
* preserved (mirror of the coalesced cancelled branch,
|
|
27
|
+
* run-coalesced-task-group.ts:367, and the R13-S1 terminal-preserve class
|
|
28
|
+
* in team-tool api). Without this, a late worker settle overwrote the
|
|
29
|
+
* leader's cancellation with `completed` and post-execution then emitted
|
|
30
|
+
* task.completed after crew.run.orphan_cancelled — cancelled run holding a
|
|
31
|
+
* completed task, event-log vs tasks.json divergence. Incoming fields other
|
|
32
|
+
* than status/error (result artifacts, finishedAt, usage, …) are still taken
|
|
33
|
+
* from the incoming record — the worker DID run; only the terminal verdict
|
|
34
|
+
* is not ours to overwrite. Non-terminal disk status → incoming wins as
|
|
35
|
+
* before (normal settle path); incoming non-terminal → untouched (the
|
|
36
|
+
* heartbeat resurrect window is guarded upstream by live-executor's
|
|
37
|
+
* isCurrent disk-status check, R13-S1 style).
|
|
38
|
+
*
|
|
39
|
+
* R6-4 allocation merge — delegate budget accounting (crew-broker writes
|
|
40
|
+
* task.allocation.tokensSpent under the run lock: reserve = disk + reserved
|
|
41
|
+
* at admission, roll-up = − reserved + actual on the then-current base at
|
|
42
|
+
* completion) was a lost update vs the runner's ~1s heartbeat whole-record
|
|
43
|
+
* replace which carries the dispatch-time (pre-reserve) copy. Rule chosen:
|
|
44
|
+
* `tokensSpent = max(disk, incoming)`. Verification against the broker
|
|
45
|
+
* arithmetic (read before choosing — crew-broker.ts reserve ~:1575-1602,
|
|
46
|
+
* roll-up ~:1690-1708, P2-8c refund ~:1745): every broker write is computed
|
|
47
|
+
* off the then-current DISK base; each roll-up subtracts exactly the amount
|
|
48
|
+
* its own reserve previously added and adds back actual ∈ [0, reserved], so
|
|
49
|
+
* disk(t) ≥ any runner snapshot for snapshots that predate their reserves'
|
|
50
|
+
* roll-ups — and the runner NEVER writes allocation (zero writers in
|
|
51
|
+
* task-runner/team-runner; its in-memory copy is dispatch-time lineage that
|
|
52
|
+
* never saw a reserve). Hence max() degenerates to "disk wins" in every
|
|
53
|
+
* broker-consistent interleaving, AND self-heals two disk-corruption shapes
|
|
54
|
+
* verbatim-disk would not: (a) a pre-fix heartbeat that already clobbered a
|
|
55
|
+
* reserve down, (b) the P2-8c refund over-subtracting a reserve that never
|
|
56
|
+
* landed. The one theoretical max() hazard — a runner snapshot taken BETWEEN
|
|
57
|
+
* a reserve and its roll-up, then rolled up with actual < reserved — cannot
|
|
58
|
+
* occur today because no task-runner path reloads the in-memory task from
|
|
59
|
+
* disk mid-execution. `tokensGranted` is admission-authoritative write-once
|
|
60
|
+
* (broker passes it through verbatim) → disk wins verbatim. A heartbeat
|
|
61
|
+
* snapshot that lacks `allocation` entirely (pre-admission copy) keeps the
|
|
62
|
+
* disk allocation instead of deleting it — same lost-update class.
|
|
63
|
+
*/
|
|
64
|
+
function mergeTaskUpdateWithDisk(diskTasks: TeamTaskState[], incoming: TeamTaskState): TeamTaskState {
|
|
65
|
+
const diskTask = diskTasks.find((t) => t.id === incoming.id);
|
|
66
|
+
if (!diskTask) return incoming;
|
|
67
|
+
|
|
68
|
+
let effective = incoming;
|
|
69
|
+
// R6-1: incoming terminal vs disk DIFFERENT terminal → disk wins for the
|
|
70
|
+
// terminal verdict (status + error).
|
|
71
|
+
if (isTerminalTaskStatus(incoming.status) && isTerminalTaskStatus(diskTask.status) && diskTask.status !== incoming.status) {
|
|
72
|
+
effective = { ...incoming, status: diskTask.status, error: diskTask.error };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// R6-4: field-wise allocation merge (see docblock above for the max()
|
|
76
|
+
// derivation). Applies on every persist (heartbeat, checkpoint, terminal
|
|
77
|
+
// settle) — all runner paths carry the same stale dispatch-time copy.
|
|
78
|
+
const incomingAllocation = incoming.allocation;
|
|
79
|
+
const diskAllocation = diskTask.allocation;
|
|
80
|
+
if (diskAllocation && !incomingAllocation) {
|
|
81
|
+
// Heartbeat snapshot predates admission — keep the broker's accounting
|
|
82
|
+
// instead of deleting it via whole-record replace.
|
|
83
|
+
effective = { ...effective, allocation: diskAllocation };
|
|
84
|
+
} else if (diskAllocation && incomingAllocation) {
|
|
85
|
+
effective = {
|
|
86
|
+
...effective,
|
|
87
|
+
allocation: {
|
|
88
|
+
tokensGranted: diskAllocation.tokensGranted,
|
|
89
|
+
tokensSpent: Math.max(diskAllocation.tokensSpent ?? 0, incomingAllocation.tokensSpent ?? 0),
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
return effective;
|
|
94
|
+
}
|
|
95
|
+
|
|
14
96
|
/**
|
|
15
97
|
* Persist a single task update using compare-and-swap under the run lock.
|
|
16
98
|
*
|
|
@@ -37,6 +119,12 @@ export function updateTask(tasks: TeamTaskState[], updated: TeamTaskState): Team
|
|
|
37
119
|
* (durability "best-effort") — tasks.json is reconstructible from the
|
|
38
120
|
* fsync'd event log, so a crash loses at most the un-flushed tail.
|
|
39
121
|
*
|
|
122
|
+
* R6-1/R6-4 (DR-03, 2026-10-16): the per-task merge is NOT an unconditional
|
|
123
|
+
* replace-by-id anymore — mergeTaskUpdateWithDisk (above) preserves a disk
|
|
124
|
+
* terminal status (vs a different incoming terminal) and merges
|
|
125
|
+
* `allocation.tokensSpent` with max(disk, incoming). The mtime CAS retry
|
|
126
|
+
* loop, the run lock, and the BUG-028 always-load-disk policy are unchanged.
|
|
127
|
+
*
|
|
40
128
|
* @param checkpointPhase - Optional checkpoint phase to include in the task state alongside the update.
|
|
41
129
|
*/
|
|
42
130
|
export function persistSingleTaskUpdate(
|
|
@@ -109,11 +197,13 @@ export function persistSingleTaskUpdate(
|
|
|
109
197
|
// stat, i.e. staleness acquired AFTER that baseline — not fallback
|
|
110
198
|
// staleness that predates it. Loading disk here makes sibling
|
|
111
199
|
// state authoritative (matching mergeUnitResult / bug-027 policy)
|
|
112
|
-
// while `updated` still wins for THIS task via updateTask
|
|
200
|
+
// while `updated` still wins for THIS task via updateTask — EXCEPT
|
|
201
|
+
// where R6-1/R6-4 (DR-03) preserve disk authority for the terminal
|
|
202
|
+
// verdict and delegate budget accounting (mergeTaskUpdateWithDisk).
|
|
113
203
|
// fallbackTasks remains the fallback when the run state is absent
|
|
114
204
|
// (fresh run, manifest not yet on disk).
|
|
115
205
|
const latest = loadRunManifestById(manifest.cwd, manifest.runId)?.tasks ?? fallbackTasks;
|
|
116
|
-
merged = updateTask(latest, taskWithCheckpoint);
|
|
206
|
+
merged = updateTask(latest, mergeTaskUpdateWithDisk(latest, taskWithCheckpoint));
|
|
117
207
|
|
|
118
208
|
// F2: collapsed from 3 redundant statSync calls into 1. The previous
|
|
119
209
|
// implementation re-checked mtime twice more after load and before
|
|
@@ -68,12 +68,23 @@ export function usageStateFromLifetime(lifetime: LifetimeUsage): UsageState {
|
|
|
68
68
|
};
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
// DR-02 C2: crash/abandon paths never run clearTrackedTaskUsage, so per-task
|
|
72
|
+
// ledger entries could accumulate for the whole process lifetime. Bounded
|
|
73
|
+
// FIFO eviction on insert — Map preserves insertion order, so deleting the
|
|
74
|
+
// first key evicts the first-INSERTED task (mirrors the sibling
|
|
75
|
+
// agentEventSeqCache cap in crew-agent-records.ts). Exported for tests.
|
|
76
|
+
export const MAX_TRACKED_TASK_USAGE = 5_000;
|
|
71
77
|
const taskUsageMap = new Map<string, LifetimeUsage>();
|
|
72
78
|
|
|
73
79
|
export function trackTaskUsage(taskId: string, delta: UsageDelta): void {
|
|
74
80
|
const existing = taskUsageMap.get(taskId) ?? emptyLifetimeUsage();
|
|
75
81
|
addUsage(existing, delta);
|
|
76
82
|
taskUsageMap.set(taskId, existing);
|
|
83
|
+
while (taskUsageMap.size > MAX_TRACKED_TASK_USAGE) {
|
|
84
|
+
const oldest = taskUsageMap.keys().next().value;
|
|
85
|
+
if (oldest === undefined) break;
|
|
86
|
+
taskUsageMap.delete(oldest);
|
|
87
|
+
}
|
|
77
88
|
}
|
|
78
89
|
|
|
79
90
|
export function getTrackedTaskUsage(taskId: string): LifetimeUsage {
|
|
@@ -33,6 +33,18 @@ export function inferGreenLevelFromTask(success: boolean, contract: Verification
|
|
|
33
33
|
return contract.allowManualEvidence ? contract.requiredGreenLevel : "targeted";
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
+
/** R4-5 (DR-02 wave 2): true when the contract REQUIRES a green level but
|
|
37
|
+
* configures zero verification commands — the default contract shape for
|
|
38
|
+
* every `verify: true` step (task-packet defaultVerificationContract).
|
|
39
|
+
* With no commands to run, satisfaction can only ever come from
|
|
40
|
+
* self-attestation (allowManualEvidence → inferGreenLevelFromTask returns
|
|
41
|
+
* the required level), which silently reports e.g. "targeted/targeted".
|
|
42
|
+
* Surface that as `decorative: true` on the evidence instead of changing
|
|
43
|
+
* the satisfied logic or the defaults (detection-first, no behavior flip). */
|
|
44
|
+
export function isDecorativeVerification(contract: VerificationContract): boolean {
|
|
45
|
+
return contract.requiredGreenLevel !== "none" && contract.commands.length === 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
36
48
|
export function createVerificationEvidence(contract: VerificationContract, success: boolean, notes: string): VerificationEvidence {
|
|
37
49
|
const observedGreenLevel = inferGreenLevelFromTask(success, contract);
|
|
38
50
|
const outcome = evaluateGreenContract(contract, {
|
|
@@ -42,6 +54,10 @@ export function createVerificationEvidence(contract: VerificationContract, succe
|
|
|
42
54
|
commands: [],
|
|
43
55
|
notes,
|
|
44
56
|
});
|
|
57
|
+
// R4-5: stamp the decorative flag so consumers (post-execution evidence
|
|
58
|
+
// builder, policy engine, UI) can tell self-attested satisfaction from
|
|
59
|
+
// command-backed satisfaction. Satisfied logic is untouched.
|
|
60
|
+
const decorative = isDecorativeVerification(contract);
|
|
45
61
|
return {
|
|
46
62
|
requiredGreenLevel: contract.requiredGreenLevel,
|
|
47
63
|
observedGreenLevel,
|
|
@@ -51,5 +67,6 @@ export function createVerificationEvidence(contract: VerificationContract, succe
|
|
|
51
67
|
status: "not_run" as const,
|
|
52
68
|
})),
|
|
53
69
|
notes,
|
|
70
|
+
...(decorative ? { decorative: true } : {}),
|
|
54
71
|
};
|
|
55
72
|
}
|
|
@@ -144,6 +144,16 @@ export const CARGO_RUST_GATES: Array<{
|
|
|
144
144
|
/**
|
|
145
145
|
* Execute a single command and capture output.
|
|
146
146
|
*/
|
|
147
|
+
/**
|
|
148
|
+
* P-4 (DR-04 round-7): accumulation cap while STREAMING child output. The
|
|
149
|
+
* final `slice(-100000)` already bounds the RESULT, but the live `output`
|
|
150
|
+
* buffer used to concatenate every chunk until close/timeout — a command
|
|
151
|
+
* emitting gigabytes held them ALL in memory (plus O(n²) string concat).
|
|
152
|
+
* Past this cap we stop concatenating and pin a "[truncated]" marker in the
|
|
153
|
+
* buffer; the final tail slice is unchanged.
|
|
154
|
+
*/
|
|
155
|
+
const OUTPUT_ACCUMULATION_CAP_CHARS = 200 * 1024;
|
|
156
|
+
const OUTPUT_TRUNCATION_MARKER = "\n[truncated]";
|
|
147
157
|
/** Characters/patterns that indicate dangerous shell metacharacters. */
|
|
148
158
|
// Round 25 (VULN-3/VULN-4): also block raw newlines (sh -c treats \n as a
|
|
149
159
|
// command separator -> injection) and bare $VARNAME references (can exfiltrate
|
|
@@ -253,8 +263,20 @@ export async function executeCommand(
|
|
|
253
263
|
|
|
254
264
|
const start = Date.now();
|
|
255
265
|
let output = "";
|
|
266
|
+
let capped = false;
|
|
256
267
|
let exitCode: number | null = null;
|
|
257
268
|
|
|
269
|
+
// P-4: single capped appender for BOTH streams — once the cap trips, later
|
|
270
|
+
// chunks are dropped (marker stays in the buffer; final slice keeps it).
|
|
271
|
+
const appendCapped = (chunk: Buffer): void => {
|
|
272
|
+
if (capped) return;
|
|
273
|
+
output += chunk.toString();
|
|
274
|
+
if (output.length > OUTPUT_ACCUMULATION_CAP_CHARS) {
|
|
275
|
+
capped = true;
|
|
276
|
+
output += OUTPUT_TRUNCATION_MARKER;
|
|
277
|
+
}
|
|
278
|
+
};
|
|
279
|
+
|
|
258
280
|
return new Promise((resolve) => {
|
|
259
281
|
const shell = spawn("sh", ["-c", command], {
|
|
260
282
|
cwd,
|
|
@@ -263,13 +285,9 @@ export async function executeCommand(
|
|
|
263
285
|
detached: true, // CORE-10: own process group so group-wide SIGKILL reaches children
|
|
264
286
|
});
|
|
265
287
|
|
|
266
|
-
shell.stdout?.on("data",
|
|
267
|
-
output += data.toString();
|
|
268
|
-
});
|
|
288
|
+
shell.stdout?.on("data", appendCapped);
|
|
269
289
|
|
|
270
|
-
shell.stderr?.on("data",
|
|
271
|
-
output += data.toString();
|
|
272
|
-
});
|
|
290
|
+
shell.stderr?.on("data", appendCapped);
|
|
273
291
|
|
|
274
292
|
// M-9 fix (code-review 2026-06-23): store the timeout handle so it can be
|
|
275
293
|
// cleared when the process exits normally, and unref it so it does not
|