pi-crew 0.11.24 → 0.11.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.md +2 -2
- package/agents/designer.md +23 -0
- package/agents/orchestrator.md +17 -0
- package/agents/verifier.md +1 -1
- package/dist/index.mjs +568 -559
- package/{workflows/chain.workflow.md → docs/chain-workflow.md} +8 -0
- package/docs/dynamic-workflows.md +23 -0
- package/docs/publishing.md +34 -5
- package/docs/troubleshooting.md +50 -2
- package/docs/usage.md +7 -2
- package/install.mjs +2 -1
- package/package.json +4 -2
- package/skills/read-only-explorer/SKILL.md +15 -5
- package/skills/real-test-pi-crew/SKILL.md +7 -7
- package/src/config/config-validation.ts +58 -0
- package/src/config/config.ts +90 -5
- package/src/config/env-vars.ts +14 -7
- package/src/config/role-tools.ts +7 -5
- package/src/config/types.ts +4 -1
- package/src/extension/crew-cleanup.ts +6 -0
- package/src/extension/project-init.ts +19 -1
- package/src/extension/register.ts +68 -30
- package/src/extension/registration/artifact-cleanup.ts +52 -11
- package/src/extension/registration/commands/manage.ts +9 -2
- package/src/extension/registration/hook-registration.ts +3 -0
- package/src/extension/registration/lazy-configurers.ts +2 -0
- package/src/extension/registration/lifecycle-handlers.ts +79 -18
- package/src/extension/registration/render-loop.ts +27 -2
- package/src/extension/registration/tool-loop-guard.ts +5 -3
- package/src/extension/registration/verify-gate-tool.ts +7 -3
- package/src/extension/team-tool/api/read.ts +12 -3
- package/src/extension/team-tool/cancel.ts +11 -0
- package/src/extension/team-tool/config-patch.ts +56 -11
- package/src/extension/team-tool/destructive-gate.ts +72 -12
- package/src/extension/team-tool/dispatch/manage.ts +9 -1
- package/src/extension/team-tool/handle-schedule.ts +38 -4
- package/src/extension/team-tool/handle-settings.ts +76 -6
- package/src/extension/team-tool/health-monitor.ts +14 -2
- package/src/extension/team-tool/lifecycle-actions.ts +28 -10
- package/src/extension/team-tool/respond.ts +12 -5
- package/src/extension/team-tool/run-deadline.ts +29 -2
- package/src/extension/team-tool/run.ts +11 -0
- package/src/extension/team-tool/status.ts +64 -8
- package/src/extension/team-tool/workflow-manage.ts +5 -1
- package/src/extension/team-tool.ts +3 -0
- package/src/i18n.ts +18 -14
- package/src/prompt/inbox-poll.ts +14 -2
- package/src/prompt/message-tool.ts +1 -1
- package/src/prompt/prompt-runtime.ts +25 -3
- package/src/prompt/scratchpad-lifecycle.ts +14 -13
- package/src/prompt/surface-worker.ts +35 -9
- package/src/runtime/broker/crew-broker.ts +113 -111
- package/src/runtime/broker/delegate/delegate-abort.ts +118 -0
- package/src/runtime/broker/delegate/delegate-event.ts +3 -1
- package/src/runtime/broker/delegate/delegate-finalize.ts +179 -0
- package/src/runtime/broker/protocol/wait-resolve.ts +71 -5
- package/src/runtime/budget-enforcement.ts +26 -6
- package/src/runtime/chain-parser.ts +5 -3
- package/src/runtime/chain-runner.ts +41 -22
- package/src/runtime/child-pi/mock-fixtures.ts +12 -5
- package/src/runtime/compaction/tool-output-pruner.ts +8 -3
- package/src/runtime/delegate-spawn.ts +61 -8
- package/src/runtime/dispatch-batch.ts +10 -4
- package/src/runtime/effectiveness.ts +31 -3
- package/src/runtime/goal-workflow/plan-templates.ts +2 -2
- package/src/runtime/live-session/live-agent-control.ts +89 -6
- package/src/runtime/live-session/live-agent-manager.ts +94 -10
- package/src/runtime/live-session/live-session-runtime.ts +9 -2
- package/src/runtime/model/model-fallback.ts +6 -9
- package/src/runtime/model/pi-args.ts +3 -3
- package/src/runtime/model/runtime-policy.ts +4 -2
- package/src/runtime/model/runtime-resolver.ts +6 -2
- package/src/runtime/process/orphan-reaper.ts +143 -0
- package/src/runtime/process-status.ts +38 -16
- package/src/runtime/resilient-edit.ts +170 -93
- package/src/runtime/role-permission.ts +3 -1
- package/src/runtime/run-tracker.ts +9 -1
- package/src/runtime/sanitize/worker-text.ts +98 -0
- package/src/runtime/scheduling/run-coalesced-task-group.ts +82 -28
- package/src/runtime/spawn-policy.ts +190 -8
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/subagent-manager.ts +29 -0
- package/src/runtime/surface/herdr-provider.ts +31 -1
- package/src/runtime/task-output-context.ts +58 -11
- package/src/runtime/task-runner/post-execution.ts +150 -0
- package/src/runtime/task-runner/prompt-builder.ts +41 -8
- package/src/runtime/task-runner.ts +16 -3
- package/src/runtime/verification/classifier-pre-gate.ts +27 -6
- package/src/runtime/verification/verify-gate.ts +153 -30
- package/src/state/atomic-write.ts +73 -4
- package/src/state/contracts.ts +7 -0
- package/src/state/coordination/mailbox.ts +40 -11
- package/src/state/event-log/event-log-rotation.ts +28 -6
- package/src/state/event-log/poison-marker.ts +16 -1
- package/src/state/stores/artifact-store.ts +181 -6
- package/src/state/stores/manifest-io.ts +126 -8
- package/src/state/stores/state-store.ts +30 -31
- package/src/state/types.ts +13 -0
- package/src/state/usage.ts +41 -0
- package/src/ui/run-dashboard.ts +3 -3
- package/src/ui/settings-overlay.ts +19 -1
- package/src/utils/internal-error.ts +5 -1
- package/src/utils/timer-limits.ts +30 -0
- package/src/utils/token-counter.ts +47 -0
- package/src/workflows/discover-workflows.ts +9 -1
- package/src/worktree/worktree-manager.ts +128 -18
- package/workflows/default.workflow.md +1 -1
- package/workflows/fast-fix.workflow.md +1 -1
- package/workflows/plan-execute.workflow.md +1 -1
- package/workflows/review.workflow.md +1 -1
- package/workflows/strict-fast-fix.workflow.md +1 -1
|
@@ -40,6 +40,20 @@
|
|
|
40
40
|
* never changes the FAILED↔spawn logic; flag off keeps the gate
|
|
41
41
|
* bit-for-bit identical to pre-U13-ADJ.
|
|
42
42
|
*
|
|
43
|
+
* GROUND TRUTH (M1-7, R47-1 fix 2026-10-19): the gate NEVER derives checks
|
|
44
|
+
* from the task cwd's package.json at gate time — a worker-mutable file
|
|
45
|
+
* (shared-cwd mode) whose tampered `scripts.typecheck: "exit 0"` used to
|
|
46
|
+
* yield PASS + skipVerifier (worker self-grading). Ground truth is
|
|
47
|
+
* host-owned: `resolveHostVerifyGateSpec(cwd)` snapshots the project
|
|
48
|
+
* manifests + derives the check preset AT DISPATCH (host process, before
|
|
49
|
+
* any worker runs) and the spec is pinned on the run manifest; the spawn
|
|
50
|
+
* site passes it as `hostSpec`. The gate re-hashes the manifests BEFORE any
|
|
51
|
+
* command runs (drift ⇒ FAILED — tamper evidence, verifier spawns) and
|
|
52
|
+
* AFTER a PASS (drift ⇒ INCONCLUSIVE — mid-gate tamper), mirroring the P1a
|
|
53
|
+
* bookend (verification-integrity.ts). Absence of a spec is fail-closed:
|
|
54
|
+
* no spec + no explicit checks ⇒ zero checks ⇒ INCONCLUSIVE ⇒ the LLM
|
|
55
|
+
* verifier spawns — absence can never produce a skip.
|
|
56
|
+
*
|
|
43
57
|
* SAFETY: a gate that cannot run (disabled by env, no checks resolved,
|
|
44
58
|
* executor failure) degrades to "spawn the LLM verifier as today" — the
|
|
45
59
|
* pre-gate can only SAVE a spawn on positive deterministic evidence, never
|
|
@@ -56,8 +70,11 @@ import * as fs from "node:fs";
|
|
|
56
70
|
import { getCrewEnvBool, getCrewEnvInt } from "../../config/env-vars.ts";
|
|
57
71
|
import type { MetricRegistry } from "../../observability/metric-registry.ts";
|
|
58
72
|
import type { VerifierPreGateResult } from "./classifier-pre-gate.ts";
|
|
59
|
-
import { classifyVerifierPreGate } from "./classifier-pre-gate.ts";
|
|
60
73
|
import { executeCommand } from "./verification-gates.ts";
|
|
74
|
+
// Static edge is graph-free: verification-integrity.ts is already in the
|
|
75
|
+
// index.bundle value-import graph (goal.ts/goal-wrap.ts/goal-loop-runner.ts
|
|
76
|
+
// import it), so this adds no bundle-graph file (check-lazy-imports budget).
|
|
77
|
+
import { compareSnapshot, snapshotManifests } from "./verification-integrity.ts";
|
|
61
78
|
|
|
62
79
|
/** One deterministic check to run. Mirrors the phase-gate shape
|
|
63
80
|
* (verification-gates.ts) but kept local so the tool schema can reuse it. */
|
|
@@ -97,6 +114,10 @@ export interface VerifyGateOutcome {
|
|
|
97
114
|
* detected flakiness or a rerun executor failure — the note is also
|
|
98
115
|
* appended to evidenceLines and shapes the rendered context trailer. */
|
|
99
116
|
determinismNote?: string;
|
|
117
|
+
/** M1-7 (R47-1): set when the dispatch-pinned manifest snapshot DRIFTED —
|
|
118
|
+
* the note is also appended to evidenceLines and switches the rendered
|
|
119
|
+
* context trailer to "results UNTRUSTED". */
|
|
120
|
+
integrityNote?: string;
|
|
100
121
|
}
|
|
101
122
|
|
|
102
123
|
/** Injected command executor. The TOOL path wraps ctx.executeTool("bash");
|
|
@@ -147,8 +168,8 @@ export function resolveVerifyDeterminismEnabled(explicit?: boolean): boolean {
|
|
|
147
168
|
}
|
|
148
169
|
|
|
149
170
|
/**
|
|
150
|
-
* Default check preset — the spec's "check cấu hình" set, resolved from
|
|
151
|
-
*
|
|
171
|
+
* Default check preset — the spec's "check cấu hình" set, resolved from a
|
|
172
|
+
* package.json's scripts:
|
|
152
173
|
* - `typecheck` script present → `npm run --silent typecheck` (critical);
|
|
153
174
|
* - `test:critical` script present → `npm run --silent test:critical`
|
|
154
175
|
* (critical);
|
|
@@ -157,6 +178,11 @@ export function resolveVerifyDeterminismEnabled(explicit?: boolean): boolean {
|
|
|
157
178
|
* `grep -c "<marker>" dist/index.mjs`) rather than defaulted.
|
|
158
179
|
* Repos without either script resolve ZERO checks → INCONCLUSIVE → the
|
|
159
180
|
* classifier layer (when enabled) or the LLM verifier spawn runs as today.
|
|
181
|
+
*
|
|
182
|
+
* HOST-SIDE ONLY (M1-7, R47-1): this reads a worker-MUTABLE file. It may run
|
|
183
|
+
* only in the host process at DISPATCH time (via resolveHostVerifyGateSpec,
|
|
184
|
+
* before any worker exists) — never as a gate-time fallback from a task cwd.
|
|
185
|
+
* Gate-time consumers must use the dispatch-pinned hostSpec instead.
|
|
160
186
|
*/
|
|
161
187
|
export function defaultVerifyGateChecks(dir: string): VerifyGateCheck[] {
|
|
162
188
|
let scripts: Record<string, unknown> = {};
|
|
@@ -177,6 +203,39 @@ export function defaultVerifyGateChecks(dir: string): VerifyGateCheck[] {
|
|
|
177
203
|
return checks;
|
|
178
204
|
}
|
|
179
205
|
|
|
206
|
+
/** Host-owned verify-gate spec (M1-7, R47-1 fix): captured ONCE at dispatch
|
|
207
|
+
* in the HOST process (pre-worker) and pinned on the run manifest.
|
|
208
|
+
* `snapshot` is the P1a manifest-hash set (verification-integrity.ts);
|
|
209
|
+
* `checks` is the preset derived from the host's own read of package.json.
|
|
210
|
+
* At gate time the task cwd is verified against `snapshot` instead of ever
|
|
211
|
+
* re-reading the worker-mutable package.json. Structural shape mirrors
|
|
212
|
+
* TeamRunManifest's `verifyGateSpec` field (the state layer must not import
|
|
213
|
+
* runtime types). */
|
|
214
|
+
export interface VerifyGateHostSpec {
|
|
215
|
+
snapshot: Record<string, string>;
|
|
216
|
+
checks: VerifyGateCheck[];
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/** HOST-SIDE (dispatch) — build the verify spec for `cwd` (the project root
|
|
220
|
+
* as the host sees it BEFORE any worker runs). Returns undefined when the
|
|
221
|
+
* directory has neither manifests to hash nor a check preset (nothing to
|
|
222
|
+
* protect or verify); the gate then resolves no checks (fail-closed). */
|
|
223
|
+
export function resolveHostVerifyGateSpec(cwd: string): VerifyGateHostSpec | undefined {
|
|
224
|
+
const snapshot = snapshotManifests(cwd);
|
|
225
|
+
const checks = defaultVerifyGateChecks(cwd);
|
|
226
|
+
if (Object.keys(snapshot).length === 0 && checks.length === 0) return undefined;
|
|
227
|
+
return { snapshot, checks };
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** Gate-time integrity probe (M1-7): re-hash the manifests in `cwd` and diff
|
|
231
|
+
* against the dispatch snapshot. Non-empty result = the drifted relative
|
|
232
|
+
* paths — the task cwd's ground truth no longer matches what the host
|
|
233
|
+
* pinned at dispatch (worker tampering or legitimate churn; both fail
|
|
234
|
+
* closed: the gate must not grade on compromised ground truth). */
|
|
235
|
+
export function detectHostSpecDrift(cwd: string, spec: VerifyGateHostSpec): string[] {
|
|
236
|
+
return compareSnapshot(spec.snapshot, snapshotManifests(cwd));
|
|
237
|
+
}
|
|
238
|
+
|
|
180
239
|
function tailOutput(output: string): string {
|
|
181
240
|
if (output.length <= OUTPUT_TAIL_MAX_CHARS) return output;
|
|
182
241
|
return `…${output.slice(-OUTPUT_TAIL_MAX_CHARS)}`;
|
|
@@ -414,11 +473,13 @@ export function renderVerifyGateContext(outcome: VerifyGateOutcome): string {
|
|
|
414
473
|
// the original wording bit-for-bit; a flaky rerun (layer 3) gets its own
|
|
415
474
|
// trailer because "do NOT re-run / ground truth" would be provably wrong.
|
|
416
475
|
const executorDeath = outcome.checks.some((c) => !c.skipped && c.exitCode === null);
|
|
417
|
-
const trailer =
|
|
418
|
-
? "
|
|
419
|
-
:
|
|
420
|
-
? "
|
|
421
|
-
:
|
|
476
|
+
const trailer = outcome.integrityNote
|
|
477
|
+
? "Manifest integrity check FAILED: project manifest files changed since the host snapshot taken at dispatch. These gate results and any transcript claims about build/test state are UNTRUSTED — inspect what changed and re-run the verification commands yourself before concluding."
|
|
478
|
+
: executorDeath
|
|
479
|
+
? "A verification command could not run (executor error — exit=err). Treat the build/test state as UNKNOWN; run the build/tests yourself before concluding."
|
|
480
|
+
: outcome.determinismNote
|
|
481
|
+
? "The gate rerun detected NONDETERMINISM (a check passed once and failed on rerun). These gate results are NOT reliable — re-run the affected command yourself and treat the build/test state as UNKNOWN."
|
|
482
|
+
: "The deterministic gate already ran these commands — do NOT re-run them; verify the WORK (diff, files, acceptance), using these results as ground truth for build/test state.";
|
|
422
483
|
lines.push("", trailer);
|
|
423
484
|
return lines.join("\n");
|
|
424
485
|
}
|
|
@@ -430,11 +491,23 @@ export function renderVerifyGateSummary(outcome: VerifyGateOutcome, reason: stri
|
|
|
430
491
|
|
|
431
492
|
export interface VerifierGatePreGateInput {
|
|
432
493
|
cwd: string;
|
|
433
|
-
/** Explicit checks
|
|
494
|
+
/** Explicit checks take precedence over every other source (tool path /
|
|
495
|
+
* tests). Without them the gate derives NO checks on its own. */
|
|
434
496
|
checks?: VerifyGateCheck[];
|
|
497
|
+
/** M1-7 (R47-1): host-owned spec pinned on the run manifest at dispatch.
|
|
498
|
+
* When provided, the gate verifies manifest integrity against the
|
|
499
|
+
* snapshot BEFORE running anything (drift ⇒ FAILED — tamper evidence,
|
|
500
|
+
* verifier spawns) and re-verifies after a PASS (drift ⇒ INCONCLUSIVE).
|
|
501
|
+
* When ABSENT there is NO fallback read of the worker-mutable task-cwd
|
|
502
|
+
* package.json: no explicit checks ⇒ zero checks ⇒ INCONCLUSIVE
|
|
503
|
+
* (fail-closed on absence — the LLM verifier spawns). */
|
|
504
|
+
hostSpec?: VerifyGateHostSpec;
|
|
435
505
|
signal?: AbortSignal;
|
|
436
506
|
timeoutMs?: number;
|
|
437
|
-
/**
|
|
507
|
+
/** @deprecated M3-28 (R47-V2): NEVER consulted — the gate's structured verdict
|
|
508
|
+
* can never be pass-shaped, so the U6B classifier could only ever escalate
|
|
509
|
+
* (see the call-site note in {@link runVerifierGatePreGate}). Kept only so the
|
|
510
|
+
* task-runner dispatch call site still type-checks; delete together with it. */
|
|
438
511
|
classifier?: {
|
|
439
512
|
enabled: boolean;
|
|
440
513
|
modelRegistry: unknown;
|
|
@@ -473,7 +546,42 @@ function incMetric(registry: MetricRegistry | undefined, name: string, help: str
|
|
|
473
546
|
* to INCONCLUSIVE checks, which escalate to the LLM verifier.
|
|
474
547
|
*/
|
|
475
548
|
export async function runVerifierGatePreGate(input: VerifierGatePreGateInput): Promise<VerifierGatePreGateResult> {
|
|
476
|
-
|
|
549
|
+
// ── M1-7 (R47-1): integrity BEFORE any command runs ────────────────────
|
|
550
|
+
// A drifted manifest means the ground truth the host pinned at dispatch is
|
|
551
|
+
// compromised — running ANY check now would be the self-grading exploit
|
|
552
|
+
// (the worker rewrote the scripts the gate was about to run). Fail FAILED
|
|
553
|
+
// (tamper evidence) → the verifier spawns with the drift in its prompt.
|
|
554
|
+
// A tampered package.json can never yield PASS + skipVerifier.
|
|
555
|
+
if (input.hostSpec) {
|
|
556
|
+
const drift = detectHostSpecDrift(input.cwd, input.hostSpec);
|
|
557
|
+
if (drift.length > 0) {
|
|
558
|
+
const note = `manifest-integrity: ${drift.join(", ")} changed since dispatch (host snapshot)`;
|
|
559
|
+
const outcome: VerifyGateOutcome = {
|
|
560
|
+
verdict: "FAILED",
|
|
561
|
+
checks: [],
|
|
562
|
+
evidenceLines: [note],
|
|
563
|
+
allPassed: false,
|
|
564
|
+
totalDurationMs: 0,
|
|
565
|
+
integrityNote: note,
|
|
566
|
+
};
|
|
567
|
+
const decision = resolveVerifierSpawnDecision(outcome);
|
|
568
|
+
incMetric(input.metricRegistry, "crew.verification.verify_gate_runs_total", "Deterministic verify_gate executions by verdict", {
|
|
569
|
+
verdict: outcome.verdict,
|
|
570
|
+
});
|
|
571
|
+
incMetric(
|
|
572
|
+
input.metricRegistry,
|
|
573
|
+
"crew.verification.verifier_spawns_total",
|
|
574
|
+
"LLM verifier spawns that still ran, labeled by the deciding layer",
|
|
575
|
+
{ reason: decision.reason },
|
|
576
|
+
);
|
|
577
|
+
return { outcome, decision, promptContext: renderVerifyGateContext(outcome) };
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
// ── M1-7: NO package.json fallback ─────────────────────────────────────
|
|
581
|
+
// Checks come from the caller or the dispatch-pinned host spec ONLY —
|
|
582
|
+
// nothing here reads the worker-mutable task cwd. Zero resolvable checks
|
|
583
|
+
// ⇒ INCONCLUSIVE ⇒ escalate (fail-closed on absence, pre-R47-1 doctrine).
|
|
584
|
+
const checks = input.checks ?? input.hostSpec?.checks ?? [];
|
|
477
585
|
const exec = createSubprocessVerifyGateExecutor();
|
|
478
586
|
let outcome = await runVerifyGate(checks, exec, {
|
|
479
587
|
cwd: input.cwd,
|
|
@@ -501,27 +609,42 @@ export async function runVerifierGatePreGate(input: VerifierGatePreGateInput): P
|
|
|
501
609
|
{ result: layer.result },
|
|
502
610
|
);
|
|
503
611
|
}
|
|
504
|
-
|
|
505
|
-
if
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
});
|
|
521
|
-
} catch {
|
|
522
|
-
preGate = undefined; // soft: escalate to the LLM verifier
|
|
612
|
+
// ── M1-7 bookend: integrity AFTER a green gate (mirrors P1a) ───────────
|
|
613
|
+
// A PASS is only trustworthy if the manifests are STILL identical to the
|
|
614
|
+
// dispatch snapshot after the commands ran — a mid-gate tamper downgrades
|
|
615
|
+
// the verdict to INCONCLUSIVE so the LLM verifier spawns. This can only
|
|
616
|
+
// ever DOWNGRADE, never upgrade (the pre-check above already ruled out
|
|
617
|
+
// pre-existing drift, so any drift now appeared during the run).
|
|
618
|
+
if (input.hostSpec && outcome.verdict === "PASS") {
|
|
619
|
+
const drift = detectHostSpecDrift(input.cwd, input.hostSpec);
|
|
620
|
+
if (drift.length > 0) {
|
|
621
|
+
const note = `manifest-integrity: ${drift.join(", ")} changed during the gate run (host snapshot)`;
|
|
622
|
+
outcome = {
|
|
623
|
+
...outcome,
|
|
624
|
+
verdict: "INCONCLUSIVE",
|
|
625
|
+
evidenceLines: [...outcome.evidenceLines, note],
|
|
626
|
+
integrityNote: note,
|
|
627
|
+
};
|
|
523
628
|
}
|
|
524
629
|
}
|
|
630
|
+
// ── M3-28 (R47-V2, fix-wave 6): the U6B classifier layer is NOT consulted ──
|
|
631
|
+
// The consult used to live here, guarded by `outcome.verdict === "INCONCLUSIVE"`
|
|
632
|
+
// and fed the literal gate verdict. That input can never be pass-shaped:
|
|
633
|
+
// `classifyVerifierPreGate` blocks only when `isPassShapedVerdict(...)` is true
|
|
634
|
+
// (classifier-pre-gate.ts — PASS vocabulary AND no failure vocabulary), and
|
|
635
|
+
// "INCONCLUSIVE" matches the failure vocabulary (…conclusive → no PASS match
|
|
636
|
+
// at all), so `action` was ALWAYS escalate_verdict_failed ⇒ blockVerifier
|
|
637
|
+
// false ⇒ the same spawn decision the no-classifier branch yields. Net effect
|
|
638
|
+
// of the consult: one model call, one task.verifier_pre_gate event and one
|
|
639
|
+
// decisions_total counter for a decision that could never differ.
|
|
640
|
+
// Wiring it "for real" is not possible from here: a state that says
|
|
641
|
+
// INCONCLUSIVE cannot be conclusive evidence of a pass, so there is nothing
|
|
642
|
+
// for the classifier to legitimately block on. Removed (spend without a
|
|
643
|
+
// decision, plus a misleading `classifier_*` reason label).
|
|
644
|
+
// `input.classifier` stays on the interface (deprecated) because the dispatch
|
|
645
|
+
// call site in task-runner.ts passes it; removing the field is a follow-up
|
|
646
|
+
// that must land with that call site.
|
|
647
|
+
const preGate: VerifierPreGateResult | undefined = undefined;
|
|
525
648
|
const decision = resolveVerifierSpawnDecision(outcome, preGate);
|
|
526
649
|
incMetric(input.metricRegistry, "crew.verification.verify_gate_runs_total", "Deterministic verify_gate executions by verdict", {
|
|
527
650
|
verdict: outcome.verdict,
|
|
@@ -990,6 +990,62 @@ interface CoalescedAtomicWrite {
|
|
|
990
990
|
}
|
|
991
991
|
const MAX_FLUSH_RETRIES = 5;
|
|
992
992
|
const pendingAtomicWrites = new Map<string, CoalescedAtomicWrite>();
|
|
993
|
+
|
|
994
|
+
/**
|
|
995
|
+
* R39-M1 (fix-wave 6, EP3): a coalesced write that exhausted MAX_FLUSH_RETRIES.
|
|
996
|
+
* Previously the flush RE-THREW from a `setTimeout` callback, which surfaced as
|
|
997
|
+
* an uncaughtException and killed the whole host process (exit 7) on a transient
|
|
998
|
+
* EACCES/EROFS/ENOSPC/EBUSY write fault — leaving the run stuck "running"
|
|
999
|
+
* forever. The write is now DROPPED and reported instead.
|
|
1000
|
+
*/
|
|
1001
|
+
export interface CoalescedFlushFailure {
|
|
1002
|
+
filePath: string;
|
|
1003
|
+
error: string;
|
|
1004
|
+
retries: number;
|
|
1005
|
+
at: number;
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
const exhaustedFlushFailures: CoalescedFlushFailure[] = [];
|
|
1009
|
+
const EXHAUSTED_FLUSH_FAILURES_MAX = 64;
|
|
1010
|
+
type CoalescedFlushExhaustedListener = (failure: CoalescedFlushFailure) => void;
|
|
1011
|
+
const coalescedFlushExhaustedListeners = new Set<CoalescedFlushExhaustedListener>();
|
|
1012
|
+
|
|
1013
|
+
/**
|
|
1014
|
+
* R39-M1: subscribe to coalesced-flush retry exhaustion (the degradation seam
|
|
1015
|
+
* a caller can use to mark the owning run degraded). Returns an unsubscribe fn.
|
|
1016
|
+
*/
|
|
1017
|
+
export function onCoalescedFlushExhausted(listener: CoalescedFlushExhaustedListener): () => void {
|
|
1018
|
+
coalescedFlushExhaustedListeners.add(listener);
|
|
1019
|
+
return () => coalescedFlushExhaustedListeners.delete(listener);
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
/** R39-M1: recently exhausted coalesced flushes (best-effort observability). */
|
|
1023
|
+
export function listExhaustedFlushFailures(): readonly CoalescedFlushFailure[] {
|
|
1024
|
+
return exhaustedFlushFailures;
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
/** @internal test hook */
|
|
1028
|
+
export function __test__clearExhaustedFlushFailures(): void {
|
|
1029
|
+
exhaustedFlushFailures.length = 0;
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
function recordExhaustedFlush(filePath: string, error: unknown, retries: number): void {
|
|
1033
|
+
const failure: CoalescedFlushFailure = {
|
|
1034
|
+
filePath,
|
|
1035
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1036
|
+
retries,
|
|
1037
|
+
at: Date.now(),
|
|
1038
|
+
};
|
|
1039
|
+
exhaustedFlushFailures.push(failure);
|
|
1040
|
+
if (exhaustedFlushFailures.length > EXHAUSTED_FLUSH_FAILURES_MAX) exhaustedFlushFailures.shift();
|
|
1041
|
+
for (const listener of coalescedFlushExhaustedListeners) {
|
|
1042
|
+
try {
|
|
1043
|
+
listener(failure);
|
|
1044
|
+
} catch (listenerError) {
|
|
1045
|
+
logInternalError("atomic-write.coalesced-flush-listener", listenerError, filePath, "warn");
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
993
1049
|
const DEFAULT_ATOMIC_COALESCE_MS = 50;
|
|
994
1050
|
/** Issue 2 fix: generation counter for coalesced writes */
|
|
995
1051
|
let writeGeneration = 0;
|
|
@@ -1093,10 +1149,22 @@ function flushOnePendingAtomicWrite(filePath: string): void {
|
|
|
1093
1149
|
const entryToRetry = current ?? entry;
|
|
1094
1150
|
entryToRetry.retryCount++;
|
|
1095
1151
|
if (entryToRetry.retryCount >= MAX_FLUSH_RETRIES) {
|
|
1096
|
-
//
|
|
1152
|
+
// R39-M1 (fix-wave 6, EP3): MAX retries exhausted — DEGRADE, never
|
|
1153
|
+
// re-throw. This code runs inside a setTimeout callback (coalesce +
|
|
1154
|
+
// backoff timers) as well as inside flushPendingAtomicWrites(); a
|
|
1155
|
+
// throw from the timer context surfaces as an uncaughtException and
|
|
1156
|
+
// killed the WHOLE host process (exit 7) on a transient write fault,
|
|
1157
|
+
// leaving the run "running" forever. Drop the buffered entry, record +
|
|
1158
|
+
// report the degradation, and keep the host alive.
|
|
1097
1159
|
pendingAtomicWrites.delete(filePath);
|
|
1098
|
-
|
|
1099
|
-
|
|
1160
|
+
recordExhaustedFlush(filePath, error, entryToRetry.retryCount);
|
|
1161
|
+
logInternalError(
|
|
1162
|
+
"atomic-write.coalesced-flush",
|
|
1163
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
1164
|
+
`${filePath} — dropped after ${entryToRetry.retryCount} retries (degraded, host stays alive)`,
|
|
1165
|
+
"error",
|
|
1166
|
+
);
|
|
1167
|
+
return;
|
|
1100
1168
|
}
|
|
1101
1169
|
// Exponential backoff: base delay * 2^(retryCount-1), capped at 30 seconds
|
|
1102
1170
|
const backoffMs = Math.min(30000, entryToRetry.coalesceMs * 2 ** (entryToRetry.retryCount - 1));
|
|
@@ -1200,7 +1268,8 @@ export function peekPendingCoalescedWrite<T>(filePath: string): T | undefined {
|
|
|
1200
1268
|
* flushed file keeps its data fsync but defers the dir fsync, and after the
|
|
1201
1269
|
* serial loop completes (all renames done → R16-B1 ordering) ONE fsync is
|
|
1202
1270
|
* issued per distinct pending dir. The trailing fsync lives in the `finally`
|
|
1203
|
-
* so a mid-drain throw (a
|
|
1271
|
+
* so a mid-drain throw (e.g. a reporting-hook failure, or any other error the
|
|
1272
|
+
* flush's catch block re-raises) still
|
|
1204
1273
|
* makes the already-renamed files crash-durable. Scoped flushes and
|
|
1205
1274
|
* coalesce-timer flushes keep the immediate per-file dir fsync.
|
|
1206
1275
|
*/
|
package/src/state/contracts.ts
CHANGED
|
@@ -151,6 +151,13 @@ export const TEAM_EVENT_TYPES = [
|
|
|
151
151
|
"delegate.completed",
|
|
152
152
|
"delegate.timed_out",
|
|
153
153
|
"delegate.rolled_up",
|
|
154
|
+
// M3-5/R43-N3 (2026-10-19 fix-wave 6): a run/owner abort reached an
|
|
155
|
+
// in-flight grandchild (broker delegate abort registry → spawner signal).
|
|
156
|
+
"delegate.cancelled",
|
|
157
|
+
// M3-5/R43-N4: the parent task was already terminal when the grandchild
|
|
158
|
+
// settled, so the fenced result was redirected to the RUN inbox instead of
|
|
159
|
+
// an unreadable parent mailbox (delegate/delegate-finalize.ts).
|
|
160
|
+
"delegate.rerouted",
|
|
154
161
|
// Budget tracking events
|
|
155
162
|
"budget.initialized",
|
|
156
163
|
"budget.warning",
|
|
@@ -96,12 +96,14 @@ export interface MailboxMessage {
|
|
|
96
96
|
export interface MailboxDeliveryState {
|
|
97
97
|
messages: Record<string, MailboxMessageStatus>;
|
|
98
98
|
updatedAt: string;
|
|
99
|
-
/** U5 (2026-10-10): durable
|
|
100
|
-
* key `${direction}:${requestId}` → id of the
|
|
101
|
-
* consumed under it
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
99
|
+
/** U5 (2026-10-10): durable delivered-requestId index —
|
|
100
|
+
* key `${direction}:${recipientTaskId}:${requestId}` → id of the message row
|
|
101
|
+
* that recipient consumed under it (M3-9/R33-1 recipient scoping: one group
|
|
102
|
+
* fan-out reuses a single requestId for every recipient, so a run-scoped key
|
|
103
|
+
* let the first acker starve all siblings — probe 1/0/0). Written by the
|
|
104
|
+
* worker inbox pickup (recordMailboxRequestIdDelivery) and read back after a
|
|
105
|
+
* restart so a redelivered/resent row is dropped (exactly-once pickup).
|
|
106
|
+
* Optional so pre-U5 delivery.json files parse unchanged. */
|
|
105
107
|
requestIds?: Record<string, string>;
|
|
106
108
|
}
|
|
107
109
|
|
|
@@ -738,8 +740,8 @@ function pruneDeliveryMessages(manifest: TeamRunManifest, state: MailboxDelivery
|
|
|
738
740
|
state.messages = Object.fromEntries(remaining.filter(([id]) => keptIds.has(id)));
|
|
739
741
|
}
|
|
740
742
|
|
|
741
|
-
/** U5: bound the durable (direction, requestId) delivered index.
|
|
742
|
-
* chronologically (first delivery under that
|
|
743
|
+
/** U5: bound the durable (direction, recipient, requestId) delivered index.
|
|
744
|
+
* Keys insert chronologically (first delivery under that key), so eviction keeps
|
|
743
745
|
* the most recent MAX_DELIVERY_MESSAGES keys — dropping a key can only
|
|
744
746
|
* re-enable delivery of an extremely old duplicate row, the same bounded
|
|
745
747
|
* risk the messages map already accepts. */
|
|
@@ -1107,22 +1109,49 @@ export async function appendMailboxMessageIdempotentAsync(
|
|
|
1107
1109
|
/** U5: durable receiver-side dedup record — the worker inbox pickup marks the
|
|
1108
1110
|
* messages it surfaced as consumed: `messages[id] = "acknowledged"` (the F09
|
|
1109
1111
|
* durable record; the inbox line itself never flips status) plus the
|
|
1110
|
-
* `${direction}:${requestId}` index. Both survive
|
|
1111
|
-
* (fresh process, empty seen-set) still drops them. */
|
|
1112
|
+
* recipient-scoped `${direction}:${taskId}:${requestId}` index. Both survive
|
|
1113
|
+
* restart, so a later poll (fresh process, empty seen-set) still drops them. */
|
|
1112
1114
|
export interface MailboxRequestIdDeliveryRecord {
|
|
1113
1115
|
direction: MailboxDirection;
|
|
1114
1116
|
messageId: string;
|
|
1117
|
+
/** The RECIPIENT mailbox this pickup consumed (the polling task).
|
|
1118
|
+
* M3-9 / R33-1: part of the index key — see {@link requestIdDeliveryKey}.
|
|
1119
|
+
* Undefined = the run-level mailbox. */
|
|
1120
|
+
taskId?: string;
|
|
1115
1121
|
/** Optional: rows without a requestId (pre-U5 legacy) only get the
|
|
1116
1122
|
* by-id acknowledged record. */
|
|
1117
1123
|
requestId?: string;
|
|
1118
1124
|
}
|
|
1119
1125
|
|
|
1126
|
+
/**
|
|
1127
|
+
* M3-9 / R33-1 — THE delivery-index key: `${direction}:${recipient}:${requestId}`.
|
|
1128
|
+
*
|
|
1129
|
+
* Round 33 probe (fan-out delivered 1/0/0): a group send mints ONE requestId
|
|
1130
|
+
* (crew-broker handleMsgSend) and reuses it for every recipient. The pickup
|
|
1131
|
+
* index was run-scoped (`${direction}:${requestId}`), so the FIRST recipient to
|
|
1132
|
+
* ack wrote the key and every SIBLING recipient was then starved at its own
|
|
1133
|
+
* pickup — one delivered, the rest silently dropped (and the
|
|
1134
|
+
* `mailbox.pickup_deduped` audit logged it as an expected duplicate).
|
|
1135
|
+
* Recipient scoping restores per-recipient exactly-once: each recipient's
|
|
1136
|
+
* mailbox gets its own entry for the same requestId, so all N copies deliver
|
|
1137
|
+
* (once each).
|
|
1138
|
+
*
|
|
1139
|
+
* Migration: keys written before this change use the legacy two-part shape and
|
|
1140
|
+
* are no longer matched. Safe — the same-row case stays covered by the
|
|
1141
|
+
* per-message `messages[id]="acknowledged"` record, and the send-side
|
|
1142
|
+
* idempotent append prevents a resend from adding a second row at all.
|
|
1143
|
+
*/
|
|
1144
|
+
export function requestIdDeliveryKey(direction: MailboxDirection, recipientTaskId: string | undefined, requestId: string): string {
|
|
1145
|
+
return `${direction}:${recipientTaskId ?? "-"}:${requestId}`;
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1120
1148
|
export function recordMailboxRequestIdDelivery(manifest: TeamRunManifest, records: MailboxRequestIdDeliveryRecord[]): MailboxDeliveryState {
|
|
1121
1149
|
return withFileLockSync(deliveryFile(manifest, true), () => {
|
|
1122
1150
|
const delivery = readDeliveryState(manifest);
|
|
1123
1151
|
for (const record of records) {
|
|
1124
1152
|
delivery.messages[record.messageId] = "acknowledged";
|
|
1125
|
-
if (record.requestId)
|
|
1153
|
+
if (record.requestId)
|
|
1154
|
+
(delivery.requestIds ??= {})[requestIdDeliveryKey(record.direction, record.taskId, record.requestId)] = record.messageId;
|
|
1126
1155
|
}
|
|
1127
1156
|
delivery.updatedAt = new Date().toISOString();
|
|
1128
1157
|
// Pickup is terminal for that message on the worker side — full durability
|
|
@@ -167,7 +167,9 @@ export interface CompactionResult {
|
|
|
167
167
|
* 3. Atomically write (atomicWriteFile handles temp-file + rename)
|
|
168
168
|
* 4. Re-read to detect events appended during the window
|
|
169
169
|
* 5. If events were lost, append them
|
|
170
|
-
* 6.
|
|
170
|
+
* 6. Bump the `.gen` generation sidecar (EP4/R32-M1) so byte-offset cursors
|
|
171
|
+
* echoing the pre-compaction generation re-read instead of resuming mid-file
|
|
172
|
+
* 7. Return compaction stats
|
|
171
173
|
*/
|
|
172
174
|
export function compactEventLog(
|
|
173
175
|
eventsPath: string,
|
|
@@ -404,6 +406,21 @@ export function applyCompactionUnlocked(
|
|
|
404
406
|
logInternalError("event-log-rotation.compact", err, `eventsPath=${eventsPath}`);
|
|
405
407
|
return undefined;
|
|
406
408
|
}
|
|
409
|
+
// EP4 (R32-M1 ≡ R4-N1/FS10, fix-wave 6): compaction REWRITES the live log at
|
|
410
|
+
// the same path (atomicWriteFile → temp+rename, a NEW inode) and drops the
|
|
411
|
+
// events before the kept window. Without a generation bump, a byte-offset
|
|
412
|
+
// cursor reader echoing the pre-compaction `generation` (readEventsCursor's
|
|
413
|
+
// cache-key, cursor.ts) still matches the live sidecar and resumes from its
|
|
414
|
+
// old offset — which now points into the MIDDLE of the rewritten file — so
|
|
415
|
+
// it silently skips the kept-window head it never delivered. The `shrank`
|
|
416
|
+
// fallback does NOT cover this: it only fires when the file is shorter than
|
|
417
|
+
// the caller's offset. Bump here so EVERY compaction path (compactEventLog /
|
|
418
|
+
// compactPreparedEventLog / applyCompactionAppendGuarded + the inline call
|
|
419
|
+
// sites in event-log.ts) invalidates stale cursors exactly like rotation
|
|
420
|
+
// (rotateEventLogUnlocked). A no-op compaction never reaches this point —
|
|
421
|
+
// prepareCompaction returns undefined when originalCount <= compactToCount —
|
|
422
|
+
// so the generation stays stable when nothing was truncated.
|
|
423
|
+
bumpGenerationUnlocked(eventsPath);
|
|
407
424
|
// U22 fix round: the compacted file is NEW content at the same path — every
|
|
408
425
|
// old poison marker is meaningless (its byteOffset now names an arbitrary
|
|
409
426
|
// line of the rewritten file). Drop the sidecar so a stale marker can never
|
|
@@ -466,9 +483,10 @@ export function applyCompactionUnlocked(
|
|
|
466
483
|
/**
|
|
467
484
|
* R-03: Path of the generation sidecar for an events file. Mirrors the
|
|
468
485
|
* `.seq` sidecar pattern. The generation is bumped every time the events
|
|
469
|
-
* file is
|
|
470
|
-
*
|
|
471
|
-
*
|
|
486
|
+
* file is REWRITTEN in place with different content — rotation (truncated +
|
|
487
|
+
* archived) or compaction (prefix dropped, kept window rewritten) — so
|
|
488
|
+
* readers holding a byte-offset cursor can detect their offset is stale and
|
|
489
|
+
* re-read from the start of the new file instead of missing events.
|
|
472
490
|
*/
|
|
473
491
|
export function generationPath(eventsPath: string): string {
|
|
474
492
|
return `${eventsPath}.gen`;
|
|
@@ -500,11 +518,15 @@ export function currentGeneration(eventsPath: string): number {
|
|
|
500
518
|
|
|
501
519
|
/**
|
|
502
520
|
* R-03: Atomically bump and persist the generation sidecar. Called by
|
|
503
|
-
* `rotateEventLogUnlocked` after truncating the events file
|
|
521
|
+
* `rotateEventLogUnlocked` after truncating the events file, by
|
|
522
|
+
* `applyCompactionUnlocked` after rewriting it (EP4/R32-M1), and by
|
|
523
|
+
* `quarantinePoisonedEventsFile` after renaming the poisoned file away
|
|
524
|
+
* (EP4/FS10 residual, wave-6 M4-R2 — a rename is the same file-instance
|
|
525
|
+
* change at the SAME path, so byte-offset cursors must reset). Assumes the
|
|
504
526
|
* caller already holds the event-log lock (or accepts the unlocked race).
|
|
505
527
|
* Returns the new generation value.
|
|
506
528
|
*/
|
|
507
|
-
function bumpGenerationUnlocked(eventsPath: string): number {
|
|
529
|
+
export function bumpGenerationUnlocked(eventsPath: string): number {
|
|
508
530
|
const next = currentGeneration(eventsPath) + 1;
|
|
509
531
|
try {
|
|
510
532
|
atomicWriteFile(generationPath(eventsPath), String(next));
|
|
@@ -63,6 +63,7 @@
|
|
|
63
63
|
*/
|
|
64
64
|
import * as fs from "node:fs";
|
|
65
65
|
import type { JsonlParseFailure } from "../../utils/incremental-reader.ts";
|
|
66
|
+
import { bumpGenerationUnlocked } from "./event-log-rotation.ts";
|
|
66
67
|
|
|
67
68
|
export type PoisonFailure = JsonlParseFailure;
|
|
68
69
|
|
|
@@ -231,7 +232,12 @@ function persistPoisonMarkers(
|
|
|
231
232
|
|
|
232
233
|
/** Last resort — quarantineCorruptFile pattern (manifest-io.ts:22-28): rename
|
|
233
234
|
* the events file to `<name>.corrupt-<ts>` (preserved for debugging) and drop
|
|
234
|
-
* the sidecar so a future log at the same path starts clean. Best-effort.
|
|
235
|
+
* the sidecar so a future log at the same path starts clean. Best-effort.
|
|
236
|
+
* EP4/FS10 residual (wave-6 M4-R2): the rename is a file-instance change at
|
|
237
|
+
* the SAME path, so bump the generation sidecar (shared helper from M2-4 /
|
|
238
|
+
* event-log-rotation) — a byte-offset cursor holding the old generation then
|
|
239
|
+
* detects the change and resets instead of resuming past the (now absent)
|
|
240
|
+
* file's old offsets. */
|
|
235
241
|
function quarantinePoisonedEventsFile(eventsPath: string): void {
|
|
236
242
|
const quarantinePath = `${eventsPath}.corrupt-${Date.now()}`;
|
|
237
243
|
try {
|
|
@@ -240,6 +246,15 @@ function quarantinePoisonedEventsFile(eventsPath: string): void {
|
|
|
240
246
|
console.error(`[poison-marker] quarantine rename failed for ${eventsPath} (file left in place):`, error);
|
|
241
247
|
return;
|
|
242
248
|
}
|
|
249
|
+
try {
|
|
250
|
+
// EP4/FS10: bump .gen AFTER a successful rename so readers see the new
|
|
251
|
+
// file instance. Upstream imported-by-us module (event-log-rotation)
|
|
252
|
+
// already imports poisonMarkerPath from here — a cycle, but both sides
|
|
253
|
+
// are hoisted function declarations, so the binding is live at call time.
|
|
254
|
+
bumpGenerationUnlocked(eventsPath);
|
|
255
|
+
} catch {
|
|
256
|
+
/* best-effort: a failed gen bump must not fail the quarantine */
|
|
257
|
+
}
|
|
243
258
|
try {
|
|
244
259
|
fs.rmSync(poisonMarkerPath(eventsPath), { force: true });
|
|
245
260
|
} catch {
|