pi-crew 0.11.24 → 0.11.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.md +2 -2
- package/agents/designer.md +23 -0
- package/agents/orchestrator.md +17 -0
- package/agents/verifier.md +1 -1
- package/dist/index.mjs +568 -559
- package/{workflows/chain.workflow.md → docs/chain-workflow.md} +8 -0
- package/docs/dynamic-workflows.md +23 -0
- package/docs/publishing.md +34 -5
- package/docs/troubleshooting.md +50 -2
- package/docs/usage.md +7 -2
- package/install.mjs +2 -1
- package/package.json +4 -2
- package/skills/read-only-explorer/SKILL.md +15 -5
- package/skills/real-test-pi-crew/SKILL.md +7 -7
- package/src/config/config-validation.ts +58 -0
- package/src/config/config.ts +90 -5
- package/src/config/env-vars.ts +14 -7
- package/src/config/role-tools.ts +7 -5
- package/src/config/types.ts +4 -1
- package/src/extension/crew-cleanup.ts +6 -0
- package/src/extension/project-init.ts +19 -1
- package/src/extension/register.ts +68 -30
- package/src/extension/registration/artifact-cleanup.ts +52 -11
- package/src/extension/registration/commands/manage.ts +9 -2
- package/src/extension/registration/hook-registration.ts +3 -0
- package/src/extension/registration/lazy-configurers.ts +2 -0
- package/src/extension/registration/lifecycle-handlers.ts +79 -18
- package/src/extension/registration/render-loop.ts +27 -2
- package/src/extension/registration/tool-loop-guard.ts +5 -3
- package/src/extension/registration/verify-gate-tool.ts +7 -3
- package/src/extension/team-tool/api/read.ts +12 -3
- package/src/extension/team-tool/cancel.ts +11 -0
- package/src/extension/team-tool/config-patch.ts +56 -11
- package/src/extension/team-tool/destructive-gate.ts +72 -12
- package/src/extension/team-tool/dispatch/manage.ts +9 -1
- package/src/extension/team-tool/handle-schedule.ts +38 -4
- package/src/extension/team-tool/handle-settings.ts +76 -6
- package/src/extension/team-tool/health-monitor.ts +14 -2
- package/src/extension/team-tool/lifecycle-actions.ts +28 -10
- package/src/extension/team-tool/respond.ts +12 -5
- package/src/extension/team-tool/run-deadline.ts +29 -2
- package/src/extension/team-tool/run.ts +11 -0
- package/src/extension/team-tool/status.ts +64 -8
- package/src/extension/team-tool/workflow-manage.ts +5 -1
- package/src/extension/team-tool.ts +3 -0
- package/src/i18n.ts +18 -14
- package/src/prompt/inbox-poll.ts +14 -2
- package/src/prompt/message-tool.ts +1 -1
- package/src/prompt/prompt-runtime.ts +25 -3
- package/src/prompt/scratchpad-lifecycle.ts +14 -13
- package/src/prompt/surface-worker.ts +35 -9
- package/src/runtime/broker/crew-broker.ts +113 -111
- package/src/runtime/broker/delegate/delegate-abort.ts +118 -0
- package/src/runtime/broker/delegate/delegate-event.ts +3 -1
- package/src/runtime/broker/delegate/delegate-finalize.ts +179 -0
- package/src/runtime/broker/protocol/wait-resolve.ts +71 -5
- package/src/runtime/budget-enforcement.ts +26 -6
- package/src/runtime/chain-parser.ts +5 -3
- package/src/runtime/chain-runner.ts +41 -22
- package/src/runtime/child-pi/mock-fixtures.ts +12 -5
- package/src/runtime/compaction/tool-output-pruner.ts +8 -3
- package/src/runtime/delegate-spawn.ts +61 -8
- package/src/runtime/dispatch-batch.ts +10 -4
- package/src/runtime/effectiveness.ts +31 -3
- package/src/runtime/goal-workflow/plan-templates.ts +2 -2
- package/src/runtime/live-session/live-agent-control.ts +89 -6
- package/src/runtime/live-session/live-agent-manager.ts +94 -10
- package/src/runtime/live-session/live-session-runtime.ts +9 -2
- package/src/runtime/model/model-fallback.ts +6 -9
- package/src/runtime/model/pi-args.ts +3 -3
- package/src/runtime/model/runtime-policy.ts +4 -2
- package/src/runtime/model/runtime-resolver.ts +6 -2
- package/src/runtime/process/orphan-reaper.ts +143 -0
- package/src/runtime/process-status.ts +38 -16
- package/src/runtime/resilient-edit.ts +170 -93
- package/src/runtime/role-permission.ts +3 -1
- package/src/runtime/run-tracker.ts +9 -1
- package/src/runtime/sanitize/worker-text.ts +98 -0
- package/src/runtime/scheduling/run-coalesced-task-group.ts +82 -28
- package/src/runtime/spawn-policy.ts +190 -8
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/subagent-manager.ts +29 -0
- package/src/runtime/surface/herdr-provider.ts +31 -1
- package/src/runtime/task-output-context.ts +58 -11
- package/src/runtime/task-runner/post-execution.ts +150 -0
- package/src/runtime/task-runner/prompt-builder.ts +41 -8
- package/src/runtime/task-runner.ts +16 -3
- package/src/runtime/verification/classifier-pre-gate.ts +27 -6
- package/src/runtime/verification/verify-gate.ts +153 -30
- package/src/state/atomic-write.ts +73 -4
- package/src/state/contracts.ts +7 -0
- package/src/state/coordination/mailbox.ts +40 -11
- package/src/state/event-log/event-log-rotation.ts +28 -6
- package/src/state/event-log/poison-marker.ts +16 -1
- package/src/state/stores/artifact-store.ts +181 -6
- package/src/state/stores/manifest-io.ts +126 -8
- package/src/state/stores/state-store.ts +30 -31
- package/src/state/types.ts +13 -0
- package/src/state/usage.ts +41 -0
- package/src/ui/run-dashboard.ts +3 -3
- package/src/ui/settings-overlay.ts +19 -1
- package/src/utils/internal-error.ts +5 -1
- package/src/utils/timer-limits.ts +30 -0
- package/src/utils/token-counter.ts +47 -0
- package/src/workflows/discover-workflows.ts +9 -1
- package/src/worktree/worktree-manager.ts +128 -18
- package/workflows/default.workflow.md +1 -1
- package/workflows/fast-fix.workflow.md +1 -1
- package/workflows/plan-execute.workflow.md +1 -1
- package/workflows/review.workflow.md +1 -1
- package/workflows/strict-fast-fix.workflow.md +1 -1
|
@@ -121,6 +121,40 @@ export function shouldAutoExit(signals: AutoExitSignals): boolean {
|
|
|
121
121
|
|
|
122
122
|
// ── parent liveness ───────────────────────────────────────────────────────
|
|
123
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Default `/proc/<pid>/stat` reader for {@link parentAlive}.
|
|
126
|
+
*
|
|
127
|
+
* R19-M1 (deep-review round 19): the R7-P1 tolerance (a THROWING reader with no
|
|
128
|
+
* recorded start time ⇒ alive-with-caveat) was UNREACHABLE in production because
|
|
129
|
+
* the default reader caught its own error and returned `undefined` — which
|
|
130
|
+
* `parentAlive` reads as “no entry ⇒ parent is dead”. On macOS (no procfs) every
|
|
131
|
+
* healthy surface worker therefore self-terminated at the first 5 s guard tick
|
|
132
|
+
* (`terminate("worker.parent-lost")`), and the R7-P1 pin only held for injected
|
|
133
|
+
* readers.
|
|
134
|
+
*
|
|
135
|
+
* The platform distinction is the whole point:
|
|
136
|
+
* - Linux with procfs: ENOENT is the DEAD signal (unchanged — keeps Linux
|
|
137
|
+
* dead-pid detection exact); any other errno (EACCES/EPERM) is “unreadable”
|
|
138
|
+
* and is surfaced as a throw so the tolerance applies.
|
|
139
|
+
* - No procfs (macOS / Windows / a Linux container without /proc): the file can
|
|
140
|
+
* never exist, so “cannot read” must NOT be reported as “dead”. Throw —
|
|
141
|
+
* `parentAlive` then keeps the worker alive and the host watchdog is the
|
|
142
|
+
* backstop (a false “alive” is recoverable; a false “dead” kills work).
|
|
143
|
+
*
|
|
144
|
+
* `platform` is injectable so the macOS path is testable on every CI OS.
|
|
145
|
+
*/
|
|
146
|
+
export function defaultProcStatReader(pid: number, platform: string = process.platform): string | undefined {
|
|
147
|
+
if (platform !== "linux" || !fs.existsSync("/proc/self/stat")) {
|
|
148
|
+
throw new Error(`procfs unavailable on ${platform}: cannot read /proc/${pid}/stat`);
|
|
149
|
+
}
|
|
150
|
+
try {
|
|
151
|
+
return fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
152
|
+
} catch (error) {
|
|
153
|
+
if ((error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") return undefined;
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
124
158
|
/**
|
|
125
159
|
* Pure parent-liveness decision.
|
|
126
160
|
* - stat unreadable → parent is GONE (/proc ENOENT means dead on Linux).
|
|
@@ -132,15 +166,7 @@ export function shouldAutoExit(signals: AutoExitSignals): boolean {
|
|
|
132
166
|
*/
|
|
133
167
|
export function parentAlive(pid: number, expectedStartTime: string | undefined, readStat?: (pid: number) => string | undefined): boolean {
|
|
134
168
|
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
135
|
-
const reader =
|
|
136
|
-
readStat ??
|
|
137
|
-
((p: number): string | undefined => {
|
|
138
|
-
try {
|
|
139
|
-
return fs.readFileSync(`/proc/${p}/stat`, "utf8");
|
|
140
|
-
} catch {
|
|
141
|
-
return undefined;
|
|
142
|
-
}
|
|
143
|
-
});
|
|
169
|
+
const reader = readStat ?? ((p: number): string | undefined => defaultProcStatReader(p));
|
|
144
170
|
let stat: string | undefined;
|
|
145
171
|
let readerThrew = false;
|
|
146
172
|
try {
|
|
@@ -39,7 +39,6 @@
|
|
|
39
39
|
import { randomUUID } from "node:crypto";
|
|
40
40
|
import * as fsp from "node:fs/promises";
|
|
41
41
|
import * as net from "node:net";
|
|
42
|
-
import { withRunLockSync } from "../../state/coordination/locks.ts";
|
|
43
42
|
import {
|
|
44
43
|
appendMailboxMessageAsync,
|
|
45
44
|
appendMailboxMessageIdempotentAsync,
|
|
@@ -60,7 +59,9 @@ import { resolveCrewMaxDepth } from "../model/pi-args.ts";
|
|
|
60
59
|
import { NestedSlotBudget } from "../scheduling/nested-slots.ts";
|
|
61
60
|
import { evaluateDelegateAdmission } from "../spawn-policy.ts";
|
|
62
61
|
import { type BrokerToken, BrokerTokenRegistry } from "./crew-broker-tokens.ts";
|
|
62
|
+
import { DelegateAbortRegistry } from "./delegate/delegate-abort.ts";
|
|
63
63
|
import { type DelegateEventTarget, type DelegateEventType, recordDelegateEvent } from "./delegate/delegate-event.ts";
|
|
64
|
+
import { delegateTerminalEvent, finalizeDelegateResult } from "./delegate/delegate-finalize.ts";
|
|
64
65
|
import { promoteShadowToRunning } from "./delegate/shadow-lifecycle.ts";
|
|
65
66
|
import { fanoutMailboxMessage } from "./mailbox-observer/mailbox-fanout.ts";
|
|
66
67
|
import type { CrewBrokerOptions, ServerConnection } from "./protocol/connection-state.ts";
|
|
@@ -155,6 +156,9 @@ export class CrewBroker {
|
|
|
155
156
|
/** T3/R5 (ADR-5 §2): nested-slot budget for delegate grandchildren — lazily
|
|
156
157
|
* sized from options (max(1, floor(globalWorkerSemaphore/2)) or override). */
|
|
157
158
|
private nestedSlots: NestedSlotBudget | undefined;
|
|
159
|
+
/** M3-5/R43-N3: abort controllers for in-flight delegate grandchildren —
|
|
160
|
+
* aborted on run terminalise (abortDelegatesForRun) and on stop(). */
|
|
161
|
+
private readonly delegateAborts = new DelegateAbortRegistry();
|
|
158
162
|
/** A single observable handshake counter (test/observability). */
|
|
159
163
|
private handshakeCount = 0;
|
|
160
164
|
/** R10-3: stat-gated manifest/tasks cache shared by all task.waitStatus
|
|
@@ -244,6 +248,30 @@ export class CrewBroker {
|
|
|
244
248
|
}
|
|
245
249
|
}
|
|
246
250
|
|
|
251
|
+
/** M3-5 / R43-N3: cancel every in-flight DELEGATE GRANDCHILD of `runId`.
|
|
252
|
+
* Called from the run-terminal paths (budget abort / cancellation via
|
|
253
|
+
* budget-enforcement.ts → abortRunDelegates) and from stop(). Before this
|
|
254
|
+
* existed the spawner was fire-and-forget: a budget abort left grandchildren
|
|
255
|
+
* running to their own deadline (up to 24h), still spending and still
|
|
256
|
+
* holding nested slots. Returns the number cancelled; idempotent. */
|
|
257
|
+
abortDelegatesForRun(runId: string, reason = "run-terminal"): number {
|
|
258
|
+
if (typeof runId !== "string" || runId.length === 0) return 0;
|
|
259
|
+
const aborted = this.delegateAborts.abortRun(runId, { reason });
|
|
260
|
+
if (aborted > 0) {
|
|
261
|
+
logInternalError(
|
|
262
|
+
"crew-broker.delegate.abort-run",
|
|
263
|
+
new Error(`${aborted} delegate grandchild(ren) cancelled`),
|
|
264
|
+
`runId=${runId} reason=${reason}`,
|
|
265
|
+
);
|
|
266
|
+
}
|
|
267
|
+
return aborted;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Number of in-flight delegate grandchildren (diagnostics / tests). */
|
|
271
|
+
get inFlightDelegateCount(): number {
|
|
272
|
+
return this.delegateAborts.size;
|
|
273
|
+
}
|
|
274
|
+
|
|
247
275
|
/** Issue the orchestrator token for `runId` (F-06). Cryptographically
|
|
248
276
|
* distinct from every per-task token — the ONLY token that grants
|
|
249
277
|
* role:'orchestrator' (required for steer.push / msg.send). */
|
|
@@ -363,6 +391,9 @@ export class CrewBroker {
|
|
|
363
391
|
async stop(): Promise<void> {
|
|
364
392
|
if (this.stopped) return;
|
|
365
393
|
this.stopped = true;
|
|
394
|
+
// M3-5/R43-N3: cancel every in-flight grandchild FIRST — broker teardown
|
|
395
|
+
// must not leave delegate spawns running to their own 24h deadline.
|
|
396
|
+
this.delegateAborts.abortAll({ reason: "broker-stop" });
|
|
366
397
|
// Phase 1.3: unregister the mailbox observer before closing connections.
|
|
367
398
|
if (this.mailboxObserverUnsub) {
|
|
368
399
|
try {
|
|
@@ -1527,6 +1558,18 @@ export class CrewBroker {
|
|
|
1527
1558
|
// ADR-5 §12: the delegate surface is an escalation — trusted only by the
|
|
1528
1559
|
// explicit user opt-in threaded from config.nesting.enabled (sensitive).
|
|
1529
1560
|
untrusted: this.options.nestingTrustedEscalation !== true,
|
|
1561
|
+
// M3-3 / R43-N1 SEED: the run-level token budget is the grant every
|
|
1562
|
+
// task inherits when its own record carries no allocation (pre-fix
|
|
1563
|
+
// state of EVERY task — nothing produced `tokensGranted`, so every
|
|
1564
|
+
// budgeted delegate was denied and every unmetered one slipped by).
|
|
1565
|
+
// `budgetUnlimited` opts out of enforcement entirely ⇒ no seed.
|
|
1566
|
+
...(fresh.manifest.budgetTotal !== undefined && fresh.manifest.budgetUnlimited !== true
|
|
1567
|
+
? { parentBudgetGrantTokens: fresh.manifest.budgetTotal }
|
|
1568
|
+
: {}),
|
|
1569
|
+
// M3-3 / R43-N1: unmetered delegation stays available (documented
|
|
1570
|
+
// pre-fix behavior for nesting runs without a budgetTotal) but is now
|
|
1571
|
+
// an EXPLICIT, audited allow (`unmetered: true` on delegate.admitted).
|
|
1572
|
+
unmeteredDelegates: "allow",
|
|
1530
1573
|
workspace: {
|
|
1531
1574
|
serializeEnabled: this.options.serializeOnPathOverlap === true,
|
|
1532
1575
|
overlappingInFlightExecutors: overlapping,
|
|
@@ -1549,20 +1592,25 @@ export class CrewBroker {
|
|
|
1549
1592
|
};
|
|
1550
1593
|
}
|
|
1551
1594
|
// Reserve the requested budget pessimistically (ADR-5 §5): tokensSpent
|
|
1552
|
-
// += budgetTokens now
|
|
1553
|
-
//
|
|
1595
|
+
// += budgetTokens now. M3-3/R43-N1: the reservation ALSO seeds the parent
|
|
1596
|
+
// allocation when the record has none — the `decision.budget.grantedTokens`
|
|
1597
|
+
// the gate admitted against is the write-once `tokensGranted`
|
|
1598
|
+
// (state-helpers.ts merge keeps disk's value verbatim). Pre-fix the
|
|
1599
|
+
// `parentAllocation` guard silently skipped the reserve for every task
|
|
1600
|
+
// that had no allocation, i.e. for every task.
|
|
1601
|
+
// M3-4/R44-N4: this reservation IS the spend — no settle-time refund.
|
|
1554
1602
|
let reserved = 0;
|
|
1555
|
-
const
|
|
1603
|
+
const budgetDecision = decision.budget;
|
|
1556
1604
|
let tasksAfterReserve = fresh.tasks;
|
|
1557
|
-
if (
|
|
1558
|
-
reserved =
|
|
1605
|
+
if (budgetDecision?.metered === true) {
|
|
1606
|
+
reserved = budgetDecision.requestedTokens ?? 0;
|
|
1559
1607
|
const updatedTasks = fresh.tasks.map((t) =>
|
|
1560
1608
|
t.id === parentTaskId
|
|
1561
1609
|
? {
|
|
1562
1610
|
...t,
|
|
1563
1611
|
allocation: {
|
|
1564
|
-
tokensGranted:
|
|
1565
|
-
tokensSpent:
|
|
1612
|
+
tokensGranted: budgetDecision.grantedTokens,
|
|
1613
|
+
tokensSpent: budgetDecision.spentTokens + reserved,
|
|
1566
1614
|
},
|
|
1567
1615
|
}
|
|
1568
1616
|
: t,
|
|
@@ -1589,7 +1637,7 @@ export class CrewBroker {
|
|
|
1589
1637
|
} satisfies TeamTaskState,
|
|
1590
1638
|
]);
|
|
1591
1639
|
// RR-023 F3: thread the parent's model out of the locked read (gc fallback).
|
|
1592
|
-
return { code: "ok" as const, decision, reserved, executionCwd, parentModel: task.model };
|
|
1640
|
+
return { code: "ok" as const, decision, reserved, executionCwd, parentModel: task.model, parentRole: task.role };
|
|
1593
1641
|
});
|
|
1594
1642
|
// RR-023 F4 (finding #4): run.lock held by a live cross-process holder —
|
|
1595
1643
|
// degrade to a typed busy frame AND a delegate.rejected event (never
|
|
@@ -1604,11 +1652,13 @@ export class CrewBroker {
|
|
|
1604
1652
|
this.sendError(conn, id, admissionOutcome.code, admissionOutcome.message);
|
|
1605
1653
|
return;
|
|
1606
1654
|
}
|
|
1607
|
-
const { decision, reserved, executionCwd, parentModel } = admissionOutcome;
|
|
1655
|
+
const { decision, reserved, executionCwd, parentModel, parentRole } = admissionOutcome;
|
|
1608
1656
|
this.recordDelegateEvent(loaded.manifest, "delegate.admitted", parentTaskId, {
|
|
1609
1657
|
subId,
|
|
1610
1658
|
childDepth: decision.childDepth,
|
|
1611
1659
|
role: requested.role ?? "explorer",
|
|
1660
|
+
// M3-3/R43-N1: an unmetered allow is EXPLICIT in the audit trail.
|
|
1661
|
+
...(decision.budget?.metered === false ? { unmetered: true } : {}),
|
|
1612
1662
|
...(reserved > 0 ? { reservedTokens: reserved } : {}),
|
|
1613
1663
|
});
|
|
1614
1664
|
// S1#1: pre-mint the grandchild-scoped token when (and only when) the
|
|
@@ -1623,6 +1673,10 @@ export class CrewBroker {
|
|
|
1623
1673
|
// RR-023 F3 (finding #3, run team_20260929041427): model-less gc spawn fell
|
|
1624
1674
|
// through to the pi GLOBAL default — inherit the parent task's model.
|
|
1625
1675
|
const gcModel = requested.model ?? parentModel;
|
|
1676
|
+
// M3-5/R43-N3: register the grandchild's abort controller BEFORE the
|
|
1677
|
+
// spawn starts, so a run terminalise / budget abort / broker stop that
|
|
1678
|
+
// lands mid-spawn still cancels it (no orphan outliving the run).
|
|
1679
|
+
const delegateAbort = this.delegateAborts.register(subId, runId);
|
|
1626
1680
|
// Background grandchild lifecycle (ADR-5 §1/§6).
|
|
1627
1681
|
const spawner = this.options.grandchildSpawner ?? spawnDelegateGrandchild;
|
|
1628
1682
|
void (async () => {
|
|
@@ -1632,6 +1686,9 @@ export class CrewBroker {
|
|
|
1632
1686
|
cwd: executionCwd,
|
|
1633
1687
|
runId,
|
|
1634
1688
|
parentTaskId,
|
|
1689
|
+
// R43-S1: transitive tier containment — the spawner applies
|
|
1690
|
+
// min(parentRole, requestedRole) as the grandchild tool policy.
|
|
1691
|
+
parentRole,
|
|
1635
1692
|
subId,
|
|
1636
1693
|
prompt: requested.prompt,
|
|
1637
1694
|
eventsPath: loaded.manifest.eventsPath,
|
|
@@ -1641,6 +1698,8 @@ export class CrewBroker {
|
|
|
1641
1698
|
...(requested.maxTurns !== undefined ? { maxTurns: requested.maxTurns } : {}),
|
|
1642
1699
|
timeoutSec: decision.timeoutSec ?? 900,
|
|
1643
1700
|
depthOverride: decision.childDepth ?? 2,
|
|
1701
|
+
// M3-5/R43-N3: owner cancellation (registry-driven).
|
|
1702
|
+
signal: delegateAbort.signal,
|
|
1644
1703
|
// RR-012 F16: promote shadow queued→running once the process exists.
|
|
1645
1704
|
onSpawn: (pid) => {
|
|
1646
1705
|
if (pid !== null) promoteShadowToRunning(cwd, runId, subId);
|
|
@@ -1649,110 +1708,32 @@ export class CrewBroker {
|
|
|
1649
1708
|
} catch (err) {
|
|
1650
1709
|
outcome = { ok: false, resultText: `delegate spawn failed: ${(err as Error).message}` };
|
|
1651
1710
|
}
|
|
1652
|
-
// S3 fence hardening: neutralize smuggled end-fence markers in the
|
|
1653
|
-
// grandchild's raw text, then wrap (the client re-fences too — the
|
|
1654
|
-
// mailbox is an unauthenticated same-uid channel by design).
|
|
1655
|
-
const sanitizedText = outcome.resultText.replace(/^--- (end )?delegate /gm, "-- ~delegate ").slice(0, 65_536);
|
|
1656
|
-
const fenced = `--- delegate ${subId} ${outcome.timedOut ? "(timed out)" : outcome.ok ? "(ok)" : "(failed)"} ---\n${sanitizedText}\n--- end delegate ${subId} ---`;
|
|
1657
|
-
// Durable delivery (ADR-5 §1): fenced result into the PARENT task's
|
|
1658
|
-
// mailbox + budget roll-up + slot release — run-locked RMW.
|
|
1659
1711
|
try {
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
// Roll-up (ADR-5 §5): reconcile the pessimistic reservation to
|
|
1681
|
-
// actual usage; refund the difference. Unattributed → keep reserve.
|
|
1682
|
-
let tasksToWrite = latest.tasks;
|
|
1683
|
-
if (reserved > 0) {
|
|
1684
|
-
const parent = latest.tasks.find((t) => t.id === parentTaskId);
|
|
1685
|
-
const parentAlloc = parent?.allocation;
|
|
1686
|
-
if (parentAlloc) {
|
|
1687
|
-
const actual = Math.max(0, Math.min(outcome.usageTokens ?? reserved, reserved));
|
|
1688
|
-
const tokensSpent = (parentAlloc.tokensSpent ?? 0) - reserved + actual;
|
|
1689
|
-
tasksToWrite = tasksToWrite.map((t) =>
|
|
1690
|
-
t.id === parentTaskId
|
|
1691
|
-
? { ...t, allocation: { tokensGranted: parentAlloc.tokensGranted, tokensSpent } }
|
|
1692
|
-
: t,
|
|
1693
|
-
);
|
|
1694
|
-
this.recordDelegateEvent(latest.manifest, "delegate.rolled_up", parentTaskId, {
|
|
1695
|
-
subId,
|
|
1696
|
-
actualTokens: actual,
|
|
1697
|
-
});
|
|
1698
|
-
}
|
|
1699
|
-
}
|
|
1700
|
-
// S1#1 (verifier N1): the terminal flip is UNCONDITIONAL — never depends
|
|
1701
|
-
// on a reservation. RR-012 F16: onSpawn promotes the record to "running"
|
|
1702
|
-
// during execution; this flip terminalizes every settle path.
|
|
1703
|
-
saveRunTasks(
|
|
1704
|
-
latest.manifest,
|
|
1705
|
-
tasksToWrite.map((t) =>
|
|
1706
|
-
t.id === subId
|
|
1707
|
-
? {
|
|
1708
|
-
...t,
|
|
1709
|
-
status: outcome.ok ? ("completed" as const) : ("failed" as const),
|
|
1710
|
-
completedAt: new Date().toISOString(),
|
|
1711
|
-
}
|
|
1712
|
-
: t,
|
|
1713
|
-
),
|
|
1714
|
-
);
|
|
1715
|
-
});
|
|
1716
|
-
}
|
|
1717
|
-
} catch (err) {
|
|
1718
|
-
logInternalError("crew-broker.delegate.finalize", err instanceof Error ? err : new Error(String(err)), `runId=${runId}`);
|
|
1719
|
-
// P2-8c: best-effort refund when finalize fails — the reservation
|
|
1720
|
-
// must not linger inflated after a roll-up write failure.
|
|
1721
|
-
if (reserved > 0) {
|
|
1722
|
-
try {
|
|
1723
|
-
const rf = loadRunManifestById(cwd, runId);
|
|
1724
|
-
if (rf) {
|
|
1725
|
-
withRunLockSync(rf.manifest, () => {
|
|
1726
|
-
const rl = loadRunManifestById(cwd, runId);
|
|
1727
|
-
if (!rl) return;
|
|
1728
|
-
const par = rl.tasks.find((t) => t.id === parentTaskId);
|
|
1729
|
-
const pa = par?.allocation;
|
|
1730
|
-
if (pa) {
|
|
1731
|
-
saveRunTasks(
|
|
1732
|
-
rl.manifest,
|
|
1733
|
-
rl.tasks.map((t) =>
|
|
1734
|
-
t.id === parentTaskId
|
|
1735
|
-
? {
|
|
1736
|
-
...t,
|
|
1737
|
-
allocation: {
|
|
1738
|
-
tokensGranted: pa.tokensGranted,
|
|
1739
|
-
tokensSpent: Math.max(0, (pa.tokensSpent ?? 0) - reserved),
|
|
1740
|
-
},
|
|
1741
|
-
}
|
|
1742
|
-
: t,
|
|
1743
|
-
),
|
|
1744
|
-
);
|
|
1745
|
-
}
|
|
1746
|
-
});
|
|
1747
|
-
}
|
|
1748
|
-
} catch {
|
|
1749
|
-
/* refund is best-effort; the startup reconcile is the follow-up */
|
|
1750
|
-
}
|
|
1751
|
-
}
|
|
1712
|
+
// S3 fence hardening: neutralize smuggled end-fence markers in the
|
|
1713
|
+
// grandchild's raw text, then wrap (the client re-fences too — the
|
|
1714
|
+
// mailbox is an unauthenticated same-uid channel by design).
|
|
1715
|
+
const sanitizedText = outcome.resultText.replace(/^--- (end )?delegate /gm, "-- ~delegate ").slice(0, 65_536);
|
|
1716
|
+
const statusTag = outcome.cancelled ? "(cancelled)" : outcome.timedOut ? "(timed out)" : outcome.ok ? "(ok)" : "(failed)";
|
|
1717
|
+
const fenced = `--- delegate ${subId} ${statusTag} ---\n${sanitizedText}\n--- end delegate ${subId} ---`;
|
|
1718
|
+
// Durable delivery + budget charge + shadow terminal flip — all
|
|
1719
|
+
// inside one run lock, off a fresh disk read (delegate-finalize.ts
|
|
1720
|
+
// owns the R43-N4 reroute and the R44-N4 broker-side charge).
|
|
1721
|
+
finalizeDelegateResult({
|
|
1722
|
+
brokerCwd: cwd,
|
|
1723
|
+
runId,
|
|
1724
|
+
parentTaskId,
|
|
1725
|
+
subId,
|
|
1726
|
+
fenced,
|
|
1727
|
+
ok: outcome.ok,
|
|
1728
|
+
...(outcome.timedOut !== undefined ? { timedOut: outcome.timedOut } : {}),
|
|
1729
|
+
reservedTokens: reserved,
|
|
1730
|
+
...(outcome.usageTokens !== undefined ? { selfReportedTokens: outcome.usageTokens } : {}),
|
|
1731
|
+
});
|
|
1752
1732
|
} finally {
|
|
1733
|
+
this.delegateAborts.release(subId);
|
|
1753
1734
|
this.nestedSlotBudget().release(subId);
|
|
1754
1735
|
}
|
|
1755
|
-
this.recordDelegateEvent(loaded.manifest, outcome
|
|
1736
|
+
this.recordDelegateEvent(loaded.manifest, delegateTerminalEvent(outcome), parentTaskId, {
|
|
1756
1737
|
subId,
|
|
1757
1738
|
ok: outcome.ok,
|
|
1758
1739
|
});
|
|
@@ -1853,10 +1834,31 @@ export class CrewBroker {
|
|
|
1853
1834
|
}
|
|
1854
1835
|
: t,
|
|
1855
1836
|
);
|
|
1837
|
+
// M4-R1(a) (fix-wave 6 residual, M3-C): `waitState` is a SINGLE
|
|
1838
|
+
// run-level slot and `isWaitAnswerPending` (stale-reconciler) reads
|
|
1839
|
+
// ONLY it — that slot is the shield keeping a parked (deliberately
|
|
1840
|
+
// non-heartbeating) worker from being stale-reaped / orphan-cancelled.
|
|
1841
|
+
// Overwriting it unconditionally with the NEWER park SHORTENED the
|
|
1842
|
+
// shield whenever the new park asked for a shorter timeout (the
|
|
1843
|
+
// clamp allows 1s..1h), so an older, longer park could lose its
|
|
1844
|
+
// protection while still awaiting an answer. Same invariant as
|
|
1845
|
+
// wait.resolve's repointedWaitState: the slot follows the park with
|
|
1846
|
+
// the LATEST deadline — a park never shortens a live shield (living
|
|
1847
|
+
// slightly long only delays repair, the fail-safe direction).
|
|
1848
|
+
const currentSlot = fresh.manifest.waitState;
|
|
1849
|
+
const currentOwner = currentSlot === undefined ? undefined : fresh.tasks.find((t) => t.id === currentSlot.taskId);
|
|
1850
|
+
const keepCurrentSlot =
|
|
1851
|
+
currentSlot !== undefined &&
|
|
1852
|
+
typeof currentSlot.deadline === "number" &&
|
|
1853
|
+
currentSlot.deadline > deadline &&
|
|
1854
|
+
currentSlot.taskId !== taskId &&
|
|
1855
|
+
currentOwner?.status === "waiting" &&
|
|
1856
|
+
currentOwner.waiting?.questionId === currentSlot.questionId;
|
|
1856
1857
|
const updatedManifest = {
|
|
1857
1858
|
...fresh.manifest,
|
|
1858
1859
|
// manifest.status stays "running" — park NEVER flips run status (ADR item 3).
|
|
1859
|
-
|
|
1860
|
+
// R2-M-A2: mirror task.waiting.deadline for shield expiry.
|
|
1861
|
+
waitState: keepCurrentSlot ? currentSlot : { taskId, questionId, askedAt, deadline },
|
|
1860
1862
|
updatedAt: askedAt,
|
|
1861
1863
|
};
|
|
1862
1864
|
saveRunTasks(updatedManifest, updatedTasks);
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* delegate-abort.ts — in-flight grandchild cancellation (M3-5 / R43-N3).
|
|
3
|
+
*
|
|
4
|
+
* R43-N3 (2026-10-18 round 43, fix-wave 6): the `delegate.request` handler
|
|
5
|
+
* spawned the grandchild inside a detached `void (async () => …)()` with NO
|
|
6
|
+
* abort path — `CrewBroker.stop()` only cleared its token maps, and the
|
|
7
|
+
* team-runner's terminalise path drained its OWN dispatch units only. So a
|
|
8
|
+
* budget abort / run cancel / broker teardown left grandchildren running with
|
|
9
|
+
* a 900s-…-86400s deadline of their own, still burning tokens and still
|
|
10
|
+
* holding nested slots (R43-N3: "orphans keep spending up to 24h").
|
|
11
|
+
*
|
|
12
|
+
* This module is the missing link: the broker registers one AbortController
|
|
13
|
+
* per live grandchild, the spawner subscribes to it (delegate-spawn.ts
|
|
14
|
+
* `signal`), and every terminal path aborts them.
|
|
15
|
+
*
|
|
16
|
+
* Two process-local wiring layers:
|
|
17
|
+
* 1. {@link DelegateAbortRegistry} — owned by the broker instance (one per
|
|
18
|
+
* root session), keyed by grandchild subId.
|
|
19
|
+
* 2. {@link setActiveDelegateAborter} / {@link abortRunDelegates} — the
|
|
20
|
+
* process-local singleton the runner side calls without holding a broker
|
|
21
|
+
* reference (mirrors `broker-issuer.ts`'s issuer/revoker registry: a
|
|
22
|
+
* function reference only, never a token).
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
export interface DelegateAbortReason {
|
|
26
|
+
/** Machine-readable cause, recorded on the `delegate.cancelled` event. */
|
|
27
|
+
reason: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Per-broker registry of live grandchild abort controllers.
|
|
32
|
+
*
|
|
33
|
+
* Invariants:
|
|
34
|
+
* - `register` is called BEFORE the spawner starts (so a terminal event that
|
|
35
|
+
* lands during the spawn still cancels it);
|
|
36
|
+
* - `release` is called from the settle `finally` — a settled delegate never
|
|
37
|
+
* lingers in the map (bounded memory, no double-abort of a reused subId —
|
|
38
|
+
* subIds are UUIDs, so reuse is impossible anyway);
|
|
39
|
+
* - `abortAll` is idempotent (AbortController.abort() is idempotent).
|
|
40
|
+
*/
|
|
41
|
+
export class DelegateAbortRegistry {
|
|
42
|
+
private readonly controllers = new Map<string, { runId: string; controller: AbortController }>();
|
|
43
|
+
|
|
44
|
+
/** Number of in-flight grandchildren (diagnostics / tests). */
|
|
45
|
+
get size(): number {
|
|
46
|
+
return this.controllers.size;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Register a controller for `subId` (owned by `runId`) for the spawner. */
|
|
50
|
+
register(subId: string, runId: string): AbortController {
|
|
51
|
+
const controller = new AbortController();
|
|
52
|
+
this.controllers.set(subId, { runId, controller });
|
|
53
|
+
return controller;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Drop the controller (settle path). Never aborts. */
|
|
57
|
+
release(subId: string): void {
|
|
58
|
+
this.controllers.delete(subId);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Abort every in-flight grandchild of ONE run. Returns the count aborted. */
|
|
62
|
+
abortRun(runId: string, reason: DelegateAbortReason): number {
|
|
63
|
+
return this.abortWhere((entry) => entry.runId === runId, reason);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Abort every in-flight grandchild (broker teardown). Returns the count. */
|
|
67
|
+
abortAll(reason: DelegateAbortReason): number {
|
|
68
|
+
return this.abortWhere(() => true, reason);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
private abortWhere(predicate: (entry: { runId: string }) => boolean, reason: DelegateAbortReason): number {
|
|
72
|
+
const victims = [...this.controllers.entries()].filter(([, entry]) => predicate(entry));
|
|
73
|
+
for (const [subId, entry] of victims) {
|
|
74
|
+
// The reason rides the signal so the spawner can tell an owner abort
|
|
75
|
+
// (R43-N3 cancellation) from its own deadline timeout.
|
|
76
|
+
entry.controller.abort(new Error(reason.reason));
|
|
77
|
+
this.controllers.delete(subId);
|
|
78
|
+
}
|
|
79
|
+
return victims.length;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Process-local aborter: `runId` — abort every in-flight grandchild of that
|
|
85
|
+
* run. A run-scoped signature (not global) keeps a multi-run root session from
|
|
86
|
+
* killing another run's grandchildren; the registry itself is per broker but
|
|
87
|
+
* tracks runId per entry via the closure the lifecycle controller installs.
|
|
88
|
+
*/
|
|
89
|
+
export type DelegateAborter = (runId: string, reason: string) => number;
|
|
90
|
+
|
|
91
|
+
let activeAborter: DelegateAborter | undefined;
|
|
92
|
+
|
|
93
|
+
/** Register the active aborter (called by the broker lifecycle controller). */
|
|
94
|
+
export function setActiveDelegateAborter(aborter: DelegateAborter | undefined): void {
|
|
95
|
+
activeAborter = aborter;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Read the active aborter, if any (undefined = no broker wired). */
|
|
99
|
+
export function getActiveDelegateAborter(): DelegateAborter | undefined {
|
|
100
|
+
return activeAborter;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Abort in-flight grandchildren of `runId` through the active broker.
|
|
105
|
+
*
|
|
106
|
+
* Returns the number aborted (0 when no broker is wired — e.g. a headless
|
|
107
|
+
* runner without a root session). Never throws: this is called from terminal /
|
|
108
|
+
* abort paths that must not fail on a broker hiccup.
|
|
109
|
+
*/
|
|
110
|
+
export function abortRunDelegates(runId: string, reason: string): number {
|
|
111
|
+
const aborter = activeAborter;
|
|
112
|
+
if (!aborter) return 0;
|
|
113
|
+
try {
|
|
114
|
+
return aborter(runId, reason);
|
|
115
|
+
} catch {
|
|
116
|
+
return 0;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -19,7 +19,9 @@ export type DelegateEventType =
|
|
|
19
19
|
| "delegate.rejected"
|
|
20
20
|
| "delegate.completed"
|
|
21
21
|
| "delegate.timed_out"
|
|
22
|
-
| "delegate.
|
|
22
|
+
| "delegate.cancelled"
|
|
23
|
+
| "delegate.rolled_up"
|
|
24
|
+
| "delegate.rerouted";
|
|
23
25
|
|
|
24
26
|
export type DelegateEventTarget = { eventsPath: string; runId: string };
|
|
25
27
|
|