pi-crew 0.11.24 → 0.11.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.md +2 -2
- package/agents/designer.md +23 -0
- package/agents/orchestrator.md +17 -0
- package/agents/verifier.md +1 -1
- package/dist/index.mjs +568 -559
- package/{workflows/chain.workflow.md → docs/chain-workflow.md} +8 -0
- package/docs/dynamic-workflows.md +23 -0
- package/docs/publishing.md +34 -5
- package/docs/troubleshooting.md +50 -2
- package/docs/usage.md +7 -2
- package/install.mjs +2 -1
- package/package.json +4 -2
- package/skills/read-only-explorer/SKILL.md +15 -5
- package/skills/real-test-pi-crew/SKILL.md +7 -7
- package/src/config/config-validation.ts +58 -0
- package/src/config/config.ts +90 -5
- package/src/config/env-vars.ts +14 -7
- package/src/config/role-tools.ts +7 -5
- package/src/config/types.ts +4 -1
- package/src/extension/crew-cleanup.ts +6 -0
- package/src/extension/project-init.ts +19 -1
- package/src/extension/register.ts +68 -30
- package/src/extension/registration/artifact-cleanup.ts +52 -11
- package/src/extension/registration/commands/manage.ts +9 -2
- package/src/extension/registration/hook-registration.ts +3 -0
- package/src/extension/registration/lazy-configurers.ts +2 -0
- package/src/extension/registration/lifecycle-handlers.ts +79 -18
- package/src/extension/registration/render-loop.ts +27 -2
- package/src/extension/registration/tool-loop-guard.ts +5 -3
- package/src/extension/registration/verify-gate-tool.ts +7 -3
- package/src/extension/team-tool/api/read.ts +12 -3
- package/src/extension/team-tool/cancel.ts +11 -0
- package/src/extension/team-tool/config-patch.ts +56 -11
- package/src/extension/team-tool/destructive-gate.ts +72 -12
- package/src/extension/team-tool/dispatch/manage.ts +9 -1
- package/src/extension/team-tool/handle-schedule.ts +38 -4
- package/src/extension/team-tool/handle-settings.ts +76 -6
- package/src/extension/team-tool/health-monitor.ts +14 -2
- package/src/extension/team-tool/lifecycle-actions.ts +28 -10
- package/src/extension/team-tool/respond.ts +12 -5
- package/src/extension/team-tool/run-deadline.ts +29 -2
- package/src/extension/team-tool/run.ts +11 -0
- package/src/extension/team-tool/status.ts +64 -8
- package/src/extension/team-tool/workflow-manage.ts +5 -1
- package/src/extension/team-tool.ts +3 -0
- package/src/i18n.ts +18 -14
- package/src/prompt/inbox-poll.ts +14 -2
- package/src/prompt/message-tool.ts +1 -1
- package/src/prompt/prompt-runtime.ts +25 -3
- package/src/prompt/scratchpad-lifecycle.ts +14 -13
- package/src/prompt/surface-worker.ts +35 -9
- package/src/runtime/broker/crew-broker.ts +113 -111
- package/src/runtime/broker/delegate/delegate-abort.ts +118 -0
- package/src/runtime/broker/delegate/delegate-event.ts +3 -1
- package/src/runtime/broker/delegate/delegate-finalize.ts +179 -0
- package/src/runtime/broker/protocol/wait-resolve.ts +71 -5
- package/src/runtime/budget-enforcement.ts +26 -6
- package/src/runtime/chain-parser.ts +5 -3
- package/src/runtime/chain-runner.ts +41 -22
- package/src/runtime/child-pi/mock-fixtures.ts +12 -5
- package/src/runtime/compaction/tool-output-pruner.ts +8 -3
- package/src/runtime/delegate-spawn.ts +61 -8
- package/src/runtime/dispatch-batch.ts +10 -4
- package/src/runtime/effectiveness.ts +31 -3
- package/src/runtime/goal-workflow/plan-templates.ts +2 -2
- package/src/runtime/live-session/live-agent-control.ts +89 -6
- package/src/runtime/live-session/live-agent-manager.ts +94 -10
- package/src/runtime/live-session/live-session-runtime.ts +9 -2
- package/src/runtime/model/model-fallback.ts +6 -9
- package/src/runtime/model/pi-args.ts +3 -3
- package/src/runtime/model/runtime-policy.ts +4 -2
- package/src/runtime/model/runtime-resolver.ts +6 -2
- package/src/runtime/process/orphan-reaper.ts +143 -0
- package/src/runtime/process-status.ts +38 -16
- package/src/runtime/resilient-edit.ts +170 -93
- package/src/runtime/role-permission.ts +3 -1
- package/src/runtime/run-tracker.ts +9 -1
- package/src/runtime/sanitize/worker-text.ts +98 -0
- package/src/runtime/scheduling/run-coalesced-task-group.ts +82 -28
- package/src/runtime/spawn-policy.ts +190 -8
- package/src/runtime/stale-reconciler.ts +36 -1
- package/src/runtime/subagent-manager.ts +29 -0
- package/src/runtime/surface/herdr-provider.ts +31 -1
- package/src/runtime/task-output-context.ts +58 -11
- package/src/runtime/task-runner/post-execution.ts +150 -0
- package/src/runtime/task-runner/prompt-builder.ts +41 -8
- package/src/runtime/task-runner.ts +16 -3
- package/src/runtime/verification/classifier-pre-gate.ts +27 -6
- package/src/runtime/verification/verify-gate.ts +153 -30
- package/src/state/atomic-write.ts +73 -4
- package/src/state/contracts.ts +7 -0
- package/src/state/coordination/mailbox.ts +40 -11
- package/src/state/event-log/event-log-rotation.ts +28 -6
- package/src/state/event-log/poison-marker.ts +16 -1
- package/src/state/stores/artifact-store.ts +181 -6
- package/src/state/stores/manifest-io.ts +126 -8
- package/src/state/stores/state-store.ts +30 -31
- package/src/state/types.ts +13 -0
- package/src/state/usage.ts +41 -0
- package/src/ui/run-dashboard.ts +3 -3
- package/src/ui/settings-overlay.ts +19 -1
- package/src/utils/internal-error.ts +5 -1
- package/src/utils/timer-limits.ts +30 -0
- package/src/utils/token-counter.ts +47 -0
- package/src/workflows/discover-workflows.ts +9 -1
- package/src/worktree/worktree-manager.ts +128 -18
- package/workflows/default.workflow.md +1 -1
- package/workflows/fast-fix.workflow.md +1 -1
- package/workflows/plan-execute.workflow.md +1 -1
- package/workflows/review.workflow.md +1 -1
- package/workflows/strict-fast-fix.workflow.md +1 -1
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* delegate-finalize.ts — broker-side SETTLE path for a delegate grandchild
|
|
3
|
+
* (ADR-5 §1/§5). Moved out of the `delegate.request` handler's fire-and-forget
|
|
4
|
+
* async IIFE (crew-broker.ts) by fix-wave 6 — behaviour change, not pure code
|
|
5
|
+
* motion: R43-N4 + R44-N4 are fixed here.
|
|
6
|
+
*
|
|
7
|
+
* R43-N4 (2026-10-19): the grandchild's fenced result used to be appended to
|
|
8
|
+
* the PARENT task's mailbox unconditionally. When the parent had already
|
|
9
|
+
* reached a terminal status (cancelled by a budget abort, failed, killed) that
|
|
10
|
+
* row was written into a mailbox nobody reads again — the result was silently
|
|
11
|
+
* dropped, permanently. {@link delegateResultTarget} now decides the
|
|
12
|
+
* destination from the parent record read INSIDE the run lock: a terminal (or
|
|
13
|
+
* missing) parent redirects the result to the RUN-level inbox (the
|
|
14
|
+
* orchestrator/leader channel) with `data.reroutedFrom` + a
|
|
15
|
+
* `delegate.rerouted` event, so a dead parent can no longer swallow an
|
|
16
|
+
* in-flight grandchild's output.
|
|
17
|
+
*
|
|
18
|
+
* R44-N4 (2026-10-19): usage roll-up is now BROKER-SIDE. Before this change
|
|
19
|
+
* the reservation written at admission was reconciled against the CHILD's own
|
|
20
|
+
* `message_end` self-report (`actual = clamp(selfReported, 0, reserved)`), so a
|
|
21
|
+
* child that under-reported refunded almost its whole reservation — the budget
|
|
22
|
+
* charge was self-attested (`probe-n4`: under-report refunded 49 999 of a
|
|
23
|
+
* 50 000 reserve). The charge is now the HOST-decided reservation, full stop:
|
|
24
|
+
* the admission reserve IS the spend, and nothing at settle time can reduce
|
|
25
|
+
* it. The child's number is still recorded on `delegate.rolled_up`
|
|
26
|
+
* (`selfReportedTokens`) for observability only — never as a charge.
|
|
27
|
+
* Consequence: the P2-8c "best-effort refund when finalize fails" path is gone
|
|
28
|
+
* with it (a failed finalize must NOT hand back budget the run may have
|
|
29
|
+
* consumed — the conservative direction).
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { isTerminalTaskStatus } from "../../../state/contracts.ts";
|
|
33
|
+
import { withRunLockSync } from "../../../state/coordination/locks.ts";
|
|
34
|
+
import { appendMailboxMessageAsync } from "../../../state/coordination/mailbox.ts";
|
|
35
|
+
import { loadRunManifestById, saveRunTasks } from "../../../state/stores/state-store.ts";
|
|
36
|
+
import type { TeamTaskState } from "../../../state/types.ts";
|
|
37
|
+
import { logInternalError } from "../../../utils/internal-error.ts";
|
|
38
|
+
import { recordDelegateEvent } from "./delegate-event.ts";
|
|
39
|
+
|
|
40
|
+
/** Destination of a settled grandchild's fenced result. */
|
|
41
|
+
export type DelegateResultTarget = "parent" | "run-mailbox";
|
|
42
|
+
|
|
43
|
+
export interface DelegateFinalizeInput {
|
|
44
|
+
/** Broker cwd (manifest lookup). */
|
|
45
|
+
brokerCwd: string;
|
|
46
|
+
runId: string;
|
|
47
|
+
parentTaskId: string;
|
|
48
|
+
/** Grandchild shadow-task id (`gc-*`) — also the mailbox `from` suffix. */
|
|
49
|
+
subId: string;
|
|
50
|
+
/** Already-fenced result body (caller neutralizes smuggled fence markers). */
|
|
51
|
+
fenced: string;
|
|
52
|
+
ok: boolean;
|
|
53
|
+
timedOut?: boolean;
|
|
54
|
+
/** Pessimistic reservation written at admission (0 = unmetered request).
|
|
55
|
+
* THIS is the broker-side charge — see the R44-N4 note above. */
|
|
56
|
+
reservedTokens: number;
|
|
57
|
+
/** Child-emitted usage, AUDIT ONLY (self-attested, never charged). */
|
|
58
|
+
selfReportedTokens?: number;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface DelegateFinalizeOutcome {
|
|
62
|
+
/** Where the result went, or "none" when the run record was gone. */
|
|
63
|
+
delivery: DelegateResultTarget | "none";
|
|
64
|
+
/** Broker-side charge applied to the parent allocation (unit: TOKENS). */
|
|
65
|
+
chargedTokens: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* R43-N4: pick the result destination from the parent task RECORD.
|
|
70
|
+
*
|
|
71
|
+
* - parent still live (queued/running/waiting) → its own mailbox (unchanged);
|
|
72
|
+
* - parent gone or terminal (completed/failed/cancelled/skipped/…) → the run
|
|
73
|
+
* mailbox, i.e. the leader channel. Terminal includes `completed`: a parent
|
|
74
|
+
* that finished before its grandchild settled cannot consume the row either.
|
|
75
|
+
*/
|
|
76
|
+
export function delegateResultTarget(parentStatus: TeamTaskState["status"] | undefined): DelegateResultTarget {
|
|
77
|
+
if (parentStatus === undefined) return "run-mailbox";
|
|
78
|
+
return isTerminalTaskStatus(parentStatus) ? "run-mailbox" : "parent";
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** M3-5/R43-N3: the terminal event for a settled grandchild — owner-abort is
|
|
82
|
+
* distinct from the grandchild's own deadline in the audit trail. */
|
|
83
|
+
export function delegateTerminalEvent(outcome: {
|
|
84
|
+
ok: boolean;
|
|
85
|
+
timedOut?: boolean;
|
|
86
|
+
cancelled?: boolean;
|
|
87
|
+
}): "delegate.completed" | "delegate.timed_out" | "delegate.cancelled" {
|
|
88
|
+
if (outcome.cancelled === true) return "delegate.cancelled";
|
|
89
|
+
return outcome.timedOut === true ? "delegate.timed_out" : "delegate.completed";
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Settle a grandchild: deliver the fenced result, charge the budget, terminal-
|
|
94
|
+
* flip the shadow record. All writes happen INSIDE one run lock, computed off
|
|
95
|
+
* a fresh disk read (never a pre-spawn snapshot).
|
|
96
|
+
*
|
|
97
|
+
* Never throws: every failure is logged (the caller is a detached async task —
|
|
98
|
+
* a throw here would become an unhandled rejection).
|
|
99
|
+
*/
|
|
100
|
+
export function finalizeDelegateResult(input: DelegateFinalizeInput): DelegateFinalizeOutcome {
|
|
101
|
+
try {
|
|
102
|
+
const loaded = loadRunManifestById(input.brokerCwd, input.runId);
|
|
103
|
+
if (!loaded) return { delivery: "none", chargedTokens: 0 };
|
|
104
|
+
// Boxed state — read back AFTER the lock closure (a plain `let` would be
|
|
105
|
+
// narrowed to its initial literal by control-flow analysis).
|
|
106
|
+
const state: { delivery: DelegateResultTarget | "none" } = { delivery: "none" };
|
|
107
|
+
withRunLockSync(loaded.manifest, () => {
|
|
108
|
+
const latest = loadRunManifestById(input.brokerCwd, input.runId);
|
|
109
|
+
if (!latest) return;
|
|
110
|
+
const parent = latest.tasks.find((t) => t.id === input.parentTaskId);
|
|
111
|
+
const delivery = delegateResultTarget(parent?.status);
|
|
112
|
+
state.delivery = delivery;
|
|
113
|
+
// Durable delivery (ADR-5 §1): fenced result into the parent task's
|
|
114
|
+
// mailbox, or into the run inbox when the parent can no longer read
|
|
115
|
+
// it (R43-N4). `appendMailboxMessageAsync` is fire-and-forget here —
|
|
116
|
+
// the append itself takes the mailbox file lock, and blocking the run
|
|
117
|
+
// lock on it would serialize every settle behind disk I/O.
|
|
118
|
+
void appendMailboxMessageAsync(latest.manifest, {
|
|
119
|
+
direction: "inbox",
|
|
120
|
+
from: `delegate:${input.subId}`,
|
|
121
|
+
to: delivery === "parent" ? input.parentTaskId : "leader",
|
|
122
|
+
...(delivery === "parent" ? { taskId: input.parentTaskId } : {}),
|
|
123
|
+
body: input.fenced,
|
|
124
|
+
kind: "response",
|
|
125
|
+
data: {
|
|
126
|
+
subId: input.subId,
|
|
127
|
+
ok: input.ok,
|
|
128
|
+
timedOut: input.timedOut === true,
|
|
129
|
+
...(delivery === "parent" ? {} : { reroutedFrom: input.parentTaskId, reason: "parent-terminal" }),
|
|
130
|
+
},
|
|
131
|
+
}).catch((err) =>
|
|
132
|
+
logInternalError(
|
|
133
|
+
"crew-broker.delegate.mailbox",
|
|
134
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
135
|
+
`runId=${input.runId} subId=${input.subId} delivery=${delivery}`,
|
|
136
|
+
),
|
|
137
|
+
);
|
|
138
|
+
if (delivery === "run-mailbox") {
|
|
139
|
+
recordDelegateEvent(latest.manifest, "delegate.rerouted", input.parentTaskId, {
|
|
140
|
+
subId: input.subId,
|
|
141
|
+
parentStatus: parent?.status ?? "missing",
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
// R44-N4: no refund, no reconciliation write — the admission reserve
|
|
145
|
+
// is the charge, so `allocation.tokensSpent` already holds it on disk.
|
|
146
|
+
// Only audit the (untrusted) child number.
|
|
147
|
+
if (input.reservedTokens > 0) {
|
|
148
|
+
recordDelegateEvent(latest.manifest, "delegate.rolled_up", input.parentTaskId, {
|
|
149
|
+
subId: input.subId,
|
|
150
|
+
chargedTokens: input.reservedTokens,
|
|
151
|
+
...(input.selfReportedTokens !== undefined ? { selfReportedTokens: input.selfReportedTokens } : {}),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
// S1#1 (verifier N1): the terminal flip is UNCONDITIONAL — never
|
|
155
|
+
// depends on a reservation. RR-012 F16: onSpawn promoted the record to
|
|
156
|
+
// "running" during execution; this flip terminalizes every settle path.
|
|
157
|
+
saveRunTasks(
|
|
158
|
+
latest.manifest,
|
|
159
|
+
latest.tasks.map((t) =>
|
|
160
|
+
t.id === input.subId
|
|
161
|
+
? {
|
|
162
|
+
...t,
|
|
163
|
+
status: input.ok ? ("completed" as const) : ("failed" as const),
|
|
164
|
+
completedAt: new Date().toISOString(),
|
|
165
|
+
}
|
|
166
|
+
: t,
|
|
167
|
+
),
|
|
168
|
+
);
|
|
169
|
+
});
|
|
170
|
+
return { delivery: state.delivery, chargedTokens: input.reservedTokens };
|
|
171
|
+
} catch (err) {
|
|
172
|
+
logInternalError(
|
|
173
|
+
"crew-broker.delegate.finalize",
|
|
174
|
+
err instanceof Error ? err : new Error(String(err)),
|
|
175
|
+
`runId=${input.runId} subId=${input.subId}`,
|
|
176
|
+
);
|
|
177
|
+
return { delivery: "none", chargedTokens: 0 };
|
|
178
|
+
}
|
|
179
|
+
}
|
|
@@ -17,11 +17,76 @@
|
|
|
17
17
|
import { withRunLockSync } from "../../../state/coordination/locks.ts";
|
|
18
18
|
import { appendEventAsync } from "../../../state/event-log/event-log.ts";
|
|
19
19
|
import { loadRunManifestById, saveRunManifest, saveRunTasks } from "../../../state/stores/state-store.ts";
|
|
20
|
+
import type { TeamRunManifest, TeamTaskState } from "../../../state/types.ts";
|
|
20
21
|
import { logInternalError } from "../../../utils/internal-error.ts";
|
|
21
22
|
import type { ServerConnection } from "./connection-state.ts";
|
|
22
23
|
import { parseWaitResolveParams } from "./request-parsers.ts";
|
|
23
24
|
import { recordWaitPolicyRejection } from "./wait-auth.ts";
|
|
24
25
|
|
|
26
|
+
/**
|
|
27
|
+
* M3-6 (R21-M1, fix-wave 6): resolve ONE park without un-shielding the others.
|
|
28
|
+
*
|
|
29
|
+
* `manifest.waitState` is a single run-level slot, but more than one task can
|
|
30
|
+
* be parked at the same time (concurrent `ask` calls / goal-loop + a worker).
|
|
31
|
+
* `isWaitAnswerPending(manifest)` reads ONLY this slot, and it is what keeps the
|
|
32
|
+
* stale-reconciler / crash-recovery from repairing (killing) a live worker that
|
|
33
|
+
* stopped heartbeating while parked. The pre-fix resolve cleared the slot
|
|
34
|
+
* whenever it pointed at the resolved park — so once the SECOND park resolved,
|
|
35
|
+
* the FIRST (still parked) task lost its shield entirely and could be
|
|
36
|
+
* stale-reaped while awaiting a human answer.
|
|
37
|
+
*
|
|
38
|
+
* Fix (single-slot contract preserved, no schema change): when the slot points at
|
|
39
|
+
* the park we just ended, RE-POINT it at the still-parked task whose shield must
|
|
40
|
+
* survive instead of blindly clearing it. The latest deadline wins so the run
|
|
41
|
+
* shield never expires while ANY park is still live (expiring early would
|
|
42
|
+
* re-open the same window; living slightly long only delays repair — the
|
|
43
|
+
* fail-safe direction the rest of this module already takes). A slot that
|
|
44
|
+
* belongs to a DIFFERENT (newer) park is left untouched, exactly as before.
|
|
45
|
+
*
|
|
46
|
+
* M4-R1(b) (fix-wave 6 residual, same class as M3-6): the identical
|
|
47
|
+
* clear-not-repoint bug lived in the SIBLING slot writers — respond.ts's
|
|
48
|
+
* requeue path (respond.ts:250-251) and dispatch-batch.ts's ask-timeout
|
|
49
|
+
* requeue (:198-199). Both scoped the CLEAR by taskId (WP-2 round-1 P3), but
|
|
50
|
+
* when the ended park owned the slot and a THIRD task was still parked they
|
|
51
|
+
* dropped the slot, re-opening the un-shielded window M3-6 had closed for
|
|
52
|
+
* wait.resolve. The helper is exported (predicate form, so the sibling writers
|
|
53
|
+
* can pass their requeued-id SET) and reused verbatim there.
|
|
54
|
+
*/
|
|
55
|
+
export function repointWaitState(
|
|
56
|
+
tasks: TeamTaskState[],
|
|
57
|
+
current: TeamRunManifest["waitState"],
|
|
58
|
+
isEnded: (taskId: string, questionId: string) => boolean,
|
|
59
|
+
): TeamRunManifest["waitState"] {
|
|
60
|
+
if (current === undefined) return undefined;
|
|
61
|
+
if (!isEnded(current.taskId, current.questionId)) return current;
|
|
62
|
+
let next: TeamTaskState | undefined;
|
|
63
|
+
for (const task of tasks) {
|
|
64
|
+
if (task.status !== "waiting" || task.waiting === undefined) continue;
|
|
65
|
+
if (isEnded(task.id, task.waiting.questionId)) continue;
|
|
66
|
+
if (next === undefined || task.waiting.deadline > (next.waiting?.deadline ?? 0)) next = task;
|
|
67
|
+
}
|
|
68
|
+
if (next?.waiting === undefined) return undefined;
|
|
69
|
+
return {
|
|
70
|
+
taskId: next.id,
|
|
71
|
+
questionId: next.waiting.questionId,
|
|
72
|
+
askedAt: next.waiting.askedAt,
|
|
73
|
+
deadline: next.waiting.deadline,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** wait.resolve's single-park view of {@link repointWaitState}. */
|
|
78
|
+
function repointedWaitState(
|
|
79
|
+
tasks: TeamTaskState[],
|
|
80
|
+
current: TeamRunManifest["waitState"],
|
|
81
|
+
endedTaskId: string,
|
|
82
|
+
endedQuestionId: string,
|
|
83
|
+
): TeamRunManifest["waitState"] {
|
|
84
|
+
return repointWaitState(tasks, current, (taskId, questionId) => taskId === endedTaskId && questionId === endedQuestionId);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** @internal M3-6 test export — exercise the scoped-clear repoint directly. */
|
|
88
|
+
export const __test__repointedWaitState = repointedWaitState;
|
|
89
|
+
|
|
25
90
|
export interface WaitResolveHelpers {
|
|
26
91
|
sendError(conn: ServerConnection, id: string, code: string, message: string): void;
|
|
27
92
|
sendResult(conn: ServerConnection, id: string, result: unknown): void;
|
|
@@ -100,13 +165,14 @@ export async function handleWaitResolve(
|
|
|
100
165
|
// the flip, so respond_delivered records the same worker-pid evidence
|
|
101
166
|
// its respond_missed twin does root-side.
|
|
102
167
|
const workerPid = task.heartbeat?.pid ?? task.checkpoint?.childPid;
|
|
103
|
-
// waitState is a single run-level slot:
|
|
104
|
-
//
|
|
105
|
-
|
|
106
|
-
|
|
168
|
+
// waitState is a single run-level slot: never clobber another task's newer
|
|
169
|
+
// park. M3-6 (R21-M1): and never un-shield a task that is STILL parked —
|
|
170
|
+
// when the slot pointed at the park we are ending, re-point it at the
|
|
171
|
+
// surviving park (repointedWaitState) instead of clearing it.
|
|
172
|
+
const waitState = repointedWaitState(fresh.tasks, fresh.manifest.waitState, taskId, parsed.questionId);
|
|
107
173
|
const updatedManifest = {
|
|
108
174
|
...fresh.manifest,
|
|
109
|
-
|
|
175
|
+
waitState,
|
|
110
176
|
updatedAt: new Date().toISOString(),
|
|
111
177
|
};
|
|
112
178
|
saveRunTasks(updatedManifest, updatedTasks);
|
|
@@ -19,8 +19,9 @@ import { flushPendingAtomicWrites } from "../state/atomic-write.ts";
|
|
|
19
19
|
import { appendEventAsync } from "../state/event-log/event-log.ts";
|
|
20
20
|
import { loadRunManifestById, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/stores/state-store.ts";
|
|
21
21
|
import type { TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
22
|
-
import {
|
|
22
|
+
import { formatTokens, taskTokenTotal } from "../state/usage.ts";
|
|
23
23
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
24
|
+
import { abortRunDelegates } from "./broker/delegate/delegate-abort.ts";
|
|
24
25
|
import { MERGE_LOCK_BUSY_RETRY_DELAYS_MS, withRunLockBusyRetryAsync } from "./broker/protocol/lock-busy.ts";
|
|
25
26
|
import { saveCrewAgents } from "./crew-agent-records.ts";
|
|
26
27
|
import { cancelNonTerminalTasks, markBlocked } from "./dispatch-batch.ts";
|
|
@@ -47,6 +48,16 @@ export interface PerTaskBudgetCheckResult {
|
|
|
47
48
|
* Check cumulative token usage against per-task budget thresholds.
|
|
48
49
|
* Returns a structured result — callers decide how to act (warn vs abort).
|
|
49
50
|
*
|
|
51
|
+
* M3-4 / R43-N2 (2026-10-19 fix-wave 6): `totalUsed` is now the FULL run cost
|
|
52
|
+
* — each task's own tokens PLUS the delegate spend booked on it
|
|
53
|
+
* (`taskTokenTotal`, i.e. `allocation.tokensSpent`). Pre-fix the check summed
|
|
54
|
+
* `task.usage` only, and grandchildren never write `task.usage` (they are
|
|
55
|
+
* broker-managed shadow records), so a nested delegate storm could burn an
|
|
56
|
+
* unbounded multiple of `budgetTotal` without ever tripping the abort
|
|
57
|
+
* (probe-verified round 43). The delegate amount is the BROKER-SIDE tally —
|
|
58
|
+
* the host-decided admission reservation — not the child's self-report
|
|
59
|
+
* (R44-N4), so an under-reporting child cannot hide its spend.
|
|
60
|
+
*
|
|
50
61
|
* Exported for unit testing.
|
|
51
62
|
*/
|
|
52
63
|
export function checkPerTaskBudget(
|
|
@@ -56,8 +67,8 @@ export function checkPerTaskBudget(
|
|
|
56
67
|
budgetAbort: number,
|
|
57
68
|
fairShareFraction = 0.5,
|
|
58
69
|
): PerTaskBudgetCheckResult {
|
|
59
|
-
|
|
60
|
-
const
|
|
70
|
+
let totalUsed = 0;
|
|
71
|
+
for (const task of tasks) totalUsed += taskTokenTotal(task);
|
|
61
72
|
const abort = totalUsed >= budgetAbort * budgetTotal;
|
|
62
73
|
const warning = !abort && totalUsed >= budgetWarning * budgetTotal;
|
|
63
74
|
// Fair share threshold based on TOTAL budget, not remaining budget.
|
|
@@ -66,8 +77,9 @@ export function checkPerTaskBudget(
|
|
|
66
77
|
const fairShareThreshold = budgetTotal * fairShareFraction;
|
|
67
78
|
const fairShareViolators: string[] = [];
|
|
68
79
|
for (const task of tasks) {
|
|
69
|
-
|
|
70
|
-
const taskTotal = (task
|
|
80
|
+
// M3-4: a delegate-heavy parent is charged for its grandchildren too.
|
|
81
|
+
const taskTotal = taskTokenTotal(task);
|
|
82
|
+
if (taskTotal === 0) continue;
|
|
71
83
|
// Only flag tasks that individually consumed a significant portion of the
|
|
72
84
|
// budget (>10% of total) AND exceeded the fair share threshold.
|
|
73
85
|
if (fairShareThreshold > 0 && taskTotal > fairShareThreshold && taskTotal > budgetTotal * 0.1) {
|
|
@@ -128,6 +140,12 @@ export async function terminaliseRunWithDrain(
|
|
|
128
140
|
// and be clobbered to "skipped" (observed CI flake: 02_b skipped on
|
|
129
141
|
// team-runner-budget-abort-inflight across v0.9.59 / cfd68d06 / 12386af2).
|
|
130
142
|
const inflightTaskIds = ctx.dispatchedTaskIds;
|
|
143
|
+
// M3-5/R43-N3: a run going terminal must also cancel its in-flight DELEGATE
|
|
144
|
+
// grandchildren. This drain only covers the run's own dispatch units; the
|
|
145
|
+
// grandchildren live in the broker's abort registry (fire-and-forget spawn
|
|
146
|
+
// before the fix, so they outlived every abort path by up to their own 24h
|
|
147
|
+
// deadline while still spending and holding nested slots).
|
|
148
|
+
abortRunDelegates(ctx.manifest.runId, opts.failedReason.slice(0, 120) || "run-terminal");
|
|
131
149
|
const outcomes = await drainPendingUnits(ctx.pendingUnits, ctx.runController);
|
|
132
150
|
const validResults: { manifest: TeamRunManifest; tasks: TeamTaskState[] }[] = [];
|
|
133
151
|
for (const outcome of outcomes) {
|
|
@@ -271,7 +289,9 @@ export async function enforceRunBudget(ctx: SchedulerContext): Promise<Scheduler
|
|
|
271
289
|
for (const violatorId of budgetCheck.fairShareViolators) {
|
|
272
290
|
const violator = tasks.find((t) => t.id === violatorId);
|
|
273
291
|
if (!violator) continue;
|
|
274
|
-
|
|
292
|
+
// M3-4/R43-N2: same full-cost rule as the threshold check (own usage
|
|
293
|
+
// + booked delegate spend) so the message matches the decision.
|
|
294
|
+
const taskTotal = taskTokenTotal(violator);
|
|
275
295
|
const message = `Task '${violatorId}' consumed ${formatTokens(taskTotal)} (${Math.round((taskTotal / input.budgetTotal) * 100)}% of total budget) — exceeds fair share`;
|
|
276
296
|
// ROUND 12: console → logInternalError, explicit "warn" severity.
|
|
277
297
|
logInternalError("team-runner.fair-share", new Error(message), `runId=${manifest.runId} taskId=${violatorId}`, "warn");
|
|
@@ -125,10 +125,12 @@ function tokenize(input: string): Token[] {
|
|
|
125
125
|
continue;
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
-
// Name
|
|
129
|
-
|
|
128
|
+
// Name (a leading '@' is part of a @team reference — R14-C1: without this
|
|
129
|
+
// the tokenizer threw on '@', the whole DSL parse fell back to the simple
|
|
130
|
+
// splitter, and parallel(@a, @b) silently degraded into garbage steps.)
|
|
131
|
+
if (/[a-zA-Z_]/.test(input[i]!) || (input[i] === "@" && /[a-zA-Z_]/.test(input[i + 1] ?? ""))) {
|
|
130
132
|
let name = "";
|
|
131
|
-
while (i < input.length && /[a-zA-Z0-9_
|
|
133
|
+
while (i < input.length && /[a-zA-Z0-9_.@-]/.test(input[i]!)) {
|
|
132
134
|
name += input[i];
|
|
133
135
|
i++;
|
|
134
136
|
}
|
|
@@ -182,6 +182,9 @@ export class ChainRunner {
|
|
|
182
182
|
const startTime = Date.now();
|
|
183
183
|
let totalTokens = 0;
|
|
184
184
|
const allHandoffs: HandoffSummary[] = [];
|
|
185
|
+
// R14-C5: a break leaves later steps UNEXECUTED. Tracked explicitly so the
|
|
186
|
+
// success predicate cannot report a truncated chain as complete.
|
|
187
|
+
let brokeEarly = false;
|
|
185
188
|
|
|
186
189
|
for (let i = 0; i < spec.steps.length; i++) {
|
|
187
190
|
const step = spec.steps[i];
|
|
@@ -228,6 +231,7 @@ export class ChainRunner {
|
|
|
228
231
|
} else {
|
|
229
232
|
// Stop chain on step failure unless configured to continue
|
|
230
233
|
if (!spec.continueOnError && !step.continueOnError) {
|
|
234
|
+
brokeEarly = true;
|
|
231
235
|
break;
|
|
232
236
|
}
|
|
233
237
|
}
|
|
@@ -261,15 +265,25 @@ export class ChainRunner {
|
|
|
261
265
|
|
|
262
266
|
// Stop chain on failure unless configured to continue
|
|
263
267
|
if (!spec.continueOnError && !step.continueOnError) {
|
|
268
|
+
brokeEarly = true;
|
|
264
269
|
break;
|
|
265
270
|
}
|
|
266
271
|
}
|
|
267
272
|
}
|
|
268
273
|
|
|
274
|
+
// R14-C5 (false-green): the old predicate was `every(outcome !== "failure")`,
|
|
275
|
+
// so a `partial` step — which breaks the chain exactly like a failure — still
|
|
276
|
+
// produced success:true, and dispatch printed "Chain completed" + status ok
|
|
277
|
+
// for a chain whose remaining steps never ran. An empty spec was vacuously
|
|
278
|
+
// green for the same reason. Success now requires: at least one step, no
|
|
279
|
+
// early break, every planned step executed, and every outcome `success`.
|
|
280
|
+
const allStepsExecuted = stepResults.length === spec.steps.length;
|
|
281
|
+
const allOutcomesSuccessful = stepResults.every((s) => s.outcome === "success");
|
|
282
|
+
|
|
269
283
|
return {
|
|
270
284
|
steps: stepResults,
|
|
271
285
|
totalDuration: Date.now() - startTime,
|
|
272
|
-
success:
|
|
286
|
+
success: spec.steps.length > 0 && !brokeEarly && allStepsExecuted && allOutcomesSuccessful,
|
|
273
287
|
totalTokens: totalTokens > 0 ? totalTokens : undefined,
|
|
274
288
|
totalHandoffs: allHandoffs,
|
|
275
289
|
};
|
|
@@ -403,34 +417,39 @@ export class ChainRunner {
|
|
|
403
417
|
|
|
404
418
|
/**
|
|
405
419
|
* Convert DSL AST steps (from chain-parser) to ChainSpec.
|
|
420
|
+
*
|
|
421
|
+
* R14-C1: `parallel(...)` groups are FLATTENED into ordered steps. They used to
|
|
422
|
+
* be parked in `step.context.parallel`, which no consumer ever read — the whole
|
|
423
|
+
* group collapsed into one step literally named "parallel" and never ran its
|
|
424
|
+
* members. Chain execution is sequential by contract (see runChain), so the
|
|
425
|
+
* members run in source order and each keeps its own step result, which means a
|
|
426
|
+
* failing member stays visible instead of vanishing into a dropped group.
|
|
406
427
|
*/
|
|
407
428
|
private dslToChainSpec(dslSteps: DSLChainStep[], chainString: string): ChainSpec {
|
|
408
|
-
const steps: ChainStep[] =
|
|
409
|
-
|
|
429
|
+
const steps: ChainStep[] = [];
|
|
430
|
+
|
|
431
|
+
// Recursive so nested groups (parallel(a, parallel(b, c))) flatten too.
|
|
432
|
+
const pushStep = (dslStep: DSLChainStep): void => {
|
|
410
433
|
if (dslStep.parallel) {
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
context: {
|
|
414
|
-
parallel: dslStep.parallel.map((p) => ({
|
|
415
|
-
name: p.name,
|
|
416
|
-
loopCount: p.loopCount,
|
|
417
|
-
withContext: p.withContext,
|
|
418
|
-
args: p.args,
|
|
419
|
-
})),
|
|
420
|
-
},
|
|
421
|
-
loopCount: dslStep.loopCount,
|
|
422
|
-
};
|
|
434
|
+
for (const member of dslStep.parallel) pushStep(member);
|
|
435
|
+
return;
|
|
423
436
|
}
|
|
437
|
+
|
|
424
438
|
const step: ChainStep = { name: dslStep.name };
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
439
|
+
|
|
440
|
+
// '@team' reference — same resolution the simple parser applies to @refs.
|
|
441
|
+
if (dslStep.name.startsWith("@")) {
|
|
442
|
+
const team = this.sanitizeIdentifier(dslStep.name.slice(1));
|
|
443
|
+
if (team.length > 0) step.team = team;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
if (dslStep.loopCount) step.context = { ...step.context, loopCount: dslStep.loopCount };
|
|
430
447
|
if (dslStep.withContext) step.context = { ...step.context, withContext: true };
|
|
431
448
|
if (dslStep.args && dslStep.args.length > 0) step.context = { ...step.context, args: dslStep.args };
|
|
432
|
-
|
|
433
|
-
}
|
|
449
|
+
steps.push(step);
|
|
450
|
+
};
|
|
451
|
+
|
|
452
|
+
for (const dslStep of dslSteps) pushStep(dslStep);
|
|
434
453
|
|
|
435
454
|
// Extract global overrides using existing logic
|
|
436
455
|
const globalModel = this.extractGlobalFlag(chainString, "global-model");
|
|
@@ -49,8 +49,15 @@ export async function runMockChildPi(
|
|
|
49
49
|
// SECURITY: Log mock mode activation prominently for audit trail
|
|
50
50
|
logInternalError("child-pi.mock", new Error(`Mock mode active: ${mock}`), "NOT running real agents");
|
|
51
51
|
|
|
52
|
+
// M3-2 (R47-3/V4): the verifier output contract requires a PASS:/FAIL:
|
|
53
|
+
// verdict line; happy-path mocks must emit contract-shaped output for
|
|
54
|
+
// verifier-role tasks or the (now real) verdict consumer marks the evidence
|
|
55
|
+
// unverified and the green contract blocks the run (fail-closed — correct
|
|
56
|
+
// for real verifiers, wrong for success mocks).
|
|
57
|
+
const verdictSuffix = input.role === "verifier" ? "\nPASS: mock verifier — mock success" : "";
|
|
58
|
+
|
|
52
59
|
if (mock === "success") {
|
|
53
|
-
const stdout = `[MOCK] Success for ${input.agent.name}\n`;
|
|
60
|
+
const stdout = `[MOCK] Success for ${input.agent.name}${verdictSuffix}\n`;
|
|
54
61
|
await observe(input, stdout);
|
|
55
62
|
return { exitCode: 0, stdout, stderr: "" };
|
|
56
63
|
}
|
|
@@ -62,7 +69,7 @@ export async function runMockChildPi(
|
|
|
62
69
|
// (default 1500ms, cap 5000ms) via PI_TEAMS_MOCK_STEER_WINDOW_MS.
|
|
63
70
|
const windowMs = Number(getCrewEnv("PI_TEAMS_MOCK_STEER_WINDOW_MS") ?? "1500");
|
|
64
71
|
await new Promise((resolve) => setTimeout(resolve, Math.min(windowMs, 5000)));
|
|
65
|
-
const text = `[MOCK] JSON success for ${input.agent.name}`;
|
|
72
|
+
const text = `[MOCK] JSON success for ${input.agent.name}${verdictSuffix}`;
|
|
66
73
|
const stdout = `${JSON.stringify({ type: "message", message: { role: "assistant", content: [{ type: "text", text }] } })}\n${JSON.stringify({ type: "message_end", usage: { input: 10, output: 5, cost: 0.001, turns: 1 } })}\n`;
|
|
67
74
|
await observe(input, stdout);
|
|
68
75
|
return { exitCode: 0, stdout, stderr: "" };
|
|
@@ -141,7 +148,7 @@ export async function runMockChildPi(
|
|
|
141
148
|
},
|
|
142
149
|
],
|
|
143
150
|
})}\nADAPTIVE_PLAN_JSON_END`
|
|
144
|
-
: `[MOCK] JSON success for ${input.agent.name}`;
|
|
151
|
+
: `[MOCK] JSON success for ${input.agent.name}${verdictSuffix}`;
|
|
145
152
|
const stdout = `${JSON.stringify({ type: "message", message: { role: "assistant", content: [{ type: "text", text }] } })}\n${JSON.stringify({ type: "message_end", usage: { input: 10, output: 5, cost: 0.001, turns: 1 } })}\n`;
|
|
146
153
|
await observe(input, stdout);
|
|
147
154
|
return { exitCode: 0, stdout, stderr: "" };
|
|
@@ -162,7 +169,7 @@ export async function runMockChildPi(
|
|
|
162
169
|
// parsers see exactly what a real child pi emits. Two events so the
|
|
163
170
|
// accumulation (sum) across messages is asserted, not a single snapshot.
|
|
164
171
|
if (mock === "json-usage-ledger") {
|
|
165
|
-
const text = `[MOCK] JSON usage ledger for ${input.agent.name}`;
|
|
172
|
+
const text = `[MOCK] JSON usage ledger for ${input.agent.name}${verdictSuffix}`;
|
|
166
173
|
const usageEvent = (tokens: { input: number; output: number; cacheRead: number; cacheWrite: number }, cost: number) =>
|
|
167
174
|
JSON.stringify({
|
|
168
175
|
type: "message_end",
|
|
@@ -282,7 +289,7 @@ export async function runMockChildPi(
|
|
|
282
289
|
}
|
|
283
290
|
// Subsequent invocations: delegate to json-success shape so the
|
|
284
291
|
// fallback chain's second attempt succeeds and the run completes.
|
|
285
|
-
const text = `[MOCK] JSON success for ${input.agent.name}`;
|
|
292
|
+
const text = `[MOCK] JSON success for ${input.agent.name}${verdictSuffix}`;
|
|
286
293
|
const stdout = `${JSON.stringify({ type: "message", message: { role: "assistant", content: [{ type: "text", text }] } })}\n${JSON.stringify({ type: "message_end", usage: { input: 10, output: 5, cost: 0.001, turns: 1 } })}\n`;
|
|
287
294
|
await observe(input, stdout);
|
|
288
295
|
return { exitCode: 0, stdout, stderr: "" };
|
|
@@ -95,13 +95,18 @@ export interface PruneResult {
|
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
// ---------------------------------------------------------------------------
|
|
98
|
-
// Token estimation
|
|
98
|
+
// Token estimation
|
|
99
99
|
// ---------------------------------------------------------------------------
|
|
100
|
+
// M3-10 / R34: the single char→token / text→token conversion lives in
|
|
101
|
+
// src/utils/token-counter.ts. This site used to carry its own wrapper around
|
|
102
|
+
// `countTokens`; it now uses the shared `estimateTokensForText` so pruner
|
|
103
|
+
// budgets, handoff budgets (task-output-context.ts) and prompt-diet estimates
|
|
104
|
+
// (prompt-builder.ts) all share one denominator and one unit (TOKENS).
|
|
100
105
|
|
|
101
|
-
import { countTokens } from "../../utils/token-counter.ts";
|
|
106
|
+
import { countTokens, estimateTokensForText } from "../../utils/token-counter.ts";
|
|
102
107
|
|
|
103
108
|
function estimateTokens(text: string): number {
|
|
104
|
-
return
|
|
109
|
+
return estimateTokensForText(text);
|
|
105
110
|
}
|
|
106
111
|
|
|
107
112
|
// ---------------------------------------------------------------------------
|