pi-crew 0.9.20 → 0.9.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/build-meta.json +72 -24
- package/dist/index.mjs +698 -307
- package/dist/index.mjs.map +4 -4
- package/package.json +1 -1
- package/src/extension/cross-extension-rpc.ts +16 -3
- package/src/extension/team-tool/run.ts +33 -4
- package/src/runtime/background-runner.ts +7 -0
- package/src/runtime/dynamic-workflow-context.ts +3 -3
- package/src/runtime/task-runner.ts +4 -4
- package/src/runtime/team-runner.ts +130 -1
- package/src/runtime/tool-output-pruner.ts +5 -3
- package/src/state/event-log.ts +168 -12
- package/src/state/types.ts +5 -0
- package/src/utils/token-counter.ts +67 -0
- package/src/worktree/worktree-manager.ts +363 -1
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
|
|
2
2
|
import type { TeamToolParamsValue } from "../schema/team-tool-schema.ts";
|
|
3
3
|
import { resolveContainedPath } from "../utils/safe-paths.ts";
|
|
4
|
+
import { withHmacVerification } from "./rpc-hmac.ts";
|
|
4
5
|
// Lazy-loaded to avoid pulling team-tool.ts (and its entire runtime chain) into module load.
|
|
5
6
|
import type { handleTeamTool as HandleTeamToolFn } from "./team-tool.ts";
|
|
6
|
-
import { withHmacVerification } from "./rpc-hmac.ts";
|
|
7
7
|
|
|
8
8
|
let _cachedHandleTeamTool: typeof HandleTeamToolFn | undefined;
|
|
9
9
|
async function handleTeamTool(
|
|
@@ -137,7 +137,7 @@ function isAllowedRpcRunParams(params: TeamToolParamsValue): {
|
|
|
137
137
|
return { ok: true };
|
|
138
138
|
}
|
|
139
139
|
|
|
140
|
-
function on(events: EventBusLike, channel: string, handler: (raw:
|
|
140
|
+
function on(events: EventBusLike, channel: string, handler: (raw: any) => unknown): () => void {
|
|
141
141
|
const unsub = events.on(channel, handler);
|
|
142
142
|
return typeof unsub === "function" ? unsub : () => {};
|
|
143
143
|
}
|
|
@@ -184,7 +184,20 @@ export function registerPiCrewRpc(
|
|
|
184
184
|
const ctx = getCtx();
|
|
185
185
|
if (!ctx) throw new Error("No active pi-crew session context.");
|
|
186
186
|
// Validate payload: only allow known fields from TeamToolParamsValue
|
|
187
|
-
const ALLOWED_RPC_RUN_KEYS = new Set([
|
|
187
|
+
const ALLOWED_RPC_RUN_KEYS = new Set([
|
|
188
|
+
"goal",
|
|
189
|
+
"team",
|
|
190
|
+
"workflow",
|
|
191
|
+
"async",
|
|
192
|
+
"cwd",
|
|
193
|
+
"config",
|
|
194
|
+
"skill",
|
|
195
|
+
"model",
|
|
196
|
+
"budgetTotal",
|
|
197
|
+
"budgetWarning",
|
|
198
|
+
"budgetAbort",
|
|
199
|
+
"budgetUnlimited",
|
|
200
|
+
]);
|
|
188
201
|
let params: TeamToolParamsValue;
|
|
189
202
|
if (raw && typeof raw === "object" && !Array.isArray(raw)) {
|
|
190
203
|
const filtered: Record<string, unknown> = {
|
|
@@ -12,7 +12,7 @@ import { createRunManifest, loadRunManifestById, updateRunStatus } from "../../s
|
|
|
12
12
|
import { allTeams, discoverTeams } from "../../teams/discover-teams.ts";
|
|
13
13
|
import { allWorkflows, discoverWorkflows } from "../../workflows/discover-workflows.ts";
|
|
14
14
|
import { validateWorkflowForTeam } from "../../workflows/validate-workflow.ts";
|
|
15
|
-
import {
|
|
15
|
+
import { assertCleanLeaderAsync, findGitRootAsync } from "../../worktree/worktree-manager.ts";
|
|
16
16
|
|
|
17
17
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- type-only import for TS inference
|
|
18
18
|
const _typeCheck: typeof ExecuteTeamRunFn = null as never as typeof ExecuteTeamRunFn;
|
|
@@ -226,7 +226,7 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
226
226
|
let resolvedCtx = ctx;
|
|
227
227
|
if (workingDir) {
|
|
228
228
|
try {
|
|
229
|
-
const gitRoot =
|
|
229
|
+
const gitRoot = await findGitRootAsync(workingDir);
|
|
230
230
|
if (gitRoot && gitRoot !== workingDir) {
|
|
231
231
|
resolvedCtx = { ...ctx, cwd: gitRoot };
|
|
232
232
|
}
|
|
@@ -241,7 +241,7 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
241
241
|
if (params.workspaceMode === "worktree") {
|
|
242
242
|
let gitRoot: string | undefined;
|
|
243
243
|
try {
|
|
244
|
-
gitRoot =
|
|
244
|
+
gitRoot = await findGitRootAsync(resolvedCtx.cwd);
|
|
245
245
|
} catch {
|
|
246
246
|
// not a git repo
|
|
247
247
|
}
|
|
@@ -256,7 +256,7 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
256
256
|
const preCheckConfig = loadConfig(resolvedCtx.cwd);
|
|
257
257
|
if (preCheckConfig.config.requireCleanWorktreeLeader !== false) {
|
|
258
258
|
try {
|
|
259
|
-
|
|
259
|
+
await assertCleanLeaderAsync(gitRoot);
|
|
260
260
|
} catch (err) {
|
|
261
261
|
const msg = err instanceof Error ? err.message : String(err);
|
|
262
262
|
return result(
|
|
@@ -573,10 +573,31 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
573
573
|
const executedConfig = effectiveRunConfig(loadedConfig.config, params.config);
|
|
574
574
|
const runtime = await resolveCrewRuntime(executedConfig);
|
|
575
575
|
const runtimeResolution = runtimeResolutionState(runtime);
|
|
576
|
+
// DEBUG: log what we received (gated to avoid stdout pollution in production)
|
|
577
|
+
if (process.env.PI_CREW_DEBUG_BUDGET === "1") {
|
|
578
|
+
console.log(
|
|
579
|
+
"[DEBUG budget] params keys:",
|
|
580
|
+
Object.keys(params),
|
|
581
|
+
"budgetTotal:",
|
|
582
|
+
params.budgetTotal,
|
|
583
|
+
"budgetWarning:",
|
|
584
|
+
params.budgetWarning,
|
|
585
|
+
"budgetAbort:",
|
|
586
|
+
params.budgetAbort,
|
|
587
|
+
);
|
|
588
|
+
}
|
|
576
589
|
const executionManifest = {
|
|
577
590
|
...updatedManifest,
|
|
578
591
|
runtimeResolution,
|
|
579
592
|
runConfig: executedConfig,
|
|
593
|
+
// Persist budget config on the manifest so it's observable post-run
|
|
594
|
+
// (events.jsonl, status reads, audits). The team-runner reads these
|
|
595
|
+
// from the input, but persisting them means consumers can verify
|
|
596
|
+
// enforcement was armed without re-parsing the original call params.
|
|
597
|
+
...(params.budgetTotal !== undefined ? { budgetTotal: params.budgetTotal } : {}),
|
|
598
|
+
...(params.budgetWarning !== undefined ? { budgetWarning: params.budgetWarning } : {}),
|
|
599
|
+
...(params.budgetAbort !== undefined ? { budgetAbort: params.budgetAbort } : {}),
|
|
600
|
+
...(params.budgetUnlimited !== undefined ? { budgetUnlimited: params.budgetUnlimited } : {}),
|
|
580
601
|
updatedAt: new Date().toISOString(),
|
|
581
602
|
};
|
|
582
603
|
atomicWriteJson(paths.manifestPath, executionManifest);
|
|
@@ -862,6 +883,10 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
862
883
|
metricRegistry: ctx.metricRegistry,
|
|
863
884
|
onJsonEvent: ctx.onJsonEvent,
|
|
864
885
|
workspaceId: ctx.sessionId ?? ctx.cwd,
|
|
886
|
+
budgetTotal: params.budgetTotal,
|
|
887
|
+
budgetWarning: params.budgetWarning,
|
|
888
|
+
budgetAbort: params.budgetAbort,
|
|
889
|
+
budgetUnlimited: params.budgetUnlimited,
|
|
865
890
|
});
|
|
866
891
|
} finally {
|
|
867
892
|
unregisterActiveRun(updatedManifest.runId);
|
|
@@ -1006,6 +1031,10 @@ export async function handleRun(params: TeamToolParamsValue, ctx: TeamContext):
|
|
|
1006
1031
|
metricRegistry: ctx.metricRegistry,
|
|
1007
1032
|
onJsonEvent: ctx.onJsonEvent,
|
|
1008
1033
|
workspaceId: ctx.cwd,
|
|
1034
|
+
budgetTotal: params.budgetTotal,
|
|
1035
|
+
budgetWarning: params.budgetWarning,
|
|
1036
|
+
budgetAbort: params.budgetAbort,
|
|
1037
|
+
budgetUnlimited: params.budgetUnlimited,
|
|
1009
1038
|
});
|
|
1010
1039
|
} finally {
|
|
1011
1040
|
unregisterActiveRun(updatedManifest.runId);
|
|
@@ -714,6 +714,9 @@ async function main(): Promise<void> {
|
|
|
714
714
|
switch (manifest.runKind ?? "team-run") {
|
|
715
715
|
default: {
|
|
716
716
|
// Existing "team-run" path — unchanged behavior.
|
|
717
|
+
// Forward budget fields from manifest (set by team-tool/run.ts
|
|
718
|
+
// from params.budgetTotal/etc.) so the team-runner's
|
|
719
|
+
// checkPerTaskBudget guard actually arms.
|
|
717
720
|
result = await executeTeamRun({
|
|
718
721
|
manifest,
|
|
719
722
|
tasks,
|
|
@@ -728,6 +731,10 @@ async function main(): Promise<void> {
|
|
|
728
731
|
reliability: runConfig.reliability,
|
|
729
732
|
workspaceId: manifest.ownerSessionId ?? manifest.cwd,
|
|
730
733
|
signal: abortController.signal,
|
|
734
|
+
...(manifest.budgetTotal !== undefined ? { budgetTotal: manifest.budgetTotal } : {}),
|
|
735
|
+
...(manifest.budgetWarning !== undefined ? { budgetWarning: manifest.budgetWarning } : {}),
|
|
736
|
+
...(manifest.budgetAbort !== undefined ? { budgetAbort: manifest.budgetAbort } : {}),
|
|
737
|
+
...(manifest.budgetUnlimited !== undefined ? { budgetUnlimited: manifest.budgetUnlimited } : {}),
|
|
731
738
|
});
|
|
732
739
|
break;
|
|
733
740
|
}
|
|
@@ -32,7 +32,7 @@ import { appendMailboxMessage, readMailbox } from "../state/mailbox.ts";
|
|
|
32
32
|
import type { TeamRunManifest } from "../state/types.ts";
|
|
33
33
|
import type { TeamConfig } from "../teams/team-config.ts";
|
|
34
34
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
35
|
-
import {
|
|
35
|
+
import { cleanupAgentWorktreeAsync, prepareAgentWorktreeAsync } from "../worktree/worktree-manager.ts";
|
|
36
36
|
import { runChildPi } from "./child-pi.ts";
|
|
37
37
|
import type { DwfCheckpointState } from "./dwf-state-store.ts";
|
|
38
38
|
import { mapConcurrent } from "./parallel-utils.ts";
|
|
@@ -342,7 +342,7 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
342
342
|
// when worktree creation is unavailable (no git repo, dirty leader).
|
|
343
343
|
let agentCwd = manifest.cwd;
|
|
344
344
|
if (call.worktree === true) {
|
|
345
|
-
const wt =
|
|
345
|
+
const wt = await prepareAgentWorktreeAsync(manifest, `dwf-agent-${Date.now()}-${randomBytes(4).toString("hex")}`);
|
|
346
346
|
if (wt?.worktreePath) {
|
|
347
347
|
agentCwd = wt.cwd;
|
|
348
348
|
worktreePath = wt.worktreePath;
|
|
@@ -431,7 +431,7 @@ export function makeWorkflowCtx(manifest: TeamRunManifest, opts: MakeWorkflowCtx
|
|
|
431
431
|
// — a leak must never crash the workflow.
|
|
432
432
|
if (worktreePath) {
|
|
433
433
|
try {
|
|
434
|
-
|
|
434
|
+
await cleanupAgentWorktreeAsync(manifest, worktreePath, worktreeBranch);
|
|
435
435
|
} catch (cleanupError) {
|
|
436
436
|
logInternalError("dynamic-workflow-context.worktree-cleanup", cleanupError, `worktreePath=${worktreePath}`);
|
|
437
437
|
}
|
|
@@ -20,7 +20,7 @@ import type {
|
|
|
20
20
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
21
21
|
import { resolveRealContainedPath } from "../utils/safe-paths.ts";
|
|
22
22
|
import type { WorkflowStep } from "../workflows/workflow-config.ts";
|
|
23
|
-
import {
|
|
23
|
+
import { captureWorktreeDiffAsync, captureWorktreeDiffStatAsync, prepareTaskWorkspaceAsync } from "../worktree/worktree-manager.ts";
|
|
24
24
|
import { reserveControlChannel } from "./agent-control.ts";
|
|
25
25
|
import { appendTaskAttentionEvent } from "./attention-events.ts";
|
|
26
26
|
import { buildSyntheticTerminalEvidence, cancellationReasonFromSignal } from "./cancellation.ts";
|
|
@@ -122,7 +122,7 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
122
122
|
let streamBridge: ReturnType<typeof registerStreamBridge> | undefined;
|
|
123
123
|
try {
|
|
124
124
|
streamBridge = registerStreamBridge(manifest.runId);
|
|
125
|
-
const workspace =
|
|
125
|
+
const workspace = await prepareTaskWorkspaceAsync(manifest, input.task, input.step.seedPaths);
|
|
126
126
|
const worktree =
|
|
127
127
|
workspace.worktreePath && workspace.branch
|
|
128
128
|
? {
|
|
@@ -905,7 +905,7 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
905
905
|
? writeArtifact(manifest.artifactsRoot, {
|
|
906
906
|
kind: "diff",
|
|
907
907
|
relativePath: `diffs/${task.id}.diff`,
|
|
908
|
-
content:
|
|
908
|
+
content: await captureWorktreeDiffAsync(workspace.worktreePath),
|
|
909
909
|
producer: task.id,
|
|
910
910
|
})
|
|
911
911
|
: undefined;
|
|
@@ -913,7 +913,7 @@ export async function runTeamTask(input: TaskRunnerInput): Promise<{ manifest: T
|
|
|
913
913
|
? writeArtifact(manifest.artifactsRoot, {
|
|
914
914
|
kind: "metadata",
|
|
915
915
|
relativePath: `metadata/${task.id}.diff-stat.json`,
|
|
916
|
-
content: `${JSON.stringify({ ...
|
|
916
|
+
content: `${JSON.stringify({ ...(await captureWorktreeDiffStatAsync(workspace.worktreePath)), syntheticPaths: workspace.syntheticPaths ?? [], nodeModulesLinked: workspace.nodeModulesLinked ?? false }, null, 2)}\n`,
|
|
917
917
|
producer: task.id,
|
|
918
918
|
})
|
|
919
919
|
: undefined;
|
|
@@ -13,7 +13,7 @@ import { HealthStore } from "../state/health-store.ts";
|
|
|
13
13
|
import { withRunLock } from "../state/locks.ts";
|
|
14
14
|
import { loadRunManifestById, saveRunManifest, saveRunManifestAsync, saveRunTasksAsync, updateRunStatus } from "../state/state-store.ts";
|
|
15
15
|
import type { ArtifactDescriptor, PolicyDecision, TaskAttemptState, TeamRunManifest, TeamTaskState } from "../state/types.ts";
|
|
16
|
-
import { aggregateUsage, formatUsage } from "../state/usage.ts";
|
|
16
|
+
import { aggregateUsage, formatTokens, formatUsage } from "../state/usage.ts";
|
|
17
17
|
import type { TeamConfig } from "../teams/team-config.ts";
|
|
18
18
|
import { logInternalError } from "../utils/internal-error.ts";
|
|
19
19
|
import type { WorkflowConfig, WorkflowStep } from "../workflows/workflow-config.ts";
|
|
@@ -130,6 +130,60 @@ export interface ExecuteTeamRunInput {
|
|
|
130
130
|
onJsonEvent?: (taskId: string, runId: string, event: unknown) => void;
|
|
131
131
|
/** Workspace where this run was initiated — used for session-scoped live-agent visibility. */
|
|
132
132
|
workspaceId: string;
|
|
133
|
+
/** Total token budget for the run. When set, enables per-task budget enforcement. */
|
|
134
|
+
budgetTotal?: number;
|
|
135
|
+
/** Budget warning threshold as a fraction (0-1). Default: 0.8 (80%). */
|
|
136
|
+
budgetWarning?: number;
|
|
137
|
+
/** Budget abort threshold as a fraction (0-1). Default: 0.95 (95%). */
|
|
138
|
+
budgetAbort?: number;
|
|
139
|
+
/** When true, skip budget enforcement entirely. */
|
|
140
|
+
budgetUnlimited?: boolean;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Result of a per-task budget check against cumulative run usage.
|
|
145
|
+
*/
|
|
146
|
+
export interface PerTaskBudgetCheckResult {
|
|
147
|
+
/** Whether the abort threshold was exceeded. */
|
|
148
|
+
abort: boolean;
|
|
149
|
+
/** Whether the warning threshold was exceeded. */
|
|
150
|
+
warning: boolean;
|
|
151
|
+
/** IDs of tasks that exceeded their fair share (>50% of remaining budget). */
|
|
152
|
+
fairShareViolators: string[];
|
|
153
|
+
/** Total tokens used so far. */
|
|
154
|
+
totalUsed: number;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Check cumulative token usage against per-task budget thresholds.
|
|
159
|
+
* Returns a structured result — callers decide how to act (warn vs abort).
|
|
160
|
+
*
|
|
161
|
+
* Exported for unit testing.
|
|
162
|
+
*/
|
|
163
|
+
export function checkPerTaskBudget(
|
|
164
|
+
tasks: TeamTaskState[],
|
|
165
|
+
budgetTotal: number,
|
|
166
|
+
budgetWarning: number,
|
|
167
|
+
budgetAbort: number,
|
|
168
|
+
fairShareFraction = 0.5,
|
|
169
|
+
): PerTaskBudgetCheckResult {
|
|
170
|
+
const usage = aggregateUsage(tasks);
|
|
171
|
+
const totalUsed = (usage?.input ?? 0) + (usage?.output ?? 0) + (usage?.cacheWrite ?? 0);
|
|
172
|
+
const abort = totalUsed >= budgetAbort * budgetTotal;
|
|
173
|
+
const warning = !abort && totalUsed >= budgetWarning * budgetTotal;
|
|
174
|
+
const remainingBudget = Math.max(0, budgetTotal - totalUsed);
|
|
175
|
+
const fairShareThreshold = remainingBudget * fairShareFraction;
|
|
176
|
+
const fairShareViolators: string[] = [];
|
|
177
|
+
for (const task of tasks) {
|
|
178
|
+
if (!task.usage) continue;
|
|
179
|
+
const taskTotal = (task.usage.input ?? 0) + (task.usage.output ?? 0) + (task.usage.cacheWrite ?? 0);
|
|
180
|
+
// Only flag tasks that individually consumed a significant portion of the
|
|
181
|
+
// budget (>10% of total) AND exceeded the fair share of remaining budget.
|
|
182
|
+
if (fairShareThreshold > 0 && taskTotal > fairShareThreshold && taskTotal > budgetTotal * 0.1) {
|
|
183
|
+
fairShareViolators.push(task.id);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return { abort, warning, fairShareViolators, totalUsed };
|
|
133
187
|
}
|
|
134
188
|
|
|
135
189
|
function findStep(workflow: WorkflowConfig, task: TeamTaskState): WorkflowStep {
|
|
@@ -586,6 +640,18 @@ export async function executeTeamRun(input: ExecuteTeamRunInput): Promise<{ mani
|
|
|
586
640
|
input.executeWorkers ? "Executing team workflow." : "Creating workflow prompts and placeholder results.",
|
|
587
641
|
);
|
|
588
642
|
|
|
643
|
+
// Persist budget fields on the manifest so all subsequent saveRunManifest
|
|
644
|
+
// calls (there are many in executeTeamRunCore) preserve the budget config.
|
|
645
|
+
// Without this, the spread in updateRunStatus and other transformations
|
|
646
|
+
// can drop budget fields before they reach the persistence layer.
|
|
647
|
+
if (input.budgetTotal !== undefined) manifest.budgetTotal = input.budgetTotal;
|
|
648
|
+
if (input.budgetWarning !== undefined) manifest.budgetWarning = input.budgetWarning;
|
|
649
|
+
if (input.budgetAbort !== undefined) manifest.budgetAbort = input.budgetAbort;
|
|
650
|
+
if (input.budgetUnlimited !== undefined) manifest.budgetUnlimited = input.budgetUnlimited;
|
|
651
|
+
if (manifest.budgetTotal !== undefined && manifest.budgetTotal > 0 && !manifest.budgetUnlimited) {
|
|
652
|
+
saveRunManifest(manifest);
|
|
653
|
+
}
|
|
654
|
+
|
|
589
655
|
void registerRunPromise(manifest.runId);
|
|
590
656
|
|
|
591
657
|
// FIX (Round 15, regression): Start a team-level heartbeat so the stale
|
|
@@ -1444,6 +1510,69 @@ async function executeTeamRunCore(
|
|
|
1444
1510
|
wfMachine = { ...wfMachine, currentPhaseIndex: pi + 1 };
|
|
1445
1511
|
}
|
|
1446
1512
|
|
|
1513
|
+
// Per-task budget enforcement: check cumulative usage after each batch merge.
|
|
1514
|
+
// This prevents a single task from consuming 100% of the budget before
|
|
1515
|
+
// abort triggers (the goal-loop only checks at turn boundaries).
|
|
1516
|
+
if (input.budgetTotal !== undefined && input.budgetTotal > 0 && input.budgetUnlimited !== true) {
|
|
1517
|
+
const warnThreshold = input.budgetWarning ?? 0.8;
|
|
1518
|
+
const abortThreshold = input.budgetAbort ?? 0.95;
|
|
1519
|
+
const budgetCheck = checkPerTaskBudget(tasks, input.budgetTotal, warnThreshold, abortThreshold);
|
|
1520
|
+
|
|
1521
|
+
if (budgetCheck.abort) {
|
|
1522
|
+
const message = `Per-task budget abort threshold exceeded: ${formatTokens(budgetCheck.totalUsed)}/${formatTokens(input.budgetTotal)} (${Math.round((budgetCheck.totalUsed / input.budgetTotal) * 100)}%)`;
|
|
1523
|
+
console.warn(`[team-runner] ${message}`);
|
|
1524
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1525
|
+
type: "run.budget_abort",
|
|
1526
|
+
runId: manifest.runId,
|
|
1527
|
+
message,
|
|
1528
|
+
data: {
|
|
1529
|
+
budgetTotal: input.budgetTotal,
|
|
1530
|
+
budgetUsed: budgetCheck.totalUsed,
|
|
1531
|
+
threshold: "abort",
|
|
1532
|
+
},
|
|
1533
|
+
});
|
|
1534
|
+
tasks = markBlocked(tasks, `Budget abort threshold exceeded: ${message}`);
|
|
1535
|
+
await saveRunTasksAsync(manifest, tasks);
|
|
1536
|
+
manifest = updateRunStatus(manifest, "failed", message);
|
|
1537
|
+
return { manifest, tasks };
|
|
1538
|
+
}
|
|
1539
|
+
|
|
1540
|
+
if (budgetCheck.warning) {
|
|
1541
|
+
const message = `Per-task budget warning threshold crossed: ${formatTokens(budgetCheck.totalUsed)}/${formatTokens(input.budgetTotal)} (${Math.round((budgetCheck.totalUsed / input.budgetTotal) * 100)}%)`;
|
|
1542
|
+
console.warn(`[team-runner] ${message}`);
|
|
1543
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1544
|
+
type: "run.budget_warning",
|
|
1545
|
+
runId: manifest.runId,
|
|
1546
|
+
message,
|
|
1547
|
+
data: {
|
|
1548
|
+
budgetTotal: input.budgetTotal,
|
|
1549
|
+
budgetUsed: budgetCheck.totalUsed,
|
|
1550
|
+
threshold: "warning",
|
|
1551
|
+
},
|
|
1552
|
+
});
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
// Fair-share warning: flag tasks that consumed >50% of remaining budget
|
|
1556
|
+
// without killing them mid-execution.
|
|
1557
|
+
for (const violatorId of budgetCheck.fairShareViolators) {
|
|
1558
|
+
const violator = tasks.find((t) => t.id === violatorId);
|
|
1559
|
+
if (!violator) continue;
|
|
1560
|
+
const taskTotal = (violator.usage?.input ?? 0) + (violator.usage?.output ?? 0) + (violator.usage?.cacheWrite ?? 0);
|
|
1561
|
+
const message = `Task '${violatorId}' consumed ${formatTokens(taskTotal)} (${Math.round((taskTotal / input.budgetTotal) * 100)}% of total budget) — exceeds fair share`;
|
|
1562
|
+
console.warn(`[team-runner.fair-share] ${message}`);
|
|
1563
|
+
await appendEventAsync(manifest.eventsPath, {
|
|
1564
|
+
type: "task.budget_fair_share",
|
|
1565
|
+
runId: manifest.runId,
|
|
1566
|
+
taskId: violatorId,
|
|
1567
|
+
message,
|
|
1568
|
+
data: {
|
|
1569
|
+
budgetTotal: input.budgetTotal,
|
|
1570
|
+
taskUsage: taskTotal,
|
|
1571
|
+
},
|
|
1572
|
+
});
|
|
1573
|
+
}
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1447
1576
|
const cancelledResult = results.find((item) => item.manifest.status === "cancelled");
|
|
1448
1577
|
if (cancelledResult || input.signal?.aborted) {
|
|
1449
1578
|
const reason = input.signal?.aborted ? cancellationReasonFromSignal(input.signal) : undefined;
|
|
@@ -90,8 +90,10 @@ export interface PruneResult {
|
|
|
90
90
|
// Token estimation (rough char/4 heuristic, matching gajae-code)
|
|
91
91
|
// ---------------------------------------------------------------------------
|
|
92
92
|
|
|
93
|
+
import { countTokens } from "../utils/token-counter.ts";
|
|
94
|
+
|
|
93
95
|
function estimateTokens(text: string): number {
|
|
94
|
-
return
|
|
96
|
+
return countTokens(text);
|
|
95
97
|
}
|
|
96
98
|
|
|
97
99
|
// ---------------------------------------------------------------------------
|
|
@@ -146,7 +148,7 @@ function createPrunedNotice(tokens: number, entry: ToolResultEntry): string {
|
|
|
146
148
|
const generic = `[Output pruned — ${tokens} tokens]`;
|
|
147
149
|
const digest = resultDigest(entry.toolName, entry.content, entry.isError);
|
|
148
150
|
if (!digest) return generic;
|
|
149
|
-
const genericTokens =
|
|
151
|
+
const genericTokens = countTokens(generic);
|
|
150
152
|
const maxTokens = Math.max(genericTokens, Math.floor(genericTokens * DIGEST_NOTICE_TOKEN_CAP_MULTIPLIER));
|
|
151
153
|
const prefix = `[Output pruned — ${tokens} tokens; `;
|
|
152
154
|
const suffix = "]";
|
|
@@ -303,7 +305,7 @@ export function pruneToolOutputs(results: ToolResultEntry[], config: PruneConfig
|
|
|
303
305
|
entry,
|
|
304
306
|
tokens,
|
|
305
307
|
notice,
|
|
306
|
-
savings: Math.max(0, tokens -
|
|
308
|
+
savings: Math.max(0, tokens - countTokens(notice)),
|
|
307
309
|
});
|
|
308
310
|
accumulatedTokens += tokens;
|
|
309
311
|
}
|
package/src/state/event-log.ts
CHANGED
|
@@ -350,6 +350,11 @@ export function checkSequenceGaps(eventsPath: string): { missing: number }[] {
|
|
|
350
350
|
* from firing and degrading live-agent responsiveness.
|
|
351
351
|
*/
|
|
352
352
|
export function appendEvent(eventsPath: string, event: AppendTeamEvent): TeamEvent {
|
|
353
|
+
// NOTE: appendEvent is a sync function that uses withEventLogLockSync (sleepSync).
|
|
354
|
+
// It cannot route through appendEventBuffered because the buffer timer requires
|
|
355
|
+
// the event loop to fire, which sleepSync blocks. Both terminal and non-terminal
|
|
356
|
+
// events use the direct sync path here. For non-terminal events, callers should
|
|
357
|
+
// prefer appendEventAsync (which routes through the buffer for coalesced writes).
|
|
353
358
|
return withEventLogLockSync(eventsPath, () => appendEventInsideLock(eventsPath, event));
|
|
354
359
|
}
|
|
355
360
|
|
|
@@ -405,6 +410,15 @@ export function resetEventLogMode(): void {
|
|
|
405
410
|
* foreground-control, etc.), prefer this over the sync `appendEvent()`.
|
|
406
411
|
*/
|
|
407
412
|
export async function appendEventAsync(eventsPath: string, event: AppendTeamEvent): Promise<TeamEvent> {
|
|
413
|
+
// Non-terminal events: route through buffer for coalesced writes (20ms default).
|
|
414
|
+
// This eliminates per-event fsync + persistSequence overhead for high-frequency events.
|
|
415
|
+
// The buffer timer is kept alive (ref'd) so standalone contexts (tests, short-lived
|
|
416
|
+
// scripts) get their events flushed before the loop drains — no per-event keepAlive
|
|
417
|
+
// intervals needed (which previously starved the event loop at high fan-out).
|
|
418
|
+
if (!TERMINAL_EVENT_TYPES.has(event.type)) {
|
|
419
|
+
return appendEventBuffered(eventsPath, event);
|
|
420
|
+
}
|
|
421
|
+
// Terminal events: direct async path for immediate durability guarantee
|
|
408
422
|
const queueKey = eventsPath;
|
|
409
423
|
const prev = asyncQueues.get(queueKey) ?? Promise.resolve();
|
|
410
424
|
const next = prev.then(async (): Promise<TeamEvent> => {
|
|
@@ -600,6 +614,131 @@ export async function appendEventAsync(eventsPath: string, event: AppendTeamEven
|
|
|
600
614
|
* `withEventLogLockSync` for `eventsPath`. Used by `appendEventBuffered` to
|
|
601
615
|
* write a whole batch of pending events under a single lock acquire.
|
|
602
616
|
*/
|
|
617
|
+
/**
|
|
618
|
+
* Batch variant used by the buffered flush path. Computes metadata for each
|
|
619
|
+
* event, writes the whole batch in a single appendFileSync + fsync, persists
|
|
620
|
+
* the sequence sidecar once with the last seq, and updates the sequence cache
|
|
621
|
+
* once. Resolves each item with its finalized event (carrying the assigned
|
|
622
|
+
* seq). This collapses N fsyncs into 1 for the buffered write path, which is
|
|
623
|
+
* the entire point of buffering — the previous per-event fsync made buffer
|
|
624
|
+
* coalescing useless and added ~30ms/event on tmpfs.
|
|
625
|
+
*/
|
|
626
|
+
async function appendEventBatchInsideLock(eventsPath: string, queue: BufferedAppend[]): Promise<void> {
|
|
627
|
+
if (queue.length === 0) return;
|
|
628
|
+
fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
|
|
629
|
+
|
|
630
|
+
// Pre-flight size check (mirrors appendEventInsideLock). We do it once for
|
|
631
|
+
// the batch instead of once per event.
|
|
632
|
+
try {
|
|
633
|
+
if (fs.existsSync(eventsPath)) {
|
|
634
|
+
const stat = fs.statSync(eventsPath);
|
|
635
|
+
if (stat.size > MAX_EVENTS_BYTES) {
|
|
636
|
+
try {
|
|
637
|
+
const prepared = prepareCompaction(eventsPath);
|
|
638
|
+
if (prepared) applyCompactionUnlocked(eventsPath, prepared);
|
|
639
|
+
} catch (error) {
|
|
640
|
+
logInternalError("event-log.batch-immediate-compact", error, `eventsPath=${eventsPath}`);
|
|
641
|
+
}
|
|
642
|
+
if (fs.existsSync(eventsPath) && fs.statSync(eventsPath).size > MAX_EVENTS_BYTES) {
|
|
643
|
+
rotateEventLogUnlocked(eventsPath);
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
} catch (error) {
|
|
648
|
+
logInternalError("event-log.batch-size-check", error, `eventsPath=${eventsPath}`);
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
// Phase 1: compute metadata + JSON lines for every event in the batch.
|
|
652
|
+
// Initialize nextSeq ONCE from nextSequence (or the first event's baseMetadata.seq),
|
|
653
|
+
// then increment locally for each subsequent event in the batch. Calling
|
|
654
|
+
// nextSequence() per-event would re-read file stat/sidecar with no writes
|
|
655
|
+
// in between — every call would see the same file state and return the same
|
|
656
|
+
// seq, breaking the "unique monotonic seq" contract. The cache update +
|
|
657
|
+
// persistSequence at the end refreshes the sidecar to the last assigned seq.
|
|
658
|
+
const startingSeq = queue[0]?.event.metadata?.seq ?? nextSequence(eventsPath);
|
|
659
|
+
let nextSeq = startingSeq;
|
|
660
|
+
const finalized: { item: BufferedAppend; line: string; fullEvent: TeamEvent }[] = [];
|
|
661
|
+
let lastSeq = 0;
|
|
662
|
+
for (const item of queue) {
|
|
663
|
+
const baseMetadata = item.event.metadata;
|
|
664
|
+
const seq = baseMetadata?.seq ?? nextSeq++;
|
|
665
|
+
let metadata: TeamEventMetadata = {
|
|
666
|
+
seq,
|
|
667
|
+
provenance: baseMetadata?.provenance ?? "team_runner",
|
|
668
|
+
...(baseMetadata?.parentEventId ? { parentEventId: baseMetadata.parentEventId } : {}),
|
|
669
|
+
...(baseMetadata?.attemptId ? { attemptId: baseMetadata.attemptId } : {}),
|
|
670
|
+
...(baseMetadata?.branchId ? { branchId: baseMetadata.branchId } : {}),
|
|
671
|
+
...(baseMetadata?.causationId ? { causationId: baseMetadata.causationId } : {}),
|
|
672
|
+
...(baseMetadata?.correlationId ? { correlationId: baseMetadata.correlationId } : {}),
|
|
673
|
+
...(baseMetadata?.sessionIdentity ? { sessionIdentity: baseMetadata.sessionIdentity } : {}),
|
|
674
|
+
...(baseMetadata?.ownership ? { ownership: baseMetadata.ownership } : {}),
|
|
675
|
+
...(baseMetadata?.nudgeId ? { nudgeId: baseMetadata.nudgeId } : {}),
|
|
676
|
+
...(baseMetadata?.confidence ? { confidence: baseMetadata.confidence } : {}),
|
|
677
|
+
};
|
|
678
|
+
const fullEvent: TeamEvent = {
|
|
679
|
+
time: new Date().toISOString(),
|
|
680
|
+
...item.event,
|
|
681
|
+
metadata,
|
|
682
|
+
};
|
|
683
|
+
if (baseMetadata?.fingerprint || TERMINAL_EVENT_TYPES.has(fullEvent.type)) {
|
|
684
|
+
metadata = {
|
|
685
|
+
...metadata,
|
|
686
|
+
fingerprint: baseMetadata?.fingerprint ?? computeEventFingerprint(fullEvent),
|
|
687
|
+
};
|
|
688
|
+
fullEvent.metadata = metadata;
|
|
689
|
+
}
|
|
690
|
+
finalized.push({ item, line: `${JSON.stringify(redactSecrets(fullEvent))}\n`, fullEvent });
|
|
691
|
+
lastSeq = seq;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
// Phase 2: single appendFileSync + single fsync + single persistSequence.
|
|
695
|
+
// Before this fix, each event in the batch triggered its own fsync, which
|
|
696
|
+
// was the dominant cost on tmpfs and CI runners.
|
|
697
|
+
try {
|
|
698
|
+
if (fs.existsSync(eventsPath) && fs.statSync(eventsPath).size > MAX_EVENTS_BYTES) {
|
|
699
|
+
logInternalError(
|
|
700
|
+
"event-log.size-limit",
|
|
701
|
+
new Error(`events file ${eventsPath} exceeds ${MAX_EVENTS_BYTES} bytes after compaction`),
|
|
702
|
+
`eventsPath=${eventsPath}`,
|
|
703
|
+
);
|
|
704
|
+
// Reject the batch — caller will surface the error per item.
|
|
705
|
+
for (const { item } of finalized) item.reject(new Error("event log size limit exceeded"));
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
} catch (error) {
|
|
709
|
+
logInternalError("event-log.batch-size-check-post", error, `eventsPath=${eventsPath}`);
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
fs.appendFileSync(eventsPath, finalized.map((f) => f.line).join(""), "utf-8");
|
|
713
|
+
const fd = fs.openSync(eventsPath, "r+");
|
|
714
|
+
try {
|
|
715
|
+
fs.fsyncSync(fd);
|
|
716
|
+
} catch {
|
|
717
|
+
// EPERM on Windows CI: best-effort flush
|
|
718
|
+
} finally {
|
|
719
|
+
fs.closeSync(fd);
|
|
720
|
+
}
|
|
721
|
+
persistSequence(eventsPath, lastSeq);
|
|
722
|
+
|
|
723
|
+
// Phase 3: cache update + resolve all promises.
|
|
724
|
+
try {
|
|
725
|
+
const stat = fs.statSync(eventsPath);
|
|
726
|
+
if (sequenceCache.size >= MAX_SEQUENCE_CACHE_ENTRIES) {
|
|
727
|
+
evictOldestSequenceCacheEntries();
|
|
728
|
+
}
|
|
729
|
+
sequenceCache.set(eventsPath, {
|
|
730
|
+
size: stat.size,
|
|
731
|
+
mtimeMs: stat.mtimeMs,
|
|
732
|
+
seq: lastSeq,
|
|
733
|
+
lastAccessMs: Date.now(),
|
|
734
|
+
});
|
|
735
|
+
} catch (error) {
|
|
736
|
+
logInternalError("event-log.batch-cache-update", error, `eventsPath=${eventsPath}`);
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
for (const { item, fullEvent } of finalized) item.resolve(fullEvent);
|
|
740
|
+
}
|
|
741
|
+
|
|
603
742
|
function appendEventInsideLock(eventsPath: string, event: AppendTeamEvent): TeamEvent {
|
|
604
743
|
fs.mkdirSync(path.dirname(eventsPath), { recursive: true });
|
|
605
744
|
const baseMetadata = event.metadata;
|
|
@@ -766,8 +905,15 @@ export function appendEventBuffered(eventsPath: string, event: AppendTeamEvent,
|
|
|
766
905
|
queue.push({ event, resolve, reject });
|
|
767
906
|
bufferedQueues.set(eventsPath, queue);
|
|
768
907
|
if (!bufferedTimers.has(eventsPath)) {
|
|
769
|
-
|
|
770
|
-
|
|
908
|
+
// Wrap flush in async IIFE so the returned Promise is awaited (avoids
|
|
909
|
+
// "floating promise" warnings under --test-force-exit and prevents
|
|
910
|
+
// the timer from being treated as done before the flush actually
|
|
911
|
+
// completes its async work).
|
|
912
|
+
const timer = setTimeout(() => {
|
|
913
|
+
flushOneEventLogBuffer(eventsPath).catch((error) => {
|
|
914
|
+
logInternalError("event-log.buffered-flush", error, `eventsPath=${eventsPath}`);
|
|
915
|
+
});
|
|
916
|
+
}, bufferMs);
|
|
771
917
|
bufferedTimers.set(eventsPath, timer);
|
|
772
918
|
}
|
|
773
919
|
});
|
|
@@ -812,15 +958,12 @@ async function flushOneEventLogBuffer(eventsPath: string): Promise<void> {
|
|
|
812
958
|
// FIX (Issue 2): Use async lock instead of withEventLogLockSync to avoid
|
|
813
959
|
// blocking the event loop. The sync lock uses sleepSync which blocks for
|
|
814
960
|
// up to 5s and prevents AbortSignal handlers from firing.
|
|
961
|
+
// FIX (P0 follow-up): Batch the file write + fsync + persistSequence across
|
|
962
|
+
// the whole queue. Previously each event triggered its own fsyncSync,
|
|
963
|
+
// turning 100 buffered events into 100 fsyncs (~3s on tmpfs). Now we do
|
|
964
|
+
// 1 appendFileSync + 1 fsync + 1 persistSequence for the whole batch.
|
|
815
965
|
await withEventLogLockAsync(eventsPath, async () => {
|
|
816
|
-
|
|
817
|
-
try {
|
|
818
|
-
const ev = appendEventInsideLock(eventsPath, item.event);
|
|
819
|
-
item.resolve(ev);
|
|
820
|
-
} catch (error) {
|
|
821
|
-
item.reject(error);
|
|
822
|
-
}
|
|
823
|
-
}
|
|
966
|
+
await appendEventBatchInsideLock(eventsPath, queue);
|
|
824
967
|
});
|
|
825
968
|
} catch (error) {
|
|
826
969
|
// Lock acquire failed — fail every queued item so callers can fall back.
|
|
@@ -848,13 +991,26 @@ export function appendEventFireAndForget(eventsPath: string, event: AppendTeamEv
|
|
|
848
991
|
// Auto-flush on process exit so buffered events do not silently leak.
|
|
849
992
|
// Defense-in-depth: SIGTERM/SIGINT use setImmediate so the handler returns
|
|
850
993
|
// immediately and the main thread is not blocked by sync I/O.
|
|
994
|
+
// FIX (P0 follow-up): Only call flushEventLogBuffer() / drainAsyncQueues() if
|
|
995
|
+
// there is actually pending work. Calling them unconditionally creates a new
|
|
996
|
+
// floating Promise that the test runner detects under --test-force-exit,
|
|
997
|
+
// failing tests with "Promise resolution is still pending but the event loop
|
|
998
|
+
// has already resolved" even when the test body completed cleanly.
|
|
851
999
|
process.on("exit", () => {
|
|
852
|
-
|
|
1000
|
+
if (bufferedQueues.size > 0) {
|
|
1001
|
+
flushEventLogBuffer().catch(() => {
|
|
1002
|
+
/* best-effort flush on exit */
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
853
1005
|
// FIX (Issue 1): Drain asyncQueues on exit to minimize event loss.
|
|
854
1006
|
// In-flight async writes are awaited (via Promise.allSettled) before
|
|
855
1007
|
// the map is cleared. This reduces but does not eliminate event loss
|
|
856
1008
|
// on crash — SIGKILL (kill -9) cannot be intercepted.
|
|
857
|
-
|
|
1009
|
+
if (asyncQueues.size > 0) {
|
|
1010
|
+
drainAsyncQueues().catch(() => {
|
|
1011
|
+
/* best-effort drain on exit */
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
858
1014
|
asyncQueues.clear();
|
|
859
1015
|
});
|
|
860
1016
|
process.on("SIGTERM", () => setImmediate(() => flushEventLogBuffer()));
|
package/src/state/types.ts
CHANGED
|
@@ -208,6 +208,11 @@ export interface TeamRunManifest {
|
|
|
208
208
|
runKind?: "team-run" | "goal-loop" | "dynamic-workflow";
|
|
209
209
|
/** round-14 P1-5: typed workflow arguments accessible in .dwf.ts scripts via ctx.args<T>(). Any JSON value; default {} when unset. */
|
|
210
210
|
args?: unknown;
|
|
211
|
+
/** Per-run token budget snapshot. budgetTotal is the cap; budgetWarning/budgetAbort are fractions in [0,1] at which the team-runner emits run.budget_warning / run.budget_abort. budgetUnlimited=true opts out of enforcement. Optional for backward compat. */
|
|
212
|
+
budgetTotal?: number;
|
|
213
|
+
budgetWarning?: number;
|
|
214
|
+
budgetAbort?: number;
|
|
215
|
+
budgetUnlimited?: boolean;
|
|
211
216
|
summary?: string;
|
|
212
217
|
policyDecisions?: PolicyDecision[];
|
|
213
218
|
/** #2 (assessment): goal-achievement verdict — kills the silent false-green. */
|