pi-harness-delegate 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -9
- package/extensions/acp-runner.ts +158 -35
- package/extensions/activity.ts +91 -1
- package/extensions/command.ts +170 -13
- package/extensions/concurrency.ts +41 -5
- package/extensions/engine.ts +604 -0
- package/extensions/fanout.ts +512 -0
- package/extensions/harnesses/amp.ts +7 -6
- package/extensions/harnesses/claude.ts +2 -0
- package/extensions/harnesses/codex.ts +15 -7
- package/extensions/harnesses/opencode.ts +14 -3
- package/extensions/harnesses/registry.ts +16 -0
- package/extensions/harnesses/types.ts +8 -8
- package/extensions/history.ts +170 -0
- package/extensions/index.ts +120 -1419
- package/extensions/progress.ts +9 -3
- package/extensions/run-registry.ts +36 -3
- package/extensions/runner.ts +75 -30
- package/extensions/stream-caps.ts +46 -0
- package/extensions/subcommands.ts +227 -0
- package/extensions/templates.ts +33 -2
- package/extensions/validate.ts +214 -0
- package/package.json +4 -2
- package/extensions/run-claude.ts +0 -61
- package/extensions/stream-parse.ts +0 -37
package/README.md
CHANGED
|
@@ -39,10 +39,38 @@ Manual delegation:
|
|
|
39
39
|
|
|
40
40
|
Only the prompt is required. A **harness as first word** and/or **mode as next word** selects them; every `--flag` is optional (harness defaults to `delegate.defaultHarness`, mode to `delegate.defaultMode`, scope to whole repo).
|
|
41
41
|
|
|
42
|
+
| Flag | Meaning |
|
|
43
|
+
| --- | --- |
|
|
44
|
+
| `--harness=<name>` | `claude`, `codex`, `opencode`, `amp` (`omp`), `devin`, `all`, or a comma list (fan-out, below) |
|
|
45
|
+
| `--mode=<template>` | Any template name (`review`, `plan`, `implement`, `security-audit`, `docs`, `general`, or your own) |
|
|
46
|
+
| `--model=<model>` | Model or alias (`economy`/`balanced`/`max`, see `modelAliases`) |
|
|
47
|
+
| `--scope=<scope>` | `diff` (current `git diff HEAD`), `pr` (`gh pr diff` for the current branch), or a path list |
|
|
48
|
+
| `--pr=<pr>` | A specific PR to scope to: a number, an http(s) PR URL (`https://<host>/<owner>/<repo>/pull/<n>`, no `user@`), or `owner/repo#123` (implies the PR diff as scope) |
|
|
49
|
+
| `--budget=<usd>` | Per-run spend cap in USD (same as the tool's `maxBudgetUsd`) |
|
|
50
|
+
| `--add-dir=<path>` | Extra directory the harness may access; repeatable (`--add-dir=../shared --add-dir=/opt/lib`) |
|
|
51
|
+
| `--resume=<session-id>` | Continue a previous delegated session (single harness only — not with a fan-out) |
|
|
52
|
+
| `--verify="<cmd>"` | Host-run check after the harness exits (see [Verify](#modes-templates)) |
|
|
53
|
+
| `--allow-dangerous` | Run this one invocation with `danger` (unrestricted) permission — required for a `permission: danger` template, and escalates any other template to `danger`. Always asks you to confirm first (one prompt for a whole fan-out); refused in a non-interactive session. Never read from config. `--allow-dangerous=true` also works; any other value is off. See [Security model](#security-model) |
|
|
54
|
+
|
|
55
|
+
Extra directories (`--add-dir`, the tool's `addDirs`, and template `addDirs:`) are merged, resolved against the working directory, and passed as each harness's native option where one exists: `--add-dir` for `claude`, `codex` (fresh runs only — `codex exec resume` has no such flag), and `amp`/`omp`; `additionalDirectories` on the ACP session for `devin`/`opencode` over ACP. `opencode run` (stdout) has no equivalent, so they're ignored there. On the `delegate` **tool**, `addDirs` entries that resolve (after `..` and symlinks) outside the working directory ask you to confirm interactively and are refused in a non-interactive session — the model alone can't widen a run to arbitrary host paths. Template `addDirs:` and `/delegate --add-dir` (both set by you) are not gated.
|
|
56
|
+
|
|
57
|
+
Flag values may be quoted (`--verify="bun test && bun run lint"`). `--resume`, `--model`, and `--pr` values are validated before anything runs — e.g. a value starting with `-` is rejected, so it can never be smuggled into a harness's command line as a flag.
|
|
58
|
+
|
|
59
|
+
**Subcommands** (work on `/delegate` and on every alias, where the alias pins the harness filter):
|
|
60
|
+
|
|
61
|
+
| Subcommand | What it does |
|
|
62
|
+
| --- | --- |
|
|
63
|
+
| `/delegate list [harness]` | Available modes/templates (all harnesses, or one) |
|
|
64
|
+
| `/delegate history [harness]` (alias `logs`) | Past transcripts, newest first; open one to read it (and see its resume hint) |
|
|
65
|
+
| `/delegate status [harness]` (aliases `health`, `doctor`, `check`) | Config provenance, project trust, per-harness detection/version/templates/active-vs-cap, spend rollup |
|
|
66
|
+
| `/delegate config` | What was read from `settings.json` and the effective config (print-only) |
|
|
67
|
+
| `/delegate config init` | Write the effective config into `settings.json`'s `delegate` key (the only write this extension does) |
|
|
68
|
+
| `/delegate watch` (alias `show`) | Re-open a minimized progress overlay |
|
|
69
|
+
|
|
42
70
|
Some modes have **default tasks** when the prompt is omitted:
|
|
43
71
|
`/delegate review` reviews the current git diff (`scope: diff`), `/delegate security-audit` audits the repo. Modes without a default (`plan`, `implement`, `docs`, `general`) print a hint asking for a prompt.
|
|
44
72
|
|
|
45
|
-
The `delegate` tool takes: `harness`, `task`, `mode`, `scope` (`diff` = git diff, `pr` = PR diff, path list, or whole repo), `model`, `maxBudgetUsd`, `allowDangerous`, `sessionId`, `pr`. (`verify` is deliberately *not* a tool parameter — see below.)
|
|
73
|
+
The `delegate` tool takes: `harness`, `task`, `mode`, `scope` (`diff` = git diff, `pr` = PR diff, path list, or whole repo), `model`, `maxBudgetUsd`, `allowDangerous`, `sessionId`, `pr`, `addDirs`. (`verify` is deliberately *not* a tool parameter — see below.) Setting `allowDangerous` from the tool always asks you to confirm interactively, and is refused outright in a non-interactive session — see [Security model](#security-model).
|
|
46
74
|
|
|
47
75
|
`claude_delegate` remains as a deprecated alias for `delegate{harness:claude}`.
|
|
48
76
|
|
|
@@ -56,10 +84,9 @@ The `delegate` tool takes: `harness`, `task`, `mode`, `scope` (`diff` = git diff
|
|
|
56
84
|
delegate({ harness: "all", mode: "review", scope: "diff" }) # tool call form
|
|
57
85
|
```
|
|
58
86
|
|
|
59
|
-
- `all` resolves
|
|
60
|
-
|
|
61
|
-
- `
|
|
62
|
-
- Each harness's run goes through the same `delegate()` engine as a single-harness call and writes its own transcript to its own `~/.pi/agent/delegate/outputs/<harness>/`. Runs are launched together and execute in parallel, bounded by `maxConcurrent` (default `4`, one slot per supported harness) — a run beyond the cap queues for a free slot instead of failing, and the cap is enforced across pi processes, not just this one. **This means fan-out spend is genuinely simultaneous**: with the default cap, a 4-harness fan-out can bill all four at once instead of one after another — budget accordingly (`maxBudgetUsd` still applies per run).
|
|
87
|
+
- `all` resolves to whatever's actually installed (`detectAll()`) — an uninstalled harness is skipped and named in the report, it doesn't fail the run. An explicit list is validated the same way; an unknown name is also reported rather than aborting the rest. With all five harnesses installed, `all` means five runs.
|
|
88
|
+
- Each harness's run goes through the same `delegate()` engine as a single-harness call and writes its own transcript to its own `~/.pi/agent/delegate/outputs/<harness>/`. Runs are launched together and execute in parallel, bounded by `maxConcurrent` (default `4`) — a run beyond the cap queues for a free slot instead of failing (so a 5-harness `all` runs four at once and the fifth when a slot frees), and the cap is enforced across pi processes, not just this one. **This means fan-out spend is genuinely simultaneous**: up to `maxConcurrent` harnesses can bill at once instead of one after another — budget accordingly (`maxBudgetUsd` still applies per run, not to the fan-out as a whole).
|
|
89
|
+
- `--resume`/`sessionId` can't be combined with a fan-out — a session id belongs to exactly one harness — and is rejected up front with a message saying so.
|
|
63
90
|
- The synthesized report is always ordered by the resolved harness list (e.g. `claude, codex, opencode`), regardless of which harness actually finishes first — it groups each harness's metrics + output and a total spend line (unknown-cost runs called out separately, same as `/delegate status`), assembled mechanically, not by asking a model to summarize.
|
|
64
91
|
- A single-harness call (`harness: "claude"`, or omitted) behaves exactly as before, including the concurrency guard: it still fails fast with "another delegate run is already in progress" at capacity rather than queueing. Fan-out is opt-in by typing `all`/a list.
|
|
65
92
|
- `/delegate all …` batches successful completions into one notification instead of one per harness; a failure is never delayed or folded into the batch — it surfaces immediately.
|
|
@@ -89,7 +116,7 @@ Detect availability: `delegate` checks `harness --version` at startup; missing h
|
|
|
89
116
|
|
|
90
117
|
### Transport
|
|
91
118
|
|
|
92
|
-
Every harness runs over its native CLI's stdout (`stdout`, the default and only option for `claude`/`codex`/`amp`). `opencode` and `devin` also speak [ACP](https://agentclientprotocol.com) (Agent Client Protocol — bidirectional JSON-RPC over stdio): Devin ships ACP-only (no stdout mode exists), and `opencode` supports both — `stdout` stays the default, `transport: "acp"` is opt-in per harness in config (see below). ACP gives `opencode` a genuine `
|
|
119
|
+
Every harness runs over its native CLI's stdout (`stdout`, the default and only option for `claude`/`codex`/`amp`). `opencode` and `devin` also speak [ACP](https://agentclientprotocol.com) (Agent Client Protocol — bidirectional JSON-RPC over stdio): Devin ships ACP-only (no stdout mode exists), and `opencode` supports both — `stdout` stays the default, `transport: "acp"` is opt-in per harness in config (see below). ACP gives `opencode` a genuine `contextWindow` (`null` over stdout) and a server-side running cost total (stdout reports cost too, summed from each step's `step_finish` — `null` only if no step reported one), plus a resume path independently proven to recall cross-process state; the tradeoff is `model`/`numTurns` staying unmeasured (`null`) either way. `amp`/`omp` has a real `acp` subcommand too, but isn't offered as a `transport` value — its ACP mode surface has only 2 permission tiers against the stdout CLI's genuine 3, a real regression, not just an unverified one. Configuring a transport a harness doesn't support fails immediately with a clear error, before anything spawns.
|
|
93
120
|
|
|
94
121
|
## Modes (templates)
|
|
95
122
|
|
|
@@ -116,6 +143,23 @@ You are a senior engineer delegated by the pi coding agent.
|
|
|
116
143
|
...
|
|
117
144
|
```
|
|
118
145
|
|
|
146
|
+
All frontmatter keys (one `key: value` per line; only `name` is required):
|
|
147
|
+
|
|
148
|
+
| Key | Meaning |
|
|
149
|
+
| --- | --- |
|
|
150
|
+
| `name` | The mode name used to select it (`--mode=`/`mode`) |
|
|
151
|
+
| `description` | Shown by `/delegate list` |
|
|
152
|
+
| `permission` | `readonly` \| `edit` \| `danger` (default `edit`). Any other value is treated as a native permission string |
|
|
153
|
+
| `permissionMode` / `sandbox` | Legacy/native escape hatch (see below) |
|
|
154
|
+
| `model` | Default model for this mode (call → template → harness → global) |
|
|
155
|
+
| `maxBudgetUsd` | Default per-run spend cap in USD (a call's `maxBudgetUsd`/`--budget` wins) |
|
|
156
|
+
| `skill` | Appends `Use the "<skill>" skill.` to the prompt |
|
|
157
|
+
| `defaultTask` | Task used when the prompt is omitted (e.g. `review` → review the current diff) |
|
|
158
|
+
| `defaultScope` | Scope used with `defaultTask` (e.g. `diff`) |
|
|
159
|
+
| `verify` | Host-run check command (see below) |
|
|
160
|
+
| `addDirs` | Comma-separated extra directories the harness may access (merged with the call's `addDirs`/`--add-dir`) |
|
|
161
|
+
| `harness` | Informational: which harness a template targets (shown by `/delegate list`) |
|
|
162
|
+
|
|
119
163
|
**Native escape hatch:** if you need a harness-specific permission not covered by the normalized set, use the native key (`permissionMode: dontAsk`, `sandbox: ...`) — it overrides `permission` for that harness.
|
|
120
164
|
|
|
121
165
|
**Verify:** `verify` is a shell command run **on the host** (never handed to the harness) right after it exits — e.g. `verify: bun test` on an `implement`/`docs`/`general` template turns "the harness says it's done" into an actual pass/fail. It's report-only: a failing verify is appended as its own section in the transcript and injected report, and surfaced in the tool result's `details.verify` (`{command, exitCode, ok}`), but it never changes whether the run itself is reported as an error — that stays whatever the harness reported. No template ships one by default — there's no universally-correct check command, so nothing is invented for you.
|
|
@@ -144,7 +188,7 @@ Custom templates are just files dropped in the above dirs — any registered nam
|
|
|
144
188
|
## Inspecting what the harness is doing
|
|
145
189
|
|
|
146
190
|
- **Live activity feed** — `▶ Bash: ... ✓/✗`, `💭 thinking…`, text tail. `/delegate` shows a framed progress window (spinner, `danger` banner for `danger` permission, `esc`×2 cancels, `m` minimizes, `watch` re-opens).
|
|
147
|
-
- **Full transcript every run** — `~/.pi/agent/delegate/outputs/<harness>/<ts>-<mode>.md` (also legacy `claude-delegate/outputs/` for claude). Tool result ends with path.
|
|
191
|
+
- **Full transcript every run** — `~/.pi/agent/delegate/outputs/<harness>/<ts>-<mode>.md` (also legacy `claude-delegate/outputs/` for claude). Tool result ends with path. Transcripts contain your prompts, diffs, and the harness's full output, so they're written owner-only (directory `0700`, files `0600`).
|
|
148
192
|
- **Resume:** every run records a session id.
|
|
149
193
|
|
|
150
194
|
```bash
|
|
@@ -157,7 +201,7 @@ Reveal thinking live with `"inspectThinking": true` in config (off by default).
|
|
|
157
201
|
|
|
158
202
|
## Config
|
|
159
203
|
|
|
160
|
-
In `~/.pi/agent/settings.json`:
|
|
204
|
+
In `~/.pi/agent/settings.json` (or `$PI_CODING_AGENT_DIR/settings.json` — `PI_CODING_AGENT_DIR`, pi's own agent-dir override, relocates everything this extension keeps under `~/.pi/agent`: settings, user templates, transcripts, and the active-run registry):
|
|
161
205
|
|
|
162
206
|
```json
|
|
163
207
|
{
|
|
@@ -198,6 +242,14 @@ Config lives as a key inside pi's own `~/.pi/agent/settings.json` rather than a
|
|
|
198
242
|
|
|
199
243
|
`autoDelegateHints` is off by default — no system-prompt bias. When `true`, explicit markers (`@harness`, `with codex`, `delegate … to claude`) and imperative review/plan phrasing append a hint.
|
|
200
244
|
|
|
245
|
+
### Budgets (`maxBudgetUsd` / `--budget`)
|
|
246
|
+
|
|
247
|
+
A per-run cap, resolved call → template → global config → per-harness config. How it's enforced depends on the harness, and the outcome is always recorded (tool result `details.budget`, a `- budget:` line in the transcript):
|
|
248
|
+
|
|
249
|
+
- **Native** — `claude` gets `--max-budget-usd` and enforces it itself.
|
|
250
|
+
- **Host-enforced (best-effort)** — harnesses with no budget flag that *do* stream a running cost (`opencode`, `amp`/`omp`, and `opencode` over ACP): the host checks the reported total each time the harness reports one and kills the run once it's over the cap, recording `budget exceeded` (`stopReason: budget_exceeded`, a `⛔ budget exceeded … run stopped` line at the top of the result, `(host-enforced, best-effort)` in the transcript). This is **not a hard cap**: cost is only visible at the step/turn boundaries the harness reports, so spend already incurred within the step that crossed the line can't be prevented — a run can overshoot by up to one step/turn (more if a single step is expensive). Use a native-budget harness (`claude`) when the cap must be strict. For `opencode` over ACP the reported cost is a session running total, so on a resumed session only the spend since this run started counts against the cap, and the reported cost is just this run's share — or `—` (unmeasured) when the session didn't report its prior total before the new prompt, rather than an under-count.
|
|
251
|
+
- **Not enforceable** — `codex` (no `$` cost on ChatGPT-plan auth) and `devin` report no cost and have no flag, so a budget can't be applied; the run proceeds and the result starts with `⚠ maxBudgetUsd … was not enforced`, never silently.
|
|
252
|
+
|
|
201
253
|
## Metrics recorded
|
|
202
254
|
|
|
203
255
|
Every run records in details + transcript: harness, mode, permission (normalized + native), cost, tokens (input/output/cache), context% (prompt ÷ window), model, turns, duration, TTFT, stop reason, session id. Token + cost feed pi's `Usage`.
|
|
@@ -210,7 +262,11 @@ One deliberate, narrow exception: pi's own `Usage` (the footer/session token+cos
|
|
|
210
262
|
|
|
211
263
|
- `readonly` — no edits (e.g. Claude `plan`, Codex `read-only`).
|
|
212
264
|
- `edit` — workspace writes auto-accepted (e.g. `acceptEdits`, `workspace-write`).
|
|
213
|
-
- `danger` — unrestricted, **only via `allowDangerous:true` on the
|
|
265
|
+
- `danger` — unrestricted, **only via an explicit per-call opt-in** — `allowDangerous:true` on the tool, or `--allow-dangerous` on `/delegate` — never a default (and never from config). Shows `⚠ danger` banner. `review`/`plan`/`security-audit` templates stay `readonly`.
|
|
266
|
+
- When the **model** sets `allowDangerous: true` on the `delegate` tool, the extension asks you to confirm it (`Allow dangerous delegation?`) before anything runs; declining aborts the call, and in a non-interactive session (no UI to ask) it is refused outright. The model alone can never grant `danger`.
|
|
267
|
+
- `/delegate --allow-dangerous` (and the `/claude`, `/codex`, … aliases) is the human-typed counterpart: it also asks you to confirm (`Allow dangerous delegation?`, naming the harness(es), mode, and that it runs with full, unrestricted permissions) — a fan-out gets one prompt covering every harness — and a decline runs nothing. In a non-interactive session it is refused outright, since there's no one to confirm with. It applies to that invocation only.
|
|
268
|
+
- Likewise, when the **model** passes `addDirs` on the `delegate` tool, entries that resolve (after `..` and symlinks) outside the working directory need your interactive confirmation (`Allow access outside the project?`) and are refused in a non-interactive session. Entries inside the working directory, template `addDirs:`, and `/delegate --add-dir` are not gated.
|
|
269
|
+
- Values that end up on a harness's command line (`sessionId`/`--resume`, `model`, `pr`) are validated first — notably, nothing starting with `-` is accepted, so a prompt-injected value can't masquerade as a CLI flag.
|
|
214
270
|
|
|
215
271
|
Review what the harness is asked to do before granting broad permissions.
|
|
216
272
|
|
package/extensions/acp-runner.ts
CHANGED
|
@@ -21,7 +21,14 @@
|
|
|
21
21
|
import { spawn } from 'node:child_process';
|
|
22
22
|
import { createInterface } from 'node:readline';
|
|
23
23
|
import { DEFAULT_TIMEOUT_MS, type Harness, type ParseState, type StreamedResult } from './harnesses/types.ts';
|
|
24
|
-
import
|
|
24
|
+
import {
|
|
25
|
+
budgetStoppedResult,
|
|
26
|
+
type HarnessResult,
|
|
27
|
+
isCostOverBudget,
|
|
28
|
+
isOverBudget,
|
|
29
|
+
type RunHarnessOptions,
|
|
30
|
+
} from './runner.ts';
|
|
31
|
+
import { appendActivities, appendStreamed } from './stream-caps.ts';
|
|
25
32
|
|
|
26
33
|
/** Bound on the initial handshake (initialize / session/new / session/set_mode) so a hung agent
|
|
27
34
|
* doesn't wedge the whole `timeoutMs` budget before `session/prompt` — the actual work — even starts. */
|
|
@@ -60,14 +67,78 @@ function supportsSessionModes(sessionResult: unknown): boolean {
|
|
|
60
67
|
return false;
|
|
61
68
|
}
|
|
62
69
|
|
|
70
|
+
/**
|
|
71
|
+
* The cost to report for *this* run of a session whose harness reports a session-cumulative running
|
|
72
|
+
* total (opencode over ACP: `usage_update.cost.amount`). On a fresh session that total is this run's
|
|
73
|
+
* spend as-is. On a resume it also includes every prior turn — reporting it unchanged would make
|
|
74
|
+
* `/delegate status`'s spend rollup count those turns again on every resume — so the delta since
|
|
75
|
+
* `baselineUsd` (the same baseline the host budget check uses; see `costBaseline` in
|
|
76
|
+
* `runAcpHarness`) is reported instead.
|
|
77
|
+
*
|
|
78
|
+
* When the baseline wasn't replayed before the prompt (`baselineFromReplay: false`) it is the first
|
|
79
|
+
* running total seen *after* the prompt, so any delta silently omits that first step's spend — an
|
|
80
|
+
* under-report that looks measured (with one sample it's even a fake "$0"). That case is reported
|
|
81
|
+
* as `null` (unmeasured), per the honest-metrics convention. The host budget check still uses that
|
|
82
|
+
* first-sample baseline on purpose (biased toward never killing a resumed run for prior turns'
|
|
83
|
+
* spend); only the *reported* figure refuses to guess. Pure.
|
|
84
|
+
*/
|
|
85
|
+
export function resumedRunCost(opts: {
|
|
86
|
+
totalCostUsd: number | null;
|
|
87
|
+
resumed: boolean;
|
|
88
|
+
baselineUsd: number | undefined;
|
|
89
|
+
baselineFromReplay: boolean;
|
|
90
|
+
postPromptCostSamples: number;
|
|
91
|
+
}): number | null {
|
|
92
|
+
const { totalCostUsd, resumed, baselineUsd, baselineFromReplay, postPromptCostSamples } = opts;
|
|
93
|
+
if (!resumed || totalCostUsd === null) return totalCostUsd;
|
|
94
|
+
// no cost reported during this run at all — whatever total we hold is prior turns' spend
|
|
95
|
+
if (baselineUsd === undefined || postPromptCostSamples === 0) return null;
|
|
96
|
+
// baseline is this run's own first sample: the true spend is unknowable, not "the delta"
|
|
97
|
+
if (!baselineFromReplay) return null;
|
|
98
|
+
return Math.max(0, totalCostUsd - baselineUsd);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Placeholder for a run the host stops mid-turn before the harness produced anything result-shaped. */
|
|
102
|
+
function emptyResult(): StreamedResult {
|
|
103
|
+
return {
|
|
104
|
+
result: '',
|
|
105
|
+
isError: false,
|
|
106
|
+
numTurns: null,
|
|
107
|
+
totalCostUsd: null,
|
|
108
|
+
sessionId: null,
|
|
109
|
+
stopReason: null,
|
|
110
|
+
permissionDenials: [],
|
|
111
|
+
durationMs: null,
|
|
112
|
+
durationApiMs: null,
|
|
113
|
+
ttftMs: null,
|
|
114
|
+
model: null,
|
|
115
|
+
contextWindow: null,
|
|
116
|
+
maxOutputTokens: null,
|
|
117
|
+
usage: null,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
63
121
|
interface PendingRequest {
|
|
64
122
|
resolve: (result: unknown) => void;
|
|
65
123
|
reject: (err: Error) => void;
|
|
66
124
|
timer?: ReturnType<typeof setTimeout>;
|
|
67
125
|
}
|
|
68
126
|
|
|
69
|
-
|
|
127
|
+
/** Test-only knobs — production callers pass only `opts`. */
|
|
128
|
+
export interface AcpRunnerInternals {
|
|
129
|
+
/** Overrides `HANDSHAKE_TIMEOUT_MS` so a hung-handshake test doesn't take 30s. */
|
|
130
|
+
handshakeTimeoutMs?: number;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export function runAcpHarness(opts: RunHarnessOptions, internals: AcpRunnerInternals = {}): Promise<HarnessResult> {
|
|
134
|
+
const handshakeTimeoutMs = internals.handshakeTimeoutMs ?? HANDSHAKE_TIMEOUT_MS;
|
|
70
135
|
return new Promise((resolve, reject) => {
|
|
136
|
+
// Already cancelled (e.g. the user hit cancel while this run was still being set up) —
|
|
137
|
+
// never spawn a process just to kill it.
|
|
138
|
+
if (opts.signal?.aborted) {
|
|
139
|
+
reject(new Error('cancelled'));
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
71
142
|
const args = opts.harness.buildArgs({
|
|
72
143
|
prompt: opts.prompt,
|
|
73
144
|
cwd: opts.cwd,
|
|
@@ -80,6 +151,10 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
80
151
|
});
|
|
81
152
|
|
|
82
153
|
const proc = spawn(opts.harness.binary, args, { cwd: opts.cwd, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
154
|
+
// A write to a child that already exited surfaces as an async EPIPE 'error' on stdin — not as a
|
|
155
|
+
// throw from write() (writeLine's try/catch can't see it). Unhandled, that event would crash the
|
|
156
|
+
// host process; the run itself already ends through the 'close'/timeout/abort paths.
|
|
157
|
+
proc.stdin.on('error', () => {});
|
|
83
158
|
|
|
84
159
|
const state: ParseState = { streamedText: '', activities: [], result: null, _harness: {} };
|
|
85
160
|
let stderr = '';
|
|
@@ -89,25 +164,48 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
89
164
|
// prompt's — set once session/prompt is actually sent, so replayed text/activities (and a
|
|
90
165
|
// replay-skewed TTFT) never reach the caller. See the handshake IIFE below.
|
|
91
166
|
let promptSent = false;
|
|
167
|
+
// Host budget accounting. opencode's ACP `usage_update.cost` is a *session* running total, so
|
|
168
|
+
// on a resume it already includes every prior turn — `costBaseline` is what the session had
|
|
169
|
+
// spent before this run's prompt, and only the delta counts against `maxBudgetUsd`. A fresh
|
|
170
|
+
// session starts at 0. On a resume it's the last cost replayed before the prompt (if any), else
|
|
171
|
+
// the first cost seen after it — which under-counts that first step, a deliberate bias toward
|
|
172
|
+
// never killing a resumed run for spend it didn't incur.
|
|
173
|
+
let prePromptCost: number | undefined;
|
|
174
|
+
let costBaseline: number | undefined;
|
|
175
|
+
let postPromptCostSamples = 0;
|
|
92
176
|
const startAt = Date.now();
|
|
93
|
-
const MAX_STREAMED = 5 * 1024 * 1024; // 5MB cap to prevent OOM on compromised harness
|
|
94
|
-
const MAX_ACTIVITIES = 5000;
|
|
95
177
|
|
|
96
178
|
let nextId = 1;
|
|
97
179
|
const pending = new Map<number, PendingRequest>();
|
|
98
180
|
|
|
99
|
-
const finish = (r: StreamedResult) => {
|
|
181
|
+
const finish = (r: StreamedResult, budgetExceeded = false) => {
|
|
100
182
|
if (settled) return;
|
|
101
183
|
settled = true;
|
|
102
184
|
clearTimeout(timer);
|
|
185
|
+
opts.signal?.removeEventListener('abort', onAbort);
|
|
103
186
|
rejectAllPending(new Error('session ended'));
|
|
104
187
|
const ttft = firstTokenAt !== null ? firstTokenAt - startAt : r.ttftMs;
|
|
105
|
-
resolve({
|
|
188
|
+
resolve({
|
|
189
|
+
...r,
|
|
190
|
+
// a resumed session's running total includes prior turns — report only this run's share
|
|
191
|
+
totalCostUsd: resumedRunCost({
|
|
192
|
+
totalCostUsd: r.totalCostUsd,
|
|
193
|
+
resumed: Boolean(opts.resumeSessionId),
|
|
194
|
+
baselineUsd: costBaseline,
|
|
195
|
+
baselineFromReplay: prePromptCost !== undefined,
|
|
196
|
+
postPromptCostSamples,
|
|
197
|
+
}),
|
|
198
|
+
ttftMs: ttft,
|
|
199
|
+
streamedText: state.streamedText,
|
|
200
|
+
harness: opts.harness.name,
|
|
201
|
+
...(budgetExceeded ? { budgetExceeded: true } : {}),
|
|
202
|
+
});
|
|
106
203
|
};
|
|
107
204
|
const fail = (err: Error) => {
|
|
108
205
|
if (settled) return;
|
|
109
206
|
settled = true;
|
|
110
207
|
clearTimeout(timer);
|
|
208
|
+
opts.signal?.removeEventListener('abort', onAbort);
|
|
111
209
|
rejectAllPending(err);
|
|
112
210
|
reject(err);
|
|
113
211
|
};
|
|
@@ -210,31 +308,56 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
210
308
|
}
|
|
211
309
|
|
|
212
310
|
const outcome = opts.harness.parseLine(line, state);
|
|
311
|
+
if (typeof outcome.runningCostUsd === 'number') {
|
|
312
|
+
if (!promptSent) prePromptCost = outcome.runningCostUsd;
|
|
313
|
+
else {
|
|
314
|
+
postPromptCostSamples++;
|
|
315
|
+
costBaseline ??= outcome.runningCostUsd;
|
|
316
|
+
if (!settled && isCostOverBudget(opts.harness, opts.maxBudgetUsd, outcome.runningCostUsd, costBaseline)) {
|
|
317
|
+
proc.kill('SIGKILL');
|
|
318
|
+
const partial = state.result ?? opts.harness.extractResult(state) ?? emptyResult();
|
|
319
|
+
const latchedId = state._harness?.sessionId;
|
|
320
|
+
finish(
|
|
321
|
+
budgetStoppedResult(
|
|
322
|
+
{
|
|
323
|
+
...partial,
|
|
324
|
+
totalCostUsd: outcome.runningCostUsd,
|
|
325
|
+
sessionId: partial.sessionId ?? (typeof latchedId === 'string' ? latchedId : null),
|
|
326
|
+
},
|
|
327
|
+
state.streamedText,
|
|
328
|
+
),
|
|
329
|
+
true,
|
|
330
|
+
);
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
213
335
|
// Discard streamed text/activities from anything that arrives before the new session/prompt
|
|
214
336
|
// is sent — on a resume that's the replayed prior conversation, not the new turn's own output.
|
|
215
337
|
if (promptSent && outcome.streamedText) {
|
|
216
338
|
if (firstTokenAt === null) firstTokenAt = Date.now();
|
|
217
|
-
|
|
218
|
-
const remaining = MAX_STREAMED - state.streamedText.length;
|
|
219
|
-
const chunk =
|
|
220
|
-
outcome.streamedText.length > remaining
|
|
221
|
-
? `${outcome.streamedText.slice(0, remaining)} [truncated ${outcome.streamedText.length - remaining} chars]`
|
|
222
|
-
: outcome.streamedText;
|
|
223
|
-
state.streamedText += chunk;
|
|
224
|
-
opts.onStream?.(chunk);
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
|
-
if (promptSent && outcome.activities) {
|
|
228
|
-
for (const a of outcome.activities) {
|
|
229
|
-
if (state.activities.length < MAX_ACTIVITIES) {
|
|
230
|
-
state.activities.push(a);
|
|
231
|
-
opts.onActivity?.(a);
|
|
232
|
-
}
|
|
233
|
-
}
|
|
339
|
+
appendStreamed(state, outcome.streamedText, opts.onStream); // capped — see stream-caps.ts
|
|
234
340
|
}
|
|
341
|
+
if (promptSent && outcome.activities) appendActivities(state, outcome.activities, opts.onActivity);
|
|
235
342
|
if (outcome.result) {
|
|
236
343
|
if (!outcome.result.result) outcome.result.result = state.streamedText;
|
|
237
344
|
state.result = outcome.result;
|
|
345
|
+
// Same host-side budget enforcement as runner.ts (see isOverBudget). Replayed history
|
|
346
|
+
// from a resume can't trip it — only once the new prompt is in flight.
|
|
347
|
+
if (
|
|
348
|
+
promptSent &&
|
|
349
|
+
!settled &&
|
|
350
|
+
isOverBudget(
|
|
351
|
+
opts.harness,
|
|
352
|
+
opts.maxBudgetUsd,
|
|
353
|
+
state.result,
|
|
354
|
+
// a resume that never saw a cost during this run has no evidence of new spend at all
|
|
355
|
+
costBaseline ?? (opts.resumeSessionId ? Number.POSITIVE_INFINITY : 0),
|
|
356
|
+
)
|
|
357
|
+
) {
|
|
358
|
+
proc.kill('SIGKILL');
|
|
359
|
+
finish(budgetStoppedResult(state.result, state.streamedText), true);
|
|
360
|
+
}
|
|
238
361
|
}
|
|
239
362
|
});
|
|
240
363
|
|
|
@@ -265,14 +388,13 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
265
388
|
}, opts.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
266
389
|
timer.unref?.();
|
|
267
390
|
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
);
|
|
391
|
+
// Removed again in finish()/fail() — a long-lived signal (e.g. one shared by a fan-out, or the
|
|
392
|
+
// tool call's own) must not accumulate a dead listener per run.
|
|
393
|
+
function onAbort(): void {
|
|
394
|
+
proc.kill('SIGKILL');
|
|
395
|
+
fail(new Error('cancelled'));
|
|
396
|
+
}
|
|
397
|
+
opts.signal?.addEventListener('abort', onAbort, { once: true });
|
|
276
398
|
|
|
277
399
|
// Drive the handshake. `session/prompt` has no separate timeout — it's the actual work,
|
|
278
400
|
// bounded by the overall `timer` above like everything else.
|
|
@@ -284,7 +406,7 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
284
406
|
protocolVersion: PROTOCOL_VERSION,
|
|
285
407
|
clientCapabilities: {}, // no fs/terminal proxying — decline those requests if asked (see handleServerRequest)
|
|
286
408
|
},
|
|
287
|
-
|
|
409
|
+
handshakeTimeoutMs,
|
|
288
410
|
);
|
|
289
411
|
if (settled) return;
|
|
290
412
|
// The client "should disconnect" (spec text) if the agent didn't echo back the version we
|
|
@@ -312,11 +434,11 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
312
434
|
sessionResult = await sendRequest(
|
|
313
435
|
'session/load',
|
|
314
436
|
{ sessionId: opts.resumeSessionId, ...sessionParams },
|
|
315
|
-
|
|
437
|
+
handshakeTimeoutMs,
|
|
316
438
|
);
|
|
317
439
|
sessionId = opts.resumeSessionId;
|
|
318
440
|
} else {
|
|
319
|
-
sessionResult = await sendRequest('session/new', sessionParams,
|
|
441
|
+
sessionResult = await sendRequest('session/new', sessionParams, handshakeTimeoutMs);
|
|
320
442
|
sessionId =
|
|
321
443
|
isRecord(sessionResult) && typeof sessionResult.sessionId === 'string' ? sessionResult.sessionId : null;
|
|
322
444
|
}
|
|
@@ -342,9 +464,10 @@ export function runAcpHarness(opts: RunHarnessOptions): Promise<HarnessResult> {
|
|
|
342
464
|
`cannot verify the "${opts.permission}" permission tier would be honored over ACP`,
|
|
343
465
|
);
|
|
344
466
|
}
|
|
345
|
-
await sendRequest('session/set_mode', { sessionId, modeId },
|
|
467
|
+
await sendRequest('session/set_mode', { sessionId, modeId }, handshakeTimeoutMs);
|
|
346
468
|
if (settled) return;
|
|
347
469
|
promptSent = true;
|
|
470
|
+
costBaseline = opts.resumeSessionId ? prePromptCost : 0;
|
|
348
471
|
await sendRequest('session/prompt', { sessionId, prompt: [{ type: 'text', text: opts.prompt }] });
|
|
349
472
|
if (settled) return;
|
|
350
473
|
// The agent doesn't exit on its own once the turn is done — an ACP session can outlive a
|
package/extensions/activity.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readdirSync, rmSync, statSync } from 'node:fs';
|
|
1
|
+
import { chmodSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import type { ActivityEvent, NormalizedPermission } from './harnesses/types.ts';
|
|
4
4
|
|
|
@@ -169,6 +169,68 @@ export function formatVerifySection(v: VerifyResult): string {
|
|
|
169
169
|
return lines.join('\n');
|
|
170
170
|
}
|
|
171
171
|
|
|
172
|
+
/** How a run's `maxBudgetUsd` was (or wasn't) enforced, and whether it was exceeded. */
|
|
173
|
+
export interface BudgetNote {
|
|
174
|
+
limitUsd: number;
|
|
175
|
+
/** `native`: the harness CLI enforces it itself. `host`: best-effort — the runner kills the run
|
|
176
|
+
* when the harness's streamed cost (checked only at the step/turn boundaries it reports) is over
|
|
177
|
+
* the cap, so spend can overshoot by up to one step/turn. `unenforced`: neither — no native flag and no cost reported. */
|
|
178
|
+
enforcement: 'native' | 'host' | 'unenforced';
|
|
179
|
+
exceeded: boolean;
|
|
180
|
+
/** Human-readable warning, or null when there's nothing to warn about. */
|
|
181
|
+
message: string | null;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Pure: describe a run's budget outcome. Undefined when no budget was set. `stoppedByHost` is the
|
|
186
|
+
* runner's own `budgetExceeded` flag; `costUsd` is the run's final `totalCostUsd` (null = unmeasured).
|
|
187
|
+
*/
|
|
188
|
+
export function describeBudget(opts: {
|
|
189
|
+
harness: string;
|
|
190
|
+
limitUsd: number | undefined;
|
|
191
|
+
native: boolean;
|
|
192
|
+
costUsd: number | null;
|
|
193
|
+
stoppedByHost: boolean;
|
|
194
|
+
}): BudgetNote | undefined {
|
|
195
|
+
const { harness, limitUsd, native, costUsd, stoppedByHost } = opts;
|
|
196
|
+
if (limitUsd === undefined) return undefined;
|
|
197
|
+
const limit = `$${limitUsd.toFixed(3)}`;
|
|
198
|
+
const cost = costUsd !== null ? `$${costUsd.toFixed(3)}` : '$—';
|
|
199
|
+
if (native) {
|
|
200
|
+
const exceeded = costUsd !== null && costUsd > limitUsd;
|
|
201
|
+
return {
|
|
202
|
+
limitUsd,
|
|
203
|
+
enforcement: 'native',
|
|
204
|
+
exceeded,
|
|
205
|
+
message: exceeded ? `⛔ budget exceeded: ${cost} spent against a ${limit} cap (enforced by ${harness})` : null,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
if (stoppedByHost) {
|
|
209
|
+
return {
|
|
210
|
+
limitUsd,
|
|
211
|
+
enforcement: 'host',
|
|
212
|
+
exceeded: true,
|
|
213
|
+
message: `⛔ budget exceeded: ${harness} reported ${cost} against a ${limit} cap — run stopped (best-effort: cost is only checked at each step/turn the harness reports, so spend can overshoot the cap)`,
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
if (costUsd === null) {
|
|
217
|
+
return {
|
|
218
|
+
limitUsd,
|
|
219
|
+
enforcement: 'unenforced',
|
|
220
|
+
exceeded: false,
|
|
221
|
+
message: `⚠ maxBudgetUsd ${limit} was not enforced — ${harness} has no native budget flag and reported no cost`,
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
return { limitUsd, enforcement: 'host', exceeded: costUsd > limitUsd, message: null };
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/** One transcript header line for a budget note. */
|
|
228
|
+
export function formatBudgetLine(b: BudgetNote): string {
|
|
229
|
+
const how =
|
|
230
|
+
b.enforcement === 'native' ? 'native' : b.enforcement === 'host' ? 'host-enforced, best-effort' : 'NOT enforced';
|
|
231
|
+
return `- budget: $${b.limitUsd.toFixed(3)} (${how})${b.exceeded ? ' · budget exceeded' : ''}`;
|
|
232
|
+
}
|
|
233
|
+
|
|
172
234
|
/** Build the markdown report content injected into the session on the next turn. */
|
|
173
235
|
export function buildReportContent(opts: {
|
|
174
236
|
harness?: string;
|
|
@@ -256,6 +318,30 @@ export function buildClaudeReportContent(opts: {
|
|
|
256
318
|
}
|
|
257
319
|
|
|
258
320
|
/** Delete oldest transcript files beyond `maxCount` (0 = keep everything). */
|
|
321
|
+
/**
|
|
322
|
+
* Write one transcript to `dir` (created if needed) and return its path. Transcripts hold the
|
|
323
|
+
* delegated prompt, repo diffs, and the harness's full output — owner-only: the directory is
|
|
324
|
+
* `0700` and each file `0600` (chmod'd explicitly too, since `mkdirSync`'s `mode` only applies to
|
|
325
|
+
* directories it creates and is subject to the umask).
|
|
326
|
+
*/
|
|
327
|
+
export function writeTranscript(dir: string, mode: string, text: string): string {
|
|
328
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
329
|
+
try {
|
|
330
|
+
chmodSync(dir, 0o700);
|
|
331
|
+
} catch {
|
|
332
|
+
// best-effort — e.g. a dir owned by someone else; the file mode below still applies
|
|
333
|
+
}
|
|
334
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
335
|
+
const file = join(dir, `${stamp}-${safeSegmentName(mode)}.md`);
|
|
336
|
+
writeFileSync(file, text, { encoding: 'utf8', mode: 0o600 });
|
|
337
|
+
try {
|
|
338
|
+
chmodSync(file, 0o600);
|
|
339
|
+
} catch {
|
|
340
|
+
// best-effort
|
|
341
|
+
}
|
|
342
|
+
return file;
|
|
343
|
+
}
|
|
344
|
+
|
|
259
345
|
export function pruneOutputs(dir: string, maxCount: number): void {
|
|
260
346
|
if (maxCount <= 0) return;
|
|
261
347
|
let files: string[];
|
|
@@ -373,6 +459,8 @@ export function buildTranscript(
|
|
|
373
459
|
output: string;
|
|
374
460
|
/** Host-run verification result, when a `verify` command was configured for this run. */
|
|
375
461
|
verify?: VerifyResult;
|
|
462
|
+
/** Budget outcome, when `maxBudgetUsd` was set for this run. */
|
|
463
|
+
budget?: BudgetNote;
|
|
376
464
|
} & Record<string, unknown>,
|
|
377
465
|
): string {
|
|
378
466
|
const harness = (opts.harness as string | undefined) ?? 'claude';
|
|
@@ -426,6 +514,8 @@ export function buildTranscript(
|
|
|
426
514
|
`- context: ${context ?? 'n/a'}`,
|
|
427
515
|
`- duration: ${duration ?? 'n/a'}`,
|
|
428
516
|
`- stop reason: ${opts.stopReason ?? 'n/a'}`,
|
|
517
|
+
...(opts.budget ? [formatBudgetLine(opts.budget)] : []),
|
|
518
|
+
...(opts.budget?.message ? ['', opts.budget.message] : []),
|
|
429
519
|
'',
|
|
430
520
|
'## Activity',
|
|
431
521
|
opts.activityLog.length > 0 ? opts.activityLog.join('\n') : '(no tool activity)',
|