run-harness 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- run_harness-0.1.0/PKG-INFO +254 -0
- run_harness-0.1.0/README.md +247 -0
- run_harness-0.1.0/pyproject.toml +36 -0
- run_harness-0.1.0/pyproject.toml.orig +29 -0
- run_harness-0.1.0/src/run_harness/__init__.py +1 -0
- run_harness-0.1.0/src/run_harness/__main__.py +3 -0
- run_harness-0.1.0/src/run_harness/adapters/__init__.py +16 -0
- run_harness-0.1.0/src/run_harness/adapters/base.py +156 -0
- run_harness-0.1.0/src/run_harness/adapters/claude.py +101 -0
- run_harness-0.1.0/src/run_harness/adapters/codex.py +43 -0
- run_harness-0.1.0/src/run_harness/adapters/opencode.py +243 -0
- run_harness-0.1.0/src/run_harness/adapters/pi.py +113 -0
- run_harness-0.1.0/src/run_harness/cli.py +182 -0
- run_harness-0.1.0/src/run_harness/request.py +30 -0
- run_harness-0.1.0/src/run_harness/runner.py +102 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: run-harness
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: One CLI for launching coding-agent harnesses non-interactively.
|
|
5
|
+
Requires-Python: >=3.12
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
|
|
8
|
+
# run-harness
|
|
9
|
+
|
|
10
|
+
`run-harness` launches one coding-agent harness (Claude Code, Codex, Pi or OpenCode)
|
|
11
|
+
non-interactively, through one set of options, and prints the harness's final message on stdout.
|
|
12
|
+
It builds the command line, starts the harness, extracts the answer and maps the exit code. It is
|
|
13
|
+
not an agent runtime, and it never reads or writes harness config files.
|
|
14
|
+
|
|
15
|
+
Callers are mostly other programs that run it as a subprocess, plus people at a terminal. There is
|
|
16
|
+
no Python library API. The detailed contract is in [docs/spec.md](docs/spec.md).
|
|
17
|
+
|
|
18
|
+
Supported: Linux and macOS, Python 3.12+. Windows is not supported.
|
|
19
|
+
|
|
20
|
+
## Install and use
|
|
21
|
+
|
|
22
|
+
```sh
|
|
23
|
+
uvx run-harness claude "Summarise the architecture of this repo"
|
|
24
|
+
uvx run-harness codex "List the public functions in src/" -m gpt-5-codex -e high
|
|
25
|
+
uvx run-harness pi "Where is the config parsed?" -C ~/code/project
|
|
26
|
+
uvx run-harness opencode "Explain the build" -m anthropic/claude-sonnet-4-5 -e high
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
You can also install it with `uv tool install run-harness`. The harness itself (`claude`, `codex`,
|
|
30
|
+
`pi`, `opencode`) must already be installed, logged in and on `PATH`. run-harness passes the
|
|
31
|
+
environment and auth through untouched.
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
run-harness <harness> [PROMPT] [-m MODEL] [-e {low,medium,high,max}] [-f PATH] [-C DIR] [--autonomous] [-- PASSTHROUGH...]
|
|
35
|
+
run-harness --version | --help
|
|
36
|
+
run-harness <harness> --help
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
Only the final message goes to stdout, with a trailing newline added if it is missing. Progress and
|
|
40
|
+
diagnostics go to stderr.
|
|
41
|
+
|
|
42
|
+
| option | meaning |
|
|
43
|
+
|---|---|
|
|
44
|
+
| `PROMPT` | the prompt text, or `-` to read the prompt from run-harness's own stdin |
|
|
45
|
+
| `-f, --prompt-file PATH` | read the prompt from a file |
|
|
46
|
+
| `-m, --model MODEL` | model, passed verbatim and never validated |
|
|
47
|
+
| `-e, --effort LEVEL` | `low`, `medium`, `high` or `max`. If omitted, the harness default applies |
|
|
48
|
+
| `-C, --cwd DIR` | directory the harness runs in (default: the current directory) |
|
|
49
|
+
| `--autonomous` | lift the read-only restrictions |
|
|
50
|
+
| `-- …` | passthrough arguments, handed to the harness verbatim |
|
|
51
|
+
|
|
52
|
+
Options may come before or after `PROMPT`. An unknown option before `--` is an error (exit 2).
|
|
53
|
+
Abbreviated long options are not accepted. `--model ""` is an error.
|
|
54
|
+
|
|
55
|
+
### Prompt source
|
|
56
|
+
|
|
57
|
+
Give exactly one of: `PROMPT`, `PROMPT` = `-` (read stdin), or `-f PATH`. None, or more than one,
|
|
58
|
+
is exit 2. Empty or whitespace-only prompts and prompts that are not valid UTF-8 are exit 2. A
|
|
59
|
+
Claude Code prompt over 10 MiB is rejected. No other harness limit is checked.
|
|
60
|
+
|
|
61
|
+
- run-harness reads its own stdin only when `PROMPT` is `-`. `-f -` is rejected.
|
|
62
|
+
- `-f PATH` is resolved against the directory you ran run-harness from, not `--cwd`. FIFOs and
|
|
63
|
+
process substitution (`-f <(...)`) work. Directories are rejected.
|
|
64
|
+
- The prompt reaches the harness on its stdin, never as an argv argument. It is passed verbatim.
|
|
65
|
+
- A prompt starting with `-` cannot be given as `PROMPT`. **Send untrusted text, and anything that
|
|
66
|
+
starts with `-`, through `-f` or stdin:**
|
|
67
|
+
|
|
68
|
+
```sh
|
|
69
|
+
printf '%s' "$UNTRUSTED" | uvx run-harness claude -
|
|
70
|
+
uvx run-harness codex -f prompt.md
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
### Working directory
|
|
74
|
+
|
|
75
|
+
`--cwd` must exist and be a directory. It is made absolute without resolving symlinks, and it is
|
|
76
|
+
applied both as the harness's working directory and as `PWD` in its environment. No harness gets
|
|
77
|
+
its own directory flag (`codex -C`, `opencode --dir`).
|
|
78
|
+
|
|
79
|
+
### Passthrough
|
|
80
|
+
|
|
81
|
+
Everything after the first `--` is appended after the arguments run-harness generates, and is never
|
|
82
|
+
inspected:
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
uvx run-harness claude "Review this" -- --max-turns 5
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Conflicts with the normalized options are your responsibility. See "do not override" below. If an
|
|
89
|
+
override breaks final-message extraction, the run exits 1.
|
|
90
|
+
|
|
91
|
+
## Read-only and autonomous
|
|
92
|
+
|
|
93
|
+
Read-only is the default. The model may read and reason, but cannot change files or reach the
|
|
94
|
+
network. It may run a command only where a sandbox stops that command from writing (Codex).
|
|
95
|
+
`--autonomous`
|
|
96
|
+
lifts that and permits writes, command execution and network access, in whatever way each harness
|
|
97
|
+
does it (see the table below).
|
|
98
|
+
|
|
99
|
+
The limits matter:
|
|
100
|
+
|
|
101
|
+
- run-harness only sets the harness's own flags. It does not sandbox anything itself.
|
|
102
|
+
- **Hooks and extensions still run.** User and project hooks run in Claude Code's read-only mode,
|
|
103
|
+
and Pi extensions still load and run their hooks. Only the tools are restricted.
|
|
104
|
+
- **User deny rules still apply** under `--autonomous` (Claude Code `deny` settings, deny rules in
|
|
105
|
+
the OpenCode config).
|
|
106
|
+
- **Codex read-only still lets the model run commands.** They run inside a sandbox that blocks
|
|
107
|
+
writes and network access.
|
|
108
|
+
- Pi has no permission system. Its read-only mode is a tool allowlist.
|
|
109
|
+
- A permission that is blocked in read-only mode is not an error. OpenCode ends such a run with
|
|
110
|
+
exit 0.
|
|
111
|
+
- Org, managed or MDM OpenCode config merges after the config run-harness injects.
|
|
112
|
+
|
|
113
|
+
## Per-harness mapping
|
|
114
|
+
|
|
115
|
+
| | Claude Code | Codex | Pi | OpenCode |
|
|
116
|
+
|---|---|---|---|---|
|
|
117
|
+
| base command | `claude -p --output-format stream-json --verbose` | `codex exec --skip-git-repo-check --color never` | `pi --mode json` | `opencode run --standalone --format json` |
|
|
118
|
+
| prompt | stdin | stdin | stdin | stdin |
|
|
119
|
+
| `--model M` | `--model M` | `-m M` | `--model M` | `-m M` |
|
|
120
|
+
| `--effort L` | `--effort L` | `-c model_reasoning_effort=L` | `--thinking L` | `-m M#L` (pre-checked) |
|
|
121
|
+
| read-only (default) | `--tools Read,Grep,Glob --permission-mode dontAsk --strict-mcp-config` | `-s read-only` | `--tools read,grep,find,ls` (plus `--no-mcp` on Pi 1.x and later) | injected agent plus `--agent run-harness-readonly` (pre-checked with `opencode api`) |
|
|
122
|
+
| `--autonomous` | `--dangerously-skip-permissions` | `--dangerously-bypass-approvals-and-sandbox` | nothing added | `--auto` |
|
|
123
|
+
| final message | `result` of the last `result` event | captured stdout | text of the last assistant `message_end` | text parts of the last assistant message |
|
|
124
|
+
| failure despite exit 0 | `is_error: true`, or no `result` | none | `stopReason` is `error` or `aborted` | an `error` event |
|
|
125
|
+
| do not override | `-p`, `--output-format`, `--verbose`, `--tools`, `--permission-mode` | `-s`, `--color`, `--json`, `-o` | `--mode`, `--tools`, `-p` | `--format`, `--agent`, `--dir` |
|
|
126
|
+
|
|
127
|
+
Argument order is: base command, permission arguments, `--model`, effort, passthrough.
|
|
128
|
+
|
|
129
|
+
### Claude Code
|
|
130
|
+
|
|
131
|
+
- `CLAUDE_CODE_EFFORT_LEVEL` passes through untouched. Whether it or `--effort` wins is unverified.
|
|
132
|
+
- A user `maxEffortLevel` setting can silently cap `max`.
|
|
133
|
+
- Read-only uses a tool allowlist, not `plan` mode, and `--strict-mcp-config` drops MCP servers from
|
|
134
|
+
user and project config. Hooks still run.
|
|
135
|
+
- Bypass mode refuses to run as root outside a sandbox. That refusal comes through with Claude's
|
|
136
|
+
exit code.
|
|
137
|
+
- An auth failure (`Not logged in`) exits 1 with the message on stderr.
|
|
138
|
+
|
|
139
|
+
### Codex
|
|
140
|
+
|
|
141
|
+
- Codex's stdout, when piped, is the final message. It is buffered and written once at the end.
|
|
142
|
+
Its stderr is its own progress, forwarded unchanged.
|
|
143
|
+
- `-s read-only` is always passed explicitly. Projects marked `trusted` in `~/.codex/config.toml`
|
|
144
|
+
would otherwise default to `workspace-write`.
|
|
145
|
+
- An unknown model makes Codex warn ("fallback metadata") and fail the turn, so the run exits 1.
|
|
146
|
+
- There is no auth pre-check. An auth failure exits 1 after roughly 15 to 20 seconds of retries.
|
|
147
|
+
|
|
148
|
+
### Pi
|
|
149
|
+
|
|
150
|
+
- Before a read-only run, run-harness runs `pi --version`. Pi 1.x and later gets `--no-mcp`,
|
|
151
|
+
because `--tools` keeps MCP tools otherwise. Pi 0.x does not get it, since the flag does not
|
|
152
|
+
exist there. Unparseable output or a non-zero exit is exit 2, because read-only cannot be
|
|
153
|
+
guaranteed. `--autonomous` skips the check.
|
|
154
|
+
- `--thinking` is clamped silently per model.
|
|
155
|
+
- A `:<thinking>` suffix on `--model` is overridden by an explicit `--effort`.
|
|
156
|
+
- Extensions still load and run their hooks in read-only mode. Only their tools are hidden.
|
|
157
|
+
- Passthrough positional arguments starting with `@` become file attachments.
|
|
158
|
+
- Pi exits 0 when the model errors in JSON mode. run-harness turns that into exit 1.
|
|
159
|
+
|
|
160
|
+
### OpenCode
|
|
161
|
+
|
|
162
|
+
- Read-only injects an agent named `run-harness-readonly` through `OPENCODE_CONFIG_CONTENT`. It is
|
|
163
|
+
merged into your value if you set one, and invalid JSON there is exit 2. The agent allows only
|
|
164
|
+
`read`, `glob`, `grep`, `list` and `lsp`, and denies everything else, including `task`. Nothing
|
|
165
|
+
is persisted. Apart from `PWD`, this is the only environment variable run-harness changes.
|
|
166
|
+
- Supports OpenCode 2.x only. Every run passes `--standalone`: 2.x otherwise talks to a shared
|
|
167
|
+
background service that never sees the injected config, so the agent would be missing.
|
|
168
|
+
- Before launch, `opencode api --standalone GET /api/config` (same env and cwd) must show the
|
|
169
|
+
agent with exactly the injected permissions (otherwise exit 2). If a run still cannot load the
|
|
170
|
+
agent, OpenCode reports `Agent not found` and run-harness exits non-zero with nothing on stdout.
|
|
171
|
+
- **`--effort` requires `--model` in `provider/model` form.** run-harness passes it as
|
|
172
|
+
`-m provider/model#level` and first reads `opencode api --standalone GET /api/model`. It exits 2
|
|
173
|
+
if the model is not listed, has no variants, or the level is not one of its variants.
|
|
174
|
+
- **The model is validated only when `--effort` is given.** Without `--effort`, an unlisted model
|
|
175
|
+
fails at run time, and the run exits 1.
|
|
176
|
+
- A permission rejected in read-only mode ends the run with exit 0.
|
|
177
|
+
- OpenCode creates `.opencode/.gitignore` and installs its plugin package at startup. Those are its
|
|
178
|
+
own side effects.
|
|
179
|
+
|
|
180
|
+
### Effort
|
|
181
|
+
|
|
182
|
+
`low`, `medium`, `high` and `max` map one to one onto every harness, with no remapping (`max` is
|
|
183
|
+
never turned into `xhigh`). Claude Code, Codex and Pi adjust a level the model does not support to
|
|
184
|
+
the nearest one it does, and run-harness does not detect that. OpenCode rejects unknown variants
|
|
185
|
+
at run time, but run-harness checks them first so you get exit 2 and the valid levels. Native-only levels (`xhigh`, `minimal`, `off`, `none`
|
|
186
|
+
and so on) are reachable only through passthrough.
|
|
187
|
+
|
|
188
|
+
## Exit codes
|
|
189
|
+
|
|
190
|
+
| situation | exit | stdout |
|
|
191
|
+
|---|---|---|
|
|
192
|
+
| usage or validation error, or a failed pre-launch check | `2` | empty |
|
|
193
|
+
| harness executable (or a pre-check helper) not on `PATH` | `127` | empty |
|
|
194
|
+
| harness exits non-zero | the harness's code, verbatim | empty |
|
|
195
|
+
| harness killed by signal N | `128 + N` | empty |
|
|
196
|
+
| harness exits 0, but the event stream reports an error | `1` | empty |
|
|
197
|
+
| harness exits 0 with no final-message event | `1` | empty |
|
|
198
|
+
| harness exits 0 with a final message | `0` | the final message |
|
|
199
|
+
| harness exits 0 with an empty final message | `0` | empty (no newline added) |
|
|
200
|
+
| Codex exits 0 | `0` | the captured stdout |
|
|
201
|
+
|
|
202
|
+
stdout is written once, after the harness exits, and only when run-harness exits 0. On failure,
|
|
203
|
+
error text taken from the stream goes to stderr.
|
|
204
|
+
|
|
205
|
+
## stderr
|
|
206
|
+
|
|
207
|
+
stderr is for people. Its format is not stable and is not a contract. The harness's own stderr is
|
|
208
|
+
forwarded unchanged. For Claude Code, Pi and OpenCode, run-harness also prints the assistant text
|
|
209
|
+
and a `→ tool: <name>` line per tool call as events arrive. Lines that are not JSON are forwarded
|
|
210
|
+
raw. Codex has no renderer, since its stderr already is its progress.
|
|
211
|
+
|
|
212
|
+
## Tested harness versions
|
|
213
|
+
|
|
214
|
+
run-harness declares tested versions, not minimums. The fixtures come from these versions run
|
|
215
|
+
against a local fake model server (OpenCode: observed on 2.0.32 with a free model), plus
|
|
216
|
+
hand-written edge cases. Claude Code's and OpenCode's `success` and `tool_call` fixtures, and
|
|
217
|
+
Claude Code's `not_logged_in`, are recorded against a real model with the recorder script below.
|
|
218
|
+
Codex was checked with the live suite:
|
|
219
|
+
|
|
220
|
+
| harness | version |
|
|
221
|
+
|---|---|
|
|
222
|
+
| Claude Code | 2.1.303 |
|
|
223
|
+
| Codex | codex-cli 0.163.0 |
|
|
224
|
+
| Pi | 0.85.1 |
|
|
225
|
+
| OpenCode | 2.0.32 (2.x only) |
|
|
226
|
+
|
|
227
|
+
There are no general runtime version checks. The one exception is Pi's `--no-mcp`, because a
|
|
228
|
+
version difference there would make read-only fail open. Other versions may work, or may break
|
|
229
|
+
extraction. The opt-in live suite (below) is how drift is detected.
|
|
230
|
+
|
|
231
|
+
## Development
|
|
232
|
+
|
|
233
|
+
```sh
|
|
234
|
+
uv run pytest # unit and fake-harness tests; no real harness needed
|
|
235
|
+
uv run ruff check
|
|
236
|
+
uv run ruff format --check
|
|
237
|
+
uv run pytest -m live # opt-in smoke run against the real harnesses on PATH
|
|
238
|
+
uv run scripts/record_fixtures.py # re-record the fixtures from the real harnesses
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
The live suite never runs in CI. Run it locally before a release, and re-record the fixtures if it
|
|
242
|
+
fails.
|
|
243
|
+
|
|
244
|
+
## Release
|
|
245
|
+
|
|
246
|
+
Releases run from the Actions tab: run the **Release** workflow on `master`, choosing a version
|
|
247
|
+
`bump` (`none` releases the version already in `pyproject.toml`) and whether to publish to `pypi`.
|
|
248
|
+
|
|
249
|
+
The workflow runs lint and tests, applies the bump and builds, refusing a version that is already
|
|
250
|
+
tagged. It publishes to TestPyPI and smoke-tests the install on Linux and macOS
|
|
251
|
+
(`scripts/smoke_release.sh`). With `pypi` ticked it then publishes the same files to PyPI, commits
|
|
252
|
+
the bump, tags `vX.Y.Z`, creates a GitHub Release with generated notes and smoke-tests the PyPI
|
|
253
|
+
install. With `pypi` unticked it stops after TestPyPI, as a dry run. Both indexes use trusted
|
|
254
|
+
publishing for `release.yml`, with no environment.
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
# run-harness
|
|
2
|
+
|
|
3
|
+
`run-harness` launches one coding-agent harness (Claude Code, Codex, Pi or OpenCode)
|
|
4
|
+
non-interactively, through one set of options, and prints the harness's final message on stdout.
|
|
5
|
+
It builds the command line, starts the harness, extracts the answer and maps the exit code. It is
|
|
6
|
+
not an agent runtime, and it never reads or writes harness config files.
|
|
7
|
+
|
|
8
|
+
Callers are mostly other programs that run it as a subprocess, plus people at a terminal. There is
|
|
9
|
+
no Python library API. The detailed contract is in [docs/spec.md](docs/spec.md).
|
|
10
|
+
|
|
11
|
+
Supported: Linux and macOS, Python 3.12+. Windows is not supported.
|
|
12
|
+
|
|
13
|
+
## Install and use
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
uvx run-harness claude "Summarise the architecture of this repo"
|
|
17
|
+
uvx run-harness codex "List the public functions in src/" -m gpt-5-codex -e high
|
|
18
|
+
uvx run-harness pi "Where is the config parsed?" -C ~/code/project
|
|
19
|
+
uvx run-harness opencode "Explain the build" -m anthropic/claude-sonnet-4-5 -e high
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
You can also install it with `uv tool install run-harness`. The harness itself (`claude`, `codex`,
|
|
23
|
+
`pi`, `opencode`) must already be installed, logged in and on `PATH`. run-harness passes the
|
|
24
|
+
environment and auth through untouched.
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
run-harness <harness> [PROMPT] [-m MODEL] [-e {low,medium,high,max}] [-f PATH] [-C DIR] [--autonomous] [-- PASSTHROUGH...]
|
|
28
|
+
run-harness --version | --help
|
|
29
|
+
run-harness <harness> --help
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Only the final message goes to stdout, with a trailing newline added if it is missing. Progress and
|
|
33
|
+
diagnostics go to stderr.
|
|
34
|
+
|
|
35
|
+
| option | meaning |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `PROMPT` | the prompt text, or `-` to read the prompt from run-harness's own stdin |
|
|
38
|
+
| `-f, --prompt-file PATH` | read the prompt from a file |
|
|
39
|
+
| `-m, --model MODEL` | model, passed verbatim and never validated |
|
|
40
|
+
| `-e, --effort LEVEL` | `low`, `medium`, `high` or `max`. If omitted, the harness default applies |
|
|
41
|
+
| `-C, --cwd DIR` | directory the harness runs in (default: the current directory) |
|
|
42
|
+
| `--autonomous` | lift the read-only restrictions |
|
|
43
|
+
| `-- …` | passthrough arguments, handed to the harness verbatim |
|
|
44
|
+
|
|
45
|
+
Options may come before or after `PROMPT`. An unknown option before `--` is an error (exit 2).
|
|
46
|
+
Abbreviated long options are not accepted. `--model ""` is an error.
|
|
47
|
+
|
|
48
|
+
### Prompt source
|
|
49
|
+
|
|
50
|
+
Give exactly one of: `PROMPT`, `PROMPT` = `-` (read stdin), or `-f PATH`. None, or more than one,
|
|
51
|
+
is exit 2. Empty or whitespace-only prompts and prompts that are not valid UTF-8 are exit 2. A
|
|
52
|
+
Claude Code prompt over 10 MiB is rejected. No other harness limit is checked.
|
|
53
|
+
|
|
54
|
+
- run-harness reads its own stdin only when `PROMPT` is `-`. `-f -` is rejected.
|
|
55
|
+
- `-f PATH` is resolved against the directory you ran run-harness from, not `--cwd`. FIFOs and
|
|
56
|
+
process substitution (`-f <(...)`) work. Directories are rejected.
|
|
57
|
+
- The prompt reaches the harness on its stdin, never as an argv argument. It is passed verbatim.
|
|
58
|
+
- A prompt starting with `-` cannot be given as `PROMPT`. **Send untrusted text, and anything that
|
|
59
|
+
starts with `-`, through `-f` or stdin:**
|
|
60
|
+
|
|
61
|
+
```sh
|
|
62
|
+
printf '%s' "$UNTRUSTED" | uvx run-harness claude -
|
|
63
|
+
uvx run-harness codex -f prompt.md
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
### Working directory
|
|
67
|
+
|
|
68
|
+
`--cwd` must exist and be a directory. It is made absolute without resolving symlinks, and it is
|
|
69
|
+
applied both as the harness's working directory and as `PWD` in its environment. No harness gets
|
|
70
|
+
its own directory flag (`codex -C`, `opencode --dir`).
|
|
71
|
+
|
|
72
|
+
### Passthrough
|
|
73
|
+
|
|
74
|
+
Everything after the first `--` is appended after the arguments run-harness generates, and is never
|
|
75
|
+
inspected:
|
|
76
|
+
|
|
77
|
+
```sh
|
|
78
|
+
uvx run-harness claude "Review this" -- --max-turns 5
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Conflicts with the normalized options are your responsibility. See "do not override" below. If an
|
|
82
|
+
override breaks final-message extraction, the run exits 1.
|
|
83
|
+
|
|
84
|
+
## Read-only and autonomous
|
|
85
|
+
|
|
86
|
+
Read-only is the default. The model may read and reason, but cannot change files or reach the
|
|
87
|
+
network. It may run a command only where a sandbox stops that command from writing (Codex).
|
|
88
|
+
`--autonomous`
|
|
89
|
+
lifts that and permits writes, command execution and network access, in whatever way each harness
|
|
90
|
+
does it (see the table below).
|
|
91
|
+
|
|
92
|
+
The limits matter:
|
|
93
|
+
|
|
94
|
+
- run-harness only sets the harness's own flags. It does not sandbox anything itself.
|
|
95
|
+
- **Hooks and extensions still run.** User and project hooks run in Claude Code's read-only mode,
|
|
96
|
+
and Pi extensions still load and run their hooks. Only the tools are restricted.
|
|
97
|
+
- **User deny rules still apply** under `--autonomous` (Claude Code `deny` settings, deny rules in
|
|
98
|
+
the OpenCode config).
|
|
99
|
+
- **Codex read-only still lets the model run commands.** They run inside a sandbox that blocks
|
|
100
|
+
writes and network access.
|
|
101
|
+
- Pi has no permission system. Its read-only mode is a tool allowlist.
|
|
102
|
+
- A permission that is blocked in read-only mode is not an error. OpenCode ends such a run with
|
|
103
|
+
exit 0.
|
|
104
|
+
- Org, managed or MDM OpenCode config merges after the config run-harness injects.
|
|
105
|
+
|
|
106
|
+
## Per-harness mapping
|
|
107
|
+
|
|
108
|
+
| | Claude Code | Codex | Pi | OpenCode |
|
|
109
|
+
|---|---|---|---|---|
|
|
110
|
+
| base command | `claude -p --output-format stream-json --verbose` | `codex exec --skip-git-repo-check --color never` | `pi --mode json` | `opencode run --standalone --format json` |
|
|
111
|
+
| prompt | stdin | stdin | stdin | stdin |
|
|
112
|
+
| `--model M` | `--model M` | `-m M` | `--model M` | `-m M` |
|
|
113
|
+
| `--effort L` | `--effort L` | `-c model_reasoning_effort=L` | `--thinking L` | `-m M#L` (pre-checked) |
|
|
114
|
+
| read-only (default) | `--tools Read,Grep,Glob --permission-mode dontAsk --strict-mcp-config` | `-s read-only` | `--tools read,grep,find,ls` (plus `--no-mcp` on Pi 1.x and later) | injected agent plus `--agent run-harness-readonly` (pre-checked with `opencode api`) |
|
|
115
|
+
| `--autonomous` | `--dangerously-skip-permissions` | `--dangerously-bypass-approvals-and-sandbox` | nothing added | `--auto` |
|
|
116
|
+
| final message | `result` of the last `result` event | captured stdout | text of the last assistant `message_end` | text parts of the last assistant message |
|
|
117
|
+
| failure despite exit 0 | `is_error: true`, or no `result` | none | `stopReason` is `error` or `aborted` | an `error` event |
|
|
118
|
+
| do not override | `-p`, `--output-format`, `--verbose`, `--tools`, `--permission-mode` | `-s`, `--color`, `--json`, `-o` | `--mode`, `--tools`, `-p` | `--format`, `--agent`, `--dir` |
|
|
119
|
+
|
|
120
|
+
Argument order is: base command, permission arguments, `--model`, effort, passthrough.
|
|
121
|
+
|
|
122
|
+
### Claude Code
|
|
123
|
+
|
|
124
|
+
- `CLAUDE_CODE_EFFORT_LEVEL` passes through untouched. Whether it or `--effort` wins is unverified.
|
|
125
|
+
- A user `maxEffortLevel` setting can silently cap `max`.
|
|
126
|
+
- Read-only uses a tool allowlist, not `plan` mode, and `--strict-mcp-config` drops MCP servers from
|
|
127
|
+
user and project config. Hooks still run.
|
|
128
|
+
- Bypass mode refuses to run as root outside a sandbox. That refusal comes through with Claude's
|
|
129
|
+
exit code.
|
|
130
|
+
- An auth failure (`Not logged in`) exits 1 with the message on stderr.
|
|
131
|
+
|
|
132
|
+
### Codex
|
|
133
|
+
|
|
134
|
+
- Codex's stdout, when piped, is the final message. It is buffered and written once at the end.
|
|
135
|
+
Its stderr is its own progress, forwarded unchanged.
|
|
136
|
+
- `-s read-only` is always passed explicitly. Projects marked `trusted` in `~/.codex/config.toml`
|
|
137
|
+
would otherwise default to `workspace-write`.
|
|
138
|
+
- An unknown model makes Codex warn ("fallback metadata") and fail the turn, so the run exits 1.
|
|
139
|
+
- There is no auth pre-check. An auth failure exits 1 after roughly 15 to 20 seconds of retries.
|
|
140
|
+
|
|
141
|
+
### Pi
|
|
142
|
+
|
|
143
|
+
- Before a read-only run, run-harness runs `pi --version`. Pi 1.x and later gets `--no-mcp`,
|
|
144
|
+
because `--tools` keeps MCP tools otherwise. Pi 0.x does not get it, since the flag does not
|
|
145
|
+
exist there. Unparseable output or a non-zero exit is exit 2, because read-only cannot be
|
|
146
|
+
guaranteed. `--autonomous` skips the check.
|
|
147
|
+
- `--thinking` is clamped silently per model.
|
|
148
|
+
- A `:<thinking>` suffix on `--model` is overridden by an explicit `--effort`.
|
|
149
|
+
- Extensions still load and run their hooks in read-only mode. Only their tools are hidden.
|
|
150
|
+
- Passthrough positional arguments starting with `@` become file attachments.
|
|
151
|
+
- Pi exits 0 when the model errors in JSON mode. run-harness turns that into exit 1.
|
|
152
|
+
|
|
153
|
+
### OpenCode
|
|
154
|
+
|
|
155
|
+
- Read-only injects an agent named `run-harness-readonly` through `OPENCODE_CONFIG_CONTENT`. It is
|
|
156
|
+
merged into your value if you set one, and invalid JSON there is exit 2. The agent allows only
|
|
157
|
+
`read`, `glob`, `grep`, `list` and `lsp`, and denies everything else, including `task`. Nothing
|
|
158
|
+
is persisted. Apart from `PWD`, this is the only environment variable run-harness changes.
|
|
159
|
+
- Supports OpenCode 2.x only. Every run passes `--standalone`: 2.x otherwise talks to a shared
|
|
160
|
+
background service that never sees the injected config, so the agent would be missing.
|
|
161
|
+
- Before launch, `opencode api --standalone GET /api/config` (same env and cwd) must show the
|
|
162
|
+
agent with exactly the injected permissions (otherwise exit 2). If a run still cannot load the
|
|
163
|
+
agent, OpenCode reports `Agent not found` and run-harness exits non-zero with nothing on stdout.
|
|
164
|
+
- **`--effort` requires `--model` in `provider/model` form.** run-harness passes it as
|
|
165
|
+
`-m provider/model#level` and first reads `opencode api --standalone GET /api/model`. It exits 2
|
|
166
|
+
if the model is not listed, has no variants, or the level is not one of its variants.
|
|
167
|
+
- **The model is validated only when `--effort` is given.** Without `--effort`, an unlisted model
|
|
168
|
+
fails at run time, and the run exits 1.
|
|
169
|
+
- A permission rejected in read-only mode ends the run with exit 0.
|
|
170
|
+
- OpenCode creates `.opencode/.gitignore` and installs its plugin package at startup. Those are its
|
|
171
|
+
own side effects.
|
|
172
|
+
|
|
173
|
+
### Effort
|
|
174
|
+
|
|
175
|
+
`low`, `medium`, `high` and `max` map one to one onto every harness, with no remapping (`max` is
|
|
176
|
+
never turned into `xhigh`). Claude Code, Codex and Pi adjust a level the model does not support to
|
|
177
|
+
the nearest one it does, and run-harness does not detect that. OpenCode rejects unknown variants
|
|
178
|
+
at run time, but run-harness checks them first so you get exit 2 and the valid levels. Native-only levels (`xhigh`, `minimal`, `off`, `none`
|
|
179
|
+
and so on) are reachable only through passthrough.
|
|
180
|
+
|
|
181
|
+
## Exit codes
|
|
182
|
+
|
|
183
|
+
| situation | exit | stdout |
|
|
184
|
+
|---|---|---|
|
|
185
|
+
| usage or validation error, or a failed pre-launch check | `2` | empty |
|
|
186
|
+
| harness executable (or a pre-check helper) not on `PATH` | `127` | empty |
|
|
187
|
+
| harness exits non-zero | the harness's code, verbatim | empty |
|
|
188
|
+
| harness killed by signal N | `128 + N` | empty |
|
|
189
|
+
| harness exits 0, but the event stream reports an error | `1` | empty |
|
|
190
|
+
| harness exits 0 with no final-message event | `1` | empty |
|
|
191
|
+
| harness exits 0 with a final message | `0` | the final message |
|
|
192
|
+
| harness exits 0 with an empty final message | `0` | empty (no newline added) |
|
|
193
|
+
| Codex exits 0 | `0` | the captured stdout |
|
|
194
|
+
|
|
195
|
+
stdout is written once, after the harness exits, and only when run-harness exits 0. On failure,
|
|
196
|
+
error text taken from the stream goes to stderr.
|
|
197
|
+
|
|
198
|
+
## stderr
|
|
199
|
+
|
|
200
|
+
stderr is for people. Its format is not stable and is not a contract. The harness's own stderr is
|
|
201
|
+
forwarded unchanged. For Claude Code, Pi and OpenCode, run-harness also prints the assistant text
|
|
202
|
+
and a `→ tool: <name>` line per tool call as events arrive. Lines that are not JSON are forwarded
|
|
203
|
+
raw. Codex has no renderer, since its stderr already is its progress.
|
|
204
|
+
|
|
205
|
+
## Tested harness versions
|
|
206
|
+
|
|
207
|
+
run-harness declares tested versions, not minimums. The fixtures come from these versions run
|
|
208
|
+
against a local fake model server (OpenCode: observed on 2.0.32 with a free model), plus
|
|
209
|
+
hand-written edge cases. Claude Code's and OpenCode's `success` and `tool_call` fixtures, and
|
|
210
|
+
Claude Code's `not_logged_in`, are recorded against a real model with the recorder script below.
|
|
211
|
+
Codex was checked with the live suite:
|
|
212
|
+
|
|
213
|
+
| harness | version |
|
|
214
|
+
|---|---|
|
|
215
|
+
| Claude Code | 2.1.303 |
|
|
216
|
+
| Codex | codex-cli 0.163.0 |
|
|
217
|
+
| Pi | 0.85.1 |
|
|
218
|
+
| OpenCode | 2.0.32 (2.x only) |
|
|
219
|
+
|
|
220
|
+
There are no general runtime version checks. The one exception is Pi's `--no-mcp`, because a
|
|
221
|
+
version difference there would make read-only fail open. Other versions may work, or may break
|
|
222
|
+
extraction. The opt-in live suite (below) is how drift is detected.
|
|
223
|
+
|
|
224
|
+
## Development
|
|
225
|
+
|
|
226
|
+
```sh
|
|
227
|
+
uv run pytest # unit and fake-harness tests; no real harness needed
|
|
228
|
+
uv run ruff check
|
|
229
|
+
uv run ruff format --check
|
|
230
|
+
uv run pytest -m live # opt-in smoke run against the real harnesses on PATH
|
|
231
|
+
uv run scripts/record_fixtures.py # re-record the fixtures from the real harnesses
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
The live suite never runs in CI. Run it locally before a release, and re-record the fixtures if it
|
|
235
|
+
fails.
|
|
236
|
+
|
|
237
|
+
## Release
|
|
238
|
+
|
|
239
|
+
Releases run from the Actions tab: run the **Release** workflow on `master`, choosing a version
|
|
240
|
+
`bump` (`none` releases the version already in `pyproject.toml`) and whether to publish to `pypi`.
|
|
241
|
+
|
|
242
|
+
The workflow runs lint and tests, applies the bump and builds, refusing a version that is already
|
|
243
|
+
tagged. It publishes to TestPyPI and smoke-tests the install on Linux and macOS
|
|
244
|
+
(`scripts/smoke_release.sh`). With `pypi` ticked it then publishes the same files to PyPI, commits
|
|
245
|
+
the bump, tags `vX.Y.Z`, creates a GitHub Release with generated notes and smoke-tests the PyPI
|
|
246
|
+
install. With `pypi` unticked it stops after TestPyPI, as a dry run. Both indexes use trusted
|
|
247
|
+
publishing for `release.yml`, with no environment.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "run-harness"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "One CLI for launching coding-agent harnesses non-interactively."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
dependencies = []
|
|
8
|
+
|
|
9
|
+
[project.scripts]
|
|
10
|
+
run-harness = "run_harness.cli:main"
|
|
11
|
+
|
|
12
|
+
[build-system]
|
|
13
|
+
requires = ["uv_build>=0.8,<0.13"]
|
|
14
|
+
build-backend = "uv_build"
|
|
15
|
+
|
|
16
|
+
[dependency-groups]
|
|
17
|
+
dev = [
|
|
18
|
+
"pytest",
|
|
19
|
+
"ruff",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
[tool.ruff]
|
|
23
|
+
line-length = 100
|
|
24
|
+
|
|
25
|
+
[tool.ruff.lint]
|
|
26
|
+
select = [
|
|
27
|
+
"E",
|
|
28
|
+
"F",
|
|
29
|
+
"I",
|
|
30
|
+
"UP",
|
|
31
|
+
"B",
|
|
32
|
+
]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
markers = ["live: runs the real harnesses (costs money); opt in with `pytest -m live`"]
|
|
36
|
+
addopts = "-m 'not live'"
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "run-harness"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "One CLI for launching coding-agent harnesses non-interactively."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
dependencies = []
|
|
8
|
+
|
|
9
|
+
[project.scripts]
|
|
10
|
+
run-harness = "run_harness.cli:main"
|
|
11
|
+
|
|
12
|
+
[build-system]
|
|
13
|
+
requires = ["uv_build>=0.8,<0.13"]
|
|
14
|
+
build-backend = "uv_build"
|
|
15
|
+
|
|
16
|
+
[dependency-groups]
|
|
17
|
+
dev = ["pytest", "ruff"]
|
|
18
|
+
|
|
19
|
+
[tool.ruff]
|
|
20
|
+
line-length = 100
|
|
21
|
+
|
|
22
|
+
[tool.ruff.lint]
|
|
23
|
+
select = ["E", "F", "I", "UP", "B"]
|
|
24
|
+
|
|
25
|
+
[tool.pytest.ini_options]
|
|
26
|
+
markers = [
|
|
27
|
+
"live: runs the real harnesses (costs money); opt in with `pytest -m live`",
|
|
28
|
+
]
|
|
29
|
+
addopts = "-m 'not live'"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""run-harness: one CLI for coding-agent harnesses."""
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Harness adapters and their registry."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from run_harness.adapters.base import Adapter
|
|
6
|
+
from run_harness.adapters.claude import ClaudeAdapter
|
|
7
|
+
from run_harness.adapters.codex import CodexAdapter
|
|
8
|
+
from run_harness.adapters.opencode import OpenCodeAdapter
|
|
9
|
+
from run_harness.adapters.pi import PiAdapter
|
|
10
|
+
|
|
11
|
+
ADAPTERS: dict[str, Adapter] = {
|
|
12
|
+
"claude": ClaudeAdapter(),
|
|
13
|
+
"codex": CodexAdapter(),
|
|
14
|
+
"opencode": OpenCodeAdapter(),
|
|
15
|
+
"pi": PiAdapter(),
|
|
16
|
+
}
|