seatrial 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. seatrial-0.1.0/LICENSE +21 -0
  2. seatrial-0.1.0/PKG-INFO +283 -0
  3. seatrial-0.1.0/README.md +262 -0
  4. seatrial-0.1.0/pyproject.toml +37 -0
  5. seatrial-0.1.0/setup.cfg +4 -0
  6. seatrial-0.1.0/src/seatrial/__init__.py +19 -0
  7. seatrial-0.1.0/src/seatrial/__main__.py +3 -0
  8. seatrial-0.1.0/src/seatrial/cases.py +214 -0
  9. seatrial-0.1.0/src/seatrial/checks.py +184 -0
  10. seatrial-0.1.0/src/seatrial/cli.py +499 -0
  11. seatrial-0.1.0/src/seatrial/config.py +214 -0
  12. seatrial-0.1.0/src/seatrial/hosts/__init__.py +69 -0
  13. seatrial-0.1.0/src/seatrial/hosts/claude.py +162 -0
  14. seatrial-0.1.0/src/seatrial/hosts/codex.py +280 -0
  15. seatrial-0.1.0/src/seatrial/hosts/samples/claude/auth-error.expected.json +1 -0
  16. seatrial-0.1.0/src/seatrial/hosts/samples/claude/auth-error.jsonl +2 -0
  17. seatrial-0.1.0/src/seatrial/hosts/samples/claude/delegated-report.expected.json +4 -0
  18. seatrial-0.1.0/src/seatrial/hosts/samples/claude/delegated-report.jsonl +18 -0
  19. seatrial-0.1.0/src/seatrial/hosts/samples/codex/delegated-report.expected.json +4 -0
  20. seatrial-0.1.0/src/seatrial/hosts/samples/codex/delegated-report.jsonl +11 -0
  21. seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-paths.expected.json +18 -0
  22. seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-paths.jsonl +8 -0
  23. seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-reads.expected.json +3 -0
  24. seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-reads.jsonl +6 -0
  25. seatrial-0.1.0/src/seatrial/hosts/samples/codex/skipped-cd.expected.json +12 -0
  26. seatrial-0.1.0/src/seatrial/hosts/samples/codex/skipped-cd.jsonl +5 -0
  27. seatrial-0.1.0/src/seatrial/hosts/samples/codex/usage-limit.expected.json +1 -0
  28. seatrial-0.1.0/src/seatrial/hosts/samples/codex/usage-limit.jsonl +4 -0
  29. seatrial-0.1.0/src/seatrial/interactive.py +121 -0
  30. seatrial-0.1.0/src/seatrial/ports.py +192 -0
  31. seatrial-0.1.0/src/seatrial/process.py +127 -0
  32. seatrial-0.1.0/src/seatrial/report.py +172 -0
  33. seatrial-0.1.0/src/seatrial/runner.py +292 -0
  34. seatrial-0.1.0/src/seatrial/setup.py +219 -0
  35. seatrial-0.1.0/src/seatrial/stats.py +85 -0
  36. seatrial-0.1.0/src/seatrial/templates.py +70 -0
  37. seatrial-0.1.0/src/seatrial/ui.py +226 -0
  38. seatrial-0.1.0/src/seatrial/workspace.py +234 -0
  39. seatrial-0.1.0/src/seatrial.egg-info/PKG-INFO +283 -0
  40. seatrial-0.1.0/src/seatrial.egg-info/SOURCES.txt +48 -0
  41. seatrial-0.1.0/src/seatrial.egg-info/dependency_links.txt +1 -0
  42. seatrial-0.1.0/src/seatrial.egg-info/entry_points.txt +2 -0
  43. seatrial-0.1.0/src/seatrial.egg-info/top_level.txt +1 -0
  44. seatrial-0.1.0/tests/test_cases.py +125 -0
  45. seatrial-0.1.0/tests/test_cli.py +215 -0
  46. seatrial-0.1.0/tests/test_host_contract.py +267 -0
  47. seatrial-0.1.0/tests/test_process.py +102 -0
  48. seatrial-0.1.0/tests/test_runner.py +303 -0
  49. seatrial-0.1.0/tests/test_stats.py +65 -0
  50. seatrial-0.1.0/tests/test_workspace.py +250 -0
seatrial-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Otavio Barros
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,283 @@
1
+ Metadata-Version: 2.4
2
+ Name: seatrial
3
+ Version: 0.1.0
4
+ Summary: Run evaluation cases for agent skills on agent CLIs and compare skill revisions.
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/otaviobarros/seatrial
7
+ Project-URL: Documentation, https://github.com/otaviobarros/seatrial/tree/main/docs
8
+ Project-URL: Issues, https://github.com/otaviobarros/seatrial/issues
9
+ Project-URL: Changelog, https://github.com/otaviobarros/seatrial/releases
10
+ Keywords: agents,skills,evaluation,claude,codex
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Environment :: Console
14
+ Classifier: Operating System :: MacOS
15
+ Classifier: Operating System :: POSIX :: Linux
16
+ Classifier: Topic :: Software Development :: Testing
17
+ Requires-Python: >=3.11
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Dynamic: license-file
21
+
22
+ <!-- markdownlint-disable-next-line MD041 -->
23
+ <p align="center">
24
+ <picture>
25
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-dark.png">
26
+ <img alt="seatrial" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-light.png" width="420">
27
+ </picture>
28
+ </p>
29
+
30
+ <p align="center">Sea trials for agent skills.</p>
31
+
32
+ <p align="center">
33
+ <a href="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml/badge.svg"></a>
34
+ <img alt="Python 3.11+" src="https://img.shields.io/badge/python-3.11%2B-1d6fa5">
35
+ <img alt="No dependencies" src="https://img.shields.io/badge/dependencies-none-0b7a8f">
36
+ <a href="https://github.com/otaviobarros/seatrial/blob/main/LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-0b2540"></a>
37
+ </p>
38
+
39
+ You changed a skill. Did the agent get better, or did it only feel better on the
40
+ one prompt you tried? seatrial answers that the way a shipyard tests a boat: it
41
+ takes the skill out on a fixed course, many times, and records what happened.
42
+
43
+ - **Real agents.** Each trial runs the actual CLI, Claude Code (`claude -p`) or
44
+ Codex (`codex exec`), in a fresh project. Add other hosts through a small
45
+ [adapter](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md).
46
+ - **Graded by code.** A case's `grade()` checks files, Git state, commands, and the
47
+ final message. No model judges the result.
48
+ - **Two revisions, one verdict.** `compare` runs `main` against your uncommitted
49
+ edits and reports `improved`, `non-inferior`, `regressed`, or `inconclusive`.
50
+ - **Nothing to install into your project.** Python 3.11+ standard library and Git.
51
+
52
+ ## How it works
53
+
54
+ <picture>
55
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-dark.svg">
56
+ <img alt="Diagram of one trial in five steps: case, mount, agent, evidence, grade. The trial repeats n times per case, host, and revision, and the results go to summary.json." src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-light.svg">
57
+ </picture>
58
+
59
+ <details>
60
+ <summary>The five steps in text</summary>
61
+
62
+ Each trial runs these steps:
63
+
64
+ 1. **Case.** `build()` in the case's `case.py` writes a small fictional project
65
+ and returns the request a user would type.
66
+ 2. **Mount.** seatrial mounts your skills at one revision, a Git ref such as
67
+ `main` or the uncommitted `WORKTREE`, as a plugin.
68
+ 3. **Agent.** The real host CLI (`claude` or `codex`) runs the request in a
69
+ fresh, isolated folder.
70
+ 4. **Evidence.** seatrial records the commands, file edits, loaded skills, and
71
+ final message in `evidence.json`.
72
+ 5. **Grade.** `grade()` checks that evidence and the project, and returns pass or
73
+ fail for each assertion in `case.toml`.
74
+
75
+ </details>
76
+
77
+ seatrial repeats each trial n times per case, host, and revision, and writes the
78
+ pass rates and verdicts to `summary.json`. Repetition turns one lucky run into a
79
+ pass rate.
80
+
81
+ ## How seatrial compares
82
+
83
+ Other tools evaluate agents too. This table reflects their public documentation
84
+ in November 2026.
85
+
86
+ | | seatrial | [`claude plugin eval`](https://code.claude.com/docs/en/plugin-evals) | [promptfoo](https://www.promptfoo.dev/docs/guides/test-agent-skills/) | [Inspect](https://inspect.aisi.org.uk) + [inspect_swe](https://meridianlabs-ai.github.io/inspect_swe) |
87
+ |---|---|---|---|---|
88
+ | Agents | Claude Code and Codex CLIs; others through an adapter | Claude Code | Claude Agent SDK, Codex SDK, OpenCode SDK | Claude Code and Codex CLI as Inspect agents, with any model |
89
+ | Compares | Two Git revisions of your skills, including uncommitted edits | The plugin loaded against no plugin | Skill versions in separate fixture folders, side by side | Models and agents on a task set |
90
+ | Grading | Your Python code over files, Git state, commands, and the final message | Regex, tool use, and file checks, plus model-judged rubrics | Assertions such as skill used, custom JavaScript, plus model-graded | Code or model scorers |
91
+ | Statistics | Fisher exact test per assertion, and a verdict per case and host | Three runs per case by default, mean scores against a threshold | `--repeat`; no significance test | Metrics such as accuracy and standard error |
92
+ | Isolation | Fresh Git project per trial, filtered environment, flags skills from outside the plugin | Fresh home, working folder, and config per run | A `working_dir` per provider | Docker sandbox |
93
+ | Runtime | Python 3.11+ standard library; no dependencies | Built into Claude Code 2.1.269+ | Node and the agent SDK packages | Python packages; Docker for the sandbox |
94
+
95
+ Choose seatrial to answer "did this skill edit make the agent better or worse"
96
+ on the real CLIs, with code-graded cases and a statistical verdict. Choose
97
+ `claude plugin eval` to check that a plugin helps compared with no plugin, inside
98
+ Claude Code. Choose promptfoo when your evals already live in promptfoo or need
99
+ model-graded rubrics. Choose Inspect for benchmark-scale runs in sandboxes or
100
+ across many models.
101
+
102
+ ## Install
103
+
104
+ Run it with [uv](https://docs.astral.sh/uv/). `uvx` builds the package into a cached,
105
+ isolated environment and runs the command; nothing is installed into your project.
106
+
107
+ ```sh
108
+ uvx seatrial --help # from PyPI
109
+ uvx --from . seatrial --help # from a checkout
110
+ uv tool install seatrial # keep `seatrial` on PATH
111
+ ```
112
+
113
+ Without uv, use `pipx install seatrial`, or run `python3 -m seatrial`
114
+ with `src/` on `PYTHONPATH`. The package has no dependencies beyond Python 3.11+.
115
+
116
+ seatrial runs on macOS and Linux. Windows is not supported; use WSL.
117
+
118
+ ## Quickstart
119
+
120
+ ```sh
121
+ cd my-skills-project
122
+ seatrial init # detect skills, write seatrial.toml and an example case
123
+ seatrial doctor # check Python, Git, the configuration, cases, and host CLIs
124
+ seatrial list
125
+ seatrial run --dry-run # print the trial plan; no model calls
126
+ seatrial run --host claude --n 1
127
+ seatrial # interactive mode on a terminal
128
+ ```
129
+
130
+ `init` finds folders that hold `<name>/SKILL.md`. Two layouts work:
131
+
132
+ - `plugin`: a Claude Code plugin root with `skills/` and `.claude-plugin/plugin.json`.
133
+ - `skills`: a plain folder of skills. seatrial mounts it as a generated plugin.
134
+
135
+ On a terminal, `init` asks which folder, namespace, case folder, and hosts to use.
136
+ With `--yes` or without a terminal, it uses the detected values. See
137
+ [configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md) for every key.
138
+
139
+ ## Commands
140
+
141
+ | Command | Does |
142
+ |---|---|
143
+ | `init` | Write `seatrial.toml` and a runnable example case |
144
+ | `doctor` | Check the environment; exits 3 when a check fails |
145
+ | `list` | List cases; accepts selectors |
146
+ | `new-case --skill S --id NAME-001` | Create a case skeleton; prompts on a terminal |
147
+ | `run [--rev REV]` | Run cases against one skill revision |
148
+ | `compare --base REV [--head REV]` | Run both revisions, interleaved, and give a verdict per case and host |
149
+ | `runs` | List retained run directories, newest first |
150
+ | `report [RUN_DIR\|latest]` | Print a run summary; `--format markdown` for a Markdown table |
151
+ | `regrade [RUN_DIR\|latest]` | Grade retained trials again with the current case code; no host calls |
152
+ | `hosts` | List host adapters, models, and CLI versions |
153
+ | `schema` | Print the configuration, case, evidence, and exit-code contract |
154
+ | `interactive` | Menu-driven mode; also the default with no arguments on a terminal |
155
+
156
+ Every command except `interactive` accepts `--json`. `seatrial COMMAND --help` lists all options.
157
+ For agents and CI, read [using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md).
158
+
159
+ ## Agent skill
160
+
161
+ [`skills/seatrial`](https://github.com/otaviobarros/seatrial/blob/main/skills/seatrial/SKILL.md) teaches an agent to set up seatrial,
162
+ write cases, run and compare within a budget, and read the results. Install it with
163
+ the [skills](https://skills.sh) CLI:
164
+
165
+ ```sh
166
+ npx skills add otaviobarros/seatrial
167
+ ```
168
+
169
+ ## Run and compare
170
+
171
+ `run` and `compare` call host CLIs and spend quota. Before a native run:
172
+
173
+ 1. Sign in to each CLI (`claude /login`, `codex login`).
174
+ 2. Print the plan with `--dry-run`.
175
+ 3. Set a budget: `--n`, `--max-runs`, `--max-minutes`, and `--max-cost-usd` for
176
+ hosts that report cost.
177
+
178
+ A revision is a Git revision or `WORKTREE`, the skill files in the checkout with
179
+ uncommitted changes. `compare` defaults `--head` to `WORKTREE`, so
180
+ `seatrial compare --base main` measures uncommitted edits. Each revision mounts
181
+ as a whole skill folder; trials alternate between revisions so that time and load
182
+ affect both alike.
183
+
184
+ Selectors combine and repeat: `--case ID`, `--skill NAME`, `--rule ID`,
185
+ `--suite capability|regression`, `--host NAME`.
186
+
187
+ <details>
188
+ <summary>Options, defaults, and limits</summary>
189
+
190
+ | Option | Default | Maximum |
191
+ |---|---|---|
192
+ | `--n` trials per case, host, and revision | `[run] n`, 3 | `[limits] max_n`, 10 |
193
+ | `--jobs` parallel trials | `[run] jobs`, 1 | `max_jobs`, 4 |
194
+ | `--max-runs` trials in the plan | `max_runs`, 100 | `max_runs` |
195
+ | `--max-minutes`; a trial starts only if its timeout fits | `max_minutes`, 90 | `max_minutes` |
196
+ | `--timeout` seconds per trial | case value, else `[run] timeout`, 180 | `max_timeout`, 600 |
197
+ | `--max-cost-usd` total host-reported cost | none | none |
198
+ | `--budget-usd` per-trial cap (Claude only) | none | none |
199
+
200
+ </details>
201
+
202
+ A usage-limit or authentication error stops the run. Finished trials are kept,
203
+ and the command exits 1.
204
+
205
+ ## Verdicts
206
+
207
+ <picture>
208
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-dark.svg">
209
+ <img alt="compare: base main passes 4 of 10 trials, head WORKTREE passes 10 of 10, verdict improved with Fisher p = 0.011" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-light.svg">
210
+ </picture>
211
+
212
+ `compare` gives each case and host one advisory verdict:
213
+
214
+ | Verdict | Meaning |
215
+ |---|---|
216
+ | `regressed` | A zero-tolerance assertion newly fails, or an assertion is significantly worse (Fisher p < 0.05) |
217
+ | `improved` | The primary assertion is significantly better, and head has no zero-tolerance failure |
218
+ | `non-inferior` | No assertion drops by more than two trials' worth |
219
+ | `inconclusive` | Otherwise, or fewer than 80% of the trials in a revision are valid |
220
+
221
+ Ten trials per revision detect a change of about 50 percentage points on the
222
+ primary assertion. With 3 trials, `improved` is not possible. Report a
223
+ non-significant result as "no detected effect". The report also shows pass^k,
224
+ median lines added, mean wall time, and host-reported cost. Use
225
+ `--fail-on-regression` to exit 4 on any `regressed` verdict.
226
+
227
+ <p align="center">
228
+ <img alt="Terminal output of seatrial compare: progress lines, then a table of valid trials, pass counts, pass^k, wall time, cost, and verdicts for two cases on Claude and Codex" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/terminal.svg" width="880">
229
+ </p>
230
+
231
+ ## Results
232
+
233
+ Each invocation writes a run directory under `[run] state_dir` (default
234
+ `$SEATRIAL_STATE`, else `~/.cache/workspaces/<timestamp>/`). The path prints at
235
+ the end of the run. Keep it out of version control. Projects can share a state
236
+ directory: `runs`, `report latest`, and `regrade` show only the current project's runs.
237
+
238
+ <details>
239
+ <summary>Files in a run directory</summary>
240
+
241
+ | File | Content |
242
+ |---|---|
243
+ | `run.json` | The run settings and revisions, written before the first trial |
244
+ | `records.jsonl` | One record per trial, written as each trial ends; `foreign_skills`, `unknown_skills`, and `outside_access` mark trials that used skills from outside the plugin, read a skill whose source the trace cannot show, or named the project path |
245
+ | `summary.json` | Per-case, per-host, per-revision summaries and verdicts; missing when the run was interrupted, until `seatrial regrade` builds `summary-regraded.json` |
246
+ | `trials/<random>/` | The trial project, `session.jsonl` trace, `stderr.txt`, `evidence.json`, `retained.json` |
247
+
248
+ </details>
249
+
250
+ Trial directories use random names so that no path the agent sees names the
251
+ evaluation. Private host state, including any copied login, is removed after each
252
+ trial.
253
+
254
+ <details>
255
+ <summary>Trial statuses</summary>
256
+
257
+ Only `completed` trials are graded. Other statuses are excluded from pass rates:
258
+
259
+ | Status | Meaning |
260
+ |---|---|
261
+ | `control` | The grader marked the trial as a control, not a measurement |
262
+ | `trace_incomplete` | The agent delegated, but at least one delegated session was not captured |
263
+ | `usage_limit`, `auth_error` | Host quota or sign-in problem; the run stops |
264
+ | `timeout`, `host_stall` | The time cap ended the trial; `host_stall` means the machine paused |
265
+ | `incomplete` | The host exited without a successful final turn |
266
+ | `harness_error` | seatrial failed; the record holds the exception |
267
+ | `stale` | From `regrade` only: the retained project files or `evidence.json` changed after the trial |
268
+
269
+ </details>
270
+
271
+ `regrade` writes `records-regraded.jsonl` and `summary-regraded.json`. Copy
272
+ retained trial files before you inspect them with tools that write.
273
+
274
+ ## Documentation
275
+
276
+ - [Configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md): `seatrial.toml` keys and defaults.
277
+ - [Writing cases](https://github.com/otaviobarros/seatrial/blob/main/docs/cases.md): `case.toml`, `case.py`, evidence, and grading helpers.
278
+ - [Hosts](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md): built-in adapters and how to add one.
279
+ - [Using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md): JSON output, exit codes, and a safe procedure.
280
+
281
+ ## License
282
+
283
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,262 @@
1
+ <!-- markdownlint-disable-next-line MD041 -->
2
+ <p align="center">
3
+ <picture>
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-dark.png">
5
+ <img alt="seatrial" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-light.png" width="420">
6
+ </picture>
7
+ </p>
8
+
9
+ <p align="center">Sea trials for agent skills.</p>
10
+
11
+ <p align="center">
12
+ <a href="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml/badge.svg"></a>
13
+ <img alt="Python 3.11+" src="https://img.shields.io/badge/python-3.11%2B-1d6fa5">
14
+ <img alt="No dependencies" src="https://img.shields.io/badge/dependencies-none-0b7a8f">
15
+ <a href="https://github.com/otaviobarros/seatrial/blob/main/LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-0b2540"></a>
16
+ </p>
17
+
18
+ You changed a skill. Did the agent get better, or did it only feel better on the
19
+ one prompt you tried? seatrial answers that the way a shipyard tests a boat: it
20
+ takes the skill out on a fixed course, many times, and records what happened.
21
+
22
+ - **Real agents.** Each trial runs the actual CLI, Claude Code (`claude -p`) or
23
+ Codex (`codex exec`), in a fresh project. Add other hosts through a small
24
+ [adapter](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md).
25
+ - **Graded by code.** A case's `grade()` checks files, Git state, commands, and the
26
+ final message. No model judges the result.
27
+ - **Two revisions, one verdict.** `compare` runs `main` against your uncommitted
28
+ edits and reports `improved`, `non-inferior`, `regressed`, or `inconclusive`.
29
+ - **Nothing to install into your project.** Python 3.11+ standard library and Git.
30
+
31
+ ## How it works
32
+
33
+ <picture>
34
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-dark.svg">
35
+ <img alt="Diagram of one trial in five steps: case, mount, agent, evidence, grade. The trial repeats n times per case, host, and revision, and the results go to summary.json." src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-light.svg">
36
+ </picture>
37
+
38
+ <details>
39
+ <summary>The five steps in text</summary>
40
+
41
+ Each trial runs these steps:
42
+
43
+ 1. **Case.** `build()` in the case's `case.py` writes a small fictional project
44
+ and returns the request a user would type.
45
+ 2. **Mount.** seatrial mounts your skills at one revision, a Git ref such as
46
+ `main` or the uncommitted `WORKTREE`, as a plugin.
47
+ 3. **Agent.** The real host CLI (`claude` or `codex`) runs the request in a
48
+ fresh, isolated folder.
49
+ 4. **Evidence.** seatrial records the commands, file edits, loaded skills, and
50
+ final message in `evidence.json`.
51
+ 5. **Grade.** `grade()` checks that evidence and the project, and returns pass or
52
+ fail for each assertion in `case.toml`.
53
+
54
+ </details>
55
+
56
+ seatrial repeats each trial n times per case, host, and revision, and writes the
57
+ pass rates and verdicts to `summary.json`. Repetition turns one lucky run into a
58
+ pass rate.
59
+
60
+ ## How seatrial compares
61
+
62
+ Other tools evaluate agents too. This table reflects their public documentation
63
+ in November 2026.
64
+
65
+ | | seatrial | [`claude plugin eval`](https://code.claude.com/docs/en/plugin-evals) | [promptfoo](https://www.promptfoo.dev/docs/guides/test-agent-skills/) | [Inspect](https://inspect.aisi.org.uk) + [inspect_swe](https://meridianlabs-ai.github.io/inspect_swe) |
66
+ |---|---|---|---|---|
67
+ | Agents | Claude Code and Codex CLIs; others through an adapter | Claude Code | Claude Agent SDK, Codex SDK, OpenCode SDK | Claude Code and Codex CLI as Inspect agents, with any model |
68
+ | Compares | Two Git revisions of your skills, including uncommitted edits | The plugin loaded against no plugin | Skill versions in separate fixture folders, side by side | Models and agents on a task set |
69
+ | Grading | Your Python code over files, Git state, commands, and the final message | Regex, tool use, and file checks, plus model-judged rubrics | Assertions such as skill used, custom JavaScript, plus model-graded | Code or model scorers |
70
+ | Statistics | Fisher exact test per assertion, and a verdict per case and host | Three runs per case by default, mean scores against a threshold | `--repeat`; no significance test | Metrics such as accuracy and standard error |
71
+ | Isolation | Fresh Git project per trial, filtered environment, flags skills from outside the plugin | Fresh home, working folder, and config per run | A `working_dir` per provider | Docker sandbox |
72
+ | Runtime | Python 3.11+ standard library; no dependencies | Built into Claude Code 2.1.269+ | Node and the agent SDK packages | Python packages; Docker for the sandbox |
73
+
74
+ Choose seatrial to answer "did this skill edit make the agent better or worse"
75
+ on the real CLIs, with code-graded cases and a statistical verdict. Choose
76
+ `claude plugin eval` to check that a plugin helps compared with no plugin, inside
77
+ Claude Code. Choose promptfoo when your evals already live in promptfoo or need
78
+ model-graded rubrics. Choose Inspect for benchmark-scale runs in sandboxes or
79
+ across many models.
80
+
81
+ ## Install
82
+
83
+ Run it with [uv](https://docs.astral.sh/uv/). `uvx` builds the package into a cached,
84
+ isolated environment and runs the command; nothing is installed into your project.
85
+
86
+ ```sh
87
+ uvx seatrial --help # from PyPI
88
+ uvx --from . seatrial --help # from a checkout
89
+ uv tool install seatrial # keep `seatrial` on PATH
90
+ ```
91
+
92
+ Without uv, use `pipx install seatrial`, or run `python3 -m seatrial`
93
+ with `src/` on `PYTHONPATH`. The package has no dependencies beyond Python 3.11+.
94
+
95
+ seatrial runs on macOS and Linux. Windows is not supported; use WSL.
96
+
97
+ ## Quickstart
98
+
99
+ ```sh
100
+ cd my-skills-project
101
+ seatrial init # detect skills, write seatrial.toml and an example case
102
+ seatrial doctor # check Python, Git, the configuration, cases, and host CLIs
103
+ seatrial list
104
+ seatrial run --dry-run # print the trial plan; no model calls
105
+ seatrial run --host claude --n 1
106
+ seatrial # interactive mode on a terminal
107
+ ```
108
+
109
+ `init` finds folders that hold `<name>/SKILL.md`. Two layouts work:
110
+
111
+ - `plugin`: a Claude Code plugin root with `skills/` and `.claude-plugin/plugin.json`.
112
+ - `skills`: a plain folder of skills. seatrial mounts it as a generated plugin.
113
+
114
+ On a terminal, `init` asks which folder, namespace, case folder, and hosts to use.
115
+ With `--yes` or without a terminal, it uses the detected values. See
116
+ [configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md) for every key.
117
+
118
+ ## Commands
119
+
120
+ | Command | Does |
121
+ |---|---|
122
+ | `init` | Write `seatrial.toml` and a runnable example case |
123
+ | `doctor` | Check the environment; exits 3 when a check fails |
124
+ | `list` | List cases; accepts selectors |
125
+ | `new-case --skill S --id NAME-001` | Create a case skeleton; prompts on a terminal |
126
+ | `run [--rev REV]` | Run cases against one skill revision |
127
+ | `compare --base REV [--head REV]` | Run both revisions, interleaved, and give a verdict per case and host |
128
+ | `runs` | List retained run directories, newest first |
129
+ | `report [RUN_DIR\|latest]` | Print a run summary; `--format markdown` for a Markdown table |
130
+ | `regrade [RUN_DIR\|latest]` | Grade retained trials again with the current case code; no host calls |
131
+ | `hosts` | List host adapters, models, and CLI versions |
132
+ | `schema` | Print the configuration, case, evidence, and exit-code contract |
133
+ | `interactive` | Menu-driven mode; also the default with no arguments on a terminal |
134
+
135
+ Every command except `interactive` accepts `--json`. `seatrial COMMAND --help` lists all options.
136
+ For agents and CI, read [using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md).
137
+
138
+ ## Agent skill
139
+
140
+ [`skills/seatrial`](https://github.com/otaviobarros/seatrial/blob/main/skills/seatrial/SKILL.md) teaches an agent to set up seatrial,
141
+ write cases, run and compare within a budget, and read the results. Install it with
142
+ the [skills](https://skills.sh) CLI:
143
+
144
+ ```sh
145
+ npx skills add otaviobarros/seatrial
146
+ ```
147
+
148
+ ## Run and compare
149
+
150
+ `run` and `compare` call host CLIs and spend quota. Before a native run:
151
+
152
+ 1. Sign in to each CLI (`claude /login`, `codex login`).
153
+ 2. Print the plan with `--dry-run`.
154
+ 3. Set a budget: `--n`, `--max-runs`, `--max-minutes`, and `--max-cost-usd` for
155
+ hosts that report cost.
156
+
157
+ A revision is a Git revision or `WORKTREE`, the skill files in the checkout with
158
+ uncommitted changes. `compare` defaults `--head` to `WORKTREE`, so
159
+ `seatrial compare --base main` measures uncommitted edits. Each revision mounts
160
+ as a whole skill folder; trials alternate between revisions so that time and load
161
+ affect both alike.
162
+
163
+ Selectors combine and repeat: `--case ID`, `--skill NAME`, `--rule ID`,
164
+ `--suite capability|regression`, `--host NAME`.
165
+
166
+ <details>
167
+ <summary>Options, defaults, and limits</summary>
168
+
169
+ | Option | Default | Maximum |
170
+ |---|---|---|
171
+ | `--n` trials per case, host, and revision | `[run] n`, 3 | `[limits] max_n`, 10 |
172
+ | `--jobs` parallel trials | `[run] jobs`, 1 | `max_jobs`, 4 |
173
+ | `--max-runs` trials in the plan | `max_runs`, 100 | `max_runs` |
174
+ | `--max-minutes`; a trial starts only if its timeout fits | `max_minutes`, 90 | `max_minutes` |
175
+ | `--timeout` seconds per trial | case value, else `[run] timeout`, 180 | `max_timeout`, 600 |
176
+ | `--max-cost-usd` total host-reported cost | none | none |
177
+ | `--budget-usd` per-trial cap (Claude only) | none | none |
178
+
179
+ </details>
180
+
181
+ A usage-limit or authentication error stops the run. Finished trials are kept,
182
+ and the command exits 1.
183
+
184
+ ## Verdicts
185
+
186
+ <picture>
187
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-dark.svg">
188
+ <img alt="compare: base main passes 4 of 10 trials, head WORKTREE passes 10 of 10, verdict improved with Fisher p = 0.011" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-light.svg">
189
+ </picture>
190
+
191
+ `compare` gives each case and host one advisory verdict:
192
+
193
+ | Verdict | Meaning |
194
+ |---|---|
195
+ | `regressed` | A zero-tolerance assertion newly fails, or an assertion is significantly worse (Fisher p < 0.05) |
196
+ | `improved` | The primary assertion is significantly better, and head has no zero-tolerance failure |
197
+ | `non-inferior` | No assertion drops by more than two trials' worth |
198
+ | `inconclusive` | Otherwise, or fewer than 80% of the trials in a revision are valid |
199
+
200
+ Ten trials per revision detect a change of about 50 percentage points on the
201
+ primary assertion. With 3 trials, `improved` is not possible. Report a
202
+ non-significant result as "no detected effect". The report also shows pass^k,
203
+ median lines added, mean wall time, and host-reported cost. Use
204
+ `--fail-on-regression` to exit 4 on any `regressed` verdict.
205
+
206
+ <p align="center">
207
+ <img alt="Terminal output of seatrial compare: progress lines, then a table of valid trials, pass counts, pass^k, wall time, cost, and verdicts for two cases on Claude and Codex" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/terminal.svg" width="880">
208
+ </p>
209
+
210
+ ## Results
211
+
212
+ Each invocation writes a run directory under `[run] state_dir` (default
213
+ `$SEATRIAL_STATE`, else `~/.cache/workspaces/<timestamp>/`). The path prints at
214
+ the end of the run. Keep it out of version control. Projects can share a state
215
+ directory: `runs`, `report latest`, and `regrade` show only the current project's runs.
216
+
217
+ <details>
218
+ <summary>Files in a run directory</summary>
219
+
220
+ | File | Content |
221
+ |---|---|
222
+ | `run.json` | The run settings and revisions, written before the first trial |
223
+ | `records.jsonl` | One record per trial, written as each trial ends; `foreign_skills`, `unknown_skills`, and `outside_access` mark trials that used skills from outside the plugin, read a skill whose source the trace cannot show, or named the project path |
224
+ | `summary.json` | Per-case, per-host, per-revision summaries and verdicts; missing when the run was interrupted, until `seatrial regrade` builds `summary-regraded.json` |
225
+ | `trials/<random>/` | The trial project, `session.jsonl` trace, `stderr.txt`, `evidence.json`, `retained.json` |
226
+
227
+ </details>
228
+
229
+ Trial directories use random names so that no path the agent sees names the
230
+ evaluation. Private host state, including any copied login, is removed after each
231
+ trial.
232
+
233
+ <details>
234
+ <summary>Trial statuses</summary>
235
+
236
+ Only `completed` trials are graded. Other statuses are excluded from pass rates:
237
+
238
+ | Status | Meaning |
239
+ |---|---|
240
+ | `control` | The grader marked the trial as a control, not a measurement |
241
+ | `trace_incomplete` | The agent delegated, but at least one delegated session was not captured |
242
+ | `usage_limit`, `auth_error` | Host quota or sign-in problem; the run stops |
243
+ | `timeout`, `host_stall` | The time cap ended the trial; `host_stall` means the machine paused |
244
+ | `incomplete` | The host exited without a successful final turn |
245
+ | `harness_error` | seatrial failed; the record holds the exception |
246
+ | `stale` | From `regrade` only: the retained project files or `evidence.json` changed after the trial |
247
+
248
+ </details>
249
+
250
+ `regrade` writes `records-regraded.jsonl` and `summary-regraded.json`. Copy
251
+ retained trial files before you inspect them with tools that write.
252
+
253
+ ## Documentation
254
+
255
+ - [Configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md): `seatrial.toml` keys and defaults.
256
+ - [Writing cases](https://github.com/otaviobarros/seatrial/blob/main/docs/cases.md): `case.toml`, `case.py`, evidence, and grading helpers.
257
+ - [Hosts](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md): built-in adapters and how to add one.
258
+ - [Using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md): JSON output, exit codes, and a safe procedure.
259
+
260
+ ## License
261
+
262
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,37 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "seatrial"
7
+ version = "0.1.0"
8
+ description = "Run evaluation cases for agent skills on agent CLIs and compare skill revisions."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ requires-python = ">=3.11"
13
+ dependencies = []
14
+ keywords = ["agents", "skills", "evaluation", "claude", "codex"]
15
+ classifiers = [
16
+ "Development Status :: 3 - Alpha",
17
+ "Programming Language :: Python :: 3",
18
+ "Environment :: Console",
19
+ "Operating System :: MacOS",
20
+ "Operating System :: POSIX :: Linux",
21
+ "Topic :: Software Development :: Testing",
22
+ ]
23
+
24
+ [project.urls]
25
+ Homepage = "https://github.com/otaviobarros/seatrial"
26
+ Documentation = "https://github.com/otaviobarros/seatrial/tree/main/docs"
27
+ Issues = "https://github.com/otaviobarros/seatrial/issues"
28
+ Changelog = "https://github.com/otaviobarros/seatrial/releases"
29
+
30
+ [project.scripts]
31
+ seatrial = "seatrial.cli:entry"
32
+
33
+ [tool.setuptools.packages.find]
34
+ where = ["src"]
35
+
36
+ [tool.setuptools.package-data]
37
+ "seatrial.hosts" = ["samples/**/*"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,19 @@
1
+ """seatrial: run evaluation cases for agent skills on agent CLIs and compare revisions.
2
+
3
+ Modules:
4
+ ports the HostAdapter port and the Evidence record every adapter returns
5
+ hosts one adapter per agent CLI, registered by name
6
+ cases folder-based case discovery and selection
7
+ checks shared fixture helpers and grading checks for case authors
8
+ process bounded process launch
9
+ workspace skill mounts, file manifests, and the run state directory
10
+ stats pass@k, pass^k, Fisher and Mann-Whitney tests, verdicts
11
+ report grading, per-arm summaries, and the Markdown report
12
+ runner scheduling and execution of trials
13
+ config seatrial.toml discovery, validation, and the active configuration
14
+ setup `init` and `doctor`
15
+ cli the seatrial command
16
+ interactive the menu-driven terminal mode
17
+ """
18
+
19
+ __version__ = '0.1.0' # x-release-please-version
@@ -0,0 +1,3 @@
1
+ from seatrial.cli import entry
2
+
3
+ entry()