seatrial 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- seatrial-0.1.0/LICENSE +21 -0
- seatrial-0.1.0/PKG-INFO +283 -0
- seatrial-0.1.0/README.md +262 -0
- seatrial-0.1.0/pyproject.toml +37 -0
- seatrial-0.1.0/setup.cfg +4 -0
- seatrial-0.1.0/src/seatrial/__init__.py +19 -0
- seatrial-0.1.0/src/seatrial/__main__.py +3 -0
- seatrial-0.1.0/src/seatrial/cases.py +214 -0
- seatrial-0.1.0/src/seatrial/checks.py +184 -0
- seatrial-0.1.0/src/seatrial/cli.py +499 -0
- seatrial-0.1.0/src/seatrial/config.py +214 -0
- seatrial-0.1.0/src/seatrial/hosts/__init__.py +69 -0
- seatrial-0.1.0/src/seatrial/hosts/claude.py +162 -0
- seatrial-0.1.0/src/seatrial/hosts/codex.py +280 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/claude/auth-error.expected.json +1 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/claude/auth-error.jsonl +2 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/claude/delegated-report.expected.json +4 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/claude/delegated-report.jsonl +18 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/delegated-report.expected.json +4 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/delegated-report.jsonl +11 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-paths.expected.json +18 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-paths.jsonl +8 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-reads.expected.json +3 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/skill-reads.jsonl +6 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/skipped-cd.expected.json +12 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/skipped-cd.jsonl +5 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/usage-limit.expected.json +1 -0
- seatrial-0.1.0/src/seatrial/hosts/samples/codex/usage-limit.jsonl +4 -0
- seatrial-0.1.0/src/seatrial/interactive.py +121 -0
- seatrial-0.1.0/src/seatrial/ports.py +192 -0
- seatrial-0.1.0/src/seatrial/process.py +127 -0
- seatrial-0.1.0/src/seatrial/report.py +172 -0
- seatrial-0.1.0/src/seatrial/runner.py +292 -0
- seatrial-0.1.0/src/seatrial/setup.py +219 -0
- seatrial-0.1.0/src/seatrial/stats.py +85 -0
- seatrial-0.1.0/src/seatrial/templates.py +70 -0
- seatrial-0.1.0/src/seatrial/ui.py +226 -0
- seatrial-0.1.0/src/seatrial/workspace.py +234 -0
- seatrial-0.1.0/src/seatrial.egg-info/PKG-INFO +283 -0
- seatrial-0.1.0/src/seatrial.egg-info/SOURCES.txt +48 -0
- seatrial-0.1.0/src/seatrial.egg-info/dependency_links.txt +1 -0
- seatrial-0.1.0/src/seatrial.egg-info/entry_points.txt +2 -0
- seatrial-0.1.0/src/seatrial.egg-info/top_level.txt +1 -0
- seatrial-0.1.0/tests/test_cases.py +125 -0
- seatrial-0.1.0/tests/test_cli.py +215 -0
- seatrial-0.1.0/tests/test_host_contract.py +267 -0
- seatrial-0.1.0/tests/test_process.py +102 -0
- seatrial-0.1.0/tests/test_runner.py +303 -0
- seatrial-0.1.0/tests/test_stats.py +65 -0
- seatrial-0.1.0/tests/test_workspace.py +250 -0
seatrial-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Otavio Barros
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
seatrial-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: seatrial
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Run evaluation cases for agent skills on agent CLIs and compare skill revisions.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/otaviobarros/seatrial
|
|
7
|
+
Project-URL: Documentation, https://github.com/otaviobarros/seatrial/tree/main/docs
|
|
8
|
+
Project-URL: Issues, https://github.com/otaviobarros/seatrial/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/otaviobarros/seatrial/releases
|
|
10
|
+
Keywords: agents,skills,evaluation,claude,codex
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Topic :: Software Development :: Testing
|
|
17
|
+
Requires-Python: >=3.11
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
<!-- markdownlint-disable-next-line MD041 -->
|
|
23
|
+
<p align="center">
|
|
24
|
+
<picture>
|
|
25
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-dark.png">
|
|
26
|
+
<img alt="seatrial" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-light.png" width="420">
|
|
27
|
+
</picture>
|
|
28
|
+
</p>
|
|
29
|
+
|
|
30
|
+
<p align="center">Sea trials for agent skills.</p>
|
|
31
|
+
|
|
32
|
+
<p align="center">
|
|
33
|
+
<a href="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml/badge.svg"></a>
|
|
34
|
+
<img alt="Python 3.11+" src="https://img.shields.io/badge/python-3.11%2B-1d6fa5">
|
|
35
|
+
<img alt="No dependencies" src="https://img.shields.io/badge/dependencies-none-0b7a8f">
|
|
36
|
+
<a href="https://github.com/otaviobarros/seatrial/blob/main/LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-0b2540"></a>
|
|
37
|
+
</p>
|
|
38
|
+
|
|
39
|
+
You changed a skill. Did the agent get better, or did it only feel better on the
|
|
40
|
+
one prompt you tried? seatrial answers that the way a shipyard tests a boat: it
|
|
41
|
+
takes the skill out on a fixed course, many times, and records what happened.
|
|
42
|
+
|
|
43
|
+
- **Real agents.** Each trial runs the actual CLI, Claude Code (`claude -p`) or
|
|
44
|
+
Codex (`codex exec`), in a fresh project. Add other hosts through a small
|
|
45
|
+
[adapter](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md).
|
|
46
|
+
- **Graded by code.** A case's `grade()` checks files, Git state, commands, and the
|
|
47
|
+
final message. No model judges the result.
|
|
48
|
+
- **Two revisions, one verdict.** `compare` runs `main` against your uncommitted
|
|
49
|
+
edits and reports `improved`, `non-inferior`, `regressed`, or `inconclusive`.
|
|
50
|
+
- **Nothing to install into your project.** Python 3.11+ standard library and Git.
|
|
51
|
+
|
|
52
|
+
## How it works
|
|
53
|
+
|
|
54
|
+
<picture>
|
|
55
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-dark.svg">
|
|
56
|
+
<img alt="Diagram of one trial in five steps: case, mount, agent, evidence, grade. The trial repeats n times per case, host, and revision, and the results go to summary.json." src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-light.svg">
|
|
57
|
+
</picture>
|
|
58
|
+
|
|
59
|
+
<details>
|
|
60
|
+
<summary>The five steps in text</summary>
|
|
61
|
+
|
|
62
|
+
Each trial runs these steps:
|
|
63
|
+
|
|
64
|
+
1. **Case.** `build()` in the case's `case.py` writes a small fictional project
|
|
65
|
+
and returns the request a user would type.
|
|
66
|
+
2. **Mount.** seatrial mounts your skills at one revision, a Git ref such as
|
|
67
|
+
`main` or the uncommitted `WORKTREE`, as a plugin.
|
|
68
|
+
3. **Agent.** The real host CLI (`claude` or `codex`) runs the request in a
|
|
69
|
+
fresh, isolated folder.
|
|
70
|
+
4. **Evidence.** seatrial records the commands, file edits, loaded skills, and
|
|
71
|
+
final message in `evidence.json`.
|
|
72
|
+
5. **Grade.** `grade()` checks that evidence and the project, and returns pass or
|
|
73
|
+
fail for each assertion in `case.toml`.
|
|
74
|
+
|
|
75
|
+
</details>
|
|
76
|
+
|
|
77
|
+
seatrial repeats each trial n times per case, host, and revision, and writes the
|
|
78
|
+
pass rates and verdicts to `summary.json`. Repetition turns one lucky run into a
|
|
79
|
+
pass rate.
|
|
80
|
+
|
|
81
|
+
## How seatrial compares
|
|
82
|
+
|
|
83
|
+
Other tools evaluate agents too. This table reflects their public documentation
|
|
84
|
+
in November 2026.
|
|
85
|
+
|
|
86
|
+
| | seatrial | [`claude plugin eval`](https://code.claude.com/docs/en/plugin-evals) | [promptfoo](https://www.promptfoo.dev/docs/guides/test-agent-skills/) | [Inspect](https://inspect.aisi.org.uk) + [inspect_swe](https://meridianlabs-ai.github.io/inspect_swe) |
|
|
87
|
+
|---|---|---|---|---|
|
|
88
|
+
| Agents | Claude Code and Codex CLIs; others through an adapter | Claude Code | Claude Agent SDK, Codex SDK, OpenCode SDK | Claude Code and Codex CLI as Inspect agents, with any model |
|
|
89
|
+
| Compares | Two Git revisions of your skills, including uncommitted edits | The plugin loaded against no plugin | Skill versions in separate fixture folders, side by side | Models and agents on a task set |
|
|
90
|
+
| Grading | Your Python code over files, Git state, commands, and the final message | Regex, tool use, and file checks, plus model-judged rubrics | Assertions such as skill used, custom JavaScript, plus model-graded | Code or model scorers |
|
|
91
|
+
| Statistics | Fisher exact test per assertion, and a verdict per case and host | Three runs per case by default, mean scores against a threshold | `--repeat`; no significance test | Metrics such as accuracy and standard error |
|
|
92
|
+
| Isolation | Fresh Git project per trial, filtered environment, flags skills from outside the plugin | Fresh home, working folder, and config per run | A `working_dir` per provider | Docker sandbox |
|
|
93
|
+
| Runtime | Python 3.11+ standard library; no dependencies | Built into Claude Code 2.1.269+ | Node and the agent SDK packages | Python packages; Docker for the sandbox |
|
|
94
|
+
|
|
95
|
+
Choose seatrial to answer "did this skill edit make the agent better or worse"
|
|
96
|
+
on the real CLIs, with code-graded cases and a statistical verdict. Choose
|
|
97
|
+
`claude plugin eval` to check that a plugin helps compared with no plugin, inside
|
|
98
|
+
Claude Code. Choose promptfoo when your evals already live in promptfoo or need
|
|
99
|
+
model-graded rubrics. Choose Inspect for benchmark-scale runs in sandboxes or
|
|
100
|
+
across many models.
|
|
101
|
+
|
|
102
|
+
## Install
|
|
103
|
+
|
|
104
|
+
Run it with [uv](https://docs.astral.sh/uv/). `uvx` builds the package into a cached,
|
|
105
|
+
isolated environment and runs the command; nothing is installed into your project.
|
|
106
|
+
|
|
107
|
+
```sh
|
|
108
|
+
uvx seatrial --help # from PyPI
|
|
109
|
+
uvx --from . seatrial --help # from a checkout
|
|
110
|
+
uv tool install seatrial # keep `seatrial` on PATH
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Without uv, use `pipx install seatrial`, or run `python3 -m seatrial`
|
|
114
|
+
with `src/` on `PYTHONPATH`. The package has no dependencies beyond Python 3.11+.
|
|
115
|
+
|
|
116
|
+
seatrial runs on macOS and Linux. Windows is not supported; use WSL.
|
|
117
|
+
|
|
118
|
+
## Quickstart
|
|
119
|
+
|
|
120
|
+
```sh
|
|
121
|
+
cd my-skills-project
|
|
122
|
+
seatrial init # detect skills, write seatrial.toml and an example case
|
|
123
|
+
seatrial doctor # check Python, Git, the configuration, cases, and host CLIs
|
|
124
|
+
seatrial list
|
|
125
|
+
seatrial run --dry-run # print the trial plan; no model calls
|
|
126
|
+
seatrial run --host claude --n 1
|
|
127
|
+
seatrial # interactive mode on a terminal
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
`init` finds folders that hold `<name>/SKILL.md`. Two layouts work:
|
|
131
|
+
|
|
132
|
+
- `plugin`: a Claude Code plugin root with `skills/` and `.claude-plugin/plugin.json`.
|
|
133
|
+
- `skills`: a plain folder of skills. seatrial mounts it as a generated plugin.
|
|
134
|
+
|
|
135
|
+
On a terminal, `init` asks which folder, namespace, case folder, and hosts to use.
|
|
136
|
+
With `--yes` or without a terminal, it uses the detected values. See
|
|
137
|
+
[configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md) for every key.
|
|
138
|
+
|
|
139
|
+
## Commands
|
|
140
|
+
|
|
141
|
+
| Command | Does |
|
|
142
|
+
|---|---|
|
|
143
|
+
| `init` | Write `seatrial.toml` and a runnable example case |
|
|
144
|
+
| `doctor` | Check the environment; exits 3 when a check fails |
|
|
145
|
+
| `list` | List cases; accepts selectors |
|
|
146
|
+
| `new-case --skill S --id NAME-001` | Create a case skeleton; prompts on a terminal |
|
|
147
|
+
| `run [--rev REV]` | Run cases against one skill revision |
|
|
148
|
+
| `compare --base REV [--head REV]` | Run both revisions, interleaved, and give a verdict per case and host |
|
|
149
|
+
| `runs` | List retained run directories, newest first |
|
|
150
|
+
| `report [RUN_DIR\|latest]` | Print a run summary; `--format markdown` for a Markdown table |
|
|
151
|
+
| `regrade [RUN_DIR\|latest]` | Grade retained trials again with the current case code; no host calls |
|
|
152
|
+
| `hosts` | List host adapters, models, and CLI versions |
|
|
153
|
+
| `schema` | Print the configuration, case, evidence, and exit-code contract |
|
|
154
|
+
| `interactive` | Menu-driven mode; also the default with no arguments on a terminal |
|
|
155
|
+
|
|
156
|
+
Every command except `interactive` accepts `--json`. `seatrial COMMAND --help` lists all options.
|
|
157
|
+
For agents and CI, read [using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md).
|
|
158
|
+
|
|
159
|
+
## Agent skill
|
|
160
|
+
|
|
161
|
+
[`skills/seatrial`](https://github.com/otaviobarros/seatrial/blob/main/skills/seatrial/SKILL.md) teaches an agent to set up seatrial,
|
|
162
|
+
write cases, run and compare within a budget, and read the results. Install it with
|
|
163
|
+
the [skills](https://skills.sh) CLI:
|
|
164
|
+
|
|
165
|
+
```sh
|
|
166
|
+
npx skills add otaviobarros/seatrial
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Run and compare
|
|
170
|
+
|
|
171
|
+
`run` and `compare` call host CLIs and spend quota. Before a native run:
|
|
172
|
+
|
|
173
|
+
1. Sign in to each CLI (`claude /login`, `codex login`).
|
|
174
|
+
2. Print the plan with `--dry-run`.
|
|
175
|
+
3. Set a budget: `--n`, `--max-runs`, `--max-minutes`, and `--max-cost-usd` for
|
|
176
|
+
hosts that report cost.
|
|
177
|
+
|
|
178
|
+
A revision is a Git revision or `WORKTREE`, the skill files in the checkout with
|
|
179
|
+
uncommitted changes. `compare` defaults `--head` to `WORKTREE`, so
|
|
180
|
+
`seatrial compare --base main` measures uncommitted edits. Each revision mounts
|
|
181
|
+
as a whole skill folder; trials alternate between revisions so that time and load
|
|
182
|
+
affect both alike.
|
|
183
|
+
|
|
184
|
+
Selectors combine and repeat: `--case ID`, `--skill NAME`, `--rule ID`,
|
|
185
|
+
`--suite capability|regression`, `--host NAME`.
|
|
186
|
+
|
|
187
|
+
<details>
|
|
188
|
+
<summary>Options, defaults, and limits</summary>
|
|
189
|
+
|
|
190
|
+
| Option | Default | Maximum |
|
|
191
|
+
|---|---|---|
|
|
192
|
+
| `--n` trials per case, host, and revision | `[run] n`, 3 | `[limits] max_n`, 10 |
|
|
193
|
+
| `--jobs` parallel trials | `[run] jobs`, 1 | `max_jobs`, 4 |
|
|
194
|
+
| `--max-runs` trials in the plan | `max_runs`, 100 | `max_runs` |
|
|
195
|
+
| `--max-minutes`; a trial starts only if its timeout fits | `max_minutes`, 90 | `max_minutes` |
|
|
196
|
+
| `--timeout` seconds per trial | case value, else `[run] timeout`, 180 | `max_timeout`, 600 |
|
|
197
|
+
| `--max-cost-usd` total host-reported cost | none | none |
|
|
198
|
+
| `--budget-usd` per-trial cap (Claude only) | none | none |
|
|
199
|
+
|
|
200
|
+
</details>
|
|
201
|
+
|
|
202
|
+
A usage-limit or authentication error stops the run. Finished trials are kept,
|
|
203
|
+
and the command exits 1.
|
|
204
|
+
|
|
205
|
+
## Verdicts
|
|
206
|
+
|
|
207
|
+
<picture>
|
|
208
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-dark.svg">
|
|
209
|
+
<img alt="compare: base main passes 4 of 10 trials, head WORKTREE passes 10 of 10, verdict improved with Fisher p = 0.011" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-light.svg">
|
|
210
|
+
</picture>
|
|
211
|
+
|
|
212
|
+
`compare` gives each case and host one advisory verdict:
|
|
213
|
+
|
|
214
|
+
| Verdict | Meaning |
|
|
215
|
+
|---|---|
|
|
216
|
+
| `regressed` | A zero-tolerance assertion newly fails, or an assertion is significantly worse (Fisher p < 0.05) |
|
|
217
|
+
| `improved` | The primary assertion is significantly better, and head has no zero-tolerance failure |
|
|
218
|
+
| `non-inferior` | No assertion drops by more than two trials' worth |
|
|
219
|
+
| `inconclusive` | Otherwise, or fewer than 80% of the trials in a revision are valid |
|
|
220
|
+
|
|
221
|
+
Ten trials per revision detect a change of about 50 percentage points on the
|
|
222
|
+
primary assertion. With 3 trials, `improved` is not possible. Report a
|
|
223
|
+
non-significant result as "no detected effect". The report also shows pass^k,
|
|
224
|
+
median lines added, mean wall time, and host-reported cost. Use
|
|
225
|
+
`--fail-on-regression` to exit 4 on any `regressed` verdict.
|
|
226
|
+
|
|
227
|
+
<p align="center">
|
|
228
|
+
<img alt="Terminal output of seatrial compare: progress lines, then a table of valid trials, pass counts, pass^k, wall time, cost, and verdicts for two cases on Claude and Codex" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/terminal.svg" width="880">
|
|
229
|
+
</p>
|
|
230
|
+
|
|
231
|
+
## Results
|
|
232
|
+
|
|
233
|
+
Each invocation writes a run directory under `[run] state_dir` (default
|
|
234
|
+
`$SEATRIAL_STATE`, else `~/.cache/workspaces/<timestamp>/`). The path prints at
|
|
235
|
+
the end of the run. Keep it out of version control. Projects can share a state
|
|
236
|
+
directory: `runs`, `report latest`, and `regrade` show only the current project's runs.
|
|
237
|
+
|
|
238
|
+
<details>
|
|
239
|
+
<summary>Files in a run directory</summary>
|
|
240
|
+
|
|
241
|
+
| File | Content |
|
|
242
|
+
|---|---|
|
|
243
|
+
| `run.json` | The run settings and revisions, written before the first trial |
|
|
244
|
+
| `records.jsonl` | One record per trial, written as each trial ends; `foreign_skills`, `unknown_skills`, and `outside_access` mark trials that used skills from outside the plugin, read a skill whose source the trace cannot show, or named the project path |
|
|
245
|
+
| `summary.json` | Per-case, per-host, per-revision summaries and verdicts; missing when the run was interrupted, until `seatrial regrade` builds `summary-regraded.json` |
|
|
246
|
+
| `trials/<random>/` | The trial project, `session.jsonl` trace, `stderr.txt`, `evidence.json`, `retained.json` |
|
|
247
|
+
|
|
248
|
+
</details>
|
|
249
|
+
|
|
250
|
+
Trial directories use random names so that no path the agent sees names the
|
|
251
|
+
evaluation. Private host state, including any copied login, is removed after each
|
|
252
|
+
trial.
|
|
253
|
+
|
|
254
|
+
<details>
|
|
255
|
+
<summary>Trial statuses</summary>
|
|
256
|
+
|
|
257
|
+
Only `completed` trials are graded. Other statuses are excluded from pass rates:
|
|
258
|
+
|
|
259
|
+
| Status | Meaning |
|
|
260
|
+
|---|---|
|
|
261
|
+
| `control` | The grader marked the trial as a control, not a measurement |
|
|
262
|
+
| `trace_incomplete` | The agent delegated, but at least one delegated session was not captured |
|
|
263
|
+
| `usage_limit`, `auth_error` | Host quota or sign-in problem; the run stops |
|
|
264
|
+
| `timeout`, `host_stall` | The time cap ended the trial; `host_stall` means the machine paused |
|
|
265
|
+
| `incomplete` | The host exited without a successful final turn |
|
|
266
|
+
| `harness_error` | seatrial failed; the record holds the exception |
|
|
267
|
+
| `stale` | From `regrade` only: the retained project files or `evidence.json` changed after the trial |
|
|
268
|
+
|
|
269
|
+
</details>
|
|
270
|
+
|
|
271
|
+
`regrade` writes `records-regraded.jsonl` and `summary-regraded.json`. Copy
|
|
272
|
+
retained trial files before you inspect them with tools that write.
|
|
273
|
+
|
|
274
|
+
## Documentation
|
|
275
|
+
|
|
276
|
+
- [Configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md): `seatrial.toml` keys and defaults.
|
|
277
|
+
- [Writing cases](https://github.com/otaviobarros/seatrial/blob/main/docs/cases.md): `case.toml`, `case.py`, evidence, and grading helpers.
|
|
278
|
+
- [Hosts](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md): built-in adapters and how to add one.
|
|
279
|
+
- [Using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md): JSON output, exit codes, and a safe procedure.
|
|
280
|
+
|
|
281
|
+
## License
|
|
282
|
+
|
|
283
|
+
MIT. See [LICENSE](LICENSE).
|
seatrial-0.1.0/README.md
ADDED
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
<!-- markdownlint-disable-next-line MD041 -->
|
|
2
|
+
<p align="center">
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-dark.png">
|
|
5
|
+
<img alt="seatrial" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/logo-light.png" width="420">
|
|
6
|
+
</picture>
|
|
7
|
+
</p>
|
|
8
|
+
|
|
9
|
+
<p align="center">Sea trials for agent skills.</p>
|
|
10
|
+
|
|
11
|
+
<p align="center">
|
|
12
|
+
<a href="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml"><img alt="CI" src="https://github.com/otaviobarros/seatrial/actions/workflows/ci.yml/badge.svg"></a>
|
|
13
|
+
<img alt="Python 3.11+" src="https://img.shields.io/badge/python-3.11%2B-1d6fa5">
|
|
14
|
+
<img alt="No dependencies" src="https://img.shields.io/badge/dependencies-none-0b7a8f">
|
|
15
|
+
<a href="https://github.com/otaviobarros/seatrial/blob/main/LICENSE"><img alt="MIT license" src="https://img.shields.io/badge/license-MIT-0b2540"></a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
You changed a skill. Did the agent get better, or did it only feel better on the
|
|
19
|
+
one prompt you tried? seatrial answers that the way a shipyard tests a boat: it
|
|
20
|
+
takes the skill out on a fixed course, many times, and records what happened.
|
|
21
|
+
|
|
22
|
+
- **Real agents.** Each trial runs the actual CLI, Claude Code (`claude -p`) or
|
|
23
|
+
Codex (`codex exec`), in a fresh project. Add other hosts through a small
|
|
24
|
+
[adapter](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md).
|
|
25
|
+
- **Graded by code.** A case's `grade()` checks files, Git state, commands, and the
|
|
26
|
+
final message. No model judges the result.
|
|
27
|
+
- **Two revisions, one verdict.** `compare` runs `main` against your uncommitted
|
|
28
|
+
edits and reports `improved`, `non-inferior`, `regressed`, or `inconclusive`.
|
|
29
|
+
- **Nothing to install into your project.** Python 3.11+ standard library and Git.
|
|
30
|
+
|
|
31
|
+
## How it works
|
|
32
|
+
|
|
33
|
+
<picture>
|
|
34
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-dark.svg">
|
|
35
|
+
<img alt="Diagram of one trial in five steps: case, mount, agent, evidence, grade. The trial repeats n times per case, host, and revision, and the results go to summary.json." src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/how-it-works-light.svg">
|
|
36
|
+
</picture>
|
|
37
|
+
|
|
38
|
+
<details>
|
|
39
|
+
<summary>The five steps in text</summary>
|
|
40
|
+
|
|
41
|
+
Each trial runs these steps:
|
|
42
|
+
|
|
43
|
+
1. **Case.** `build()` in the case's `case.py` writes a small fictional project
|
|
44
|
+
and returns the request a user would type.
|
|
45
|
+
2. **Mount.** seatrial mounts your skills at one revision, a Git ref such as
|
|
46
|
+
`main` or the uncommitted `WORKTREE`, as a plugin.
|
|
47
|
+
3. **Agent.** The real host CLI (`claude` or `codex`) runs the request in a
|
|
48
|
+
fresh, isolated folder.
|
|
49
|
+
4. **Evidence.** seatrial records the commands, file edits, loaded skills, and
|
|
50
|
+
final message in `evidence.json`.
|
|
51
|
+
5. **Grade.** `grade()` checks that evidence and the project, and returns pass or
|
|
52
|
+
fail for each assertion in `case.toml`.
|
|
53
|
+
|
|
54
|
+
</details>
|
|
55
|
+
|
|
56
|
+
seatrial repeats each trial n times per case, host, and revision, and writes the
|
|
57
|
+
pass rates and verdicts to `summary.json`. Repetition turns one lucky run into a
|
|
58
|
+
pass rate.
|
|
59
|
+
|
|
60
|
+
## How seatrial compares
|
|
61
|
+
|
|
62
|
+
Other tools evaluate agents too. This table reflects their public documentation
|
|
63
|
+
in November 2026.
|
|
64
|
+
|
|
65
|
+
| | seatrial | [`claude plugin eval`](https://code.claude.com/docs/en/plugin-evals) | [promptfoo](https://www.promptfoo.dev/docs/guides/test-agent-skills/) | [Inspect](https://inspect.aisi.org.uk) + [inspect_swe](https://meridianlabs-ai.github.io/inspect_swe) |
|
|
66
|
+
|---|---|---|---|---|
|
|
67
|
+
| Agents | Claude Code and Codex CLIs; others through an adapter | Claude Code | Claude Agent SDK, Codex SDK, OpenCode SDK | Claude Code and Codex CLI as Inspect agents, with any model |
|
|
68
|
+
| Compares | Two Git revisions of your skills, including uncommitted edits | The plugin loaded against no plugin | Skill versions in separate fixture folders, side by side | Models and agents on a task set |
|
|
69
|
+
| Grading | Your Python code over files, Git state, commands, and the final message | Regex, tool use, and file checks, plus model-judged rubrics | Assertions such as skill used, custom JavaScript, plus model-graded | Code or model scorers |
|
|
70
|
+
| Statistics | Fisher exact test per assertion, and a verdict per case and host | Three runs per case by default, mean scores against a threshold | `--repeat`; no significance test | Metrics such as accuracy and standard error |
|
|
71
|
+
| Isolation | Fresh Git project per trial, filtered environment, flags skills from outside the plugin | Fresh home, working folder, and config per run | A `working_dir` per provider | Docker sandbox |
|
|
72
|
+
| Runtime | Python 3.11+ standard library; no dependencies | Built into Claude Code 2.1.269+ | Node and the agent SDK packages | Python packages; Docker for the sandbox |
|
|
73
|
+
|
|
74
|
+
Choose seatrial to answer "did this skill edit make the agent better or worse"
|
|
75
|
+
on the real CLIs, with code-graded cases and a statistical verdict. Choose
|
|
76
|
+
`claude plugin eval` to check that a plugin helps compared with no plugin, inside
|
|
77
|
+
Claude Code. Choose promptfoo when your evals already live in promptfoo or need
|
|
78
|
+
model-graded rubrics. Choose Inspect for benchmark-scale runs in sandboxes or
|
|
79
|
+
across many models.
|
|
80
|
+
|
|
81
|
+
## Install
|
|
82
|
+
|
|
83
|
+
Run it with [uv](https://docs.astral.sh/uv/). `uvx` builds the package into a cached,
|
|
84
|
+
isolated environment and runs the command; nothing is installed into your project.
|
|
85
|
+
|
|
86
|
+
```sh
|
|
87
|
+
uvx seatrial --help # from PyPI
|
|
88
|
+
uvx --from . seatrial --help # from a checkout
|
|
89
|
+
uv tool install seatrial # keep `seatrial` on PATH
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Without uv, use `pipx install seatrial`, or run `python3 -m seatrial`
|
|
93
|
+
with `src/` on `PYTHONPATH`. The package has no dependencies beyond Python 3.11+.
|
|
94
|
+
|
|
95
|
+
seatrial runs on macOS and Linux. Windows is not supported; use WSL.
|
|
96
|
+
|
|
97
|
+
## Quickstart
|
|
98
|
+
|
|
99
|
+
```sh
|
|
100
|
+
cd my-skills-project
|
|
101
|
+
seatrial init # detect skills, write seatrial.toml and an example case
|
|
102
|
+
seatrial doctor # check Python, Git, the configuration, cases, and host CLIs
|
|
103
|
+
seatrial list
|
|
104
|
+
seatrial run --dry-run # print the trial plan; no model calls
|
|
105
|
+
seatrial run --host claude --n 1
|
|
106
|
+
seatrial # interactive mode on a terminal
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
`init` finds folders that hold `<name>/SKILL.md`. Two layouts work:
|
|
110
|
+
|
|
111
|
+
- `plugin`: a Claude Code plugin root with `skills/` and `.claude-plugin/plugin.json`.
|
|
112
|
+
- `skills`: a plain folder of skills. seatrial mounts it as a generated plugin.
|
|
113
|
+
|
|
114
|
+
On a terminal, `init` asks which folder, namespace, case folder, and hosts to use.
|
|
115
|
+
With `--yes` or without a terminal, it uses the detected values. See
|
|
116
|
+
[configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md) for every key.
|
|
117
|
+
|
|
118
|
+
## Commands
|
|
119
|
+
|
|
120
|
+
| Command | Does |
|
|
121
|
+
|---|---|
|
|
122
|
+
| `init` | Write `seatrial.toml` and a runnable example case |
|
|
123
|
+
| `doctor` | Check the environment; exits 3 when a check fails |
|
|
124
|
+
| `list` | List cases; accepts selectors |
|
|
125
|
+
| `new-case --skill S --id NAME-001` | Create a case skeleton; prompts on a terminal |
|
|
126
|
+
| `run [--rev REV]` | Run cases against one skill revision |
|
|
127
|
+
| `compare --base REV [--head REV]` | Run both revisions, interleaved, and give a verdict per case and host |
|
|
128
|
+
| `runs` | List retained run directories, newest first |
|
|
129
|
+
| `report [RUN_DIR\|latest]` | Print a run summary; `--format markdown` for a Markdown table |
|
|
130
|
+
| `regrade [RUN_DIR\|latest]` | Grade retained trials again with the current case code; no host calls |
|
|
131
|
+
| `hosts` | List host adapters, models, and CLI versions |
|
|
132
|
+
| `schema` | Print the configuration, case, evidence, and exit-code contract |
|
|
133
|
+
| `interactive` | Menu-driven mode; also the default with no arguments on a terminal |
|
|
134
|
+
|
|
135
|
+
Every command except `interactive` accepts `--json`. `seatrial COMMAND --help` lists all options.
|
|
136
|
+
For agents and CI, read [using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md).
|
|
137
|
+
|
|
138
|
+
## Agent skill
|
|
139
|
+
|
|
140
|
+
[`skills/seatrial`](https://github.com/otaviobarros/seatrial/blob/main/skills/seatrial/SKILL.md) teaches an agent to set up seatrial,
|
|
141
|
+
write cases, run and compare within a budget, and read the results. Install it with
|
|
142
|
+
the [skills](https://skills.sh) CLI:
|
|
143
|
+
|
|
144
|
+
```sh
|
|
145
|
+
npx skills add otaviobarros/seatrial
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
## Run and compare
|
|
149
|
+
|
|
150
|
+
`run` and `compare` call host CLIs and spend quota. Before a native run:
|
|
151
|
+
|
|
152
|
+
1. Sign in to each CLI (`claude /login`, `codex login`).
|
|
153
|
+
2. Print the plan with `--dry-run`.
|
|
154
|
+
3. Set a budget: `--n`, `--max-runs`, `--max-minutes`, and `--max-cost-usd` for
|
|
155
|
+
hosts that report cost.
|
|
156
|
+
|
|
157
|
+
A revision is a Git revision or `WORKTREE`, the skill files in the checkout with
|
|
158
|
+
uncommitted changes. `compare` defaults `--head` to `WORKTREE`, so
|
|
159
|
+
`seatrial compare --base main` measures uncommitted edits. Each revision mounts
|
|
160
|
+
as a whole skill folder; trials alternate between revisions so that time and load
|
|
161
|
+
affect both alike.
|
|
162
|
+
|
|
163
|
+
Selectors combine and repeat: `--case ID`, `--skill NAME`, `--rule ID`,
|
|
164
|
+
`--suite capability|regression`, `--host NAME`.
|
|
165
|
+
|
|
166
|
+
<details>
|
|
167
|
+
<summary>Options, defaults, and limits</summary>
|
|
168
|
+
|
|
169
|
+
| Option | Default | Maximum |
|
|
170
|
+
|---|---|---|
|
|
171
|
+
| `--n` trials per case, host, and revision | `[run] n`, 3 | `[limits] max_n`, 10 |
|
|
172
|
+
| `--jobs` parallel trials | `[run] jobs`, 1 | `max_jobs`, 4 |
|
|
173
|
+
| `--max-runs` trials in the plan | `max_runs`, 100 | `max_runs` |
|
|
174
|
+
| `--max-minutes`; a trial starts only if its timeout fits | `max_minutes`, 90 | `max_minutes` |
|
|
175
|
+
| `--timeout` seconds per trial | case value, else `[run] timeout`, 180 | `max_timeout`, 600 |
|
|
176
|
+
| `--max-cost-usd` total host-reported cost | none | none |
|
|
177
|
+
| `--budget-usd` per-trial cap (Claude only) | none | none |
|
|
178
|
+
|
|
179
|
+
</details>
|
|
180
|
+
|
|
181
|
+
A usage-limit or authentication error stops the run. Finished trials are kept,
|
|
182
|
+
and the command exits 1.
|
|
183
|
+
|
|
184
|
+
## Verdicts
|
|
185
|
+
|
|
186
|
+
<picture>
|
|
187
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-dark.svg">
|
|
188
|
+
<img alt="compare: base main passes 4 of 10 trials, head WORKTREE passes 10 of 10, verdict improved with Fisher p = 0.011" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/compare-light.svg">
|
|
189
|
+
</picture>
|
|
190
|
+
|
|
191
|
+
`compare` gives each case and host one advisory verdict:
|
|
192
|
+
|
|
193
|
+
| Verdict | Meaning |
|
|
194
|
+
|---|---|
|
|
195
|
+
| `regressed` | A zero-tolerance assertion newly fails, or an assertion is significantly worse (Fisher p < 0.05) |
|
|
196
|
+
| `improved` | The primary assertion is significantly better, and head has no zero-tolerance failure |
|
|
197
|
+
| `non-inferior` | No assertion drops by more than two trials' worth |
|
|
198
|
+
| `inconclusive` | Otherwise, or fewer than 80% of the trials in a revision are valid |
|
|
199
|
+
|
|
200
|
+
Ten trials per revision detect a change of about 50 percentage points on the
|
|
201
|
+
primary assertion. With 3 trials, `improved` is not possible. Report a
|
|
202
|
+
non-significant result as "no detected effect". The report also shows pass^k,
|
|
203
|
+
median lines added, mean wall time, and host-reported cost. Use
|
|
204
|
+
`--fail-on-regression` to exit 4 on any `regressed` verdict.
|
|
205
|
+
|
|
206
|
+
<p align="center">
|
|
207
|
+
<img alt="Terminal output of seatrial compare: progress lines, then a table of valid trials, pass counts, pass^k, wall time, cost, and verdicts for two cases on Claude and Codex" src="https://raw.githubusercontent.com/otaviobarros/seatrial/main/docs/images/terminal.svg" width="880">
|
|
208
|
+
</p>
|
|
209
|
+
|
|
210
|
+
## Results
|
|
211
|
+
|
|
212
|
+
Each invocation writes a run directory under `[run] state_dir` (default
|
|
213
|
+
`$SEATRIAL_STATE`, else `~/.cache/workspaces/<timestamp>/`). The path prints at
|
|
214
|
+
the end of the run. Keep it out of version control. Projects can share a state
|
|
215
|
+
directory: `runs`, `report latest`, and `regrade` show only the current project's runs.
|
|
216
|
+
|
|
217
|
+
<details>
|
|
218
|
+
<summary>Files in a run directory</summary>
|
|
219
|
+
|
|
220
|
+
| File | Content |
|
|
221
|
+
|---|---|
|
|
222
|
+
| `run.json` | The run settings and revisions, written before the first trial |
|
|
223
|
+
| `records.jsonl` | One record per trial, written as each trial ends; `foreign_skills`, `unknown_skills`, and `outside_access` mark trials that used skills from outside the plugin, read a skill whose source the trace cannot show, or named the project path |
|
|
224
|
+
| `summary.json` | Per-case, per-host, per-revision summaries and verdicts; missing when the run was interrupted, until `seatrial regrade` builds `summary-regraded.json` |
|
|
225
|
+
| `trials/<random>/` | The trial project, `session.jsonl` trace, `stderr.txt`, `evidence.json`, `retained.json` |
|
|
226
|
+
|
|
227
|
+
</details>
|
|
228
|
+
|
|
229
|
+
Trial directories use random names so that no path the agent sees names the
|
|
230
|
+
evaluation. Private host state, including any copied login, is removed after each
|
|
231
|
+
trial.
|
|
232
|
+
|
|
233
|
+
<details>
|
|
234
|
+
<summary>Trial statuses</summary>
|
|
235
|
+
|
|
236
|
+
Only `completed` trials are graded. Other statuses are excluded from pass rates:
|
|
237
|
+
|
|
238
|
+
| Status | Meaning |
|
|
239
|
+
|---|---|
|
|
240
|
+
| `control` | The grader marked the trial as a control, not a measurement |
|
|
241
|
+
| `trace_incomplete` | The agent delegated, but at least one delegated session was not captured |
|
|
242
|
+
| `usage_limit`, `auth_error` | Host quota or sign-in problem; the run stops |
|
|
243
|
+
| `timeout`, `host_stall` | The time cap ended the trial; `host_stall` means the machine paused |
|
|
244
|
+
| `incomplete` | The host exited without a successful final turn |
|
|
245
|
+
| `harness_error` | seatrial failed; the record holds the exception |
|
|
246
|
+
| `stale` | From `regrade` only: the retained project files or `evidence.json` changed after the trial |
|
|
247
|
+
|
|
248
|
+
</details>
|
|
249
|
+
|
|
250
|
+
`regrade` writes `records-regraded.jsonl` and `summary-regraded.json`. Copy
|
|
251
|
+
retained trial files before you inspect them with tools that write.
|
|
252
|
+
|
|
253
|
+
## Documentation
|
|
254
|
+
|
|
255
|
+
- [Configuration](https://github.com/otaviobarros/seatrial/blob/main/docs/configuration.md): `seatrial.toml` keys and defaults.
|
|
256
|
+
- [Writing cases](https://github.com/otaviobarros/seatrial/blob/main/docs/cases.md): `case.toml`, `case.py`, evidence, and grading helpers.
|
|
257
|
+
- [Hosts](https://github.com/otaviobarros/seatrial/blob/main/docs/hosts.md): built-in adapters and how to add one.
|
|
258
|
+
- [Using seatrial from an agent](https://github.com/otaviobarros/seatrial/blob/main/docs/agents.md): JSON output, exit codes, and a safe procedure.
|
|
259
|
+
|
|
260
|
+
## License
|
|
261
|
+
|
|
262
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "seatrial"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Run evaluation cases for agent skills on agent CLIs and compare skill revisions."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
requires-python = ">=3.11"
|
|
13
|
+
dependencies = []
|
|
14
|
+
keywords = ["agents", "skills", "evaluation", "claude", "codex"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Environment :: Console",
|
|
19
|
+
"Operating System :: MacOS",
|
|
20
|
+
"Operating System :: POSIX :: Linux",
|
|
21
|
+
"Topic :: Software Development :: Testing",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/otaviobarros/seatrial"
|
|
26
|
+
Documentation = "https://github.com/otaviobarros/seatrial/tree/main/docs"
|
|
27
|
+
Issues = "https://github.com/otaviobarros/seatrial/issues"
|
|
28
|
+
Changelog = "https://github.com/otaviobarros/seatrial/releases"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
seatrial = "seatrial.cli:entry"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
"seatrial.hosts" = ["samples/**/*"]
|
seatrial-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""seatrial: run evaluation cases for agent skills on agent CLIs and compare revisions.
|
|
2
|
+
|
|
3
|
+
Modules:
|
|
4
|
+
ports the HostAdapter port and the Evidence record every adapter returns
|
|
5
|
+
hosts one adapter per agent CLI, registered by name
|
|
6
|
+
cases folder-based case discovery and selection
|
|
7
|
+
checks shared fixture helpers and grading checks for case authors
|
|
8
|
+
process bounded process launch
|
|
9
|
+
workspace skill mounts, file manifests, and the run state directory
|
|
10
|
+
stats pass@k, pass^k, Fisher and Mann-Whitney tests, verdicts
|
|
11
|
+
report grading, per-arm summaries, and the Markdown report
|
|
12
|
+
runner scheduling and execution of trials
|
|
13
|
+
config seatrial.toml discovery, validation, and the active configuration
|
|
14
|
+
setup `init` and `doctor`
|
|
15
|
+
cli the seatrial command
|
|
16
|
+
interactive the menu-driven terminal mode
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
__version__ = '0.1.0' # x-release-please-version
|