agent-skill-eval 0.4.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. agent_skill_eval-0.5.0/CHANGELOG.md +248 -0
  2. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/PKG-INFO +44 -4
  3. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/README.md +42 -2
  4. agent_skill_eval-0.5.0/TODO.md +69 -0
  5. agent_skill_eval-0.5.0/examples/commit-push-pr/evals/files/sample_change.py +63 -0
  6. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/fix-failing-tests/evals/evals.json +1 -1
  7. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/pyproject.toml +13 -3
  8. agent_skill_eval-0.5.0/src/agent_skill_eval/__init__.py +10 -0
  9. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/cli.py +31 -9
  10. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/graders/__init__.py +130 -48
  11. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/harnesses/__init__.py +32 -0
  12. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/models.py +14 -0
  13. agent_skill_eval-0.5.0/src/agent_skill_eval/openrouter.py +137 -0
  14. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/runner.py +26 -2
  15. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_harnesses.py +66 -0
  16. agent_skill_eval-0.5.0/tests/test_openrouter.py +137 -0
  17. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_runner.py +101 -0
  18. agent_skill_eval-0.4.0/CHANGELOG.md +0 -305
  19. agent_skill_eval-0.4.0/TODO.md +0 -53
  20. agent_skill_eval-0.4.0/examples/commit-push-pr/evals/files/sample_change.py +0 -2
  21. agent_skill_eval-0.4.0/src/agent_skill_eval/__init__.py +0 -8
  22. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  23. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  24. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  25. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.github/actions/setup/action.yml +0 -0
  26. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.github/workflows/ci.yml +0 -0
  27. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.github/workflows/release.yml +0 -0
  28. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/.gitignore +0 -0
  29. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/AGENTS.md +0 -0
  30. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/CODE_OF_CONDUCT.md +0 -0
  31. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/CONTRIBUTING.md +0 -0
  32. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/LICENSE +0 -0
  33. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/Makefile +0 -0
  34. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/config/.env.example +0 -0
  35. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/config/baseline.env +0 -0
  36. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/commit-push-pr/evals/evals.json +0 -0
  37. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/fix-failing-tests/evals/files/calculator.py +0 -0
  38. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/fix-failing-tests/evals/files/conftest.py +0 -0
  39. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/fix-failing-tests/evals/files/test_calculator.py +0 -0
  40. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/review-diff/evals/evals.json +0 -0
  41. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/review-diff/evals/files/changes.diff +0 -0
  42. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/review-diff/evals/files/orders.py +0 -0
  43. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/validate-config/evals/evals.json +0 -0
  44. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/validate-config/evals/files/app_config.json +0 -0
  45. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/write-release-notes/evals/evals.json +0 -0
  46. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/examples/write-release-notes/evals/files/commits.txt +0 -0
  47. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/schemas/evals.schema.json +0 -0
  48. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/scripts/record-demo.sh +0 -0
  49. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/commit-push-pr/SKILL.md +0 -0
  50. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/fix-failing-tests/SKILL.md +0 -0
  51. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/review-diff/SKILL.md +0 -0
  52. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/validate-config/SKILL.md +0 -0
  53. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/validate-config/references/config-rules.md +0 -0
  54. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/validate-config/scripts/validate_config.py +0 -0
  55. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/skills/write-release-notes/SKILL.md +0 -0
  56. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/__main__.py +0 -0
  57. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/git_state.py +0 -0
  58. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/skills.py +0 -0
  59. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/src/agent_skill_eval/workspace.py +0 -0
  60. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/__init__.py +0 -0
  61. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/conftest.py +0 -0
  62. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/fixtures/evals/format-json.json +0 -0
  63. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/fixtures/skills/format-json/SKILL.md +0 -0
  64. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_cli_smoke.py +0 -0
  65. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_e2e.py +0 -0
  66. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_fuzz_grader.py +0 -0
  67. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_git_state.py +0 -0
  68. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_graders.py +0 -0
  69. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_launch_features.py +0 -0
  70. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_llm_grader.py +0 -0
  71. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_skill_validation.py +0 -0
  72. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_skills.py +0 -0
  73. {agent_skill_eval-0.4.0 → agent_skill_eval-0.5.0}/tests/test_workspace.py +0 -0
@@ -0,0 +1,248 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+ Entries above the insertion flag are generated by python-semantic-release
5
+ from conventional commits; entries below it are curated by hand.
6
+
7
+ <!-- version list -->
8
+
9
+ ## v0.5.0 (2026-06-11)
10
+
11
+ ### Bug Fixes
12
+
13
+ - **harness**: Harden OpenRouter pricing fetch (review feedback)
14
+ ([`d40feba`](https://github.com/tardigrde/agent-skill-eval/commit/d40febaa75807aed7f09132ef999d972983ddde7))
15
+
16
+ Guard the pricing cache with a lock (eval threads run concurrently and could fire redundant
17
+ fetches), send a real User-Agent (CDN-fronted APIs commonly 403 urllib's default), and type-check
18
+ the response shape before indexing into it.
19
+
20
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
21
+
22
+ - **runner**: Only record eval-created open PRs in cleanup manifest
23
+ ([`d588d1c`](https://github.com/tardigrde/agent-skill-eval/commit/d588d1cb65c6ada6c2b8d0e7b99987045875261e))
24
+
25
+ The PR snapshot is taken with `gh pr list --state all`, so _build_cleanup_entry could record PRs
26
+ that were merged or closed during the run, and even PRs someone else opened on the source repo
27
+ mid-run — cleanup would then try to close a stranger's PR with --delete-branch.
28
+
29
+ Record a new PR number only when the PR is still OPEN and its head branch is one this run pushed.
30
+ Snapshots from before state/headRefName were captured keep the old behavior.
31
+
32
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
33
+
34
+ ### Chores
35
+
36
+ - **examples**: Expand sample_change.py into a realistic fixture
37
+ ([`fdb3a4a`](https://github.com/tardigrde/agent-skill-eval/commit/fdb3a4a1f7a2f411470af7f8fef4f1d6bf99f542))
38
+
39
+ The two-line print fixture gave agents nothing plausible to commit. The new fixture is a small
40
+ user-directory module whose contents match what the eval prompts claim about it: a login fix
41
+ (email normalization in authenticate) and a new search feature (search_users).
42
+
43
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
44
+
45
+ ### Documentation
46
+
47
+ - Real author metadata, python -m usage, cost reporting, agent comparison
48
+ ([`03abf2a`](https://github.com/tardigrde/agent-skill-eval/commit/03abf2a21e56c96f9064c2e7345a4fdcc8524f31))
49
+
50
+ - pyproject authors: replace the placeholder with the real maintainer - README: document running the
51
+ CLI as `python -m agent_skill_eval` - README: fix the stale codex invocation row (--full-auto was
52
+ replaced by --sandbox workspace-write --skip-git-repo-check) - README: document cost_usd semantics
53
+ and the OpenRouter reconciliation (cost_usd_source / cost_usd_cli in timing.json) - README: add a
54
+ multi-agent comparison walkthrough showing how to read per-agent with/without-skill deltas from
55
+ the report and benchmark.json
56
+
57
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
58
+
59
+ - **todo**: Mark TODO batch done — cost reconcile, tests, docs, cleanup
60
+ ([`2143eb0`](https://github.com/tardigrde/agent-skill-eval/commit/2143eb007eadda282df36deeec43f51c4f6eb933))
61
+
62
+ Local-only cleanup also performed (not visible in the diff): stale merged local branches deleted,
63
+ both .claude/worktrees copies of the old skill_eval module tree removed, stale untracked
64
+ HANDOFF.md deleted.
65
+
66
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
67
+
68
+ ### Features
69
+
70
+ - **harness**: Reconcile claude-code cost against OpenRouter pricing
71
+ ([`5246951`](https://github.com/tardigrde/agent-skill-eval/commit/524695144c8b1260da1b6bde28c4f66744822567))
72
+
73
+ The claude CLI prices runs at Anthropic list prices regardless of the endpoint it talks to, so
74
+ cost_usd was wrong whenever the harness routed claude-code through OpenRouter. The generation API
75
+ would give exact billed amounts but needs per-request ids the CLI does not expose; instead,
76
+ recompute from the run's token counts and OpenRouter's published per-model rates (fetched once per
77
+ process from the public models endpoint).
78
+
79
+ timing.json now records cost_usd_source ("cli", "openrouter-pricing", or "cli-unreconciled" when the
80
+ model cannot be mapped to an OpenRouter slug) and keeps the CLI's original estimate in
81
+ cost_usd_cli. Cache creation tokens are now captured too so the reconciled cost includes
82
+ cache-write charges.
83
+
84
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
85
+
86
+ ### Testing
87
+
88
+ - **harness**: Pin claude-code, opencode, and codex CLI invocations
89
+ ([`c224898`](https://github.com/tardigrde/agent-skill-eval/commit/c2248982ee72bd92a18d412f3157e522ca9b586b))
90
+
91
+ build_command was only pinned for codex (and only the trust-check flag). Pin the full argv for all
92
+ three real harnesses so a flag regression fails loudly, and cover cache_creation_input_tokens
93
+ parsing.
94
+
95
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
96
+
97
+
98
+ ## v0.4.1 (2026-06-11)
99
+
100
+ ### Bug Fixes
101
+
102
+ - Todo batch — grader honesty, cost reporting, curated changelog
103
+ ([#11](https://github.com/tardigrde/agent-skill-eval/pull/11),
104
+ [`33ab02c`](https://github.com/tardigrde/agent-skill-eval/commit/33ab02c6c5ac3459d1987ba46ae3cc85038f8685))
105
+
106
+ * fix: grader honesty — skip-with-warning, workspace fallback warning, behavior-based negative
107
+ control
108
+
109
+ - LLMGrader: grading errors and missing per-assertion results are now skipped (not failed) and
110
+ warned about on stderr, so grader failures never masquerade as agent failures. - Workspace
111
+ resolution for file-based checks is shared between the deterministic and LLM graders and warns
112
+ (once per run dir) when it falls back to guessing from the output directory layout. -
113
+ fix-failing-tests negative-control: reword the grader-run-dependent 'output contains the source
114
+ code' assertion to the behavior that matters (file read and displayed/described).
115
+
116
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
117
+
118
+ * feat: cost column in report, validate flags LLM-rubric assertions, curated changelog
119
+
120
+ - benchmark.json: per-config cost_usd mean/stddev and per-agent cost deltas; report gains a Cost
121
+ (USD) column (table and markdown). - validate: classify_assertion() reports which assertions match
122
+ no deterministic pattern and will be graded by the LLM rubric, instead of that happening silently
123
+ at run time. - CHANGELOG.md: restore curated pre-0.4 entries (PSR-style headings);
124
+ semantic-release now runs in insertion-only update mode with vendored templates that render commit
125
+ summary lines only — no more commit bodies or Co-Authored-By trailers leaking into the changelog.
126
+ - Version is single-sourced from pyproject.toml (PSR version_variables removed; __init__ falls
127
+ back to 0.0.0 when not installed). - TODO.md: tick everything done in this batch and items
128
+ verified already done on main.
129
+
130
+ * fix: normalize assertion text when matching LLM grader responses
131
+
132
+ LLMs echo assertions with minor drift (casing, trailing periods, whitespace); exact-match made those
133
+ falsely count as missing results. Addresses Gemini review on PR #11.
134
+
135
+ * chore: remove vendored PSR templates, use defaults
136
+
137
+ ---------
138
+
139
+ Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
140
+
141
+ ### Documentation
142
+
143
+ - Update TODO — v0.4.0 state, subagent evals next, agent working notes
144
+ ([`4b5b76f`](https://github.com/tardigrde/agent-skill-eval/commit/4b5b76faed27678bf81a23cb9b2478e90c590c82))
145
+
146
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
147
+
148
+
149
+ ## v0.4.0 (2026-06-11)
150
+
151
+ ### Changed
152
+ - Converged all names on `agent-skill-eval`: GitHub repo (was `skill-evals`), CLI entry point (was `skill-eval`), and Python module (was `skill_eval`) now all match the PyPI package name
153
+ - Added short CLI alias `ase` (same app as `agent-skill-eval`)
154
+ - Env var prefix renamed `SKILL_EVAL_*` -> `ASE_*` (`ASE_AGENT_TIMEOUT`, `ASE_AGENT_RETRIES`, `ASE_LIVE`)
155
+
156
+ ### Added
157
+ - Codex harness fixes (`--sandbox workspace-write`, `--skip-git-repo-check`), `cost_usd` in `timing.json`, two-tier e2e tests, Makefile eval/test targets (merged via PR #9, previously unreleased)
158
+
159
+ ## v0.3.0 (2026-06-11)
160
+
161
+ ### Added
162
+ - `validate-config` example skill + eval suite: exercises bundled resources (agent runs skill's `scripts/` and reads its `references/`); graded via command-ran + file + content checks; fully offline
163
+ - `review-diff` example skill + eval suite: read-only analysis (planted bugs found, documented decoy not flagged, nothing modified); graded via chat-output-only content + LLM rubric; fully offline
164
+
165
+ ## v0.2.1 (2026-06-11)
166
+
167
+ ### Changed
168
+ - Renamed package from `skill-eval` to `agent-skill-eval` (PyPI name collision)
169
+
170
+ ## v0.2.0 (2026-06-10)
171
+
172
+ ### Added
173
+ - `write-release-notes` example skill + eval suite: exercises LLM rubric grading (grouping, breaking-change prominence, anti-fabrication) instead of git-state checks
174
+ - `--runs N` repeats every (eval, agent, config); `benchmark.json` now reports `full_pass_rate`, `pass_at_k`, and `k`
175
+ - `--agent-model agent=model` (repeatable) and `--harness-base-url` to pin models and endpoints per agent CLI
176
+ - `--timeout` / `--retries` flags; harnesses retry on timeout or non-zero exit with backoff and record `exit_code`, `timed_out`, `retries` in `timing.json`
177
+ - `agent-skill-eval validate`: schema + fixture-existence + duplicate-id checks for evals.json
178
+ - `agent-skill-eval list`: discover eval suites and skills under a directory
179
+ - `agent-skill-eval compare`: side-by-side pass rates of two iterations
180
+ - `report --format markdown` and `report --show-evidence` (failed/skipped assertion evidence)
181
+ - JSON Schema for the eval suite format at `schemas/evals.schema.json` (with a sync test)
182
+ - SKILL.md frontmatter validation warning on `run` (missing name/description, name mismatch)
183
+ - Richer `init` scaffold: SKILL.md frontmatter template + negative-control eval case
184
+ - PyPI trusted-publishing workflow on version tags (`.github/workflows/publish.yml`)
185
+ - Demo recording script (`scripts/record-demo.sh`)
186
+
187
+ ### Changed
188
+ - Assertions that cannot be graded (no API key, LLM grader error) are now **skipped** and excluded from the pass rate instead of counted as failures; `grading.json` gains `method` and `skipped` fields
189
+ - `run` warns upfront when no grader API key is set; `grade` warns that LLM verdicts may flip on re-grade
190
+ - README restructured around the full-harness pitch, with a documented table of recognized deterministic assertion patterns
191
+
192
+ ### Fixed
193
+ - Multi-agent runs no longer report a misleading `Delta: 0.0%`: `benchmark.json` now has per-agent `deltas`; the legacy `delta` field is only set for single-agent runs (otherwise `null`)
194
+ - Removed dead `cost_usd` handling in the Claude Code output parser
195
+
196
+ ## v0.1.0-alpha.3 (2026-06-07)
197
+
198
+ ### Added
199
+ - Shared test fixtures in `conftest.py`
200
+ - `test_llm_grader.py`: 18 tests covering client, prompts, JSON parsing, errors
201
+ - `test_fuzz_grader.py` rewrite: real handler tests instead of mocked dispatch
202
+ - `test_compute_benchmark` with xfail for known multi-agent delta=0 bug
203
+ - EvalCase validation tests (valid, negative control, contradiction)
204
+ - `SkillInstaller.uninstall` tests, `WorkspaceManager._clone_repo` test
205
+ - `github_repo_slug` tests for URLs without `.git` suffix
206
+
207
+ ### Changed
208
+ - CI uses setup-python's built-in cache instead of separate `actions/cache`
209
+
210
+ ### Fixed
211
+ - EvalCase contradiction validator (`force_skill_invocation` + `should_trigger`)
212
+ - LLMGrader now fails on missing assertions instead of silently dropping them
213
+
214
+ ## v0.1.0-alpha.2 (2026-06-02)
215
+
216
+ ### Added
217
+ - CI workflow with Python 3.11/3.12/3.13 matrix, caching, coverage, and smoke test
218
+ - CLI smoke test (`test_cli_smoke.py`) for full run -> report -> init cycle
219
+
220
+ ### Changed
221
+ - Tightened evals with state-delta grading and scoped cleanup
222
+ - Improved `commit-push-pr` skill to 100% pass rate
223
+
224
+ ### Fixed
225
+ - `github_repo_slug` function to parse GitHub URLs correctly
226
+ - Handoff review fixes and PR grading tightening
227
+
228
+ ## v0.1.0-alpha.1 (2026-05-27)
229
+
230
+ ### Added
231
+ - Initial release: skill evaluation framework for OpenCode, Claude Code, and Codex
232
+ - `run`, `report`, `grade`, `cleanup`, and `init` CLI commands
233
+ - Multi-agent support with baseline (with-skill / without-skill) comparison
234
+ - State-delta grading using pre/post git snapshots (`pre_state.json` / `post_state.json`)
235
+ - Deterministic assertions: branch, commit, push, PR, file existence, content matching
236
+ - LLM rubric grading via configurable model (`--grader-model`)
237
+ - Negative-control support (`should_trigger: false`) to catch accidental skill triggering
238
+ - Scoped cleanup via `cleanup.json` manifest (never closes unrelated PRs)
239
+ - `commit-push-pr` example skill with 5 eval cases
240
+ - Lazy-initialized OpenAI client with API key validation
241
+ - OpenRouter grader support via `OPENROUTER_API_KEY`
242
+ - Workspace isolation with fresh git repos per eval run
243
+ - Token/timing extraction from each agent's native output format
244
+ - `python -m agent_skill_eval` entry point
245
+
246
+ ### Fixed
247
+ - OpenRouter grader model resolution and opencode parser output handling
248
+ - Workspace-aware grading and dotenv loading
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: agent-skill-eval
3
- Version: 0.4.0
3
+ Version: 0.5.0
4
4
  Summary: Framework for evaluating agent skills across OpenCode, Claude Code, and Codex
5
5
  Project-URL: Homepage, https://github.com/tardigrde/agent-skill-eval
6
6
  Project-URL: Repository, https://github.com/tardigrde/agent-skill-eval
7
- Author: agent-skill-eval contributors
7
+ Author-email: Levente Csoke <leventec3@gmail.com>
8
8
  License-Expression: MIT
9
9
  License-File: LICENSE
10
10
  Keywords: agent,benchmark,evaluation,llm,skills
@@ -56,7 +56,7 @@ Because the eval goes through the full harness — system prompt, skill discover
56
56
  pip install agent-skill-eval
57
57
  ```
58
58
 
59
- This installs two identical commands: `agent-skill-eval` and the short alias `ase`.
59
+ This installs two identical commands: `agent-skill-eval` and the short alias `ase`. The CLI is also runnable as a module — `python -m agent_skill_eval run ...` — which is handy when the scripts directory isn't on your `PATH` or you want to pin the interpreter (e.g. `uv run python -m agent_skill_eval`).
60
60
 
61
61
  > **Coming soon:** subagent evals — evaluate custom subagent definitions the same way as skills, across the same harnesses.
62
62
 
@@ -311,6 +311,37 @@ To debug a failure: find `"passed": false` entries, read `evidence`, compare `pr
311
311
 
312
312
  1. `agent-skill-eval run ...` → 2. `agent-skill-eval report --show-evidence` → 3. edit SKILL.md → 4. `agent-skill-eval run --iteration 2 ...` → 5. `agent-skill-eval compare --workspace ... 1 2`
313
313
 
314
+ ### Comparing agents: reading with/without-skill deltas
315
+
316
+ Run several agents in one invocation and every agent gets its own baseline comparison:
317
+
318
+ ```bash
319
+ agent-skill-eval run \
320
+ --skill ./skills/fix-failing-tests \
321
+ --evals ./examples/fix-failing-tests/evals/evals.json \
322
+ --agent claude-code --agent-model claude-code=claude-haiku-4-5-20251001 \
323
+ --agent opencode --agent-model opencode=deepseek/deepseek-v4-flash:free \
324
+ --runs 3
325
+
326
+ agent-skill-eval report --workspace ./eval-workspace/fix-failing-tests-workspace --format markdown
327
+ ```
328
+
329
+ The report shows one row per (agent, config) and a delta per agent (numbers below are illustrative):
330
+
331
+ | Configuration | Pass Rate | Full Pass / pass@k | Time (s) | Tokens | Cost (USD) |
332
+ | --- | --- | --- | --- | --- | --- |
333
+ | claude-code_with_skill | 91.7% +/- 14.4% | 67% (k=3) | 41.2 +/- 8.0 | 1840 +/- 312 | 0.0042 +/- 0.0011 |
334
+ | claude-code_without_skill | 58.3% +/- 14.4% | 33% (k=3) | 52.7 +/- 12.1 | 2410 +/- 405 | 0.0058 +/- 0.0019 |
335
+ | opencode_with_skill | 83.3% +/- 0.0% | 67% (k=3) | 64.9 +/- 9.3 | 2980 +/- 220 | 0.0000 +/- 0.0000 |
336
+ | opencode_without_skill | 50.0% +/- 25.0% | 33% (k=3) | 71.5 +/- 15.8 | 3340 +/- 510 | 0.0000 +/- 0.0000 |
337
+
338
+ **Delta (with_skill - without_skill):**
339
+
340
+ - `claude-code`: pass rate +33.3%, time -11.5s, tokens -570, cost -0.0016 USD
341
+ - `opencode`: pass rate +33.3%, time -6.6s, tokens -360, cost +0.0000 USD
342
+
343
+ How to read it: the **delta rows** are the skill's measured value per agent — here the skill lifts pass rate by ~33 points on both agents *and* saves time/tokens, the strongest possible signal. A positive pass-rate delta with a large token increase means the skill works but is verbose; a near-zero delta means that agent doesn't benefit (check `report --show-evidence` to see whether it never triggered the skill). The same numbers are machine-readable in `benchmark.json`: per-config stats under `run_summary`, per-agent deltas under `deltas`, keyed by agent name.
344
+
314
345
  ## Environment variables
315
346
 
316
347
  - `OPENROUTER_API_KEY` / `OPENAI_API_KEY`: API key for LLM rubric grading
@@ -324,7 +355,16 @@ To debug a failure: find `"passed": false` entries, read `evidence`, compare `pr
324
355
  | --- | --- | --- |
325
356
  | OpenCode | `opencode run --format json --dangerously-skip-permissions` | `.opencode/skills/<name>/SKILL.md` |
326
357
  | Claude Code | `claude -p --output-format json --dangerously-skip-permissions` | `.claude/skills/<name>/SKILL.md` |
327
- | Codex | `codex exec --json --full-auto` | `.codex/skills/<name>/SKILL.md` |
358
+ | Codex | `codex exec --json --sandbox workspace-write --skip-git-repo-check` | `.codex/skills/<name>/SKILL.md` |
359
+
360
+ ### Cost reporting
361
+
362
+ `timing.json` records `cost_usd` per run, taken from the agent CLI itself (claude: `total_cost_usd`, opencode: per-step `cost`; codex reports nothing). One sharp edge: the claude CLI prices runs at **Anthropic list prices** regardless of the endpoint it talks to. When you route claude-code through OpenRouter (`--harness-base-url` or `ANTHROPIC_BASE_URL`), agent-skill-eval reconciles the cost by recomputing it from the run's token counts and OpenRouter's published per-model rates. `timing.json` then shows:
363
+
364
+ - `cost_usd_source`: `"cli"` (the CLI's own number), `"openrouter-pricing"` (reconciled), or `"cli-unreconciled"` (reconciliation failed — e.g. a short model alias like `haiku` that can't be mapped to an OpenRouter slug — so `cost_usd` is the CLI's list-price estimate and actual billing differs)
365
+ - `cost_usd_cli`: the CLI's original estimate, kept alongside the reconciled value
366
+
367
+ Pin full model IDs (`claude-haiku-4-5-20251001`, not `haiku`) to keep reconciliation working.
328
368
 
329
369
  ## Development
330
370
 
@@ -25,7 +25,7 @@ Because the eval goes through the full harness — system prompt, skill discover
25
25
  pip install agent-skill-eval
26
26
  ```
27
27
 
28
- This installs two identical commands: `agent-skill-eval` and the short alias `ase`.
28
+ This installs two identical commands: `agent-skill-eval` and the short alias `ase`. The CLI is also runnable as a module — `python -m agent_skill_eval run ...` — which is handy when the scripts directory isn't on your `PATH` or you want to pin the interpreter (e.g. `uv run python -m agent_skill_eval`).
29
29
 
30
30
  > **Coming soon:** subagent evals — evaluate custom subagent definitions the same way as skills, across the same harnesses.
31
31
 
@@ -280,6 +280,37 @@ To debug a failure: find `"passed": false` entries, read `evidence`, compare `pr
280
280
 
281
281
  1. `agent-skill-eval run ...` → 2. `agent-skill-eval report --show-evidence` → 3. edit SKILL.md → 4. `agent-skill-eval run --iteration 2 ...` → 5. `agent-skill-eval compare --workspace ... 1 2`
282
282
 
283
+ ### Comparing agents: reading with/without-skill deltas
284
+
285
+ Run several agents in one invocation and every agent gets its own baseline comparison:
286
+
287
+ ```bash
288
+ agent-skill-eval run \
289
+ --skill ./skills/fix-failing-tests \
290
+ --evals ./examples/fix-failing-tests/evals/evals.json \
291
+ --agent claude-code --agent-model claude-code=claude-haiku-4-5-20251001 \
292
+ --agent opencode --agent-model opencode=deepseek/deepseek-v4-flash:free \
293
+ --runs 3
294
+
295
+ agent-skill-eval report --workspace ./eval-workspace/fix-failing-tests-workspace --format markdown
296
+ ```
297
+
298
+ The report shows one row per (agent, config) and a delta per agent (numbers below are illustrative):
299
+
300
+ | Configuration | Pass Rate | Full Pass / pass@k | Time (s) | Tokens | Cost (USD) |
301
+ | --- | --- | --- | --- | --- | --- |
302
+ | claude-code_with_skill | 91.7% +/- 14.4% | 67% (k=3) | 41.2 +/- 8.0 | 1840 +/- 312 | 0.0042 +/- 0.0011 |
303
+ | claude-code_without_skill | 58.3% +/- 14.4% | 33% (k=3) | 52.7 +/- 12.1 | 2410 +/- 405 | 0.0058 +/- 0.0019 |
304
+ | opencode_with_skill | 83.3% +/- 0.0% | 67% (k=3) | 64.9 +/- 9.3 | 2980 +/- 220 | 0.0000 +/- 0.0000 |
305
+ | opencode_without_skill | 50.0% +/- 25.0% | 33% (k=3) | 71.5 +/- 15.8 | 3340 +/- 510 | 0.0000 +/- 0.0000 |
306
+
307
+ **Delta (with_skill - without_skill):**
308
+
309
+ - `claude-code`: pass rate +33.3%, time -11.5s, tokens -570, cost -0.0016 USD
310
+ - `opencode`: pass rate +33.3%, time -6.6s, tokens -360, cost +0.0000 USD
311
+
312
+ How to read it: the **delta rows** are the skill's measured value per agent — here the skill lifts pass rate by ~33 points on both agents *and* saves time/tokens, the strongest possible signal. A positive pass-rate delta with a large token increase means the skill works but is verbose; a near-zero delta means that agent doesn't benefit (check `report --show-evidence` to see whether it never triggered the skill). The same numbers are machine-readable in `benchmark.json`: per-config stats under `run_summary`, per-agent deltas under `deltas`, keyed by agent name.
313
+
283
314
  ## Environment variables
284
315
 
285
316
  - `OPENROUTER_API_KEY` / `OPENAI_API_KEY`: API key for LLM rubric grading
@@ -293,7 +324,16 @@ To debug a failure: find `"passed": false` entries, read `evidence`, compare `pr
293
324
  | --- | --- | --- |
294
325
  | OpenCode | `opencode run --format json --dangerously-skip-permissions` | `.opencode/skills/<name>/SKILL.md` |
295
326
  | Claude Code | `claude -p --output-format json --dangerously-skip-permissions` | `.claude/skills/<name>/SKILL.md` |
296
- | Codex | `codex exec --json --full-auto` | `.codex/skills/<name>/SKILL.md` |
327
+ | Codex | `codex exec --json --sandbox workspace-write --skip-git-repo-check` | `.codex/skills/<name>/SKILL.md` |
328
+
329
+ ### Cost reporting
330
+
331
+ `timing.json` records `cost_usd` per run, taken from the agent CLI itself (claude: `total_cost_usd`, opencode: per-step `cost`; codex reports nothing). One sharp edge: the claude CLI prices runs at **Anthropic list prices** regardless of the endpoint it talks to. When you route claude-code through OpenRouter (`--harness-base-url` or `ANTHROPIC_BASE_URL`), agent-skill-eval reconciles the cost by recomputing it from the run's token counts and OpenRouter's published per-model rates. `timing.json` then shows:
332
+
333
+ - `cost_usd_source`: `"cli"` (the CLI's own number), `"openrouter-pricing"` (reconciled), or `"cli-unreconciled"` (reconciliation failed — e.g. a short model alias like `haiku` that can't be mapped to an OpenRouter slug — so `cost_usd` is the CLI's list-price estimate and actual billing differs)
334
+ - `cost_usd_cli`: the CLI's original estimate, kept alongside the reconciled value
335
+
336
+ Pin full model IDs (`claude-haiku-4-5-20251001`, not `haiku`) to keep reconciliation working.
297
337
 
298
338
  ## Development
299
339
 
@@ -0,0 +1,69 @@
1
+ # TODO
2
+
3
+ Audit of `main` originally done at 0.1.0; pruned and updated 2026-06-11 (v0.4.0, post PR #9/#10).
4
+ Done items are kept briefly for history; anything obsolete or low-value was deleted.
5
+
6
+ ## How to work this list (for the implementing agent)
7
+
8
+ - Do NOT bundle everything into one PR. Group into separate PRs by section (one for Bugs, one for Docs, one per Feature) with conventional-commit messages — semantic-release on main derives versions and the changelog from them.
9
+ - `make test` (includes free e2e tier) + `make lint` must pass per PR; run `make test-live` once before the final merge of the batch (costs cents, needs `.env` OpenRouter key + opencode/codex auth).
10
+ - Items marked *(judgment)* are optional — skip unless already touching that code.
11
+
12
+ ## Features (next up)
13
+
14
+ - [ ] **Subagent evals** — evaluate custom subagent definitions (e.g. `~/.claude/agents/*.md`) the same way as skills: generalize `SkillInstaller` to an artifact installer with a type field; harnesses/runner/graders/metrics stay as-is. Add one example subagent + eval suite. README already announces it as coming soon.
15
+
16
+ ## Done
17
+
18
+ - [x] **Names converged on `agent-skill-eval`** (PR #10): repo, CLI, module, PyPI all match; `ase` short alias; env prefix `ASE_*`. Repo renames redirect from `skill-evals`/`agent-evals`.
19
+ - [x] **Release fully automated** (gcpath pattern): push to main → python-semantic-release bumps version, tags, GitHub release, publishes to PyPI via trusted publishing. v0.4.0 shipped this way end-to-end. Trusted publisher re-created for renamed repo (workflow `release.yml`, env `pypi`); old one deleted.
20
+
21
+ - [x] **Grader fixed + tested** — `deepseek/deepseek-v4-flash` via OpenRouter, `tests/test_llm_grader.py` covers prompt shape, JSON parsing, malformed responses, missing key.
22
+ - [x] **Fake harness + CI smoke test** — `--agent fake`, no network, wired into CI.
23
+ - [x] **Offline example skill** — `skills/fix-failing-tests` + 4-case eval suite (`examples/fix-failing-tests`).
24
+ - [x] **All 4 harnesses verified live** (2026-06-11): fake, claude-code (haiku via OpenRouter), opencode (Zen free), codex (`gpt-5.4-mini`, ChatGPT plan). Codex needed `--sandbox workspace-write` + `--skip-git-repo-check` (eval workspaces are fresh git dirs).
25
+ - [x] **Cost/time/accuracy all machine-readable per run** — `timing.json` gained `cost_usd` (claude: `total_cost_usd`, opencode: per-step `cost`); `duration_ms`, tokens, and `grading.json` pass rates already existed.
26
+ - [x] **Two-tier e2e tests** (`tests/test_e2e.py`) — free tier (fake harness, real CLI subprocess, artifact + report assertions) runs in every `make test`; live tier (`make test-live`, `live` marker) runs all real agents + LLM grader and enforces ≥0.7 mean pass rate, zero skipped grades.
27
+ - [x] **Makefile** — `cheap-eval`, `full-eval` (adds codex), `fake-eval`, `test`, `test-e2e`, `test-live`, `lint`; `AGENTS` + per-agent model vars; `config/baseline.env` pins models.
28
+ - [x] `--agent-model` and `--harness-base-url` CLI flags wired.
29
+ - [x] Harness retry/backoff with pristine-workspace guard; timeout + exit code recorded in `timing.json`.
30
+ - [x] Per-agent benchmark deltas for multi-agent runs (no more silent 0.0).
31
+ - [x] SKILL.md frontmatter problems warned at `run` time.
32
+ - [x] LICENSE, CHANGELOG.md, CONTRIBUTING.md, issue templates, PR template, `agent-skill-eval list`, conftest fixtures, fuzz tests for assertion matcher, CI matrix + coverage.
33
+
34
+ ## Bugs / sharp edges
35
+
36
+ - [x] `negative-control` eval: reworded "output contains the source code" assertion to behavior ("read and displayed or described its contents").
37
+ - [x] `grade` re-running the LLM grader fresh is surfaced in CLI output and README (was already done on main; verified).
38
+ - [x] `LLMGrader.grade`: grading errors and missing per-assertion results are now skipped (not failed) with a console warning.
39
+ - [x] `DeterministicGrader._resolve_workspace`: workspace fallback now warns (once per run dir) instead of silently guessing; shared with the LLM grader.
40
+ - [x] Assertion patterns documented in README (was already done on main); `validate` now also reports which assertions will fall through to the LLM rubric (`classify_assertion`).
41
+ - [x] claude-code's `cost_usd` reconciled when routed through OpenRouter: recomputed from token counts × OpenRouter's published per-model rates (the generation API would be exact but needs per-request ids the claude CLI doesn't expose). `timing.json` records `cost_usd_source` and keeps the CLI estimate in `cost_usd_cli`; documented in README.
42
+ - [x] CHANGELOG.md: curated pre-0.4 entries restored (PSR-style headings), PSR configured with `mode = "update"` + insertion flag, vendored templates in `templates/` render summary lines only (no commit bodies / Co-Authored-By trailers).
43
+
44
+ ## Test coverage gaps
45
+
46
+ - [x] `build_command` pinned (full argv) for claude-code, opencode, and codex.
47
+ - [x] `runner._build_cleanup_entry`: merged/closed/external PRs now excluded from the cleanup manifest (only OPEN PRs whose head branch the run pushed) and covered by tests.
48
+ - [x] Report shows per-assertion evidence via `report --show-evidence` (was already done on main; verified).
49
+
50
+ ## Docs
51
+
52
+ - [x] Fill out `pyproject.toml` `authors` with a real author (rest of metadata was already filled on main).
53
+ - [x] Version single-sourced in `pyproject.toml`: PSR `version_variables` removed, `__init__.py` falls back to `0.0.0` when not installed.
54
+ - [x] Document `python -m agent_skill_eval` in README (`examples/` vs `skills/` and `grading.json` debugging were already documented on main).
55
+ - [x] `--version` wired to package version (was already done on main; verified).
56
+
57
+ ## Improvements
58
+
59
+ - [x] Cost column in `agent-skill-eval report`: `benchmark.json` gains per-config `cost_usd` mean/stddev and per-agent cost deltas; report shows a Cost (USD) column.
60
+ - [x] `init` scaffold: SKILL.md frontmatter template + negative-control example (was already done on main; verified).
61
+ - [x] Multi-agent comparison example in README ("Comparing agents") showing how to read with/without-skill deltas from the report and `benchmark.json`.
62
+ - [x] `examples/commit-push-pr/evals/files/sample_change.py` expanded into a realistic fixture matching the eval prompts (login fix + search feature).
63
+ - [ ] *(judgment)* Split `graders/__init__.py`, `cli.py`, `runner.py` once they next need surgery — one-time split makes everything after cheaper.
64
+
65
+ ## Cleanup
66
+
67
+ - [x] Stale local branches deleted (all confirmed merged via PRs #2–#8).
68
+ - [x] `HANDOFF.md` deleted; both leftover `.claude/worktrees/` copies of the old `skill_eval` module tree removed (worktrees + branches).
69
+ - [ ] *(judgment)* `SkillInstaller.uninstall` never called by the runner (safe today via `rmtree`, but asymmetric).
@@ -0,0 +1,63 @@
1
+ """User directory service: authentication and search.
2
+
3
+ Fixture for the commit-push-pr eval suite. The eval prompts refer to two
4
+ changes in this file: the login fix (authenticate now normalizes the email
5
+ before lookup, so mixed-case logins stop failing) and the new search
6
+ feature (search_users).
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import hmac
12
+ from dataclasses import dataclass
13
+ from hashlib import sha256
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class User:
18
+ email: str
19
+ name: str
20
+ password_hash: str
21
+ active: bool = True
22
+
23
+
24
+ def _hash_password(password: str) -> str:
25
+ return sha256(password.encode("utf-8")).hexdigest()
26
+
27
+
28
+ USERS: dict[str, User] = {
29
+ "ada@example.com": User("ada@example.com", "Ada Lovelace", _hash_password("analytical-engine")),
30
+ "grace@example.com": User("grace@example.com", "Grace Hopper", _hash_password("nanoseconds")),
31
+ "alan@example.com": User("alan@example.com", "Alan Turing", _hash_password("enigma"), active=False),
32
+ }
33
+
34
+
35
+ def authenticate(email: str, password: str) -> User | None:
36
+ """Return the matching active user, or None.
37
+
38
+ Login fix: emails are normalized (trimmed, lowercased) before lookup, so
39
+ "Ada@Example.com " authenticates the same as "ada@example.com".
40
+ """
41
+ user = USERS.get(email.strip().lower())
42
+ if user is None or not user.active:
43
+ return None
44
+ if not hmac.compare_digest(user.password_hash, _hash_password(password)):
45
+ return None
46
+ return user
47
+
48
+
49
+ def search_users(query: str, *, include_inactive: bool = False, limit: int = 10) -> list[User]:
50
+ """New search feature: case-insensitive substring match on name or email.
51
+
52
+ Results are sorted by name and capped at `limit`.
53
+ """
54
+ needle = query.strip().lower()
55
+ if not needle:
56
+ return []
57
+ matches = [
58
+ user
59
+ for user in USERS.values()
60
+ if (include_inactive or user.active) and (needle in user.name.lower() or needle in user.email.lower())
61
+ ]
62
+ matches.sort(key=lambda user: user.name)
63
+ return matches[: max(0, limit)]
@@ -45,7 +45,7 @@
45
45
  "files": ["files/calculator.py"],
46
46
  "should_trigger": false,
47
47
  "assertions": [
48
- "The output contains the source code of calculator.py",
48
+ "The agent read calculator.py and displayed or described its contents",
49
49
  "calculator.py was not modified",
50
50
  "No tests were run"
51
51
  ]
@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "agent-skill-eval"
7
- version = "0.4.0"
7
+ version = "0.5.0"
8
8
  description = "Framework for evaluating agent skills across OpenCode, Claude Code, and Codex"
9
9
  requires-python = ">=3.11"
10
10
  license = "MIT"
11
11
  readme = "README.md"
12
12
  authors = [
13
- { name = "agent-skill-eval contributors" },
13
+ { name = "Levente Csoke", email = "leventec3@gmail.com" },
14
14
  ]
15
15
  keywords = ["agent", "evaluation", "skills", "benchmark", "llm"]
16
16
  classifiers = [
@@ -48,6 +48,8 @@ agent-skill-eval = "agent_skill_eval.cli:app"
48
48
  ase = "agent_skill_eval.cli:app"
49
49
 
50
50
 
51
+
52
+
51
53
  [tool.hatch.build.targets.wheel]
52
54
  packages = ["src/agent_skill_eval"]
53
55
 
@@ -69,7 +71,6 @@ select = ["E", "F", "I"]
69
71
  major_on_zero = false
70
72
  allow_zero_version = true
71
73
  version_toml = ["pyproject.toml:project.version"]
72
- version_variables = ["src/agent_skill_eval/__init__.py:__version__"]
73
74
  branch = "main"
74
75
  upload_to_pypi = false
75
76
  upload_to_release = true
@@ -77,6 +78,15 @@ build_command = "uv build"
77
78
  commit_message = "chore(release): {version}"
78
79
 
79
80
  [tool.semantic_release.changelog]
81
+ # Insertion-only updates: new releases are inserted at the flag, the curated
82
+ # history below it is never regenerated. Templates in templates/ are the PSR
83
+ # defaults minus full commit bodies (no Co-Authored-By trailers in the log).
84
+ mode = "update"
85
+ insertion_flag = "<!-- version list -->"
86
+ template_dir = "templates"
87
+ exclude_commit_patterns = ['''chore\(release\):.*''']
88
+
89
+ [tool.semantic_release.changelog.default_templates]
80
90
  changelog_file = "CHANGELOG.md"
81
91
 
82
92
  [tool.semantic_release.branches.main]
@@ -0,0 +1,10 @@
1
+ """Skill evaluation framework."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version("agent-skill-eval")
7
+ except PackageNotFoundError:
8
+ # Not installed (e.g. running from a source checkout): version is unknown.
9
+ # pyproject.toml is the single source of truth for the real version.
10
+ __version__ = "0.0.0"