agent-skill-eval 0.4.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. agent_skill_eval-0.4.1/CHANGELOG.md +159 -0
  2. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/PKG-INFO +1 -1
  3. agent_skill_eval-0.4.1/TODO.md +69 -0
  4. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/evals.json +1 -1
  5. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/pyproject.toml +11 -2
  6. agent_skill_eval-0.4.1/src/agent_skill_eval/__init__.py +10 -0
  7. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/cli.py +31 -9
  8. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/graders/__init__.py +130 -48
  9. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/models.py +3 -0
  10. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/runner.py +8 -0
  11. agent_skill_eval-0.4.0/CHANGELOG.md +0 -305
  12. agent_skill_eval-0.4.0/TODO.md +0 -53
  13. agent_skill_eval-0.4.0/src/agent_skill_eval/__init__.py +0 -8
  14. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
  15. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
  16. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
  17. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/actions/setup/action.yml +0 -0
  18. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/workflows/ci.yml +0 -0
  19. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/workflows/release.yml +0 -0
  20. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.gitignore +0 -0
  21. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/AGENTS.md +0 -0
  22. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/CODE_OF_CONDUCT.md +0 -0
  23. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/CONTRIBUTING.md +0 -0
  24. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/LICENSE +0 -0
  25. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/Makefile +0 -0
  26. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/README.md +0 -0
  27. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/config/.env.example +0 -0
  28. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/config/baseline.env +0 -0
  29. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/commit-push-pr/evals/evals.json +0 -0
  30. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/commit-push-pr/evals/files/sample_change.py +0 -0
  31. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/files/calculator.py +0 -0
  32. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/files/conftest.py +0 -0
  33. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/files/test_calculator.py +0 -0
  34. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/review-diff/evals/evals.json +0 -0
  35. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/review-diff/evals/files/changes.diff +0 -0
  36. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/review-diff/evals/files/orders.py +0 -0
  37. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/validate-config/evals/evals.json +0 -0
  38. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/validate-config/evals/files/app_config.json +0 -0
  39. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/write-release-notes/evals/evals.json +0 -0
  40. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/write-release-notes/evals/files/commits.txt +0 -0
  41. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/schemas/evals.schema.json +0 -0
  42. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/scripts/record-demo.sh +0 -0
  43. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/commit-push-pr/SKILL.md +0 -0
  44. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/fix-failing-tests/SKILL.md +0 -0
  45. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/review-diff/SKILL.md +0 -0
  46. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/validate-config/SKILL.md +0 -0
  47. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/validate-config/references/config-rules.md +0 -0
  48. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/validate-config/scripts/validate_config.py +0 -0
  49. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/write-release-notes/SKILL.md +0 -0
  50. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/__main__.py +0 -0
  51. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/git_state.py +0 -0
  52. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/harnesses/__init__.py +0 -0
  53. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/skills.py +0 -0
  54. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/workspace.py +0 -0
  55. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/__init__.py +0 -0
  56. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/conftest.py +0 -0
  57. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/fixtures/evals/format-json.json +0 -0
  58. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/fixtures/skills/format-json/SKILL.md +0 -0
  59. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_cli_smoke.py +0 -0
  60. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_e2e.py +0 -0
  61. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_fuzz_grader.py +0 -0
  62. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_git_state.py +0 -0
  63. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_graders.py +0 -0
  64. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_harnesses.py +0 -0
  65. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_launch_features.py +0 -0
  66. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_llm_grader.py +0 -0
  67. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_runner.py +0 -0
  68. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_skill_validation.py +0 -0
  69. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_skills.py +0 -0
  70. {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_workspace.py +0 -0
@@ -0,0 +1,159 @@
1
+ # Changelog
2
+
3
+ All notable changes to this project will be documented in this file.
4
+ Entries above the insertion flag are generated by python-semantic-release
5
+ from conventional commits; entries below it are curated by hand.
6
+
7
+ <!-- version list -->
8
+
9
+ ## v0.4.1 (2026-06-11)
10
+
11
+ ### Bug Fixes
12
+
13
+ - Todo batch — grader honesty, cost reporting, curated changelog
14
+ ([#11](https://github.com/tardigrde/agent-skill-eval/pull/11),
15
+ [`33ab02c`](https://github.com/tardigrde/agent-skill-eval/commit/33ab02c6c5ac3459d1987ba46ae3cc85038f8685))
16
+
17
+ * fix: grader honesty — skip-with-warning, workspace fallback warning, behavior-based negative
18
+ control
19
+
20
+ - LLMGrader: grading errors and missing per-assertion results are now skipped (not failed) and
21
+ warned about on stderr, so grader failures never masquerade as agent failures. - Workspace
22
+ resolution for file-based checks is shared between the deterministic and LLM graders and warns
23
+ (once per run dir) when it falls back to guessing from the output directory layout. -
24
+ fix-failing-tests negative-control: reword the grader-run-dependent 'output contains the source
25
+ code' assertion to the behavior that matters (file read and displayed/described).
26
+
27
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
28
+
29
+ * feat: cost column in report, validate flags LLM-rubric assertions, curated changelog
30
+
31
+ - benchmark.json: per-config cost_usd mean/stddev and per-agent cost deltas; report gains a Cost
32
+ (USD) column (table and markdown). - validate: classify_assertion() reports which assertions match
33
+ no deterministic pattern and will be graded by the LLM rubric, instead of that happening silently
34
+ at run time. - CHANGELOG.md: restore curated pre-0.4 entries (PSR-style headings);
35
+ semantic-release now runs in insertion-only update mode with vendored templates that render commit
36
+ summary lines only — no more commit bodies or Co-Authored-By trailers leaking into the changelog.
37
+ - Version is single-sourced from pyproject.toml (PSR version_variables removed; __init__ falls
38
+ back to 0.0.0 when not installed). - TODO.md: tick everything done in this batch and items
39
+ verified already done on main.
40
+
41
+ * fix: normalize assertion text when matching LLM grader responses
42
+
43
+ LLMs echo assertions with minor drift (casing, trailing periods, whitespace); exact-match made those
44
+ falsely count as missing results. Addresses Gemini review on PR #11.
45
+
46
+ * chore: remove vendored PSR templates, use defaults
47
+
48
+ ---------
49
+
50
+ Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
51
+
52
+ ### Documentation
53
+
54
+ - Update TODO — v0.4.0 state, subagent evals next, agent working notes
55
+ ([`4b5b76f`](https://github.com/tardigrde/agent-skill-eval/commit/4b5b76faed27678bf81a23cb9b2478e90c590c82))
56
+
57
+ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
58
+
59
+
60
+ ## v0.4.0 (2026-06-11)
61
+
62
+ ### Changed
63
+ - Converged all names on `agent-skill-eval`: GitHub repo (was `skill-evals`), CLI entry point (was `skill-eval`), and Python module (was `skill_eval`) now all match the PyPI package name
64
+ - Added short CLI alias `ase` (same app as `agent-skill-eval`)
65
+ - Env var prefix renamed `SKILL_EVAL_*` -> `ASE_*` (`ASE_AGENT_TIMEOUT`, `ASE_AGENT_RETRIES`, `ASE_LIVE`)
66
+
67
+ ### Added
68
+ - Codex harness fixes (`--sandbox workspace-write`, `--skip-git-repo-check`), `cost_usd` in `timing.json`, two-tier e2e tests, Makefile eval/test targets (merged via PR #9, previously unreleased)
69
+
70
+ ## v0.3.0 (2026-06-11)
71
+
72
+ ### Added
73
+ - `validate-config` example skill + eval suite: exercises bundled resources (agent runs skill's `scripts/` and reads its `references/`); graded via command-ran + file + content checks; fully offline
74
+ - `review-diff` example skill + eval suite: read-only analysis (planted bugs found, documented decoy not flagged, nothing modified); graded via chat-output-only content + LLM rubric; fully offline
75
+
76
+ ## v0.2.1 (2026-06-11)
77
+
78
+ ### Changed
79
+ - Renamed package from `skill-eval` to `agent-skill-eval` (PyPI name collision)
80
+
81
+ ## v0.2.0 (2026-06-10)
82
+
83
+ ### Added
84
+ - `write-release-notes` example skill + eval suite: exercises LLM rubric grading (grouping, breaking-change prominence, anti-fabrication) instead of git-state checks
85
+ - `--runs N` repeats every (eval, agent, config); `benchmark.json` now reports `full_pass_rate`, `pass_at_k`, and `k`
86
+ - `--agent-model agent=model` (repeatable) and `--harness-base-url` to pin models and endpoints per agent CLI
87
+ - `--timeout` / `--retries` flags; harnesses retry on timeout or non-zero exit with backoff and record `exit_code`, `timed_out`, `retries` in `timing.json`
88
+ - `agent-skill-eval validate`: schema + fixture-existence + duplicate-id checks for evals.json
89
+ - `agent-skill-eval list`: discover eval suites and skills under a directory
90
+ - `agent-skill-eval compare`: side-by-side pass rates of two iterations
91
+ - `report --format markdown` and `report --show-evidence` (failed/skipped assertion evidence)
92
+ - JSON Schema for the eval suite format at `schemas/evals.schema.json` (with a sync test)
93
+ - SKILL.md frontmatter validation warning on `run` (missing name/description, name mismatch)
94
+ - Richer `init` scaffold: SKILL.md frontmatter template + negative-control eval case
95
+ - PyPI trusted-publishing workflow on version tags (`.github/workflows/publish.yml`)
96
+ - Demo recording script (`scripts/record-demo.sh`)
97
+
98
+ ### Changed
99
+ - Assertions that cannot be graded (no API key, LLM grader error) are now **skipped** and excluded from the pass rate instead of counted as failures; `grading.json` gains `method` and `skipped` fields
100
+ - `run` warns upfront when no grader API key is set; `grade` warns that LLM verdicts may flip on re-grade
101
+ - README restructured around the full-harness pitch, with a documented table of recognized deterministic assertion patterns
102
+
103
+ ### Fixed
104
+ - Multi-agent runs no longer report a misleading `Delta: 0.0%`: `benchmark.json` now has per-agent `deltas`; the legacy `delta` field is only set for single-agent runs (otherwise `null`)
105
+ - Removed dead `cost_usd` handling in the Claude Code output parser
106
+
107
+ ## v0.1.0-alpha.3 (2026-06-07)
108
+
109
+ ### Added
110
+ - Shared test fixtures in `conftest.py`
111
+ - `test_llm_grader.py`: 18 tests covering client, prompts, JSON parsing, errors
112
+ - `test_fuzz_grader.py` rewrite: real handler tests instead of mocked dispatch
113
+ - `test_compute_benchmark` with xfail for known multi-agent delta=0 bug
114
+ - EvalCase validation tests (valid, negative control, contradiction)
115
+ - `SkillInstaller.uninstall` tests, `WorkspaceManager._clone_repo` test
116
+ - `github_repo_slug` tests for URLs without `.git` suffix
117
+
118
+ ### Changed
119
+ - CI uses setup-python's built-in cache instead of separate `actions/cache`
120
+
121
+ ### Fixed
122
+ - EvalCase contradiction validator (`force_skill_invocation` + `should_trigger`)
123
+ - LLMGrader now fails on missing assertions instead of silently dropping them
124
+
125
+ ## v0.1.0-alpha.2 (2026-06-02)
126
+
127
+ ### Added
128
+ - CI workflow with Python 3.11/3.12/3.13 matrix, caching, coverage, and smoke test
129
+ - CLI smoke test (`test_cli_smoke.py`) for full run -> report -> init cycle
130
+
131
+ ### Changed
132
+ - Tightened evals with state-delta grading and scoped cleanup
133
+ - Improved `commit-push-pr` skill to 100% pass rate
134
+
135
+ ### Fixed
136
+ - `github_repo_slug` function to parse GitHub URLs correctly
137
+ - Handoff review fixes and PR grading tightening
138
+
139
+ ## v0.1.0-alpha.1 (2026-05-27)
140
+
141
+ ### Added
142
+ - Initial release: skill evaluation framework for OpenCode, Claude Code, and Codex
143
+ - `run`, `report`, `grade`, `cleanup`, and `init` CLI commands
144
+ - Multi-agent support with baseline (with-skill / without-skill) comparison
145
+ - State-delta grading using pre/post git snapshots (`pre_state.json` / `post_state.json`)
146
+ - Deterministic assertions: branch, commit, push, PR, file existence, content matching
147
+ - LLM rubric grading via configurable model (`--grader-model`)
148
+ - Negative-control support (`should_trigger: false`) to catch accidental skill triggering
149
+ - Scoped cleanup via `cleanup.json` manifest (never closes unrelated PRs)
150
+ - `commit-push-pr` example skill with 5 eval cases
151
+ - Lazy-initialized OpenAI client with API key validation
152
+ - OpenRouter grader support via `OPENROUTER_API_KEY`
153
+ - Workspace isolation with fresh git repos per eval run
154
+ - Token/timing extraction from each agent's native output format
155
+ - `python -m agent_skill_eval` entry point
156
+
157
+ ### Fixed
158
+ - OpenRouter grader model resolution and opencode parser output handling
159
+ - Workspace-aware grading and dotenv loading
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: agent-skill-eval
3
- Version: 0.4.0
3
+ Version: 0.4.1
4
4
  Summary: Framework for evaluating agent skills across OpenCode, Claude Code, and Codex
5
5
  Project-URL: Homepage, https://github.com/tardigrde/agent-skill-eval
6
6
  Project-URL: Repository, https://github.com/tardigrde/agent-skill-eval
@@ -0,0 +1,69 @@
1
+ # TODO
2
+
3
+ Audit of `main` originally done at 0.1.0; pruned and updated 2026-06-11 (v0.4.0, post PR #9/#10).
4
+ Done items are kept briefly for history; anything obsolete or low-value was deleted.
5
+
6
+ ## How to work this list (for the implementing agent)
7
+
8
+ - Do NOT bundle everything into one PR. Group into separate PRs by section (one for Bugs, one for Docs, one per Feature) with conventional-commit messages — semantic-release on main derives versions and the changelog from them.
9
+ - `make test` (includes free e2e tier) + `make lint` must pass per PR; run `make test-live` once before the final merge of the batch (costs cents, needs `.env` OpenRouter key + opencode/codex auth).
10
+ - Items marked *(judgment)* are optional — skip unless already touching that code.
11
+
12
+ ## Features (next up)
13
+
14
+ - [ ] **Subagent evals** — evaluate custom subagent definitions (e.g. `~/.claude/agents/*.md`) the same way as skills: generalize `SkillInstaller` to an artifact installer with a type field; harnesses/runner/graders/metrics stay as-is. Add one example subagent + eval suite. README already announces it as coming soon.
15
+
16
+ ## Done
17
+
18
+ - [x] **Names converged on `agent-skill-eval`** (PR #10): repo, CLI, module, PyPI all match; `ase` short alias; env prefix `ASE_*`. Repo renames redirect from `skill-evals`/`agent-evals`.
19
+ - [x] **Release fully automated** (gcpath pattern): push to main → python-semantic-release bumps version, tags, GitHub release, publishes to PyPI via trusted publishing. v0.4.0 shipped this way end-to-end. Trusted publisher re-created for renamed repo (workflow `release.yml`, env `pypi`); old one deleted.
20
+
21
+ - [x] **Grader fixed + tested** — `deepseek/deepseek-v4-flash` via OpenRouter, `tests/test_llm_grader.py` covers prompt shape, JSON parsing, malformed responses, missing key.
22
+ - [x] **Fake harness + CI smoke test** — `--agent fake`, no network, wired into CI.
23
+ - [x] **Offline example skill** — `skills/fix-failing-tests` + 4-case eval suite (`examples/fix-failing-tests`).
24
+ - [x] **All 4 harnesses verified live** (2026-06-11): fake, claude-code (haiku via OpenRouter), opencode (Zen free), codex (`gpt-5.4-mini`, ChatGPT plan). Codex needed `--sandbox workspace-write` + `--skip-git-repo-check` (eval workspaces are fresh git dirs).
25
+ - [x] **Cost/time/accuracy all machine-readable per run** — `timing.json` gained `cost_usd` (claude: `total_cost_usd`, opencode: per-step `cost`); `duration_ms`, tokens, and `grading.json` pass rates already existed.
26
+ - [x] **Two-tier e2e tests** (`tests/test_e2e.py`) — free tier (fake harness, real CLI subprocess, artifact + report assertions) runs in every `make test`; live tier (`make test-live`, `live` marker) runs all real agents + LLM grader and enforces ≥0.7 mean pass rate, zero skipped grades.
27
+ - [x] **Makefile** — `cheap-eval`, `full-eval` (adds codex), `fake-eval`, `test`, `test-e2e`, `test-live`, `lint`; `AGENTS` + per-agent model vars; `config/baseline.env` pins models.
28
+ - [x] `--agent-model` and `--harness-base-url` CLI flags wired.
29
+ - [x] Harness retry/backoff with pristine-workspace guard; timeout + exit code recorded in `timing.json`.
30
+ - [x] Per-agent benchmark deltas for multi-agent runs (no more silent 0.0).
31
+ - [x] SKILL.md frontmatter problems warned at `run` time.
32
+ - [x] LICENSE, CHANGELOG.md, CONTRIBUTING.md, issue templates, PR template, `agent-skill-eval list`, conftest fixtures, fuzz tests for assertion matcher, CI matrix + coverage.
33
+
34
+ ## Bugs / sharp edges
35
+
36
+ - [x] `negative-control` eval: reworded "output contains the source code" assertion to behavior ("read and displayed or described its contents").
37
+ - [x] `grade` re-running the LLM grader fresh is surfaced in CLI output and README (was already done on main; verified).
38
+ - [x] `LLMGrader.grade`: grading errors and missing per-assertion results are now skipped (not failed) with a console warning.
39
+ - [x] `DeterministicGrader._resolve_workspace`: workspace fallback now warns (once per run dir) instead of silently guessing; shared with the LLM grader.
40
+ - [x] Assertion patterns documented in README (was already done on main); `validate` now also reports which assertions will fall through to the LLM rubric (`classify_assertion`).
41
+ - [ ] claude-code's `cost_usd` is the CLI's Anthropic-list-price estimate; actual OpenRouter billing differs. Document or reconcile via OpenRouter generation API.
42
+ - [x] CHANGELOG.md: curated pre-0.4 entries restored (PSR-style headings), PSR configured with `mode = "update"` + insertion flag, vendored templates in `templates/` render summary lines only (no commit bodies / Co-Authored-By trailers).
43
+
44
+ ## Test coverage gaps
45
+
46
+ - [ ] `build_command` tested for codex only; pin claude-code and opencode invocations too.
47
+ - [ ] `runner._build_cleanup_entry`: merged/closed/external PR-number handling uncovered.
48
+ - [x] Report shows per-assertion evidence via `report --show-evidence` (was already done on main; verified).
49
+
50
+ ## Docs
51
+
52
+ - [ ] Fill out `pyproject.toml` `authors` with a real author (rest of metadata was already filled on main).
53
+ - [x] Version single-sourced in `pyproject.toml`: PSR `version_variables` removed, `__init__.py` falls back to `0.0.0` when not installed.
54
+ - [ ] Document `python -m agent_skill_eval` in README (`examples/` vs `skills/` and `grading.json` debugging were already documented on main).
55
+ - [x] `--version` wired to package version (was already done on main; verified).
56
+
57
+ ## Improvements
58
+
59
+ - [x] Cost column in `agent-skill-eval report`: `benchmark.json` gains per-config `cost_usd` mean/stddev and per-agent cost deltas; report shows a Cost (USD) column.
60
+ - [x] `init` scaffold: SKILL.md frontmatter template + negative-control example (was already done on main; verified).
61
+ - [ ] Multi-agent comparison example showing how to read with/without-skill deltas.
62
+ - [ ] Expand `examples/commit-push-pr/evals/files/sample_change.py` into a realistic fixture.
63
+ - [ ] *(judgment)* Split `graders/__init__.py`, `cli.py`, `runner.py` once they next need surgery — one-time split makes everything after cheaper.
64
+
65
+ ## Cleanup
66
+
67
+ - [ ] Delete stale local branches (`feat/improve-commit-push-pr-skill`, `fix/login-bug`, `smoke-test-ci`, merged feature branches) after confirming merged.
68
+ - [ ] Delete `HANDOFF.md` (untracked, stale — its mission completed 2026-06-11) and the leftover `.claude/worktrees/` copy of the old module tree.
69
+ - [ ] *(judgment)* `SkillInstaller.uninstall` never called by the runner (safe today via `rmtree`, but asymmetric).
@@ -45,7 +45,7 @@
45
45
  "files": ["files/calculator.py"],
46
46
  "should_trigger": false,
47
47
  "assertions": [
48
- "The output contains the source code of calculator.py",
48
+ "The agent read calculator.py and displayed or described its contents",
49
49
  "calculator.py was not modified",
50
50
  "No tests were run"
51
51
  ]
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "agent-skill-eval"
7
- version = "0.4.0"
7
+ version = "0.4.1"
8
8
  description = "Framework for evaluating agent skills across OpenCode, Claude Code, and Codex"
9
9
  requires-python = ">=3.11"
10
10
  license = "MIT"
@@ -48,6 +48,7 @@ agent-skill-eval = "agent_skill_eval.cli:app"
48
48
  ase = "agent_skill_eval.cli:app"
49
49
 
50
50
 
51
+
51
52
  [tool.hatch.build.targets.wheel]
52
53
  packages = ["src/agent_skill_eval"]
53
54
 
@@ -69,7 +70,6 @@ select = ["E", "F", "I"]
69
70
  major_on_zero = false
70
71
  allow_zero_version = true
71
72
  version_toml = ["pyproject.toml:project.version"]
72
- version_variables = ["src/agent_skill_eval/__init__.py:__version__"]
73
73
  branch = "main"
74
74
  upload_to_pypi = false
75
75
  upload_to_release = true
@@ -77,6 +77,15 @@ build_command = "uv build"
77
77
  commit_message = "chore(release): {version}"
78
78
 
79
79
  [tool.semantic_release.changelog]
80
+ # Insertion-only updates: new releases are inserted at the flag, the curated
81
+ # history below it is never regenerated. Templates in templates/ are the PSR
82
+ # defaults minus full commit bodies (no Co-Authored-By trailers in the log).
83
+ mode = "update"
84
+ insertion_flag = "<!-- version list -->"
85
+ template_dir = "templates"
86
+ exclude_commit_patterns = ['''chore\(release\):.*''']
87
+
88
+ [tool.semantic_release.changelog.default_templates]
80
89
  changelog_file = "CHANGELOG.md"
81
90
 
82
91
  [tool.semantic_release.branches.main]
@@ -0,0 +1,10 @@
1
+ """Skill evaluation framework."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ try:
6
+ __version__ = version("agent-skill-eval")
7
+ except PackageNotFoundError:
8
+ # Not installed (e.g. running from a source checkout): version is unknown.
9
+ # pyproject.toml is the single source of truth for the real version.
10
+ __version__ = "0.0.0"
@@ -492,12 +492,14 @@ def _collect_results(workspace: Path) -> dict[str, dict]:
492
492
  return results
493
493
 
494
494
 
495
- def _benchmark_rows(benchmark: dict) -> list[tuple[str, str, str, str, str]]:
495
+ def _benchmark_rows(benchmark: dict) -> list[tuple[str, str, str, str, str, str]]:
496
496
  rows = []
497
497
  for config, stats in benchmark.get("run_summary", {}).items():
498
498
  pass_rate = stats.get("pass_rate", {})
499
499
  time_s = stats.get("time_seconds", {})
500
500
  tokens = stats.get("tokens", {})
501
+ # benchmark.json from runs before cost stats existed lacks the field.
502
+ cost = stats.get("cost_usd") or {}
501
503
  k = stats.get("k", 1)
502
504
  pass_at_k = f"{stats.get('pass_at_k', 0):.0%} (k={k})" if k > 1 else f"{stats.get('full_pass_rate', 0):.0%}"
503
505
  rows.append(
@@ -507,6 +509,7 @@ def _benchmark_rows(benchmark: dict) -> list[tuple[str, str, str, str, str]]:
507
509
  pass_at_k,
508
510
  f"{time_s.get('mean', 0):.1f} +/- {time_s.get('stddev', 0):.1f}",
509
511
  f"{tokens.get('mean', 0):.0f} +/- {tokens.get('stddev', 0):.0f}",
512
+ f"{cost.get('mean', 0):.4f} +/- {cost.get('stddev', 0):.4f}",
510
513
  )
511
514
  )
512
515
  return rows
@@ -519,7 +522,7 @@ def _benchmark_deltas(benchmark: dict) -> dict[str, dict]:
519
522
  return deltas
520
523
 
521
524
 
522
- REPORT_COLUMNS = ["Configuration", "Pass Rate", "Full Pass / pass@k", "Time (s)", "Tokens"]
525
+ REPORT_COLUMNS = ["Configuration", "Pass Rate", "Full Pass / pass@k", "Time (s)", "Tokens", "Cost (USD)"]
523
526
 
524
527
 
525
528
  def _print_markdown_report(iter_name: str, benchmark: dict) -> None:
@@ -534,7 +537,8 @@ def _print_markdown_report(iter_name: str, benchmark: dict) -> None:
534
537
  for agent, delta in deltas.items():
535
538
  print(
536
539
  f"- `{agent}`: pass rate {delta.get('pass_rate', 0):+.1%}, "
537
- f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}"
540
+ f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}, "
541
+ f"cost {delta.get('cost_usd', 0):+.4f} USD"
538
542
  )
539
543
 
540
544
 
@@ -542,11 +546,9 @@ def _print_table_report(iter_name: str, benchmark: dict) -> None:
542
546
  console.print(f"\n[bold]{iter_name}[/bold]")
543
547
 
544
548
  table = Table(title="Results Summary")
545
- table.add_column(REPORT_COLUMNS[0], style="cyan")
546
- table.add_column(REPORT_COLUMNS[1], style="green")
547
- table.add_column(REPORT_COLUMNS[2], style="magenta")
548
- table.add_column(REPORT_COLUMNS[3], style="yellow")
549
- table.add_column(REPORT_COLUMNS[4], style="blue")
549
+ styles = ["cyan", "green", "magenta", "yellow", "blue", "white"]
550
+ for column, style in zip(REPORT_COLUMNS, styles):
551
+ table.add_column(column, style=style)
550
552
  for row in _benchmark_rows(benchmark):
551
553
  table.add_row(*row)
552
554
  console.print(table)
@@ -557,7 +559,8 @@ def _print_table_report(iter_name: str, benchmark: dict) -> None:
557
559
  for agent, delta in deltas.items():
558
560
  console.print(
559
561
  f" {agent}: pass rate {delta.get('pass_rate', 0):+.1%}, "
560
- f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}"
562
+ f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}, "
563
+ f"cost {delta.get('cost_usd', 0):+.4f} USD"
561
564
  )
562
565
 
563
566
 
@@ -720,7 +723,26 @@ def validate(
720
723
  console.print(f"[red]Duplicate eval ids: {duplicates}[/red]")
721
724
  raise typer.Exit(1)
722
725
 
726
+ from agent_skill_eval.graders import classify_assertion
727
+
728
+ llm_assertions: list[tuple[str, str]] = []
729
+ total_assertions = 0
730
+ for eval_case in suite.evals:
731
+ for assertion in eval_case.assertions:
732
+ total_assertions += 1
733
+ if classify_assertion(assertion) == "llm":
734
+ llm_assertions.append((str(eval_case.id), assertion))
735
+
723
736
  console.print(f"[green]Valid: {len(suite.evals)} eval(s) for skill '{suite.skill_name}'[/green]")
737
+ if llm_assertions:
738
+ console.print(
739
+ f"[yellow]{len(llm_assertions)}/{total_assertions} assertion(s) match no deterministic "
740
+ f"pattern and will be graded by the LLM rubric (needs an API key; verdicts can vary):[/yellow]"
741
+ )
742
+ for eval_id, assertion in llm_assertions:
743
+ console.print(f" [dim]{eval_id}: {assertion}[/dim]")
744
+ elif total_assertions:
745
+ console.print(f"[green]All {total_assertions} assertion(s) match deterministic patterns[/green]")
724
746
 
725
747
 
726
748
  @app.command(name="list")
@@ -8,10 +8,97 @@ from pathlib import Path
8
8
  from typing import Optional
9
9
 
10
10
  from openai import OpenAI, OpenAIError
11
+ from rich.console import Console
11
12
 
12
13
  from agent_skill_eval.git_state import capture_git_state, github_repo_slug, state_diff
13
14
  from agent_skill_eval.models import AssertionResult, GitStateSnapshot, GradingResult, GradingSummary
14
15
 
16
+ console = Console(stderr=True)
17
+
18
+
19
+ def classify_assertion(assertion: str) -> str:
20
+ """Return the deterministic check an assertion text routes to, or ``"llm"``.
21
+
22
+ Mirrors the routing in ``DeterministicGrader._check_assertion`` exactly.
23
+ ``validate`` uses this to show suite authors which assertions will fall
24
+ through to the LLM rubric (nondeterministic, needs an API key) instead of
25
+ that happening silently at run time.
26
+ """
27
+ assertion_lower = assertion.lower()
28
+
29
+ if "branch" in assertion_lower and (
30
+ "created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
31
+ ):
32
+ return "git-branch"
33
+
34
+ if "commit" in assertion_lower and (
35
+ "created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
36
+ ):
37
+ return "git-commit"
38
+
39
+ if "push" in assertion_lower and (
40
+ "remote" in assertion_lower or "branch" in assertion_lower or "pushed" in assertion_lower
41
+ ):
42
+ return "pushed"
43
+
44
+ # Word-boundary match: a bare "pr" substring would also match
45
+ # "prominently", "present", "approach", etc.
46
+ if re.search(r"\bprs?\b|\bpull request", assertion_lower):
47
+ return "pr-created"
48
+
49
+ if (
50
+ "file exists" in assertion_lower
51
+ or ("file" in assertion_lower and "exists" in assertion_lower)
52
+ or ("created" in assertion_lower and any(c in assertion_lower for c in [".", "file"]))
53
+ ):
54
+ return "file-exists"
55
+
56
+ if "ran" in assertion_lower and (
57
+ "command" in assertion_lower or any(cmd in assertion_lower for cmd in ["npm", "git", "python", "cargo", "go"])
58
+ ):
59
+ return "command-ran"
60
+
61
+ if "contains" in assertion_lower or "includes" in assertion_lower:
62
+ # Without a quoted/backticked pattern this is a prose assertion that
63
+ # happens to contain "contains"/"includes"; it goes to the LLM rubric.
64
+ if re.findall(r'"([^"]+)"', assertion) or re.findall(r"`([^`]+)`", assertion):
65
+ return "content-contains"
66
+ return "llm"
67
+
68
+ if "valid json" in assertion_lower:
69
+ return "valid-json"
70
+
71
+ return "llm"
72
+
73
+
74
+ def _resolve_workspace_path(output_dir: Path, workspace: Path | None) -> Path:
75
+ """Resolve the agent workspace used for file-based checks.
76
+
77
+ Prefers the explicitly passed workspace. Falls back to guessing from the
78
+ output directory layout, which is unreliable (e.g. when re-grading after
79
+ the per-eval workspace was deleted) — so the fallback warns instead of
80
+ happening silently.
81
+ """
82
+ if workspace is not None and workspace.exists():
83
+ return workspace
84
+ if "with_skill" in str(output_dir) or "without_skill" in str(output_dir):
85
+ guess = output_dir.parent.parent
86
+ else:
87
+ guess = output_dir.parent
88
+ key = str(output_dir)
89
+ if key not in _warned_fallback_dirs:
90
+ _warned_fallback_dirs.add(key)
91
+ console.print(
92
+ f"[yellow]Warning: no live workspace for {output_dir}; file-based checks fall back to "
93
+ f"{guess} (saved artifacts only). Results may differ from the original run.[/yellow]"
94
+ )
95
+ return guess
96
+
97
+
98
+ # Output dirs we already warned about, so the deterministic and LLM graders
99
+ # don't each repeat the same workspace-fallback warning for one run.
100
+ _warned_fallback_dirs: set[str] = set()
101
+
15
102
 
16
103
  def _is_open_pr_state(state: Optional[str]) -> bool:
17
104
  """True if a PR state string represents an open PR.
@@ -64,6 +151,16 @@ def _fetch_pr_for_branch(branch: str, source_repo: str) -> Optional[dict]:
64
151
  return None
65
152
 
66
153
 
154
+ def _normalize_assertion_text(text: str) -> str:
155
+ """Normalize assertion text for matching LLM grader responses.
156
+
157
+ LLMs often echo assertions with minor drift (casing, trailing periods,
158
+ collapsed whitespace); without normalization those would be falsely
159
+ treated as missing results and skipped.
160
+ """
161
+ return " ".join(text.lower().split()).rstrip(".")
162
+
163
+
67
164
  def _load_state(path: Path) -> Optional[GitStateSnapshot]:
68
165
  if not path.exists():
69
166
  return None
@@ -143,45 +240,23 @@ class DeterministicGrader:
143
240
  should_trigger: bool,
144
241
  diff: dict,
145
242
  ) -> AssertionResult:
146
- assertion_lower = assertion.lower()
243
+ method = classify_assertion(assertion)
147
244
 
148
- if "branch" in assertion_lower and (
149
- "created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
150
- ):
245
+ if method == "git-branch":
151
246
  return self._check_git_branch(assertion, diff, should_trigger)
152
-
153
- if "commit" in assertion_lower and (
154
- "created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
155
- ):
247
+ if method == "git-commit":
156
248
  return self._check_git_commit(assertion, diff, should_trigger)
157
-
158
- if "push" in assertion_lower and (
159
- "remote" in assertion_lower or "branch" in assertion_lower or "pushed" in assertion_lower
160
- ):
249
+ if method == "pushed":
161
250
  return self._check_pushed(assertion, diff, agent_output, should_trigger)
162
-
163
- # Word-boundary match: a bare "pr" substring would also match
164
- # "prominently", "present", "approach", etc.
165
- if re.search(r"\bprs?\b|\bpull request", assertion_lower):
251
+ if method == "pr-created":
166
252
  return self._check_pr_created(assertion, diff, agent_output, should_trigger, output_dir)
167
-
168
- if (
169
- "file exists" in assertion_lower
170
- or ("file" in assertion_lower and "exists" in assertion_lower)
171
- or ("created" in assertion_lower and any(c in assertion_lower for c in [".", "file"]))
172
- ):
253
+ if method == "file-exists":
173
254
  return self._check_file_exists(assertion, output_dir, workspace)
174
-
175
- if "ran" in assertion_lower and (
176
- "command" in assertion_lower
177
- or any(cmd in assertion_lower for cmd in ["npm", "git", "python", "cargo", "go"])
178
- ):
255
+ if method == "command-ran":
179
256
  return self._check_command_ran(assertion, output_dir)
180
-
181
- if "contains" in assertion_lower or "includes" in assertion_lower:
257
+ if method == "content-contains":
182
258
  return self._check_content_contains(assertion, agent_output)
183
-
184
- if "valid json" in assertion_lower:
259
+ if method == "valid-json":
185
260
  return self._check_valid_json(assertion, output_dir, agent_output, workspace)
186
261
 
187
262
  return AssertionResult(
@@ -192,11 +267,7 @@ class DeterministicGrader:
192
267
  )
193
268
 
194
269
  def _resolve_workspace(self, output_dir: Path, workspace: Path | None) -> Path:
195
- if workspace:
196
- return workspace
197
- if "with_skill" in str(output_dir) or "without_skill" in str(output_dir):
198
- return output_dir.parent.parent
199
- return output_dir.parent
270
+ return _resolve_workspace_path(output_dir, workspace)
200
271
 
201
272
  def _invert(self, should_trigger: bool) -> bool:
202
273
  """For negative controls, branch/commit/push/pr assertions are inverted."""
@@ -629,8 +700,7 @@ class LLMGrader:
629
700
  if not assertions:
630
701
  return []
631
702
 
632
- if workspace is None or not workspace.exists():
633
- workspace = output_dir.parent.parent if "with_skill" in str(output_dir) else output_dir.parent
703
+ workspace = _resolve_workspace_path(output_dir, workspace)
634
704
  file_listing = self._list_workspace_files(workspace)
635
705
  file_contents = self._read_workspace_files(workspace)
636
706
 
@@ -679,21 +749,33 @@ Be strict: only mark PASS if there is clear evidence. Quote the output or file c
679
749
  method="llm",
680
750
  )
681
751
  )
682
- returned_texts.add(r["text"])
683
- for a in assertions:
684
- if a not in returned_texts:
685
- results.append(
686
- AssertionResult(
687
- text=a,
688
- passed=False,
689
- evidence="LLM grader did not return a result for this assertion",
690
- method="llm",
691
- )
752
+ returned_texts.add(_normalize_assertion_text(r["text"]))
753
+ missing = [a for a in assertions if _normalize_assertion_text(a) not in returned_texts]
754
+ if missing:
755
+ # The grader dropping an assertion is a grader failure, not an
756
+ # agent failure: skip it so it doesn't drag down the pass rate.
757
+ console.print(
758
+ f"[yellow]Warning: LLM grader returned no result for {len(missing)} assertion(s); "
759
+ f"skipped (not failed)[/yellow]"
760
+ )
761
+ for a in missing:
762
+ results.append(
763
+ AssertionResult(
764
+ text=a,
765
+ passed=False,
766
+ evidence="LLM grader did not return a result for this assertion (skipped, not failed)",
767
+ method="llm",
768
+ skipped=True,
692
769
  )
770
+ )
693
771
  return results
694
772
  except Exception as e:
695
773
  # A grader failure is not an agent failure: mark the assertions as
696
774
  # skipped so they don't drag down the pass rate.
775
+ console.print(
776
+ f"[yellow]Warning: LLM grading failed ({e}); {len(assertions)} assertion(s) skipped, "
777
+ f"not failed[/yellow]"
778
+ )
697
779
  return [
698
780
  AssertionResult(
699
781
  text=a,
@@ -122,6 +122,8 @@ class BenchmarkStats(BaseModel):
122
122
  pass_rate: StatsPair
123
123
  time_seconds: StatsPair
124
124
  tokens: StatsPair
125
+ # USD cost per run as reported by the agent CLI (see TimingData.cost_usd).
126
+ cost_usd: StatsPair = Field(default_factory=lambda: StatsPair(mean=0.0, stddev=0.0))
125
127
  # Fraction of runs where ALL assertions passed (pass@1 estimate).
126
128
  full_pass_rate: float = 0.0
127
129
  # Fraction of evals where at least one of k runs fully passed.
@@ -147,3 +149,4 @@ class DeltaStats(BaseModel):
147
149
  pass_rate: float
148
150
  time_seconds: float
149
151
  tokens: float
152
+ cost_usd: float = 0.0