agent-skill-eval 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_skill_eval-0.4.1/CHANGELOG.md +159 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/PKG-INFO +1 -1
- agent_skill_eval-0.4.1/TODO.md +69 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/evals.json +1 -1
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/pyproject.toml +11 -2
- agent_skill_eval-0.4.1/src/agent_skill_eval/__init__.py +10 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/cli.py +31 -9
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/graders/__init__.py +130 -48
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/models.py +3 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/runner.py +8 -0
- agent_skill_eval-0.4.0/CHANGELOG.md +0 -305
- agent_skill_eval-0.4.0/TODO.md +0 -53
- agent_skill_eval-0.4.0/src/agent_skill_eval/__init__.py +0 -8
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/PULL_REQUEST_TEMPLATE.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/actions/setup/action.yml +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/workflows/ci.yml +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.github/workflows/release.yml +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/.gitignore +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/AGENTS.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/CODE_OF_CONDUCT.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/CONTRIBUTING.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/LICENSE +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/Makefile +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/README.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/config/.env.example +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/config/baseline.env +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/commit-push-pr/evals/evals.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/commit-push-pr/evals/files/sample_change.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/files/calculator.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/files/conftest.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/files/test_calculator.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/review-diff/evals/evals.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/review-diff/evals/files/changes.diff +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/review-diff/evals/files/orders.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/validate-config/evals/evals.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/validate-config/evals/files/app_config.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/write-release-notes/evals/evals.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/write-release-notes/evals/files/commits.txt +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/schemas/evals.schema.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/scripts/record-demo.sh +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/commit-push-pr/SKILL.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/fix-failing-tests/SKILL.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/review-diff/SKILL.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/validate-config/SKILL.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/validate-config/references/config-rules.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/validate-config/scripts/validate_config.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/skills/write-release-notes/SKILL.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/__main__.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/git_state.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/harnesses/__init__.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/skills.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/src/agent_skill_eval/workspace.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/__init__.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/conftest.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/fixtures/evals/format-json.json +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/fixtures/skills/format-json/SKILL.md +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_cli_smoke.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_e2e.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_fuzz_grader.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_git_state.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_graders.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_harnesses.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_launch_features.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_llm_grader.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_runner.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_skill_validation.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_skills.py +0 -0
- {agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/tests/test_workspace.py +0 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
Entries above the insertion flag are generated by python-semantic-release
|
|
5
|
+
from conventional commits; entries below it are curated by hand.
|
|
6
|
+
|
|
7
|
+
<!-- version list -->
|
|
8
|
+
|
|
9
|
+
## v0.4.1 (2026-06-11)
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
- Todo batch — grader honesty, cost reporting, curated changelog
|
|
14
|
+
([#11](https://github.com/tardigrde/agent-skill-eval/pull/11),
|
|
15
|
+
[`33ab02c`](https://github.com/tardigrde/agent-skill-eval/commit/33ab02c6c5ac3459d1987ba46ae3cc85038f8685))
|
|
16
|
+
|
|
17
|
+
* fix: grader honesty — skip-with-warning, workspace fallback warning, behavior-based negative
|
|
18
|
+
control
|
|
19
|
+
|
|
20
|
+
- LLMGrader: grading errors and missing per-assertion results are now skipped (not failed) and
|
|
21
|
+
warned about on stderr, so grader failures never masquerade as agent failures. - Workspace
|
|
22
|
+
resolution for file-based checks is shared between the deterministic and LLM graders and warns
|
|
23
|
+
(once per run dir) when it falls back to guessing from the output directory layout. -
|
|
24
|
+
fix-failing-tests negative-control: reword the grader-run-dependent 'output contains the source
|
|
25
|
+
code' assertion to the behavior that matters (file read and displayed/described).
|
|
26
|
+
|
|
27
|
+
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
|
|
28
|
+
|
|
29
|
+
* feat: cost column in report, validate flags LLM-rubric assertions, curated changelog
|
|
30
|
+
|
|
31
|
+
- benchmark.json: per-config cost_usd mean/stddev and per-agent cost deltas; report gains a Cost
|
|
32
|
+
(USD) column (table and markdown). - validate: classify_assertion() reports which assertions match
|
|
33
|
+
no deterministic pattern and will be graded by the LLM rubric, instead of that happening silently
|
|
34
|
+
at run time. - CHANGELOG.md: restore curated pre-0.4 entries (PSR-style headings);
|
|
35
|
+
semantic-release now runs in insertion-only update mode with vendored templates that render commit
|
|
36
|
+
summary lines only — no more commit bodies or Co-Authored-By trailers leaking into the changelog.
|
|
37
|
+
- Version is single-sourced from pyproject.toml (PSR version_variables removed; __init__ falls
|
|
38
|
+
back to 0.0.0 when not installed). - TODO.md: tick everything done in this batch and items
|
|
39
|
+
verified already done on main.
|
|
40
|
+
|
|
41
|
+
* fix: normalize assertion text when matching LLM grader responses
|
|
42
|
+
|
|
43
|
+
LLMs echo assertions with minor drift (casing, trailing periods, whitespace); exact-match made those
|
|
44
|
+
falsely count as missing results. Addresses Gemini review on PR #11.
|
|
45
|
+
|
|
46
|
+
* chore: remove vendored PSR templates, use defaults
|
|
47
|
+
|
|
48
|
+
---------
|
|
49
|
+
|
|
50
|
+
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
|
|
51
|
+
|
|
52
|
+
### Documentation
|
|
53
|
+
|
|
54
|
+
- Update TODO — v0.4.0 state, subagent evals next, agent working notes
|
|
55
|
+
([`4b5b76f`](https://github.com/tardigrde/agent-skill-eval/commit/4b5b76faed27678bf81a23cb9b2478e90c590c82))
|
|
56
|
+
|
|
57
|
+
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
## v0.4.0 (2026-06-11)
|
|
61
|
+
|
|
62
|
+
### Changed
|
|
63
|
+
- Converged all names on `agent-skill-eval`: GitHub repo (was `skill-evals`), CLI entry point (was `skill-eval`), and Python module (was `skill_eval`) now all match the PyPI package name
|
|
64
|
+
- Added short CLI alias `ase` (same app as `agent-skill-eval`)
|
|
65
|
+
- Env var prefix renamed `SKILL_EVAL_*` -> `ASE_*` (`ASE_AGENT_TIMEOUT`, `ASE_AGENT_RETRIES`, `ASE_LIVE`)
|
|
66
|
+
|
|
67
|
+
### Added
|
|
68
|
+
- Codex harness fixes (`--sandbox workspace-write`, `--skip-git-repo-check`), `cost_usd` in `timing.json`, two-tier e2e tests, Makefile eval/test targets (merged via PR #9, previously unreleased)
|
|
69
|
+
|
|
70
|
+
## v0.3.0 (2026-06-11)
|
|
71
|
+
|
|
72
|
+
### Added
|
|
73
|
+
- `validate-config` example skill + eval suite: exercises bundled resources (agent runs skill's `scripts/` and reads its `references/`); graded via command-ran + file + content checks; fully offline
|
|
74
|
+
- `review-diff` example skill + eval suite: read-only analysis (planted bugs found, documented decoy not flagged, nothing modified); graded via chat-output-only content + LLM rubric; fully offline
|
|
75
|
+
|
|
76
|
+
## v0.2.1 (2026-06-11)
|
|
77
|
+
|
|
78
|
+
### Changed
|
|
79
|
+
- Renamed package from `skill-eval` to `agent-skill-eval` (PyPI name collision)
|
|
80
|
+
|
|
81
|
+
## v0.2.0 (2026-06-10)
|
|
82
|
+
|
|
83
|
+
### Added
|
|
84
|
+
- `write-release-notes` example skill + eval suite: exercises LLM rubric grading (grouping, breaking-change prominence, anti-fabrication) instead of git-state checks
|
|
85
|
+
- `--runs N` repeats every (eval, agent, config); `benchmark.json` now reports `full_pass_rate`, `pass_at_k`, and `k`
|
|
86
|
+
- `--agent-model agent=model` (repeatable) and `--harness-base-url` to pin models and endpoints per agent CLI
|
|
87
|
+
- `--timeout` / `--retries` flags; harnesses retry on timeout or non-zero exit with backoff and record `exit_code`, `timed_out`, `retries` in `timing.json`
|
|
88
|
+
- `agent-skill-eval validate`: schema + fixture-existence + duplicate-id checks for evals.json
|
|
89
|
+
- `agent-skill-eval list`: discover eval suites and skills under a directory
|
|
90
|
+
- `agent-skill-eval compare`: side-by-side pass rates of two iterations
|
|
91
|
+
- `report --format markdown` and `report --show-evidence` (failed/skipped assertion evidence)
|
|
92
|
+
- JSON Schema for the eval suite format at `schemas/evals.schema.json` (with a sync test)
|
|
93
|
+
- SKILL.md frontmatter validation warning on `run` (missing name/description, name mismatch)
|
|
94
|
+
- Richer `init` scaffold: SKILL.md frontmatter template + negative-control eval case
|
|
95
|
+
- PyPI trusted-publishing workflow on version tags (`.github/workflows/publish.yml`)
|
|
96
|
+
- Demo recording script (`scripts/record-demo.sh`)
|
|
97
|
+
|
|
98
|
+
### Changed
|
|
99
|
+
- Assertions that cannot be graded (no API key, LLM grader error) are now **skipped** and excluded from the pass rate instead of counted as failures; `grading.json` gains `method` and `skipped` fields
|
|
100
|
+
- `run` warns upfront when no grader API key is set; `grade` warns that LLM verdicts may flip on re-grade
|
|
101
|
+
- README restructured around the full-harness pitch, with a documented table of recognized deterministic assertion patterns
|
|
102
|
+
|
|
103
|
+
### Fixed
|
|
104
|
+
- Multi-agent runs no longer report a misleading `Delta: 0.0%`: `benchmark.json` now has per-agent `deltas`; the legacy `delta` field is only set for single-agent runs (otherwise `null`)
|
|
105
|
+
- Removed dead `cost_usd` handling in the Claude Code output parser
|
|
106
|
+
|
|
107
|
+
## v0.1.0-alpha.3 (2026-06-07)
|
|
108
|
+
|
|
109
|
+
### Added
|
|
110
|
+
- Shared test fixtures in `conftest.py`
|
|
111
|
+
- `test_llm_grader.py`: 18 tests covering client, prompts, JSON parsing, errors
|
|
112
|
+
- `test_fuzz_grader.py` rewrite: real handler tests instead of mocked dispatch
|
|
113
|
+
- `test_compute_benchmark` with xfail for known multi-agent delta=0 bug
|
|
114
|
+
- EvalCase validation tests (valid, negative control, contradiction)
|
|
115
|
+
- `SkillInstaller.uninstall` tests, `WorkspaceManager._clone_repo` test
|
|
116
|
+
- `github_repo_slug` tests for URLs without `.git` suffix
|
|
117
|
+
|
|
118
|
+
### Changed
|
|
119
|
+
- CI uses setup-python's built-in cache instead of separate `actions/cache`
|
|
120
|
+
|
|
121
|
+
### Fixed
|
|
122
|
+
- EvalCase contradiction validator (`force_skill_invocation` + `should_trigger`)
|
|
123
|
+
- LLMGrader now fails on missing assertions instead of silently dropping them
|
|
124
|
+
|
|
125
|
+
## v0.1.0-alpha.2 (2026-06-02)
|
|
126
|
+
|
|
127
|
+
### Added
|
|
128
|
+
- CI workflow with Python 3.11/3.12/3.13 matrix, caching, coverage, and smoke test
|
|
129
|
+
- CLI smoke test (`test_cli_smoke.py`) for full run -> report -> init cycle
|
|
130
|
+
|
|
131
|
+
### Changed
|
|
132
|
+
- Tightened evals with state-delta grading and scoped cleanup
|
|
133
|
+
- Improved `commit-push-pr` skill to 100% pass rate
|
|
134
|
+
|
|
135
|
+
### Fixed
|
|
136
|
+
- `github_repo_slug` function to parse GitHub URLs correctly
|
|
137
|
+
- Handoff review fixes and PR grading tightening
|
|
138
|
+
|
|
139
|
+
## v0.1.0-alpha.1 (2026-05-27)
|
|
140
|
+
|
|
141
|
+
### Added
|
|
142
|
+
- Initial release: skill evaluation framework for OpenCode, Claude Code, and Codex
|
|
143
|
+
- `run`, `report`, `grade`, `cleanup`, and `init` CLI commands
|
|
144
|
+
- Multi-agent support with baseline (with-skill / without-skill) comparison
|
|
145
|
+
- State-delta grading using pre/post git snapshots (`pre_state.json` / `post_state.json`)
|
|
146
|
+
- Deterministic assertions: branch, commit, push, PR, file existence, content matching
|
|
147
|
+
- LLM rubric grading via configurable model (`--grader-model`)
|
|
148
|
+
- Negative-control support (`should_trigger: false`) to catch accidental skill triggering
|
|
149
|
+
- Scoped cleanup via `cleanup.json` manifest (never closes unrelated PRs)
|
|
150
|
+
- `commit-push-pr` example skill with 5 eval cases
|
|
151
|
+
- Lazy-initialized OpenAI client with API key validation
|
|
152
|
+
- OpenRouter grader support via `OPENROUTER_API_KEY`
|
|
153
|
+
- Workspace isolation with fresh git repos per eval run
|
|
154
|
+
- Token/timing extraction from each agent's native output format
|
|
155
|
+
- `python -m agent_skill_eval` entry point
|
|
156
|
+
|
|
157
|
+
### Fixed
|
|
158
|
+
- OpenRouter grader model resolution and opencode parser output handling
|
|
159
|
+
- Workspace-aware grading and dotenv loading
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: agent-skill-eval
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Framework for evaluating agent skills across OpenCode, Claude Code, and Codex
|
|
5
5
|
Project-URL: Homepage, https://github.com/tardigrde/agent-skill-eval
|
|
6
6
|
Project-URL: Repository, https://github.com/tardigrde/agent-skill-eval
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# TODO
|
|
2
|
+
|
|
3
|
+
Audit of `main` originally done at 0.1.0; pruned and updated 2026-06-11 (v0.4.0, post PR #9/#10).
|
|
4
|
+
Done items are kept briefly for history; anything obsolete or low-value was deleted.
|
|
5
|
+
|
|
6
|
+
## How to work this list (for the implementing agent)
|
|
7
|
+
|
|
8
|
+
- Do NOT bundle everything into one PR. Group into separate PRs by section (one for Bugs, one for Docs, one per Feature) with conventional-commit messages — semantic-release on main derives versions and the changelog from them.
|
|
9
|
+
- `make test` (includes free e2e tier) + `make lint` must pass per PR; run `make test-live` once before the final merge of the batch (costs cents, needs `.env` OpenRouter key + opencode/codex auth).
|
|
10
|
+
- Items marked *(judgment)* are optional — skip unless already touching that code.
|
|
11
|
+
|
|
12
|
+
## Features (next up)
|
|
13
|
+
|
|
14
|
+
- [ ] **Subagent evals** — evaluate custom subagent definitions (e.g. `~/.claude/agents/*.md`) the same way as skills: generalize `SkillInstaller` to an artifact installer with a type field; harnesses/runner/graders/metrics stay as-is. Add one example subagent + eval suite. README already announces it as coming soon.
|
|
15
|
+
|
|
16
|
+
## Done
|
|
17
|
+
|
|
18
|
+
- [x] **Names converged on `agent-skill-eval`** (PR #10): repo, CLI, module, PyPI all match; `ase` short alias; env prefix `ASE_*`. Repo renames redirect from `skill-evals`/`agent-evals`.
|
|
19
|
+
- [x] **Release fully automated** (gcpath pattern): push to main → python-semantic-release bumps version, tags, GitHub release, publishes to PyPI via trusted publishing. v0.4.0 shipped this way end-to-end. Trusted publisher re-created for renamed repo (workflow `release.yml`, env `pypi`); old one deleted.
|
|
20
|
+
|
|
21
|
+
- [x] **Grader fixed + tested** — `deepseek/deepseek-v4-flash` via OpenRouter, `tests/test_llm_grader.py` covers prompt shape, JSON parsing, malformed responses, missing key.
|
|
22
|
+
- [x] **Fake harness + CI smoke test** — `--agent fake`, no network, wired into CI.
|
|
23
|
+
- [x] **Offline example skill** — `skills/fix-failing-tests` + 4-case eval suite (`examples/fix-failing-tests`).
|
|
24
|
+
- [x] **All 4 harnesses verified live** (2026-06-11): fake, claude-code (haiku via OpenRouter), opencode (Zen free), codex (`gpt-5.4-mini`, ChatGPT plan). Codex needed `--sandbox workspace-write` + `--skip-git-repo-check` (eval workspaces are fresh git dirs).
|
|
25
|
+
- [x] **Cost/time/accuracy all machine-readable per run** — `timing.json` gained `cost_usd` (claude: `total_cost_usd`, opencode: per-step `cost`); `duration_ms`, tokens, and `grading.json` pass rates already existed.
|
|
26
|
+
- [x] **Two-tier e2e tests** (`tests/test_e2e.py`) — free tier (fake harness, real CLI subprocess, artifact + report assertions) runs in every `make test`; live tier (`make test-live`, `live` marker) runs all real agents + LLM grader and enforces ≥0.7 mean pass rate, zero skipped grades.
|
|
27
|
+
- [x] **Makefile** — `cheap-eval`, `full-eval` (adds codex), `fake-eval`, `test`, `test-e2e`, `test-live`, `lint`; `AGENTS` + per-agent model vars; `config/baseline.env` pins models.
|
|
28
|
+
- [x] `--agent-model` and `--harness-base-url` CLI flags wired.
|
|
29
|
+
- [x] Harness retry/backoff with pristine-workspace guard; timeout + exit code recorded in `timing.json`.
|
|
30
|
+
- [x] Per-agent benchmark deltas for multi-agent runs (no more silent 0.0).
|
|
31
|
+
- [x] SKILL.md frontmatter problems warned at `run` time.
|
|
32
|
+
- [x] LICENSE, CHANGELOG.md, CONTRIBUTING.md, issue templates, PR template, `agent-skill-eval list`, conftest fixtures, fuzz tests for assertion matcher, CI matrix + coverage.
|
|
33
|
+
|
|
34
|
+
## Bugs / sharp edges
|
|
35
|
+
|
|
36
|
+
- [x] `negative-control` eval: reworded "output contains the source code" assertion to behavior ("read and displayed or described its contents").
|
|
37
|
+
- [x] `grade` re-running the LLM grader fresh is surfaced in CLI output and README (was already done on main; verified).
|
|
38
|
+
- [x] `LLMGrader.grade`: grading errors and missing per-assertion results are now skipped (not failed) with a console warning.
|
|
39
|
+
- [x] `DeterministicGrader._resolve_workspace`: workspace fallback now warns (once per run dir) instead of silently guessing; shared with the LLM grader.
|
|
40
|
+
- [x] Assertion patterns documented in README (was already done on main); `validate` now also reports which assertions will fall through to the LLM rubric (`classify_assertion`).
|
|
41
|
+
- [ ] claude-code's `cost_usd` is the CLI's Anthropic-list-price estimate; actual OpenRouter billing differs. Document or reconcile via OpenRouter generation API.
|
|
42
|
+
- [x] CHANGELOG.md: curated pre-0.4 entries restored (PSR-style headings), PSR configured with `mode = "update"` + insertion flag, vendored templates in `templates/` render summary lines only (no commit bodies / Co-Authored-By trailers).
|
|
43
|
+
|
|
44
|
+
## Test coverage gaps
|
|
45
|
+
|
|
46
|
+
- [ ] `build_command` tested for codex only; pin claude-code and opencode invocations too.
|
|
47
|
+
- [ ] `runner._build_cleanup_entry`: merged/closed/external PR-number handling uncovered.
|
|
48
|
+
- [x] Report shows per-assertion evidence via `report --show-evidence` (was already done on main; verified).
|
|
49
|
+
|
|
50
|
+
## Docs
|
|
51
|
+
|
|
52
|
+
- [ ] Fill out `pyproject.toml` `authors` with a real author (rest of metadata was already filled on main).
|
|
53
|
+
- [x] Version single-sourced in `pyproject.toml`: PSR `version_variables` removed, `__init__.py` falls back to `0.0.0` when not installed.
|
|
54
|
+
- [ ] Document `python -m agent_skill_eval` in README (`examples/` vs `skills/` and `grading.json` debugging were already documented on main).
|
|
55
|
+
- [x] `--version` wired to package version (was already done on main; verified).
|
|
56
|
+
|
|
57
|
+
## Improvements
|
|
58
|
+
|
|
59
|
+
- [x] Cost column in `agent-skill-eval report`: `benchmark.json` gains per-config `cost_usd` mean/stddev and per-agent cost deltas; report shows a Cost (USD) column.
|
|
60
|
+
- [x] `init` scaffold: SKILL.md frontmatter template + negative-control example (was already done on main; verified).
|
|
61
|
+
- [ ] Multi-agent comparison example showing how to read with/without-skill deltas.
|
|
62
|
+
- [ ] Expand `examples/commit-push-pr/evals/files/sample_change.py` into a realistic fixture.
|
|
63
|
+
- [ ] *(judgment)* Split `graders/__init__.py`, `cli.py`, `runner.py` once they next need surgery — one-time split makes everything after cheaper.
|
|
64
|
+
|
|
65
|
+
## Cleanup
|
|
66
|
+
|
|
67
|
+
- [ ] Delete stale local branches (`feat/improve-commit-push-pr-skill`, `fix/login-bug`, `smoke-test-ci`, merged feature branches) after confirming merged.
|
|
68
|
+
- [ ] Delete `HANDOFF.md` (untracked, stale — its mission completed 2026-06-11) and the leftover `.claude/worktrees/` copy of the old module tree.
|
|
69
|
+
- [ ] *(judgment)* `SkillInstaller.uninstall` never called by the runner (safe today via `rmtree`, but asymmetric).
|
{agent_skill_eval-0.4.0 → agent_skill_eval-0.4.1}/examples/fix-failing-tests/evals/evals.json
RENAMED
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"files": ["files/calculator.py"],
|
|
46
46
|
"should_trigger": false,
|
|
47
47
|
"assertions": [
|
|
48
|
-
"The
|
|
48
|
+
"The agent read calculator.py and displayed or described its contents",
|
|
49
49
|
"calculator.py was not modified",
|
|
50
50
|
"No tests were run"
|
|
51
51
|
]
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "agent-skill-eval"
|
|
7
|
-
version = "0.4.
|
|
7
|
+
version = "0.4.1"
|
|
8
8
|
description = "Framework for evaluating agent skills across OpenCode, Claude Code, and Codex"
|
|
9
9
|
requires-python = ">=3.11"
|
|
10
10
|
license = "MIT"
|
|
@@ -48,6 +48,7 @@ agent-skill-eval = "agent_skill_eval.cli:app"
|
|
|
48
48
|
ase = "agent_skill_eval.cli:app"
|
|
49
49
|
|
|
50
50
|
|
|
51
|
+
|
|
51
52
|
[tool.hatch.build.targets.wheel]
|
|
52
53
|
packages = ["src/agent_skill_eval"]
|
|
53
54
|
|
|
@@ -69,7 +70,6 @@ select = ["E", "F", "I"]
|
|
|
69
70
|
major_on_zero = false
|
|
70
71
|
allow_zero_version = true
|
|
71
72
|
version_toml = ["pyproject.toml:project.version"]
|
|
72
|
-
version_variables = ["src/agent_skill_eval/__init__.py:__version__"]
|
|
73
73
|
branch = "main"
|
|
74
74
|
upload_to_pypi = false
|
|
75
75
|
upload_to_release = true
|
|
@@ -77,6 +77,15 @@ build_command = "uv build"
|
|
|
77
77
|
commit_message = "chore(release): {version}"
|
|
78
78
|
|
|
79
79
|
[tool.semantic_release.changelog]
|
|
80
|
+
# Insertion-only updates: new releases are inserted at the flag, the curated
|
|
81
|
+
# history below it is never regenerated. Templates in templates/ are the PSR
|
|
82
|
+
# defaults minus full commit bodies (no Co-Authored-By trailers in the log).
|
|
83
|
+
mode = "update"
|
|
84
|
+
insertion_flag = "<!-- version list -->"
|
|
85
|
+
template_dir = "templates"
|
|
86
|
+
exclude_commit_patterns = ['''chore\(release\):.*''']
|
|
87
|
+
|
|
88
|
+
[tool.semantic_release.changelog.default_templates]
|
|
80
89
|
changelog_file = "CHANGELOG.md"
|
|
81
90
|
|
|
82
91
|
[tool.semantic_release.branches.main]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Skill evaluation framework."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("agent-skill-eval")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
# Not installed (e.g. running from a source checkout): version is unknown.
|
|
9
|
+
# pyproject.toml is the single source of truth for the real version.
|
|
10
|
+
__version__ = "0.0.0"
|
|
@@ -492,12 +492,14 @@ def _collect_results(workspace: Path) -> dict[str, dict]:
|
|
|
492
492
|
return results
|
|
493
493
|
|
|
494
494
|
|
|
495
|
-
def _benchmark_rows(benchmark: dict) -> list[tuple[str, str, str, str, str]]:
|
|
495
|
+
def _benchmark_rows(benchmark: dict) -> list[tuple[str, str, str, str, str, str]]:
|
|
496
496
|
rows = []
|
|
497
497
|
for config, stats in benchmark.get("run_summary", {}).items():
|
|
498
498
|
pass_rate = stats.get("pass_rate", {})
|
|
499
499
|
time_s = stats.get("time_seconds", {})
|
|
500
500
|
tokens = stats.get("tokens", {})
|
|
501
|
+
# benchmark.json from runs before cost stats existed lacks the field.
|
|
502
|
+
cost = stats.get("cost_usd") or {}
|
|
501
503
|
k = stats.get("k", 1)
|
|
502
504
|
pass_at_k = f"{stats.get('pass_at_k', 0):.0%} (k={k})" if k > 1 else f"{stats.get('full_pass_rate', 0):.0%}"
|
|
503
505
|
rows.append(
|
|
@@ -507,6 +509,7 @@ def _benchmark_rows(benchmark: dict) -> list[tuple[str, str, str, str, str]]:
|
|
|
507
509
|
pass_at_k,
|
|
508
510
|
f"{time_s.get('mean', 0):.1f} +/- {time_s.get('stddev', 0):.1f}",
|
|
509
511
|
f"{tokens.get('mean', 0):.0f} +/- {tokens.get('stddev', 0):.0f}",
|
|
512
|
+
f"{cost.get('mean', 0):.4f} +/- {cost.get('stddev', 0):.4f}",
|
|
510
513
|
)
|
|
511
514
|
)
|
|
512
515
|
return rows
|
|
@@ -519,7 +522,7 @@ def _benchmark_deltas(benchmark: dict) -> dict[str, dict]:
|
|
|
519
522
|
return deltas
|
|
520
523
|
|
|
521
524
|
|
|
522
|
-
REPORT_COLUMNS = ["Configuration", "Pass Rate", "Full Pass / pass@k", "Time (s)", "Tokens"]
|
|
525
|
+
REPORT_COLUMNS = ["Configuration", "Pass Rate", "Full Pass / pass@k", "Time (s)", "Tokens", "Cost (USD)"]
|
|
523
526
|
|
|
524
527
|
|
|
525
528
|
def _print_markdown_report(iter_name: str, benchmark: dict) -> None:
|
|
@@ -534,7 +537,8 @@ def _print_markdown_report(iter_name: str, benchmark: dict) -> None:
|
|
|
534
537
|
for agent, delta in deltas.items():
|
|
535
538
|
print(
|
|
536
539
|
f"- `{agent}`: pass rate {delta.get('pass_rate', 0):+.1%}, "
|
|
537
|
-
f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}"
|
|
540
|
+
f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}, "
|
|
541
|
+
f"cost {delta.get('cost_usd', 0):+.4f} USD"
|
|
538
542
|
)
|
|
539
543
|
|
|
540
544
|
|
|
@@ -542,11 +546,9 @@ def _print_table_report(iter_name: str, benchmark: dict) -> None:
|
|
|
542
546
|
console.print(f"\n[bold]{iter_name}[/bold]")
|
|
543
547
|
|
|
544
548
|
table = Table(title="Results Summary")
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
table.add_column(REPORT_COLUMNS[3], style="yellow")
|
|
549
|
-
table.add_column(REPORT_COLUMNS[4], style="blue")
|
|
549
|
+
styles = ["cyan", "green", "magenta", "yellow", "blue", "white"]
|
|
550
|
+
for column, style in zip(REPORT_COLUMNS, styles):
|
|
551
|
+
table.add_column(column, style=style)
|
|
550
552
|
for row in _benchmark_rows(benchmark):
|
|
551
553
|
table.add_row(*row)
|
|
552
554
|
console.print(table)
|
|
@@ -557,7 +559,8 @@ def _print_table_report(iter_name: str, benchmark: dict) -> None:
|
|
|
557
559
|
for agent, delta in deltas.items():
|
|
558
560
|
console.print(
|
|
559
561
|
f" {agent}: pass rate {delta.get('pass_rate', 0):+.1%}, "
|
|
560
|
-
f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}"
|
|
562
|
+
f"time {delta.get('time_seconds', 0):+.1f}s, tokens {delta.get('tokens', 0):+.0f}, "
|
|
563
|
+
f"cost {delta.get('cost_usd', 0):+.4f} USD"
|
|
561
564
|
)
|
|
562
565
|
|
|
563
566
|
|
|
@@ -720,7 +723,26 @@ def validate(
|
|
|
720
723
|
console.print(f"[red]Duplicate eval ids: {duplicates}[/red]")
|
|
721
724
|
raise typer.Exit(1)
|
|
722
725
|
|
|
726
|
+
from agent_skill_eval.graders import classify_assertion
|
|
727
|
+
|
|
728
|
+
llm_assertions: list[tuple[str, str]] = []
|
|
729
|
+
total_assertions = 0
|
|
730
|
+
for eval_case in suite.evals:
|
|
731
|
+
for assertion in eval_case.assertions:
|
|
732
|
+
total_assertions += 1
|
|
733
|
+
if classify_assertion(assertion) == "llm":
|
|
734
|
+
llm_assertions.append((str(eval_case.id), assertion))
|
|
735
|
+
|
|
723
736
|
console.print(f"[green]Valid: {len(suite.evals)} eval(s) for skill '{suite.skill_name}'[/green]")
|
|
737
|
+
if llm_assertions:
|
|
738
|
+
console.print(
|
|
739
|
+
f"[yellow]{len(llm_assertions)}/{total_assertions} assertion(s) match no deterministic "
|
|
740
|
+
f"pattern and will be graded by the LLM rubric (needs an API key; verdicts can vary):[/yellow]"
|
|
741
|
+
)
|
|
742
|
+
for eval_id, assertion in llm_assertions:
|
|
743
|
+
console.print(f" [dim]{eval_id}: {assertion}[/dim]")
|
|
744
|
+
elif total_assertions:
|
|
745
|
+
console.print(f"[green]All {total_assertions} assertion(s) match deterministic patterns[/green]")
|
|
724
746
|
|
|
725
747
|
|
|
726
748
|
@app.command(name="list")
|
|
@@ -8,10 +8,97 @@ from pathlib import Path
|
|
|
8
8
|
from typing import Optional
|
|
9
9
|
|
|
10
10
|
from openai import OpenAI, OpenAIError
|
|
11
|
+
from rich.console import Console
|
|
11
12
|
|
|
12
13
|
from agent_skill_eval.git_state import capture_git_state, github_repo_slug, state_diff
|
|
13
14
|
from agent_skill_eval.models import AssertionResult, GitStateSnapshot, GradingResult, GradingSummary
|
|
14
15
|
|
|
16
|
+
console = Console(stderr=True)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def classify_assertion(assertion: str) -> str:
|
|
20
|
+
"""Return the deterministic check an assertion text routes to, or ``"llm"``.
|
|
21
|
+
|
|
22
|
+
Mirrors the routing in ``DeterministicGrader._check_assertion`` exactly.
|
|
23
|
+
``validate`` uses this to show suite authors which assertions will fall
|
|
24
|
+
through to the LLM rubric (nondeterministic, needs an API key) instead of
|
|
25
|
+
that happening silently at run time.
|
|
26
|
+
"""
|
|
27
|
+
assertion_lower = assertion.lower()
|
|
28
|
+
|
|
29
|
+
if "branch" in assertion_lower and (
|
|
30
|
+
"created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
|
|
31
|
+
):
|
|
32
|
+
return "git-branch"
|
|
33
|
+
|
|
34
|
+
if "commit" in assertion_lower and (
|
|
35
|
+
"created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
|
|
36
|
+
):
|
|
37
|
+
return "git-commit"
|
|
38
|
+
|
|
39
|
+
if "push" in assertion_lower and (
|
|
40
|
+
"remote" in assertion_lower or "branch" in assertion_lower or "pushed" in assertion_lower
|
|
41
|
+
):
|
|
42
|
+
return "pushed"
|
|
43
|
+
|
|
44
|
+
# Word-boundary match: a bare "pr" substring would also match
|
|
45
|
+
# "prominently", "present", "approach", etc.
|
|
46
|
+
if re.search(r"\bprs?\b|\bpull request", assertion_lower):
|
|
47
|
+
return "pr-created"
|
|
48
|
+
|
|
49
|
+
if (
|
|
50
|
+
"file exists" in assertion_lower
|
|
51
|
+
or ("file" in assertion_lower and "exists" in assertion_lower)
|
|
52
|
+
or ("created" in assertion_lower and any(c in assertion_lower for c in [".", "file"]))
|
|
53
|
+
):
|
|
54
|
+
return "file-exists"
|
|
55
|
+
|
|
56
|
+
if "ran" in assertion_lower and (
|
|
57
|
+
"command" in assertion_lower or any(cmd in assertion_lower for cmd in ["npm", "git", "python", "cargo", "go"])
|
|
58
|
+
):
|
|
59
|
+
return "command-ran"
|
|
60
|
+
|
|
61
|
+
if "contains" in assertion_lower or "includes" in assertion_lower:
|
|
62
|
+
# Without a quoted/backticked pattern this is a prose assertion that
|
|
63
|
+
# happens to contain "contains"/"includes"; it goes to the LLM rubric.
|
|
64
|
+
if re.findall(r'"([^"]+)"', assertion) or re.findall(r"`([^`]+)`", assertion):
|
|
65
|
+
return "content-contains"
|
|
66
|
+
return "llm"
|
|
67
|
+
|
|
68
|
+
if "valid json" in assertion_lower:
|
|
69
|
+
return "valid-json"
|
|
70
|
+
|
|
71
|
+
return "llm"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _resolve_workspace_path(output_dir: Path, workspace: Path | None) -> Path:
|
|
75
|
+
"""Resolve the agent workspace used for file-based checks.
|
|
76
|
+
|
|
77
|
+
Prefers the explicitly passed workspace. Falls back to guessing from the
|
|
78
|
+
output directory layout, which is unreliable (e.g. when re-grading after
|
|
79
|
+
the per-eval workspace was deleted) — so the fallback warns instead of
|
|
80
|
+
happening silently.
|
|
81
|
+
"""
|
|
82
|
+
if workspace is not None and workspace.exists():
|
|
83
|
+
return workspace
|
|
84
|
+
if "with_skill" in str(output_dir) or "without_skill" in str(output_dir):
|
|
85
|
+
guess = output_dir.parent.parent
|
|
86
|
+
else:
|
|
87
|
+
guess = output_dir.parent
|
|
88
|
+
key = str(output_dir)
|
|
89
|
+
if key not in _warned_fallback_dirs:
|
|
90
|
+
_warned_fallback_dirs.add(key)
|
|
91
|
+
console.print(
|
|
92
|
+
f"[yellow]Warning: no live workspace for {output_dir}; file-based checks fall back to "
|
|
93
|
+
f"{guess} (saved artifacts only). Results may differ from the original run.[/yellow]"
|
|
94
|
+
)
|
|
95
|
+
return guess
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# Output dirs we already warned about, so the deterministic and LLM graders
|
|
99
|
+
# don't each repeat the same workspace-fallback warning for one run.
|
|
100
|
+
_warned_fallback_dirs: set[str] = set()
|
|
101
|
+
|
|
15
102
|
|
|
16
103
|
def _is_open_pr_state(state: Optional[str]) -> bool:
|
|
17
104
|
"""True if a PR state string represents an open PR.
|
|
@@ -64,6 +151,16 @@ def _fetch_pr_for_branch(branch: str, source_repo: str) -> Optional[dict]:
|
|
|
64
151
|
return None
|
|
65
152
|
|
|
66
153
|
|
|
154
|
+
def _normalize_assertion_text(text: str) -> str:
|
|
155
|
+
"""Normalize assertion text for matching LLM grader responses.
|
|
156
|
+
|
|
157
|
+
LLMs often echo assertions with minor drift (casing, trailing periods,
|
|
158
|
+
collapsed whitespace); without normalization those would be falsely
|
|
159
|
+
treated as missing results and skipped.
|
|
160
|
+
"""
|
|
161
|
+
return " ".join(text.lower().split()).rstrip(".")
|
|
162
|
+
|
|
163
|
+
|
|
67
164
|
def _load_state(path: Path) -> Optional[GitStateSnapshot]:
|
|
68
165
|
if not path.exists():
|
|
69
166
|
return None
|
|
@@ -143,45 +240,23 @@ class DeterministicGrader:
|
|
|
143
240
|
should_trigger: bool,
|
|
144
241
|
diff: dict,
|
|
145
242
|
) -> AssertionResult:
|
|
146
|
-
|
|
243
|
+
method = classify_assertion(assertion)
|
|
147
244
|
|
|
148
|
-
if "branch"
|
|
149
|
-
"created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
|
|
150
|
-
):
|
|
245
|
+
if method == "git-branch":
|
|
151
246
|
return self._check_git_branch(assertion, diff, should_trigger)
|
|
152
|
-
|
|
153
|
-
if "commit" in assertion_lower and (
|
|
154
|
-
"created" in assertion_lower or "exists" in assertion_lower or "new" in assertion_lower
|
|
155
|
-
):
|
|
247
|
+
if method == "git-commit":
|
|
156
248
|
return self._check_git_commit(assertion, diff, should_trigger)
|
|
157
|
-
|
|
158
|
-
if "push" in assertion_lower and (
|
|
159
|
-
"remote" in assertion_lower or "branch" in assertion_lower or "pushed" in assertion_lower
|
|
160
|
-
):
|
|
249
|
+
if method == "pushed":
|
|
161
250
|
return self._check_pushed(assertion, diff, agent_output, should_trigger)
|
|
162
|
-
|
|
163
|
-
# Word-boundary match: a bare "pr" substring would also match
|
|
164
|
-
# "prominently", "present", "approach", etc.
|
|
165
|
-
if re.search(r"\bprs?\b|\bpull request", assertion_lower):
|
|
251
|
+
if method == "pr-created":
|
|
166
252
|
return self._check_pr_created(assertion, diff, agent_output, should_trigger, output_dir)
|
|
167
|
-
|
|
168
|
-
if (
|
|
169
|
-
"file exists" in assertion_lower
|
|
170
|
-
or ("file" in assertion_lower and "exists" in assertion_lower)
|
|
171
|
-
or ("created" in assertion_lower and any(c in assertion_lower for c in [".", "file"]))
|
|
172
|
-
):
|
|
253
|
+
if method == "file-exists":
|
|
173
254
|
return self._check_file_exists(assertion, output_dir, workspace)
|
|
174
|
-
|
|
175
|
-
if "ran" in assertion_lower and (
|
|
176
|
-
"command" in assertion_lower
|
|
177
|
-
or any(cmd in assertion_lower for cmd in ["npm", "git", "python", "cargo", "go"])
|
|
178
|
-
):
|
|
255
|
+
if method == "command-ran":
|
|
179
256
|
return self._check_command_ran(assertion, output_dir)
|
|
180
|
-
|
|
181
|
-
if "contains" in assertion_lower or "includes" in assertion_lower:
|
|
257
|
+
if method == "content-contains":
|
|
182
258
|
return self._check_content_contains(assertion, agent_output)
|
|
183
|
-
|
|
184
|
-
if "valid json" in assertion_lower:
|
|
259
|
+
if method == "valid-json":
|
|
185
260
|
return self._check_valid_json(assertion, output_dir, agent_output, workspace)
|
|
186
261
|
|
|
187
262
|
return AssertionResult(
|
|
@@ -192,11 +267,7 @@ class DeterministicGrader:
|
|
|
192
267
|
)
|
|
193
268
|
|
|
194
269
|
def _resolve_workspace(self, output_dir: Path, workspace: Path | None) -> Path:
|
|
195
|
-
|
|
196
|
-
return workspace
|
|
197
|
-
if "with_skill" in str(output_dir) or "without_skill" in str(output_dir):
|
|
198
|
-
return output_dir.parent.parent
|
|
199
|
-
return output_dir.parent
|
|
270
|
+
return _resolve_workspace_path(output_dir, workspace)
|
|
200
271
|
|
|
201
272
|
def _invert(self, should_trigger: bool) -> bool:
|
|
202
273
|
"""For negative controls, branch/commit/push/pr assertions are inverted."""
|
|
@@ -629,8 +700,7 @@ class LLMGrader:
|
|
|
629
700
|
if not assertions:
|
|
630
701
|
return []
|
|
631
702
|
|
|
632
|
-
|
|
633
|
-
workspace = output_dir.parent.parent if "with_skill" in str(output_dir) else output_dir.parent
|
|
703
|
+
workspace = _resolve_workspace_path(output_dir, workspace)
|
|
634
704
|
file_listing = self._list_workspace_files(workspace)
|
|
635
705
|
file_contents = self._read_workspace_files(workspace)
|
|
636
706
|
|
|
@@ -679,21 +749,33 @@ Be strict: only mark PASS if there is clear evidence. Quote the output or file c
|
|
|
679
749
|
method="llm",
|
|
680
750
|
)
|
|
681
751
|
)
|
|
682
|
-
returned_texts.add(r["text"])
|
|
683
|
-
for a in assertions
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
752
|
+
returned_texts.add(_normalize_assertion_text(r["text"]))
|
|
753
|
+
missing = [a for a in assertions if _normalize_assertion_text(a) not in returned_texts]
|
|
754
|
+
if missing:
|
|
755
|
+
# The grader dropping an assertion is a grader failure, not an
|
|
756
|
+
# agent failure: skip it so it doesn't drag down the pass rate.
|
|
757
|
+
console.print(
|
|
758
|
+
f"[yellow]Warning: LLM grader returned no result for {len(missing)} assertion(s); "
|
|
759
|
+
f"skipped (not failed)[/yellow]"
|
|
760
|
+
)
|
|
761
|
+
for a in missing:
|
|
762
|
+
results.append(
|
|
763
|
+
AssertionResult(
|
|
764
|
+
text=a,
|
|
765
|
+
passed=False,
|
|
766
|
+
evidence="LLM grader did not return a result for this assertion (skipped, not failed)",
|
|
767
|
+
method="llm",
|
|
768
|
+
skipped=True,
|
|
692
769
|
)
|
|
770
|
+
)
|
|
693
771
|
return results
|
|
694
772
|
except Exception as e:
|
|
695
773
|
# A grader failure is not an agent failure: mark the assertions as
|
|
696
774
|
# skipped so they don't drag down the pass rate.
|
|
775
|
+
console.print(
|
|
776
|
+
f"[yellow]Warning: LLM grading failed ({e}); {len(assertions)} assertion(s) skipped, "
|
|
777
|
+
f"not failed[/yellow]"
|
|
778
|
+
)
|
|
697
779
|
return [
|
|
698
780
|
AssertionResult(
|
|
699
781
|
text=a,
|
|
@@ -122,6 +122,8 @@ class BenchmarkStats(BaseModel):
|
|
|
122
122
|
pass_rate: StatsPair
|
|
123
123
|
time_seconds: StatsPair
|
|
124
124
|
tokens: StatsPair
|
|
125
|
+
# USD cost per run as reported by the agent CLI (see TimingData.cost_usd).
|
|
126
|
+
cost_usd: StatsPair = Field(default_factory=lambda: StatsPair(mean=0.0, stddev=0.0))
|
|
125
127
|
# Fraction of runs where ALL assertions passed (pass@1 estimate).
|
|
126
128
|
full_pass_rate: float = 0.0
|
|
127
129
|
# Fraction of evals where at least one of k runs fully passed.
|
|
@@ -147,3 +149,4 @@ class DeltaStats(BaseModel):
|
|
|
147
149
|
pass_rate: float
|
|
148
150
|
time_seconds: float
|
|
149
151
|
tokens: float
|
|
152
|
+
cost_usd: float = 0.0
|