agent-skill-eval 0.2.1__tar.gz

agent_skill_eval-0.2.1/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,38 @@
+name: Bug Report
+description: Report a bug or unexpected behavior
+labels: ["bug"]
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: A clear description of the bug
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce
+      description: Minimal steps to reproduce the behavior
+      placeholder: |
+        1. skill-eval run --skill ... --evals ... --agent opencode
+        2. ...
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: What you expected to happen
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: Output of `skill-eval --version`
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+      description: Output of `python --version`

agent_skill_eval-0.2.1/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,23 @@
+name: Feature Request
+description: Suggest a new feature or improvement
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: Problem
+      description: What problem does this feature solve?
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+      description: How would this feature work?
+    validations:
+      required: true
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives considered
+      description: Any alternative approaches you've considered

agent_skill_eval-0.2.1/.github/actions/setup/action.yml

@@ -0,0 +1,19 @@
+name: Setup Python and install dependencies
+description: Shared setup for lint and test jobs
+
+inputs:
+  python-version:
+    description: Python version to use
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: 'pip'
+
+    - name: Install dependencies
+      shell: bash
+      run: pip install -e ".[dev]"

agent_skill_eval-0.2.1/.github/workflows/ci.yml

@@ -0,0 +1,62 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: ruff check src/ tests/
+      - run: ruff format --check src/ tests/
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pytest -v --junitxml=results.xml --cov=skill_eval --cov-report=xml:coverage.xml
+      - name: Upload test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.python-version }}
+          path: results.xml
+      - name: Upload coverage
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.python-version }}
+          path: coverage.xml
+
+  smoke:
+    runs-on: ubuntu-latest
+    needs: [lint, test]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+        with:
+          python-version: "3.12"
+      - run: pytest -v tests/test_cli_smoke.py

agent_skill_eval-0.2.1/.github/workflows/publish.yml

@@ -0,0 +1,50 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Check tag matches package version
+        run: |
+          version=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
+          tag="${GITHUB_REF_NAME#v}"
+          if [ "$tag" != "$version" ]; then
+            echo "Tag v$tag does not match pyproject.toml version $version" >&2
+            exit 1
+          fi
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install --upgrade build
+          python -m build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/skill-eval
+    permissions:
+      # Required for PyPI trusted publishing (OIDC).
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

agent_skill_eval-0.2.1/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+*.egg-info/
+dist/
+build/
+eval-workspace/
+.env
+uv.lock
+.coverage

agent_skill_eval-0.2.1/AGENTS.md

@@ -0,0 +1,66 @@
+# AGENTS.md
+
+## Project
+
+Python CLI framework (`skill-eval`) for evaluating agent skills across OpenCode, Claude Code, and Codex. Uses Typer, Pydantic, OpenAI SDK, and Rich.
+
+## Quick commands
+
+```bash
+# Install (editable + dev deps)
+uv pip install -e ".[dev]"
+
+# Run tests
+uv run --extra dev pytest -q
+
+# Lint + format check (CI runs both)
+uv run --extra dev ruff check src/ tests/
+uv run --extra dev ruff format --check src/ tests/
+
+# Run all checks before committing
+uv run --extra dev ruff check src/ tests/ && uv run --extra dev ruff format --check src/ tests/ && uv run --extra dev pytest -q
+```
+
+## Architecture
+
+```
+src/skill_eval/
+├── cli.py            # Typer CLI app (run, report, grade, cleanup, init)
+├── runner.py         # EvalRunner orchestrates eval execution
+├── graders/__init__.py   # DeterministicGrader + LLMGrader (700+ lines, single file)
+├── harnesses/__init__.py # Agent harnesses (opencode, claude-code, codex, fake)
+├── models.py         # Pydantic models for eval config
+├── git_state.py      # Pre/post git/PR state snapshots
+├── skills.py         # Skill installer
+└── workspace.py      # Workspace creation (fresh git init or clone)
+```
+
+- Entry point: `skill_eval.cli:app` (registered as `skill-eval` console script)
+- Package layout: `src/skill_eval/` (hatchling build, wheel packages `src/skill_eval`)
+- The `fake` harness (used in tests) is a no-op agent that produces deterministic output
+
+## Testing
+
+- Test files mirror source: `test_runner.py`, `test_graders.py`, `test_harnesses.py`, etc.
+- Fixtures live in `tests/fixtures/` — skills and evals JSON used by smoke tests
+- `test_cli_smoke.py` is the integration test: runs full `run -> report -> init` cycle with `--agent fake`
+- Tests use `tmp_path` for workspace isolation; no shared state
+- Shared fixtures in `tests/conftest.py` (`_init_git_workspace`, `grader`, `evals_path`)
+
+## Conventions
+
+- Python 3.11+ required (`requires-python = ">=3.11"`)
+- Ruff config: `line-length = 120`, rules `["E", "F", "I"]`
+- All imports use `from __future__ import annotations`
+- Pydantic v2 models throughout
+- CLI uses Typer with `CliRunner` for testing
+- `.env` file loaded via `python-dotenv` (not committed)
+
+## Key gotchas
+
+- Default grader model `deepseek/deepseek-v4-flash` is a valid OpenRouter model. Override with `--grader-model` or set a real model.
+- `graders/__init__.py` is a monolith (700+ lines) containing both deterministic and LLM graders
+- `harnesses/__init__.py` has a dead `cost_usd` read at line ~140 (no-op code)
+- `compute_benchmark` returns delta=0.0 for multi-agent runs (known bug)
+- Workspace paths must contain literal `with_skill`/`without_skill` substrings or grading breaks silently
+- `uv.lock` is gitignored — not tracked in version control

agent_skill_eval-0.2.1/CHANGELOG.md

@@ -0,0 +1,83 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## [0.2.0] - 2026-06-10
+
+### Added
+- `write-release-notes` example skill + eval suite: exercises LLM rubric grading (grouping, breaking-change prominence, anti-fabrication) instead of git-state checks
+- `--runs N` repeats every (eval, agent, config); `benchmark.json` now reports `full_pass_rate`, `pass_at_k`, and `k`
+- `--agent-model agent=model` (repeatable) and `--harness-base-url` to pin models and endpoints per agent CLI
+- `--timeout` / `--retries` flags; harnesses retry on timeout or non-zero exit with backoff and record `exit_code`, `timed_out`, `retries` in `timing.json`
+- `skill-eval validate`: schema + fixture-existence + duplicate-id checks for evals.json
+- `skill-eval list`: discover eval suites and skills under a directory
+- `skill-eval compare`: side-by-side pass rates of two iterations
+- `report --format markdown` and `report --show-evidence` (failed/skipped assertion evidence)
+- JSON Schema for the eval suite format at `schemas/evals.schema.json` (with a sync test)
+- SKILL.md frontmatter validation warning on `run` (missing name/description, name mismatch)
+- Richer `init` scaffold: SKILL.md frontmatter template + negative-control eval case
+- PyPI trusted-publishing workflow on version tags (`.github/workflows/publish.yml`)
+- Demo recording script (`scripts/record-demo.sh`)
+
+### Changed
+- Assertions that cannot be graded (no API key, LLM grader error) are now **skipped** and excluded from the pass rate instead of counted as failures; `grading.json` gains `method` and `skipped` fields
+- `run` warns upfront when no grader API key is set; `grade` warns that LLM verdicts may flip on re-grade
+- README restructured around the full-harness pitch, with a documented table of recognized deterministic assertion patterns
+
+### Fixed
+- Multi-agent runs no longer report a misleading `Delta: 0.0%`: `benchmark.json` now has per-agent `deltas`; the legacy `delta` field is only set for single-agent runs (otherwise `null`)
+- Removed dead `cost_usd` handling in the Claude Code output parser
+
+## [0.1.0-alpha.3] - 2026-06-07
+
+### Added
+- Shared test fixtures in `conftest.py`
+- `test_llm_grader.py`: 18 tests covering client, prompts, JSON parsing, errors
+- `test_fuzz_grader.py` rewrite: real handler tests instead of mocked dispatch
+- `test_compute_benchmark` with xfail for known multi-agent delta=0 bug
+- EvalCase validation tests (valid, negative control, contradiction)
+- `SkillInstaller.uninstall` tests, `WorkspaceManager._clone_repo` test
+- `github_repo_slug` tests for URLs without `.git` suffix
+
+### Changed
+- CI uses setup-python's built-in cache instead of separate `actions/cache`
+
+### Fixed
+- EvalCase contradiction validator (`force_skill_invocation` + `should_trigger`)
+- LLMGrader now fails on missing assertions instead of silently dropping them
+
+## [0.1.0-alpha.2] - 2026-06-02
+
+### Added
+- CI workflow with Python 3.11/3.12/3.13 matrix, caching, coverage, and smoke test
+- CLI smoke test (`test_cli_smoke.py`) for full run -> report -> init cycle
+
+### Changed
+- Tightened evals with state-delta grading and scoped cleanup
+- Improved `commit-push-pr` skill to 100% pass rate
+
+### Fixed
+- `github_repo_slug` function to parse GitHub URLs correctly
+- Handoff review fixes and PR grading tightening
+
+## [0.1.0-alpha.1] - 2026-05-27
+
+### Added
+- Initial release: skill evaluation framework for OpenCode, Claude Code, and Codex
+- `run`, `report`, `grade`, `cleanup`, and `init` CLI commands
+- Multi-agent support with baseline (with-skill / without-skill) comparison
+- State-delta grading using pre/post git snapshots (`pre_state.json` / `post_state.json`)
+- Deterministic assertions: branch, commit, push, PR, file existence, content matching
+- LLM rubric grading via configurable model (`--grader-model`)
+- Negative-control support (`should_trigger: false`) to catch accidental skill triggering
+- Scoped cleanup via `cleanup.json` manifest (never closes unrelated PRs)
+- `commit-push-pr` example skill with 5 eval cases
+- Lazy-initialized OpenAI client with API key validation
+- OpenRouter grader support via `OPENROUTER_API_KEY`
+- Workspace isolation with fresh git repos per eval run
+- Token/timing extraction from each agent's native output format
+- `python -m skill_eval` entry point
+
+### Fixed
+- OpenRouter grader model resolution and opencode parser output handling
+- Workspace-aware grading and dotenv loading

agent_skill_eval-0.2.1/CODE_OF_CONDUCT.md

@@ -0,0 +1,39 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment:
+
+- Using welcoming and inclusive language
+- Being respectful of differing viewpoints and experiences
+- Gracefully accepting constructive criticism
+- Focusing on what is best for the community
+- Showing empathy towards other community members
+
+Examples of unacceptable behavior:
+
+- The use of sexualized language or imagery and unwelcome sexual attention or advances
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information without explicit permission
+- Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the project maintainers. All complaints will be reviewed and
+investigated and will result in a response that is deemed necessary and
+appropriate to the circumstances.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 2.1.

agent_skill_eval-0.2.1/CONTRIBUTING.md

@@ -0,0 +1,42 @@
+# Contributing
+
+## Setup
+
+```bash
+git clone https://github.com/tardigrde/skill-evals.git
+cd skill-evals
+uv venv
+uv pip install -e ".[dev]"
+```
+
+## Running checks
+
+```bash
+uv run --extra dev pytest -q
+uv run --extra dev ruff check src/ tests/
+uv run --extra dev ruff format --check src/ tests/
+```
+
+Run all three before committing:
+
+```bash
+uv run --extra dev ruff check src/ tests/ && uv run --extra dev ruff format --check src/ tests/ && uv run --extra dev pytest -q
+```
+
+## Code style
+
+- Python 3.11+ required
+- Ruff for linting and formatting (`line-length = 120`)
+- All imports use `from __future__ import annotations`
+- Pydantic v2 models throughout
+
+## Pull requests
+
+1. Fork the repo and create a feature branch.
+2. Add tests for new functionality.
+3. Ensure all checks pass.
+4. Open a PR with a clear description of the change.
+
+## Issues
+
+Use the GitHub issue templates to report bugs or request features.

agent_skill_eval-0.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 skill-eval contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.