skillproof 0.1.0__tar.gz

skillproof-0.1.0/.github/workflows/action-selftest.yml

@@ -0,0 +1,31 @@
+# Runs this repo's own action.yml on a mock skill (no API keys), including the
+# PR comment, so changes to the action are tested end to end.
+name: action self-test
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  selftest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - id: skillproof
+        uses: ./
+        with:
+          path: tests/selftest-skill
+          min-lift: "20"
+          artifact-name: skillproof-selftest
+      - name: Check outputs
+        env:
+          VERDICT: ${{ steps.skillproof.outputs.verdict }}
+          LIFT: ${{ steps.skillproof.outputs.lift }}
+        run: |
+          echo "verdict=$VERDICT lift=$LIFT"
+          test "$VERDICT" = "helps"

skillproof-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,35 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: python -m pip install -e ".[dev]"
+      - run: ruff check .
+      - run: ruff format --check .
+      - run: pytest -q
+
+  action-js:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: node --test action/test/comment.test.js

skillproof-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,76 @@
+# Publish a release: push a tag like v0.1.0 and this workflow
+#   1. builds the sdist and wheel and checks the tag matches the package version,
+#   2. publishes them to PyPI with Trusted Publishing (no API token stored here),
+#   3. creates a GitHub Release with the files and moves the major tag (v0) to it,
+#      so `uses: akhilgarg07/skillproof@v0` picks up the new action.
+#
+# One-time setup on PyPI: add a trusted publisher for project "skillproof",
+# owner "Akhilgarg07", repository "skillproof", workflow "release.yml",
+# environment "pypi". See RELEASING.md.
+name: release
+
+on:
+  push:
+    tags: ["v[0-9]+.[0-9]+.[0-9]+"]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: python -m pip install build twine
+      - name: Check the tag matches the package version
+        run: |
+          version=$(python -c "import re; print(re.search(r'__version__ = \"(.+)\"', open('src/skillproof/__init__.py').read()).group(1))")
+          if [ "v$version" != "$GITHUB_REF_NAME" ]; then
+            echo "Tag $GITHUB_REF_NAME does not match __version__ $version in src/skillproof/__init__.py"
+            exit 1
+          fi
+      - run: python -m build
+      - run: python -m twine check --strict dist/*
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/skillproof
+    permissions:
+      id-token: write  # Trusted Publishing
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    needs: publish
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # create the release and move the major tag
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Create the GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release create "$GITHUB_REF_NAME" dist/* --title "$GITHUB_REF_NAME" --generate-notes
+      - name: Move the major tag (e.g. v0) to this release
+        run: |
+          major="${GITHUB_REF_NAME%%.*}"
+          git tag -f "$major" "$GITHUB_REF_NAME"
+          git push -f origin "refs/tags/$major"

skillproof-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.py[cod]
+.venv/
+dist/
+build/
+*.egg-info/
+.pytest_cache/
+.ruff_cache/
+.skillproof/
+.skilltest/

skillproof-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,65 @@
+# Contributing to skillproof
+
+Thanks for helping! skillproof is meant to stay small and easy to read: prefer
+plain code over clever abstractions, and explain *why* in comments when the
+code can't.
+
+## Setup
+
+```bash
+git clone https://github.com/akhilgarg07/skillproof && cd skillproof
+uv venv && uv pip install -e ".[dev]"      # or: python -m venv .venv && pip install -e ".[dev]"
+.venv/bin/pytest
+.venv/bin/ruff check . && .venv/bin/ruff format --check .
+node --test action/test/comment.test.js     # only if you touch action/
+```
+
+Tests never call real APIs or agents. `tests/conftest.py` makes any test that
+tries to start a real Claude Code or Codex session fail. Runner tests use
+recorded transcripts in `tests/fixtures/` and fake executables or clients.
+
+## Where things live
+
+| path | what |
+|---|---|
+| `src/skillproof/cli.py` | the `skillproof` command (typer) |
+| `src/skillproof/config.py` | the `evals.yaml` schema (pydantic) and loader |
+| `src/skillproof/engine.py` | runs trials with/without the skill, sandboxes, fail-fast, saving |
+| `src/skillproof/runners/` | one module per agent, plus `base.py` (the `Runner` interface) |
+| `src/skillproof/parsers/` | turn an agent's transcript into an `AgentRun` (pure functions) |
+| `src/skillproof/assertions/` | assertion types (`builtin.py`, `llm_judge.py`) |
+| `src/skillproof/judges/` | judge models for `llm_judge` |
+| `src/skillproof/stats.py` | pass rates, intervals, lift, verdicts |
+| `src/skillproof/report/` | `summary.py` (all numbers, computed once) and one renderer per format |
+| `action.yml`, `action/` | the GitHub Action |
+| `examples/`, `leaderboard/` | example skills and the leaderboard tooling |
+
+## Adding a runner (e.g. Gemini CLI)
+
+1. Read the agent's **current** docs for its non-interactive mode, JSON output
+   and how it discovers skills. Agent CLIs change often: don't rely on memory.
+2. Create `src/skillproof/runners/<agent>.py` with a `Runner` subclass. Read the
+   rules at the top of `runners/base.py`: only touch the sandbox, keep the user's
+   personal config out of *both* variants, put infrastructure failures in `error`.
+3. Put transcript parsing in `src/skillproof/parsers/<agent>.py` as pure
+   functions, including trigger detection.
+4. Register it in `RUNNERS` in `runners/__init__.py` and give it a sensible
+   entry in `DEFAULT_COST_PER_RUN` (`estimate.py`).
+5. Record one real transcript with and without a skill, scrub paths and IDs, save
+   it under `tests/fixtures/`, and test the parser on it. Test the runner with a
+   fake CLI (see `tests/test_codex.py`) or a fake client.
+6. Add a row to the runners table in the README, including how triggers are detected.
+
+## Adding an assertion type
+
+Add a `BaseAssertion` subclass in `assertions/builtin.py` with a unique
+`type: Literal["..."]` and an async `check()`, add it to the `Assertion` union in
+`assertions/__init__.py`, document it in the README table and in the `init`
+template comments, and add tests to `tests/test_assertions.py`.
+
+## Pull requests
+
+- Keep each PR to one change, with tests.
+- Run ruff and pytest locally; CI runs them on Python 3.10–3.13.
+- If you change `summary.json` fields, keep old ones (it's a contract for CI users)
+  or bump `SCHEMA_VERSION` in `report/summary.py`.

skillproof-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Akhil Garg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

skillproof-0.1.0/PKG-INFO

@@ -0,0 +1,262 @@
+Metadata-Version: 2.5
+Name: skillproof
+Version: 0.1.0
+Summary: pytest for agent skills: prove your skill actually works.
+Project-URL: Homepage, https://github.com/akhilgarg07/skillproof
+Project-URL: Issues, https://github.com/akhilgarg07/skillproof/issues
+Project-URL: Source, https://github.com/akhilgarg07/skillproof
+Author: Akhil Garg
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent-skills,claude-code,codex,evals,llm,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: anthropic>=1.15
+Requires-Dist: claude-agent-sdk>=0.2.168
+Requires-Dist: openai>=3.0
+Requires-Dist: pydantic>=2.6
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# skillproof
+
+**pytest for agent skills: prove your skill actually works.**
+
+[![ci](https://github.com/akhilgarg07/skillproof/actions/workflows/ci.yml/badge.svg)](https://github.com/akhilgarg07/skillproof/actions/workflows/ci.yml)
+[![license: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/akhilgarg07/skillproof/blob/main/LICENSE)
+
+![skillproof demo](https://raw.githubusercontent.com/akhilgarg07/skillproof/main/docs/demo.gif)
+<!-- Record with: vhs docs/demo.tape -->
+
+```bash
+pip install skillproof
+```
+
+**Works with:** Claude Code · Codex · any OpenAI-compatible endpoint
+
+---
+
+Agent skills (`SKILL.md` folders for Claude Code, Codex, Cursor, Gemini CLI,
+OpenCode...) are everywhere, but almost nobody can show that a skill helps. Often
+it isn't even loaded: the agent never decides to use it. **skillproof** runs your
+tasks N times **with** the skill installed and N times **without** it, then tells
+you:
+
+- **Trigger rate**: did the agent actually load the skill?
+- **Lift**: pass rate with the skill minus pass rate without it, with a 95% confidence interval
+- **Cost**: extra (or saved) tokens, dollars and seconds per run
+- **A verdict**: *helps*, *hurts*, *no clear effect*, or *too few runs* to say
+
+## 60-second quickstart
+
+```bash
+pip install skillproof
+export ANTHROPIC_API_KEY=sk-ant-...      # or use your `claude` login
+
+skillproof init ./my-skill                # writes my-skill/evals.yaml
+# edit evals.yaml: 2-3 real tasks, how to check them, one task the skill should ignore
+skillproof run ./my-skill                 # prints the plan and cost estimate, then runs
+```
+
+`uvx skillproof run ./my-skill` works too, without installing anything.
+
+## Example output
+
+This is a real run of the [`release-notes`](https://github.com/akhilgarg07/skillproof/blob/main/examples/release-notes/) example
+(Claude Code, 5 trials per variant):
+
+```
+acme-release-notes · runner claude-code · model claude-opus-5-5
+
+ case                             ┃        triggered ┃ pass with ┃ pass without ┃  lift ┃ tokens Δ ┃ latency Δ
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━
+ release notes from merged PRs    │              5/5 │       5/5 │          0/5 │ +100% │      +7% │       −3%
+ release with no breaking changes │              5/5 │       5/5 │          0/5 │ +100% │      +7% │       +3%
+ unrelated question               │ 0/5 (should not) │       4/5 │          5/5 │  −20% │      ±0% │       −2%
+
+skill lift: +100% (95% CI +61% to +100%)  |  triggers 10/10  |  false triggers 0/5  |  tokens +7%  |  cost +10%
+verdict: the skill helps (95% CI for lift: +61% to +100%)
+• Small sample: as few as 5 completed run(s) per variant. Treat these numbers as rough; use --trials 10 or more
+  before quoting them.
+```
+
+Every run also saves a self-contained **HTML report** (with each run's checks and
+answer), `report.md`, `summary.json` and the full transcripts under
+`.skillproof/runs/<timestamp>/`.
+
+[`examples/`](https://github.com/akhilgarg07/skillproof/blob/main/examples/) has skills that **help**, **do nothing** and **hurt**, so
+you can see all three outcomes.
+
+## How it works
+
+```
+for each case in evals.yaml, N trials, interleaved:
+    with skill:     fresh temp dir + fixtures + skill installed  ->  agent  ->  checks
+    without skill:  fresh temp dir + fixtures, no skill          ->  agent  ->  checks
+```
+
+- Every run gets a **fresh temp directory** with your fixtures copied in. Your
+  project is never touched.
+- Your **personal** skills, settings and instructions (`~/.claude`, `~/.codex`,
+  `~/.agents`) are not loaded for either variant, so the only difference is the skill.
+- The agent can't see your `evals.yaml` or its checks.
+- Transcripts, token usage and tool calls are saved for every run, with API keys redacted.
+
+## evals.yaml
+
+```yaml
+skill: ./                 # folder containing SKILL.md
+runner: claude-code       # claude-code | codex | openai-compatible
+# model: claude-sonnet-5-5  # optional; the agent's default otherwise
+trials: 5                 # runs per case, per variant
+timeout: 300              # seconds per run
+
+cases:
+  - name: "summarizes stack trace"
+    prompt: "Here is an error log in error.log. What's wrong?"  # don't name the skill
+    fixtures: ./fixtures/case1/     # copied into each run's temp dir
+    assert:
+      - type: contains
+        value: "NullPointerException"
+      - type: file_exists
+        path: fix.patch
+      - type: command               # runs in the temp dir; passes on exit code 0
+        run: "python -m pytest -q"
+      - type: max_tokens
+        value: 20000
+      - type: llm_judge             # optional; graded by a model from another vendor
+        rubric: "States the root cause in the first sentence."
+
+  - name: "unrelated question"      # catches skills that load when they shouldn't
+    prompt: "What's the capital of France?"
+    should_trigger: false
+    assert:
+      - {type: contains, value: "Paris"}
+```
+
+| assertion | passes when |
+|---|---|
+| `contains` / `not_contains` | the final answer contains / doesn't contain `value` (case-insensitive by default) |
+| `regex` | the final answer matches `pattern` |
+| `file_exists` / `file_contains` | a file in the temp dir exists / contains `value` |
+| `command` | `run` exits 0 in the temp dir (API keys are removed from its environment) |
+| `max_tokens` | the run used at most `value` tokens (input + output + cache) |
+| `llm_judge` | a judge model says the answer meets `rubric` |
+
+Use deterministic checks first. `llm_judge` defaults to a judge from a **different
+vendor** than the agent (Claude agents are graded by an OpenAI model and vice
+versa), so no model grades its own family. Change it with
+`judge: {provider: anthropic, model: ...}`.
+
+## Runners
+
+| runner | how it runs | where the skill goes | how a trigger is detected |
+|---|---|---|---|
+| `claude-code` | [Claude Agent SDK](https://code.claude.com/docs/en/agent-sdk/overview) | `<temp>/.claude/skills/<name>/` | a `Skill` tool call naming the skill |
+| `codex` | `codex exec --json` | `<temp>/.agents/skills/<name>/` | a shell command that reads the skill's folder |
+| `openai-compatible` | a small tool-using loop over Chat Completions | SKILL.md pasted into the system prompt | not measured (**approximate**) |
+
+Keys come from environment variables only: `ANTHROPIC_API_KEY` (or your `claude`
+login), `CODEX_API_KEY` / `OPENAI_API_KEY` (or `codex login`). Runner settings go
+under `runner_options:`; see each runner's module docstring.
+
+## Reports, CI and badges
+
+```bash
+skillproof run ./my-skill --format md -o report.md    # also: html, json (to stdout)
+skillproof report                                      # re-render the latest run
+skillproof run ./my-skill --min-lift 0 --min-trigger-rate 80   # exit 1 if not met
+skillproof run ./my-skill --dry-run                    # plan + cost estimate only
+```
+
+**Cost guard:** before every run skillproof prints the number of runs and an
+estimate based on your previous runs. Above $5 it asks first; in CI it refuses
+unless you pass `--yes`. If the first runs all fail (bad key, wrong model), it
+stops and tells you what to fix.
+
+**GitHub Action:** comments the results on pull requests and keeps the comment up to date.
+
+```yaml
+# .github/workflows/skillproof.yml (full example: action/example-workflow.yml)
+on: pull_request
+permissions: {contents: read, pull-requests: write}
+jobs:
+  skillproof:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: akhilgarg07/skillproof@v0
+        with: {path: my-skill, trials: "10", min-lift: "0"}
+        env: {ANTHROPIC_API_KEY: "${{ secrets.ANTHROPIC_API_KEY }}"}
+```
+
+**Badge for your skill's README:**
+
+```bash
+skillproof badge ./my-skill
+# [![skillproof: +100% lift · 10/10 triggered](https://img.shields.io/badge/skillproof-...-brightgreen)](...)
+```
+
+It is green when the skill helps, red when it hurts and grey when the result is
+not significant or comes from fewer than 5 runs per variant. `--json` prints a
+[shields.io endpoint](https://shields.io/badges/endpoint-badge) file instead, so CI
+can keep a badge up to date.
+
+## Honest numbers: what this can and can't tell you
+
+- **It measures your tasks, not the skill in general.** Lift depends on the prompts
+  and checks you write. Publish your `evals.yaml` next to any number you share.
+- **Small samples are noisy.** Every rate comes with a 95% interval (Wilson for pass
+  rates, Newcombe for the lift). Below 5 runs per variant there's no verdict at all,
+  and below 10 the report says the numbers are rough.
+- **Results depend on the agent and model.** Reports record the model that actually
+  ran. In our runs, Claude Code read a library's source, noticed that a skill's docs
+  were stale and ignored them ([`examples/stale-docs`](https://github.com/akhilgarg07/skillproof/blob/main/examples/stale-docs/)); a
+  weaker model may not.
+- **Trigger detection is per agent and heuristic.** Claude Code has an explicit
+  `Skill` tool call. Codex reads skills with ordinary shell commands, so a "trigger"
+  there means a command touched the skill's folder.
+- **The `openai-compatible` runner is an approximation.** It pastes SKILL.md into the
+  system prompt instead of letting the agent discover and load it, so it can't
+  measure trigger rate and overstates how often a skill is "used". Reports label it.
+- **Token deltas include cache reads,** which are cheap. Look at the cost delta when
+  the agent reports cost (Claude Code does).
+- **`llm_judge` is a model's opinion.** It's cross-vendor by default, but still noisy;
+  prefer deterministic checks.
+- **Agents run real commands.** Runs happen in temp directories, but agents run
+  without permission prompts, so a command could still reach other files on your
+  machine. Test untrusted skills in a container or VM.
+
+## Leaderboard
+
+[`leaderboard/`](https://github.com/akhilgarg07/skillproof/blob/main/leaderboard/) runs skillproof over a list of public skills (using
+evals you write, never the repo's own) and renders a markdown leaderboard.
+
+## Contributing
+
+Contributions are very welcome; see [CONTRIBUTING.md](https://github.com/akhilgarg07/skillproof/blob/main/CONTRIBUTING.md). Good first issues:
+
+- **New runners:** Gemini CLI, Cursor (cursor-agent), OpenCode, Aider
+- **New assertions:** `tool_called` (the agent used a given tool), `json_schema`,
+  `file_matches` (regex on a file), `max_cost_usd`
+- **`skillproof compare`:** two versions of a skill head to head
+- **Windows support** and a Docker sandbox option for untrusted skills
+- **A local judge:** `llm_judge` over Chat Completions, for local models (Ollama, vLLM)
+
+## License
+
+MIT