proofrag 0.3.0__tar.gz

proofrag-0.3.0/.claude-plugin/marketplace.json

@@ -0,0 +1,20 @@
+{
+  "name": "proofrag",
+  "owner": { "name": "Ansh Dawda", "url": "https://github.com/unshDee" },
+  "metadata": {
+    "description": "RAG/LLM evaluation skill — golden sets, LLM-as-judge, scorecards.",
+    "version": "0.1.0"
+  },
+  "plugins": [
+    {
+      "name": "proofrag",
+      "source": "./",
+      "description": "Evaluate a RAG/LLM app: golden set from your docs + LLM-as-judge + retrieval metrics + shareable scorecard + CI gate.",
+      "version": "0.1.0",
+      "author": { "name": "Ansh Dawda" },
+      "homepage": "https://github.com/unshDee/proofrag",
+      "license": "MIT",
+      "keywords": ["rag", "llm", "evaluation", "llm-as-judge", "skill"]
+    }
+  ]
+}

proofrag-0.3.0/.claude-plugin/plugin.json

@@ -0,0 +1,10 @@
+{
+  "name": "proofrag",
+  "version": "0.1.0",
+  "description": "Evaluate a RAG/LLM app: generate a golden set from your docs, run LLM-as-judge + retrieval metrics, and produce a shareable HTML scorecard with a CI gate.",
+  "author": { "name": "Ansh Dawda", "email": "ansh.dawda@gmail.com" },
+  "homepage": "https://github.com/unshDee/proofrag",
+  "repository": "https://github.com/unshDee/proofrag",
+  "license": "MIT",
+  "keywords": ["rag", "llm", "evaluation", "llm-as-judge", "retrieval", "skill"]
+}

proofrag-0.3.0/.env.example

@@ -0,0 +1,27 @@
+# Copy this file to `.env` and fill in your key:  cp .env.example .env
+# `.env` is gitignored — never commit real keys.
+#
+# proofrag does not auto-load .env. Load it before running, e.g.:
+#   set -a && source .env && set +a
+# (or just `export` the vars yourself).
+
+# --- Backend (pick ONE) ------------------------------------------------------
+
+# Anthropic — the default backend (cheap Haiku judge).
+ANTHROPIC_API_KEY=
+
+# OpenAI-compatible — also covers local servers (Ollama, vLLM, LM Studio)
+# via OPENAI_BASE_URL. Needed for `evaluate --semantic` (embeddings).
+# OPENAI_API_KEY=
+# OPENAI_BASE_URL=http://localhost:11434/v1
+
+# --- Optional overrides ------------------------------------------------------
+
+# Force a provider instead of auto-detecting from the keys above.
+# PROOFRAG_PROVIDER=anthropic        # or: openai
+
+# Judge & generator model (defaults: Haiku for Anthropic, gpt-4o-mini for OpenAI).
+# PROOFRAG_MODEL=
+
+# Embedding model used by `evaluate --semantic`.
+# PROOFRAG_EMBED_MODEL=text-embedding-3-small

proofrag-0.3.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,25 @@
+---
+name: Bug report
+about: Something isn't working as expected
+title: "bug: "
+labels: bug
+---
+
+**What happened**
+<!-- A clear description of the bug. -->
+
+**Steps to reproduce**
+1.
+2.
+
+**Expected**
+<!-- What you expected instead. -->
+
+**Environment**
+- proofrag version: <!-- `proofrag --version` -->
+- Python:
+- Backend: <!-- anthropic / openai / local -->
+- OS:
+
+**Logs / scorecard**
+<!-- Paste error output, or attach the scorecard HTML/JSON if relevant. -->

proofrag-0.3.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: true

proofrag-0.3.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,15 @@
+---
+name: Feature request
+about: Suggest a capability or improvement
+title: "feat: "
+labels: enhancement
+---
+
+**Problem**
+<!-- What are you trying to do that proofrag doesn't support today? -->
+
+**Proposed solution**
+<!-- What would the command / metric / output look like? -->
+
+**Alternatives considered**
+<!-- Other tools or approaches, and why they fall short. -->

proofrag-0.3.0/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,17 @@
+## Summary
+
+<!-- What does this PR do, and why? -->
+
+## Type
+
+- [ ] feat — new capability
+- [ ] fix — bug fix
+- [ ] docs — documentation only
+- [ ] chore / refactor / test
+
+## Checklist
+
+- [ ] `make lint` passes
+- [ ] `make test` passes
+- [ ] Updated `CHANGELOG.md` under `## [Unreleased]` (if user-facing)
+- [ ] Updated docs / `SKILL.md` if behavior or commands changed

proofrag-0.3.0/.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # uv-dynamic-versioning needs tags/history
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install
+        run: uv sync --all-extras
+      - name: Lint
+        run: uv run python devtools/lint.py
+      - name: Test
+        run: uv run pytest
+      - name: Demo scorecard renders without an API key
+        run: uv run proofrag demo --out /tmp/scorecard.html

proofrag-0.3.0/.github/workflows/publish.yml

@@ -0,0 +1,26 @@
+name: Publish
+
+# Publishes proofrag to PyPI when a GitHub Release is published.
+# Uses PyPI Trusted Publishing (OIDC) — configure the publisher once at
+# https://pypi.org/manage/project/proofrag/settings/publishing/ (no token needed).
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for trusted publishing
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # uv-dynamic-versioning derives the version from tags
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Build
+        run: uv build
+      - name: Publish to PyPI
+        run: uv publish

proofrag-0.3.0/.gitignore

@@ -0,0 +1,19 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+build/
+dist/
+.venv/
+venv/
+.pytest_cache/
+.DS_Store
+# eval artifacts (commit goldenset.jsonl deliberately, ignore the rest)
+results.json
+predictions.jsonl
+scorecard.html
+!docs/scorecard.png
+# secrets — never commit
+.env
+.env.*
+!.env.example
+.venv/

proofrag-0.3.0/.python-version

@@ -0,0 +1 @@
+3.12

proofrag-0.3.0/AGENTS.md

@@ -0,0 +1,36 @@
+# Agents
+
+This repo ships **proofrag** as a portable [Agent Skill](https://agentskills.io):
+`skills/proofrag/SKILL.md`. The skill is the interface; the `proofrag` Python CLI
+(`src/proofrag/`) is the engine it drives.
+
+## Use it as a skill
+
+**Claude Code (plugin):**
+```
+/plugin marketplace add unshDee/proofrag
+/plugin install proofrag@proofrag
+```
+Then just ask: *"evaluate my RAG"* — Claude auto-loads the skill. Or type `/proofrag`.
+
+**Claude Code (manual):** copy the skill folder where Claude discovers skills:
+```
+cp -r skills/proofrag ~/.claude/skills/        # personal
+cp -r skills/proofrag .claude/skills/          # this project only
+```
+
+**Codex / other agents (open standard):** drop the skill into your agent's skills
+directory (e.g. `.agents/skills/` or your tool's equivalent):
+```
+cp -r skills/proofrag .agents/skills/
+```
+
+## Install the engine
+
+The skill calls the `proofrag` CLI. Install it once, or run ad-hoc with `uvx`:
+```
+uv tool install "proofrag[anthropic]"     # or: pipx install "proofrag[anthropic]"
+uvx "proofrag[anthropic]" demo            # no install
+```
+Set `ANTHROPIC_API_KEY` (default Haiku) or `OPENAI_API_KEY` (`OPENAI_BASE_URL` for
+local/Ollama). No key needed for `proofrag demo`.

proofrag-0.3.0/CHANGELOG.md

@@ -0,0 +1,45 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based on
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres
+to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+- `proofrag diff` — compare a run against a committed baseline results.json and
+  fail on regression (per-metric delta table, `--tolerance`, refuses to compare
+  across different judge models unless `--allow-judge-mismatch`).
+- Reusable composite GitHub Action (`action.yml`): `uses: unshDee/proofrag@v0`
+  installs the CLI, evaluates, writes the scorecard, and gates on the absolute
+  floor and/or the baseline. Example workflow in `examples/ci/`.
+
+## [0.2.0] - 2026-05-31
+
+### Added
+- Rank-aware retrieval metrics: Recall@k, Precision@k, NDCG@k, MRR, with a
+  pluggable relevance matcher (`metrics.py`).
+- Optional embedding-based semantic matcher (`embeddings.py`); `evaluate --semantic`
+  and `--k` to set the cutoff.
+- Scorecard split into Generation and Retrieval panels with an NDCG@k headline.
+- Animated demo GIF of the full eval loop (`docs/demo.gif`, reproducible via
+  `docs/demo.tape`).
+- Installable as a Claude Code plugin: `.claude-plugin/` manifests, `/proofrag`
+  slash command, `AGENTS.md`, and skill-discovery layout under `skills/proofrag/`.
+
+### Changed
+- Unanswerable cases skip retrieval scoring so they don't skew the averages.
+
+## [0.1.0] - 2026-05-31
+
+### Added
+- Golden-set generator from a corpus, with single-doc / multi-doc / unanswerable
+  difficulty tiers.
+- LLM-as-judge scoring (groundedness, correctness, completeness, citation quality),
+  pinned and fingerprinted.
+- Self-contained, shareable HTML scorecard, plus a keyless `demo` command.
+- `--fail-under` CI gate; provider-agnostic backend (Anthropic / OpenAI / local).
+
+[Unreleased]: https://github.com/unshDee/proofrag/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/unshDee/proofrag/compare/v0.1.0...v0.2.0
+[0.1.0]: https://github.com/unshDee/proofrag/releases/tag/v0.1.0

proofrag-0.3.0/CONTRIBUTING.md

@@ -0,0 +1,61 @@
+# Contributing to proofrag
+
+Thanks for considering a contribution! proofrag is an Agent Skill + Python CLI for
+evaluating RAG/LLM apps. This guide covers the dev setup and the workflow.
+
+## Dev setup
+
+Uses [uv](https://docs.astral.sh/uv/).
+
+```bash
+git clone https://github.com/unshDee/proofrag && cd proofrag
+uv sync --all-extras          # installs the package + both backends + dev tools
+```
+
+Run the checks (CI runs exactly these):
+
+```bash
+make test     # or: uv run pytest
+make lint     # or: uv run python devtools/lint.py   (ruff + codespell + basedpyright)
+```
+
+No API key needed for tests — they're fully offline. For a live end-to-end run, copy
+the env template and add a key, then load it:
+
+```bash
+cp .env.example .env          # then put your key in .env
+set -a && source .env && set +a
+```
+
+`.env` is gitignored; never commit real keys.
+
+## Workflow (GitHub Flow)
+
+`main` is always green and releasable. All changes land via pull request.
+
+1. Branch off `main`. Name it by type:
+   - `feat/<short-name>` — new capability
+   - `fix/<short-name>` — bug fix
+   - `docs/<short-name>` — docs only
+   - `chore/<short-name>` — tooling, deps, CI, refactors
+2. Make focused commits using [Conventional Commits](https://www.conventionalcommits.org/):
+   `feat: …`, `fix: …`, `docs: …`, `chore: …`, `refactor: …`, `test: …`.
+3. Keep the change scoped — one logical thing per PR.
+4. Make sure `make lint` and `make test` pass locally.
+5. Open a PR into `main`. CI (lint + tests on Python 3.11–3.13) must pass.
+6. PRs are **squash-merged** — your PR becomes one clean commit on `main`.
+7. Note user-facing changes under `## [Unreleased]` in [CHANGELOG.md](CHANGELOG.md).
+
+## Project layout
+
+- `skills/proofrag/SKILL.md` — the Agent Skill (the interface agents load)
+- `src/proofrag/` — the engine: `corpus`, `goldenset`, `judge`, `metrics`,
+  `embeddings`, `scorecard`, `llm`, `cli`
+- `examples/docs-rag/` — a runnable end-to-end example
+- `.claude-plugin/` — plugin + marketplace manifests
+- `tests/` — offline smoke tests
+
+## Releases
+
+Maintainer cuts a [SemVer](https://semver.org/) tag and a GitHub Release from `main`;
+that triggers the PyPI publish workflow. Versions are derived from git tags.

proofrag-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ansh Dawda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

proofrag-0.3.0/Makefile

@@ -0,0 +1,22 @@
+.PHONY: default install lint test build clean
+
+default: install lint test
+
+install:
+	uv sync --all-extras
+
+lint:
+	uv run python devtools/lint.py
+
+test:
+	uv run pytest
+
+build:
+	uv build
+
+upgrade:
+	uv sync --upgrade --all-extras
+
+clean:
+	-rm -rf dist/ build/ *.egg-info/ .pytest_cache/ .ruff_cache/ .coverage htmlcov/
+	-find . -type d -name __pycache__ -exec rm -rf {} +

proofrag-0.3.0/PKG-INFO

@@ -0,0 +1,183 @@
+Metadata-Version: 2.5
+Name: proofrag
+Version: 0.3.0
+Summary: Point your agent at your docs and your RAG app; get a golden test set + an LLM-as-judge & retrieval scorecard, in one command.
+Project-URL: Repository, https://github.com/unshDee/proofrag
+Project-URL: Issues, https://github.com/unshDee/proofrag/issues
+Author-email: Ansh Dawda <ansh.dawda@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: agent-skills,claude,codex,evaluation,llm,llm-as-judge,rag,retrieval
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: <4.0,>=3.11
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.40; extra == 'anthropic'
+Provides-Extra: openai
+Requires-Dist: openai>=1.40; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# proofrag
+
+[![CI](https://github.com/unshDee/proofrag/actions/workflows/ci.yml/badge.svg)](https://github.com/unshDee/proofrag/actions/workflows/ci.yml)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+**Point your agent at your docs and your RAG app. Get a golden test set, an
+LLM-as-judge + retrieval scorecard, and a CI gate — in one command.**
+
+Evaluation is the #1 unmet pain in production RAG/LLM work, and the hardest part
+is building a good test set in the first place. `proofrag` generates one from
+*your own corpus*, judges your system on it, and emits a shareable HTML scorecard.
+It's an [Agent Skill](https://agentskills.io) (works in Claude Code, Codex, Cursor)
+**and** a plain Python CLI — wrapping the eval loop, not reinventing the metrics.
+
+<p align="center">
+  <img src="docs/demo.gif" alt="proofrag — generate a golden set, judge, and score in one loop" width="820">
+</p>
+
+<p align="center"><em>…and the scorecard it produces:</em></p>
+<p align="center">
+  <img src="docs/scorecard.png" alt="RAG eval scorecard" width="760">
+</p>
+
+<p align="center"><em>Try it now — no API key needed:</em></p>
+
+```bash
+git clone https://github.com/unshDee/proofrag && cd proofrag
+uv run proofrag demo --out scorecard.html && open scorecard.html
+```
+
+> Uses [uv](https://docs.astral.sh/uv/). `uv run` auto-creates the environment on
+> first call — nothing else to install. Prefer pip? `pipx install proofrag`.
+
+## Install as an Agent Skill
+
+`proofrag` is a skill (the [agentskills.io](https://agentskills.io) open standard) backed
+by a real CLI — so any agent can run *"evaluate my RAG"* and get a reproducible scorecard.
+
+**Claude Code (plugin):**
+```
+/plugin marketplace add unshDee/proofrag
+/plugin install proofrag@proofrag
+```
+Then ask *"evaluate my RAG"* (auto-triggered) or type `/proofrag`.
+
+**Claude Code (manual)** — `cp -r skills/proofrag ~/.claude/skills/`
+**Codex / other agents** — `cp -r skills/proofrag .agents/skills/`
+
+The skill drives the `proofrag` CLI; install it with `uv tool install "proofrag[anthropic]"`
+(or `pipx install`, or run ad-hoc via `uvx`). See [AGENTS.md](AGENTS.md) for details.
+
+## Why this exists
+
+> "Running evals aren't the problem — the problem is acquiring or building a
+> high-quality, non-contaminated dataset."
+
+Most RAG systems reach production with no evals because writing a balanced golden
+set by hand is tedious. So teams ship prompt and model changes blind. This closes
+that loop: **change something → re-run → see if quality moved → gate the merge.**
+
+## The loop
+
+```bash
+# 1. Generate a golden set from YOUR docs (questions + gold answers + gold contexts)
+proofrag generate --corpus ./docs --out goldenset.jsonl --n 20
+
+# 2. Run your RAG over each question -> predictions.jsonl  (one line per question)
+#    {"id": "q000", "answer": "...", "retrieved_contexts": ["...", "..."]}
+#    See examples/docs-rag/naive_rag.py for a runnable driver.
+
+# 3. Judge: groundedness, correctness, completeness, citation quality + retrieval metrics
+proofrag evaluate --goldenset goldenset.jsonl --predictions predictions.jsonl --out results.json
+
+# 4. Shareable HTML scorecard
+proofrag report --results results.json --out scorecard.html
+```
+
+Run the whole thing end-to-end against the bundled example:
+
+```bash
+uv sync --extra anthropic && export ANTHROPIC_API_KEY=...
+uv run proofrag generate --corpus examples/docs-rag/corpus --out goldenset.jsonl --n 8
+uv run python examples/docs-rag/naive_rag.py --goldenset goldenset.jsonl --corpus examples/docs-rag/corpus --out predictions.jsonl
+uv run proofrag evaluate --goldenset goldenset.jsonl --predictions predictions.jsonl --out results.json
+uv run proofrag report --results results.json --out scorecard.html
+```
+
+## CI gate
+
+Two kinds of gate. An **absolute** floor:
+
+```bash
+proofrag evaluate --goldenset goldenset.jsonl --predictions predictions.jsonl \
+  --out results.json --fail-under 0.7      # non-zero exit if overall score drops below 0.7
+```
+
+…and a **regression** gate against a committed baseline (a known-good results.json):
+
+```bash
+proofrag diff --baseline baseline.json --candidate results.json --tolerance 0.02
+# prints a per-metric delta table; exits 1 if any metric dropped > tolerance.
+# Refuses to compare across different judge models unless --allow-judge-mismatch.
+```
+
+### GitHub Action
+
+Drop proofrag into any repo's CI in a few lines — it installs the CLI, evaluates,
+writes the scorecard, and gates on both the floor and the baseline:
+
+```yaml
+- uses: unshDee/proofrag@v0
+  env:
+    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+  with:
+    goldenset: eval/goldenset.jsonl
+    predictions: predictions.jsonl     # produced by your RAG earlier in the job
+    baseline: eval/baseline.json        # optional regression gate
+    fail-under: "0.7"                   # optional absolute gate
+```
+
+Full runnable workflow (with artifact upload): [`examples/ci/proofrag-eval.yml`](examples/ci/proofrag-eval.yml).
+
+## What makes it different
+
+- **Golden set from your corpus** — the wedge. Difficulty tiers: single-doc,
+  multi-doc, and *unanswerable* (so you catch hallucination-instead-of-refusal).
+- **Retriever vs generator split** — rank-aware retrieval metrics (Recall@k,
+  Precision@k, NDCG@k, MRR) separate "the context never arrived / ranked too low"
+  from "the model fluffed it." Lexical by default; `--semantic` for embedding match.
+- **Pinned, fingerprinted judge** — every scorecard records its judge model, so you
+  never compare scores produced by different judges.
+- **Cheap & portable** — defaults to a small model; Anthropic, OpenAI, or local/Ollama
+  (`OPENAI_BASE_URL`). Self-contained HTML, zero JS, zero external assets.
+- **Agent-native** — drop it in as a skill and say *"evaluate my RAG"*; the agent
+  wires your pipeline to the kit.
+
+## Configuration
+
+| Env | Default | Purpose |
+|-----|---------|---------|
+| `ANTHROPIC_API_KEY` | — | Anthropic backend (default) |
+| `OPENAI_API_KEY` / `OPENAI_BASE_URL` | — | OpenAI-compatible / local |
+| `PROOFRAG_PROVIDER` | auto | `anthropic` or `openai` |
+| `PROOFRAG_MODEL` | Haiku / gpt-4o-mini | judge & generator model |
+| `PROOFRAG_EMBED_MODEL` | text-embedding-3-small | embeddings for `--semantic` retrieval match |
+
+## Roadmap
+
+- [x] v0.1 — golden-set generator, LLM-as-judge, retrieval recall, HTML scorecard, CI gate
+- [x] v0.2 — rank-aware retrieval metrics (Recall@k / Precision@k / NDCG@k / MRR), lexical + optional embedding match
+- [ ] v0.3 — GitHub Action + baseline diffing (regression-aware gate)
+- [ ] v0.4 — A/B comparator (vector vs GraphRAG) with blind judging
+- [ ] v0.5 — Ragas / DeepEval backends as pluggable scorers
+
+Issues and PRs welcome. MIT licensed.