agent-regress-cli 0.1.0__tar.gz

agent_regress_cli-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,30 @@
+name: Bug Report
+description: File a bug report
+title: "[Bug]: "
+labels: ["bug", "needs-triage"]
+body:
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+    validations:
+      required: true
+  - type: textarea
+    id: to-reproduce
+    attributes:
+      label: Minimal reproduction
+      render: python
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: agent-regress version
+    validations:
+      required: true
+  - type: input
+    id: python
+    attributes:
+      label: Python version
+    validations:
+      required: true

agent_regress_cli-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,17 @@
+name: Feature Request
+description: Request a new feature
+title: "[Feature]: "
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: What problem does this solve?
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Proposed solution
+    validations:
+      required: true

agent_regress_cli-0.1.0/.github/dependabot.yml

@@ -0,0 +1,16 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: /
+    schedule:
+      interval: weekly
+      day: monday
+    groups:
+      patch-updates:
+        update-types:
+          - patch
+
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly

agent_regress_cli-0.1.0/.github/pull_request_template.md

@@ -0,0 +1,22 @@
+## Description
+
+What does this PR do? (one sentence)
+
+## Type of change
+
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Performance improvement
+- [ ] Documentation update
+
+## Checklist
+
+- [ ] Tests pass locally
+- [ ] Added tests for new behavior
+- [ ] CHANGELOG entry added for user-facing changes
+- [ ] Benchmark output included below if performance-sensitive code changed
+- [ ] AI-assisted code has been reviewed and validated line by line
+
+## Benchmark output (if applicable)
+
+paste output here

agent_regress_cli-0.1.0/.github/workflows/benchmark.yml

@@ -0,0 +1,33 @@
+name: Benchmark
+
+on:
+  push:
+    branches: ["main"]
+
+jobs:
+  benchmark:
+    name: Statistical overhead
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+      - run: uv sync --extra dev
+      - name: Run benchmarks
+        run: |
+          uv run pytest benchmarks/test_stat_overhead.py \
+            --benchmark-json=output.json \
+            --benchmark-only \
+            -v
+      - uses: benchmark-action/github-action-benchmark@4de1bed97a47d2e7a675c37e04bcf5f29cb28bdb  # v1.20.4
+        with:
+          tool: pytest
+          output-file-path: output.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: "120%"
+          comment-on-alert: true
+          fail-on-alert: true
+          gh-pages-branch: gh-pages
+          benchmark-data-dir-path: dev/bench

agent_regress_cli-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,116 @@
+name: CI
+
+on:
+  push:
+    branches: ["main", "dev"]
+  pull_request:
+    branches: ["main"]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  quality:
+    name: Quality (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Lint
+        run: uv run ruff check src/ tests/ benchmarks/
+
+      - name: Format check
+        run: uv run ruff format --check src/ tests/ benchmarks/
+
+      - name: Type check
+        run: uv run mypy src/ --strict
+
+      - name: Unit tests with coverage
+        run: |
+          uv run pytest tests/unit/ \
+            --cov=src/ \
+            --cov-report=xml \
+            --cov-report=term-missing \
+            --cov-fail-under=80 \
+            -v
+
+      - name: Stats module coverage gate (95%)
+        run: |
+          uv run pytest tests/unit/stats/ \
+            --cov=src/agent_regress/stats/ \
+            --cov-fail-under=95 \
+            -q
+
+      - name: Security scan
+        uses: aquasecurity/trivy-action@v0.36.0
+        with:
+          scan-type: fs
+          format: sarif
+          output: trivy-results.sarif
+          severity: HIGH,CRITICAL
+          exit-code: "1"
+          ignore-unfixed: true
+
+      - name: Upload Trivy SARIF
+        uses: github/codeql-action/upload-sarif@v4
+        if: always()
+        with:
+          sarif_file: trivy-results.sarif
+
+  leaderboard-schema:
+    name: Validate leaderboard schema
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - run: uv sync --extra dev
+      - name: Validate submissions
+        run: |
+          python -c "
+          import json, jsonschema, pathlib, sys
+          schema_path = pathlib.Path('leaderboard/schema.json')
+          if not schema_path.exists():
+              sys.exit(0)
+          schema = json.loads(schema_path.read_text())
+          results_dir = pathlib.Path('leaderboard/results')
+          json_files = [f for f in results_dir.glob('*.json') if f.name != '.gitkeep']
+          if not json_files:
+              print('No leaderboard results to validate')
+              sys.exit(0)
+          for f in json_files:
+              jsonschema.validate(json.loads(f.read_text()), schema)
+              print(f'OK: {f.name}')
+          "
+
+  codeql:
+    name: CodeQL Security Analysis
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@v7
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: python
+      - uses: github/codeql-action/autobuild@v4
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:python"

agent_regress_cli-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,39 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  release:
+    name: Build and publish
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: write
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - name: Build
+        run: uv build
+      - name: Publish to PyPI
+        run: uv publish
+        env:
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
+      - name: Generate SBOM
+        run: |
+          pip install cyclonedx-bom
+          uv export --no-dev --format requirements-txt -o /tmp/prod-requirements.txt
+          cyclonedx-py requirements --output-format json -o sbom.json /tmp/prod-requirements.txt
+      - name: Sign with Sigstore
+        uses: sigstore/gh-action-sigstore-python@f514d46b907ebcd5bedc05145c03b69c1edd8b46  # v3.0.0
+        with:
+          inputs: dist/*.whl dist/*.tar.gz
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b  # v3.0.1
+        with:
+          files: |
+            sbom.json
+            dist/*.whl
+            dist/*.tar.gz

agent_regress_cli-0.1.0/.github/workflows/scorecard.yml

@@ -0,0 +1,29 @@
+name: OpenSSF Scorecard
+
+on:
+  push:
+    branches: ["main"]
+  schedule:
+    - cron: "30 1 * * 6"
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      id-token: write
+      contents: read
+      actions: read
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: ossf/scorecard-action@v2.4.3
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+      - uses: github/codeql-action/upload-sarif@v4
+        with:
+          sarif_file: results.sarif

agent_regress_cli-0.1.0/.gitignore

@@ -0,0 +1,39 @@
+# Python
+__pycache__/
+*.py[cod]
+*.so
+*.egg
+*.egg-info/
+dist/
+build/
+.eggs/
+.venv/
+venv/
+env/
+.env
+
+# Testing
+.pytest_cache/
+.coverage
+coverage.xml
+htmlcov/
+
+# Type checking
+.mypy_cache/
+
+# Ruff
+.ruff_cache/
+
+# Claude Code settings
+.claude/
+
+# IDE
+.vscode/
+.idea/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Benchmark output
+output.json

agent_regress_cli-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: check-toml
+      - id: check-json
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: no-commit-to-branch
+        args: [--branch, main]
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

agent_regress_cli-0.1.0/CHANGELOG.md

@@ -0,0 +1,60 @@
+# Changelog
+
+All notable changes documented here.
+
+Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
+Versions follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+- PyPI distribution renamed to `agent-regress-cli` (import path is unchanged: `import agent_regress`) -- `agent-regress` on PyPI was blocked as too similar to an existing unrelated project
+- `agent-regress` CLI (`src/agent_regress/cli.py`), wired up via `[project.scripts]`
+- `agent-regress compare --version-a-results <path.json> --version-b-results <path.json>` -- file-based entry point that runs two JSON arrays of pre-computed per-run scores through the same Mann-Whitney U / bootstrap CI / Cohen's d pipeline `compare()` uses
+- `--json` flag on `compare` for clean, machine-readable `Report` output (warnings routed to stderr)
+- `--fail-on-regression` flag on `compare` for CI use (exit 1 on a `REGRESSED` verdict)
+- `agent-regress --version`
+
+### Fixed
+
+- `mann_whitney_u`: raise `ValueError` on NaN input instead of silently returning `p=1.0`
+- `rank_biserial_r`: propagate `nan` instead of silently returning `-1.0` on NaN input
+- `run_suite`: warn threshold raised from n<10 to n<30 to match recommendation message
+- `_get_expected`: fix `KeyError` when `expected` key is present but set to `None`
+- `f1_scorer`: use `Counter` multiset intersection instead of set intersection — fixes wrong F1 when tokens repeat
+- `compare`: align `_MIN_N_WARN` threshold (was 30, now 50) with warning message and `n_runs` default
+- `compare` / `Report`: store `p_threshold` and `min_effect` on `Report` so `assert_stable()` uses the right defaults
+- `gaia.py`: fix false-positive accuracy when agent raises and `expected_answer` is `'none'`
+- `tau_bench.py`: raise clear `ValueError` on empty `k_values` instead of opaque `max()` error
+- `langchain` / `langgraph` integrations: fix fallback adapters that were wrapping `test_case` in a nested dict
+- `openai_agents_runner`: narrow `RuntimeError` catch to avoid masking real errors and double-invoking the agent
+- `benchmarks/__init__`: export `GAIALevelResult`, `SWEBenchResult`, `TauBenchResult`
+- `core/__init__` / top-level `__init__`: export `AgentCallable` and `ScorerCallable` type aliases
+
+## [0.1.0] - 2026-06-19
+
+### Added
+
+- `compare(version_a, version_b, test_suite, n_runs=50)` -- primary public API
+- `run_suite(agent, test_suite, n_runs)` -- run agent N times on a test suite
+- `Report` dataclass with `verdict`, `p_value`, `effect_size`, `ci_lower`, `ci_upper`
+- `Verdict` enum: `REGRESSED`, `STABLE`, `IMPROVED`, `INSUFFICIENT_DATA`
+- Mann-Whitney U test with continuity correction (`stats.mann_whitney_u`)
+- Bootstrap confidence intervals at 95%, N=1000 resamples (`stats.bootstrap_mean_ci`)
+- Cohen's d and rank-biserial r effect size (`stats.compute_effect_sizes`)
+- `RegressionGate` and `assert_no_regression` for pytest CI integration
+- `exact_match_scorer` and `f1_scorer` built-in scorers
+- LangGraph integration (`integrations.langgraph_runner`)
+- LangChain LCEL integration (`integrations.langchain_runner`)
+- OpenAI Agents SDK integration (`integrations.openai_agents_runner`)
+- CrewAI integration (`integrations.crewai_runner`)
+- Tau-bench pass^k harness (`benchmarks.TauBenchHarness`)
+- GAIA Level 1-3 split harness (`benchmarks.GAIAHarness`)
+- SWE-bench Verified scaffold score harness (`benchmarks.SWEBenchHarness`)
+- Docker Compose self-host setup
+- Sample size warnings: warns (not fails) when n < 30 per version
+- Apache 2.0 license
+- OpenSSF Scorecard CI workflow
+- SLSA Level 2 release signing via Sigstore
+- SBOM generation on release (CycloneDX JSON format)

agent_regress_cli-0.1.0/CLAUDE.md

@@ -0,0 +1,71 @@
+# CLAUDE.md -- agent-eval
+
+## Git Workflow
+
+When asked to commit, push, or "update GitHub" -- just do it. No questions.
+
+- git add relevant files -> git commit -> git push origin main
+- "commit" means commit AND push -- always both
+- Never ask "should I push?"
+
+Every commit message must end with:
+
+Built by Rudrendu Paul and Sourav Nandy, developed with Claude Code
+
+Never use Co-Authored-By: lines.
+
+## Project Overview
+
+agent-eval: Python statistical regression testing framework for LLM agents.
+
+Core angle: run agent N times at version A, N times at version B, apply
+Mann-Whitney U + bootstrap CI, report p-value and Cohen's d.
+
+A/B testing for agent quality. Not DeepEval (threshold-based). Not single-run
+benchmarks. Statistical comparison of two version distributions.
+
+License: Apache 2.0 core.
+Primary integrations: LangGraph (P0), OpenAI Agents SDK (P0).
+
+## Repo Layout
+
+src/agent_regress/
+- core/: runner.py, scorer.py, report.py, compare.py
+- stats/: mann_whitney.py, bootstrap.py, effect_size.py (pure Python + scipy ONLY)
+- integrations/: lazy-imported optional deps
+- ci/: gate.py (AssertionError when p < threshold AND d > min_effect)
+- benchmarks/: tau_bench.py, gaia.py, swebench.py
+tests/unit/: no LLM calls
+tests/integration/: real APIs, tagged @pytest.mark.integration
+benchmarks/: standalone scripts
+leaderboard/: git-committed JSON results
+
+## Engineering Standards
+
+uv run ruff check src/ tests/ benchmarks/
+uv run ruff format --check src/ tests/ benchmarks/
+uv run mypy src/ --strict
+uv run pytest tests/unit/ --cov=src/ --cov-fail-under=80
+
+## Key Invariants
+
+- stats/ has ZERO LLM calls. Pure Python + scipy on lists of floats.
+- runner.py: any callable dict->float. No framework coupling.
+- gate.py: AssertionError when p < threshold AND d > min_effect.
+- gate.py: WARN (not fail) when n < 30.
+- All public API in __init__.py.
+- mypy --strict must pass.
+- leaderboard/results/ schema-validated in CI.
+
+## Session Start
+
+1. git status && git log --oneline -5
+2. uv run pytest tests/unit/ -q
+3. Read docs/lessons.md
+
+## Anti-Sycophancy
+
+- No performance claims without benchmark output
+- stats/ is a scipy one-liner -- not the moat
+- The moat is leaderboard governance and hosted regression history
+- Braintrust closes the p-value gap in one sprint -- say this proactively

agent_regress_cli-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,56 @@
+# Contributing to agent-eval
+
+Thanks for your interest. This covers everything to go from zero to a merged PR.
+
+## Fastest path
+
+1. Clone and run dev setup below
+2. Pick an issue labeled `good first issue`
+3. Open a draft PR early -- we can save you wasted work
+4. Mark ready for review when tests pass
+
+We aim to review PRs within 72 hours (weekdays).
+
+## Dev setup
+
+```bash
+git clone https://github.com/RudrenduPaul/agent-eval
+cd agent-eval
+pip install uv
+uv sync --extra dev
+pre-commit install
+uv run pytest tests/unit/ -q  # should pass 100% on a clean clone
+```
+
+## Engineering standards
+
+Before submitting:
+
+```bash
+uv run ruff check src/ tests/ benchmarks/
+uv run ruff format --check src/ tests/ benchmarks/
+uv run mypy src/ --strict
+uv run pytest tests/unit/ --cov=src/ --cov-fail-under=80
+```
+
+## What makes a PR merge quickly
+
+- Tests added for every behavior change
+- CHANGELOG entry for any user-facing change
+- One focused change per PR
+
+## What we will not merge
+
+- PRs that break existing tests without a documented reason
+- AI-generated code submitted without running and validating the output
+
+## Maintainer SLAs
+
+- Bug reports: acknowledged within 24 hours (weekdays)
+- Feature requests: triage label within 72 hours
+- PRs: first review within 72 hours
+- Security: acknowledged within 48 hours
+
+## Community
+
+Discord: discord.gg/agent-eval (#contributing channel)