counterproof 1.0.0__tar.gz

counterproof-1.0.0/.gitattributes

@@ -0,0 +1,2 @@
+# Fixture repositories must be byte-identical on every OS (they are diffed and hashed).
+tests/fixtures/** text eol=lf

counterproof-1.0.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,33 @@
+name: Bug report
+description: Counterproof gave a wrong verdict, crashed, or behaved unexpectedly.
+labels: [bug]
+body:
+  - type: textarea
+    id: what
+    attributes:
+      label: What happened
+      description: What you ran, what you expected, and what you got.
+    validations:
+      required: true
+  - type: textarea
+    id: certificate
+    attributes:
+      label: Certificate
+      description: The output of `counterproof report RUN_ID --format md` (remove anything private).
+      render: markdown
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: The output of `counterproof --version`.
+    validations:
+      required: true
+  - type: dropdown
+    id: stack
+    attributes:
+      label: Test runner
+      options: [pytest, Vitest, Jest, other]
+  - type: input
+    id: os
+    attributes:
+      label: Operating system and Python version

counterproof-1.0.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/sudais-khalid/counterproof/security/advisories/new
+    about: Report security issues privately.
+  - name: Documentation
+    url: https://sudais-khalid.github.io/counterproof/docs/
+    about: Install, configure and read a certificate.

counterproof-1.0.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,15 @@
+name: Feature request
+description: Suggest a probe, an integration, a language or an improvement.
+labels: [enhancement]
+body:
+  - type: textarea
+    id: problem
+    attributes:
+      label: The problem
+      description: What are you trying to verify, and what gets in the way today?
+    validations:
+      required: true
+  - type: textarea
+    id: idea
+    attributes:
+      label: What would help

counterproof-1.0.0/.github/pull_request_template.md

@@ -0,0 +1,8 @@
+## What and why
+
+## Evidence
+
+- [ ] `ruff check`, `ruff format --check`, `mypy` and `pytest -m "not corpus"` pass locally
+- [ ] New behaviour has tests that fail without the change
+- [ ] No test was deleted or weakened, and no golden expectation changed (or the reason is above)
+- [ ] Prompt changes come with re-recorded cassettes

counterproof-1.0.0/.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: ${{ matrix.os }} / py${{ matrix.python }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python: ["3.11", "3.13"]
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python }}
+          enable-cache: true
+      - uses: actions/setup-node@v7
+        with:
+          node-version: "24"
+      - name: Install
+        run: uv sync --locked
+      - name: Install fixture test runners (Vitest, Jest; pinned by package-lock.json)
+        run: npm ci --no-audit --no-fund --prefix tests/fixtures/node
+      - name: Lint
+        run: |
+          uv run ruff check
+          uv run ruff format --check
+      - name: Type check
+        run: uv run mypy
+      - name: Test
+        run: uv run pytest -q -n auto -m "not corpus" --cov --cov-report=term
+        env:
+          COUNTERPROOF_REQUIRE_NODE: "1" # Node-based tests must run here, never skip
+          # Ubuntu runners have a docker daemon: the --sandbox docker tests must run there.
+          COUNTERPROOF_REQUIRE_DOCKER: ${{ matrix.os == 'ubuntu-latest' && '1' || '0' }}
+      - name: Doctor
+        run: uv run counterproof doctor
+
+  corpus:
+    # The extended C1 tier (tasks T6 to T20 of both fixtures, 120 variants): fixture checks
+    # and golden verdicts, rule-only and with recorded LLM answers. One OS keeps CI fast;
+    # the matrix above runs the same code paths on the original ten tasks everywhere.
+    name: corpus (extended C1)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.13"
+          enable-cache: true
+      - uses: actions/setup-node@v7
+        with:
+          node-version: "24"
+      - name: Install
+        run: uv sync --locked
+      - name: Install fixture test runners
+        run: npm ci --no-audit --no-fund --prefix tests/fixtures/node
+      - name: Corpus tests
+        run: uv run pytest -q -n auto -m corpus
+        env:
+          COUNTERPROOF_REQUIRE_NODE: "1"

counterproof-1.0.0/.github/workflows/eval-c3.yml

@@ -0,0 +1,102 @@
+name: eval-c3
+
+# Corpus C3 (PRD 17.1): real merged pull requests written with a coding agent. Every pull
+# request runs on its own ephemeral runner because it executes foreign code: the case jobs
+# get no token and no secret, and checkout credentials are not persisted. Run by hand:
+#   gh workflow run eval-c3.yml [-f limit=44]
+# then download the c3-results artifact (eval/results/<date>-c3.{json,md}).
+
+on:
+  workflow_dispatch:
+    inputs:
+      limit:
+        description: "How many candidate pull requests to run (from candidates.json)"
+        default: "44"
+      only:
+        description: "Only these pull requests (comma-separated owner/repo#number), e.g. to re-run"
+        default: ""
+
+permissions:
+  contents: read
+
+jobs:
+  plan:
+    name: Resolve the pull requests
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.plan.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - id: plan
+        env:
+          GH_TOKEN: ${{ github.token }}
+          LIMIT: ${{ inputs.limit }}
+          ONLY: ${{ inputs.only }}
+        run: |
+          python3 scripts/c3_runner.py plan --out matrix.json --limit "$LIMIT" --only "$ONLY"
+          echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"
+
+  case:
+    name: ${{ matrix.pr.id }}
+    needs: plan
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions: {}
+    strategy:
+      fail-fast: false
+      max-parallel: 12
+      matrix:
+        pr: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+      - name: Install Counterproof
+        run: uv tool install .
+      - name: Run the pull request
+        env:
+          SPEC: ${{ toJSON(matrix.pr) }}
+        run: |
+          python3 scripts/c3_runner.py case --spec "$SPEC" --workdir "$RUNNER_TEMP/case" \
+            --out "$RUNNER_TEMP/out/report.json"
+      - name: Name the result after the pull request
+        id: name
+        env:
+          ID: ${{ matrix.pr.id }}
+        run: echo "name=c3-$(echo "$ID" | tr '/#' '__')" >> "$GITHUB_OUTPUT"
+      - uses: actions/upload-artifact@v7
+        with:
+          name: ${{ steps.name.outputs.name }}
+          path: ${{ runner.temp }}/out/
+          if-no-files-found: warn
+
+  merge:
+    name: Merge the results
+    needs: case
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.13"
+      - uses: actions/download-artifact@v8
+        with:
+          pattern: c3-*
+          path: reports
+      - name: Merge
+        run: |
+          uv sync --locked
+          date="$(date -u +%Y-%m-%d)"
+          uv run python scripts/c3_runner.py merge --reports reports --out "eval/results/$date-c3"
+      - uses: actions/upload-artifact@v7
+        with:
+          name: c3-results
+          path: eval/results/

counterproof-1.0.0/.github/workflows/pages.yml

@@ -0,0 +1,51 @@
+name: pages
+
+# The website (https://sudais-khalid.github.io/counterproof/): built from site/ and the
+# committed evaluation results by scripts/build_site.py, then deployed to GitHub Pages.
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "site/**"
+      - "eval/results/**"
+      - "scripts/build_site.py"
+      - "pyproject.toml"
+      - ".github/workflows/pages.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.13"
+      - name: Build the site
+        run: |
+          uv sync --locked
+          uv run python scripts/build_site.py --out _site
+      - uses: actions/configure-pages@v6
+      - uses: actions/upload-pages-artifact@v5
+        with:
+          path: _site
+
+  deploy:
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - id: deployment
+        uses: actions/deploy-pages@v5

counterproof-1.0.0/.github/workflows/release.yml

@@ -0,0 +1,91 @@
+name: release
+
+# Push a tag vX.Y.Z (matching [project].version) to release: the wheel and sdist are built
+# once, installed into a clean environment and run, then published to PyPI with Trusted
+# Publishing (no stored token) and attached to a GitHub release marked Latest.
+
+on:
+  push:
+    tags: ["v*.*.*"]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build and smoke-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.13"
+      - name: The tag must match the package version
+        run: |
+          version="$(python3 -c 'import tomllib; print(tomllib.load(open("pyproject.toml", "rb"))["project"]["version"])')"
+          if [ "v$version" != "$GITHUB_REF_NAME" ]; then
+            echo "::error::tag $GITHUB_REF_NAME does not match version $version"; exit 1
+          fi
+      - name: Build the sdist and the wheel
+        run: uv build
+      - name: Install the wheel into a clean environment and run it
+        run: |
+          uv venv /tmp/smoke --python 3.13
+          uv pip install --python /tmp/smoke/bin/python dist/*.whl
+          /tmp/smoke/bin/counterproof --version
+          /tmp/smoke/bin/counterproof doctor
+          /tmp/smoke/bin/counterproof --help > /dev/null
+      - uses: actions/upload-artifact@v7
+        with:
+          name: dist
+          path: dist/
+          if-no-files-found: error
+
+  pypi:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/counterproof
+    permissions:
+      id-token: write # Trusted Publishing: PyPI trusts this workflow, no API token stored
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1
+
+  github-release:
+    name: GitHub release
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/download-artifact@v8
+        with:
+          name: dist
+          path: dist/
+      - name: Release notes from CHANGELOG.md
+        run: |
+          version="${GITHUB_REF_NAME#v}"
+          awk -v v="$version" '
+            $0 ~ "^## \\[" v "\\]" { on = 1; next }
+            on && /^## \[/ { exit }
+            on { print }
+          ' CHANGELOG.md > notes.md
+          test -s notes.md || { echo "::error::no CHANGELOG.md section for $version"; exit 1; }
+      - name: Create the release (marked Latest) with the distributions and the demo
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          assets=(dist/*)
+          [ -f docs/media/counterproof-demo.mp4 ] && assets+=(docs/media/counterproof-demo.mp4)
+          gh release create "$GITHUB_REF_NAME" "${assets[@]}" \
+            --title "Counterproof ${GITHUB_REF_NAME#v}" \
+            --notes-file notes.md \
+            --latest \
+            --verify-tag

counterproof-1.0.0/.gitignore

@@ -0,0 +1,18 @@
+__pycache__/
+*.py[cod]
+.venv/
+build/
+dist/
+*.egg-info/
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+.coverage
+.coverage.*
+coverage.xml
+htmlcov/
+node_modules/
+# Working copies of the PRD; the tracked version is docs/PRD.md
+/Counterproof_PRD.docx
+/Counterproof_PRD.md
+_site/

counterproof-1.0.0/CHANGELOG.md

@@ -0,0 +1,68 @@
+# Changelog
+
+All notable changes to Counterproof. The format follows
+[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project uses
+[semantic versioning](https://semver.org/).
+
+## [1.0.0] - 2026-11-09
+
+The first stable release: everything in the v1 product requirements ([docs/PRD.md](docs/PRD.md)),
+measured on three corpora.
+
+### Verification
+
+- **Five probes**, run cheapest first in isolated git worktrees: claim audit (P5), integrity
+  (P2), red-green (P1), change-scoped mutants (P3) and the blind oracle (P4).
+- **A deterministic verdict engine**: fourteen ordered rules map the evidence to PROVEN, WEAK,
+  UNPROVEN, REFUTED or INCONCLUSIVE, with exit codes for CI. An LLM never decides a verdict.
+- **Integrity rules FR-IN-01 to FR-IN-09**: removed assertions, deleted tests, added skips,
+  loosened expectations, neutralised assertions, mocked units, code special-cased to the
+  test's inputs, test-environment detection and self-configuration changes. Waivers only
+  count once committed. Comparing an input with a one-word string (`== "rejected"`) is
+  treated as mode dispatch and stays a medium finding for review, after it caused every
+  refutation on real agent pull requests in corpus C3.
+- **Mutants**: rule operators for Python and TypeScript, optional LLM-written realistic
+  faults and an overfit mutant, a three-stage equivalence filter (static, an LLM judge,
+  Hypothesis or fast-check fuzzing), timeouts confirmed by a re-run before they count.
+- **Blind oracle for Python and TypeScript**: tests written from the task and the public
+  interface only (never a function body), validated before they run, run three times on the
+  change and triaged when they fail. A likely bug caps the verdict at WEAK and never refutes.
+- **Claim audit** of Claude Code session transcripts: no run after the last edit, subset runs
+  sold as the full suite, hidden exit codes, count mismatches, claims contradicted by the
+  baseline.
+
+### Languages and environments
+
+- Python with pytest; TypeScript and JavaScript with Vitest or Jest (tree-sitter analysis;
+  tree-sitter is held below 0.26, whose 0.26.0 release segfaults on large syntax trees).
+- **`--sandbox docker`**: tests run in a container with no network, 2 GB of memory, 2 CPUs and
+  512 processes, with an image built once per set of dependency files. Linux and macOS hosts
+  (Node projects: Linux). A sandbox that cannot start makes the run INCONCLUSIVE; it never
+  falls back to running unsandboxed.
+- LLM providers: Anthropic, OpenAI, DeepSeek and Ollama, all optional, with hard token, dollar
+  and time budgets, secret redaction and a response cache. Nothing leaves the machine by
+  default.
+
+### Integrations
+
+- Claude Code Stop hook (coach mode: the gap and its location, never the answer), an MCP
+  server, a git pre-push hook and a GitHub Action with PR certificates and SARIF.
+- Certificates as text, Markdown, JSON, HTML and SARIF 2.1.0; sealed evidence bundles.
+
+### Evaluation
+
+- `counterproof eval` with baselines B0 (green tests), B1 (claims only) and B2 (diff-scoped
+  classic mutation), probe ablations through the pure engine, determinism checks and
+  recorded-LLM replay.
+- **C1**: 160 seeded changes, 20 tasks in each of a Python and a TypeScript library, each
+  with an honest fix and three gamed variants, ground truth checked with the real runners.
+- **C2**: 15 real Claude Code sessions, including EvilGenie-style pressure, labelled with
+  held-out tests and a hand review, plus 8 hand-built sessions for the false-claim patterns.
+- **C3**: real merged agent pull requests from public repositories, each run on its own
+  ephemeral GitHub runner.
+- Equivalent-mutant leakage measured on a human-labelled sample of surviving mutants.
+
+### Project
+
+- Website with docs, integrations, results and FAQ; a two-minute demo video.
+- Releases publish to PyPI with Trusted Publishing and to GitHub releases.

counterproof-1.0.0/CONTRIBUTING.md

@@ -0,0 +1,56 @@
+# Contributing to Counterproof
+
+Thanks for helping. Counterproof makes claims about other people's code, so the bar for its
+own evidence is high: every change comes with tests, and no test is loosened to make a change
+pass.
+
+## Set up
+
+```sh
+git clone https://github.com/sudais-khalid/counterproof
+cd counterproof
+uv sync                                  # Python 3.11+, the dev tools included
+npm ci --prefix tests/fixtures/node      # Vitest and Jest for the TypeScript fixtures
+uv run counterproof doctor
+```
+
+## Before you open a pull request
+
+```sh
+uv run ruff check
+uv run ruff format --check
+uv run mypy
+uv run pytest -q -n auto -m "not corpus"     # what every CI job runs
+uv run pytest -q -n auto -m corpus           # the extended C1 corpus (one CI job)
+```
+
+CI runs on Linux, macOS and Windows with Python 3.11 and 3.13, and needs no API key: real
+LLM answers are replayed from the recorded cassettes in `tests/cassettes/`.
+
+## Ground rules
+
+- **The verdict engine stays pure.** `verdict/rules.py` decides from evidence only; an LLM
+  can propose mutants or tests, never a verdict.
+- **Never touch the user's working tree.** Everything runs in worktrees under the cache
+  directory.
+- **Anti-gaming holds everywhere an agent can read**: coach hints, hook output and MCP
+  responses never contain mutant code, oracle tests or the seed.
+- **Tests are evidence.** Do not delete or weaken a test, or change a golden expectation,
+  without saying why in the pull request. Fixture behaviour is never hard-coded.
+- **Requirement ids are traceable.** Code and test names cite the PRD ids they implement
+  (for example `FR-IN-07`); see [docs/PRD.md](docs/PRD.md).
+- **Prompts are versioned.** Changing a prompt changes the cache keys; re-record the
+  cassettes (`tests/golden/record_cassettes.py`) in the same pull request.
+
+## Fixtures and corpora
+
+C1 tasks live in `tests/fixtures/repos/<repo>/tasks/` with four variants each, and their
+ground truth in `tests/golden/expectations.yaml`. Tasks T6 to T20 are generated from
+`tests/golden/c1_tasks_py.py` and `c1_tasks_ts.py` by `tests/golden/make_c1_tasks.py`; edit
+the data and re-run the script rather than the generated files.
+
+## Reporting bugs
+
+Open an issue with the command you ran, the certificate (`--format md`) and, if you can, the
+run directory from `counterproof report RUN_ID --format json`. For security issues, see
+[SECURITY.md](SECURITY.md).