reviewgate 0.1.0__tar.gz

reviewgate-0.1.0/.code-review-graph/.gitignore

@@ -0,0 +1,3 @@
+# Auto-generated by code-review-graph — do not commit database files.
+# The graph.db contains absolute paths and code structure metadata.
+*

reviewgate-0.1.0/.github/workflows/agentboard-review.yml

@@ -0,0 +1,100 @@
+name: agentboard review
+
+# Advisory pre-merge review: an LLM proposes edge-case tests, a deterministic
+# gate executes them against this PR's code, and the evidence lands as a PR
+# comment. Never blocking: confirmed gaps live in the comment, not the exit
+# code. A human decides.
+#
+# Fork PRs are skipped by design: they cannot access OPENAI_API_KEY (repo
+# secrets are withheld from forks), and that is correct; do not work around
+# it.
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  review:
+    # Same-repo PRs only: forks have no access to the key.
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Install agentboard
+        # In agentboard's own repo this installs the PR's code (true
+        # dogfooding); in any other repo the local install fails fast and
+        # the fallback pulls the published tool.
+        run: pip install -e . 2>/dev/null || pip install git+https://github.com/anp0429/agentboard
+
+      - name: Pick a reviewable target from the PR's changed files
+        id: target
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          target=$(git diff --name-only "$BASE_SHA...$HEAD_SHA" \
+            | grep -E '\.(ts|tsx|js|jsx|mjs)$' \
+            | grep -vE '\.test\.|\.spec\.|/tests?/|__tests__' \
+            | head -1 || true)
+          echo "target=$target" >> "$GITHUB_OUTPUT"
+          if [ -z "$target" ]; then
+            echo "No reviewable source change in this PR; skipping quietly."
+          fi
+
+      - name: Review
+        if: steps.target.outputs.target != ''
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          # Filenames are PR-controlled data. Passing through env (not inline
+          # expression interpolation) keeps shell metacharacters inert.
+          TARGET: ${{ steps.target.outputs.target }}
+        run: |
+          agentboard review \
+            --repo . \
+            --target "$TARGET" \
+            --base "$BASE_SHA" \
+            --head "$HEAD_SHA" \
+            --json-out agentboard-run.json \
+            --board agentboard-board.html
+
+      - name: Render comment
+        if: steps.target.outputs.target != ''
+        run: python3 scripts/render_pr_comment.py agentboard-run.json > agentboard-comment.md
+
+      - name: Post or update the sticky comment
+        if: steps.target.outputs.target != ''
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require("fs");
+            const body = fs.readFileSync("agentboard-comment.md", "utf8");
+            const marker = "<!-- agentboard-review -->";
+            const { owner, repo } = context.repo;
+            const issue_number = context.payload.pull_request.number;
+            const { data: comments } = await github.rest.issues.listComments(
+              { owner, repo, issue_number, per_page: 100 });
+            const mine = comments.find(c => c.body && c.body.includes(marker));
+            if (mine) {
+              await github.rest.issues.updateComment(
+                { owner, repo, comment_id: mine.id, body });
+            } else {
+              await github.rest.issues.createComment(
+                { owner, repo, issue_number, body });
+            }

reviewgate-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and test dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+          pip install pytest
+
+      # The verifier/gate is unit-testable without any API key — that's the point.
+      - name: Run unit tests (no API key required)
+        env:
+          PYTHONPATH: src
+        run: python -m pytest tests/ -q
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install ruff
+        run: pip install ruff
+      - name: Lint
+        run: ruff check src/ tests/ examples/
+        continue-on-error: true  # advisory for now; tighten after first green run

reviewgate-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+
+__pycache__/
+.DS_Store
+review_board.html
+.env*
+node_modules/
+package-lock.json
+agentboard_demo_board.html
+review_board.html
+paired_board_*.html
+zod_board_*.html

reviewgate-0.1.0/BENCHMARK.md

@@ -0,0 +1,163 @@
+# Benchmark
+
+Can agentboard find a real bug in code it has never seen, from an intent
+that does not name the bug? This measures that, on recent merged bugfix PRs
+from active TypeScript repositories, and publishes the misses.
+
+## Method
+
+For each row:
+
+1. Pick a bugfix PR that is already merged into an active repo.
+2. Check out the PR's **parent** commit — the code exactly as it was before
+   the fix. The bug is present.
+3. Run agentboard with a **neutral intent** that describes what the module
+   is for, never what the bug is. PR titles telegraph the answer, so the PR
+   and its tests are ground truth, not input.
+4. Score the confirmed gaps against the fix, by reading source.
+
+The intent discipline is the whole experiment. "recursively merge an object
+with default values" is a fair description of defu; "prevent prototype
+pollution via `__proto__`" would be handing over the answer. If a skeptic
+can argue the intent leaked the bug, the row does not count.
+
+No row uses the bug agentboard was originally developed against
+(supabase/mcp #317). Training on your test set is not a benchmark; #317
+appears only as a disclosed reference case in the README.
+
+Every gap in this table was adjudicated by reading the fix diff or the
+source at the parent commit. The auditor's advisory calls were **not**
+trusted as scores — see the auditor-inversion note below for why that
+matters.
+
+## Results
+
+Tool version `e175535`. 12 distinct bugs across 8 repos (two rows are
+security-axis twins). Zero environment failures.
+
+| # | Repo | PR | Target module | Outcome |
+|---|------|----|--------------|---------|
+| s1 | supabase/mcp | #269 | content-api graphql | in-area catch |
+| s2 | supabase/mcp | #329 | management-api errors | miss (4 clean false positives) |
+| r1 | unjs/ufo | #335 | url base join/strip | missed #335, **re-found #360** |
+| r2 | unjs/ufo | #313 | url base prefix match | **strict catch** |
+| r3 | unjs/defu | #156 | recursive default merge | **strict catch** (`__proto__` pollution) |
+| r4 | unjs/pathe | #246 | path resolve | **strict catch** (UNC authority) |
+| r5 | unjs/pathe | #241 | path parse | in-area catch (node divergence) |
+| r6 | pmndrs/jotai | #3354 | store subscribe | miss (subtle async) |
+| r7 | pmndrs/jotai | #3326 | atomWithStorage | miss (component harness) |
+| r8 | vueuse/vueuse | #5525 | useFetch | **in-area catch + 2 bonus bugs** |
+| r9 | TanStack/query | #10812 | devtools sort | **strict catch** |
+| r10 | colinhacks/zod | #5898 | object catchall | miss (wrong-remedy assertion) |
+
+Security-axis twins: r3s (defu) reproduced the strict catch and added a
+nested `__proto__` variant the default axis missed; r10s (zod) missed the
+same way the default did.
+
+**Summary: 8 of 12 rows produced a real confirmed bug. 4 were exact strict
+catches (ufo #313, defu #156, pathe #246, query #10812). r8 over-delivered:
+pointed at vueuse before a known fix, it caught the fix's neighborhood and
+two additional real bugs the PR never touched (a falsy-`initialData` bug
+from `||` where `??` was meant, and a header-array-merge bug). 4 rows
+missed. 0 environment failures.**
+
+### The two bonus bugs (r8)
+
+vueuse `useFetch`, at the parent of #5525, neutral intent "fetch a URL
+reactively with abort and refetch support":
+
+- `const data = shallowRef<T | null>(initialData || null)` — `||` coerces a
+  valid falsy `initialData` (`0`, `''`, `false`) to `null`. Should be `??`.
+- `headersToObject` only handles `Headers` instances, silently dropping
+  array-form `HeadersInit` (`[['authorization', ...]]`).
+
+Neither is what #5525 fixed. Both are real, both were found from a neutral
+intent, both have an executed failing test attached.
+
+## The auditor inverted, twice — and that is the point
+
+The auditor is a second model (Claude) that reads the source and flags
+confirmed gaps it believes assert something the code never promised. It is
+**advisory**: it never changes a verdict.
+
+On two rows it called a **real strict catch** a false positive:
+
+- **r2 (ufo #313):** the code matched `/api` as a prefix of `/api-v2`. The
+  auditor called the failing test a wrong assumption.
+- **r9 (query #10812):** the comparator `(a, b) => a < b ? 1 : -1` never
+  returns `0` for ties, violating the comparator contract — the exact thing
+  #10812 fixed. The auditor flagged it a false positive **and cited the
+  buggy line as evidence**, mistaking the bug for the contract.
+
+A calibration pass (a false-positive call must now quote the source line it
+relies on, or it downgrades) made the auditor's output more useful but did
+**not** stop this: it can cite the buggy code as if that code were the
+specification. This is the benchmark's most important finding, and it is the
+thesis proving itself inside the tool. The judging model was confidently
+wrong. Execution was right. That is exactly why the verdict comes from the
+gate and never from a model — the auditor is a lead, not a ruling, and every
+gap in this table was adjudicated by a human reading source.
+
+## Misses, in daylight
+
+- **s2 (supabase #329):** four gaps, all false positives — the proposals
+  asserted `.length` on a formatted string result. The real fix (a 403
+  org-scoping error message) was never approached. The auditor correctly
+  flagged all four.
+- **r6 (jotai #3354):** a subscriber-notification bug that only manifests
+  after a nested `store.set` with sub/unsub inside a write. Too subtle for
+  the proposer; two false positives instead.
+- **r7 (jotai #3326):** a React component target. The proposer wrote React
+  Testing Library tests with wrong DOM assumptions; the gate correctly
+  classified them `broken_test` rather than gaps. Complex component harnesses
+  are a real reviewer-capability limit.
+- **r10 (zod #5898):** the `__proto__` catchall bug is present, but the
+  proposer asserted the wrong remedy (expected `__proto__` as a visible data
+  key) rather than the real defect (prototype pollution). The bug class was
+  right, the assertion was wrong. Same bug class as defu #156, opposite
+  outcome — see below.
+
+## Assertion quality is the frontier
+
+defu #156 and zod #5898 are both `__proto__` bugs. defu's proposal asserted
+*the prototype was not polluted* and caught the bug. zod's asserted
+*`__proto__` should appear as a data key* and missed it, producing a false
+positive on a real defect. Same vulnerability, opposite result, decided
+entirely by what the model chose to assert. Improving assertion quality —
+not reaching the topic — is the open problem this benchmark surfaces, and it
+is the training signal behind the roadmap's model work.
+
+## Determinism receipts
+
+Every run prints a fingerprint over its verdicts. Six rows were run on two
+separate days, through a proposal-cache hit and a changed advisory layer,
+and reproduced byte-identical fingerprints:
+
+| Row | fingerprint (both runs) |
+|-----|-------------------------|
+| s1 | `d64b434d0833afcf` |
+| s2 | `3811dc33363f9751` |
+| r1 | `6e9bb41299792a83` |
+| r3 | `b6a7bd31e37e5bb7` |
+| r4 | `35d6c88385a680c2` |
+| r10 | `f7214bf3b971ad1c` |
+
+The auditor changed between the two runs; the fingerprints did not, because
+the auditor is advisory and never enters the verdict. When verdict-side code
+changed, every row was rerun on one tool version rather than mixing.
+
+## Reproduce
+
+Each row is a fresh checkout at the parent commit, `--base HEAD` (empty
+change, stranger-mode review), a neutral intent, and the auditor on:
+
+```
+git clone <repo> && cd <repo> && git checkout <parent-sha>
+agentboard review --repo . --target <file> --tests <tests> \
+  --head HEAD --base HEAD --intent "<neutral intent>" \
+  --audit-model claude-sonnet-5 --json-out run.json
+```
+
+Parent SHAs, targets, and neutral intents for all rows are in
+`bench/run_bench.sh`. Environment failures, when they occur, are reported as
+failures, never retried until green.

reviewgate-0.1.0/CONTRIBUTING.md

@@ -0,0 +1,70 @@
+# Contributing to agentboard
+
+Thanks for looking. agentboard is a review gate with one non-negotiable idea:
+**a model may propose, but only executed tests decide.** Contributions are
+welcome as long as they hold that line.
+
+## The invariants (do not break these)
+
+These are the whole reason the tool is trustworthy. A change that violates one
+will not be merged, however useful it looks otherwise.
+
+1. **The verdict is deterministic and model-free.** No LLM sits in the
+   pass/fail path. A behavior becomes a `confirmed_gap` only if a proposed test
+   compiles, runs, and fails its assertion on the real code.
+2. **A garbage test cannot manufacture a finding.** A test that crashes or
+   fails to load is classified against the test (`broken_test`), never against
+   the code under review. Only a named assertion failure is a gap.
+3. **The auditor is advisory.** A second model may flag a likely false
+   positive, with a quoted source line. It never changes a verdict. It has been
+   wrong (see `BENCHMARK.md`); the gate has the final word, always.
+4. **Per-finding isolation.** Every proposed test runs against a clean checkout
+   of the repo; no finding sees another's injected test.
+5. **The engine knows nothing about GitHub.** CI and MCP are adapters over the
+   `--json-out` schema. Keep PR-shaped concepts out of the core.
+
+If your change touches the gate, the classifier, or the cache key, add a test
+that pins the behavior. `tests/` has examples of the style, including
+falsifier invariants (an impossible test must always read as a real failure).
+
+## Development
+
+```
+git clone https://github.com/anp0429/agentboard
+cd agentboard
+pip install -e ".[dev]"
+PYTHONPATH=src python -m pytest tests/ -q
+```
+
+The demo runs with no API key and exercises the deterministic gate:
+
+```
+agentboard demo
+agentboard demo --fixed
+```
+
+A live review needs `OPENAI_API_KEY` (reviewer) and, for the auditor,
+`ANTHROPIC_API_KEY`. Reviews on real repos run vitest, so Node is required for
+those; the Python test suite does not need it.
+
+## Before you open a PR
+
+- `PYTHONPATH=src python -m pytest tests/ -q` passes.
+- `ruff check src/ tests/` is clean (advisory today, tightening over time).
+- New behavior has a test. New verdict-path behavior has a test that would fail
+  without your change.
+- If you changed anything the benchmark exercises, say so; `BENCHMARK.md`
+  documents the method and the honest misses, and we keep it honest.
+
+## What is especially welcome
+
+- New `RepoProfile` presets or `.agentboard.toml` recipes for repos that need
+  scoping (multiple vitest projects, per-package monorepos).
+- A pytest gate (the current gate is vitest-only; `PytestVerifier` exists but
+  speaks the older protocol).
+- Benchmark rows: a merged bugfix PR, run at its parent commit with a neutral
+  intent, scored honestly including misses.
+
+## Reporting a security issue
+
+Please use GitHub's private vulnerability reporting rather than a public issue.

reviewgate-0.1.0/LICENSE

@@ -0,0 +1,5 @@
+MIT License
+
+Copyright (c) 2026 Ankita
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction...