reprove 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- reprove-0.1.0/LICENSE +21 -0
- reprove-0.1.0/PKG-INFO +179 -0
- reprove-0.1.0/README.md +156 -0
- reprove-0.1.0/pyproject.toml +43 -0
- reprove-0.1.0/reprove/__init__.py +3 -0
- reprove-0.1.0/reprove/_pytest_context.py +33 -0
- reprove-0.1.0/reprove/cli.py +126 -0
- reprove-0.1.0/reprove/coverage_map.py +281 -0
- reprove-0.1.0/reprove/diff.py +271 -0
- reprove-0.1.0/reprove/fsutil.py +66 -0
- reprove-0.1.0/reprove/hypotheses.py +217 -0
- reprove-0.1.0/reprove/mutate.py +432 -0
- reprove-0.1.0/reprove/pipeline.py +179 -0
- reprove-0.1.0/reprove/report.py +662 -0
- reprove-0.1.0/reprove/runner.py +228 -0
- reprove-0.1.0/reprove.egg-info/PKG-INFO +179 -0
- reprove-0.1.0/reprove.egg-info/SOURCES.txt +27 -0
- reprove-0.1.0/reprove.egg-info/dependency_links.txt +1 -0
- reprove-0.1.0/reprove.egg-info/entry_points.txt +2 -0
- reprove-0.1.0/reprove.egg-info/requires.txt +9 -0
- reprove-0.1.0/reprove.egg-info/top_level.txt +1 -0
- reprove-0.1.0/setup.cfg +4 -0
- reprove-0.1.0/tests/test_cli.py +109 -0
- reprove-0.1.0/tests/test_coverage_map.py +85 -0
- reprove-0.1.0/tests/test_diff.py +126 -0
- reprove-0.1.0/tests/test_hypotheses.py +184 -0
- reprove-0.1.0/tests/test_mutate.py +212 -0
- reprove-0.1.0/tests/test_report.py +164 -0
- reprove-0.1.0/tests/test_runner.py +97 -0
reprove-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Miguel Jardim
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
reprove-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: reprove
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Verify a pull request with evidence, not opinions.
|
|
5
|
+
Author: Miguel Jardim
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/ping-dev-ui/reprove
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Software Development :: Testing
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Requires-Dist: coverage>=7.0
|
|
16
|
+
Provides-Extra: llm
|
|
17
|
+
Requires-Dist: anthropic; extra == "llm"
|
|
18
|
+
Provides-Extra: test
|
|
19
|
+
Requires-Dist: pytest; extra == "test"
|
|
20
|
+
Requires-Dist: pytest-cov; extra == "test"
|
|
21
|
+
Requires-Dist: ruff; extra == "test"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# reprove
|
|
25
|
+
|
|
26
|
+
**Verify a pull request with evidence, not opinions.**
|
|
27
|
+
|
|
28
|
+
`reprove` runs on a repository and a pull request and answers one question:
|
|
29
|
+
*what evidence exists that this change is safe to release, and what evidence
|
|
30
|
+
is missing?* It is built for the case where the change was written quickly,
|
|
31
|
+
often by an AI tool, and nobody has checked whether the tests actually bite.
|
|
32
|
+
|
|
33
|
+
Every finding carries a status:
|
|
34
|
+
|
|
35
|
+
| status | meaning |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `reproduced` | reprove ran it and saw it |
|
|
38
|
+
| `inferred` | reasoned from the diff, not executed |
|
|
39
|
+
| `needs-evidence` | could not be checked here; the report says what would be needed |
|
|
40
|
+
|
|
41
|
+
reprove never claims more than it ran.
|
|
42
|
+
|
|
43
|
+
## What it does
|
|
44
|
+
|
|
45
|
+
1. **Diff.** `git diff base...head`. Each changed file is classified as source, test or config. Changed source lines are the unit everything else measures against.
|
|
46
|
+
2. **Tests.** Detects and runs the project's test command (`pytest` if `pyproject.toml`/`pytest.ini`/`tests/` exist; `npm test`, `vitest` or `jest` from `package.json`). Captures pass/fail counts, duration and the tail of the output, with a configurable timeout.
|
|
47
|
+
3. **Diff coverage.** Python: the suite runs once under `coverage` with per-test contexts, and executed lines are mapped onto the changed lines: *changed lines executed by tests: X of Y*, listing the uncovered ones. JS/TS: `c8` or `nyc` if the project already has one, otherwise `needs-evidence` with the exact command that would produce it.
|
|
48
|
+
4. **Mutation of changed lines only** (Python). One AST mutator at a time on the changed lines: comparison swap (`<`/`<=`, `==`/`!=`, `>`/`>=`), arithmetic swap (`+`/`-`, `*`/`/`), numeric boundary (`n` to `n+1`, `n-1`), boolean flip, negation removal, `and`/`or`. Each mutant is written into a temp copy of the repository and the tests that cover that line are run (the full suite when coverage is unavailable). Result: `killed` or `survived`, with the exact mutation and line. **A surviving mutant on a changed line is the headline finding: the tests do not detect that change.** Budgets: `--max-mutants` (default 50) and `--time-budget` seconds.
|
|
49
|
+
5. **Hypotheses (optional, bring your own key).** Only if `ANTHROPIC_API_KEY` is set and `--no-llm` is not passed: the diff, test summary and surviving mutants go to the model (default `claude-sonnet-5`, `--model` to change), which proposes a few specific risks, each with a pytest reproduction. reprove writes each reproduction into a temp copy and runs it. A hypothesis is shown **only with its run result**: `reproduced` (the test failed as predicted), `not-reproduced`, or `could-not-run`. Without a key this stage is skipped and the report says so in one line.
|
|
50
|
+
6. **Report.** Markdown sized for a PR comment, and JSON. A verdict line (`release-safe evidence: strong | partial | weak` with the specific gaps), findings ordered by severity (file:line, what was checked, the command run, an output excerpt, status), then the coverage table, the mutant table, the hypotheses table, and *what reprove could not check here*.
|
|
51
|
+
|
|
52
|
+
## Honesty rules
|
|
53
|
+
|
|
54
|
+
- reprove proves what it ran. It does not prove absence of bugs.
|
|
55
|
+
- Coverage of a line is not correctness of a line; a killed mutant is one specific change the tests noticed.
|
|
56
|
+
- Surviving mutants are the point: they are places where a change could ship unnoticed.
|
|
57
|
+
- The hypothesis stage is the only part that uses a model, it is off by default without a key, and every hypothesis is executed before it is shown.
|
|
58
|
+
|
|
59
|
+
## Install
|
|
60
|
+
|
|
61
|
+
Python 3.10 or newer. Install reprove into the **same environment as your project's test dependencies** (it runs `python -m pytest` with its own interpreter).
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install reprove # core: coverage only
|
|
65
|
+
pip install "reprove[llm]" # + anthropic, for the optional hypothesis stage
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
From a checkout:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
git clone https://github.com/ping-dev-ui/reprove
|
|
72
|
+
cd reprove
|
|
73
|
+
python -m venv .venv
|
|
74
|
+
# Windows: .venv\Scripts\activate Linux/macOS: source .venv/bin/activate
|
|
75
|
+
pip install -e ".[llm,test]"
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Usage
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
reprove verify [--base REF] [--head REF] [--format md|json|both] [--out PATH]
|
|
82
|
+
[--max-mutants N] [--time-budget S] [--test-timeout S]
|
|
83
|
+
[--no-llm] [--model NAME] [--max-hypotheses N]
|
|
84
|
+
[--fail-on weak|partial|never] [--repo PATH]
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Defaults: base is `origin/main` if it exists, else `main`; head is `HEAD`; `--fail-on never`; Markdown to stdout.
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
# the last commit, no model
|
|
91
|
+
reprove verify --base HEAD~1 --head HEAD --no-llm
|
|
92
|
+
|
|
93
|
+
# a branch against main, both formats, fail the job on a weak verdict
|
|
94
|
+
reprove verify --base origin/main --format both --out reprove-report --fail-on weak
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Exit codes: `0` ok, `1` the verdict hit `--fail-on`, `2` reprove could not run (not a git repository, unknown ref).
|
|
98
|
+
|
|
99
|
+
Tests, coverage and mutation run against the **working tree**. If the working tree is not at `--head`, or has uncommitted changes, the report says so under *what reprove could not check here*.
|
|
100
|
+
|
|
101
|
+
## GitHub Action
|
|
102
|
+
|
|
103
|
+
```yaml
|
|
104
|
+
name: reprove
|
|
105
|
+
on:
|
|
106
|
+
pull_request:
|
|
107
|
+
|
|
108
|
+
permissions:
|
|
109
|
+
contents: read
|
|
110
|
+
pull-requests: write # to create/update the sticky comment
|
|
111
|
+
|
|
112
|
+
jobs:
|
|
113
|
+
verify:
|
|
114
|
+
runs-on: ubuntu-latest
|
|
115
|
+
steps:
|
|
116
|
+
- uses: actions/checkout@v4
|
|
117
|
+
with:
|
|
118
|
+
ref: ${{ github.event.pull_request.head.sha }} # run against the PR head, not the merge commit
|
|
119
|
+
fetch-depth: 0
|
|
120
|
+
- uses: actions/setup-python@v5
|
|
121
|
+
with:
|
|
122
|
+
python-version: "3.12"
|
|
123
|
+
- run: pip install -e ".[test]" # your project's test dependencies
|
|
124
|
+
- uses: ping-dev-ui/reprove@main
|
|
125
|
+
with:
|
|
126
|
+
max-mutants: "50"
|
|
127
|
+
time-budget: "300"
|
|
128
|
+
fail-on: never # weak | partial | never
|
|
129
|
+
no-llm: "false"
|
|
130
|
+
env:
|
|
131
|
+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # optional (BYOK)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
The action installs reprove, runs `verify` with base `${{ github.event.pull_request.base.sha }}` and head `${{ github.event.pull_request.head.sha }}`, writes `reprove-report.md`/`.json`, uploads them as an artifact, and creates or updates **one sticky PR comment** (marker `<!-- reprove -->`) using the `gh` CLI with `GITHUB_TOKEN`. Inputs: `max-mutants`, `time-budget`, `fail-on`, `no-llm`.
|
|
135
|
+
|
|
136
|
+
**Bring your own key.** The hypothesis stage runs only if you add your own `ANTHROPIC_API_KEY` secret. reprove reads the key from the environment only; it never writes, logs or stores it, and never sends anything anywhere else. Without the secret the stage is skipped and the report says so.
|
|
137
|
+
|
|
138
|
+
## What it proves, and what it cannot
|
|
139
|
+
|
|
140
|
+
Proves (status `reproduced`):
|
|
141
|
+
|
|
142
|
+
- the test suite passes or fails on this working tree, with counts and the command;
|
|
143
|
+
- which changed lines the tests execute, and which they never touch;
|
|
144
|
+
- for each mutant on a changed line, whether the covering tests noticed it;
|
|
145
|
+
- for each model hypothesis, whether its reproduction actually fails.
|
|
146
|
+
|
|
147
|
+
Cannot prove (and says so):
|
|
148
|
+
|
|
149
|
+
- absence of bugs, or correctness of any line: only that specific changes were or were not noticed;
|
|
150
|
+
- anything the tests do not exercise, including code paths reachable only in production;
|
|
151
|
+
- whether a **test-only** change made the tests weaker: a diff with no changed source lines gets at most a `partial` verdict;
|
|
152
|
+
- mutation and hypothesis execution for JS/TS in v1 (detected, reported as `needs-evidence`);
|
|
153
|
+
- a mutant survives trivially when no test executes its line: the report labels these *no test executes this line*;
|
|
154
|
+
- a file that the tests import from somewhere other than the working tree (for example an installed copy of a `src/` package). reprove checks this with an import sentinel: it breaks the file in the temp copy first, and if the covering tests still pass, that file's mutants are reported as `needs-evidence` instead of `survived`.
|
|
155
|
+
|
|
156
|
+
Model-written reproductions are executed like any other test in your project, inside a temp copy: only enable the hypothesis stage on repositories whose tests you would run anyway.
|
|
157
|
+
|
|
158
|
+
## Verdict
|
|
159
|
+
|
|
160
|
+
- **strong**: tests pass, every executable changed line is executed, every mutant run was killed, nothing skipped, no reproduced hypothesis.
|
|
161
|
+
- **partial**: tests pass but something is missing: uncovered changed lines, a few survivors, mutants skipped by a budget, coverage unavailable, or a test-only change.
|
|
162
|
+
- **weak**: no test command, failing or timed-out tests, fewer than half the changed lines executed, at least half the mutants surviving, or a reproduced hypothesis.
|
|
163
|
+
|
|
164
|
+
The gaps are always listed next to the verdict.
|
|
165
|
+
|
|
166
|
+
## Developing
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
pip install -e ".[llm,test]"
|
|
170
|
+
python -m pytest -q # ~40 s; builds real git repos under tmp_path
|
|
171
|
+
ruff check .
|
|
172
|
+
reprove verify --base HEAD~1 --head HEAD --no-llm # run reprove on itself
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
Tests never call the network; the model is mocked. See `docs/demo-self.md` for reprove run on its own repository.
|
|
176
|
+
|
|
177
|
+
## License
|
|
178
|
+
|
|
179
|
+
MIT, copyright 2026 Miguel Jardim.
|
reprove-0.1.0/README.md
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
# reprove
|
|
2
|
+
|
|
3
|
+
**Verify a pull request with evidence, not opinions.**
|
|
4
|
+
|
|
5
|
+
`reprove` runs on a repository and a pull request and answers one question:
|
|
6
|
+
*what evidence exists that this change is safe to release, and what evidence
|
|
7
|
+
is missing?* It is built for the case where the change was written quickly,
|
|
8
|
+
often by an AI tool, and nobody has checked whether the tests actually bite.
|
|
9
|
+
|
|
10
|
+
Every finding carries a status:
|
|
11
|
+
|
|
12
|
+
| status | meaning |
|
|
13
|
+
|---|---|
|
|
14
|
+
| `reproduced` | reprove ran it and saw it |
|
|
15
|
+
| `inferred` | reasoned from the diff, not executed |
|
|
16
|
+
| `needs-evidence` | could not be checked here; the report says what would be needed |
|
|
17
|
+
|
|
18
|
+
reprove never claims more than it ran.
|
|
19
|
+
|
|
20
|
+
## What it does
|
|
21
|
+
|
|
22
|
+
1. **Diff.** `git diff base...head`. Each changed file is classified as source, test or config. Changed source lines are the unit everything else measures against.
|
|
23
|
+
2. **Tests.** Detects and runs the project's test command (`pytest` if `pyproject.toml`/`pytest.ini`/`tests/` exist; `npm test`, `vitest` or `jest` from `package.json`). Captures pass/fail counts, duration and the tail of the output, with a configurable timeout.
|
|
24
|
+
3. **Diff coverage.** Python: the suite runs once under `coverage` with per-test contexts, and executed lines are mapped onto the changed lines: *changed lines executed by tests: X of Y*, listing the uncovered ones. JS/TS: `c8` or `nyc` if the project already has one, otherwise `needs-evidence` with the exact command that would produce it.
|
|
25
|
+
4. **Mutation of changed lines only** (Python). One AST mutator at a time on the changed lines: comparison swap (`<`/`<=`, `==`/`!=`, `>`/`>=`), arithmetic swap (`+`/`-`, `*`/`/`), numeric boundary (`n` to `n+1`, `n-1`), boolean flip, negation removal, `and`/`or`. Each mutant is written into a temp copy of the repository and the tests that cover that line are run (the full suite when coverage is unavailable). Result: `killed` or `survived`, with the exact mutation and line. **A surviving mutant on a changed line is the headline finding: the tests do not detect that change.** Budgets: `--max-mutants` (default 50) and `--time-budget` seconds.
|
|
26
|
+
5. **Hypotheses (optional, bring your own key).** Only if `ANTHROPIC_API_KEY` is set and `--no-llm` is not passed: the diff, test summary and surviving mutants go to the model (default `claude-sonnet-5`, `--model` to change), which proposes a few specific risks, each with a pytest reproduction. reprove writes each reproduction into a temp copy and runs it. A hypothesis is shown **only with its run result**: `reproduced` (the test failed as predicted), `not-reproduced`, or `could-not-run`. Without a key this stage is skipped and the report says so in one line.
|
|
27
|
+
6. **Report.** Markdown sized for a PR comment, and JSON. A verdict line (`release-safe evidence: strong | partial | weak` with the specific gaps), findings ordered by severity (file:line, what was checked, the command run, an output excerpt, status), then the coverage table, the mutant table, the hypotheses table, and *what reprove could not check here*.
|
|
28
|
+
|
|
29
|
+
## Honesty rules
|
|
30
|
+
|
|
31
|
+
- reprove proves what it ran. It does not prove absence of bugs.
|
|
32
|
+
- Coverage of a line is not correctness of a line; a killed mutant is one specific change the tests noticed.
|
|
33
|
+
- Surviving mutants are the point: they are places where a change could ship unnoticed.
|
|
34
|
+
- The hypothesis stage is the only part that uses a model, it is off by default without a key, and every hypothesis is executed before it is shown.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
Python 3.10 or newer. Install reprove into the **same environment as your project's test dependencies** (it runs `python -m pytest` with its own interpreter).
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install reprove # core: coverage only
|
|
42
|
+
pip install "reprove[llm]" # + anthropic, for the optional hypothesis stage
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
From a checkout:
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
git clone https://github.com/ping-dev-ui/reprove
|
|
49
|
+
cd reprove
|
|
50
|
+
python -m venv .venv
|
|
51
|
+
# Windows: .venv\Scripts\activate Linux/macOS: source .venv/bin/activate
|
|
52
|
+
pip install -e ".[llm,test]"
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Usage
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
reprove verify [--base REF] [--head REF] [--format md|json|both] [--out PATH]
|
|
59
|
+
[--max-mutants N] [--time-budget S] [--test-timeout S]
|
|
60
|
+
[--no-llm] [--model NAME] [--max-hypotheses N]
|
|
61
|
+
[--fail-on weak|partial|never] [--repo PATH]
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
Defaults: base is `origin/main` if it exists, else `main`; head is `HEAD`; `--fail-on never`; Markdown to stdout.
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
# the last commit, no model
|
|
68
|
+
reprove verify --base HEAD~1 --head HEAD --no-llm
|
|
69
|
+
|
|
70
|
+
# a branch against main, both formats, fail the job on a weak verdict
|
|
71
|
+
reprove verify --base origin/main --format both --out reprove-report --fail-on weak
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Exit codes: `0` ok, `1` the verdict hit `--fail-on`, `2` reprove could not run (not a git repository, unknown ref).
|
|
75
|
+
|
|
76
|
+
Tests, coverage and mutation run against the **working tree**. If the working tree is not at `--head`, or has uncommitted changes, the report says so under *what reprove could not check here*.
|
|
77
|
+
|
|
78
|
+
## GitHub Action
|
|
79
|
+
|
|
80
|
+
```yaml
|
|
81
|
+
name: reprove
|
|
82
|
+
on:
|
|
83
|
+
pull_request:
|
|
84
|
+
|
|
85
|
+
permissions:
|
|
86
|
+
contents: read
|
|
87
|
+
pull-requests: write # to create/update the sticky comment
|
|
88
|
+
|
|
89
|
+
jobs:
|
|
90
|
+
verify:
|
|
91
|
+
runs-on: ubuntu-latest
|
|
92
|
+
steps:
|
|
93
|
+
- uses: actions/checkout@v4
|
|
94
|
+
with:
|
|
95
|
+
ref: ${{ github.event.pull_request.head.sha }} # run against the PR head, not the merge commit
|
|
96
|
+
fetch-depth: 0
|
|
97
|
+
- uses: actions/setup-python@v5
|
|
98
|
+
with:
|
|
99
|
+
python-version: "3.12"
|
|
100
|
+
- run: pip install -e ".[test]" # your project's test dependencies
|
|
101
|
+
- uses: ping-dev-ui/reprove@main
|
|
102
|
+
with:
|
|
103
|
+
max-mutants: "50"
|
|
104
|
+
time-budget: "300"
|
|
105
|
+
fail-on: never # weak | partial | never
|
|
106
|
+
no-llm: "false"
|
|
107
|
+
env:
|
|
108
|
+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} # optional (BYOK)
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The action installs reprove, runs `verify` with base `${{ github.event.pull_request.base.sha }}` and head `${{ github.event.pull_request.head.sha }}`, writes `reprove-report.md`/`.json`, uploads them as an artifact, and creates or updates **one sticky PR comment** (marker `<!-- reprove -->`) using the `gh` CLI with `GITHUB_TOKEN`. Inputs: `max-mutants`, `time-budget`, `fail-on`, `no-llm`.
|
|
112
|
+
|
|
113
|
+
**Bring your own key.** The hypothesis stage runs only if you add your own `ANTHROPIC_API_KEY` secret. reprove reads the key from the environment only; it never writes, logs or stores it, and never sends anything anywhere else. Without the secret the stage is skipped and the report says so.
|
|
114
|
+
|
|
115
|
+
## What it proves, and what it cannot
|
|
116
|
+
|
|
117
|
+
Proves (status `reproduced`):
|
|
118
|
+
|
|
119
|
+
- the test suite passes or fails on this working tree, with counts and the command;
|
|
120
|
+
- which changed lines the tests execute, and which they never touch;
|
|
121
|
+
- for each mutant on a changed line, whether the covering tests noticed it;
|
|
122
|
+
- for each model hypothesis, whether its reproduction actually fails.
|
|
123
|
+
|
|
124
|
+
Cannot prove (and says so):
|
|
125
|
+
|
|
126
|
+
- absence of bugs, or correctness of any line: only that specific changes were or were not noticed;
|
|
127
|
+
- anything the tests do not exercise, including code paths reachable only in production;
|
|
128
|
+
- whether a **test-only** change made the tests weaker: a diff with no changed source lines gets at most a `partial` verdict;
|
|
129
|
+
- mutation and hypothesis execution for JS/TS in v1 (detected, reported as `needs-evidence`);
|
|
130
|
+
- a mutant survives trivially when no test executes its line: the report labels these *no test executes this line*;
|
|
131
|
+
- a file that the tests import from somewhere other than the working tree (for example an installed copy of a `src/` package). reprove checks this with an import sentinel: it breaks the file in the temp copy first, and if the covering tests still pass, that file's mutants are reported as `needs-evidence` instead of `survived`.
|
|
132
|
+
|
|
133
|
+
Model-written reproductions are executed like any other test in your project, inside a temp copy: only enable the hypothesis stage on repositories whose tests you would run anyway.
|
|
134
|
+
|
|
135
|
+
## Verdict
|
|
136
|
+
|
|
137
|
+
- **strong**: tests pass, every executable changed line is executed, every mutant run was killed, nothing skipped, no reproduced hypothesis.
|
|
138
|
+
- **partial**: tests pass but something is missing: uncovered changed lines, a few survivors, mutants skipped by a budget, coverage unavailable, or a test-only change.
|
|
139
|
+
- **weak**: no test command, failing or timed-out tests, fewer than half the changed lines executed, at least half the mutants surviving, or a reproduced hypothesis.
|
|
140
|
+
|
|
141
|
+
The gaps are always listed next to the verdict.
|
|
142
|
+
|
|
143
|
+
## Developing
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
pip install -e ".[llm,test]"
|
|
147
|
+
python -m pytest -q # ~40 s; builds real git repos under tmp_path
|
|
148
|
+
ruff check .
|
|
149
|
+
reprove verify --base HEAD~1 --head HEAD --no-llm # run reprove on itself
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Tests never call the network; the model is mocked. See `docs/demo-self.md` for reprove run on its own repository.
|
|
153
|
+
|
|
154
|
+
## License
|
|
155
|
+
|
|
156
|
+
MIT, copyright 2026 Miguel Jardim.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "reprove"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Verify a pull request with evidence, not opinions."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = {text = "MIT"}
|
|
11
|
+
authors = [{name = "Miguel Jardim"}]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dependencies = ["coverage>=7.0"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Topic :: Software Development :: Testing",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.optional-dependencies]
|
|
22
|
+
llm = ["anthropic"]
|
|
23
|
+
test = ["pytest", "pytest-cov", "ruff"]
|
|
24
|
+
|
|
25
|
+
[project.scripts]
|
|
26
|
+
reprove = "reprove.cli:main"
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/ping-dev-ui/reprove"
|
|
30
|
+
|
|
31
|
+
[tool.setuptools.packages.find]
|
|
32
|
+
include = ["reprove*"]
|
|
33
|
+
|
|
34
|
+
[tool.pytest.ini_options]
|
|
35
|
+
testpaths = ["tests"]
|
|
36
|
+
addopts = "-p no:cacheprovider"
|
|
37
|
+
|
|
38
|
+
[tool.ruff]
|
|
39
|
+
line-length = 100
|
|
40
|
+
target-version = "py310"
|
|
41
|
+
|
|
42
|
+
[tool.ruff.lint]
|
|
43
|
+
select = ["E", "F", "W", "I", "UP", "B"]
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""pytest plugin: label coverage data with the running test's node id.
|
|
2
|
+
|
|
3
|
+
Loaded with `pytest -p reprove._pytest_context` inside `coverage run`, so each
|
|
4
|
+
executed line records which tests executed it. Those node ids are what the
|
|
5
|
+
mutation stage re-runs per mutant.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections.abc import Iterator
|
|
11
|
+
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
import coverage
|
|
16
|
+
except ImportError: # pragma: no cover - coverage is a hard dependency of reprove
|
|
17
|
+
coverage = None # type: ignore[assignment]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _current_coverage(): # type: ignore[no-untyped-def]
|
|
21
|
+
if coverage is None:
|
|
22
|
+
return None
|
|
23
|
+
return coverage.Coverage.current()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@pytest.hookimpl(hookwrapper=True)
|
|
27
|
+
def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None) -> Iterator[None]:
|
|
28
|
+
cov = _current_coverage()
|
|
29
|
+
if cov is not None:
|
|
30
|
+
cov.switch_context(item.nodeid)
|
|
31
|
+
yield
|
|
32
|
+
if cov is not None:
|
|
33
|
+
cov.switch_context("")
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Command line: `reprove verify [...]`. Exit code follows --fail-on."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
from reprove import __version__, hypotheses, mutate, report
|
|
10
|
+
from reprove.diff import DiffError
|
|
11
|
+
from reprove.pipeline import VerifyOptions, verify
|
|
12
|
+
from reprove.runner import DEFAULT_TIMEOUT
|
|
13
|
+
|
|
14
|
+
EXIT_OK = 0
|
|
15
|
+
EXIT_FAIL_ON = 1
|
|
16
|
+
EXIT_ERROR = 2
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
20
|
+
parser = argparse.ArgumentParser(
|
|
21
|
+
prog="reprove", description="Verify a pull request with evidence."
|
|
22
|
+
)
|
|
23
|
+
parser.add_argument("--version", action="version", version=f"reprove {__version__}")
|
|
24
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
25
|
+
v = sub.add_parser("verify", help="verify base...head of a repository")
|
|
26
|
+
v.add_argument("--repo", default=".", help="repository path (default: current directory)")
|
|
27
|
+
v.add_argument(
|
|
28
|
+
"--base", default=None, help="base ref (default: origin/main if it exists, else main)"
|
|
29
|
+
)
|
|
30
|
+
v.add_argument("--head", default="HEAD", help="head ref (default: HEAD)")
|
|
31
|
+
v.add_argument("--format", choices=("md", "json", "both"), default="md")
|
|
32
|
+
v.add_argument("--max-mutants", type=int, default=mutate.DEFAULT_MAX_MUTANTS)
|
|
33
|
+
v.add_argument("--time-budget", type=float, default=None, help="seconds for the mutation stage")
|
|
34
|
+
v.add_argument(
|
|
35
|
+
"--test-timeout", type=float, default=DEFAULT_TIMEOUT, help="seconds per test run"
|
|
36
|
+
)
|
|
37
|
+
v.add_argument(
|
|
38
|
+
"--no-llm", action="store_true", help="skip the hypothesis stage even if a key is set"
|
|
39
|
+
)
|
|
40
|
+
v.add_argument("--model", default=hypotheses.DEFAULT_MODEL)
|
|
41
|
+
v.add_argument("--max-hypotheses", type=int, default=hypotheses.DEFAULT_MAX_HYPOTHESES)
|
|
42
|
+
v.add_argument("--fail-on", choices=("weak", "partial", "never"), default="never")
|
|
43
|
+
v.add_argument(
|
|
44
|
+
"--out",
|
|
45
|
+
default=None,
|
|
46
|
+
help="write the report here (with --format both: PATH.md and PATH.json)",
|
|
47
|
+
)
|
|
48
|
+
return parser
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def main(argv: list[str] | None = None) -> int:
|
|
52
|
+
args = build_parser().parse_args(argv)
|
|
53
|
+
if args.command == "verify":
|
|
54
|
+
return cmd_verify(args)
|
|
55
|
+
return EXIT_ERROR # pragma: no cover - argparse enforces the subcommand
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def cmd_verify(args: argparse.Namespace) -> int:
|
|
59
|
+
repo = Path(args.repo)
|
|
60
|
+
if not (repo / ".git").exists() and not _inside_git(repo):
|
|
61
|
+
print(f"reprove: {repo} is not a git repository", file=sys.stderr)
|
|
62
|
+
return EXIT_ERROR
|
|
63
|
+
opts = VerifyOptions(
|
|
64
|
+
base=args.base,
|
|
65
|
+
head=args.head,
|
|
66
|
+
max_mutants=args.max_mutants,
|
|
67
|
+
time_budget=args.time_budget,
|
|
68
|
+
test_timeout=args.test_timeout,
|
|
69
|
+
no_llm=args.no_llm,
|
|
70
|
+
model=args.model,
|
|
71
|
+
max_hypotheses=args.max_hypotheses,
|
|
72
|
+
)
|
|
73
|
+
try:
|
|
74
|
+
result = verify(repo, opts)
|
|
75
|
+
except DiffError as exc:
|
|
76
|
+
print(f"reprove: {exc}", file=sys.stderr)
|
|
77
|
+
return EXIT_ERROR
|
|
78
|
+
write_outputs(result, args.format, args.out)
|
|
79
|
+
return exit_code(result.verdict.level, args.fail_on)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def write_outputs(result: report.Report, fmt: str, out: str | None) -> None:
|
|
83
|
+
md = report.to_markdown(result)
|
|
84
|
+
js = report.to_json_text(result)
|
|
85
|
+
if out is None:
|
|
86
|
+
if fmt in ("md", "both"):
|
|
87
|
+
sys.stdout.write(md)
|
|
88
|
+
if fmt == "json":
|
|
89
|
+
sys.stdout.write(js)
|
|
90
|
+
if fmt == "both":
|
|
91
|
+
Path("reprove-report.json").write_text(js, encoding="utf-8", newline="\n")
|
|
92
|
+
return
|
|
93
|
+
path = Path(out)
|
|
94
|
+
if fmt == "md":
|
|
95
|
+
path.write_text(md, encoding="utf-8", newline="\n")
|
|
96
|
+
elif fmt == "json":
|
|
97
|
+
path.write_text(js, encoding="utf-8", newline="\n")
|
|
98
|
+
else:
|
|
99
|
+
stem = path.with_suffix("") if path.suffix in (".md", ".json") else path
|
|
100
|
+
stem.with_suffix(".md").write_text(md, encoding="utf-8", newline="\n")
|
|
101
|
+
stem.with_suffix(".json").write_text(js, encoding="utf-8", newline="\n")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def exit_code(level: str, fail_on: str) -> int:
|
|
105
|
+
if fail_on == "never":
|
|
106
|
+
return EXIT_OK
|
|
107
|
+
if fail_on == "weak":
|
|
108
|
+
return EXIT_FAIL_ON if level == "weak" else EXIT_OK
|
|
109
|
+
return EXIT_FAIL_ON if level in ("weak", "partial") else EXIT_OK
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _inside_git(repo: Path) -> bool:
|
|
113
|
+
import subprocess
|
|
114
|
+
|
|
115
|
+
proc = subprocess.run(
|
|
116
|
+
["git", "rev-parse", "--git-dir"],
|
|
117
|
+
cwd=str(repo),
|
|
118
|
+
capture_output=True,
|
|
119
|
+
encoding="utf-8",
|
|
120
|
+
errors="replace",
|
|
121
|
+
)
|
|
122
|
+
return proc.returncode == 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__": # pragma: no cover
|
|
126
|
+
sys.exit(main())
|