jitcatch 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jitcatch-0.2.0/LICENSE +21 -0
- jitcatch-0.2.0/PKG-INFO +164 -0
- jitcatch-0.2.0/README.md +134 -0
- jitcatch-0.2.0/jitcatch/__init__.py +2 -0
- jitcatch-0.2.0/jitcatch/__main__.py +5 -0
- jitcatch-0.2.0/jitcatch/adapters/__init__.py +36 -0
- jitcatch-0.2.0/jitcatch/adapters/base.py +137 -0
- jitcatch-0.2.0/jitcatch/adapters/golang.py +88 -0
- jitcatch-0.2.0/jitcatch/adapters/java.py +136 -0
- jitcatch-0.2.0/jitcatch/adapters/javascript.py +165 -0
- jitcatch-0.2.0/jitcatch/adapters/python.py +187 -0
- jitcatch-0.2.0/jitcatch/assessors/__init__.py +94 -0
- jitcatch-0.2.0/jitcatch/assessors/failure_parse.py +126 -0
- jitcatch-0.2.0/jitcatch/assessors/llm_judge.py +75 -0
- jitcatch-0.2.0/jitcatch/assessors/rules.py +343 -0
- jitcatch-0.2.0/jitcatch/cli.py +338 -0
- jitcatch-0.2.0/jitcatch/config.py +259 -0
- jitcatch-0.2.0/jitcatch/engine.py +521 -0
- jitcatch-0.2.0/jitcatch/github.py +90 -0
- jitcatch-0.2.0/jitcatch/gitutil.py +231 -0
- jitcatch-0.2.0/jitcatch/hook.py +98 -0
- jitcatch-0.2.0/jitcatch/jsonutil.py +87 -0
- jitcatch-0.2.0/jitcatch/llm/__init__.py +21 -0
- jitcatch-0.2.0/jitcatch/llm/base.py +100 -0
- jitcatch-0.2.0/jitcatch/llm/providers.py +127 -0
- jitcatch-0.2.0/jitcatch/models.py +150 -0
- jitcatch-0.2.0/jitcatch/prompts.py +191 -0
- jitcatch-0.2.0/jitcatch/report.py +163 -0
- jitcatch-0.2.0/jitcatch/targeting.py +44 -0
- jitcatch-0.2.0/jitcatch/templates.py +76 -0
- jitcatch-0.2.0/jitcatch/userconfig.py +188 -0
- jitcatch-0.2.0/jitcatch.egg-info/PKG-INFO +164 -0
- jitcatch-0.2.0/jitcatch.egg-info/SOURCES.txt +43 -0
- jitcatch-0.2.0/jitcatch.egg-info/dependency_links.txt +1 -0
- jitcatch-0.2.0/jitcatch.egg-info/entry_points.txt +2 -0
- jitcatch-0.2.0/jitcatch.egg-info/requires.txt +3 -0
- jitcatch-0.2.0/jitcatch.egg-info/top_level.txt +1 -0
- jitcatch-0.2.0/pyproject.toml +46 -0
- jitcatch-0.2.0/setup.cfg +4 -0
- jitcatch-0.2.0/tests/test_cli.py +212 -0
- jitcatch-0.2.0/tests/test_e2e_languages.py +219 -0
- jitcatch-0.2.0/tests/test_e2e_python.py +202 -0
- jitcatch-0.2.0/tests/test_llm_clients.py +148 -0
- jitcatch-0.2.0/tests/test_seamless.py +158 -0
- jitcatch-0.2.0/tests/test_units.py +173 -0
jitcatch-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ahmad Waqar
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
jitcatch-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: jitcatch
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Just-in-Time catching test generation: LLM-generated tests that fail on risky code changes, with rule-based and LLM-judge false-positive filtering.
|
|
5
|
+
Author: Ahmad Waqar
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/khawjaahmad/jitcatch
|
|
8
|
+
Project-URL: Repository, https://github.com/khawjaahmad/jitcatch
|
|
9
|
+
Project-URL: Issues, https://github.com/khawjaahmad/jitcatch/issues
|
|
10
|
+
Project-URL: Paper, https://arxiv.org/abs/2601.22832
|
|
11
|
+
Keywords: testing,regression,llm,mutation-testing,test-generation,code-review
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Software Development :: Testing
|
|
23
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
24
|
+
Requires-Python: >=3.11
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# JiTCatch
|
|
32
|
+
|
|
33
|
+
Just-in-Time **catching** test generation for any Git repository, modelled on
|
|
34
|
+
Becker et al., *"Just-in-Time Catching Test Generation at Meta"* (FSE Companion '26, arXiv:2601.22832).
|
|
35
|
+
|
|
36
|
+
Hardening tests pass today and guard the future. **Catching tests are generated for one specific change and
|
|
37
|
+
are meant to fail on it**: they pass on the parent revision and fail on the change, exposing a regression
|
|
38
|
+
before it merges. JiTCatch then filters out false positives and asks the author a plain question:
|
|
39
|
+
*"`can_delete()` now returns False for a resource's owner. Is this expected?"*
|
|
40
|
+
|
|
41
|
+
| Paper concept | Where it lives |
|
|
42
|
+
|---|---|
|
|
43
|
+
| Diff risk targeting (DRS) | `targeting.py` (transparent heuristic; `run.min_risk_score`) |
|
|
44
|
+
| Intent-aware workflow: intent → risks → mutants of parent → tests that kill them | `engine.py` `_infer_intent`, `_make_mutant`, `_generate_test` |
|
|
45
|
+
| Mutant must build and pass existing tests | `run.check_mutants_with_existing_tests` |
|
|
46
|
+
| "Dodgy diff" workflow: treat the change itself as a mutant | `engine.py` `_dodgy_diff` |
|
|
47
|
+
| Weak catch = passes on parent, fails on change | `models.GeneratedTest.is_weak_catch` |
|
|
48
|
+
| RubFake rule-based assessor (Tables 2 & 3) | `assessors/rules.py` |
|
|
49
|
+
| LLM-as-judge ensemble, High/Medium/Low → 1/0/−1, **median** | `assessors/llm_judge.py` |
|
|
50
|
+
| Normalised token-probability Yes/No score | `assessors/llm_judge.py` `logprob_score` |
|
|
51
|
+
| Score in [−1, 1], rank, threshold | `assessors/__init__.py`, `run.report_threshold` |
|
|
52
|
+
| Low-friction "is this expected?" report | `report.py` (Markdown, JSON, PR comment) |
|
|
53
|
+
| Coincidental hardening tests as a by-product | `.jitcatch/runs/<id>/hardening/` |
|
|
54
|
+
|
|
55
|
+
## Supported stacks
|
|
56
|
+
|
|
57
|
+
| Language | Runner | Generated test location |
|
|
58
|
+
|---|---|---|
|
|
59
|
+
| Python | pytest | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
60
|
+
| TypeScript / JavaScript | Jest or Vitest (auto-detected per `package.json`) | `<name>.jitcatch-<id>.test.<ext>` |
|
|
61
|
+
| Java | JUnit 4/5 via Maven or Gradle (wrappers preferred) | `src/test/java/<pkg>/<Class>JitcatchTest<ID>.java` |
|
|
62
|
+
| Go | `go test` | `<name>_jitcatch_<id>_test.go`, functions `TestJitcatch<ID>…` |
|
|
63
|
+
|
|
64
|
+
LLM providers: **Anthropic Messages**, **OpenAI Chat Completions**, **OpenAI Responses**, and anything compatible
|
|
65
|
+
with them, with presets for Z.ai GLM Coding Plan (`zai-coding` → `https://api.z.ai/api/coding/paas/v4`,
|
|
66
|
+
`zai-coding-anthropic`, `zai-coding-responses`) and local servers (`ollama`, `lmstudio`, `vllm`).
|
|
67
|
+
No third-party Python dependencies; Python ≥ 3.11.
|
|
68
|
+
|
|
69
|
+
## Install
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
pipx install jitcatch # or: uv tool install jitcatch / pip install jitcatch
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Use
|
|
76
|
+
|
|
77
|
+
Inside any git project, run:
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
jitcatch
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The first time, it asks at most three questions (provider, model, API key) and saves the answers in
|
|
84
|
+
`~/.config/jitcatch/` for every project. Questions whose answer is already known are skipped; for example,
|
|
85
|
+
the key isn't asked for if `ZAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` is already set.
|
|
86
|
+
Run `jitcatch setup --reset` to change provider or model.
|
|
87
|
+
|
|
88
|
+
What gets tested is picked automatically:
|
|
89
|
+
|
|
90
|
+
1. uncommitted changes (including new files), compared with the last commit;
|
|
91
|
+
2. otherwise, commits on your branch that aren't on the default branch yet;
|
|
92
|
+
3. otherwise, the last commit.
|
|
93
|
+
|
|
94
|
+
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index, and run
|
|
95
|
+
artifacts in `.jitcatch/` are hidden through `.git/info/exclude`. To compare specific revisions:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
jitcatch run --base origin/main --head HEAD
|
|
99
|
+
jitcatch run --base main --head feature --json report.json --markdown report.md --fail-on-catch
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The project's own toolchain must work (`node_modules`, Maven/Gradle dependencies, Go modules). For Python,
|
|
103
|
+
tests run in the project's `.venv` so its dependencies are importable; if that environment has no pytest,
|
|
104
|
+
jitcatch installs a private copy into `~/.cache/jitcatch/` and leaves your project untouched.
|
|
105
|
+
`jitcatch doctor --ping` checks configuration, keys, toolchains and provider round-trips.
|
|
106
|
+
|
|
107
|
+
**GitHub Action** – copy `examples/github-workflow.yml`. It needs `fetch-depth: 0`,
|
|
108
|
+
`permissions: pull-requests: write`, your project's normal setup steps, and API keys as secrets.
|
|
109
|
+
It posts one sticky PR comment (updated on each push), writes a job summary, and exposes
|
|
110
|
+
`catches` / `weak-catches` outputs. jitcatch is installed in its own venv so it never shadows your toolchain.
|
|
111
|
+
|
|
112
|
+
**Pre-push hook**
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
jitcatch install-hook # reports only
|
|
116
|
+
jitcatch install-hook --blocking # a reported catch blocks the push
|
|
117
|
+
JITCATCH_SKIP=1 git push # bypass once
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Tool or provider failures never block a push.
|
|
121
|
+
|
|
122
|
+
## How scoring works
|
|
123
|
+
|
|
124
|
+
Every weak catch is scored by up to three assessors, each in [−1 (false positive), 1 (true positive)]:
|
|
125
|
+
|
|
126
|
+
1. **rules**: deterministic false-positive patterns from the paper (broken runner, reflection, type
|
|
127
|
+
mismatch, mocks, private/protected members, NotImplemented, key ordering, undefined names, call-count
|
|
128
|
+
expectations, network down, flakiness) and true-positive patterns (unexpected key/index error, emptied
|
|
129
|
+
container, construction failure, flipped boolean, behaviour change in a refactor, dead-code removal or
|
|
130
|
+
additive change, unexpected null, RBAC). A **high** false-positive pattern discards the catch before any
|
|
131
|
+
judge is called (`assessors.discard_on_high_fp_rule`).
|
|
132
|
+
2. **llm_bucket**: median of the judges' High/Medium/Low.
|
|
133
|
+
3. **llm_logprob** (optional): P(Yes) vs P(No) for the first answer token from an `openai_chat` provider
|
|
134
|
+
with `logprobs = true`.
|
|
135
|
+
|
|
136
|
+
The combined score is a weighted mean of the available scores (`assessors.weights`); catches at or above
|
|
137
|
+
`run.report_threshold` are reported, best first. All judge rationales are kept individually.
|
|
138
|
+
|
|
139
|
+
## Output
|
|
140
|
+
|
|
141
|
+
`.jitcatch/runs/<timestamp>-<sha>/` contains `report.json` (everything: intent, risks, mutants, every test with
|
|
142
|
+
its runs on parent / mutant / change, all assessor details), `report.md`, `weak_catches/` and `hardening/`
|
|
143
|
+
(tests that pass on both revisions; candidates to add to your suite).
|
|
144
|
+
|
|
145
|
+
## Cost and safety notes
|
|
146
|
+
|
|
147
|
+
* LLM calls per changed file are about 1 intent call (shared) + 2 × risks (mutant + test) + dodgy tests +
|
|
148
|
+
repairs, plus judges for each weak catch that is not auto-discarded. Bound it with `max_target_files`,
|
|
149
|
+
`max_risks_per_file`, `dodgy_tests_per_file`, `max_fix_attempts`, `min_risk_score`.
|
|
150
|
+
* **Generated tests are LLM-written code executed on the machine** (CI runner or, with the hook, your laptop).
|
|
151
|
+
Run it where that is acceptable.
|
|
152
|
+
* Targeting is a heuristic stand-in for Meta's learned risk model, which needs historical data.
|
|
153
|
+
* Logprob judging only reads Chat Completions logprobs; whether a given server returns them varies
|
|
154
|
+
(`jitcatch doctor --ping` reports it).
|
|
155
|
+
|
|
156
|
+
## Development
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
pip install -e '.[dev]' && pytest
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
The suite runs the full pipeline end-to-end with a scripted LLM against real pytest, Go, Vitest (TypeScript)
|
|
163
|
+
and Jest (JavaScript) projects, and against HTTP fake servers for the provider and GitHub wire formats.
|
|
164
|
+
Java/JUnit execution is covered by placement/classification unit tests only.
|
jitcatch-0.2.0/README.md
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# JiTCatch
|
|
2
|
+
|
|
3
|
+
Just-in-Time **catching** test generation for any Git repository, modelled on
|
|
4
|
+
Becker et al., *"Just-in-Time Catching Test Generation at Meta"* (FSE Companion '26, arXiv:2601.22832).
|
|
5
|
+
|
|
6
|
+
Hardening tests pass today and guard the future. **Catching tests are generated for one specific change and
|
|
7
|
+
are meant to fail on it**: they pass on the parent revision and fail on the change, exposing a regression
|
|
8
|
+
before it merges. JiTCatch then filters out false positives and asks the author a plain question:
|
|
9
|
+
*"`can_delete()` now returns False for a resource's owner. Is this expected?"*
|
|
10
|
+
|
|
11
|
+
| Paper concept | Where it lives |
|
|
12
|
+
|---|---|
|
|
13
|
+
| Diff risk targeting (DRS) | `targeting.py` (transparent heuristic; `run.min_risk_score`) |
|
|
14
|
+
| Intent-aware workflow: intent → risks → mutants of parent → tests that kill them | `engine.py` `_infer_intent`, `_make_mutant`, `_generate_test` |
|
|
15
|
+
| Mutant must build and pass existing tests | `run.check_mutants_with_existing_tests` |
|
|
16
|
+
| "Dodgy diff" workflow: treat the change itself as a mutant | `engine.py` `_dodgy_diff` |
|
|
17
|
+
| Weak catch = passes on parent, fails on change | `models.GeneratedTest.is_weak_catch` |
|
|
18
|
+
| RubFake rule-based assessor (Tables 2 & 3) | `assessors/rules.py` |
|
|
19
|
+
| LLM-as-judge ensemble, High/Medium/Low → 1/0/−1, **median** | `assessors/llm_judge.py` |
|
|
20
|
+
| Normalised token-probability Yes/No score | `assessors/llm_judge.py` `logprob_score` |
|
|
21
|
+
| Score in [−1, 1], rank, threshold | `assessors/__init__.py`, `run.report_threshold` |
|
|
22
|
+
| Low-friction "is this expected?" report | `report.py` (Markdown, JSON, PR comment) |
|
|
23
|
+
| Coincidental hardening tests as a by-product | `.jitcatch/runs/<id>/hardening/` |
|
|
24
|
+
|
|
25
|
+
## Supported stacks
|
|
26
|
+
|
|
27
|
+
| Language | Runner | Generated test location |
|
|
28
|
+
|---|---|---|
|
|
29
|
+
| Python | pytest | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
30
|
+
| TypeScript / JavaScript | Jest or Vitest (auto-detected per `package.json`) | `<name>.jitcatch-<id>.test.<ext>` |
|
|
31
|
+
| Java | JUnit 4/5 via Maven or Gradle (wrappers preferred) | `src/test/java/<pkg>/<Class>JitcatchTest<ID>.java` |
|
|
32
|
+
| Go | `go test` | `<name>_jitcatch_<id>_test.go`, functions `TestJitcatch<ID>…` |
|
|
33
|
+
|
|
34
|
+
LLM providers: **Anthropic Messages**, **OpenAI Chat Completions**, **OpenAI Responses**, and anything compatible
|
|
35
|
+
with them, with presets for Z.ai GLM Coding Plan (`zai-coding` → `https://api.z.ai/api/coding/paas/v4`,
|
|
36
|
+
`zai-coding-anthropic`, `zai-coding-responses`) and local servers (`ollama`, `lmstudio`, `vllm`).
|
|
37
|
+
No third-party Python dependencies; Python ≥ 3.11.
|
|
38
|
+
|
|
39
|
+
## Install
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pipx install jitcatch # or: uv tool install jitcatch / pip install jitcatch
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Use
|
|
46
|
+
|
|
47
|
+
Inside any git project, run:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
jitcatch
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The first time, it asks at most three questions (provider, model, API key) and saves the answers in
|
|
54
|
+
`~/.config/jitcatch/` for every project. Questions whose answer is already known are skipped; for example,
|
|
55
|
+
the key isn't asked for if `ZAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` is already set.
|
|
56
|
+
Run `jitcatch setup --reset` to change provider or model.
|
|
57
|
+
|
|
58
|
+
What gets tested is picked automatically:
|
|
59
|
+
|
|
60
|
+
1. uncommitted changes (including new files), compared with the last commit;
|
|
61
|
+
2. otherwise, commits on your branch that aren't on the default branch yet;
|
|
62
|
+
3. otherwise, the last commit.
|
|
63
|
+
|
|
64
|
+
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index, and run
|
|
65
|
+
artifacts in `.jitcatch/` are hidden through `.git/info/exclude`. To compare specific revisions:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
jitcatch run --base origin/main --head HEAD
|
|
69
|
+
jitcatch run --base main --head feature --json report.json --markdown report.md --fail-on-catch
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The project's own toolchain must work (`node_modules`, Maven/Gradle dependencies, Go modules). For Python,
|
|
73
|
+
tests run in the project's `.venv` so its dependencies are importable; if that environment has no pytest,
|
|
74
|
+
jitcatch installs a private copy into `~/.cache/jitcatch/` and leaves your project untouched.
|
|
75
|
+
`jitcatch doctor --ping` checks configuration, keys, toolchains and provider round-trips.
|
|
76
|
+
|
|
77
|
+
**GitHub Action** – copy `examples/github-workflow.yml`. It needs `fetch-depth: 0`,
|
|
78
|
+
`permissions: pull-requests: write`, your project's normal setup steps, and API keys as secrets.
|
|
79
|
+
It posts one sticky PR comment (updated on each push), writes a job summary, and exposes
|
|
80
|
+
`catches` / `weak-catches` outputs. jitcatch is installed in its own venv so it never shadows your toolchain.
|
|
81
|
+
|
|
82
|
+
**Pre-push hook**
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
jitcatch install-hook # reports only
|
|
86
|
+
jitcatch install-hook --blocking # a reported catch blocks the push
|
|
87
|
+
JITCATCH_SKIP=1 git push # bypass once
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Tool or provider failures never block a push.
|
|
91
|
+
|
|
92
|
+
## How scoring works
|
|
93
|
+
|
|
94
|
+
Every weak catch is scored by up to three assessors, each in [−1 (false positive), 1 (true positive)]:
|
|
95
|
+
|
|
96
|
+
1. **rules**: deterministic false-positive patterns from the paper (broken runner, reflection, type
|
|
97
|
+
mismatch, mocks, private/protected members, NotImplemented, key ordering, undefined names, call-count
|
|
98
|
+
expectations, network down, flakiness) and true-positive patterns (unexpected key/index error, emptied
|
|
99
|
+
container, construction failure, flipped boolean, behaviour change in a refactor, dead-code removal or
|
|
100
|
+
additive change, unexpected null, RBAC). A **high** false-positive pattern discards the catch before any
|
|
101
|
+
judge is called (`assessors.discard_on_high_fp_rule`).
|
|
102
|
+
2. **llm_bucket**: median of the judges' High/Medium/Low.
|
|
103
|
+
3. **llm_logprob** (optional): P(Yes) vs P(No) for the first answer token from an `openai_chat` provider
|
|
104
|
+
with `logprobs = true`.
|
|
105
|
+
|
|
106
|
+
The combined score is a weighted mean of the available scores (`assessors.weights`); catches at or above
|
|
107
|
+
`run.report_threshold` are reported, best first. All judge rationales are kept individually.
|
|
108
|
+
|
|
109
|
+
## Output
|
|
110
|
+
|
|
111
|
+
`.jitcatch/runs/<timestamp>-<sha>/` contains `report.json` (everything: intent, risks, mutants, every test with
|
|
112
|
+
its runs on parent / mutant / change, all assessor details), `report.md`, `weak_catches/` and `hardening/`
|
|
113
|
+
(tests that pass on both revisions; candidates to add to your suite).
|
|
114
|
+
|
|
115
|
+
## Cost and safety notes
|
|
116
|
+
|
|
117
|
+
* LLM calls per changed file are about 1 intent call (shared) + 2 × risks (mutant + test) + dodgy tests +
|
|
118
|
+
repairs, plus judges for each weak catch that is not auto-discarded. Bound it with `max_target_files`,
|
|
119
|
+
`max_risks_per_file`, `dodgy_tests_per_file`, `max_fix_attempts`, `min_risk_score`.
|
|
120
|
+
* **Generated tests are LLM-written code executed on the machine** (CI runner or, with the hook, your laptop).
|
|
121
|
+
Run it where that is acceptable.
|
|
122
|
+
* Targeting is a heuristic stand-in for Meta's learned risk model, which needs historical data.
|
|
123
|
+
* Logprob judging only reads Chat Completions logprobs; whether a given server returns them varies
|
|
124
|
+
(`jitcatch doctor --ping` reports it).
|
|
125
|
+
|
|
126
|
+
## Development
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
pip install -e '.[dev]' && pytest
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
The suite runs the full pipeline end-to-end with a scripted LLM against real pytest, Go, Vitest (TypeScript)
|
|
133
|
+
and Jest (JavaScript) projects, and against HTTP fake servers for the provider and GitHub wire formats.
|
|
134
|
+
Java/JUnit execution is covered by placement/classification unit tests only.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .base import Adapter, TestPlacement
|
|
7
|
+
from .golang import GoAdapter
|
|
8
|
+
from .java import JavaAdapter
|
|
9
|
+
from .javascript import TypeScriptAdapter
|
|
10
|
+
from .python import PythonAdapter
|
|
11
|
+
|
|
12
|
+
ADAPTER_CLASSES: dict[str, type[Adapter]] = {
|
|
13
|
+
"python": PythonAdapter,
|
|
14
|
+
"typescript": TypeScriptAdapter,
|
|
15
|
+
"java": JavaAdapter,
|
|
16
|
+
"go": GoAdapter,
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def build_adapters(settings: dict[str, Any], repo: Path) -> list[Adapter]:
|
|
21
|
+
out = []
|
|
22
|
+
for name, cls in ADAPTER_CLASSES.items():
|
|
23
|
+
s = settings.get(name, {})
|
|
24
|
+
if s.get("enabled", True):
|
|
25
|
+
out.append(cls(s, repo))
|
|
26
|
+
return out
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def adapter_for(adapters: list[Adapter], rel_path: str) -> Adapter | None:
|
|
30
|
+
for a in adapters:
|
|
31
|
+
if a.handles(rel_path):
|
|
32
|
+
return a
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
__all__ = ["Adapter", "TestPlacement", "build_adapters", "adapter_for", "ADAPTER_CLASSES"]
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""Language adapter interface: where generated tests go, how they run, and how outcomes are classified."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
import subprocess
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from ..models import Outcome, TestRun
|
|
13
|
+
|
|
14
|
+
SKIP_DIRS = {".git", ".jitcatch", "node_modules", ".venv", "venv", "target", "build", "dist", "__pycache__", "vendor"}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class TestPlacement:
|
|
19
|
+
__test__ = False
|
|
20
|
+
rel_path: str # repo-relative path of the generated test file
|
|
21
|
+
instructions: str # language/framework-specific constraints for the LLM
|
|
22
|
+
test_name_prefix: str = "" # e.g. Go test function prefix used to select the test
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Adapter:
|
|
26
|
+
name: str = "base"
|
|
27
|
+
fence_langs: tuple[str, ...] = ()
|
|
28
|
+
source_exts: tuple[str, ...] = ()
|
|
29
|
+
|
|
30
|
+
def __init__(self, settings: dict[str, Any], repo: Path):
|
|
31
|
+
self.settings = settings
|
|
32
|
+
self.repo = repo
|
|
33
|
+
|
|
34
|
+
# -- classification ---------------------------------------------------------------
|
|
35
|
+
def handles(self, rel_path: str) -> bool:
|
|
36
|
+
return rel_path.endswith(self.source_exts) and not self.is_test_file(rel_path)
|
|
37
|
+
|
|
38
|
+
def is_test_file(self, rel_path: str) -> bool:
|
|
39
|
+
raise NotImplementedError
|
|
40
|
+
|
|
41
|
+
# -- placement & prompting --------------------------------------------------------
|
|
42
|
+
def placement(self, source_rel: str, test_id: str, source_text: str) -> TestPlacement:
|
|
43
|
+
raise NotImplementedError
|
|
44
|
+
|
|
45
|
+
def related_test(self, source_rel: str) -> str | None:
|
|
46
|
+
"""Repo-relative path of an existing test that most likely tests `source_rel`."""
|
|
47
|
+
stem = Path(source_rel).stem.lower()
|
|
48
|
+
best: tuple[int, str] | None = None
|
|
49
|
+
src_dir = Path(source_rel).parent
|
|
50
|
+
for dirpath, dirnames, filenames in os.walk(self.repo):
|
|
51
|
+
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
|
52
|
+
for fn in filenames:
|
|
53
|
+
rel = str(Path(dirpath, fn).relative_to(self.repo))
|
|
54
|
+
if not self.is_test_file(rel) or "jitcatch" in fn.lower():
|
|
55
|
+
continue
|
|
56
|
+
name = fn.lower()
|
|
57
|
+
if stem and stem in name:
|
|
58
|
+
# prefer tests closest to the source file
|
|
59
|
+
distance = len(set(Path(rel).parent.parts) ^ set(src_dir.parts))
|
|
60
|
+
score = distance
|
|
61
|
+
if best is None or score < best[0]:
|
|
62
|
+
best = (score, rel)
|
|
63
|
+
return best[1] if best else None
|
|
64
|
+
|
|
65
|
+
def any_test_example(self) -> str | None:
|
|
66
|
+
for dirpath, dirnames, filenames in os.walk(self.repo):
|
|
67
|
+
dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
|
|
68
|
+
for fn in sorted(filenames):
|
|
69
|
+
rel = str(Path(dirpath, fn).relative_to(self.repo))
|
|
70
|
+
if self.is_test_file(rel) and "jitcatch" not in fn.lower():
|
|
71
|
+
return rel
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
# -- execution --------------------------------------------------------------------
|
|
75
|
+
def run_test(self, worktree: Path, test_rel: str, placement: TestPlacement, timeout: float) -> TestRun:
|
|
76
|
+
raise NotImplementedError
|
|
77
|
+
|
|
78
|
+
def run_existing_tests(self, worktree: Path, source_rel: str, timeout: float) -> TestRun | None:
|
|
79
|
+
"""Run the project's existing tests relevant to `source_rel`. None = unsupported."""
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
def available(self) -> tuple[bool, str]:
|
|
83
|
+
"""Is the toolchain usable? Returns (ok, reason)."""
|
|
84
|
+
return True, ""
|
|
85
|
+
|
|
86
|
+
def can_test(self, rel_path: str) -> tuple[bool, str]:
|
|
87
|
+
"""Can this particular file be tested (e.g. its package has a supported runner)? Returns (ok, reason)."""
|
|
88
|
+
return True, ""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# -- helpers ------------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
def run_cmd(cmd: list[str], cwd: Path, timeout: float, env_extra: dict[str, str] | None = None) -> tuple[int, str, float, bool]:
|
|
94
|
+
"""Run a command; returns (exit_code, combined_output, seconds, timed_out)."""
|
|
95
|
+
env = dict(os.environ)
|
|
96
|
+
env.update({"CI": "1", "NO_COLOR": "1", "FORCE_COLOR": "0", "PYTHONDONTWRITEBYTECODE": "1"})
|
|
97
|
+
if env_extra:
|
|
98
|
+
env.update(env_extra)
|
|
99
|
+
t0 = time.monotonic()
|
|
100
|
+
try:
|
|
101
|
+
proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, encoding="utf-8",
|
|
102
|
+
errors="replace", timeout=timeout, env=env)
|
|
103
|
+
out = (proc.stdout or "") + ("\n" + proc.stderr if proc.stderr else "")
|
|
104
|
+
return proc.returncode, strip_ansi(out), time.monotonic() - t0, False
|
|
105
|
+
except subprocess.TimeoutExpired as e:
|
|
106
|
+
out = (e.stdout or b"")
|
|
107
|
+
out = out.decode("utf-8", "replace") if isinstance(out, bytes) else out
|
|
108
|
+
return -1, strip_ansi(out) + f"\n[jitcatch] TIMEOUT after {timeout}s", time.monotonic() - t0, True
|
|
109
|
+
except FileNotFoundError as e:
|
|
110
|
+
return -2, f"[jitcatch] command not found: {e}", time.monotonic() - t0, False
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
_ANSI = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def strip_ansi(s: str) -> str:
|
|
117
|
+
return _ANSI.sub("", s)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def make_run(outcome: Outcome, code: int, out: str, secs: float, cmd: list[str]) -> TestRun:
|
|
121
|
+
return TestRun(outcome=outcome, exit_code=code, output=truncate_output(out), duration_s=round(secs, 2), command=cmd)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def truncate_output(out: str, head: int = 3000, tail: int = 12000) -> str:
|
|
125
|
+
if len(out) <= head + tail:
|
|
126
|
+
return out
|
|
127
|
+
return out[:head] + f"\n... [{len(out) - head - tail} chars truncated] ...\n" + out[-tail:]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def nearest_dir_with(start: Path, stop: Path, names: tuple[str, ...]) -> Path | None:
|
|
131
|
+
d = start
|
|
132
|
+
while True:
|
|
133
|
+
if any((d / n).exists() for n in names):
|
|
134
|
+
return d
|
|
135
|
+
if d == stop or d.parent == d:
|
|
136
|
+
return None
|
|
137
|
+
d = d.parent
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"""Go adapter (go test)."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
|
|
8
|
+
from ..models import Outcome, TestRun
|
|
9
|
+
from .base import Adapter, TestPlacement, make_run, nearest_dir_with, run_cmd
|
|
10
|
+
|
|
11
|
+
_PKG_RE = re.compile(r"^\s*package\s+(\w+)", re.M)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class GoAdapter(Adapter):
|
|
15
|
+
name = "go"
|
|
16
|
+
fence_langs = ("go", "golang")
|
|
17
|
+
source_exts = (".go",)
|
|
18
|
+
|
|
19
|
+
def is_test_file(self, rel_path: str) -> bool:
|
|
20
|
+
return rel_path.endswith("_test.go")
|
|
21
|
+
|
|
22
|
+
def handles(self, rel_path: str) -> bool:
|
|
23
|
+
return super().handles(rel_path) and "/vendor/" not in f"/{rel_path}" and "/testdata/" not in f"/{rel_path}"
|
|
24
|
+
|
|
25
|
+
def related_test(self, source_rel: str) -> str | None:
|
|
26
|
+
p = PurePosixPath(source_rel)
|
|
27
|
+
cand = p.with_name(p.stem + "_test.go")
|
|
28
|
+
if (self.repo / cand).exists():
|
|
29
|
+
return str(cand)
|
|
30
|
+
for f in sorted((self.repo / p.parent).glob("*_test.go")):
|
|
31
|
+
if "jitcatch" not in f.name:
|
|
32
|
+
return str(f.relative_to(self.repo))
|
|
33
|
+
return None
|
|
34
|
+
|
|
35
|
+
def available(self) -> tuple[bool, str]:
|
|
36
|
+
code, _, _, _ = run_cmd(["go", "version"], self.repo, 60)
|
|
37
|
+
return (code == 0, "" if code == 0 else "go not available")
|
|
38
|
+
|
|
39
|
+
def placement(self, source_rel: str, test_id: str, source_text: str) -> TestPlacement:
|
|
40
|
+
p = PurePosixPath(source_rel)
|
|
41
|
+
m = _PKG_RE.search(source_text)
|
|
42
|
+
pkg = m.group(1) if m else "main"
|
|
43
|
+
rel = str(p.parent / f"{p.stem}_jitcatch_{test_id}_test.go")
|
|
44
|
+
prefix = f"TestJitcatch{test_id.upper()}"
|
|
45
|
+
instr = (
|
|
46
|
+
f"Framework: Go's standard `testing` package. The file must start with `package {pkg}` (same package, "
|
|
47
|
+
f"so unexported identifiers are accessible) and every test function name must start with `{prefix}` "
|
|
48
|
+
f"(e.g. `func {prefix}Something(t *testing.T)`). Only import the standard library and modules already "
|
|
49
|
+
"in go.mod. Do not use network, sleeps, randomness or time.Now; avoid reflect."
|
|
50
|
+
)
|
|
51
|
+
return TestPlacement(rel_path=rel, instructions=instr, test_name_prefix=prefix)
|
|
52
|
+
|
|
53
|
+
def _module(self, base: Path, rel: str) -> Path:
|
|
54
|
+
return nearest_dir_with((base / rel).parent, base, ("go.mod",)) or base
|
|
55
|
+
|
|
56
|
+
def run_test(self, worktree: Path, test_rel: str, placement: TestPlacement, timeout: float) -> TestRun:
|
|
57
|
+
mod = self._module(worktree, test_rel)
|
|
58
|
+
pkg_dir = os.path.relpath(worktree / PurePosixPath(test_rel).parent, mod)
|
|
59
|
+
target = "./" + pkg_dir.replace(os.sep, "/") if pkg_dir != "." else "."
|
|
60
|
+
cmd = ["go", "test", target, "-run", f"^{placement.test_name_prefix}", "-count=1", "-v",
|
|
61
|
+
*self.settings.get("extra_args", [])]
|
|
62
|
+
code, out, secs, timed_out = run_cmd(cmd, mod, timeout)
|
|
63
|
+
return make_run(self._classify(code, out, timed_out), code, out, secs, cmd)
|
|
64
|
+
|
|
65
|
+
def run_existing_tests(self, worktree: Path, source_rel: str, timeout: float) -> TestRun | None:
|
|
66
|
+
mod = self._module(worktree, source_rel)
|
|
67
|
+
pkg_dir = os.path.relpath(worktree / PurePosixPath(source_rel).parent, mod)
|
|
68
|
+
target = "./" + pkg_dir.replace(os.sep, "/") if pkg_dir != "." else "."
|
|
69
|
+
cmd = ["go", "test", target, "-count=1"]
|
|
70
|
+
code, out, secs, timed_out = run_cmd(cmd, mod, timeout)
|
|
71
|
+
outcome = Outcome.PASS if code == 0 else self._classify(code, out, timed_out)
|
|
72
|
+
return make_run(outcome, code, out, secs, cmd)
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _classify(code: int, out: str, timed_out: bool) -> Outcome:
|
|
76
|
+
if timed_out or code == -2:
|
|
77
|
+
return Outcome.INFRA
|
|
78
|
+
if re.search(r"\[(build|setup) failed\]|^# \S+|cannot find package|no required module provides", out, re.M):
|
|
79
|
+
return Outcome.ERROR
|
|
80
|
+
if code == 0:
|
|
81
|
+
if "no tests to run" in out or "[no test files]" in out:
|
|
82
|
+
return Outcome.NO_TESTS
|
|
83
|
+
return Outcome.PASS
|
|
84
|
+
if re.search(r"^--- FAIL|^panic:|^FAIL\s", out, re.M):
|
|
85
|
+
return Outcome.FAIL
|
|
86
|
+
if re.search(r"go: (downloading|error loading)|dial tcp|proxy\.golang\.org", out):
|
|
87
|
+
return Outcome.INFRA
|
|
88
|
+
return Outcome.ERROR
|