jitcatch 0.2.0__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jitcatch-0.2.0 → jitcatch-0.2.1}/PKG-INFO +60 -17
- {jitcatch-0.2.0 → jitcatch-0.2.1}/README.md +59 -16
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/__init__.py +1 -1
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/templates.py +3 -1
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch.egg-info/PKG-INFO +60 -17
- {jitcatch-0.2.0 → jitcatch-0.2.1}/pyproject.toml +1 -1
- {jitcatch-0.2.0 → jitcatch-0.2.1}/LICENSE +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/__main__.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/adapters/__init__.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/adapters/base.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/adapters/golang.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/adapters/java.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/adapters/javascript.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/adapters/python.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/assessors/__init__.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/assessors/failure_parse.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/assessors/llm_judge.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/assessors/rules.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/cli.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/config.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/engine.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/github.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/gitutil.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/hook.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/jsonutil.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/llm/__init__.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/llm/base.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/llm/providers.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/models.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/prompts.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/report.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/targeting.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch/userconfig.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch.egg-info/SOURCES.txt +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch.egg-info/dependency_links.txt +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch.egg-info/entry_points.txt +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch.egg-info/requires.txt +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/jitcatch.egg-info/top_level.txt +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/setup.cfg +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/tests/test_cli.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/tests/test_e2e_languages.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/tests/test_e2e_python.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/tests/test_llm_clients.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/tests/test_seamless.py +0 -0
- {jitcatch-0.2.0 → jitcatch-0.2.1}/tests/test_units.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: jitcatch
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Just-in-Time catching test generation: LLM-generated tests that fail on risky code changes, with rule-based and LLM-judge false-positive filtering.
|
|
5
5
|
Author: Ahmad Waqar
|
|
6
6
|
License: MIT
|
|
@@ -56,8 +56,8 @@ before it merges. JiTCatch then filters out false positives and asks the author
|
|
|
56
56
|
|
|
57
57
|
| Language | Runner | Generated test location |
|
|
58
58
|
|---|---|---|
|
|
59
|
-
| Python | pytest | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
60
|
-
| TypeScript / JavaScript | Jest or Vitest
|
|
59
|
+
| Python | pytest, in the project's `.venv` (see below) | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
60
|
+
| TypeScript / JavaScript | Jest or Vitest, detected from `package.json`; files in packages using other runners (e.g. Playwright only) are skipped | `<name>.jitcatch-<id>.test.<ext>` |
|
|
61
61
|
| Java | JUnit 4/5 via Maven or Gradle (wrappers preferred) | `src/test/java/<pkg>/<Class>JitcatchTest<ID>.java` |
|
|
62
62
|
| Go | `go test` | `<name>_jitcatch_<id>_test.go`, functions `TestJitcatch<ID>…` |
|
|
63
63
|
|
|
@@ -80,10 +80,17 @@ Inside any git project, run:
|
|
|
80
80
|
jitcatch
|
|
81
81
|
```
|
|
82
82
|
|
|
83
|
-
The first time, it asks at most three questions
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
The first time, it asks at most three questions and saves the answers in `~/.config/jitcatch/` for every
|
|
84
|
+
project:
|
|
85
|
+
|
|
86
|
+
1. provider: Z.ai GLM Coding Plan, Anthropic, OpenAI, or Ollama (local, no key);
|
|
87
|
+
2. model: Enter accepts the suggested default (`glm-5.3-flash`, `claude-sonnet-5-5`, `gpt-5`, `qwen2.5-coder:32b`);
|
|
88
|
+
3. API key: input is hidden, and the key is stored in `~/.config/jitcatch/credentials.toml`, readable only by you.
|
|
89
|
+
|
|
90
|
+
Questions whose answer is already known are skipped. For example, the key isn't asked for if
|
|
91
|
+
`ZAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` is set, and nothing is asked once setup is complete.
|
|
92
|
+
Without a terminal (CI, git hooks) jitcatch never prompts; it says what is missing.
|
|
93
|
+
Run `jitcatch setup` to set up ahead of time, or `jitcatch setup --reset` to change provider or model.
|
|
87
94
|
|
|
88
95
|
What gets tested is picked automatically:
|
|
89
96
|
|
|
@@ -91,8 +98,12 @@ What gets tested is picked automatically:
|
|
|
91
98
|
2. otherwise, commits on your branch that aren't on the default branch yet;
|
|
92
99
|
3. otherwise, the last commit.
|
|
93
100
|
|
|
94
|
-
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index,
|
|
95
|
-
artifacts in `.jitcatch/` are hidden through
|
|
101
|
+
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index, generated
|
|
102
|
+
tests run in temporary git worktrees, and run artifacts in `.jitcatch/` are hidden through
|
|
103
|
+
`.git/info/exclude`. Each step is logged as it happens (`risk 2/3: generating mutant…`,
|
|
104
|
+
`model replied in 41s`). Ctrl-C stops cleanly and removes the temporary worktrees.
|
|
105
|
+
|
|
106
|
+
To compare specific revisions:
|
|
96
107
|
|
|
97
108
|
```bash
|
|
98
109
|
jitcatch run --base origin/main --head HEAD
|
|
@@ -104,10 +115,40 @@ tests run in the project's `.venv` so its dependencies are importable; if that e
|
|
|
104
115
|
jitcatch installs a private copy into `~/.cache/jitcatch/` and leaves your project untouched.
|
|
105
116
|
`jitcatch doctor --ping` checks configuration, keys, toolchains and provider round-trips.
|
|
106
117
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
118
|
+
| Command | What it does |
|
|
119
|
+
|---|---|
|
|
120
|
+
| `jitcatch` | Same as `jitcatch run`: test the current change, running first-time setup if needed |
|
|
121
|
+
| `jitcatch run [--base REV] [--head REV]` | Test a change; `--json`, `--markdown`, `--fail-on-catch`, `--force` |
|
|
122
|
+
| `jitcatch setup [--reset]` | Choose provider, model and API key (once per machine) |
|
|
123
|
+
| `jitcatch doctor [--ping]` | Check config, keys and toolchains; `--ping` sends a tiny request to each provider |
|
|
124
|
+
| `jitcatch init` | Write a full `.jitcatch.toml` into the repository (see Configuration) |
|
|
125
|
+
| `jitcatch install-hook` / `uninstall-hook` | Manage the git pre-push hook |
|
|
126
|
+
| `jitcatch github` | Run inside GitHub Actions on a `pull_request` event (used by the action) |
|
|
127
|
+
|
|
128
|
+
## Configuration
|
|
129
|
+
|
|
130
|
+
Settings are layered: built-in defaults, then your user settings from `jitcatch setup`
|
|
131
|
+
(`~/.config/jitcatch/config.toml`), then a repository `.jitcatch.toml` if one exists. API keys come from
|
|
132
|
+
the environment variable named by the provider, falling back to the key saved by `jitcatch setup`.
|
|
133
|
+
|
|
134
|
+
A repository `.jitcatch.toml` is optional for local use. You need one for CI, where there are no user
|
|
135
|
+
settings, or to change the pipeline per project. `jitcatch init` writes a commented starter file. It
|
|
136
|
+
defines a Claude generator and a Claude + GLM judge ensemble, so edit `[providers]`/`[roles]` to match the
|
|
137
|
+
keys you have. Every option is documented in [`examples/jitcatch.toml`](https://github.com/khawjaahmad/jitcatch/blob/main/examples/jitcatch.toml), e.g.
|
|
138
|
+
`max_target_files`, `max_risks_per_file`, `report_threshold`, and `[adapters.python] python = ...` to force an
|
|
139
|
+
interpreter.
|
|
140
|
+
|
|
141
|
+
GLM-5.x models always think. The `zai-coding*` presets use Z.ai's documented defaults (`max_tokens = 65536`,
|
|
142
|
+
`temperature = 1.0`). To trade depth for speed and Coding Plan quota, add
|
|
143
|
+
`extra_body = { reasoning_effort = "high" }` (or `"low"`) to the provider.
|
|
144
|
+
|
|
145
|
+
## CI and hooks
|
|
146
|
+
|
|
147
|
+
**GitHub Action**: commit a `.jitcatch.toml` and copy [`examples/github-workflow.yml`](https://github.com/khawjaahmad/jitcatch/blob/main/examples/github-workflow.yml). It needs
|
|
148
|
+
`fetch-depth: 0`, `permissions: pull-requests: write`, your project's normal setup steps, and the API keys
|
|
149
|
+
named in your config as secrets. It posts one sticky PR comment (updated on each push), writes a job
|
|
150
|
+
summary, and exposes `catches` / `weak-catches` outputs. jitcatch is installed in its own venv so it never
|
|
151
|
+
shadows your toolchain.
|
|
111
152
|
|
|
112
153
|
**Pre-push hook**
|
|
113
154
|
|
|
@@ -138,14 +179,16 @@ The combined score is a weighted mean of the available scores (`assessors.weight
|
|
|
138
179
|
|
|
139
180
|
## Output
|
|
140
181
|
|
|
141
|
-
`.jitcatch/runs/<timestamp>-<sha>/` contains `report.json`
|
|
142
|
-
its runs on parent / mutant / change, all assessor
|
|
143
|
-
|
|
182
|
+
The Markdown report is printed to stdout. `.jitcatch/runs/<timestamp>-<sha>/` contains `report.json`
|
|
183
|
+
(everything: intent, risks, mutants, every test with its runs on parent / mutant / change, all assessor
|
|
184
|
+
details), `report.md`, `weak_catches/`, and `hardening/` when there are any (tests that pass on both
|
|
185
|
+
revisions; candidates to add to your suite).
|
|
144
186
|
|
|
145
187
|
## Cost and safety notes
|
|
146
188
|
|
|
147
189
|
* LLM calls per changed file are about 1 intent call (shared) + 2 × risks (mutant + test) + dodgy tests +
|
|
148
|
-
repairs, plus judges for each weak catch that is not auto-discarded.
|
|
190
|
+
repairs, plus judges for each weak catch that is not auto-discarded. That is typically 10–20 calls, so
|
|
191
|
+
with thinking models a run takes several minutes. Bound it with `max_target_files`,
|
|
149
192
|
`max_risks_per_file`, `dodgy_tests_per_file`, `max_fix_attempts`, `min_risk_score`.
|
|
150
193
|
* **Generated tests are LLM-written code executed on the machine** (CI runner or, with the hook, your laptop).
|
|
151
194
|
Run it where that is acceptable.
|
|
@@ -26,8 +26,8 @@ before it merges. JiTCatch then filters out false positives and asks the author
|
|
|
26
26
|
|
|
27
27
|
| Language | Runner | Generated test location |
|
|
28
28
|
|---|---|---|
|
|
29
|
-
| Python | pytest | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
30
|
-
| TypeScript / JavaScript | Jest or Vitest
|
|
29
|
+
| Python | pytest, in the project's `.venv` (see below) | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
30
|
+
| TypeScript / JavaScript | Jest or Vitest, detected from `package.json`; files in packages using other runners (e.g. Playwright only) are skipped | `<name>.jitcatch-<id>.test.<ext>` |
|
|
31
31
|
| Java | JUnit 4/5 via Maven or Gradle (wrappers preferred) | `src/test/java/<pkg>/<Class>JitcatchTest<ID>.java` |
|
|
32
32
|
| Go | `go test` | `<name>_jitcatch_<id>_test.go`, functions `TestJitcatch<ID>…` |
|
|
33
33
|
|
|
@@ -50,10 +50,17 @@ Inside any git project, run:
|
|
|
50
50
|
jitcatch
|
|
51
51
|
```
|
|
52
52
|
|
|
53
|
-
The first time, it asks at most three questions
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
The first time, it asks at most three questions and saves the answers in `~/.config/jitcatch/` for every
|
|
54
|
+
project:
|
|
55
|
+
|
|
56
|
+
1. provider: Z.ai GLM Coding Plan, Anthropic, OpenAI, or Ollama (local, no key);
|
|
57
|
+
2. model: Enter accepts the suggested default (`glm-5.3-flash`, `claude-sonnet-5-5`, `gpt-5`, `qwen2.5-coder:32b`);
|
|
58
|
+
3. API key: input is hidden, and the key is stored in `~/.config/jitcatch/credentials.toml`, readable only by you.
|
|
59
|
+
|
|
60
|
+
Questions whose answer is already known are skipped. For example, the key isn't asked for if
|
|
61
|
+
`ZAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` is set, and nothing is asked once setup is complete.
|
|
62
|
+
Without a terminal (CI, git hooks) jitcatch never prompts; it says what is missing.
|
|
63
|
+
Run `jitcatch setup` to set up ahead of time, or `jitcatch setup --reset` to change provider or model.
|
|
57
64
|
|
|
58
65
|
What gets tested is picked automatically:
|
|
59
66
|
|
|
@@ -61,8 +68,12 @@ What gets tested is picked automatically:
|
|
|
61
68
|
2. otherwise, commits on your branch that aren't on the default branch yet;
|
|
62
69
|
3. otherwise, the last commit.
|
|
63
70
|
|
|
64
|
-
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index,
|
|
65
|
-
artifacts in `.jitcatch/` are hidden through
|
|
71
|
+
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index, generated
|
|
72
|
+
tests run in temporary git worktrees, and run artifacts in `.jitcatch/` are hidden through
|
|
73
|
+
`.git/info/exclude`. Each step is logged as it happens (`risk 2/3: generating mutant…`,
|
|
74
|
+
`model replied in 41s`). Ctrl-C stops cleanly and removes the temporary worktrees.
|
|
75
|
+
|
|
76
|
+
To compare specific revisions:
|
|
66
77
|
|
|
67
78
|
```bash
|
|
68
79
|
jitcatch run --base origin/main --head HEAD
|
|
@@ -74,10 +85,40 @@ tests run in the project's `.venv` so its dependencies are importable; if that e
|
|
|
74
85
|
jitcatch installs a private copy into `~/.cache/jitcatch/` and leaves your project untouched.
|
|
75
86
|
`jitcatch doctor --ping` checks configuration, keys, toolchains and provider round-trips.
|
|
76
87
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
88
|
+
| Command | What it does |
|
|
89
|
+
|---|---|
|
|
90
|
+
| `jitcatch` | Same as `jitcatch run`: test the current change, running first-time setup if needed |
|
|
91
|
+
| `jitcatch run [--base REV] [--head REV]` | Test a change; `--json`, `--markdown`, `--fail-on-catch`, `--force` |
|
|
92
|
+
| `jitcatch setup [--reset]` | Choose provider, model and API key (once per machine) |
|
|
93
|
+
| `jitcatch doctor [--ping]` | Check config, keys and toolchains; `--ping` sends a tiny request to each provider |
|
|
94
|
+
| `jitcatch init` | Write a full `.jitcatch.toml` into the repository (see Configuration) |
|
|
95
|
+
| `jitcatch install-hook` / `uninstall-hook` | Manage the git pre-push hook |
|
|
96
|
+
| `jitcatch github` | Run inside GitHub Actions on a `pull_request` event (used by the action) |
|
|
97
|
+
|
|
98
|
+
## Configuration
|
|
99
|
+
|
|
100
|
+
Settings are layered: built-in defaults, then your user settings from `jitcatch setup`
|
|
101
|
+
(`~/.config/jitcatch/config.toml`), then a repository `.jitcatch.toml` if one exists. API keys come from
|
|
102
|
+
the environment variable named by the provider, falling back to the key saved by `jitcatch setup`.
|
|
103
|
+
|
|
104
|
+
A repository `.jitcatch.toml` is optional for local use. You need one for CI, where there are no user
|
|
105
|
+
settings, or to change the pipeline per project. `jitcatch init` writes a commented starter file. It
|
|
106
|
+
defines a Claude generator and a Claude + GLM judge ensemble, so edit `[providers]`/`[roles]` to match the
|
|
107
|
+
keys you have. Every option is documented in [`examples/jitcatch.toml`](https://github.com/khawjaahmad/jitcatch/blob/main/examples/jitcatch.toml), e.g.
|
|
108
|
+
`max_target_files`, `max_risks_per_file`, `report_threshold`, and `[adapters.python] python = ...` to force an
|
|
109
|
+
interpreter.
|
|
110
|
+
|
|
111
|
+
GLM-5.x models always think. The `zai-coding*` presets use Z.ai's documented defaults (`max_tokens = 65536`,
|
|
112
|
+
`temperature = 1.0`). To trade depth for speed and Coding Plan quota, add
|
|
113
|
+
`extra_body = { reasoning_effort = "high" }` (or `"low"`) to the provider.
|
|
114
|
+
|
|
115
|
+
## CI and hooks
|
|
116
|
+
|
|
117
|
+
**GitHub Action**: commit a `.jitcatch.toml` and copy [`examples/github-workflow.yml`](https://github.com/khawjaahmad/jitcatch/blob/main/examples/github-workflow.yml). It needs
|
|
118
|
+
`fetch-depth: 0`, `permissions: pull-requests: write`, your project's normal setup steps, and the API keys
|
|
119
|
+
named in your config as secrets. It posts one sticky PR comment (updated on each push), writes a job
|
|
120
|
+
summary, and exposes `catches` / `weak-catches` outputs. jitcatch is installed in its own venv so it never
|
|
121
|
+
shadows your toolchain.
|
|
81
122
|
|
|
82
123
|
**Pre-push hook**
|
|
83
124
|
|
|
@@ -108,14 +149,16 @@ The combined score is a weighted mean of the available scores (`assessors.weight
|
|
|
108
149
|
|
|
109
150
|
## Output
|
|
110
151
|
|
|
111
|
-
`.jitcatch/runs/<timestamp>-<sha>/` contains `report.json`
|
|
112
|
-
its runs on parent / mutant / change, all assessor
|
|
113
|
-
|
|
152
|
+
The Markdown report is printed to stdout. `.jitcatch/runs/<timestamp>-<sha>/` contains `report.json`
|
|
153
|
+
(everything: intent, risks, mutants, every test with its runs on parent / mutant / change, all assessor
|
|
154
|
+
details), `report.md`, `weak_catches/`, and `hardening/` when there are any (tests that pass on both
|
|
155
|
+
revisions; candidates to add to your suite).
|
|
114
156
|
|
|
115
157
|
## Cost and safety notes
|
|
116
158
|
|
|
117
159
|
* LLM calls per changed file are about 1 intent call (shared) + 2 × risks (mutant + test) + dodgy tests +
|
|
118
|
-
repairs, plus judges for each weak catch that is not auto-discarded.
|
|
160
|
+
repairs, plus judges for each weak catch that is not auto-discarded. That is typically 10–20 calls, so
|
|
161
|
+
with thinking models a run takes several minutes. Bound it with `max_target_files`,
|
|
119
162
|
`max_risks_per_file`, `dodgy_tests_per_file`, `max_fix_attempts`, `min_risk_score`.
|
|
120
163
|
* **Generated tests are LLM-written code executed on the machine** (CI runner or, with the hook, your laptop).
|
|
121
164
|
Run it where that is acceptable.
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""JiTCatch: Just-in-Time catching test generation (after Becker et al., FSE '26)."""
|
|
2
|
-
__version__ = "0.2.
|
|
2
|
+
__version__ = "0.2.1"
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
CONFIG_TEMPLATE = '''\
|
|
2
|
-
# JiTCatch configuration
|
|
2
|
+
# JiTCatch repository configuration (optional locally; needed in CI). It overrides the user settings
|
|
3
|
+
# saved by `jitcatch setup`. API keys come from the environment variables named below, falling back to
|
|
4
|
+
# keys saved by `jitcatch setup`.
|
|
3
5
|
|
|
4
6
|
# ---------------------------------------------------------------- LLM providers
|
|
5
7
|
# type: "anthropic" | "openai_chat" | "openai_responses" (or use a preset)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: jitcatch
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: Just-in-Time catching test generation: LLM-generated tests that fail on risky code changes, with rule-based and LLM-judge false-positive filtering.
|
|
5
5
|
Author: Ahmad Waqar
|
|
6
6
|
License: MIT
|
|
@@ -56,8 +56,8 @@ before it merges. JiTCatch then filters out false positives and asks the author
|
|
|
56
56
|
|
|
57
57
|
| Language | Runner | Generated test location |
|
|
58
58
|
|---|---|---|
|
|
59
|
-
| Python | pytest | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
60
|
-
| TypeScript / JavaScript | Jest or Vitest
|
|
59
|
+
| Python | pytest, in the project's `.venv` (see below) | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
|
|
60
|
+
| TypeScript / JavaScript | Jest or Vitest, detected from `package.json`; files in packages using other runners (e.g. Playwright only) are skipped | `<name>.jitcatch-<id>.test.<ext>` |
|
|
61
61
|
| Java | JUnit 4/5 via Maven or Gradle (wrappers preferred) | `src/test/java/<pkg>/<Class>JitcatchTest<ID>.java` |
|
|
62
62
|
| Go | `go test` | `<name>_jitcatch_<id>_test.go`, functions `TestJitcatch<ID>…` |
|
|
63
63
|
|
|
@@ -80,10 +80,17 @@ Inside any git project, run:
|
|
|
80
80
|
jitcatch
|
|
81
81
|
```
|
|
82
82
|
|
|
83
|
-
The first time, it asks at most three questions
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
83
|
+
The first time, it asks at most three questions and saves the answers in `~/.config/jitcatch/` for every
|
|
84
|
+
project:
|
|
85
|
+
|
|
86
|
+
1. provider: Z.ai GLM Coding Plan, Anthropic, OpenAI, or Ollama (local, no key);
|
|
87
|
+
2. model: Enter accepts the suggested default (`glm-5.3-flash`, `claude-sonnet-5-5`, `gpt-5`, `qwen2.5-coder:32b`);
|
|
88
|
+
3. API key: input is hidden, and the key is stored in `~/.config/jitcatch/credentials.toml`, readable only by you.
|
|
89
|
+
|
|
90
|
+
Questions whose answer is already known are skipped. For example, the key isn't asked for if
|
|
91
|
+
`ZAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` is set, and nothing is asked once setup is complete.
|
|
92
|
+
Without a terminal (CI, git hooks) jitcatch never prompts; it says what is missing.
|
|
93
|
+
Run `jitcatch setup` to set up ahead of time, or `jitcatch setup --reset` to change provider or model.
|
|
87
94
|
|
|
88
95
|
What gets tested is picked automatically:
|
|
89
96
|
|
|
@@ -91,8 +98,12 @@ What gets tested is picked automatically:
|
|
|
91
98
|
2. otherwise, commits on your branch that aren't on the default branch yet;
|
|
92
99
|
3. otherwise, the last commit.
|
|
93
100
|
|
|
94
|
-
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index,
|
|
95
|
-
artifacts in `.jitcatch/` are hidden through
|
|
101
|
+
Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index, generated
|
|
102
|
+
tests run in temporary git worktrees, and run artifacts in `.jitcatch/` are hidden through
|
|
103
|
+
`.git/info/exclude`. Each step is logged as it happens (`risk 2/3: generating mutant…`,
|
|
104
|
+
`model replied in 41s`). Ctrl-C stops cleanly and removes the temporary worktrees.
|
|
105
|
+
|
|
106
|
+
To compare specific revisions:
|
|
96
107
|
|
|
97
108
|
```bash
|
|
98
109
|
jitcatch run --base origin/main --head HEAD
|
|
@@ -104,10 +115,40 @@ tests run in the project's `.venv` so its dependencies are importable; if that e
|
|
|
104
115
|
jitcatch installs a private copy into `~/.cache/jitcatch/` and leaves your project untouched.
|
|
105
116
|
`jitcatch doctor --ping` checks configuration, keys, toolchains and provider round-trips.
|
|
106
117
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
118
|
+
| Command | What it does |
|
|
119
|
+
|---|---|
|
|
120
|
+
| `jitcatch` | Same as `jitcatch run`: test the current change, running first-time setup if needed |
|
|
121
|
+
| `jitcatch run [--base REV] [--head REV]` | Test a change; `--json`, `--markdown`, `--fail-on-catch`, `--force` |
|
|
122
|
+
| `jitcatch setup [--reset]` | Choose provider, model and API key (once per machine) |
|
|
123
|
+
| `jitcatch doctor [--ping]` | Check config, keys and toolchains; `--ping` sends a tiny request to each provider |
|
|
124
|
+
| `jitcatch init` | Write a full `.jitcatch.toml` into the repository (see Configuration) |
|
|
125
|
+
| `jitcatch install-hook` / `uninstall-hook` | Manage the git pre-push hook |
|
|
126
|
+
| `jitcatch github` | Run inside GitHub Actions on a `pull_request` event (used by the action) |
|
|
127
|
+
|
|
128
|
+
## Configuration
|
|
129
|
+
|
|
130
|
+
Settings are layered: built-in defaults, then your user settings from `jitcatch setup`
|
|
131
|
+
(`~/.config/jitcatch/config.toml`), then a repository `.jitcatch.toml` if one exists. API keys come from
|
|
132
|
+
the environment variable named by the provider, falling back to the key saved by `jitcatch setup`.
|
|
133
|
+
|
|
134
|
+
A repository `.jitcatch.toml` is optional for local use. You need one for CI, where there are no user
|
|
135
|
+
settings, or to change the pipeline per project. `jitcatch init` writes a commented starter file. It
|
|
136
|
+
defines a Claude generator and a Claude + GLM judge ensemble, so edit `[providers]`/`[roles]` to match the
|
|
137
|
+
keys you have. Every option is documented in [`examples/jitcatch.toml`](https://github.com/khawjaahmad/jitcatch/blob/main/examples/jitcatch.toml), e.g.
|
|
138
|
+
`max_target_files`, `max_risks_per_file`, `report_threshold`, and `[adapters.python] python = ...` to force an
|
|
139
|
+
interpreter.
|
|
140
|
+
|
|
141
|
+
GLM-5.x models always think. The `zai-coding*` presets use Z.ai's documented defaults (`max_tokens = 65536`,
|
|
142
|
+
`temperature = 1.0`). To trade depth for speed and Coding Plan quota, add
|
|
143
|
+
`extra_body = { reasoning_effort = "high" }` (or `"low"`) to the provider.
|
|
144
|
+
|
|
145
|
+
## CI and hooks
|
|
146
|
+
|
|
147
|
+
**GitHub Action**: commit a `.jitcatch.toml` and copy [`examples/github-workflow.yml`](https://github.com/khawjaahmad/jitcatch/blob/main/examples/github-workflow.yml). It needs
|
|
148
|
+
`fetch-depth: 0`, `permissions: pull-requests: write`, your project's normal setup steps, and the API keys
|
|
149
|
+
named in your config as secrets. It posts one sticky PR comment (updated on each push), writes a job
|
|
150
|
+
summary, and exposes `catches` / `weak-catches` outputs. jitcatch is installed in its own venv so it never
|
|
151
|
+
shadows your toolchain.
|
|
111
152
|
|
|
112
153
|
**Pre-push hook**
|
|
113
154
|
|
|
@@ -138,14 +179,16 @@ The combined score is a weighted mean of the available scores (`assessors.weight
|
|
|
138
179
|
|
|
139
180
|
## Output
|
|
140
181
|
|
|
141
|
-
`.jitcatch/runs/<timestamp>-<sha>/` contains `report.json`
|
|
142
|
-
its runs on parent / mutant / change, all assessor
|
|
143
|
-
|
|
182
|
+
The Markdown report is printed to stdout. `.jitcatch/runs/<timestamp>-<sha>/` contains `report.json`
|
|
183
|
+
(everything: intent, risks, mutants, every test with its runs on parent / mutant / change, all assessor
|
|
184
|
+
details), `report.md`, `weak_catches/`, and `hardening/` when there are any (tests that pass on both
|
|
185
|
+
revisions; candidates to add to your suite).
|
|
144
186
|
|
|
145
187
|
## Cost and safety notes
|
|
146
188
|
|
|
147
189
|
* LLM calls per changed file are about 1 intent call (shared) + 2 × risks (mutant + test) + dodgy tests +
|
|
148
|
-
repairs, plus judges for each weak catch that is not auto-discarded.
|
|
190
|
+
repairs, plus judges for each weak catch that is not auto-discarded. That is typically 10–20 calls, so
|
|
191
|
+
with thinking models a run takes several minutes. Bound it with `max_target_files`,
|
|
149
192
|
`max_risks_per_file`, `dodgy_tests_per_file`, `max_fix_attempts`, `min_risk_score`.
|
|
150
193
|
* **Generated tests are LLM-written code executed on the machine** (CI runner or, with the hook, your laptop).
|
|
151
194
|
Run it where that is acceptable.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "jitcatch"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.1"
|
|
8
8
|
description = "Just-in-Time catching test generation: LLM-generated tests that fail on risky code changes, with rule-based and LLM-judge false-positive filtering."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|