jitcatch 0.2.0__py3-none-any.whl

jitcatch/userconfig.py

@@ -0,0 +1,188 @@
+"""Per-user settings: provider choice, stored API keys, and the interactive first-run setup.
+
+Lives in ~/.config/jitcatch/ (or $JITCATCH_CONFIG_HOME / $XDG_CONFIG_HOME/jitcatch):
+  config.toml       provider + roles, merged under any repo .jitcatch.toml
+  credentials.toml  API keys (mode 0600); environment variables always take precedence
+"""
+from __future__ import annotations
+
+import getpass
+import json
+import os
+import sys
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Callable
+
+from .config import ConfigError
+
+
+def config_home() -> Path:
+    if os.environ.get("JITCATCH_CONFIG_HOME"):
+        return Path(os.environ["JITCATCH_CONFIG_HOME"])
+    base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config")
+    return Path(base) / "jitcatch"
+
+
+def cache_home() -> Path:
+    if os.environ.get("JITCATCH_CACHE_DIR"):
+        return Path(os.environ["JITCATCH_CACHE_DIR"])
+    base = os.environ.get("XDG_CACHE_HOME") or str(Path.home() / ".cache")
+    return Path(base) / "jitcatch"
+
+
+def user_config_path() -> Path:
+    return config_home() / "config.toml"
+
+
+def credentials_path() -> Path:
+    return config_home() / "credentials.toml"
+
+
+def _read_toml(p: Path) -> dict[str, Any]:
+    if not p.is_file():
+        return {}
+    try:
+        with open(p, "rb") as f:
+            return tomllib.load(f)
+    except (OSError, tomllib.TOMLDecodeError):
+        return {}
+
+
+def load_user_config() -> dict[str, Any]:
+    return _read_toml(user_config_path())
+
+
+def stored_key(env_name: str) -> str:
+    v = _read_toml(credentials_path()).get(env_name, "")
+    return v if isinstance(v, str) else ""
+
+
+def _toml_str(s: str) -> str:
+    return json.dumps(s)  # a JSON string is a valid TOML basic string
+
+
+def save_key(env_name: str, value: str) -> Path:
+    p = credentials_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    data = {k: v for k, v in _read_toml(p).items() if isinstance(v, str)}
+    data[env_name] = value
+    body = "# jitcatch API keys (environment variables take precedence)\n" + "".join(
+        f"{k} = {_toml_str(v)}\n" for k, v in sorted(data.items()))
+    fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "w", encoding="utf-8") as f:
+        f.write(body)
+    os.chmod(p, 0o600)
+    return p
+
+
+def save_user_provider(name: str, preset: str, model: str) -> Path:
+    p = user_config_path()
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(
+        "# jitcatch user settings (written by `jitcatch setup`; a repo .jitcatch.toml overrides these)\n\n"
+        f"[providers.{name}]\npreset = {_toml_str(preset)}\nmodel = {_toml_str(model)}\n\n"
+        f"[roles]\ngenerator = {_toml_str(name)}\njudges = [{_toml_str(name)}]\n",
+        encoding="utf-8")
+    return p
+
+
+# ------------------------------------------------------------------------- interactive setup
+
+@dataclass(frozen=True)
+class Choice:
+    label: str
+    name: str      # provider name written to config
+    preset: str
+    model: str     # suggested default model
+    key_env: str   # "" = no key needed
+
+
+CHOICES = [
+    Choice("Z.ai GLM Coding Plan", "glm", "zai-coding", "glm-5.3-flash", "ZAI_API_KEY"),
+    Choice("Anthropic (Claude)", "claude", "anthropic", "claude-sonnet-5-5", "ANTHROPIC_API_KEY"),
+    Choice("OpenAI", "gpt", "openai", "gpt-5", "OPENAI_API_KEY"),
+    Choice("Ollama (local, no key)", "local", "ollama", "qwen2.5-coder:32b", ""),
+]
+
+
+class SetupNeeded(ConfigError):
+    """Setup is incomplete and we cannot ask (no terminal)."""
+
+
+def is_interactive() -> bool:
+    return sys.stdin.isatty() and sys.stdout.isatty()
+
+
+def missing_keys(cfg) -> list[tuple[str, str]]:
+    """(provider name, env var) for every provider used by the roles that has no API key."""
+    roles = cfg.roles
+    used = [roles.get("generator"), *(roles.get("judges") or []), roles.get("logprob_judge")]
+    if not roles.get("generator") and len(cfg.providers) == 1:
+        used.append(next(iter(cfg.providers)))
+    out = []
+    for n in dict.fromkeys(u for u in used if u):
+        p = cfg.providers.get(n)
+        if p is not None and p.api_key_env and not p.resolve_api_key():
+            out.append((n, p.api_key_env))
+    return out
+
+
+def ensure_setup(load: Callable[[], Any], ask: Callable[[str], str] = input,
+                 ask_secret: Callable[[str], str] = getpass.getpass,
+                 interactive: bool | None = None, out=None) -> Any:
+    """Return a ready config, asking at most 3 questions (provider, model, key) for whatever is missing.
+
+    Steps that are already satisfied are skipped; when nothing is missing nothing is asked."""
+    cfg = load()
+    need_provider = not cfg.providers
+    keys = [] if need_provider else missing_keys(cfg)
+    if not need_provider and not keys:
+        return cfg
+    out = out or sys.stderr
+    if interactive is None:
+        interactive = is_interactive()
+    if not interactive:
+        what = "no LLM provider configured" if need_provider else \
+            "missing API key " + ", ".join(env for _, env in keys)
+        raise SetupNeeded(f"{what}: run `jitcatch setup` in a terminal (or set the environment variable)")
+
+    if need_provider:
+        print("\njitcatch setup (asked once; saved for every project)", file=out)
+        print("\n1. Which LLM provider?", file=out)
+        for i, c in enumerate(CHOICES, 1):
+            print(f"   {i}) {c.label}", file=out)
+        choice = CHOICES[0]
+        while True:
+            ans = ask("   Choose 1-4 [1]: ").strip()
+            if not ans:
+                break
+            if ans.isdigit() and 1 <= int(ans) <= len(CHOICES):
+                choice = CHOICES[int(ans) - 1]
+                break
+            print("   please enter a number from the list", file=out)
+
+        print("\n2. Which model?", file=out)
+        model = ask(f"   Model [{choice.model}]: ").strip() or choice.model
+        path = save_user_provider(choice.name, choice.preset, model)
+
+        if choice.key_env and not (os.environ.get(choice.key_env) or stored_key(choice.key_env)):
+            print(f"\n3. API key ({choice.key_env})", file=out)
+            _ask_key(choice.key_env, ask_secret, out)
+        print(f"\nSaved to {path}. You won't be asked again.\n", file=out)
+    else:
+        for _, env in keys:
+            print(f"\njitcatch needs an API key ({env}); asked once.", file=out)
+            _ask_key(env, ask_secret, out)
+    return load()
+
+
+def _ask_key(env: str, ask_secret: Callable[[str], str], out) -> None:
+    while True:
+        key = ask_secret("   Paste key (input hidden): ").strip()
+        if key:
+            p = save_key(env, key)
+            print(f"  saved to {p} (only you can read it)", file=out)
+            return
+        print("  the key cannot be empty", file=out)

jitcatch-0.2.0.dist-info/METADATA

@@ -0,0 +1,164 @@
+Metadata-Version: 2.4
+Name: jitcatch
+Version: 0.2.0
+Summary: Just-in-Time catching test generation: LLM-generated tests that fail on risky code changes, with rule-based and LLM-judge false-positive filtering.
+Author: Ahmad Waqar
+License: MIT
+Project-URL: Homepage, https://github.com/khawjaahmad/jitcatch
+Project-URL: Repository, https://github.com/khawjaahmad/jitcatch
+Project-URL: Issues, https://github.com/khawjaahmad/jitcatch/issues
+Project-URL: Paper, https://arxiv.org/abs/2601.22832
+Keywords: testing,regression,llm,mutation-testing,test-generation,code-review
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Dynamic: license-file
+
+# JiTCatch
+
+Just-in-Time **catching** test generation for any Git repository, modelled on
+Becker et al., *"Just-in-Time Catching Test Generation at Meta"* (FSE Companion '26, arXiv:2601.22832).
+
+Hardening tests pass today and guard the future. **Catching tests are generated for one specific change and
+are meant to fail on it**: they pass on the parent revision and fail on the change, exposing a regression
+before it merges. JiTCatch then filters out false positives and asks the author a plain question:
+*"`can_delete()` now returns False for a resource's owner. Is this expected?"*
+
+| Paper concept | Where it lives |
+|---|---|
+| Diff risk targeting (DRS) | `targeting.py` (transparent heuristic; `run.min_risk_score`) |
+| Intent-aware workflow: intent → risks → mutants of parent → tests that kill them | `engine.py` `_infer_intent`, `_make_mutant`, `_generate_test` |
+| Mutant must build and pass existing tests | `run.check_mutants_with_existing_tests` |
+| "Dodgy diff" workflow: treat the change itself as a mutant | `engine.py` `_dodgy_diff` |
+| Weak catch = passes on parent, fails on change | `models.GeneratedTest.is_weak_catch` |
+| RubFake rule-based assessor (Tables 2 & 3) | `assessors/rules.py` |
+| LLM-as-judge ensemble, High/Medium/Low → 1/0/−1, **median** | `assessors/llm_judge.py` |
+| Normalised token-probability Yes/No score | `assessors/llm_judge.py` `logprob_score` |
+| Score in [−1, 1], rank, threshold | `assessors/__init__.py`, `run.report_threshold` |
+| Low-friction "is this expected?" report | `report.py` (Markdown, JSON, PR comment) |
+| Coincidental hardening tests as a by-product | `.jitcatch/runs/<id>/hardening/` |
+
+## Supported stacks
+
+| Language | Runner | Generated test location |
+|---|---|---|
+| Python | pytest | next to the related existing test (else next to the source): `test_jitcatch_<id>.py` |
+| TypeScript / JavaScript | Jest or Vitest (auto-detected per `package.json`) | `<name>.jitcatch-<id>.test.<ext>` |
+| Java | JUnit 4/5 via Maven or Gradle (wrappers preferred) | `src/test/java/<pkg>/<Class>JitcatchTest<ID>.java` |
+| Go | `go test` | `<name>_jitcatch_<id>_test.go`, functions `TestJitcatch<ID>…` |
+
+LLM providers: **Anthropic Messages**, **OpenAI Chat Completions**, **OpenAI Responses**, and anything compatible
+with them, with presets for Z.ai GLM Coding Plan (`zai-coding` → `https://api.z.ai/api/coding/paas/v4`,
+`zai-coding-anthropic`, `zai-coding-responses`) and local servers (`ollama`, `lmstudio`, `vllm`).
+No third-party Python dependencies; Python ≥ 3.11.
+
+## Install
+
+```bash
+pipx install jitcatch        # or: uv tool install jitcatch   /   pip install jitcatch
+```
+
+## Use
+
+Inside any git project, run:
+
+```bash
+jitcatch
+```
+
+The first time, it asks at most three questions (provider, model, API key) and saves the answers in
+`~/.config/jitcatch/` for every project. Questions whose answer is already known are skipped; for example,
+the key isn't asked for if `ZAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` is already set.
+Run `jitcatch setup --reset` to change provider or model.
+
+What gets tested is picked automatically:
+
+1. uncommitted changes (including new files), compared with the last commit;
+2. otherwise, commits on your branch that aren't on the default branch yet;
+3. otherwise, the last commit.
+
+Nothing in your checkout is modified. Uncommitted work is snapshotted with a throwaway index, and run
+artifacts in `.jitcatch/` are hidden through `.git/info/exclude`. To compare specific revisions:
+
+```bash
+jitcatch run --base origin/main --head HEAD
+jitcatch run --base main --head feature --json report.json --markdown report.md --fail-on-catch
+```
+
+The project's own toolchain must work (`node_modules`, Maven/Gradle dependencies, Go modules). For Python,
+tests run in the project's `.venv` so its dependencies are importable; if that environment has no pytest,
+jitcatch installs a private copy into `~/.cache/jitcatch/` and leaves your project untouched.
+`jitcatch doctor --ping` checks configuration, keys, toolchains and provider round-trips.
+
+**GitHub Action** – copy `examples/github-workflow.yml`. It needs `fetch-depth: 0`,
+`permissions: pull-requests: write`, your project's normal setup steps, and API keys as secrets.
+It posts one sticky PR comment (updated on each push), writes a job summary, and exposes
+`catches` / `weak-catches` outputs. jitcatch is installed in its own venv so it never shadows your toolchain.
+
+**Pre-push hook**
+
+```bash
+jitcatch install-hook              # reports only
+jitcatch install-hook --blocking   # a reported catch blocks the push
+JITCATCH_SKIP=1 git push           # bypass once
+```
+
+Tool or provider failures never block a push.
+
+## How scoring works
+
+Every weak catch is scored by up to three assessors, each in [−1 (false positive), 1 (true positive)]:
+
+1. **rules**: deterministic false-positive patterns from the paper (broken runner, reflection, type
+   mismatch, mocks, private/protected members, NotImplemented, key ordering, undefined names, call-count
+   expectations, network down, flakiness) and true-positive patterns (unexpected key/index error, emptied
+   container, construction failure, flipped boolean, behaviour change in a refactor, dead-code removal or
+   additive change, unexpected null, RBAC). A **high** false-positive pattern discards the catch before any
+   judge is called (`assessors.discard_on_high_fp_rule`).
+2. **llm_bucket**: median of the judges' High/Medium/Low.
+3. **llm_logprob** (optional): P(Yes) vs P(No) for the first answer token from an `openai_chat` provider
+   with `logprobs = true`.
+
+The combined score is a weighted mean of the available scores (`assessors.weights`); catches at or above
+`run.report_threshold` are reported, best first. All judge rationales are kept individually.
+
+## Output
+
+`.jitcatch/runs/<timestamp>-<sha>/` contains `report.json` (everything: intent, risks, mutants, every test with
+its runs on parent / mutant / change, all assessor details), `report.md`, `weak_catches/` and `hardening/`
+(tests that pass on both revisions; candidates to add to your suite).
+
+## Cost and safety notes
+
+* LLM calls per changed file are about 1 intent call (shared) + 2 × risks (mutant + test) + dodgy tests +
+  repairs, plus judges for each weak catch that is not auto-discarded. Bound it with `max_target_files`,
+  `max_risks_per_file`, `dodgy_tests_per_file`, `max_fix_attempts`, `min_risk_score`.
+* **Generated tests are LLM-written code executed on the machine** (CI runner or, with the hook, your laptop).
+  Run it where that is acceptable.
+* Targeting is a heuristic stand-in for Meta's learned risk model, which needs historical data.
+* Logprob judging only reads Chat Completions logprobs; whether a given server returns them varies
+  (`jitcatch doctor --ping` reports it).
+
+## Development
+
+```bash
+pip install -e '.[dev]' && pytest
+```
+
+The suite runs the full pipeline end-to-end with a scripted LLM against real pytest, Go, Vitest (TypeScript)
+and Jest (JavaScript) projects, and against HTTP fake servers for the provider and GitHub wire formats.
+Java/JUnit execution is covered by placement/classification unit tests only.

jitcatch-0.2.0.dist-info/RECORD

@@ -0,0 +1,34 @@
+jitcatch/__init__.py,sha256=V7QhGRoQxOSVqzU4u_94DRbqQl804T-ht0Mjcx4Ttbg,108
+jitcatch/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
+jitcatch/cli.py,sha256=bUoKYZYsSF_AKpvYOgGyxr-H07KJo5n_AdlKwrzzxmg,14485
+jitcatch/config.py,sha256=W0MQPHUOgzwKyb8xJeXdPPMkTRJMEbT3FTdXi4LdXcM,8667
+jitcatch/engine.py,sha256=pQM-RaVPO0Rr_Xt1McHkzGcxvmnOJQImxS54aSrzXvo,25347
+jitcatch/github.py,sha256=hGs24ZJiyvYL0h2yAK6M85hbt9mp2aGr1iU-f5aBKsc,3247
+jitcatch/gitutil.py,sha256=B5x3lwUnXVLT5184rMsoIkv6jFoT77p-J9OOx0wlEAU,10136
+jitcatch/hook.py,sha256=1SaTccKMWXNIVYjWxjf_l4RmMtQCDtJKY6xe3WWM1gA,3410
+jitcatch/jsonutil.py,sha256=qny5utgwztYArpnZaAWHN33JVQfJ7ICT4LXXjO1Y8Ss,2620
+jitcatch/models.py,sha256=j9ue5Z5tiM2361b_dLNMfBw9ERxHQ5BS6q-3A_Gz1ws,4342
+jitcatch/prompts.py,sha256=_BryDEQh1HqOE01lMuQrCac_UjFG7drds_tIk5zB60M,6175
+jitcatch/report.py,sha256=E9PSmj0ZbvEi20pBJ1kym80Sj4uxfE59tYYzAWkbAk8,7365
+jitcatch/targeting.py,sha256=4INqRzqe7J85AUXrI3h-jq8Bz-Kf1Rra6mA7EvGRiKo,1812
+jitcatch/templates.py,sha256=JKwtJZ7eTWUAxRNAXrF7dFUFMs03bOF4zffmeZMODWw,2993
+jitcatch/userconfig.py,sha256=exZxyW3hsNw5a_BaPXsxfmKoJ5cvG4TIPfOQPs9LtVQ,6795
+jitcatch/adapters/__init__.py,sha256=OI7WyKVxfgmiaeBa3jTBctnTz0k0dF0AB73Po7krS7M,943
+jitcatch/adapters/base.py,sha256=XcMDys_k4C9orUPVElGhV_bw4CCKrBdy6VZfj_KA2Ig,5767
+jitcatch/adapters/golang.py,sha256=KoYrPMIQDf6iXg3iRMFadkHbRLZQDFSX9k7YW5hXvOM,4157
+jitcatch/adapters/java.py,sha256=m1eIlhv9rgsQzfhRzxFzLix_i70WnSto2yhLJsUt3RE,6562
+jitcatch/adapters/javascript.py,sha256=4w_DaB2rOL3uZnBUsuvfyTGZpN-L6fbcyvsFA1kxNmk,8107
+jitcatch/adapters/python.py,sha256=kRvUA3DPZXAG9G1NAr4pXMtoV9DZ_SItRV_sbNmclYU,8548
+jitcatch/assessors/__init__.py,sha256=1-Y3mQya3dRQFZLjxVSlZz9ozqWe-JL7SERnGAkudlg,4158
+jitcatch/assessors/failure_parse.py,sha256=TSENddQeTdSg29243AzPf_OU6QparRWhX1NfeZGMdtQ,4976
+jitcatch/assessors/llm_judge.py,sha256=ZlUG6gviLqhszjR-02d53gS2kPC6Gm91g44jwKQevi0,3664
+jitcatch/assessors/rules.py,sha256=XlgVCZkaeAprt-rpHYf0FXjyjPDdV2JPnn6lTBV6d2k,16133
+jitcatch/llm/__init__.py,sha256=SCUlx_RRBc0Isf2ZKX-Wd0PGx3PMJ69glfJ8HotUGAI,627
+jitcatch/llm/base.py,sha256=4IZDEmkyQAX3n6fio1of10W42cXSRy0bprXn3ZezIEo,3984
+jitcatch/llm/providers.py,sha256=HX5Do6zccOx4Kp9pyn5wnooXzQFOZ2Rx_VjcXQ7e98A,5533
+jitcatch-0.2.0.dist-info/licenses/LICENSE,sha256=xLd32VBsKO-bgUFqBy2I8vphK6fwHdx3bQPT00XtoWk,1068
+jitcatch-0.2.0.dist-info/METADATA,sha256=hcUkQyu81Eurq8t8dgimW5gKJt-3wym_TiqYXF6KTKA,8466
+jitcatch-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
+jitcatch-0.2.0.dist-info/entry_points.txt,sha256=JfFzcIGORqr97JwBPSw9FGe3ezdmavAnr0WNBR4Epks,47
+jitcatch-0.2.0.dist-info/top_level.txt,sha256=SKJKVPRcvS1U_PkrlD1eAcYM96O-q0oyIQQNtyJRB1c,9
+jitcatch-0.2.0.dist-info/RECORD,,

jitcatch-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.2.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

jitcatch-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+jitcatch = jitcatch.cli:main

jitcatch-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ahmad Waqar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

jitcatch-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+jitcatch