jitcatch 0.2.0__py3-none-any.whl

jitcatch/models.py

@@ -0,0 +1,150 @@
+"""Core data structures shared across the pipeline.
+
+Terminology follows Becker et al., "Just-in-Time Catching Test Generation at Meta" (FSE '26):
+  * weak catch   -- a generated test that passes on the parent revision and fails on the change
+  * strong catch -- a weak catch whose failure reveals a real bug (true positive)
+  * strictly weak catch -- a weak catch whose failure is a false positive
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field, asdict
+from enum import Enum
+from typing import Any
+
+
+class Outcome(str, Enum):
+    PASS = "pass"
+    FAIL = "fail"            # test ran, at least one assertion/runtime failure
+    ERROR = "error"          # test could not build / collect / compile
+    INFRA = "infra"          # runner itself broken, timeout, missing tool
+    NO_TESTS = "no_tests"    # runner found no tests in the generated file
+
+
+@dataclass
+class TestRun:
+    __test__ = False  # not a pytest test class
+    outcome: Outcome
+    exit_code: int
+    output: str
+    duration_s: float = 0.0
+    command: list[str] = field(default_factory=list)
+
+    @property
+    def passed(self) -> bool:
+        return self.outcome == Outcome.PASS
+
+
+@dataclass
+class ChangedFile:
+    path: str                 # repo-relative path at head
+    old_path: str             # repo-relative path at base (same as path unless renamed)
+    status: str               # M, A, D, R
+    patch: str                # unified diff for this file
+    added_lines: int = 0
+    removed_lines: int = 0
+
+
+@dataclass
+class ChangeContext:
+    base_sha: str
+    head_sha: str
+    title: str
+    description: str
+    files: list[ChangedFile]
+    full_patch: str
+
+
+@dataclass
+class Risk:
+    id: str
+    description: str
+    file: str
+
+
+@dataclass
+class Intent:
+    summary: str
+    change_kind: str          # refactor | dead_code_removal | additive | bugfix | feature | other
+    risks: list[Risk]
+
+
+@dataclass
+class Mutant:
+    id: str
+    risk: Risk
+    file: str
+    original: str
+    mutated: str
+    survived_existing_tests: bool | None = None
+    note: str = ""
+
+
+@dataclass
+class GeneratedTest:
+    id: str
+    workflow: str             # intent_aware | dodgy_diff
+    target_file: str          # source file under test (repo-relative)
+    test_path: str            # where the test is written (repo-relative)
+    code: str
+    risk: Risk | None = None
+    mutant_id: str | None = None
+    base_run: TestRun | None = None
+    base_rerun: TestRun | None = None
+    mutant_run: TestRun | None = None
+    head_run: TestRun | None = None
+    head_rerun: TestRun | None = None
+    attempts: int = 0
+    discarded_reason: str = ""
+
+    @property
+    def is_weak_catch(self) -> bool:
+        return (
+            self.base_run is not None and self.base_run.passed
+            and self.head_run is not None and not self.head_run.passed
+            and self.head_run.outcome != Outcome.INFRA
+        )
+
+    @property
+    def is_hardening(self) -> bool:
+        return (
+            self.base_run is not None and self.base_run.passed
+            and self.head_run is not None and self.head_run.passed
+        )
+
+    @property
+    def is_flaky(self) -> bool:
+        for a, b in ((self.base_run, self.base_rerun), (self.head_run, self.head_rerun)):
+            if a is not None and b is not None and a.outcome != b.outcome:
+                return True
+        return False
+
+
+@dataclass
+class AssessorScore:
+    name: str
+    score: float | None       # -1.0 (false positive) .. 1.0 (true positive); None = no information
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Assessment:
+    test_id: str
+    scores: list[AssessorScore]
+    combined: float
+    discarded: bool
+    discard_reason: str = ""
+    behavior_change: str = ""     # plain-language "used to X, now Y"
+    question: str = ""            # "is this expected?" message for the author
+    rationales: list[dict[str, str]] = field(default_factory=list)
+
+
+def to_jsonable(obj: Any) -> Any:
+    if isinstance(obj, Enum):
+        return obj.value
+    if hasattr(obj, "__dataclass_fields__"):
+        return {k: to_jsonable(v) for k, v in asdict(obj).items()}
+    if isinstance(obj, dict):
+        return {k: to_jsonable(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return [to_jsonable(v) for v in obj]
+    return obj

jitcatch/prompts.py

@@ -0,0 +1,191 @@
+"""Prompt templates for every LLM step in the pipeline."""
+from __future__ import annotations
+
+GEN_SYSTEM = (
+    "You are a meticulous senior software engineer specialising in finding regressions in code changes "
+    "before they are merged. You follow output-format instructions exactly."
+)
+
+INTENT_USER = """\
+A developer submitted the code change below. Infer what the change is trying to do (its intent), classify it,
+and list the concrete *risks*: specific ways an attempt to implement this intent could introduce a bug that
+changes observable behaviour of the code that existed BEFORE the change and that a unit test could detect.
+Prioritise risks with severe consequences (data loss, wrong results, security/permission mistakes, crashes).
+Do not list style issues or risks that only concern brand-new code paths.
+
+## Title
+{title}
+
+## Description
+{description}
+
+## Diff (parent -> change)
+```diff
+{patch}
+```
+
+## Changed files after the change
+{files}
+
+Respond with ONLY a JSON object:
+{{
+  "intent": "<one or two sentences>",
+  "change_kind": "refactor" | "dead_code_removal" | "additive" | "bugfix" | "feature" | "other",
+  "risks": [{{"file": "<repo-relative path of a changed file listed above>", "description": "<specific risk>"}}]
+}}
+List at most {max_risks} risks per file, most severe first.
+"""
+
+MUTANT_USER = """\
+We simulate a bug that could be introduced while implementing a change. Below is the PARENT version (before the
+change) of `{path}`. Create a *mutant*: a minimal, realistic modification of this parent file that exhibits the
+risk described. The mutant must still compile/parse, must change observable behaviour, and must look like a
+plausible developer mistake (not a deliberate sabotage like `return None` everywhere).
+
+## Intended change
+{intent}
+
+## Risk to simulate
+{risk}
+
+## Parent version of `{path}`
+```{lang}
+{source}
+```
+
+Respond with ONLY a JSON object:
+{{
+  "explanation": "<what the mutant changes and why it reflects the risk>",
+  "edits": [{{"search": "<exact, unique snippet copied verbatim from the parent file>", "replace": "<new snippet>"}}]
+}}
+Each "search" string must appear exactly once in the parent file. Keep edits small.
+"""
+
+TEST_FOR_MUTANT_USER = """\
+Write a unit test file for `{path}` that PASSES on the parent version shown below and FAILS on the mutant
+(the mutant diff is shown). The test must assert the parent's correct behaviour on the code path the mutant
+affects, so that the risk "{risk}" would be caught. Prefer simple, deterministic inputs and precise assertions
+on returned values or observable state. Test only the public behaviour.
+
+## Test-file constraints
+{instructions}
+The file will be written to `{test_path}`.
+
+## Parent version of `{path}`
+```{lang}
+{source}
+```
+
+## Mutant diff (parent -> mutant)
+```diff
+{mutant_diff}
+```
+{example}
+Respond with ONLY the complete test file in a single ```{lang} code block.
+"""
+
+TEST_DODGY_USER = """\
+Treat the change below as if it were a mutant of its parent. Write a unit test file for `{path}` that PASSES on
+the parent version (shown in full) and would FAIL if the code behaved the way the changed version behaves.
+Focus on the behaviour of the functions/branches touched by the diff. Assert precise parent-version outputs for
+concrete inputs. Test only the public behaviour; keep it deterministic.
+
+## Test-file constraints
+{instructions}
+The file will be written to `{test_path}`.
+
+## Parent version of `{path}`
+```{lang}
+{source}
+```
+
+## Diff (parent -> change)
+```diff
+{patch}
+```
+{example}{variation}
+Respond with ONLY the complete test file in a single ```{lang} code block.
+"""
+
+TEST_FIX_USER = """\
+The test file below was supposed to PASS on the parent version of `{path}` but it did not.
+Fix the test so that it passes on the parent version while still asserting meaningful, precise behaviour on the
+same code path (do not delete the assertions that matter or make the test trivial). If an assertion encodes a
+wrong expectation, correct the expected value to the parent's real behaviour.
+
+## Test-file constraints
+{instructions}
+
+## Parent version of `{path}`
+```{lang}
+{source}
+```
+
+## Current test file (`{test_path}`)
+```{lang}
+{test_code}
+```
+
+## Output when run on the parent version (outcome: {outcome})
+```
+{output}
+```
+
+Respond with ONLY the complete corrected test file in a single ```{lang} code block.
+"""
+
+EXAMPLE_BLOCK = """
+## Existing test in this repository (follow its imports, setup and style)
+`{path}`:
+```{lang}
+{code}
+```
+"""
+
+JUDGE_SYSTEM = (
+    "You are an experienced code reviewer judging whether an automatically generated test failure reveals a "
+    "real bug introduced by a code change, or whether the behaviour change is intended / the test is wrong. "
+    "You are careful, concrete and sceptical of brittle tests."
+)
+
+JUDGE_USER = """\
+A generated test PASSES on the parent revision and FAILS on the changed revision.
+Decide how likely it is that this failure reveals an UNEXPECTED behaviour change (a bug) introduced by the change,
+as opposed to an intended behaviour change or a problem with the test itself (brittle assertions on
+implementation details, wrong mocks, API/signature changes, private members, ordering of unordered data, etc.).
+
+## Inferred intent of the change
+{intent}
+
+## Title / description supplied by the author
+{title}
+{description}
+
+## Diff (parent -> change)
+```diff
+{patch}
+```
+
+## Generated test (`{test_path}`)
+```{lang}
+{test_code}
+```
+
+## Test execution on the changed revision (outcome: {outcome})
+```
+{output}
+```
+
+Respond with ONLY a JSON object:
+{{
+  "likelihood": "High" | "Medium" | "Low",
+  "behavior_change": "<one sentence: what the code did before the change vs what it does now, in concrete terms>",
+  "question": "<a short, friendly message to the change author describing the observed behaviour change and ending with 'Is this expected?'>",
+  "rationale": "<2-4 sentences: why this is or is not likely an unintended bug, given the diff and intent>"
+}}
+"""
+
+LOGPROB_USER = JUDGE_USER.split("Respond with ONLY")[0] + (
+    "Question: Is this test failure most likely an UNEXPECTED bug introduced by the change?\n"
+    "Answer with exactly one word: Yes or No."
+)

jitcatch/report.py

@@ -0,0 +1,163 @@
+"""JSON + Markdown reports and on-disk artifacts."""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path, PurePosixPath
+
+from .engine import RunResult
+from .models import Assessment, GeneratedTest, to_jsonable
+
+COMMENT_MARKER = "<!-- jitcatch-report -->"
+
+
+def to_dict(r: RunResult) -> dict:
+    return {
+        "base_sha": r.ctx.base_sha,
+        "head_sha": r.ctx.head_sha,
+        "title": r.ctx.title,
+        "risk": to_jsonable(r.risk),
+        "intent": to_jsonable(r.intent),
+        "targets": r.targets,
+        "skipped": r.skipped,
+        "threshold": r.threshold,
+        "summary": summary_counts(r),
+        "candidates": [
+            {"test_id": t.id, "target_file": t.target_file, "score": a.combined, "question": a.question,
+             "behavior_change": a.behavior_change}
+            for t, a in r.candidates()
+        ],
+        "mutants": [to_jsonable(_mutant_summary(m)) for m in r.mutants],
+        "tests": [to_jsonable(t) for t in r.tests],
+        "assessments": {k: to_jsonable(v) for k, v in r.assessments.items()},
+        "errors": r.errors,
+        "llm_calls": r.llm_calls,
+        "duration_s": round((r.finished_at or time.time()) - r.started_at, 1),
+    }
+
+
+def _mutant_summary(m):
+    return {"id": m.id, "risk": m.risk, "file": m.file, "survived_existing_tests": m.survived_existing_tests,
+            "note": m.note, "changed": m.original != m.mutated}
+
+
+def summary_counts(r: RunResult) -> dict:
+    tests = r.tests
+    return {
+        "generated_tests": len(tests),
+        "passed_on_parent": sum(1 for t in tests if t.base_run is not None and t.base_run.passed and not t.discarded_reason),
+        "weak_catches": len(r.weak_catches),
+        "discarded_by_rules": sum(1 for a in r.assessments.values() if a.discarded),
+        "reported": len(r.candidates()),
+        "hardening_tests": len(r.hardening),
+        "mutants_used": sum(1 for m in r.mutants if not m.note.startswith("invalid") and m.survived_existing_tests is not False),
+    }
+
+
+def write_artifacts(r: RunResult, out_root: Path) -> Path:
+    run_dir = out_root / "runs" / f"{time.strftime('%Y%m%d-%H%M%S')}-{r.ctx.head_sha[:8]}"
+    run_dir.mkdir(parents=True, exist_ok=True)
+    (run_dir / "report.json").write_text(json.dumps(to_dict(r), indent=2), encoding="utf-8")
+    (run_dir / "report.md").write_text(markdown(r), encoding="utf-8")
+    for t in r.weak_catches:
+        _save_test(run_dir / "weak_catches", t)
+    for t in r.hardening:
+        _save_test(run_dir / "hardening", t)
+    return run_dir
+
+
+def _save_test(d: Path, t: GeneratedTest) -> None:
+    dest = d / t.test_path
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    dest.write_text(t.code, encoding="utf-8")
+
+
+def _fmt(score) -> str:
+    return "n/a" if score is None else f"{score:+.2f}"
+
+
+def _lang_for(path: str) -> str:
+    ext = PurePosixPath(path).suffix
+    return {".py": "python", ".ts": "ts", ".tsx": "tsx", ".js": "js", ".jsx": "jsx", ".mjs": "js",
+            ".java": "java", ".go": "go"}.get(ext, "")
+
+
+def _candidate_md(i: int, t: GeneratedTest, a: Assessment) -> str:
+    lines = [f"#### {i}. `{t.target_file}` — confidence {a.combined:+.2f}"]
+    if a.question:
+        lines.append(f"> {a.question}")
+    elif a.behavior_change:
+        lines.append(f"> {a.behavior_change} Is this expected?")
+    if t.risk:
+        lines.append(f"\n_Risk probed:_ {t.risk.description}")
+    lines.append("\n<details><summary>Evidence</summary>\n")
+    for s in a.scores:
+        extra = ""
+        if s.name == "rules":
+            fps = [h["rule"] for h in s.details.get("false_positive_patterns", [])]
+            tps = [h["rule"] for h in s.details.get("true_positive_patterns", [])]
+            extra = f" — TP patterns: {', '.join(tps) or 'none'}; FP patterns: {', '.join(fps) or 'none'}"
+        elif s.name == "llm_logprob" and s.score is not None:
+            extra = f" — P(yes)={s.details.get('p_yes')}, P(no)={s.details.get('p_no')}"
+        lines.append(f"- **{s.name}**: {_fmt(s.score)}{extra}")
+    for rat in a.rationales:
+        if rat.get("error"):
+            lines.append(f"- _{rat.get('judge')}_: error — {rat['error'][:200]}")
+        else:
+            lines.append(f"- _{rat.get('judge')}_ ({rat.get('likelihood')}): {rat.get('rationale', '')}")
+    lines.append(f"\nWorkflow: `{t.workflow}` · test id `{t.id}` · passes on parent, "
+                 f"`{t.head_run.outcome.value if t.head_run else '?'}` on this change"
+                 + (f" · {'kills' if t.mutant_run and not t.mutant_run.passed else 'does not kill'} its mutant"
+                    if t.mutant_id else ""))
+    out = (t.head_run.output if t.head_run else "").strip()
+    if len(out) > 3000:
+        out = "…\n" + out[-3000:]
+    lines.append(f"\n**Failure on this change**\n```\n{out}\n```")
+    lines.append(f"\n**Generated test** (`{t.test_path}`)\n```{_lang_for(t.test_path)}\n{t.code.rstrip()}\n```")
+    lines.append("\n</details>\n")
+    return "\n".join(lines)
+
+
+def markdown(r: RunResult, for_comment: bool = False) -> str:
+    c = summary_counts(r)
+    cands = r.candidates()
+    md = [COMMENT_MARKER if for_comment else "", "## JiTCatch: just-in-time catching tests", ""]
+    md.append(f"Change `{r.ctx.base_sha[:8]}..{r.ctx.head_sha[:8]}`"
+              + (f" · risk score {r.risk.score:.2f}" if r.risk else ""))
+    if r.intent:
+        md.append(f"\n**Inferred intent** ({r.intent.change_kind}): {r.intent.summary}")
+    md.append("")
+    if cands:
+        md.append(f"### {len(cands)} behaviour change(s) worth a quick check")
+        md.append("Each item is a generated test that passes on the parent revision but fails with this change. "
+                  "If the change is intended, just dismiss it.\n")
+        for i, (t, a) in enumerate(cands, 1):
+            md.append(_candidate_md(i, t, a))
+    elif r.tests:
+        md.append("No likely regressions found by the generated tests.")
+    else:
+        md.append("No tests were generated for this change.")
+    md.append("\n| generated | pass on parent | weak catches | auto-discarded (rules) | reported | hardening |")
+    md.append("|---:|---:|---:|---:|---:|---:|")
+    md.append(f"| {c['generated_tests']} | {c['passed_on_parent']} | {c['weak_catches']} | "
+              f"{c['discarded_by_rules']} | {c['reported']} | {c['hardening_tests']} |")
+    low = [(a, next(t for t in r.tests if t.id == a.test_id)) for a in r.assessments.values()
+           if a.discarded or a.combined < r.threshold]
+    if low:
+        md.append("\n<details><summary>Weak catches judged unlikely to be bugs "
+                  f"({len(low)})</summary>\n")
+        for a, t in sorted(low, key=lambda x: -x[0].combined):
+            why = a.discard_reason or f"score {a.combined:+.2f} < threshold {r.threshold:+.2f}"
+            md.append(f"- `{t.target_file}` test `{t.id}` ({t.workflow}): {why}")
+        md.append("\n</details>")
+    if r.skipped:
+        md.append("\n<details><summary>Skipped files</summary>\n")
+        for s in r.skipped:
+            md.append(f"- `{s['path']}`: {s['reason']}")
+        md.append("\n</details>")
+    if r.errors:
+        md.append("\n<details><summary>Errors</summary>\n")
+        for e in r.errors:
+            md.append(f"- {e[:300]}")
+        md.append("\n</details>")
+    return "\n".join(x for x in md if x is not None).strip() + "\n"

jitcatch/targeting.py

@@ -0,0 +1,44 @@
+"""Change-risk targeting.
+
+The paper targets diffs with a learned Diff Risk Score model trained on historical changes. Without
+that history, this module uses a transparent heuristic (size, sensitive paths, untested change,
+spread, deletion ratio). Set `run.min_risk_score` to skip low-risk changes.
+"""
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass, field
+
+from .models import ChangeContext
+
+
+@dataclass
+class RiskScore:
+    score: float
+    reasons: list[str] = field(default_factory=list)
+
+
+def score_change(ctx: ChangeContext, sensitive_paths: list[str], is_test_file) -> RiskScore:
+    reasons: list[str] = []
+    files = [f for f in ctx.files if f.status != "D"]
+    src = [f for f in files if not is_test_file(f.path)]
+    tests_touched = any(is_test_file(f.path) for f in files)
+    changed = sum(f.added_lines + f.removed_lines for f in src)
+    removed = sum(f.removed_lines for f in src)
+
+    size = min(1.0, math.log10(1 + changed) / 3)          # ~1000 changed lines => 1.0
+    reasons.append(f"{changed} changed source lines")
+    sens_hits = sorted({p for f in src for p in sensitive_paths if re.search(rf"(^|[/_.\-]){re.escape(p)}", f.path, re.I)})
+    sensitive = 1.0 if sens_hits else 0.0
+    if sens_hits:
+        reasons.append("touches sensitive areas: " + ", ".join(sens_hits))
+    untested = 1.0 if src and not tests_touched else 0.0
+    if untested:
+        reasons.append("no tests changed alongside source changes")
+    spread = min(1.0, len(src) / 10)
+    deletion = removed / changed if changed else 0.0
+    if deletion > 0.5:
+        reasons.append(f"{deletion:.0%} of changed lines are deletions")
+    score = 0.35 * size + 0.25 * sensitive + 0.2 * untested + 0.1 * spread + 0.1 * deletion
+    return RiskScore(round(score, 3), reasons)

jitcatch/templates.py

@@ -0,0 +1,76 @@
+CONFIG_TEMPLATE = '''\
+# JiTCatch configuration. API keys are read from the environment variables named below.
+
+# ---------------------------------------------------------------- LLM providers
+# type: "anthropic" | "openai_chat" | "openai_responses"   (or use a preset)
+# presets: anthropic, openai, openai-chat, zai-coding, zai-coding-anthropic,
+#          zai-coding-responses, ollama, lmstudio, vllm
+
+[providers.claude]
+preset = "anthropic"                 # ANTHROPIC_API_KEY
+model = "claude-sonnet-5-5"
+
+[providers.glm]
+preset = "zai-coding"                # https://api.z.ai/api/coding/paas/v4, ZAI_API_KEY
+model = "glm-5.3-flash"              # preset sets max_tokens = 65536, temperature = 1.0 (Z.ai defaults)
+# GLM-5.3 / 5.3-Flash always think; trade depth for Coding Plan quota with:
+# extra_body = { reasoning_effort = "high" }   # "max" (default) | "high" | "low"
+
+# [providers.gpt]
+# preset = "openai"                  # Responses API, OPENAI_API_KEY
+# model = "gpt-5"
+# omit_params = ["temperature"]      # for models that reject temperature
+
+# [providers.local]
+# preset = "ollama"                  # http://localhost:11434/v1
+# model = "qwen2.5-coder:32b"
+# logprobs = true                    # enables the token-probability judge (if the server supports it)
+
+# [providers.proxy]                  # any Anthropic-compatible endpoint
+# type = "anthropic"
+# base_url = "https://my-gateway.example.com"
+# api_key_env = "GATEWAY_KEY"
+# auth = "bearer"                    # or "x-api-key"
+# model = "claude-sonnet-5-5"
+
+[roles]
+generator = "claude"                 # writes intents, mutants and tests
+judges = ["claude", "glm"]           # LLM-as-judge ensemble (median of High/Medium/Low)
+# logprob_judge = "local"            # optional; must be openai_chat with logprobs = true
+
+# ---------------------------------------------------------------- pipeline
+[run]
+workflows = ["intent_aware", "dodgy_diff"]
+max_target_files = 5
+max_risks_per_file = 4
+dodgy_tests_per_file = 2
+max_fix_attempts = 2
+check_mutants_with_existing_tests = true
+flaky_reruns = 1
+test_timeout_s = 300
+min_risk_score = 0.0                 # raise to only run on riskier changes
+report_threshold = 0.2               # combined score (-1..1) needed to report a catch
+link_paths = ["node_modules"]        # symlinked into the temporary worktrees
+exclude = ["**/generated/**"]
+
+[assessors]
+weights = { rules = 1.0, llm_bucket = 1.0, llm_logprob = 1.0 }
+discard_on_high_fp_rule = true
+# rule_overrides = { flakiness = "off", changed_bool = "0.5" }
+
+# ---------------------------------------------------------------- languages
+[adapters.python]
+# python = ".venv/bin/python"        # interpreter with pytest + project deps
+
+[adapters.typescript]
+runner = "auto"                      # auto | jest | vitest
+
+[adapters.java]
+build = "auto"                       # auto | maven | gradle
+
+[adapters.go]
+
+[hook]
+blocking = false                     # true: a reported catch blocks the push
+# default_branch = "main"
+'''