jitcatch 0.2.0__py3-none-any.whl

jitcatch/adapters/python.py

@@ -0,0 +1,187 @@
+"""pytest adapter."""
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import sys
+from pathlib import Path, PurePosixPath
+
+from ..models import Outcome, TestRun
+from .base import Adapter, TestPlacement, make_run, run_cmd
+
+log = logging.getLogger("jitcatch")
+
+
+class PythonAdapter(Adapter):
+    name = "python"
+    fence_langs = ("python", "py")
+    source_exts = (".py",)
+    _resolved: str | None = None
+
+    def __init__(self, settings, repo):
+        super().__init__(settings, repo)
+        self._pytest_path = _UNSET
+
+    def is_test_file(self, rel_path: str) -> bool:
+        p = PurePosixPath(rel_path)
+        n = p.name
+        return (
+            n.startswith("test_") or n.endswith("_test.py") or n == "conftest.py"
+            or any(part in ("tests", "test") for part in p.parts[:-1])
+        )
+
+    def handles(self, rel_path: str) -> bool:
+        if not super().handles(rel_path):
+            return False
+        n = PurePosixPath(rel_path).name
+        return n not in ("setup.py", "noxfile.py", "manage.py") and "/migrations/" not in f"/{rel_path}"
+
+    def _project_envs(self) -> list[str]:
+        """The project's own interpreters (they have its dependencies): repo venvs, then the active venv."""
+        envs: list[str] = []
+        for rel in (".venv/bin/python", "venv/bin/python", ".venv/Scripts/python.exe"):
+            p = self.repo / rel
+            if p.exists():
+                envs.append(str(p))
+        venv = os.environ.get("VIRTUAL_ENV")
+        if venv and (Path(venv) / "bin" / "python").exists():
+            envs.append(str(Path(venv) / "bin" / "python"))
+        return list(dict.fromkeys(envs))
+
+    def _candidates(self) -> list[str]:
+        cands = [c for c in (shutil.which("python3"), shutil.which("python"), sys.executable) if c]
+        return list(dict.fromkeys(cands))
+
+    def python_exe(self) -> str:
+        """Interpreter used to run tests: the configured one, else the project's venv (it has the project's
+        dependencies; pytest is supplied by jitcatch if missing), else the first PATH interpreter with pytest."""
+        if self.settings.get("python"):
+            return self.settings["python"]
+        if self._resolved is None:
+            envs = self._project_envs()
+            if envs:
+                self._resolved = envs[0]
+            else:
+                cands = self._candidates()
+                self._resolved = next((c for c in cands if _has_pytest(c, self.repo)), cands[0])
+        return self._resolved
+
+    def pytest_path(self) -> str | None:
+        """Extra PYTHONPATH entry that provides pytest: "" if the interpreter already has it, a jitcatch cache
+        directory if jitcatch installed it there (the project is never modified), None if unavailable."""
+        if self._pytest_path is _UNSET:
+            exe = self.python_exe()
+            self._pytest_path = "" if _has_pytest(exe, self.repo) else _provision_pytest(exe, self.repo)
+        return self._pytest_path
+
+    def available(self) -> tuple[bool, str]:
+        exe = self.python_exe()
+        if self.pytest_path() is not None:
+            return True, ""
+        return False, (f"pytest is not available for {exe} and jitcatch could not install its own copy "
+                       "(needs uv or pip and network once); install pytest there or set "
+                       "[adapters.python] python = ...")
+
+    def module_name(self, source_rel: str) -> str:
+        parts = list(PurePosixPath(source_rel).with_suffix("").parts)
+        if parts and parts[0] == "src":
+            parts = parts[1:]
+        if parts and parts[-1] == "__init__":
+            parts = parts[:-1]
+        return ".".join(parts)
+
+    def placement(self, source_rel: str, test_id: str, source_text: str) -> TestPlacement:
+        related = self.related_test(source_rel)
+        test_dir = PurePosixPath(related).parent if related else PurePosixPath(source_rel).parent
+        rel = str(test_dir / f"test_jitcatch_{test_id}.py")
+        instr = (
+            "Framework: pytest. Write a self-contained pytest module (plain `def test_...` functions; "
+            "fixtures/monkeypatch allowed). The test runs from the repository root with the root and `src/` "
+            f"on sys.path, so import the module under test as `{self.module_name(source_rel)}` "
+            "(mirror the import style of the example test if one is shown). Do not use network, sleeps, "
+            "randomness or the current time. Do not access private names (leading underscore) of the module "
+            "under test unless there is no public alternative."
+        )
+        return TestPlacement(rel_path=rel, instructions=instr)
+
+    def _env(self, worktree: Path) -> dict[str, str]:
+        paths = [str(worktree)]
+        if (worktree / "src").is_dir():
+            paths.append(str(worktree / "src"))
+        if self.pytest_path():
+            paths.append(self.pytest_path())
+        return {"PYTHONPATH": os.pathsep.join(paths)}
+
+    def run_test(self, worktree: Path, test_rel: str, placement: TestPlacement, timeout: float) -> TestRun:
+        cmd = [self.python_exe(), "-m", "pytest", test_rel, "-q", "-p", "no:cacheprovider",
+               "--rootdir", str(worktree), "-o", "addopts=", "--tb=short", *self.settings.get("extra_args", [])]
+        code, out, secs, timed_out = run_cmd(cmd, worktree, timeout, self._env(worktree))
+        return make_run(self._classify(code, out, timed_out), code, out, secs, cmd)
+
+    def run_existing_tests(self, worktree: Path, source_rel: str, timeout: float) -> TestRun | None:
+        cmd = [self.python_exe(), "-m", "pytest", "-q", "-x", "-p", "no:cacheprovider", "--tb=line",
+               "--ignore-glob=**/test_jitcatch_*.py"]
+        code, out, secs, timed_out = run_cmd(cmd, worktree, timeout, self._env(worktree))
+        outcome = Outcome.PASS if code in (0, 5) else self._classify(code, out, timed_out)
+        return make_run(outcome, code, out, secs, cmd)
+
+    @staticmethod
+    def _classify(code: int, out: str, timed_out: bool) -> Outcome:
+        if timed_out or code in (-2, 3, 4):
+            return Outcome.INFRA
+        if code == 0:
+            return Outcome.PASS
+        if code == 5:
+            return Outcome.NO_TESTS
+        if code == 2:  # interrupted: collection / import errors
+            return Outcome.ERROR
+        if code == 1:
+            return Outcome.FAIL
+        return Outcome.INFRA
+
+
+def _has_pytest(exe: str, cwd: Path, extra_path: str = "") -> bool:
+    code, _, _, _ = run_cmd([exe, "-c", "import pytest"], cwd, 60, {"PYTHONPATH": extra_path} if extra_path else None)
+    return code == 0
+
+
+_UNSET = object()
+
+
+def _py_version(exe: str, cwd: Path) -> str | None:
+    code, out, _, _ = run_cmd([exe, "-c", "import sys; print('%d.%d' % sys.version_info[:2])"], cwd, 60)
+    v = out.strip().splitlines()[-1] if code == 0 and out.strip() else ""
+    return v if v.count(".") == 1 else None
+
+
+def _provision_pytest(exe: str, cwd: Path) -> str | None:
+    """Install pytest into jitcatch's cache (keyed by Python version) and return that directory.
+    It is only added to PYTHONPATH of the test processes; the project's environment is left untouched."""
+    from ..userconfig import cache_home
+    ver = _py_version(exe, cwd)
+    if ver is None:
+        return None
+    target = cache_home() / f"pytest-py{ver}"
+    if (target / "pytest").exists() and _has_pytest(exe, cwd, str(target)):
+        return str(target)
+    log.info("pytest is not installed for %s; installing a private copy into %s (your project is not changed)…",
+             exe, target)
+    tmp = target.with_name(target.name + ".tmp")
+    shutil.rmtree(tmp, ignore_errors=True)
+    attempts = []
+    if shutil.which("uv"):
+        attempts.append(["uv", "pip", "install", "--quiet", "--target", str(tmp), "--python", exe, "pytest"])
+    attempts.append([exe, "-m", "pip", "install", "--quiet", "--disable-pip-version-check",
+                     "--target", str(tmp), "pytest"])
+    attempts.append([sys.executable, "-m", "pip", "install", "--quiet", "--disable-pip-version-check",
+                     "--target", str(tmp), "--python-version", ver, "--only-binary=:all:", "pytest"])
+    for cmd in attempts:
+        code, out, _, _ = run_cmd(cmd, cwd, 600)
+        if code == 0 and _has_pytest(exe, cwd, str(tmp)):
+            shutil.rmtree(target, ignore_errors=True)
+            tmp.rename(target)
+            return str(target)
+        shutil.rmtree(tmp, ignore_errors=True)
+    log.warning("could not install pytest for %s", exe)
+    return None

jitcatch/assessors/__init__.py

@@ -0,0 +1,94 @@
+"""Combine the assessors into a single true/false-positive score per weak catch."""
+from __future__ import annotations
+
+from typing import Any
+
+from ..llm import LLMClient
+from ..models import Assessment, AssessorScore, ChangeContext, GeneratedTest, Intent
+from .. import prompts
+from . import failure_parse, llm_judge, rules
+
+
+def assess_weak_catch(
+    test: GeneratedTest,
+    ctx: ChangeContext,
+    intent: Intent | None,
+    language: str,
+    fence: str,
+    judges: list[LLMClient],
+    logprob_judge: LLMClient | None,
+    settings: dict[str, Any],
+    max_patch_chars: int,
+) -> Assessment:
+    assert test.head_run is not None
+    file_patch = next((f.patch for f in ctx.files if f.path == test.target_file), "") or ctx.full_patch
+    rin = rules.RuleInput(test=test, output=test.head_run.output, patch=file_patch, intent=intent,
+                          title=ctx.title, language=language)
+    rule_score = rules.assess(rin, settings.get("rule_overrides") or {})
+    scores: list[AssessorScore] = [rule_score]
+
+    if rule_score.details.get("high_fp") and settings.get("discard_on_high_fp_rule", True):
+        names = ", ".join(h["rule"] for h in rule_score.details["false_positive_patterns"] if h["level"] == "high")
+        return Assessment(test_id=test.id, scores=scores, combined=-1.0, discarded=True,
+                          discard_reason=f"rule-based false-positive pattern: {names}",
+                          behavior_change=_fallback_behavior(test))
+
+    user = prompts.JUDGE_USER.format(
+        intent=(f"{intent.summary} (kind: {intent.change_kind})" if intent else "(not inferred)"),
+        title=ctx.title or "(none)", description=(ctx.description or "")[:4000],
+        patch=_clip(file_patch, max_patch_chars), test_path=test.test_path, lang=fence,
+        test_code=test.code, outcome=test.head_run.outcome.value, output=_clip(test.head_run.output, 8000),
+    )
+    bucket, rationales = llm_judge.bucket_ensemble(judges, user)
+    scores.append(bucket)
+    lp_user = prompts.LOGPROB_USER.format(
+        intent=(f"{intent.summary} (kind: {intent.change_kind})" if intent else "(not inferred)"),
+        title=ctx.title or "(none)", description=(ctx.description or "")[:4000],
+        patch=_clip(file_patch, max_patch_chars), test_path=test.test_path, lang=fence,
+        test_code=test.code, outcome=test.head_run.outcome.value, output=_clip(test.head_run.output, 8000),
+    )
+    scores.append(llm_judge.logprob_score(logprob_judge, lp_user))
+
+    weights = settings.get("weights") or {}
+    num = den = 0.0
+    for s in scores:
+        if s.score is None:
+            continue
+        w = float(weights.get(s.name, 1.0))
+        num += w * s.score
+        den += w
+    combined = round(num / den, 4) if den else 0.0
+
+    behavior, question = "", ""
+    valid = [r for r in rationales if "value" in r]
+    if valid and bucket.score is not None:
+        pick = min(valid, key=lambda r: abs(r["value"] - bucket.score))
+        behavior, question = pick.get("behavior_change", ""), pick.get("question", "")
+    if not behavior:
+        behavior = _fallback_behavior(test)
+    if not question and behavior:
+        question = f"{behavior} Is this expected?"
+    return Assessment(
+        test_id=test.id, scores=scores, combined=combined, discarded=False,
+        behavior_change=behavior, question=question,
+        rationales=[{k: str(v) for k, v in r.items() if k in ("judge", "model", "likelihood", "rationale", "error")}
+                    for r in rationales],
+    )
+
+
+def _fallback_behavior(test: GeneratedTest) -> str:
+    if test.head_run is None:
+        return ""
+    comps = failure_parse.comparisons(test.head_run.output)
+    if comps:
+        c = comps[0]
+        return f"A value that was `{c.expected[:120]}` before the change is now `{c.actual[:120]}`."
+    exc = failure_parse.exceptions(test.head_run.output)
+    if exc:
+        name, msg = exc[0]
+        return f"Code that ran successfully before the change now raises `{name}{': ' + msg[:120] if msg else ''}`."
+    return ""
+
+
+def _clip(s: str, n: int) -> str:
+    return s if len(s) <= n else s[: n // 2] + "\n... [truncated] ...\n" + s[-n // 2 :]

jitcatch/assessors/failure_parse.py

@@ -0,0 +1,126 @@
+"""Extract structured facts from test-runner output across pytest, Jest/Vitest, JUnit and go test."""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+
+@dataclass
+class Comparison:
+    actual: str
+    expected: str
+    source: str   # which pattern produced it
+
+
+_PYTEST_ASSERT = re.compile(r"^E\s+(?:AssertionError: )?assert (.+?) (==|!=|is|is not) (.+?)\s*$", re.M)
+_PYTEST_BARE = re.compile(r"^E\s+(?:AssertionError: )?assert (not )?(True|False|None|0|''|\"\"|\[\]|\{\})\s*$", re.M)
+_JEST = re.compile(r"Expected(?: value)?:\s*(.+?)\s*\n\s*Received(?: value)?:\s*(.+?)\s*$", re.M)
+_JUNIT = re.compile(r"expected:? ?<(.*?)> but was:? ?<(.*?)>", re.S)
+_JUNIT5_ALT = re.compile(r"==> expected: <(.*?)> but was: <(.*?)>", re.S)
+_TESTIFY = re.compile(r"expected:\s*(.+?)\s*\n\s*actual\s*:\s*(.+?)\s*$", re.M)
+_GOT_WANT = re.compile(r"got:?\s*(.+?)[,;]?\s+(?:but\s+)?want(?:ed)?:?\s*([^\n]+)", re.I)
+_WANT_GOT = re.compile(r"want(?:ed)?:?\s*(.+?)[,;]?\s+(?:but\s+)?got:?\s*([^\n]+)", re.I)
+
+
+def comparisons(output: str) -> list[Comparison]:
+    out: list[Comparison] = []
+    for m in _PYTEST_ASSERT.finditer(output):
+        out.append(Comparison(actual=m.group(1).strip(), expected=m.group(3).strip(), source="pytest"))
+    for m in _PYTEST_BARE.finditer(output):
+        negated, val = bool(m.group(1)), m.group(2)
+        if val in ("True", "False"):
+            # `assert False`    -> the asserted expression evaluated to False; True was expected.
+            # `assert not True` -> the expression evaluated to True; False was expected.
+            opposite = "False" if val == "True" else "True"
+            out.append(Comparison(actual=val, expected=opposite, source="pytest-bare"))
+        elif negated:
+            continue
+        else:
+            out.append(Comparison(actual=val, expected="<truthy>", source="pytest-bare"))
+    for m in _JEST.finditer(output):
+        out.append(Comparison(actual=m.group(2).strip(), expected=m.group(1).strip(), source="jest"))
+    for rx in (_JUNIT5_ALT, _JUNIT):
+        for m in rx.finditer(output):
+            out.append(Comparison(actual=m.group(2).strip(), expected=m.group(1).strip(), source="junit"))
+    for m in _TESTIFY.finditer(output):
+        out.append(Comparison(actual=m.group(2).strip(), expected=m.group(1).strip(), source="testify"))
+    for m in _GOT_WANT.finditer(output):
+        out.append(Comparison(actual=m.group(1).strip(), expected=m.group(2).strip(), source="got-want"))
+    for m in _WANT_GOT.finditer(output):
+        out.append(Comparison(actual=m.group(2).strip(), expected=m.group(1).strip(), source="want-got"))
+    # de-duplicate
+    seen, uniq = set(), []
+    for c in out:
+        k = (c.actual, c.expected)
+        if k not in seen:
+            seen.add(k)
+            uniq.append(c)
+    return uniq
+
+
+_BOOL = {"true": True, "false": False}
+
+
+def as_bool(s: str) -> bool | None:
+    return _BOOL.get(s.strip().strip("<>").lower())
+
+
+_NULLS = {"none", "null", "undefined", "nil", "<nil>"}
+
+
+def is_null(s: str) -> bool:
+    return s.strip().strip("<>").lower() in _NULLS
+
+
+_EMPTY = {"[]", "{}", "()", "set()", "''", '""', "array []", "object {}", "map[]", "[]{}", "0"}
+
+
+def is_empty_container(s: str) -> bool:
+    v = s.strip()
+    return v in _EMPTY or bool(re.fullmatch(r"(Array|Object|Map|Set)\s*(\(0\))?\s*(\[\]|\{\})", v))
+
+
+def same_elements_different_order(a: str, b: str) -> bool:
+    """True when two rendered collections contain the same items in a different order."""
+    if a == b:
+        return False
+    ta = _tokens(a)
+    tb = _tokens(b)
+    return len(ta) >= 2 and sorted(ta) == sorted(tb) and ta != tb
+
+
+def _tokens(s: str) -> list[str]:
+    inner = s.strip()
+    if len(inner) >= 2 and inner[0] in "[{(" and inner[-1] in "]})":
+        inner = inner[1:-1]
+    return [t.strip() for t in re.split(r",\s*", inner) if t.strip()]
+
+
+# -- exceptions --------------------------------------------------------------------------
+
+_EXC_PATTERNS = [
+    re.compile(r"^E\s+([A-Za-z_][\w.]*(?:Error|Exception|Exit|Interrupt))(?::\s*(.*))?$", re.M),            # pytest
+    re.compile(r"^\s*([A-Z][\w.]*(?:Error|Exception))(?::\s*(.*))?$", re.M),                                    # JS / Java / traceback tail
+    re.compile(r"^panic:\s*(runtime error):?\s*(.*)$", re.M),                                                   # Go
+]
+
+
+def exceptions(output: str) -> list[tuple[str, str]]:
+    found: list[tuple[str, str]] = []
+    for rx in _EXC_PATTERNS:
+        for m in rx.finditer(output):
+            name = m.group(1).split(".")[-1]
+            msg = (m.group(2) or "").strip()
+            if (name, msg) not in found:
+                found.append((name, msg))
+    return found
+
+
+def changed_lines(patch: str) -> list[str]:
+    lines = []
+    for ln in patch.splitlines():
+        if ln.startswith(("+++", "---", "@@", "diff ", "index ")):
+            continue
+        if ln.startswith(("+", "-")):
+            lines.append(ln[1:])
+    return lines

jitcatch/assessors/llm_judge.py

@@ -0,0 +1,75 @@
+"""LLM-as-judge ensemble (paper §6.1).
+
+* Ensemble categorical likelihood ("Bucket Med"): each judge answers High/Medium/Low, mapped to
+  1 / 0 / -1, aggregated with the median (ordinal data).
+* Normalised token-probability score ("TP Prob"): a judge with token log-probabilities answers
+  Yes/No; P(Yes) and P(No) for the first answer token give a score in [-1, 1].
+* Rationales from every judge are kept individually (not merged), as in the paper.
+"""
+from __future__ import annotations
+
+import math
+import statistics
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from ..jsonutil import LLMFormatError, extract_json
+from ..llm import LLMClient, LLMError
+from ..models import AssessorScore
+from .. import prompts
+
+BUCKETS = {"high": 1.0, "medium": 0.0, "low": -1.0}
+
+
+def _one_judge(client: LLMClient, user: str) -> dict[str, Any]:
+    try:
+        resp = client.complete(prompts.JUDGE_SYSTEM, user, max_tokens=1500)
+        data = extract_json(resp.text)
+        if not isinstance(data, dict):
+            raise LLMFormatError("judge output is not a JSON object")
+        level = str(data.get("likelihood", "")).strip().lower()
+        if level not in BUCKETS:
+            raise LLMFormatError(f"invalid likelihood {data.get('likelihood')!r}")
+        return {
+            "judge": client.name, "model": resp.model, "likelihood": level.capitalize(),
+            "value": BUCKETS[level],
+            "behavior_change": str(data.get("behavior_change", "")).strip(),
+            "question": str(data.get("question", "")).strip(),
+            "rationale": str(data.get("rationale", "")).strip(),
+        }
+    except (LLMError, LLMFormatError) as e:
+        return {"judge": client.name, "error": str(e)[:500]}
+
+
+def bucket_ensemble(judges: list[LLMClient], user: str) -> tuple[AssessorScore, list[dict[str, Any]]]:
+    if not judges:
+        return AssessorScore("llm_bucket", None, {"note": "no judges configured"}), []
+    with ThreadPoolExecutor(max_workers=min(8, len(judges))) as ex:
+        results = list(ex.map(lambda c: _one_judge(c, user), judges))
+    values = [r["value"] for r in results if "value" in r]
+    score = statistics.median(values) if values else None
+    return AssessorScore("llm_bucket", score, {"judges": results, "n_valid": len(values)}), results
+
+
+def logprob_score(client: LLMClient | None, user: str) -> AssessorScore:
+    if client is None:
+        return AssessorScore("llm_logprob", None, {"note": "no logprob judge configured"})
+    try:
+        resp = client.complete(prompts.JUDGE_SYSTEM, user, max_tokens=5, want_logprobs=True)
+    except LLMError as e:
+        return AssessorScore("llm_logprob", None, {"error": str(e)[:500]})
+    if not resp.logprobs:
+        return AssessorScore("llm_logprob", None, {"error": "server returned no logprobs", "text": resp.text[:50]})
+    first = next((t for t in resp.logprobs if t.token.strip()), None)
+    if first is None:
+        return AssessorScore("llm_logprob", None, {"error": "empty answer"})
+    candidates = dict(first.top)
+    candidates.setdefault(first.token, first.logprob)
+    p_yes = sum(math.exp(lp) for tok, lp in candidates.items() if tok.strip().lower().startswith("yes"))
+    p_no = sum(math.exp(lp) for tok, lp in candidates.items() if tok.strip().lower().startswith("no"))
+    if p_yes + p_no == 0:
+        return AssessorScore("llm_logprob", None, {"error": f"first token {first.token!r} is neither Yes nor No"})
+    score = (p_yes - p_no) / (p_yes + p_no)
+    return AssessorScore("llm_logprob", round(score, 4), {
+        "answer": first.token.strip(), "p_yes": round(p_yes, 4), "p_no": round(p_no, 4), "judge": client.name,
+    })