jittest 0.2.1__py3-none-any.whl

jittest/__init__.py

@@ -0,0 +1,13 @@
+"""jittest - just-in-time catching tests for pull requests.
+
+The open-source reference implementation of Just-in-Time catching test
+generation, the method published by Meta in 2026 (arXiv 2601.22832).
+
+A catching test is not a test that passes. It is a test that PASSES on the
+base commit and FAILS on the head commit, proving the change broke something.
+That distinction is the whole product.
+"""
+from __future__ import annotations
+
+__version__ = "0.2.1"
+__all__ = ["__version__"]

jittest/_minirunner.py

@@ -0,0 +1,121 @@
+"""A ~100 line test runner, used only when pytest is not importable.
+
+Why this exists: jittest must be able to answer the question "does this test
+pass here?" in environments where we cannot install anything - hardened CI
+images, air-gapped runners, or a first-time user who has not created a virtual
+environment yet. Refusing to run is a worse answer than running a smaller
+runner.
+
+Exit codes deliberately mirror pytest so the oracle needs no special cases:
+
+    0  all tests passed
+    1  at least one test failed
+    2  the file could not be imported or collected
+    5  no tests were collected
+"""
+from __future__ import annotations
+
+import asyncio
+import importlib.util
+import inspect
+import sys
+import traceback
+from pathlib import Path
+
+EXIT_OK = 0
+EXIT_FAILED = 1
+EXIT_COLLECTION_ERROR = 2
+EXIT_NO_TESTS = 5
+
+
+def _load(path: Path):
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"cannot load {path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[path.stem] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def _call(fn) -> None:
+    if inspect.iscoroutinefunction(fn):
+        asyncio.run(fn())
+    else:
+        fn()
+
+
+def _collect(module) -> list[tuple[str, object]]:
+    tests: list[tuple[str, object]] = []
+    for name in dir(module):
+        obj = getattr(module, name)
+        if name.startswith("test_") and callable(obj):
+            tests.append((name, obj))
+        elif name.startswith("Test") and inspect.isclass(obj):
+            for method_name in dir(obj):
+                if method_name.startswith("test_"):
+                    tests.append((f"{name}.{method_name}", (obj, method_name)))
+    return tests
+
+
+def run_file(path: Path) -> int:
+    try:
+        module = _load(path)
+    except BaseException:
+        print(f"COLLECTION ERROR in {path.name}", file=sys.stderr)
+        traceback.print_exc()
+        return EXIT_COLLECTION_ERROR
+
+    tests = _collect(module)
+    if not tests:
+        print("no tests collected", file=sys.stderr)
+        return EXIT_NO_TESTS
+
+    failures = 0
+    for label, target in tests:
+        try:
+            if isinstance(target, tuple):
+                cls, method_name = target
+                instance = cls()
+                if hasattr(instance, "setup_method"):
+                    instance.setup_method(getattr(instance, method_name))
+                _call(getattr(instance, method_name))
+                if hasattr(instance, "teardown_method"):
+                    instance.teardown_method(getattr(instance, method_name))
+            else:
+                sig = inspect.signature(target)
+                if sig.parameters:
+                    print(f"SKIP {label}: fixtures are not supported by the "
+                          "mini-runner; install pytest", file=sys.stderr)
+                    continue
+                _call(target)
+            print(f"PASS {label}")
+        except BaseException as exc:
+            failures += 1
+            print(f"FAIL {label}: {type(exc).__name__}: {exc}", file=sys.stderr)
+            traceback.print_exc()
+
+    return EXIT_FAILED if failures else EXIT_OK
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = list(sys.argv[1:] if argv is None else argv)
+    paths = [Path(a) for a in args if not a.startswith("-")]
+    if not paths:
+        print("usage: python -m jittest._minirunner <test_file.py>", file=sys.stderr)
+        return EXIT_COLLECTION_ERROR
+
+    worst = EXIT_NO_TESTS
+    for path in paths:
+        code = run_file(path)
+        if code == EXIT_COLLECTION_ERROR:
+            return code
+        if code == EXIT_FAILED:
+            worst = EXIT_FAILED
+        elif code == EXIT_OK and worst != EXIT_FAILED:
+            worst = EXIT_OK
+    return worst
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

jittest/assess.py

@@ -0,0 +1,89 @@
+"""The assessor layer.
+
+Meta reports that assessor agents cut reviewer load by roughly 70% (arXiv
+2601.22832). That number is the difference between a tool people keep switched
+on and one they mute in week two: independent 2026 measurements put 15-30% of
+AI review comments in the low-value or incorrect bucket.
+
+The oracle decides whether a test is *true*. The assessor decides whether it is
+*worth saying*. Only a confident real_regression is reported by default.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+__all__ = ["Assessment", "parse_assessment", "MIN_CONFIDENCE", "VALID_VERDICTS"]
+
+MIN_CONFIDENCE = 0.70
+VALID_VERDICTS = ("real_regression", "intended_change", "unclear")
+VALID_SEVERITY = ("low", "medium", "high")
+
+
+def _coerce_confidence(raw: object) -> float:
+    try:
+        value = float(raw)  # type: ignore[arg-type]
+    except (TypeError, ValueError):
+        return 0.0
+    if value > 1.0:                 # models like answering "85"
+        value = value / 100.0
+    return round(min(1.0, max(0.0, value)), 3)
+
+
+@dataclass
+class Assessment:
+    verdict: str = "unclear"
+    confidence: float = 0.0
+    severity: str = "medium"
+    summary: str = ""
+    reviewer_question: str = ""
+    raw: str = ""
+
+    @property
+    def should_report(self) -> bool:
+        return self.verdict == "real_regression" and self.confidence >= MIN_CONFIDENCE
+
+    @property
+    def badge(self) -> str:
+        return {
+            "real_regression": "likely regression",
+            "intended_change": "looks intended",
+            "unclear": "needs a human",
+        }.get(self.verdict, "needs a human")
+
+    def as_dict(self) -> dict:
+        return {
+            "verdict": self.verdict,
+            "confidence": self.confidence,
+            "severity": self.severity,
+            "summary": self.summary,
+            "reviewer_question": self.reviewer_question,
+            "should_report": self.should_report,
+        }
+
+
+def parse_assessment(payload: dict | None, raw: str = "") -> Assessment:
+    """Never raise. A malformed assessor reply degrades to `unclear`, which is
+    not reported - failing closed is the whole point of this layer."""
+    if not isinstance(payload, dict):
+        return Assessment(summary="assessor returned unparseable output", raw=raw)
+
+    verdict = str(payload.get("verdict", "")).strip().lower()
+    if verdict not in VALID_VERDICTS:
+        verdict = "unclear"
+
+    severity = str(payload.get("severity", "medium")).strip().lower()
+    if severity not in VALID_SEVERITY:
+        severity = "medium"
+
+    confidence = _coerce_confidence(payload.get("confidence", 0.0))
+    if verdict == "unclear":
+        confidence = min(confidence, MIN_CONFIDENCE - 0.01)
+
+    return Assessment(
+        verdict=verdict,
+        confidence=confidence,
+        severity=severity,
+        summary=str(payload.get("summary", "")).strip()[:300],
+        reviewer_question=str(payload.get("reviewer_question", "")).strip()[:300],
+        raw=raw,
+    )

jittest/cli.py

@@ -0,0 +1,232 @@
+"""Command line interface. argparse, because a CI tool has no business dragging
+Typer, Click and Rich into somebody else's dependency graph.
+
+    jittest run --base main --head HEAD
+    jittest run --dry-run          # whole pipeline, no model, no key, no cost
+    jittest doctor                 # can this environment run jittest?
+    jittest stats                  # what the ledger has learned
+    jittest outcome <hash> fixed_code
+    jittest export corpus.jsonl
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from . import __version__
+from .config import load_config
+from .execute import detect_runner
+from .ledger import HUMAN_OUTCOMES, Ledger
+from .llm import LLMError, build_llm
+from .pipeline import run as run_pipeline
+from .report import to_markdown, to_terminal
+
+
+def _add_run_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--repo", default=".", help="path to the git repository")
+    p.add_argument("--base", default=os.getenv("JITTEST_BASE", "origin/main"))
+    p.add_argument("--head", default=os.getenv("JITTEST_HEAD", "HEAD"))
+    p.add_argument("--model", default=None)
+    p.add_argument("--candidates", type=int, default=None,
+                   help="candidate tests generated per target")
+    p.add_argument("--max-targets", type=int, default=None)
+    p.add_argument("--risk-threshold", type=float, default=None)
+    p.add_argument("--budget", type=float, default=None, help="hard USD cap")
+    p.add_argument("--timeout", type=int, default=None, help="per test run, seconds")
+    p.add_argument("--reruns", type=int, default=None,
+                   help="flakiness reruns on head (default 2)")
+    p.add_argument("--latent", action="store_true",
+                   help="also report faults that fail on base too")
+    p.add_argument("--dry-run", action="store_true",
+                   help="run everything with a stub model: no API key, no cost")
+    p.add_argument("--comment", action="store_true", help="upsert a PR comment")
+    p.add_argument("--fail-on-regression", action="store_true",
+                   help="exit 1 when a confident regression is found")
+    p.add_argument("--json", dest="as_json", action="store_true")
+    p.add_argument("--markdown", metavar="PATH", default=None,
+                   help="also write the markdown report to this file")
+    p.add_argument("--quiet", action="store_true")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="jittest",
+        description="Generate tests that FAIL on your pull request and PASS on main.",
+    )
+    parser.add_argument("--version", action="version", version=f"jittest {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    _add_run_args(sub.add_parser("run", help="analyse a diff"))
+
+    st = sub.add_parser("stats", help="summarise the local ledger")
+    st.add_argument("--repo", default=".")
+    st.add_argument("--json", dest="as_json", action="store_true")
+
+    ex = sub.add_parser("export", help="export the corpus as JSONL")
+    ex.add_argument("out")
+    ex.add_argument("--repo", default=".")
+    ex.add_argument("--with-source", action="store_true",
+                    help="include repo names and test code (off by default)")
+
+    oc = sub.add_parser("outcome", help="label what a human did with a finding")
+    oc.add_argument("test_hash")
+    oc.add_argument("outcome", choices=list(HUMAN_OUTCOMES))
+    oc.add_argument("--note", default="")
+    oc.add_argument("--repo", default=".")
+
+    dr = sub.add_parser("doctor", help="check that this environment can run jittest")
+    dr.add_argument("--repo", default=".")
+
+    sub.add_parser("version", help="print the version")
+    return parser
+
+
+def _cmd_run(args: argparse.Namespace) -> int:
+    repo = Path(args.repo).resolve()
+    cfg = load_config(repo, overrides={
+        "model": args.model,
+        "candidates_per_target": args.candidates,
+        "max_targets": args.max_targets,
+        "risk_threshold": args.risk_threshold,
+        "budget_usd": args.budget,
+        "timeout_s": args.timeout,
+        "reruns": args.reruns,
+        "latent_mode": True if args.latent else None,
+        "fail_on_regression": True if args.fail_on_regression else None,
+    })
+
+    from .github import pr_context, upsert_pr_comment
+    pr_title, pr_body = pr_context()
+
+    try:
+        llm = build_llm(cfg.model, dry_run=args.dry_run, budget_usd=cfg.budget_usd,
+                        temperature=cfg.temperature, cache_path=repo / cfg.cache_path)
+    except LLMError as exc:
+        print(f"jittest: {exc}", file=sys.stderr)
+        return 2
+
+    emit = (lambda m: None) if args.quiet else (lambda m: print(f"  {m}", file=sys.stderr))
+    report = run_pipeline(repo, args.base, args.head, cfg, llm,
+                          pr_title=pr_title, pr_body=pr_body,
+                          pr_ref=os.getenv("JITTEST_PR_NUMBER", ""),
+                          on_event=emit)
+
+    markdown = to_markdown(report)
+    if args.as_json:
+        print(json.dumps(report.as_dict(), indent=2))
+    else:
+        print(to_terminal(report))
+
+    if args.markdown:
+        Path(args.markdown).write_text(markdown or "", encoding="utf-8")
+
+    if args.comment:
+        print(f"  github: {upsert_pr_comment(markdown)}", file=sys.stderr)
+
+    if os.getenv("GITHUB_OUTPUT"):
+        with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fh:
+            fh.write(f"regressions={'true' if report.has_regression else 'false'}\n")
+            fh.write(f"findings={len(report.findings)}\n")
+            fh.write(f"cost_usd={report.cost_usd:.4f}\n")
+
+    if (args.fail_on_regression or cfg.fail_on_regression) and report.has_regression:
+        return 1
+    return 0
+
+
+def _cmd_stats(args: argparse.Namespace) -> int:
+    cfg = load_config(Path(args.repo))
+    with Ledger(Path(args.repo) / cfg.ledger_path) as ledger:
+        stats = ledger.stats()
+    if args.as_json:
+        print(json.dumps(stats, indent=2))
+        return 0
+    print("jittest ledger")
+    for key, value in stats.items():
+        print(f"  {key:22} {value}")
+    if not stats["labelled"]:
+        print("\n  No human outcomes recorded yet, so the precision numbers above")
+        print("  are unknown rather than good. Label findings with `jittest outcome`.")
+    return 0
+
+
+def _cmd_export(args: argparse.Namespace) -> int:
+    cfg = load_config(Path(args.repo))
+    with Ledger(Path(args.repo) / cfg.ledger_path) as ledger:
+        n = ledger.export_jsonl(args.out, anonymise=not args.with_source)
+    print(f"wrote {n} record(s) to {args.out} "
+          f"({'with source' if args.with_source else 'anonymised'})")
+    return 0
+
+
+def _cmd_outcome(args: argparse.Namespace) -> int:
+    cfg = load_config(Path(args.repo))
+    with Ledger(Path(args.repo) / cfg.ledger_path) as ledger:
+        n = ledger.mark_outcome_by_hash(args.test_hash, args.outcome, args.note)
+    print(f"labelled {n} record(s) as {args.outcome}")
+    return 0 if n else 1
+
+
+def _cmd_doctor(args: argparse.Namespace) -> int:
+    repo = Path(args.repo).resolve()
+    cfg = load_config(repo)
+    ok = True
+
+    def check(label: str, passed: bool, detail: str = "") -> None:
+        nonlocal ok
+        ok = ok and passed
+        print(f"  [{'ok  ' if passed else 'FAIL'}] {label}"
+              f"{(' - ' + detail) if detail else ''}")
+
+    print(f"jittest {__version__} doctor")
+    check("python >= 3.11", sys.version_info >= (3, 11), sys.version.split()[0])
+
+    git = subprocess.run(["git", "--version"], capture_output=True, text=True)
+    check("git available", git.returncode == 0, git.stdout.strip())
+
+    inside = subprocess.run(
+        ["git", "-C", str(repo), "rev-parse", "--is-inside-work-tree"],
+        capture_output=True, text=True)
+    check("inside a git repository", inside.stdout.strip() == "true", str(repo))
+
+    runner = detect_runner()
+    using_pytest = "pytest" in " ".join(runner)
+    print(f"  [ok  ] test runner: "
+          f"{'pytest' if using_pytest else 'built-in mini-runner'}")
+    if not using_pytest:
+        print("         pytest is not importable here, so the oracle will use the")
+        print("         stdlib fallback, which does not support fixtures.")
+
+    has_key = any(os.getenv(k) for k in
+                  ("JITTEST_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"))
+    print(f"  [{'ok  ' if has_key else 'warn'}] model API key present"
+          f"{'' if has_key else ' - only --dry-run will work'}")
+    print(f"  [ok  ] model {cfg.model}, budget ${cfg.budget_usd:.2f}, "
+          f"max targets {cfg.max_targets}")
+    print(f"  [ok  ] ledger {repo / cfg.ledger_path}")
+    print(f"  [ok  ] {len(cfg.ignore)} ignore pattern(s)")
+    return 0 if ok else 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command == "run":
+        return _cmd_run(args)
+    if args.command == "stats":
+        return _cmd_stats(args)
+    if args.command == "export":
+        return _cmd_export(args)
+    if args.command == "outcome":
+        return _cmd_outcome(args)
+    if args.command == "doctor":
+        return _cmd_doctor(args)
+    print(__version__)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

jittest/config.py

@@ -0,0 +1,127 @@
+"""Configuration with a boring, predictable precedence chain:
+
+    dataclass defaults  ->  .jittest.toml  ->  [tool.jittest] in pyproject.toml
+    ->  JITTEST_* environment variables  ->  command line flags
+
+Read with stdlib `tomllib`. No PyYAML, no pydantic.
+"""
+from __future__ import annotations
+
+import os
+import tomllib
+from dataclasses import asdict, dataclass, field
+from fnmatch import fnmatch
+from pathlib import Path
+
+__all__ = ["Config", "load_config", "DEFAULT_IGNORES"]
+
+# Generated, vendored or throwaway code. Testing it wastes money and reviewer
+# patience in equal measure.
+DEFAULT_IGNORES = [
+    "*/migrations/*", "*/vendor/*", "*/vendored/*", "*/node_modules/*",
+    "*/.venv/*", "*/venv/*", "*/build/*", "*/dist/*", "*/__pycache__/*",
+    "*_pb2.py", "*_pb2_grpc.py", "*.pyi", "setup.py", "conftest.py",
+    "*/docs/*", "*/examples/*",
+]
+
+_ENV = {
+    "model": ("JITTEST_MODEL", str),
+    "budget_usd": ("JITTEST_BUDGET_USD", float),
+    "max_targets": ("JITTEST_MAX_TARGETS", int),
+    "candidates_per_target": ("JITTEST_CANDIDATES", int),
+    "risk_threshold": ("JITTEST_RISK_THRESHOLD", float),
+    "timeout_s": ("JITTEST_TIMEOUT", int),
+    "reruns": ("JITTEST_RERUNS", int),
+    "min_confidence": ("JITTEST_MIN_CONFIDENCE", float),
+    "ledger_path": ("JITTEST_LEDGER", str),
+}
+
+
+@dataclass
+class Config:
+    model: str = "anthropic/claude-sonnet-4-5"
+    budget_usd: float = 1.00
+    max_targets: int = 5
+    candidates_per_target: int = 4
+    risk_threshold: float = 0.35
+    timeout_s: int = 120
+    reruns: int = 2
+    temperature: float = 0.8
+    min_confidence: float = 0.70
+    repair_attempts: int = 1
+    latent_mode: bool = False
+    fail_on_regression: bool = False
+    ledger_path: str = ".jittest/ledger.db"
+    cache_path: str = ".jittest/cache.db"
+    ignore: list[str] = field(default_factory=lambda: list(DEFAULT_IGNORES))
+
+    def is_ignored(self, path: str) -> bool:
+        p = path.replace("\\", "/")
+        name = p.split("/")[-1]
+        for pattern in self.ignore:
+            if fnmatch(p, pattern) or fnmatch(name, pattern) or fnmatch("/" + p, pattern):
+                return True
+        return False
+
+    def as_dict(self) -> dict:
+        return asdict(self)
+
+
+def _from_toml(repo: Path) -> dict:
+    for filename, section in ((".jittest.toml", None), ("pyproject.toml", "jittest")):
+        path = repo / filename
+        if not path.exists():
+            continue
+        try:
+            with path.open("rb") as fh:
+                data = tomllib.load(fh)
+        except (OSError, tomllib.TOMLDecodeError):
+            continue
+        if section:
+            data = data.get("tool", {}).get(section, {})
+        elif "jittest" in data and isinstance(data["jittest"], dict):
+            data = data["jittest"]
+        if data:
+            return dict(data)
+    return {}
+
+
+def _from_env() -> dict:
+    out: dict = {}
+    for key, (env_name, caster) in _ENV.items():
+        raw = os.getenv(env_name)
+        if raw is None or raw == "":
+            continue
+        try:
+            out[key] = caster(raw)
+        except ValueError:
+            continue
+    return out
+
+
+def load_config(repo: Path | str = ".", overrides: dict | None = None) -> Config:
+    repo = Path(repo)
+    values: dict = {}
+    values.update(_from_toml(repo))
+    values.update(_from_env())
+    for key, value in (overrides or {}).items():
+        if value is not None:
+            values[key] = value
+
+    valid = {f for f in Config.__dataclass_fields__}
+    extra_ignores = values.pop("ignore", None)
+    cfg = Config(**{k: v for k, v in values.items() if k in valid})
+
+    if extra_ignores:
+        cfg.ignore = list(DEFAULT_IGNORES) + list(extra_ignores)
+
+    ignore_file = repo / ".jittestignore"
+    if ignore_file.exists():
+        try:
+            lines = ignore_file.read_text(encoding="utf-8").splitlines()
+        except OSError:
+            lines = []
+        cfg.ignore += [ln.strip() for ln in lines
+                       if ln.strip() and not ln.strip().startswith("#")]
+
+    return cfg