gitmole 0.12.0__tar.gz → 0.14.0__tar.gz

{gitmole-0.12.0 → gitmole-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitmole
-Version: 0.12.0
+Version: 0.14.0
 Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
 License: MIT
 Project-URL: Homepage, https://github.com/antvinni/gitmole

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/__init__.py

@@ -1,3 +1,3 @@
 """gitmole: offline git repository analysis with a terminal report."""
 
-__version__ = "0.12.0"
+__version__ = "0.14.0"

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/cli.py

@@ -45,8 +45,16 @@ def parse_args(argv):
     p.add_argument("--risk", metavar="BASE", help="score the files changed since BASE (merge base with HEAD) by their share of the repository's revisions × lines of code; needs a local path")
     p.add_argument("--risk-threshold", type=float, metavar="N", help="with --risk: exit 3 when the changed files hold more than N percent of the repository's revisions × lines of code")
     p.add_argument("--compare", metavar="BEFORE_JSON", help="add a 'Since last report' section against an earlier --json export of the same clone")
+    p.add_argument("--hook", action="store_true", help="with --no-run: read an agent hook's JSON on stdin (or files after --), score the files it names like --risk, "
+                                                       "print a summary the agent reads back, exit 2 when --risk-threshold is exceeded")
     p.add_argument("--version", action="version", version=f"gitmole {__version__}")
-    return p.parse_args(argv)
+    # --hook takes the files to score after --, pre-commit's way. Split them off here: Python 3.9's argparse
+    # cannot give a second positional a value once optionals sit between it and the first.
+    argv = list(argv)
+    files = argv[argv.index("--") + 1:] if "--" in argv else []
+    args = p.parse_args(argv[:argv.index("--")] if "--" in argv else argv)
+    args.files = files
+    return args
 
 
 _control = run.Control()
@@ -58,7 +66,7 @@ def interrupt(*_):
 
 
 def main(argv=None, console: Console = None, tool_check=run.missing_tools, planner=run.plan, estimator=run.estimate_blames,
-         lister=run.list_repos, cloner=run.clone, lizard_check=run.has_lizard, ask=None) -> int:
+         lister=run.list_repos, cloner=run.clone, lizard_check=run.has_lizard, ask=None, stdin=None) -> int:
     global _control
     _control = run.Control()
     if threading.current_thread() is threading.main_thread():
@@ -80,7 +88,7 @@ def main(argv=None, console: Console = None, tool_check=run.missing_tools, plann
         return rc
 
     if args.no_run:
-        return _no_run(args, console, ui, err)
+        return _no_run(args, console, ui, err, stdin)
 
     try:
         kind, target = run.classify_target(args.target)
@@ -128,7 +136,8 @@ def _check_args(args, err, kind=None) -> int | None:
     if kind is None:
         bad = ("--yes needs --clean" if args.yes and not args.clean else
                "target required" if args.target is None and not args.clean else
-               "--risk-threshold needs --risk" if args.risk_threshold is not None and not args.risk else
+               "--hook needs --no-run and an output directory" if args.hook and not args.no_run else
+               "--risk-threshold needs --risk" if args.risk_threshold is not None and not args.risk and not args.hook else
                "--compare: no such file: " + args.compare if args.compare and not os.path.isfile(args.compare) else None)
     elif kind == "path":
         bad = None
@@ -164,8 +173,9 @@ def _resolve_time(args, err, ui):
     return None, now
 
 
-def _no_run(args, console, ui, err) -> int:
-    """Handle --no-run: re-render an existing output directory instead of running the pipeline."""
+def _no_run(args, console, ui, err, stdin=None) -> int:
+    """Handle --no-run: re-render an existing output directory instead of running the pipeline, or,
+    with --hook, score the files an agent's hook names against it."""
     if args.since:
         err.print("[red]--since needs a run:[/red] a re-render cannot narrow an earlier analysis")
         return 2
@@ -173,11 +183,39 @@ def _no_run(args, console, ui, err) -> int:
     if not os.path.isfile(os.path.join(out_dir, "meta.json")):
         err.print(f"[red]no gitmole output found in {out_dir}[/red] (expected meta.json)")
         return 2
+    if args.hook:
+        return _hook(out_dir, args, console, err, sys.stdin if stdin is None else stdin)
     if ui.is_terminal:
         ui.print(banner.neon(version=__version__))
     return _render(out_dir, console, ui, args, err)
 
 
+def _hook(out_dir: str, args, console: Console, err: Console, stdin) -> int:
+    """The agent-hook gate (see hook.py): 2 over the threshold, 0 otherwise, silent when the event
+    names no file in the repository."""
+    from . import hook, watch
+    try:
+        report = load.load_report(out_dir)
+    except load.Unreadable as e:
+        err.print(f"[red]{e}[/red]", soft_wrap=True)
+        return 0   # a broken output directory must not block an edit
+    repo = report["meta"].get("path") or os.getcwd()
+    event = {} if args.files else hook.read_event(stdin)
+    files = [os.path.relpath(os.path.abspath(f), os.path.realpath(repo)) if os.path.isabs(f) else f for f in args.files] or hook.paths_in(event, repo)
+    if not files:
+        return 0
+    risk = watch.change_risk(report, files)
+    lines = hook.summary(risk, args.risk_threshold)
+    if args.files:
+        console.print("\n".join(lines), soft_wrap=True, markup=False, highlight=False)
+    else:
+        console.print(hook.hook_output(event, lines), soft_wrap=True, markup=False, highlight=False)
+    if args.risk_threshold is not None and risk["total"] > args.risk_threshold:
+        err.print("\n".join(lines), soft_wrap=True, markup=False, highlight=False)   # exit 2: what the agent is told
+        return 2
+    return 0
+
+
 def _clean(args, console: Console, ask) -> int:
     """Handle --clean: list what gitmole left behind, ask once, remove. ask(prompt) returns the answer."""
     from . import clean, render
@@ -516,12 +554,12 @@ def _render(out_dir: str, console: Console, ui: Console, args, err: Console) ->
     risk = None
     if args.risk:
         try:
-            files = run.changed_files(report["meta"].get("path") or os.getcwd(), args.risk)
+            stats = run.change_stats(report["meta"].get("path") or os.getcwd(), args.risk)
         except ValueError as e:
             err.print(f"[red]--risk {args.risk}:[/red] {e}", soft_wrap=True)
             return 2
         from . import watch
-        risk = {"base": args.risk, **watch.change_risk(report, files)}
+        risk = {"base": args.risk, **watch.change_risk(report, stats["files"], stats)}
     comparison = None
     if args.compare:
         from . import compare as _compare

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/evaluate.py

@@ -20,7 +20,7 @@ import os
 import subprocess
 import sys
 
-from . import backtest, filetypes, identity, load, maat, trend, watch
+from . import backtest, filetypes, identity, load, maat, szz, trend, watch
 
 
 def cutoffs(last_date: str, windows: int, horizon: int) -> list:
@@ -44,6 +44,32 @@ def fixed_between(commits: list, start: str, end: str) -> set:
             for p, _, _ in c["files"] if not filetypes.is_test_path(p)}
 
 
+def induced_between(repo: str, commits: list, start: str, end: str, exclude=None) -> set:
+    """Source files a commit before `start` made buggy, by R-SZZ over the fixes landing on or after
+    `start` and before `end`: the outcome is defect insertion the list could have known about, not
+    fix locality. A fix whose bug-inducing commit is inside the horizon is not counted, since no
+    list drawn before `start` could have named it."""
+    out = set()
+    for c in maat.in_window(maat.fix_commits(commits), start, end):
+        found = szz.bug_inducing(repo, c["hash"], exclude)
+        if found and found["date"] < start:
+            out.update(p for p in found["files"] if not filetypes.is_test_path(p))
+    return out
+
+
+def labelled_between(commits: list, labels: dict, start: str, end: str) -> set:
+    """Source files an independently labelled bug-inducing commit touched on or after `start` and
+    before `end` (ApacheJIT, Defectors: see szz.read_labels): the paths the label names, or every
+    source file the commit touched. Either side may be abbreviated."""
+    out = set()
+    for c in maat.in_window(commits, start, end):
+        paths = next((v for k, v in labels.items() if k.startswith(c["hash"]) or c["hash"].startswith(k)), "none")
+        if paths == "none":
+            continue
+        out.update(p for p in (paths if paths is not None else [p for p, _, _ in c["files"]]) if not filetypes.is_test_path(p))
+    return out
+
+
 def report_at(commits: list, t: str, size: dict, meta: dict, generated: list, vendored: list, ignored: set = frozenset()) -> dict:
     """The report watch.risks reads, from the commits before `t` (less the sweeps and the declared, as
     the pipeline leaves them out), scc's listing of the tree at `t`, and that tree's own generated and
@@ -54,7 +80,8 @@ def report_at(commits: list, t: str, size: dict, meta: dict, generated: list, ve
     ownership = [r for r in maat.entity_ownership(past) if r["author"] not in bots and not identity.is_bot(r["author"])]
     return {"meta": {"now": t, "generated": generated, "vendored": vendored}, "size": size, "revisions": maat.revisions(past),
             "plumbing": maat.plumbing(past), "authors": maat.authors(past), "ownership": ownership,
-            "fixes": maat.fixes(past, now=t), "coupling": [], "functions": []}
+            "fixes": maat.fixes(past, now=t), "coupling": [], "functions": [],
+            "entropy": maat.entropy(past, now=t)}
 
 
 SOLO_WEIGHT = 1.5       # how much single ownership lifts a factor product
@@ -110,6 +137,8 @@ def variants(report: dict) -> dict:
     for name, key in watch.BASELINES.items():
         out[name] = watch.ranked_by(rows, key)
     out["recent fixes"] = watch.ranked_by(rows, lambda r: (r["recent_fixes"], r["revs"]))
+    hcm = {e["entity"]: e["hcm"] for e in report.get("entropy") or []}
+    out["change entropy (HCM)"] = watch.ranked_by(rows, lambda r: (hcm.get(r["file"], 0.0), r["revs"]))   # Hassan's decayed HCM, the one metric with published evidence of beating churn
     return out
 
 
@@ -122,10 +151,11 @@ def score(report: dict, fixed: set, top: int) -> dict:
     return out
 
 
-def table(results: list) -> str:
-    """results: [(t, files fixed that were in the pool, pool size, {variant: hits})], oldest first -> Markdown."""
+def table(results: list, noun: str = "fixed") -> str:
+    """results: [(t, files fixed that were in the pool, pool size, {variant: hits})], oldest first -> Markdown.
+    `noun` says what the outcome is: fixed, bug-inducing (R-SZZ) or labelled."""
     names = list(results[0][3]) if results else []
-    head = "| variant | " + " | ".join(f"{t} ({fixed} of {pool} fixed)" for t, fixed, pool, _ in results) + " | total |"
+    head = "| variant | " + " | ".join(f"{t} ({fixed} of {pool} {noun})" for t, fixed, pool, _ in results) + " | total |"
     rule = "|---|" + "---:|" * (len(results) + 1)
     rows = []
     for name in names:
@@ -152,6 +182,8 @@ def main(argv=None) -> int:
     p.add_argument("--windows", type=int, default=6)
     p.add_argument("--horizon", type=int, default=6, metavar="MONTHS")
     p.add_argument("--top", type=int, default=watch.WATCH_TOP)
+    p.add_argument("--szz", action="store_true", help="also score against R-SZZ bug-inducing commits (one git blame per fix and file: minutes)")
+    p.add_argument("--labels", metavar="CSV", help="also score against independent bug-inducing labels (ApacheJIT's CSV, Defectors' file rows, or one hash per line)")
     args = p.parse_args(argv)
     meta = load._read_json(args.out, "meta.json", {})
     log_path = os.path.join(args.out, "log.txt")
@@ -165,7 +197,11 @@ def main(argv=None) -> int:
     from . import run
     declared = maat.read_ignore_revs(run.ignore_revs_files(args.repo))
     ignored = {c["hash"] for c in commits if declared and maat.is_ignored(c["hash"], declared)}
-    results = []
+    labels = szz.read_labels(args.labels) if args.labels else None
+    if args.labels and not labels:
+        print(f"evaluate: no bug-inducing commits read from {args.labels}", file=sys.stderr)
+        return 2
+    results, induced_results, labelled_results = [], [], []
     for t in cutoffs(meta["last_date"], args.windows, args.horizon):
         rev = trend.rev_before(args.repo, t, end_of_day=False)
         if not rev:
@@ -173,9 +209,17 @@ def main(argv=None) -> int:
         size_json, generated, vendored = backtest.snapshot_at(args.repo, rev, args.out)
         size = load.parse_scc(size_json, types)
         report = report_at(commits, t, size, meta, generated, vendored, ignored)
-        fixed = fixed_between(commits, t, months_after(t, args.horizon))
+        end = months_after(t, args.horizon)
+        fixed = fixed_between(commits, t, end)
         pool = set(variants(report)["churn"])
         results.append((t, len(fixed & pool), len(pool), score(report, fixed, args.top)))
+        if args.szz:
+            vendor = tuple(vendored)
+            induced = induced_between(args.repo, commits, t, end, exclude=lambda p: p in generated or filetypes.is_vendored(p, vendor) or filetypes.is_sample_path(p))
+            induced_results.append((t, len(induced & pool), len(pool), score(report, induced, args.top)))
+        if labels:
+            marked = labelled_between(commits, labels, t, end)
+            labelled_results.append((t, len(marked & pool), len(pool), score(report, marked, args.top)))
         print(f"evaluate: {t} done", file=sys.stderr)
     if not results:
         print("evaluate: no cut-off falls inside the history", file=sys.stderr)
@@ -183,6 +227,12 @@ def main(argv=None) -> int:
     spread = ref_spread(args.repo)
     print(f"### {meta.get('name', args.repo)}, top {args.top}, {args.horizon}-month horizon\n")
     print(table(results))
+    if induced_results:
+        print(f"\nAgainst the files a commit before the cut-off made buggy, by R-SZZ over the fixes that followed (the most recent commit each fix's removed lines blame to):\n")
+        print(table(induced_results, noun="bug-inducing"))
+    if labelled_results:
+        print(f"\nAgainst the files the bug-inducing commits labelled in {os.path.basename(args.labels)} touched inside each window:\n")
+        print(table(labelled_results, noun="labelled"))
     print(f"\n`--all` exports {spread['all']:,} commits ({spread['fix_all']:,} fixes); HEAD reaches {spread['head']:,} ({spread['fix_head']:,} fixes).")
     return 0
 

gitmole-0.14.0/gitmole/hook.py

@@ -0,0 +1,99 @@
+"""The agent-hook gate: `gitmole OUT_DIR --no-run --hook [--risk-threshold N] [-- FILE...]`.
+
+Deterministic findings should run before inference, so the model reasons over a short list rather
+than rediscovering known issues. A linter says what is wrong in the diff; gitmole says what history
+says about the files the diff touches, and no other tool supplies those fields. The gate has the same
+shape everywhere: the hook's JSON on stdin (Claude Code's PostToolUse, Cursor's afterFileEdit, Gemini
+CLI's AfterTool), the files it names scored like `--risk`, a one-line summary per file with the
+companions the edit left untouched, and exit 2 when the total is over the threshold, which is what
+every one of those hooks reads as "block". Files on the command line (pre-commit's way) print the
+summary as plain text instead of the hook JSON."""
+from __future__ import annotations
+
+import json
+import os
+
+_PATH_KEYS = ("file_path", "filePath", "path", "notebook_path")
+_LIST_KEYS = ("file_paths", "filePaths", "files", "paths")
+_CONTAINERS = ("tool_input", "tool_response", "input", "output")
+
+
+def paths_in(event, repo: str) -> list:
+    """The files a hook event names, relative to `repo`, each once, in order: the path keys the
+    agents use at the top level and under tool_input/tool_response. A path outside the repository is
+    not this repository's risk and is left out; a relative path is taken as repository-relative."""
+    if not isinstance(event, dict):
+        return []
+    found = []
+
+    def take(value):
+        if isinstance(value, str) and value:
+            found.append(value)
+        elif isinstance(value, list):
+            for v in value:
+                if isinstance(v, str) and v:
+                    found.append(v)
+                elif isinstance(v, dict):
+                    for k in _PATH_KEYS:
+                        if isinstance(v.get(k), str):
+                            found.append(v[k])
+
+    for holder in (event, *[event.get(c) for c in _CONTAINERS if isinstance(event.get(c), dict)]):
+        for k in _PATH_KEYS:
+            take(holder.get(k))
+        for k in _LIST_KEYS:
+            take(holder.get(k))
+    root = os.path.realpath(repo)
+    out = []
+    for p in found:
+        full = os.path.realpath(p if os.path.isabs(p) else os.path.join(root, p))
+        if full == root or not full.startswith(root + os.sep):
+            continue
+        rel = os.path.relpath(full, root)
+        if rel not in out:
+            out.append(rel)
+    return out
+
+
+def read_event(stdin) -> dict:
+    """The hook's JSON from stdin, or {} for nothing or for anything that is not JSON: garbage on
+    stdin is not a reason to block an edit."""
+    try:
+        text = stdin.read() if stdin is not None else ""
+    except (OSError, ValueError):
+        return {}
+    if not text or not text.strip():
+        return {}
+    try:
+        event = json.loads(text)
+    except ValueError:
+        return {}
+    return event if isinstance(event, dict) else {}
+
+
+def summary(risk: dict, threshold=None) -> list:
+    """One line per file, worst first, then the companions the change left untouched, then the total
+    against the threshold when there is one."""
+    lines = []
+    for r in risk["files"]:
+        if r.get("rank"):
+            where = f"rank {r['rank']} of {risk.get('pool', '?')}" + (", on the watch list" if r.get("watched") else "")
+            lines.append(f"{r['file']}: {r['score']:.1f}% of the repository's revisions × lines of code ({where}); " + "; ".join(r["reasons"]))
+        else:
+            lines.append(f"{r['file']}: not scored ({r['reason']})")
+    gaps = risk.get("coupling_gaps") or []
+    if gaps:
+        from . import render
+        lines.append(render.gaps_line(gaps))
+    total = f"total {risk['total']:.1f}%"
+    if threshold is not None:
+        total += f", over the {threshold:g}% threshold" if risk["total"] > threshold else f", under the {threshold:g}% threshold"
+    lines.append(total)
+    return lines
+
+
+def hook_output(event: dict, lines: list) -> str:
+    """The JSON Claude Code reads back (hookSpecificOutput.additionalContext, a soft warning); Cursor
+    and Gemini CLI ignore stdout and read the exit code, so the same document serves all three."""
+    name = event.get("hook_event_name") or "PostToolUse"
+    return json.dumps({"hookSpecificOutput": {"hookEventName": name, "additionalContext": "\n".join(lines)}})

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/load.py

@@ -82,7 +82,7 @@ def parse_scc(text: str, types=None) -> dict:
 
 
 NUMERIC_COLUMNS = {"n-revs", "degree", "average-revs", "n-authors", "age-months", "added", "deleted", "n-fixes", "recent-fixes", "tiny-revs",
-                   "minor", "soc", "partners", "n-sets", "with-tests"}
+                   "minor", "soc", "partners", "n-sets", "with-tests", "periods"}
 
 
 def parse_maat_csv(text: str) -> list:
@@ -309,6 +309,7 @@ def load_report(out_dir: str, nested: bool = True) -> dict:
         "coupling": parse_maat_csv(_read(out_dir, "maat-coupling.csv")),
         "soc": parse_maat_csv(_read(out_dir, "maat-soc.csv")),   # sum of coupling; empty for an output directory from before 0.11
         "tests": parse_maat_csv(_read(out_dir, "maat-tests.csv")),   # test co-change per production file; empty before 0.12
+        "entropy": parse_maat_csv(_read(out_dir, "maat-entropy.csv")),   # Hassan's change entropy per file; empty before 0.13
         "authors": parse_maat_csv(_read(out_dir, "maat-authors.csv")),
         "age": parse_maat_csv(_read(out_dir, "maat-age.csv")),
         "ownership": ownership,

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/maat.py

@@ -349,6 +349,36 @@ def test_cochange(commits: list) -> list:
     return rows
 
 
+ENTROPY_DECAY = 0.5   # a period's weight halves for every month it lies before the reference date
+
+
+def entropy(commits: list, now: str = None, decay: float = ENTROPY_DECAY) -> list:
+    """Hassan's history complexity metric (ICSE 2009), decayed: for each calendar month, the Shannon
+    entropy of the files' shares of that month's changes, normalised by log2 of the files changed;
+    a file's score is the sum over months of its share times that entropy, each month weighted by
+    `decay` to the power of its distance from `now`. Changes scattered over many files in a month are
+    hard to keep track of; a month spent on one file is not. `periods` is how many months the file
+    changed in."""
+    now = now or dt.date.today().isoformat()
+    by_month = defaultdict(Counter)
+    for c in commits:
+        for p, _, _ in c["files"]:
+            by_month[c["date"][:7]][p] += 1
+    y0, m0 = int(now[:4]), int(now[5:7])
+    scores, periods = defaultdict(float), Counter()
+    for month, counts in by_month.items():
+        total, n = sum(counts.values()), len(counts)
+        h = -sum((v / total) * math.log2(v / total) for v in counts.values()) / math.log2(n) if n > 1 else 0.0
+        back = (y0 - int(month[:4])) * 12 + (m0 - int(month[5:7]))
+        weight = decay ** max(0, back)
+        for p, v in counts.items():
+            periods[p] += 1
+            scores[p] += (v / total) * h * weight
+    rows = [{"entity": p, "periods": periods[p], "hcm": round(scores[p], 6)} for p in periods]
+    rows.sort(key=lambda r: (-r["hcm"], -r["periods"], r["entity"]))
+    return rows
+
+
 RECENT_MONTHS = 6
 OVERSIZED_PERCENTILE = 0.99   # a fix changing more lines than this share of the history's commits credits nothing
 OVERSIZED_FLOOR = 500         # ...and never under this many lines, so a small repository's percentile does not bite
@@ -526,8 +556,9 @@ ANALYSES = {
     "age": (age, ["entity", "age-months"]),
     "entity-ownership": (entity_ownership, ["entity", "author", "added", "deleted"]),
     "fixes": (fixes, ["entity", "n-fixes", "last-fix", "recent-fixes"]),
+    "entropy": (entropy, ["entity", "periods", "hcm"]),
 }
-NEEDS_NOW = {"age", "fixes"}
+NEEDS_NOW = {"age", "fixes", "entropy"}
 
 
 def aliases_from_meta(path: str) -> dict:

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/render.py

@@ -359,10 +359,25 @@ def risk_section(risk: dict, base: str, full=True) -> dict:
     more = _more(len(rows_all), limit)
     if more:
         notes.append(more)
+    factors = (risk.get("change") or {}).get("reasons") or []
+    if rows and factors:
+        notes.append("; ".join(factors))
+    gaps = risk.get("coupling_gaps") or []
+    if rows and gaps:
+        notes.append(gaps_line(gaps))
     return _section(f"Change risk ({len(rows_all)} files since {base})", columns, rows,
                     note=None if rows else f"no files changed since {base}", caption="\n".join(notes) or None)
 
 
+def gaps_line(gaps: list) -> str:
+    """'not touched: core/ast.py, which changes with core/parser.py 72% of the time, and core/lexer.py (55%)':
+    the companions a change left out, strongest first."""
+    first = gaps[0]
+    rest = [f"{g['companion']} ({g['degree']}%)" for g in gaps[1:4]]
+    line = f"not touched: {first['companion']}, which changes with {first['file']} {first['degree']}% of the time"
+    return line + (", and " + textfmt.join_and(rest) if rest else "") + (f" and {len(gaps) - 4} more" if len(gaps) > 4 else "")
+
+
 def size_section(report: dict, full: bool = True, width=None) -> dict:
     langs = report["size"]["languages"]
     total = report["size"]["total_code"]

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/run.py

@@ -542,6 +542,31 @@ def changed_files(repo_dir: str, base: str) -> list:
     return sorted(p.decode("utf-8", "surrogateescape") for p in proc.stdout.split(b"\0") if p)
 
 
+def change_stats(repo_dir: str, base: str) -> dict:
+    """What a change is, for the Kamei factors: the files that differ between the merge base with
+    `base` and HEAD, lines added and deleted per file (whitespace ignored, as the change log is), the
+    author of HEAD and how many commits the change spans. ValueError when git refuses."""
+    files = changed_files(repo_dir, base)
+    proc = subprocess.run([*filetypes.GIT, "diff", "--numstat", "-w", "--ignore-blank-lines", f"{base}...HEAD"], cwd=repo_dir, capture_output=True)
+    if proc.returncode != 0:
+        raise ValueError((proc.stderr.decode("utf-8", "replace").strip() or f"git diff {base}...HEAD failed"))
+    added, deleted = {f: 0 for f in files}, {f: 0 for f in files}
+    for line in proc.stdout.decode("utf-8", "surrogateescape").split("\n"):
+        parts = line.split("\t", 2)
+        if len(parts) != 3:
+            continue
+        a, d, path = parts
+        path = filetypes.unquote(path)
+        if " => " in path:
+            path = path.split(" => ", 1)[1].rstrip("}") if "{" not in path else path
+        added[path] = int(a) if a.isdigit() else 0
+        deleted[path] = int(d) if d.isdigit() else 0
+    author = subprocess.run(["git", "log", "-1", "--use-mailmap", "--format=%aN", "HEAD"], cwd=repo_dir, capture_output=True).stdout.decode("utf-8", "replace").strip()
+    count = subprocess.run(["git", "rev-list", "--count", f"{base}..HEAD"], cwd=repo_dir, capture_output=True, text=True).stdout.strip()
+    return {"files": files, "added": {f: added.get(f, 0) for f in files}, "deleted": {f: deleted.get(f, 0) for f in files},
+            "author": author, "commits": int(count) if count.isdigit() else 0}
+
+
 def save_meta(meta: dict, out_dir: str) -> None:
     with open(os.path.join(out_dir, "meta.json"), "w") as fh:
         json.dump(meta, fh, indent=2)

gitmole-0.14.0/gitmole/szz.py

@@ -0,0 +1,135 @@
+"""Bug-inducing commits by R-SZZ, from git alone, for the backtest's second outcome.
+
+The watch list's backtest scores "a fix-labelled commit touched the file in the six months after
+the cut-off": fix locality, which a changelog or a config file scores without ever being wrong.
+The SZZ family gives the other label, defect insertion: the commit that wrote the lines a fix
+removed. Rosa et al. (ICSE 2021, JSS 2023) measured the git-only variants against a
+developer-informed oracle and found R-SZZ, which keeps only the most recent candidate per fix,
+the best of them at precision 0.66 against 0.39 for keeping every candidate. This is that variant:
+
+1. the lines a fix removed or changed, per modified file, from `git diff -U0 fix^ fix`;
+2. `git blame -w -C -C` of the parent at those lines, so whitespace and moved code do not mislead;
+3. files the caller excludes (tests, vendored, generated) are not candidates;
+4. of the commits blamed, the most recent by author date is the bug-inducing one, with the files
+   whose blamed lines it wrote.
+
+An insert-only fix blames nothing and R-SZZ finds nothing for it; that is the known blind spot.
+A development tool behind `python -m gitmole.evaluate --szz`, not a pipeline step: one blame per
+(fix, file) is minutes over a large history, and the report's own backtest stays the cheap one."""
+from __future__ import annotations
+
+import csv
+import datetime as dt
+import re
+import subprocess
+
+from . import filetypes
+
+_HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+\d+(?:,\d+)? @@")
+_HEADER = re.compile(r"^([0-9a-f]{40,64}) \d+ \d+(?: \d+)?$")
+_HASH = re.compile(r"^[0-9a-f]{7,64}$")
+
+
+def _git(repo: str, *args) -> str:
+    return subprocess.run([*filetypes.GIT, *args], cwd=repo, capture_output=True, check=True).stdout.decode("utf-8", "replace")
+
+
+def deleted_ranges(repo: str, fix: str) -> dict:
+    """{path: [(start, count)]} of the parent-side lines a fix removed or changed, for every file it
+    modified (added and deleted files have no parent version to blame). An empty list is a file
+    that only gained lines."""
+    out, path = {}, None
+    text = _git(repo, "diff", "-U0", "--diff-filter=M", "--no-color", f"{fix}^", fix)
+    for line in text.split("\n"):
+        if line.startswith("diff --git "):
+            path = None
+        elif line.startswith("+++ b/"):
+            path = filetypes.unquote(line[6:])
+            out.setdefault(path, [])
+        elif path is not None and line.startswith("@@"):
+            m = _HUNK.match(line)
+            if not m:
+                continue
+            start, count = int(m.group(1)), int(m.group(2)) if m.group(2) is not None else 1
+            if count:
+                out[path].append((start, count))
+    return out
+
+
+def blame_lines(repo: str, rev: str, path: str, ranges: list) -> dict:
+    """{commit: {"time": author epoch, "lines": n}} for the given parent-side line ranges, blamed
+    with whitespace ignored and moves followed (-w -C -C: AG-SZZ's robustness for free)."""
+    if not ranges:
+        return {}
+    argv = ["blame", "-w", "-C", "-C", "--line-porcelain"]
+    for start, count in ranges:
+        argv += ["-L", f"{start},{start + count - 1}"]
+    try:
+        text = _git(repo, *argv, rev, "--", path)
+    except subprocess.CalledProcessError:
+        return {}
+    out, current = {}, None
+    for line in text.split("\n"):
+        m = _HEADER.match(line)
+        if m:
+            current = out.setdefault(m.group(1), {"time": 0, "lines": 0})
+            current["lines"] += 1
+        elif current is not None and line.startswith("author-time "):
+            current["time"] = int(line[12:])
+    return out
+
+
+def bug_inducing(repo: str, fix: str, exclude=None):
+    """{"commit", "date", "files"} for the R-SZZ pick of a fix: the most recent commit its deleted
+    lines blame to, and the files whose lines it wrote; None when nothing blames (an insert-only
+    fix, or every touched file excluded). `exclude(path)` says which files are not candidates; test
+    files never are."""
+    candidates = {}
+    for path, ranges in deleted_ranges(repo, fix).items():
+        if filetypes.is_test_path(path) or (exclude and exclude(path)):
+            continue
+        for commit, info in blame_lines(repo, f"{fix}^", path, ranges).items():
+            c = candidates.setdefault(commit, {"time": info["time"], "files": set()})
+            c["time"] = max(c["time"], info["time"])
+            c["files"].add(path)
+    if not candidates:
+        return None
+    commit = max(candidates, key=lambda c: (candidates[c]["time"], c))
+    when = dt.datetime.fromtimestamp(candidates[commit]["time"], dt.timezone.utc).date().isoformat()
+    return {"commit": commit, "date": when, "files": sorted(candidates[commit]["files"])}
+
+
+_TRUE = {"true", "1", "yes", "buggy"}
+_COMMIT_COLUMNS = ("commit_id", "commit", "hash", "sha", "bug_commit", "bic")
+_PATH_COLUMNS = ("path", "file", "filepath", "file_path", "filename")
+
+
+def read_labels(path: str) -> dict:
+    """Independent bug-inducing labels: {commit: [paths] or None}. Three shapes: ApacheJIT's CSV
+    (a `commit_id` column and a `buggy` flag; only the buggy rows count), Defectors' file-level
+    rows (a commit column and a path column), or a bare list of hashes, one per line."""
+    out = {}
+    with open(path, encoding="utf-8", errors="replace", newline="") as fh:
+        rows = list(csv.reader(fh))
+    if not rows:
+        return out
+    head = [h.strip().lower() for h in rows[0]]
+    commit_col = next((i for i, h in enumerate(head) if h in _COMMIT_COLUMNS), None)
+    if commit_col is None:
+        for row in rows:
+            if row and _HASH.match(row[0].strip()):
+                out[row[0].strip()] = None
+        return out
+    buggy_col = next((i for i, h in enumerate(head) if h == "buggy"), None)
+    path_col = next((i for i, h in enumerate(head) if h in _PATH_COLUMNS), None)
+    for row in rows[1:]:
+        if len(row) <= commit_col or not row[commit_col].strip():
+            continue
+        if buggy_col is not None and (len(row) <= buggy_col or row[buggy_col].strip().lower() not in _TRUE):
+            continue
+        commit = row[commit_col].strip()
+        if path_col is not None and len(row) > path_col and row[path_col].strip():
+            out[commit] = (out.get(commit) or []) + [row[path_col].strip()]
+        else:
+            out.setdefault(commit, None)
+    return out

{gitmole-0.12.0 → gitmole-0.14.0}/gitmole/textfmt.py

@@ -100,3 +100,8 @@ def tally(findings: list) -> str:
     if counts["info"]:
         parts.append(f"{counts['info']} note" + ("s" if counts["info"] != 1 else ""))
     return ", ".join(parts) if parts else "nothing flagged"
+
+
+def count(n: int, word: str, plural: str = None) -> str:
+    """'1 file', '3 files', '2 directories': a number with its noun."""
+    return f"{n:,} {word if n == 1 else (plural or word + 's')}"