gitmole 0.3.0__py3-none-any.whl

gitmole/findings.py

@@ -0,0 +1,405 @@
+"""Heuristics that turn a loaded report into a short list of flagged findings."""
+from __future__ import annotations
+
+import re
+
+from . import filetypes, hotspots, knowledge, leaks, loss, textfmt, trend
+
+SEVERITIES = ["critical", "warning", "info"]
+
+PLACEHOLDER_NAMES = {"your name", "unknown", "root", "user"}
+PLACEHOLDER_EMAIL = re.compile(r"(@example\.(com|org|net)$|^you@|^user@|^root@|@localhost$)")
+
+
+def _f(severity: str, title: str, statement: str, advice: str) -> dict:
+    """A finding: the facts, then the next step. `detail` is the two joined for anyone reading the
+    JSON; `advice` says which part is the step so the report can show it on its own line."""
+    return {"severity": severity, "title": title, "detail": f"{statement.rstrip()} {advice}", "advice": advice}
+
+
+def _pct(part, whole) -> str:
+    return f"{round(100 * part / whole)}%" if whole else "0%"
+
+
+def _plural(n: int, word: str) -> str:
+    return f"{n} {word}{'' if n == 1 else 's'}"
+
+
+def _secret_statement(groups: list) -> str:
+    """'N distinct values in M places: rule in file (commits), ...' with at most three values named."""
+    def one(g):
+        others = len(g["files"]) - 1
+        where = g["files"][0] + (f" and {_plural(others, 'other file')}" if others else "")
+        commits = ", ".join(g["commits"][:2]) + (f" and {len(g['commits']) - 2} more" if len(g["commits"]) > 2 else "")
+        return f"{g['rule']} in {where} ({commits})"
+    places = sum(g["places"] for g in groups)
+    sample = "; ".join(one(g) for g in groups[:3])
+    more = f" and {len(groups) - 3} more" if len(groups) > 3 else ""
+    return f"{_plural(len(groups), 'distinct value')} in {_plural(places, 'place')}: {sample}{more}."
+
+
+def secrets_found(report: dict) -> list:
+    """Secrets grouped by value. A value anywhere in source is critical; one that only ever appears in
+    test files (fixtures, saved pages) is a warning, so a critical gate does not trip on test data.
+    Version strings and shortened tokens were flagged as placeholders and are not a finding."""
+    groups = leaks.group(report.get("secrets") or [])
+    source = [g for g in groups if not g["test"]]
+    tests = [g for g in groups if g["test"]]
+    ignore = "Add the fingerprint of any false positive from secrets.json to .gitleaksignore in the repository."
+    out = []
+    if source:
+        out.append(_f("critical", f"{len(source)} secret(s) in history", _secret_statement(source),
+                      f"Rotate them; deleting the file does not remove them from git. {ignore}"))
+    if tests:
+        out.append(_f("warning", f"{len(tests)} secret(s) only in test files", _secret_statement(tests),
+                      f"Confirm they are fixtures, not live keys. {ignore}"))
+    return out
+
+
+def _all_identities(report: dict):
+    """Every identity row plus its aliases, flattened."""
+    for i in report["meta"].get("identities") or []:
+        yield i
+        for a in i.get("aliases") or []:
+            yield a
+
+
+def placeholder_identity(report: dict, min_share: float = 0.01) -> list:
+    """A placeholder name or mailbox with a real share of the commits. One stray commit in thousands
+    is not worth the panel space."""
+    total = sum(i["commits"] for i in report["meta"].get("identities") or [])
+    out = []
+    for i in _all_identities(report):
+        if total and i["commits"] / total < min_share:
+            continue
+        if i["name"].strip().lower() in PLACEHOLDER_NAMES or PLACEHOLDER_EMAIL.search(i["email"].lower()):
+            out.append(_f("warning", "Unconfigured git identity",
+                          f"\"{i['name']} <{i['email']}>\" made {i['commits']} commits ({_pct(i['commits'], total)}).",
+                          "Set user.name and user.email; consider a .mailmap for history."))
+    return out
+
+
+def _source_ownership(report: dict) -> list:
+    """Ownership rows for source files. Test files are left out of every rule that names a next
+    step: owning the tests is not the knowledge risk. The default tables leave them out too."""
+    return [r for r in report.get("ownership") or [] if not filetypes.is_test_path(r["entity"])]
+
+
+def bus_factor(report: dict, threshold: float = 0.7, min_lines: int = 200) -> list:
+    """One author owns most of the surviving code (whole history). The areas named in the advice
+    come from lines added, which `--since` windows, so the advice says so when it applies."""
+    shares = report.get("theseus_authors") or {}
+    total = sum(shares.values())
+    if not total:
+        return []
+    name, lines = max(shares.items(), key=lambda kv: kv[1])
+    if lines / total <= threshold:
+        return []
+    theirs = []
+    for a in knowledge.areas(_source_ownership(report)):
+        owned = dict(a["owners"]).get(name, 0)
+        if a["lines"] >= min_lines and owned / a["lines"] >= 0.8:
+            theirs.append((a["area"], round(100 * owned / a["lines"])))
+    if theirs:
+        areas = " and ".join(t[0] for t in theirs[:2])
+        shares_ = " and ".join(f"{t[1]}%" for t in theirs[:2])
+        since = report["meta"].get("since")
+        advice = (f"Pair someone with {name} on {areas} first; {'they are' if len(theirs) > 1 else 'it is'} {shares_} theirs"
+                  f"{f' since {since}' if since else ''}.")
+    else:
+        advice = f"Pair someone with {name} before they are unavailable."
+    return [_f("warning", "Bus factor of one", f"{name} wrote {_pct(lines, total)} of the code that survives today.", advice)]
+
+
+def _sizer_advice(row: dict) -> str:
+    """The remedy for one git-sizer row, keyed on the loader's "section: metric" name. Big blobs
+    want LFS, many refs want pruning, a wide tree wants splitting, a big checkout wants a sparse
+    checkout; everything else that grows is history, and a shallow clone is the answer to that."""
+    section, _, metric = row["name"].partition(": ")
+    if section == "Blobs" and metric in ("Maximum size", "Total size"):
+        return "Move large files to Git LFS or rewrite them out of history."
+    if section in ("References", "Annotated tags"):
+        return "Consider pruning old branches and tags."
+    if section == "Biggest checkouts":
+        return "Consider a sparse checkout for CI; the tree is the cost."
+    if section == "Trees" and metric == "Maximum entries":
+        where = row.get("ref") or "the widest directory"
+        return f"Split {where} into subdirectories; a directory that wide slows every checkout and diff."
+    if section == "Commits" and metric in ("Maximum size", "Maximum parents"):
+        return "Look at that commit; oversized commits are usually imports or octopus merges."
+    return "Consider a shallow clone for CI; the history is the cost."
+
+
+def sizer_concerns(report: dict) -> list:
+    out = []
+    for row in report.get("sizer") or []:
+        sev = "warning" if row["concern"] >= 2 else "info"
+        where = f" at {row['ref']}" if row.get("ref") else ""
+        out.append(_f(sev, "Repo health", f"{row['name']} is {row['value']}{where}. git-sizer level of concern {row['concern']}.", _sizer_advice(row)))
+    return out
+
+
+def hotspot_dominance(report: dict, ratio: float = 2.0, minimum: int = 20) -> list:
+    """One source file takes most of the churn. Test files are left out: they change with everything."""
+    revs = sorted((r for r in report.get("revisions") or [] if not filetypes.is_test_path(r["entity"])), key=lambda r: -r["n-revs"])
+    if len(revs) < 2 or revs[0]["n-revs"] < minimum or revs[0]["n-revs"] < ratio * revs[1]["n-revs"]:
+        return []
+    top, nxt = revs[0], revs[1]
+    return [_f("info", "One file dominates the churn",
+               f"{top['entity']} changed {top['n-revs']} times, versus {nxt['n-revs']} for the next file ({nxt['entity']}).",
+               f"Consider splitting {top['entity']}; every change lands there.")]
+
+
+def tight_coupling(report: dict, min_degree: int = 80, min_revs: int = 5) -> list:
+    """A file and its test are expected to change together, so pairs with a test file on either side are left out."""
+    pairs = [p for p in report.get("coupling") or [] if p["degree"] >= min_degree and p["average-revs"] >= min_revs
+             and not (filetypes.is_test_path(p["entity"]) or filetypes.is_test_path(p["coupled"]))]
+    if not pairs:
+        return []
+    pairs.sort(key=lambda p: (-p["degree"], -p["average-revs"]))
+    top = "; ".join(f"{p['entity']} + {p['coupled']} ({p['degree']}%)" for p in pairs[:3])
+    count = f"{len(pairs)} pair changes" if len(pairs) == 1 else f"{len(pairs)} pairs change"
+    first = pairs[0]
+    return [_f("info", "Files that always change together",
+               f"{count} together at least {min_degree}% of the time, e.g. {top}.",
+               f"Review {first['entity']} and {first['coupled']} first: a shared layout or a hidden dependency links them.")]
+
+
+def stale_files(report: dict, months: int = 12, share: float = 0.3) -> list:
+    """Files still in the tree that nobody has touched. The age table covers every path in the
+    history, so paths that were deleted are left out here; they are not dead code, they are gone."""
+    age = report.get("age") or []
+    tree = (report.get("size") or {}).get("files") or {}
+    if tree:
+        age = [a for a in age if a["entity"] in tree]
+    if not age:
+        return []
+    stale = [a for a in age if a["age-months"] >= months]
+    if len(stale) / len(age) <= share:
+        return []
+    return [_f("info", "A large share of files is untouched",
+               f"{_pct(len(stale), len(age))} of files ({len(stale)}) have not changed in {months} months or more.",
+               "Consider deleting what nobody has needed; dead code hides in untouched files.")]
+
+
+def bug_magnets(report: dict, min_recent: int = 3, warn_at: int = 5) -> list:
+    """Source files with a run of recent fix commits. Test files are left out: they change with every fix."""
+    hot = [f for f in report.get("fixes") or [] if f["recent-fixes"] >= min_recent and not filetypes.is_test_path(f["entity"])]
+    if not hot:
+        return []
+    hot.sort(key=lambda f: (-f["recent-fixes"], -f["n-fixes"], f["entity"]))
+    sev = "warning" if hot[0]["recent-fixes"] >= warn_at else "info"
+    listed = "; ".join(f"{f['entity']} ({f['recent-fixes']} recent, {f['n-fixes']} total)" for f in hot[:5])
+    more = f" and {len(hot) - 5} more" if len(hot) > 5 else ""
+    first = " and ".join(f["entity"] for f in hot[:2])
+    return [_f(sev, "Bug magnets",
+               f"{len(hot)} file(s) were fixed {min_recent}+ times in the last six months: {listed}{more}.",
+               f"Review {first} before the next release; expect the next bug there.")]
+
+
+def reverts(report: dict, min_share: float = 0.05, min_count: int = 5, warn_share: float = 0.10) -> list:
+    """Commits backed out with git revert. The file most often reverted is where a check before merge pays."""
+    act = report.get("activity") or {}
+    n = act.get("revert_commits") or 0
+    total = report["meta"].get("commits") or 0
+    if not n or not total or (n < min_count and n / total < min_share):
+        return []
+    sev = "warning" if total and n / total >= warn_share else "info"
+    reverted = act.get("reverted") or {}
+    # source files lead: a test file at the top of the table would otherwise be the one named first
+    items = sorted(reverted.items(), key=lambda kv: filetypes.is_test_path(kv[0]))[:3]
+    parts = []
+    for i, (p, c) in enumerate(items):
+        if i == 0:
+            parts.append(f"{p} was reverted {textfmt.times(c)}")
+        else:
+            parts.append(f"{p} {textfmt.times(c)}")
+    listed = ", ".join(parts)
+    statement = f"{n} of {total} commits are reverts" + (f"; {listed}." if listed else ".")
+    source = [p for p in reverted if not filetypes.is_test_path(p)]
+    if source:
+        advice = f"Add a check before merge for {source[0]}; it is the file most often backed out."
+    else:
+        advice = "Look at why they were backed out; only test files were touched."
+    return [_f(sev, "Reverts", statement, advice)]
+
+
+def knowledge_islands(report: dict, min_lines: int = 200, min_share: float = 0.9) -> list:
+    areas = knowledge.areas(_source_ownership(report))
+    islands = knowledge.islands(areas, min_lines=min_lines, min_share=min_share)
+    if not islands:
+        return []
+    total = sum(a["lines"] for a in areas)
+    covered = sum(i["lines"] for i in islands)
+    sev = "warning" if total and covered / total > 0.5 else "info"
+    listed = "; ".join(f"{i['area']} ({i['owner']} {i['share']}%)" for i in islands[:5])
+    more = f" and {len(islands) - 5} more" if len(islands) > 5 else ""
+    largest = max(islands, key=lambda i: i["lines"])
+    return [_f(sev, "Knowledge islands",
+               f"{len(islands)} area(s) with at least {min_lines} lines were written almost entirely by one person: {listed}{more}. "
+               f"That is {_pct(covered, total)} of all lines added.",
+               f"Pair someone with {largest['owner']} on {largest['area']} first; it is the largest at {largest['lines']:,} lines.")]
+
+
+LIVE_MONTHS = 12
+
+
+def _is_live(area: str, age_rows: list) -> bool:
+    """Has anything in this area changed in the last year? `age` covers every path in the history,
+    so an area whose files are all idle is knowledge about code nobody is touching."""
+    for row in age_rows:
+        if row["age-months"] >= LIVE_MONTHS:
+            continue
+        entity = row["entity"]
+        in_area = "/" not in entity if area == knowledge.ROOT else entity.startswith(area)
+        if in_area:
+            return True
+    return False
+
+
+def _loss_totals(report: dict, names: set, source_rows: list) -> tuple[int, int, dict, str]:
+    """(lost, total, by_person, basis): share of surviving code from the blame pass; when that did
+    not run, share of lines added instead, with the basis clause that says so."""
+    lost, total = loss.surviving(report, names)
+    by_person = {n: v for n, v in (report.get("theseus_authors") or {}).items() if n in names}
+    basis = "of the code that survives today"
+    if not total:
+        areas_all = loss.areas(source_rows, names)
+        total = sum(a["lines"] for a in areas_all)
+        lost = sum(a["lost"] for a in areas_all)
+        by_person = {}
+        for r in (r for r in source_rows if r["author"] in names):
+            by_person[r["author"]] = by_person.get(r["author"], 0) + r["added"]
+        basis = "of all lines added (from lines added, not a blame)"
+    return lost, total, by_person, basis
+
+
+def _loss_people(by_person: dict, total: int) -> str:
+    """The "Bob (25%), Cat (2%) and 3 others (1%)" clause, or "N people at under 1% each" when
+    nobody's individual share rounds to 1% or more."""
+    people = sorted(by_person.items(), key=lambda kv: (-kv[1], kv[0]))
+    named = [(n, v) for n, v in people if round(100 * v / total) >= 1][:3]
+    if named:
+        named_names = {n for n, _ in named}
+        rest = [(n, v) for n, v in people if n not in named_names]
+        listed = ", ".join(f"{n} ({_pct(v, total)})" for n, v in named)
+        if rest:
+            listed += f" and {_plural(len(rest), 'other')} ({_pct(sum(v for _, v in rest), total)})"
+    else:
+        listed = f"{len(people)} {'person' if len(people) == 1 else 'people'} at under 1% each"
+    return listed
+
+
+def _loss_areas(report: dict, names: set, source_rows: list) -> list:
+    """Areas at 200+ lines where 80%+ of the surviving code is theirs, tagged live or not and
+    sorted live-first: that is where the gap bites soonest."""
+    theirs = [a for a in loss.areas(source_rows, names) if a["lines"] >= 200 and a["lost_share"] >= 0.8]
+    for a in theirs:
+        a["live"] = _is_live(a["area"], report.get("age") or [])
+    theirs.sort(key=lambda a: (not a["live"], -a["lines"], a["area"]))   # a live area first: that is where the gap bites
+    return theirs
+
+
+def knowledge_loss(report: dict, min_share: float = 0.10, warn_share: float = 0.30) -> list:
+    """Code written by people who have stopped committing. Share of surviving code from the blame
+    pass; when that did not run, share of lines added, and the statement says so."""
+    months = report["meta"].get("gone_months", loss.DEFAULT_MONTHS)
+    gone = loss.gone(report, months)
+    if not gone:
+        return []
+    names = {g["name"] for g in gone}
+    source_rows = _source_ownership(report)
+    lost, total, by_person, basis = _loss_totals(report, names, source_rows)
+    if not total or lost / total < min_share:
+        return []
+    sev = "warning" if lost / total >= warn_share else "info"
+    listed = _loss_people(by_person, total)
+    theirs = _loss_areas(report, names, source_rows)
+    statement = (f"People with no commits since {loss.cutoff(report, months)} "
+                 f"wrote {_pct(lost, total)} {basis}: {listed}.")
+    if theirs:
+        listed_areas = ", ".join(f"{a['area']} ({round(100 * a['lost_share'])}%)" for a in theirs[:3])
+        more = f" and {len(theirs) - 3} more" if len(theirs) > 3 else ""
+        statement += f" Areas mostly theirs: {listed_areas}{more}."
+    if theirs and theirs[0]["live"]:
+        advice = f"Pair someone on {theirs[0]['area']} first; nobody who wrote it is around to ask."
+    else:   # nothing there has been touched in a year: pairing on it would be work nobody has asked for
+        top = min(by_person, key=lambda n: (-by_person[n], n))
+        advice = f"Pair someone with the people who worked with {top} before the rest of that knowledge goes."
+    return [_f(sev, "Knowledge loss", statement, advice)]
+
+
+def _partial_functions(report: dict) -> str:
+    """A sentence when the lizard step stopped part way, so what it measured is not the whole code."""
+    status = (report["meta"].get("functions") or {}).get("status")
+    reason = {"timeout": "timed out", "failed": "failed"}.get(status)
+    return f" Function metrics {reason} part way, so there may be more." if reason else ""
+
+
+def brain_methods(report: dict, min_ccn: int = 15, min_lines: int = 100) -> list:
+    """Functions that are both long and complex, in source files. A warning when one sits in a hotspot."""
+    big = [f for f in report.get("functions") or [] if f["ccn"] >= min_ccn and f["nloc"] >= min_lines and not filetypes.is_test_path(f["file"])]
+    if not big:
+        return []
+    big.sort(key=lambda f: (-f["ccn"], -f["nloc"], f["file"], f["function"], f["start"]))
+    hot = hotspots.top(report)
+    sev = "warning" if any(f["file"] in hot for f in big) else "info"
+    listed = "; ".join(f"{f['function']} ({f['file']}) complexity {f['ccn']}, {f['nloc']} lines, {f['params']} params" for f in big[:5])
+    more = f" and {len(big) - 5} more" if len(big) > 5 else ""
+    return [_f(sev, "Brain methods",
+               f"{len(big)} function(s) are both long and complex: {listed}{more}.{_partial_functions(report)}",
+               f"Split {big[0]['function']} in {big[0]['file']} first, before the next change lands there.")]
+
+
+def complexity_growth(report: dict, min_growers: int = 3, min_pct: int = 25, top_n: int = 10) -> list:
+    """The top_n source hotspots whose complexity grew over the last year, from the trend samples.
+    Test files are left out: a growing test file is not the problem the finding is about."""
+    series = (report.get("trend") or {}).get("files") or {}
+    last = report["meta"].get("last_date") or ""
+    if not series or not last:
+        return []
+    top = [h["entity"] for h in hotspots.ranked(report)
+           if h["code"] is not None and not filetypes.is_test_path(h["entity"])][:top_n]
+    grown = []
+    for path in top:
+        change = trend.change_over_year(series.get(path) or [], last)
+        if change.startswith("+") and int(change[1:-1]) >= min_pct:
+            grown.append((path, int(change[1:-1])))
+    if len(grown) < min_growers:
+        return []
+    sev = "warning" if top and grown[0][0] == top[0] else "info"
+    listed = ", ".join(f"{p} (+{g}%)" for p, g in grown[:5]) + (f" and {len(grown) - 5} more" if len(grown) > 5 else "")
+    first = grown[0]
+    return [_f(sev, "Hotspots getting more complex",
+               f"{len(grown)} of the {len(top)} top source hotspots grew by {min_pct}% or more in a year: {listed}.",
+               f"Split {first[0]} before the next change; its complexity grew {first[1]}% in a year.")]
+
+
+def duplication(report: dict, min_lines: int = 30) -> list:
+    dup = report.get("duplicates") or {}
+    blocks = [b for b in dup.get("blocks") or [] if b["lines"] >= min_lines]
+    if not blocks:
+        return []
+    blocks.sort(key=lambda b: (-b["lines"], b["places"]))
+    def place(b):
+        return " and ".join(f"{p}:{start}" for p, start, _ in b["places"][:3])
+    listed = "; ".join(f"{b['lines']} lines in {place(b)}" for b in blocks[:3])
+    more = f" and {len(blocks) - 3} more" if len(blocks) > 3 else ""
+    rate = f" Overall {dup['rate']}% of lines are duplicated." if dup.get("rate") is not None else ""
+    first = blocks[0]
+    files = list(dict.fromkeys(p for p, _, _ in first["places"]))   # each file once, in place order
+    where = f"repeated within {files[0]}" if len(files) == 1 else f"shared by {files[0]} and {files[1]}"
+    return [_f("info", "Duplicated code", f"{len(blocks)} block(s) of {min_lines}+ duplicated lines: {listed}{more}.{rate}{_partial_functions(report)}",
+               f"Extract the {first['lines']}-line block {where} first.")]
+
+
+RULES = [secrets_found, placeholder_identity, bus_factor, sizer_concerns, hotspot_dominance, bug_magnets, reverts, brain_methods, complexity_growth,
+         tight_coupling, duplication, stale_files, knowledge_islands, knowledge_loss]
+
+
+def evaluate(report: dict) -> list:
+    found = []
+    for rule in RULES:
+        found.extend(rule(report))
+    found.sort(key=lambda f: SEVERITIES.index(f["severity"]))
+    return found

gitmole/functions.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""Function-level metrics and duplicated blocks from lizard, over the tracked code files only.
+
+Runs as its own process (a pipeline step) and drives lizard through its Python API rather
+than its command line: the file list never touches a shell or a list file, the analysed
+repository is never on sys.path, only files lizard has a reader for are measured, and the
+CSV is streamed so a killed step still leaves what was measured."""
+from __future__ import annotations
+
+import argparse
+import csv
+import os
+import sys
+
+import lizard
+from lizard_ext.lizardduplicate import LizardExtension as Duplicates
+
+try:
+    from . import blame, filetypes
+except ImportError:  # run as a script: the package directory is sys.path[0]
+    import blame
+    import filetypes
+
+
+def select_files(repo: str, ignore=(), types_spec: str = None) -> list:
+    """Tracked text files lizard can parse. Without --file-types that is every language lizard
+    knows (a superset of gitmole's default code list, e.g. Fortran); with it, the intersection."""
+    types = filetypes.parse(types_spec)
+    files = blame.text_files(repo, ignore) if types_spec is None else blame.code_files(repo, ignore, types)
+    return [f for f in files if lizard.get_reader_for(f) is not None]
+
+
+def csv_row(info, fn) -> list:
+    """The columns `lizard --csv` prints, so the loader does not care which produced the file."""
+    return [fn.nloc, fn.cyclomatic_complexity, fn.token_count, fn.parameter_count, fn.length,
+            f"{fn.name}@{fn.start_line}-{fn.end_line}@{info.filename}", info.filename, fn.name, fn.long_name, fn.start_line, fn.end_line]
+
+
+def write_duplicates(dup: Duplicates, fh) -> None:
+    """The layout `lizard -Eduplicate` prints, snippets in a stable order."""
+    fh.write("Duplicates\n===================================\n")
+    for block in dup.get_duplicates():
+        fh.write("Duplicate block:\n--------------------------\n")
+        for s in sorted(block, key=lambda s: (s.file_name, s.start_line)):
+            fh.write(f"{s.file_name}:{s.start_line} ~ {s.end_line}\n")
+        fh.write("^^^^^^^^^^^^^^^^^^^^^^^^^^\n\n")
+    fh.write(f"Total duplicate rate: {(dup.duplicate_rate() or 0.0) * 100:.2f}%\n")
+    fh.write(f"Total unique rate: {(dup.unique_rate() or 0.0) * 100:.2f}%\n")
+
+
+def analyze(files: list, procs: int, exts: list):
+    """lizard.analyze_files, with one step between the per-file analysis and the cross-file
+    extensions: a file lizard could not read or finish (gone from disk, RecursionError) comes back
+    without the duplicate finder's hash_nodes, which would stop the whole pass at that file."""
+    def with_hash_nodes(infos):
+        for info in infos:
+            if not hasattr(info, "hash_nodes"):
+                info.hash_nodes = []
+            yield info
+    result = with_hash_nodes(lizard.map_files_to_analyzer(files, lizard.FileAnalyzer(exts), procs))
+    for ext in exts:
+        if hasattr(ext, "cross_file_process"):
+            result = ext.cross_file_process(result)
+    return result
+
+
+def measure(repo: str, files: list, out: str, procs: int, duplicates: bool = False) -> int:
+    """Stream functions.csv while lizard runs, then duplicates.txt when the finder was on. Returns 0,
+    or 1 when lizard gave up on a file (whatever was measured by then stays on disk)."""
+    exts = lizard.get_extensions(["duplicate"] if duplicates else [])   # lizard's metric extensions, plus the duplicate finder on request
+    dup = next((e for e in exts if isinstance(e, Duplicates)), None)
+    rc = 0
+    cwd = os.getcwd()
+    os.chdir(repo)   # lizard opens the paths as given; relative ones keep the CSV repo-relative
+    try:
+        with open(os.path.join(out, "functions.csv"), "w", encoding="utf-8", newline="") as fh:
+            writer = csv.writer(fh, quoting=csv.QUOTE_NONNUMERIC)
+            try:
+                for info in analyze(files, procs, exts):
+                    for fn in info.function_list:
+                        writer.writerow(csv_row(info, fn))
+                    fh.flush()
+            except Exception as e:  # lizard re-raises its parse failures; keep what we have
+                print(f"lizard stopped: {e!r}", file=sys.stderr)
+                rc = 1
+        if dup is not None:
+            with open(os.path.join(out, "duplicates.txt"), "w", encoding="utf-8") as fh:
+                write_duplicates(dup, fh)
+    finally:
+        os.chdir(cwd)
+    return rc
+
+
+def main(argv=None) -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("repo")
+    p.add_argument("out")
+    p.add_argument("--procs", type=int, default=1)
+    p.add_argument("--ignore", action="append", default=[])
+    p.add_argument("--types", default=None, help="file types spec as for gitmole --file-types")
+    p.add_argument("--duplicates", action="store_true", help="also run the duplicate finder (slow and memory-hungry on a large repo)")
+    args = p.parse_args(argv)
+    files = select_files(args.repo, args.ignore, args.types)
+    return measure(os.path.abspath(args.repo), files, os.path.abspath(args.out), max(1, args.procs), args.duplicates)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

gitmole/hotspots.py

@@ -0,0 +1,18 @@
+"""One ranking of hotspots, shared by the table and the findings that talk about them."""
+from __future__ import annotations
+
+
+def ranked(report: dict) -> list:
+    """Files by revisions × lines of code, Tornhill-style. Files no longer in the tree score -1 and sort last."""
+    files = (report.get("size") or {}).get("files") or {}
+    out = []
+    for r in report.get("revisions") or []:
+        info = files.get(r["entity"])
+        out.append({"entity": r["entity"], "revs": r["n-revs"], "code": info["code"] if info else None,
+                    "complexity": info["complexity"] if info else None, "score": r["n-revs"] * info["code"] if info else -1})
+    out.sort(key=lambda h: (-h["score"], -h["revs"], h["entity"]))
+    return out
+
+
+def top(report: dict, n: int = 10) -> set:
+    return {h["entity"] for h in ranked(report)[:n]}

gitmole/identity.py

@@ -0,0 +1,66 @@
+"""Merge git identities that belong to the same person."""
+from __future__ import annotations
+
+import re
+
+
+_BOT_WORDS = ("dependabot", "renovate", "github-actions", "github actions")
+
+
+def is_bot(name: str, email: str = "") -> bool:
+    """A commit author that is a service, not a person: GitHub's *[bot] suffix, or one of the
+    common automation names in the name or the mailbox."""
+    n, local = name.strip().lower(), email.strip().lower().split("@")[0]
+    if n.endswith("[bot]") or local.endswith("[bot]"):
+        return True
+    return any(w in n for w in _BOT_WORDS) or any(w in local for w in _BOT_WORDS) or email.strip().lower() == "actions@github.com"
+
+
+def _tokens(name: str) -> set:
+    return {t for t in re.split(r"[^a-z0-9]+", name.lower()) if len(t) >= 3}
+
+
+def same_person(a: dict, b: dict) -> bool:
+    return a["email"].lower() == b["email"].lower() or len(_tokens(a["name"]) & _tokens(b["name"])) >= 2
+
+
+def merge(identities: list) -> list:
+    """Group identities by shared name tokens or email. Each row keeps the most-committed
+    variant's name and email, sums the commits, and lists the other variants as aliases.
+
+    Grouping is transitive: an identity that matches two groups joins them into one, so
+    "Hayden <h@noreply>" and "hay-kot <h@pm.me>" end up together once "hay-kot <h@noreply>"
+    shows up to link them. Without that the same person appears twice."""
+    groups = []
+    for i in identities:
+        matched = [g for g in groups if any(same_person(i, j) for j in g)]
+        if not matched:
+            groups.append([i])
+            continue
+        first = matched[0]
+        first.append(i)
+        for other in matched[1:]:
+            first.extend(other)
+            groups.remove(other)
+    merged = []
+    for g in groups:
+        g = sorted(g, key=lambda x: -x["commits"])
+        head = g[0]
+        merged.append({
+            "name": head["name"],
+            "email": head["email"],
+            "commits": sum(x["commits"] for x in g),
+            "aliases": [{"name": x["name"], "email": x["email"], "commits": x["commits"]} for x in g[1:]],
+        })
+    merged.sort(key=lambda m: (-m["commits"], m["name"]))
+    return merged
+
+
+def canonical_names(merged: list) -> dict:
+    """alias name -> merged name, including the merged names themselves."""
+    out = {}
+    for m in merged:
+        out[m["name"]] = m["name"]
+        for a in m.get("aliases", []):
+            out[a["name"]] = m["name"]
+    return out

gitmole/knowledge.py

@@ -0,0 +1,55 @@
+"""Who knows which part of the tree: ownership aggregated by directory."""
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+
+
+ROOT = "(root files)"
+
+
+def _area(entity: str, depth: int) -> str:
+    dirs = entity.split("/")[:-1]
+    if not dirs:
+        return ROOT
+    return "/".join(dirs[:depth]) + "/"
+
+
+def _aggregate(rows: list, depth: int) -> list:
+    lines, per_author = Counter(), defaultdict(Counter)
+    for r in rows:
+        a = _area(r["entity"], depth)
+        lines[a] += r["added"]
+        per_author[a][r["author"]] += r["added"]
+    out = []
+    for a, n in lines.items():
+        owners = sorted(per_author[a].items(), key=lambda kv: (-kv[1], kv[0]))
+        out.append({"area": a, "lines": n, "authors": len(owners), "owners": owners})
+    out.sort(key=lambda x: (-x["lines"], x["area"]))
+    return out
+
+
+def areas(ownership_rows: list, dominant: float = 0.8) -> list:
+    """Areas of the tree by lines added, with per-author ownership.
+
+    Top-level directories, unless one of them holds `dominant` of all lines
+    (a lone src/ or the like), in which case its subdirectories are used."""
+    rows = [r for r in ownership_rows if r.get("added", 0) > 0]
+    if not rows:
+        return []
+    top = _aggregate(rows, 1)
+    total = sum(a["lines"] for a in top)
+    if top[0]["area"] != ROOT and top[0]["lines"] >= dominant * total:
+        return _aggregate(rows, 2)
+    return top
+
+
+def islands(areas_list: list, min_lines: int = 200, min_share: float = 0.9) -> list:
+    """Areas of at least `min_lines` where one author wrote at least `min_share` of them."""
+    out = []
+    for a in areas_list:
+        if a["lines"] < min_lines or not a["owners"]:
+            continue
+        owner, n = a["owners"][0]
+        if n / a["lines"] >= min_share:
+            out.append({"area": a["area"], "owner": owner, "share": round(100 * n / a["lines"]), "lines": a["lines"]})
+    return out