gitmole 0.20.0__tar.gz → 0.22.0__tar.gz

{gitmole-0.20.0 → gitmole-0.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitmole
-Version: 0.20.0
+Version: 0.22.0
 Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
 License: MIT
 Project-URL: Homepage, https://github.com/antvinni/gitmole
@@ -74,6 +74,7 @@ gitmole . --json report.json           # every table, the watch list and the fin
 gitmole . --fail-on warning            # exit 3 if any finding is a warning or worse
 gitmole . --risk main --risk-threshold 10  # exit 3 if the files changed since main hold over 10% of the risk
 gitmole . --sarif gitmole.sarif        # the findings for GitHub code scanning or GitLab
+gitmole . --sbom sbom.cdx.json         # a CycloneDX SBOM of every package the lock files pin
 gitmole . --compare last.json          # what changed since an earlier --json export
 gitmole analysis-repo --no-run --hook  # a coding agent's edit hook: history's view of the files it just touched
 gitmole . --since 2y --full            # the current team, every row and column
@@ -92,9 +93,9 @@ Reports on repositories you know, each at a pinned commit, published as gitmole
 
 | Repository | Commit | Commits | Lines | gitmole run |
 |---|---|---:|---:|---:|
-| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 57 s |
-| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 132 s |
-| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 151 s |
+| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 59 s |
+| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 135 s |
+| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 157 s |
 
 Run times are one `gitmole CLONE` with every default step, on a MacBook Pro (M4, 16 GB).
 

{gitmole-0.20.0 → gitmole-0.22.0}/README.md

@@ -41,6 +41,7 @@ gitmole . --json report.json           # every table, the watch list and the fin
 gitmole . --fail-on warning            # exit 3 if any finding is a warning or worse
 gitmole . --risk main --risk-threshold 10  # exit 3 if the files changed since main hold over 10% of the risk
 gitmole . --sarif gitmole.sarif        # the findings for GitHub code scanning or GitLab
+gitmole . --sbom sbom.cdx.json         # a CycloneDX SBOM of every package the lock files pin
 gitmole . --compare last.json          # what changed since an earlier --json export
 gitmole analysis-repo --no-run --hook  # a coding agent's edit hook: history's view of the files it just touched
 gitmole . --since 2y --full            # the current team, every row and column
@@ -59,9 +60,9 @@ Reports on repositories you know, each at a pinned commit, published as gitmole
 
 | Repository | Commit | Commits | Lines | gitmole run |
 |---|---|---:|---:|---:|
-| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 57 s |
-| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 132 s |
-| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 151 s |
+| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 59 s |
+| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 135 s |
+| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 157 s |
 
 Run times are one `gitmole CLONE` with every default step, on a MacBook Pro (M4, 16 GB).
 

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/__init__.py

@@ -1,3 +1,3 @@
 """gitmole: offline git repository analysis with a terminal report."""
 
-__version__ = "0.20.0"
+__version__ = "0.22.0"

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/evaluate.py

@@ -62,8 +62,14 @@ def labelled_between(commits: list, labels: dict, start: str, end: str) -> set:
     before `end` (ApacheJIT, Defectors: see szz.read_labels): the paths the label names, or every
     source file the commit touched. Either side may be abbreviated."""
     out = set()
+    short = [k for k in labels if len(k) < 40]   # abbreviated labels need a prefix match; full hashes are a lookup
     for c in maat.in_window(commits, start, end):
-        paths = next((v for k, v in labels.items() if k.startswith(c["hash"]) or c["hash"].startswith(k)), "none")
+        if c["hash"] in labels:
+            paths = labels[c["hash"]]
+        elif len(c["hash"]) < 40:   # an abbreviated hash in the log: scan for the label it abbreviates
+            paths = next((v for k, v in labels.items() if k.startswith(c["hash"])), "none")
+        else:
+            paths = next((labels[k] for k in short if c["hash"].startswith(k)), "none")
         if paths == "none":
             continue
         out.update(p for p in (paths if paths is not None else [p for p, _, _ in c["files"]]) if not filetypes.is_test_path(p))
@@ -151,6 +157,32 @@ def score(report: dict, fixed: set, top: int) -> dict:
     return out
 
 
+def effort(report: dict, outcome: set, top: int) -> dict:
+    """variant -> (IFA, lines): how many of its first `top` files come before the first one in the
+    outcome (initial false alarms; `top` when none is), and the lines of code those files hold at the
+    cut-off, the inspection budget. A list that ranks small files first can look good on hits per line
+    and still send a reviewer through many files before one matters, so both are shown."""
+    files = (report.get("size") or {}).get("files") or {}
+    out = {}
+    for name, ranked in variants(report).items():
+        head = ranked[:top]
+        ifa = next((i for i, f in enumerate(head) if f in outcome), len(head))
+        out[name] = (ifa, sum((files.get(f) or {}).get("code", 0) for f in head))
+    return out
+
+
+def effort_table(efforts: list) -> str:
+    """efforts: [{variant: (ifa, lines)}] per cut-off -> Markdown with the median of each over the cut-offs."""
+    import statistics
+    names = list(efforts[0]) if efforts else []
+    rows = ["| variant | IFA, median | lines of code in the list, median |", "|---|---:|---:|"]
+    for name in names:
+        ifas = [e[name][0] for e in efforts]
+        lines = [e[name][1] for e in efforts]
+        rows.append(f"| {name} | {statistics.median(ifas):g} | {int(statistics.median(lines)):,} |")
+    return "\n".join(rows)
+
+
 def table(results: list, noun: str = "fixed") -> str:
     """results: [(t, files fixed that were in the pool, pool size, {variant: hits})], oldest first -> Markdown.
     `noun` says what the outcome is: fixed, bug-inducing (R-SZZ) or labelled."""
@@ -184,6 +216,7 @@ def main(argv=None) -> int:
     p.add_argument("--top", type=int, default=watch.WATCH_TOP)
     p.add_argument("--szz", action="store_true", help="also score against R-SZZ bug-inducing commits (one git blame per fix and file: minutes)")
     p.add_argument("--labels", metavar="CSV", help="also score against independent bug-inducing labels (ApacheJIT's CSV, Defectors' file rows, or one hash per line)")
+    p.add_argument("--end", metavar="YYYY-MM-DD", help="count the cut-offs back from this date rather than the last commit: a labelled dataset that stops earlier (ApacheJIT ends in 2019)")
     args = p.parse_args(argv)
     meta = load._read_json(args.out, "meta.json", {})
     log_path = os.path.join(args.out, "log.txt")
@@ -201,8 +234,8 @@ def main(argv=None) -> int:
     if args.labels and not labels:
         print(f"evaluate: no bug-inducing commits read from {args.labels}", file=sys.stderr)
         return 2
-    results, induced_results, labelled_results = [], [], []
-    for t in cutoffs(meta["last_date"], args.windows, args.horizon):
+    results, induced_results, labelled_results, labelled_effort = [], [], [], []
+    for t in cutoffs(args.end or meta["last_date"], args.windows, args.horizon):
         rev = trend.rev_before(args.repo, t, end_of_day=False)
         if not rev:
             continue                      # the history does not reach back this far
@@ -220,6 +253,7 @@ def main(argv=None) -> int:
         if labels:
             marked = labelled_between(commits, labels, t, end)
             labelled_results.append((t, len(marked & pool), len(pool), score(report, marked, args.top)))
+            labelled_effort.append(effort(report, marked, args.top))
         print(f"evaluate: {t} done", file=sys.stderr)
     if not results:
         print("evaluate: no cut-off falls inside the history", file=sys.stderr)
@@ -233,6 +267,8 @@ def main(argv=None) -> int:
     if labelled_results:
         print(f"\nAgainst the files the bug-inducing commits labelled in {os.path.basename(args.labels)} touched inside each window:\n")
         print(table(labelled_results, noun="labelled"))
+        print(f"\nWhat each list costs a reviewer against the same labels: initial false alarms before the first labelled file, and the lines of code in its top {args.top}:\n")
+        print(effort_table(labelled_effort))
     print(f"\n`--all` exports {spread['all']:,} commits ({spread['fix_all']:,} fixes); HEAD reaches {spread['head']:,} ({spread['fix_head']:,} fixes).")
     return 0
 

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/findings.py

@@ -1009,6 +1009,78 @@ def debt_in_hotspots(report: dict, min_markers: int = 3, min_files: int = 2, top
                evidence={"files": [{"file": f, "markers": n} for f, n in flagged[:10]]})]
 
 
+def _shape_files(report: dict, key: str) -> list:
+    """[(file, shapes)] for this repository's own source files holding a shape: tests, examples,
+    documentation, vendored and generated files left out, since a fixture or a sample may do on purpose
+    what the source should not."""
+    s = _structure(report)
+    if not s:
+        return []
+    generated, vendored = _generated(report), filetypes.vendor_dirs(report)
+    return [(p, v["shapes"]) for p, v in sorted((s.get("files") or {}).items()) if (v.get("shapes") or {}).get(key)
+            and not (filetypes.is_test_path(p) or filetypes.is_sample_path(p) or filetypes.is_doc_path(p)
+                     or filetypes.is_vendored(p, vendored) or p in generated)]
+
+
+def swallowed_errors(report: dict, min_count: int = 5, top_n: int = 10) -> list:
+    """Catch, except and rescue blocks that do nothing and say nothing: no statement, no comment. In
+    Python only a bare `except:` or one catching Exception or BaseException counts, since `except
+    KeyError: pass` is the language's idiom. A warning when one sits in a top hotspot, where an error
+    that vanishes is the hardest to trace."""
+    rows = _shape_files(report, "empty_catch")
+    total = sum(sh.get("empty_catch_count", len(sh["empty_catch"])) for _, sh in rows)
+    if total < min_count:
+        return []
+    rows.sort(key=lambda r: (-r[1].get("empty_catch_count", 0), r[0]))
+    top = set(_scored_top(report, top_n))
+    hot = [p for p, _ in rows if p in top]
+    listed = _files_list([f"{p}:{sh['empty_catch'][0]}" + (f" and {sh['empty_catch_count'] - 1} more there" if sh.get("empty_catch_count", 1) > 1 else "") for p, sh in rows])
+    bare = sum(sh.get("bare_except_count", 0) for _, sh in rows)
+    first = hot[0] if hot else rows[0][0]
+    return [_f("warning" if hot else "info", "Errors caught and dropped",
+               f"{_plural(total, 'empty catch block')} in {_plural(len(rows), 'source file')}"
+               + (f", {bare} of them a bare except" if bare else "") + f": {listed}."
+               + (f" {textfmt.join_and(hot[:3])} {'is a top hotspot' if len(hot) == 1 else 'are top hotspots'}." if hot else ""),
+               f"Log or rethrow in {first} first, or say in a comment why the error is ignored; an error dropped without a trace is the hardest kind to find.",
+               rule={"id": "swallowed_errors", "min_count": min_count, "measure": "tree-sitter", "python": "bare, Exception or BaseException only"},
+               evidence={"count": total, "bare_except": bare, "hotspots": hot[:10],
+                         "files": [{"file": p, "start": sh["empty_catch"][0], "count": sh.get("empty_catch_count", 1)} for p, sh in rows[:10]]})]
+
+
+def hardcoded_addresses(report: dict) -> list:
+    """IPv4 addresses written into string literals in source files: a host that moves, or an environment
+    wired into the code. Loopback, unspecified, broadcast, netmask-shaped, documentation-range (RFC 5737)
+    and object-identifier-shaped values are not counted (structure.py)."""
+    rows = _shape_files(report, "addresses")
+    if not rows:
+        return []
+    total = sum(sh.get("addresses_count", len(sh["addresses"])) for _, sh in rows)
+    places = [(p, a) for p, sh in rows for a in sh["addresses"]]
+    listed = _files_list([f"{a['value']} at {p}:{a['line']}" for p, a in places])
+    return [_f("info", "Addresses written into the code",
+               f"{_plural(total, 'IPv4 address')} in string literals in {_plural(len(rows), 'source file')}: {listed}.",
+               f"Move {places[0][1]['value']} in {places[0][0]} into configuration, or a name that DNS resolves; an address in code has to be edited and shipped to change.",
+               rule={"id": "hardcoded_addresses", "measure": "tree-sitter", "left_out": "loopback, 0.0.0.0, broadcast, RFC 5737, x.x.x.0, first octet 0-2"},
+               evidence={"count": total, "files": [{"file": p, "start": a["line"], "value": a["value"]} for p, a in places[:10]]})]
+
+
+def commented_out_code(report: dict, min_lines: int = 10) -> list:
+    """Source files with ten or more lines of code left in comments: blocks of line or block comments in
+    which four lines in five read as statements and one starts right at the comment marker. Worked
+    examples in prose and documentation comments are not counted. The history already keeps old code."""
+    rows = [(p, sh) for p, sh in _shape_files(report, "commented_code") if sh["commented_code"] >= min_lines]
+    if not rows:
+        return []
+    rows.sort(key=lambda r: (-r[1]["commented_code"], r[0]))
+    listed = _files_list([f"{p} ({sh['commented_code']} lines from line {sh['commented_sample'][0]})" for p, sh in rows])
+    first = rows[0]
+    return [_f("info", "Code left in comments",
+               f"{_plural(len(rows), 'source file')} {'holds' if len(rows) == 1 else 'hold'} {min_lines} or more lines of commented-out code: {listed}.",
+               f"Delete the block at {first[0]}:{first[1]['commented_sample'][0]}; git keeps the old version, and a reader cannot tell whether it is meant to come back.",
+               rule={"id": "commented_out_code", "min_lines": min_lines, "measure": "tree-sitter", "code_share": 0.8},
+               evidence={"files": [{"file": p, "start": sh["commented_sample"][0], "lines": sh["commented_code"]} for p, sh in rows[:10]]})]
+
+
 def deep_nesting(report: dict, min_nesting: int = 5, min_bumps: int = 3, top_n: int = 10) -> list:
     """Functions nested five levels or more, or with three or more separate chunks of nested logic (a
     bumpy road), in this repository's own source: CodeScene's nesting and bumpy-road factors, measured
@@ -1270,7 +1342,7 @@ RULES = [dormant, secrets_found, credential_files, vulnerable_dependencies, plac
          minor_contributors, reverts, brain_methods, complexity_growth, tight_coupling, duplication, stale_files, knowledge_islands, knowledge_loss,
          sweeping_commits, tangled_commits, hygiene_findings, debt_in_hotspots, deep_nesting, hidden_coupling, unreferenced_files,
          agent_approval_disabled, agent_local_settings, mcp_literal_env, agent_instructions_drift, signoff_by_co_author,
-         truck_factor, authors_gone, component_coupling]
+         truck_factor, authors_gone, component_coupling, swallowed_errors, hardcoded_addresses, commented_out_code]
 
 
 def evaluate(report: dict) -> list:

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/provenance.py

@@ -12,6 +12,12 @@ writes provenance.json:
   changed again by another commit within two weeks. This repository against itself, with the share
   of commits the cohort covers beside it; no prior from elsewhere, since the best-controlled study
   found the spread between agents larger than the pooled difference.
+- lines: the lines added to code files in the last year and the year before, with the share git's
+  own moved-code detection marks as moved (`--color-moved`, blocks of twenty or more characters) and
+  the share deleted again within two weeks, in the same file with the same text: GitClear's moved and
+  churned lines, as a direction over this repository rather than a comparison with anyone else's.
+  The same two numbers per cohort, and each cohort's watch-list hit rate: the share of its commits
+  touching a file on the watch list's top fifteen.
 - shape: neutral descriptors (commits landing in bursts, conventional-commit subjects, how many hours
   of the day commits come in). Every one has a benign cause, and none is labelled.
 - agents: the agent instruction files by path convention (AGENTS.md, CLAUDE.md, GEMINI.md,
@@ -28,7 +34,7 @@ import os
 import re
 import subprocess
 import sys
-from collections import Counter
+from collections import Counter, deque
 
 try:
     from . import filetypes, leaks
@@ -44,6 +50,16 @@ _IDENT = re.compile(r"^\s*(?P<name>[^<]*?)\s*<(?P<email>[^>]*)>\s*$")
 _CONVENTIONAL = re.compile(r"^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([^)]*\))?!?: \S")
 BURST_SIZE, BURST_SECONDS = 5, 600
 RETOUCH_DAYS = 14
+YEAR = 365 * 86400
+WATCH_TOP = 15
+# data that inflates line counts without being code (the same set the code-age pass leaves out)
+DATA_EXCLUDES = [":(exclude,glob)**/*.json", ":(exclude,glob)**/*.lock", ":(exclude,glob)**/*.min.js", ":(exclude,glob)**/*.min.css",
+                 ":(exclude,glob)**/*.svg", ":(exclude,glob)**/*.map", ":(exclude,glob)**/*.csv", ":(exclude,glob)**/*.snap"]
+_COLORS = ["-c", "color.diff.new=green", "-c", "color.diff.newMoved=cyan", "-c", "color.diff.old=red", "-c", "color.diff.oldMoved=magenta",
+           "-c", "color.diff.meta=normal", "-c", "color.diff.frag=normal", "-c", "color.diff.func=normal", "-c", "color.diff.context=normal",
+           "-c", "color.diff.whitespace=normal", "-c", "color.diff.commit=normal"]
+_ADDED, _MOVED, _DELETED, _DELETED_MOVED = "\x1b[32m+", "\x1b[36m+", "\x1b[31m-", "\x1b[35m-"
+_ANSI = re.compile(r"\x1b\[[0-9;]*m")
 
 
 def read_commits(repo: str) -> list:
@@ -94,8 +110,8 @@ def trailers(commits: list) -> dict:
             "with_any": sum(1 for c in commits if c["trailers"]), "never_author": listing[:50], "signoff_by_co_author": signoff[:50]}
 
 
-def cohort(commits: list, inventory: dict) -> dict:
-    """The commits an `Assisted-by` trailer or a never-authoring co-author marks, against the rest."""
+def marker(inventory: dict):
+    """The predicate that marks a commit: an `Assisted-by` trailer, or a co-author who never authors."""
     marked_emails = {x["email"] for x in inventory["never_author"]}
 
     def marked(c):
@@ -106,6 +122,13 @@ def cohort(commits: list, inventory: dict) -> dict:
             if k.lower() == "co-authored-by" and ident and ident[1] in marked_emails:
                 return True
         return False
+    return marked
+
+
+def cohort(commits: list, inventory: dict, watch_files=None) -> dict:
+    """The commits an `Assisted-by` trailer or a never-authoring co-author marks, against the rest; with
+    `watch_files`, how many of each touched a file on the watch list."""
+    marked = marker(inventory)
 
     reverted = {c["subject"][len('Revert "'):-1] for c in commits if c["subject"].startswith('Revert "') and c["subject"].endswith('"')}
     by_file = {}
@@ -125,12 +148,107 @@ def cohort(commits: list, inventory: dict) -> dict:
                 again = True
                 break
         s["retouched"] += again
+        if watch_files:
+            s["watch"] += any(f in watch_files for f in c["files"])
     total = len(commits)
     return {"definition": "an Assisted-by trailer, or a co-author who never authors a commit here",
             "share": round(stats[True]["commits"] / total, 3) if total else 0.0,
             "cohort": dict(stats[True]) or {"commits": 0}, "rest": dict(stats[False]) or {"commits": 0}}
 
 
+def _code_path(path: str, generated: set, vendored) -> bool:
+    return filetypes.matches(path, filetypes.DEFAULT) and path not in generated and not filetypes.is_vendored(path, vendored)
+
+
+def lines(repo: str, end: int, marked_hashes: set, generated=frozenset(), vendored=()) -> dict:
+    """Added, moved and churned lines in code files over the two years before `end` (a timestamp), from
+    one `git log -p` with git's moved-code colouring. A line is churned when a later commit, within two
+    weeks, deletes a line with the same text from the same file; blank lines and lines without three
+    letters or digits (a lone brace) are not matched, since any brace would pair with any other."""
+    start = end - 2 * YEAR
+    argv = ["git", *_COLORS, "-c", "core.quotePath=false", "log", "HEAD", "--reverse", "--no-merges", "-p", "-U0", "-M", "--color=always",
+            "--color-moved=blocks", "--color-moved-ws=allow-indentation-change", f"--since=@{start}", f"--until=@{end}",
+            f"--format={END}%H{SEP}%at", "--", ".", *DATA_EXCLUDES]
+    proc = subprocess.Popen(argv, cwd=repo, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    windows = {w: Counter() for w in ("last", "before")}
+    groups = {True: Counter(), False: Counter()}
+    pending = {}   # (path, text) -> [(time, window, marked)] of additions not yet churned
+    order = deque()   # (time, key) in the order added, so what is past two weeks can be dropped and memory stays bounded
+    t, window, is_marked, path, keep = 0, "last", False, None, False
+    for raw in proc.stdout:
+        line = raw.decode("utf-8", "replace").rstrip("\n")
+        if line.startswith(END):
+            h, _, at = line[1:].partition(SEP)
+            t = int(at or 0)
+            window = "last" if t > end - YEAR else "before"
+            is_marked = h in marked_hashes
+            windows[window]["commits"] += 1
+            groups[is_marked]["commits"] += 1
+            path = None
+            while order and t - order[0][0] > RETOUCH_DAYS * 86400:
+                old_t, key = order.popleft()
+                adds = pending.get(key)
+                if adds and adds[0][0] == old_t:
+                    adds.pop(0)
+                if not adds:
+                    pending.pop(key, None)
+            continue
+        if line.startswith("diff --git "):
+            plain = _ANSI.sub("", line)   # git ends even an uncoloured header with a reset
+            path = filetypes.unquote(plain.rsplit(" b/", 1)[-1]) if " b/" in plain else None
+            keep = bool(path) and _code_path(path, generated, vendored)
+            continue
+        if not keep:
+            continue
+        if line.startswith((_ADDED, _MOVED)):
+            text = _ANSI.sub("", line)[1:].strip()
+            moved = line.startswith(_MOVED)
+            for c in (windows[window], groups[is_marked]):
+                c["added"] += 1
+                c["moved"] += moved
+            if sum(ch.isalnum() for ch in text) >= 3:
+                pending.setdefault((path, text), []).append((t, window, is_marked))
+                order.append((t, (path, text)))
+        elif line.startswith((_DELETED, _DELETED_MOVED)):
+            text = _ANSI.sub("", line)[1:].strip()
+            adds = pending.get((path, text))
+            while adds and t - adds[0][0] > RETOUCH_DAYS * 86400:
+                adds.pop(0)   # too old to count, and older than any later deletion will reach
+            if adds and adds[0][0] < t:
+                at_, w, m = adds.pop(0)
+                windows[w]["churned"] += 1
+                groups[m]["churned"] += 1
+    proc.stdout.close()
+    proc.wait()
+
+    def shares(c):
+        added = c.get("added", 0)
+        return {"commits": c.get("commits", 0), "added": added, "moved": c.get("moved", 0), "churned": c.get("churned", 0),
+                "moved_share": round(c.get("moved", 0) / added, 4) if added else None,
+                "churn_share": round(c.get("churned", 0) / added, 4) if added else None}
+    day = lambda ts: dt.datetime.fromtimestamp(ts, dt.timezone.utc).date().isoformat()   # noqa: E731
+    return {"windows": [{"label": "last year", "from": day(end - YEAR), "to": day(end), **shares(windows["last"])},
+                        {"label": "the year before", "from": day(start), "to": day(end - YEAR), **shares(windows["before"])}],
+            "cohort": {"marked": shares(groups[True]), "rest": shares(groups[False])},
+            "churn_days": RETOUCH_DAYS, "moved": "git --color-moved=blocks"}
+
+
+def _watch_files(out_dir: str):
+    """The watch list's top files, when the change analysis and scc have written their outputs; None
+    otherwise, or when this runs as a script outside the package."""
+    if not (os.path.exists(os.path.join(out_dir, "size.json")) and os.path.exists(os.path.join(out_dir, "maat-revisions.csv"))):
+        return None
+    try:
+        from . import load, watch
+    except ImportError:
+        return None
+    try:
+        report = load.load_report(out_dir, nested=False)
+    except load.Unreadable:
+        return None
+    return {r["file"] for r in watch.risks(report)[:WATCH_TOP]}
+
+
 def shape(commits: list) -> dict:
     """How commits arrive, as neutral numbers: the share landing in runs of five or more by one author
     within ten minutes of each other, the share with conventional-commit subjects, and how many hours
@@ -231,7 +349,20 @@ def main(argv=None) -> int:
         print(f"provenance.py: {(e.stderr or b'').decode('utf-8', 'replace').strip() or e}", file=sys.stderr)
         return 1
     inventory = trailers(commits)
-    result = {"trailers": inventory, "cohort": cohort(commits, inventory), "shape": shape(commits), "agents": agents(repo)}
+    watch_files = _watch_files(args[0])
+    meta = {}
+    try:
+        with open(os.path.join(args[0], "meta.json"), encoding="utf-8") as fh:
+            meta = json.load(fh)
+    except (OSError, ValueError):
+        pass
+    marked = marker(inventory)
+    result = {"trailers": inventory, "cohort": cohort(commits, inventory, watch_files), "shape": shape(commits), "agents": agents(repo)}
+    if watch_files is not None:
+        result["cohort"]["watch_top"] = WATCH_TOP
+    if commits:
+        result["lines"] = lines(repo, commits[-1]["time"], {c["hash"] for c in commits if marked(c)}, set(meta.get("generated") or []),
+                                filetypes.vendor_dirs({"meta": meta}))
     with open(os.path.join(args[0], "provenance.json"), "w", encoding="utf-8") as fh:
         json.dump(result, fh)
     return 0

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/render.py

@@ -548,14 +548,35 @@ def trailers_section(report: dict, full: bool = True, width=None) -> dict:
     if marked.get("commits"):
         def pair(key):
             return f"{_pct(marked.get(key, 0), marked['commits'])} against {_pct(rest.get(key, 0), rest.get('commits') or 0)}"
+        watch_part = f", touched a file on the watch list's top {co['watch_top']} {pair('watch')}" if co.get("watch_top") else ""
         notes.append(f"marked commits ({co.get('definition')}): {marked['commits']:,}, {round(100 * co.get('share', 0))}% of the history; "
-                     f"reverted {pair('reverted')} for the rest, fixes {pair('fixes')}, a file changed again within two weeks {pair('retouched')}")
+                     f"reverted {pair('reverted')} for the rest, fixes {pair('fixes')}, a file changed again within two weeks {pair('retouched')}{watch_part}")
     if sh:
         notes.append(f"{round(100 * sh.get('burst_share', 0))}% of commits land in bursts of five or more within ten minutes; "
                      f"{round(100 * sh.get('conventional_share', 0))}% have conventional-commit subjects; commits come in {sh.get('hours_used', 0)} hours of the day")
     return _section("Trailers", columns, rows, note=None if rows else "no trailers", caption="\n".join(notes) or None)
 
 
+def lines_section(report: dict, full: bool = True, width=None) -> dict:
+    """Lines added to code files in the last year and the year before, the share git marks as moved and
+    the share deleted again within two weeks, and the same for the marked cohort against the rest:
+    --full and Markdown only. A direction for this repository, not a score."""
+    ln = (report.get("provenance") or {}).get("lines") or {}
+
+    def share(x):
+        return "-" if x is None else f"{100 * x:.1f}%"
+    rows = [(f"{w['label']} ({w['from']} to {w['to']})", w["commits"], w["added"], share(w.get("moved_share")), share(w.get("churn_share")))
+            for w in ln.get("windows") or []]
+    co = ln.get("cohort") or {}
+    if (co.get("marked") or {}).get("commits"):
+        rows += [(label, c["commits"], c["added"], share(c.get("moved_share")), share(c.get("churn_share")))
+                 for label, c in (("marked commits, both years", co["marked"]), ("the rest, both years", co["rest"]))]
+    columns = [("period", {"overflow": "fold"}), ("commits", RIGHT), ("lines added", RIGHT), ("moved", RIGHT), (f"churned in {ln.get('churn_days', 14)} days", RIGHT)]
+    return _section("Changed lines", columns, rows, note=None if rows else "no history in the last two years",
+                    caption="code files only; moved: lines git's moved-code detection marks (--color-moved=blocks); churned: deleted again "
+                            "within two weeks from the same file with the same text" if rows else None)
+
+
 def hotspots_section(report: dict, full: bool = True, width=None) -> dict:
     """Change frequency times size, Tornhill-style. Files no longer in the tree sort last. Drawn
     under `--full` and in the Markdown export only; the default terminal report leaves it to the
@@ -791,10 +812,10 @@ def compare_section(result: dict) -> dict:
 
 
 BUILDERS = [watch_section, watch_by_component_section, size_section, people_section, knowledge_section, activity_section, timeline_section,
-            hotspots_section, coupling_section, signing_section, trailers_section, age_section, functions_section, health_section, osps_section]
+            hotspots_section, coupling_section, signing_section, trailers_section, lines_section, age_section, functions_section, health_section, osps_section]
 # `--full` and Markdown only: Size, Activity and Code age are interesting once and rarely change what you
 # do next; Hotspots ranks the files the watch list already leads with, by the same product.
-FULL_ONLY = {"size", "activity", "age", "hotspots", "signing", "trailers", "watch_by_component", "osps"}
+FULL_ONLY = {"size", "activity", "age", "hotspots", "signing", "trailers", "lines", "watch_by_component", "osps"}
 
 
 def sections(report: dict, full: bool = True, width=None) -> list:

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/run.py

@@ -263,7 +263,7 @@ def plan(repo_dir: str, out_dir: str, branch: str = "HEAD", age: bool = True, pl
         {"name": "change analysis", "argv": [sys.executable, MAAT_SCRIPT, log, out_dir, *type_args, *(["--now", now] if now else []), *(["--since", since] if since else []), "--aliases", o("meta.json"), *revs_args], "stdout": None, "deps": ["git-log"]},
         {"name": "signing", "argv": [sys.executable, "-m", "gitmole.signing", out_dir], "stdout": None, "deps": []},   # the gpgsig headers, no keyring
         {"name": "hygiene", "argv": [sys.executable, "-m", "gitmole.hygiene", out_dir], "stdout": None, "deps": []},   # the Scorecard checks, from the clone
-        {"name": "provenance", "argv": [sys.executable, "-m", "gitmole.provenance", out_dir], "stdout": None, "deps": []},   # trailers, cohorts, agent files
+        {"name": "provenance", "argv": [sys.executable, "-m", "gitmole.provenance", out_dir], "stdout": None, "deps": ["scc", "change analysis"]},   # trailers, cohorts, agent files; the watch list for the hit rate
     ]
     workers = procs or blame.default_procs()
     if lizard:

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole/structure.py

@@ -36,7 +36,7 @@ try:
 except ImportError:  # run as a script: the package directory is sys.path[0]
     import filetypes
 
-ANALYSER = "2"   # bump whenever what a file yields changes (a metric, an import's shape): the cache key carries it
+ANALYSER = "3"   # bump whenever what a file yields changes (a metric, an import's shape): the cache key carries it
 MAX_BYTES = 1_000_000
 FUNCTIONS_KEPT = 3000
 
@@ -73,6 +73,89 @@ NO_INCREMENT = {"try_statement"}   # the try nests its body; the catch is what S
 LOGICAL = {"&&", "||", "and", "or"}
 DEBT = re.compile(r"\b(TODO|FIXME|XXX|HACK)\b")
 
+# --- shapes: an error swallowed, an address in a literal, code left in a comment -----------------
+CATCH = {"catch_clause", "except_clause", "rescue"}
+BODY = {"block", "statement_block", "compound_statement", "then"}
+EMPTY_STATEMENTS = {"pass_statement", "empty_statement"}
+STRING = {"string", "string_literal", "interpreted_string_literal", "raw_string_literal", "template_string", "encapsed_string"}
+ATTRIBUTE = {"attribute", "attribute_list", "annotation", "marker_annotation", "decorator", "attribute_item"}
+_QUOTED = re.compile(r"""^[A-Za-z@$]*(["'`]+)(.*?)\1$""", re.S)
+_IPV4 = re.compile(r"^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})(?::\d{1,5})?$")
+# a comment line that reads as a statement: an assignment, a call, a keyword that opens one, a brace
+_CODE_LINE = re.compile(r"""^(?:(?:return|if|elif|else|for|while|import|from|var|let|const|def|function|class|#include|throw|raise|break|continue|await)\b.*[;:{})\]]|[A-Za-z_$][\w.$\[\]]*\s*(?:=|\+=|-=|\|=|:=)\s*\S.*|[\w.$]+(?:\.[\w$]+)*\(.*\)\s*;?|[{}]\s*[;)]?|.*[;{]|\}\s*else\b.*)$""")
+_COMMENT_MARK = re.compile(r"^\s*(?://+|/\*+|\*+/?|#+|--|;+) ?")
+CODE_SHARE = 0.8   # of a comment block's lines that read as code, for the block to count as code left in a comment
+_DIRECTIVE = re.compile(r"^(?:eslint|prettier|istanbul|noqa|type:|pylint|@ts-|jshint|global |c8 |nolint|NOLINT|clang-format|fmt:|pragma|region|endregion|-\*-|SPDX-|Copyright|http)", re.I)
+
+
+BROAD_PYTHON = {"Exception", "BaseException"}   # the language's own root classes
+
+
+def _is_empty_catch(node, src: bytes = b"") -> bool:
+    """A catch, except or rescue whose body does nothing and says nothing: no statement but `pass`, and
+    no comment, since a comment is the author saying the error is ignored on purpose (Sonar's S108).
+    Python's `except SomeError: pass` is the language's idiom for an expected failure (EAFP), so there
+    only a bare `except:` or one that catches Exception or BaseException counts."""
+    if node.type == "except_clause":
+        caught = [c for c in node.named_children if c.type not in BODY and c.type != "comment"]
+        if caught and not all(_text(src, c).strip("() ") in BROAD_PYTHON or _text(src, c).split(" as ")[0].strip("() ") in BROAD_PYTHON for c in caught):
+            return False
+    body = None
+    for c in node.named_children:
+        if c.type == "comment":
+            return False
+        if c.type in BODY:
+            body = c
+    if body is None:
+        return node.type == "rescue"   # Ruby: an empty rescue has no `then` at all
+    for c in body.named_children:
+        if c.type == "comment" or c.type not in EMPTY_STATEMENTS:
+            return False
+    return True
+
+
+def _address(text: str):
+    """The IPv4 address a string literal is, with its port, or None: loopback, the unspecified and
+    broadcast addresses, netmasks, the documentation ranges (RFC 5737), a trailing .0 (a network, or a
+    four-part version like 1.0.0.0), and a first octet of 0, 1 or 2, which is how an ASN.1 object
+    identifier (2.5.4.3) starts, are left out."""
+    m = _QUOTED.match(text.strip())
+    value = (m.group(2) if m else text).strip()
+    ip = _IPV4.match(value)
+    if not ip:
+        return None
+    octets = [int(x) for x in ip.groups()]
+    if any(o > 255 for o in octets) or octets[0] <= 2 or octets[0] in (127, 255) or octets[3] in (0, 255):   # 0-2: an object identifier's first arc
+        return None
+    if octets[:3] in ([192, 0, 2], [198, 51, 100], [203, 0, 113]):
+        return None
+    return value
+
+
+def _code_like(line: str) -> bool:
+    s = line.strip()
+    return bool(s) and not _DIRECTIVE.match(s) and not s.endswith(".") and bool(_CODE_LINE.match(s)) and not re.match(r"^[A-Za-z]+(?: [a-z]+){3,}", s)
+
+
+def commented_code_lines(text: str) -> int:
+    """How many lines of one comment block (a block comment, or a run of line comments on consecutive
+    lines) read as code; 0 unless four in five of its lines do and one of them starts right after the
+    comment marker, as an editor's comment-out leaves it. Prose with a worked example under it (indented,
+    or introduced by a line ending in a colon), and documentation comments, whose examples are meant to
+    be there, are not code left behind."""
+    if text.lstrip().startswith(("/**", "///", "//!", "#!", "/*!")):
+        return 0
+    lines = [re.sub(r"\s*\*/\s*$", "", _COMMENT_MARK.sub("", l)) for l in text.split("\n")]
+    lines = [l.rstrip() for l in lines if l.strip() and l.strip() not in ("*/", "*")]
+    if not lines:
+        return 0
+    code = [l for l in lines if _code_like(l)]
+    if len(code) < CODE_SHARE * len(lines) or not any(not l[:1].isspace() for l in code):
+        return 0
+    if any(l.strip().endswith(":") and not _code_like(l) for l in lines):
+        return 0   # "Build the dispatcher function:" introduces an example
+    return len(code)
+
 
 def cache_root() -> str | None:
     explicit = os.environ.get("GITMOLE_CACHE")
@@ -217,6 +300,17 @@ def analyse(src: bytes, lang_name: str, language) -> dict:
     funcs, stack, done = [], [], []   # stack: one frame per ancestor on the way down
     comments = comment_lines = definitions = 0
     debt, imports = [], []
+    shapes = {"empty_catch": [], "bare_except": [], "addresses": [], "commented_code": 0, "commented_sample": []}
+    block = []   # the open comment block: [first line, last line, text, made of line comments]
+
+    def flush():
+        if block:
+            code = commented_code_lines(block[2])
+            if code:
+                shapes["commented_code"] += code
+                if len(shapes["commented_sample"]) < 5:
+                    shapes["commented_sample"].append(block[0] + 1)
+            block.clear()
     main_guard = False
     chains = []   # open runs of logical operators: [count]
     while True:
@@ -263,6 +357,22 @@ def analyse(src: bytes, lang_name: str, language) -> dict:
             m = DEBT.search(text)
             if m:
                 debt.append({"line": node.start_point[0] + 1, "tag": m.group(1), "text": " ".join(text.split())[:100]})
+            own_line = not src[src.rfind(b"\n", 0, node.start_byte) + 1:node.start_byte].strip()   # not trailing code
+            line_comment = text.startswith(("//", "#", "--")) and "\n" not in text.rstrip()
+            if block and line_comment and block[3] and own_line and node.start_point[0] == block[1] + 1:
+                block[1], block[2] = node.end_point[0], block[2] + "\n" + text
+            else:
+                flush()
+                if own_line:
+                    block.extend([node.start_point[0], node.end_point[0], text, line_comment])
+        elif t in CATCH and _is_empty_catch(node, src):
+            shapes["empty_catch"].append(node.start_point[0] + 1)
+            if t == "except_clause" and not any(c.type not in BODY and c.type != "comment" for c in node.named_children):
+                shapes["bare_except"].append(node.start_point[0] + 1)
+        elif t in STRING and node.end_byte - node.start_byte <= 30 and not any(s[0] in ATTRIBUTE for s in stack):
+            value = _address(_text(src, node))
+            if value:
+                shapes["addresses"].append({"line": node.start_point[0] + 1, "value": value})
         found = _import(node, src, lang_name)
         if found:
             imports.extend(found)
@@ -278,7 +388,8 @@ def analyse(src: bytes, lang_name: str, language) -> dict:
             if cursor.goto_next_sibling():
                 break
             if not cursor.goto_parent():
-                return _result(done, comments, comment_lines, definitions, debt, imports, main_guard, src, tree)
+                flush()
+                return _result(done, comments, comment_lines, definitions, debt, imports, main_guard, src, tree, shapes)
 
 
 def _in_chain(node) -> bool:
@@ -304,10 +415,10 @@ def _leave(frame, funcs, done, chains):
         done.append(funcs.pop())
 
 
-def _result(done, comments, comment_lines, definitions, debt, imports, main_guard, src, tree) -> dict:
+def _result(done, comments, comment_lines, definitions, debt, imports, main_guard, src, tree, shapes) -> dict:
     lines = src.count(b"\n") + (1 if src and not src.endswith(b"\n") else 0)
     return {"lines": lines, "comments": comments, "comment_lines": comment_lines, "definitions": definitions,
-            "debt": debt, "imports": imports, "main": main_guard or src.startswith(b"#!"), "errors": tree.root_node.has_error,
+            "debt": debt, "imports": imports, "main": main_guard or src.startswith(b"#!"), "errors": tree.root_node.has_error, "shapes": shapes,
             "functions": [{"name": f.name, "start": f.start, "end": f.end, "nesting": f.max_nesting, "cognitive": f.cognitive,
                            "complex_conditions": f.complex, "bumps": f.bumps} for f in done]}
 
@@ -518,6 +629,23 @@ def unreferenced(files: dict, edges: dict, resolved: dict, entries: set) -> list
     return [p for p in out if files[p]["language"] not in loud]
 
 
+def _slim_shapes(s: dict) -> dict:
+    """The shapes one file holds, as structure.json keeps them: counts and the first few lines; empty
+    lists and zero counts are left out so a clean file costs nothing."""
+    out = {}
+    for key in ("empty_catch", "bare_except"):
+        if s.get(key):
+            out[key] = s[key][:5]
+            out[key + "_count"] = len(s[key])
+    if s.get("addresses"):
+        out["addresses"] = s["addresses"][:5]
+        out["addresses_count"] = len(s["addresses"])
+    if s.get("commented_code"):
+        out["commented_code"] = s["commented_code"]
+        out["commented_sample"] = s.get("commented_sample") or []
+    return out
+
+
 def entries_dirs(entries: set) -> set:
     return {e for e in entries if e.endswith("package.json")}
 
@@ -576,6 +704,7 @@ def collect(repo: str, procs: int = None, vendored=()) -> dict:
     slim = {p: {"language": v["language"], "lines": v.get("lines", 0), "comments": v.get("comments", 0), "comment_lines": v.get("comment_lines", 0),
                 "definitions": v.get("definitions", 0), "debt": len(v.get("debt") or []), "debt_sample": (v.get("debt") or [])[:5],
                 "main": v.get("main", False), "imports": edges.get(p, []), "errors": v.get("errors", False),
+                "shapes": _slim_shapes(v.get("shapes") or {}),
                 "max_nesting": max((f["nesting"] for f in v.get("functions") or []), default=0),
                 "max_cognitive": max((f["cognitive"] for f in v.get("functions") or []), default=0)}
             for p, v in files.items() if "failed" not in v}

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: gitmole
-Version: 0.20.0
+Version: 0.22.0
 Summary: Offline git repository analysis with a terminal report: hotspots, coupling, ownership, code age, secrets, repo health.
 License: MIT
 Project-URL: Homepage, https://github.com/antvinni/gitmole
@@ -74,6 +74,7 @@ gitmole . --json report.json           # every table, the watch list and the fin
 gitmole . --fail-on warning            # exit 3 if any finding is a warning or worse
 gitmole . --risk main --risk-threshold 10  # exit 3 if the files changed since main hold over 10% of the risk
 gitmole . --sarif gitmole.sarif        # the findings for GitHub code scanning or GitLab
+gitmole . --sbom sbom.cdx.json         # a CycloneDX SBOM of every package the lock files pin
 gitmole . --compare last.json          # what changed since an earlier --json export
 gitmole analysis-repo --no-run --hook  # a coding agent's edit hook: history's view of the files it just touched
 gitmole . --since 2y --full            # the current team, every row and column
@@ -92,9 +93,9 @@ Reports on repositories you know, each at a pinned commit, published as gitmole
 
 | Repository | Commit | Commits | Lines | gitmole run |
 |---|---|---:|---:|---:|
-| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 57 s |
-| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 132 s |
-| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 151 s |
+| [curl](https://github.com/antvinni/gitmole/blob/main/docs/examples/curl.md) | [`540ee5b5`](https://github.com/curl/curl/commit/540ee5b560cc6e775e11317048a13cc7e355bf91) | 39,758 | 247,179 | 59 s |
+| [django](https://github.com/antvinni/gitmole/blob/main/docs/examples/django.md) | [`8cbdd4a8`](https://github.com/django/django/commit/8cbdd4a814397f81adf0129288f32b615bd1f94f) | 34,933 | 431,749 | 135 s |
+| [react](https://github.com/antvinni/gitmole/blob/main/docs/examples/react.md) | [`2b19aecd`](https://github.com/facebook/react/commit/2b19aecd0e9111b774fad0fad9862e50bcb5bc8a) | 21,703 | 681,078 | 157 s |
 
 Run times are one `gitmole CLONE` with every default step, on a MacBook Pro (M4, 16 GB).
 

{gitmole-0.20.0 → gitmole-0.22.0}/gitmole.egg-info/SOURCES.txt

@@ -77,6 +77,7 @@ tests/test_render.py
 tests/test_render_examples.py
 tests/test_run.py
 tests/test_sarif.py
+tests/test_shapes.py
 tests/test_signing.py
 tests/test_structure.py
 tests/test_szz.py

{gitmole-0.20.0 → gitmole-0.22.0}/tests/test_evaluate.py

@@ -70,6 +70,16 @@ class Induced(unittest.TestCase):
 
 
 class Score(unittest.TestCase):
+    def test_effort_is_the_false_alarms_before_the_first_hit_and_the_lines_read(self):
+        r = evaluate.report_at(COMMITS, "2025-06-01", SIZE, {"bots": []}, [], [])
+        ranked = evaluate.variants(r)["watch list (hotspot)"]
+        out = evaluate.effort(r, {ranked[1]}, 15)["watch list (hotspot)"]
+        self.assertEqual(out[0], 1, "one file before the first labelled one")
+        self.assertEqual(out[1], sum(SIZE["files"][f]["code"] for f in ranked[:15]))
+        self.assertEqual(evaluate.effort(r, set(), 15)["watch list (hotspot)"][0], len(ranked[:15]), "no hit: every file was a false alarm")
+        table = evaluate.effort_table([{"a": (1, 100)}, {"a": (3, 300)}])
+        self.assertIn("| a | 2 | 200 |", table)
+
     def test_the_report_at_t_knows_nothing_after_t(self):
         r = evaluate.report_at(COMMITS, "2025-06-01", SIZE, {"bots": []}, [], [])
         self.assertEqual({x["entity"]: x["n-revs"] for x in r["revisions"]}, {"core/a.py": 3, "core/b.py": 2, "tests/test_a.py": 1})

{gitmole-0.20.0 → gitmole-0.22.0}/tests/test_provenance.py

@@ -110,6 +110,46 @@ class Agents(unittest.TestCase):
         self.assertNotIn("p4ss", json.dumps(out))
 
 
+class Lines(unittest.TestCase):
+    def _write(self, r, path, text, message, date):
+        with open(os.path.join(r.d, path), "w") as fh:
+            fh.write(text)
+        r.git("add", "-A", date=date)
+        r.git("commit", "-q", "-m", message, date=date)
+
+    def test_moved_and_churned_lines_by_year_and_cohort(self):
+        body = "".join(f"def function_number_{i}(argument):\n    return argument * {i} + compute_offset({i})\n" for i in range(6))
+        with tempfile.TemporaryDirectory() as d:
+            r = Repo(d)
+            self._write(r, "c.py", "unrelated_value = compute_something()\n", "start", "2024-06-01T10:00:00")   # the year before
+            self._write(r, "a.py", body + "keep_this_line = 1\ntemporary_line = 2\n", "add a", "2025-06-01T10:00:00")
+            self._write(r, "a.py", body + "keep_this_line = 1\n", "drop the temporary line\n\nAssisted-by: Tool", "2025-06-05T10:00:00")
+            with open(os.path.join(d, "c.py"), "a") as fh:
+                fh.write(body)
+            self._write(r, "a.py", "keep_this_line = 1\n", "move the functions into c.py", "2025-07-01T10:00:00")
+            commits = provenance.read_commits(d)
+            marked = provenance.marker(provenance.trailers(commits))
+            out = provenance.lines(d, commits[-1]["time"], {c["hash"] for c in commits if marked(c)})
+        last, before = out["windows"]
+        self.assertEqual((before["commits"], before["added"]), (1, 1))
+        self.assertEqual(last["added"], len(body.splitlines()) * 2 + 2)
+        self.assertEqual(last["churned"], 1, "the temporary line went within four days; the kept line and the moved functions are not churn")
+        self.assertEqual(last["moved"], len(body.splitlines()), "git marks the functions as moved from a.py to b.py")
+        self.assertEqual(out["cohort"]["marked"]["commits"], 1)
+        self.assertEqual(out["cohort"]["rest"]["churned"], 1, "the churn belongs to the commit that added the line")
+
+
+class WatchHits(unittest.TestCase):
+    def test_each_cohort_counts_its_commits_touching_a_watched_file(self):
+        with tempfile.TemporaryDirectory() as d:
+            history(d)
+            commits = provenance.read_commits(d)
+        inv = provenance.trailers(commits)
+        out = provenance.cohort(commits, inv, {"a.py"})
+        self.assertEqual((out["cohort"]["watch"], out["rest"]["watch"]), (1, 2), "the helper commit is marked; start and the fix are not")
+        self.assertNotIn("watch", provenance.cohort(commits, inv)["rest"], "no watch list, no count")
+
+
 class Step(unittest.TestCase):
     def test_the_step_writes_provenance_json(self):
         with tempfile.TemporaryDirectory() as d:
@@ -121,7 +161,7 @@ class Step(unittest.TestCase):
             self.assertEqual(p.returncode, 0, p.stderr)
             with open(os.path.join(out, "provenance.json")) as fh:
                 data = json.load(fh)
-        self.assertEqual(set(data), {"trailers", "cohort", "shape", "agents"})
+        self.assertEqual(set(data), {"trailers", "cohort", "shape", "agents", "lines"})
 
 
 if __name__ == "__main__":

{gitmole-0.20.0 → gitmole-0.22.0}/tests/test_render.py

@@ -1502,3 +1502,18 @@ class Excerpt(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class ChangedLines(unittest.TestCase):
+    def test_two_windows_and_the_cohorts_when_there_are_marked_commits(self):
+        w = {"commits": 10, "added": 200, "moved": 20, "churned": 10, "moved_share": 0.1, "churn_share": 0.05}
+        rep = {"provenance": {"lines": {"windows": [{"label": "last year", "from": "2025-01-01", "to": "2026-01-01", **w},
+                                                    {"label": "the year before", "from": "2024-01-01", "to": "2025-01-01", **w, "added": 0, "moved_share": None, "churn_share": None}],
+                                        "cohort": {"marked": {**w, "commits": 2}, "rest": w}, "churn_days": 14},
+                               "cohort": {"cohort": {"commits": 2, "watch": 1}, "rest": {"commits": 8, "watch": 2}, "watch_top": 15, "share": 0.2}}}
+        sec = render.lines_section(rep)
+        self.assertEqual(sec["rows"][0], ["last year (2025-01-01 to 2026-01-01)", "10", "200", "10.0%", "5.0%"])
+        self.assertEqual(sec["rows"][1][3:], ["-", "-"])
+        self.assertEqual([r[0] for r in sec["rows"][2:]], ["marked commits, both years", "the rest, both years"])
+        self.assertIn("lines", render.FULL_ONLY)
+        self.assertIn("touched a file on the watch list's top 15 50% against 25%", render.trailers_section(rep)["caption"] or render.trailers_section(rep)["note"] or "")

gitmole-0.22.0/tests/test_shapes.py

@@ -0,0 +1,88 @@
+"""The shape rules on the tree-sitter pass: an error caught and dropped, an address written into a string
+literal, code left in a comment. The parsing tests need gitmole[structure]; the findings do not."""
+import unittest
+
+from gitmole import findings, structure
+from tests.test_findings import report
+from tests.test_structure import HAVE, parse
+
+
+class CommentedCode(unittest.TestCase):
+    """The comment classifier is plain text, no grammar needed."""
+
+    def test_code_left_by_an_editor_counts(self):
+        self.assertEqual(structure.commented_code_lines("// const old = legacy(x);\n// console.log(state);\n// if (a) {\n//   b();\n// }"), 5)
+        self.assertEqual(structure.commented_code_lines("/* legacy(x);\n   other(y); */"), 2)
+        self.assertEqual(structure.commented_code_lines("# x = compute(a, b)"), 1)
+
+    def test_prose_examples_directives_and_doc_comments_do_not(self):
+        self.assertEqual(structure.commented_code_lines("# Some examples:\n#     SomeModel.objects.annotate(x)\n#     foo(bar)"), 0)
+        self.assertEqual(structure.commented_code_lines("//   function f(fn) {\n//     return g();\n//   }"), 0, "indented under the marker: an example")
+        self.assertEqual(structure.commented_code_lines("// Build the dispatcher:\n// function Foo(a) {\n// return x;\n// }\n// }"), 0)
+        self.assertEqual(structure.commented_code_lines("/** @example foo(1); */"), 0)
+        self.assertEqual(structure.commented_code_lines("// eslint-disable-next-line no-console"), 0)
+        self.assertEqual(structure.commented_code_lines("/* 35 = OBSOLETE */"), 0)
+        self.assertEqual(structure.commented_code_lines("# This explains why we do it."), 0)
+
+    def test_addresses_leave_out_what_is_not_a_host(self):
+        self.assertEqual(structure._address("'10.1.2.3'"), "10.1.2.3")
+        self.assertEqual(structure._address("`203.0.114.9:8080`"), "203.0.114.9:8080")
+        for literal in ("'127.0.0.1'", "'0.0.0.0'", "'255.255.255.0'", "'1.0.0.0'", "'2.5.4.3'", "'192.0.2.10'", "'300.1.1.1'", "'v1.2.3.4'"):
+            self.assertIsNone(structure._address(literal), literal)
+
+
+@unittest.skipUnless(HAVE, "gitmole[structure] not installed")
+class Shapes(unittest.TestCase):
+    def test_python_counts_only_the_broad_except_that_does_nothing(self):
+        s = parse(".py", "try:\n    x()\nexcept:\n    pass\ntry:\n    y()\nexcept ValueError:\n    pass\n"
+                         "try:\n    z()\nexcept Exception as e:\n    pass\ntry:\n    w()\nexcept Exception:\n    # ignored on purpose\n    pass\n")["shapes"]
+        self.assertEqual(s["empty_catch"], [3, 11])
+        self.assertEqual(s["bare_except"], [3])
+
+    def test_empty_catch_in_other_languages_and_a_comment_saves_it(self):
+        self.assertEqual(parse(".js", "try { a() } catch (e) {}\ntry { b() } catch { /* ok */ }\n")["shapes"]["empty_catch"], [1])
+        self.assertEqual(parse(".java", "class A { void f() { try { g(); } catch (Exception e) { } } }\n")["shapes"]["empty_catch"], [1])
+        self.assertEqual(parse(".rb", "begin\n  x\nrescue\nend\nbegin\n  y\nrescue => e\n  # ok\nend\n")["shapes"]["empty_catch"], [3])
+
+    def test_addresses_in_literals_but_not_in_attributes(self):
+        s = parse(".cs", '[assembly: AssemblyVersion("1.2.3.4")]\nclass A { void F() { var s = "8.8.4.4"; } }\n')["shapes"]
+        self.assertEqual(s["addresses"], [{"line": 2, "value": "8.8.4.4"}])
+
+    def test_line_comments_on_consecutive_lines_are_one_block_and_trailing_comments_are_not_code(self):
+        s = parse(".py", "x = 1  # y = 2\n# a = f(b)\n# c = g(d)\n\n# prose about it\n")["shapes"]
+        self.assertEqual((s["commented_code"], s["commented_sample"]), (2, [2]))
+
+
+def _report(files, **over):
+    return report(structure={"status": "run", "files": files}, **over)
+
+
+class ShapeFindings(unittest.TestCase):
+    def test_swallowed_errors_from_five_and_never_in_tests(self):
+        files = {"src/a.py": {"shapes": {"empty_catch": [3, 9], "empty_catch_count": 4, "bare_except": [3], "bare_except_count": 1}},
+                 "src/b.js": {"shapes": {"empty_catch": [7], "empty_catch_count": 1}},
+                 "tests/test_a.py": {"shapes": {"empty_catch": [1], "empty_catch_count": 9}}}
+        f = findings.swallowed_errors(_report(files))
+        self.assertEqual([(x["rule"]["id"], x["severity"]) for x in f], [("swallowed_errors", "info")])
+        self.assertIn("5 empty catch blocks in 2 source files, 1 of them a bare except", f[0]["detail"])
+        self.assertEqual(f[0]["evidence"]["files"][0], {"file": "src/a.py", "start": 3, "count": 4})
+        del files["src/b.js"]
+        self.assertEqual(findings.swallowed_errors(_report(files)), [], "four in source is below the floor")
+
+    def test_addresses_and_commented_code(self):
+        files = {"src/net.go": {"shapes": {"addresses": [{"line": 4, "value": "10.0.0.7"}], "addresses_count": 1}},
+                 "src/old.js": {"shapes": {"commented_code": 12, "commented_sample": [40, 90]}},
+                 "src/some.js": {"shapes": {"commented_code": 3, "commented_sample": [5]}}}
+        f = findings.hardcoded_addresses(_report(files))
+        self.assertIn("10.0.0.7 at src/net.go:4", f[0]["detail"])
+        f = findings.commented_out_code(_report(files))
+        self.assertEqual([x["file"] for x in f[0]["evidence"]["files"]], ["src/old.js"])
+        self.assertIn("src/old.js (12 lines from line 40)", f[0]["detail"])
+
+    def test_nothing_without_the_structure_step(self):
+        for rule in (findings.swallowed_errors, findings.hardcoded_addresses, findings.commented_out_code):
+            self.assertEqual(rule(report()), [])
+
+
+if __name__ == "__main__":
+    unittest.main()