gitmole 0.3.0__py3-none-any.whl

gitmole/leaks.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""gitleaks, with the secret values kept out of the output directory.
+
+gitmole runs this as the gitleaks step: `python3 leaks.py OUT_JSON`, from inside the repository.
+gitleaks writes its JSON report to our stdout, so the raw report is never a file; each value is
+replaced by a short keyed hash (enough to tell one value repeated in many places from many values)
+and a flag for shapes that cannot be a live secret, and only that is written. The key is random,
+made for one report and never stored, so a hash in secrets.json cannot be checked against a list of
+likely values; the price is that hashes from two runs cannot be compared, which nothing does. The report never needed the
+values: it names the rule, the file and the commit. Standalone, like maat.py and blame.py.
+
+The same module groups the loaded rows for the findings and the report footer.
+"""
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+try:
+    from . import filetypes
+except ImportError:  # run as a script: the package directory is sys.path[0]
+    import filetypes
+
+ARGV = ["gitleaks", "git", "--no-banner", "--report-format", "json", "--report-path", "-", "--exit-code", "0"]
+RAW_FIELDS = ("Secret", "Match", "Line", "Message")   # the value, the text around it, and the commit message, which can quote it
+
+# A version string (5.0.0-1667386184.dfbbb54) and a token shortened with an ellipsis are the only shapes
+# skipped. Nothing is skipped by prefix: a public and a private key of the same service often share one.
+_VERSION = re.compile(r"^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$")
+
+
+def new_key() -> bytes:
+    """A random key for one report. Never written anywhere."""
+    return os.urandom(32)
+
+
+def digest(value: str, key: bytes) -> str:
+    """HMAC-SHA256 of the value under the report's key, cut to 12 hex characters: equal values in one
+    report share it, and without the key it says nothing about the value."""
+    return hmac.new(key, value.encode("utf-8", "surrogateescape"), hashlib.sha256).hexdigest()[:12]
+
+
+def is_placeholder(value: str) -> bool:
+    value = value or ""
+    return bool(_VERSION.match(value)) or value.endswith("...") or value.endswith("…")
+
+
+def sanitise(rows: list) -> list:
+    key = new_key()   # one key for the whole report, so repeats of a value still group
+    out = []
+    for r in rows:
+        value = r.get("Secret") or ""
+        clean = {k: v for k, v in r.items() if k not in RAW_FIELDS}
+        clean["SecretHash"] = digest(value, key)
+        clean["Placeholder"] = is_placeholder(value)
+        out.append(clean)
+    return out
+
+
+def group(rows: list) -> list:
+    """One entry per distinct secret value (placeholders left out): its rule, the files and commits it
+    appears in, the number of distinct places (commit, file, line), and whether every place is a test
+    file. Values that appear in source come first, then the most widespread."""
+    groups, order = {}, []
+    for i, r in enumerate(rows):
+        if r.get("placeholder"):
+            continue
+        key = r.get("value") or ("row", i)
+        if key not in groups:
+            groups[key] = {"value": r.get("value"), "rule": r["rule"], "files": [], "commits": [], "_places": set(), "test": True}
+            order.append(key)
+        g = groups[key]
+        if r["file"] not in g["files"]:
+            g["files"].append(r["file"])
+        if r["commit"] not in g["commits"]:
+            g["commits"].append(r["commit"])
+        g["_places"].add((r["commit"], r["file"], r.get("line")))
+        g["test"] = g["test"] and filetypes.is_test_path(r["file"])
+    out = []
+    for key in order:
+        g = groups[key]
+        places = g.pop("_places")
+        out.append({**g, "places": len(places)})
+    out.sort(key=lambda g: (g["test"], -g["places"]))   # stable: first-seen order breaks ties
+    return out
+
+
+def placeholders(rows: list) -> int:
+    """Distinct places whose value had a placeholder shape."""
+    return len({(r["commit"], r["file"], r.get("line")) for r in rows if r.get("placeholder")})
+
+
+def main(argv=None) -> int:
+    args = sys.argv[1:] if argv is None else argv
+    if len(args) != 1:
+        print("usage: leaks.py OUT_JSON", file=sys.stderr)
+        return 2
+    target = args[0]
+    # stderr is inherited, so gitleaks' own log lands in run.log as before
+    proc = subprocess.run(ARGV, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE)
+    if proc.returncode != 0:
+        print(f"leaks.py: gitleaks exited {proc.returncode}; no report written", file=sys.stderr)
+        return proc.returncode
+    text = proc.stdout.decode("utf-8", "surrogateescape").strip()
+    rows = sanitise(json.loads(text) if text else [])
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(os.path.abspath(target)), prefix=".secrets-", suffix=".json")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            json.dump(rows, fh, indent=1)
+        os.replace(tmp, target)
+    finally:
+        if os.path.exists(tmp):
+            os.remove(tmp)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

gitmole/load.py

@@ -0,0 +1,253 @@
+"""Parsers for the files the tools write. Each takes text and returns plain data."""
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+import re
+from collections import Counter, OrderedDict
+
+from . import filetypes, identity, leaks
+
+
+def _rel(path: str) -> str:
+    """Tools started in the repo print './x'; the log and blame say 'x'."""
+    return path[2:] if path.startswith("./") else path
+
+
+def _only(rows: list, types) -> list:
+    """scc's language rows with the files outside `types` dropped and the totals rebuilt from what is
+    left. A row without per-file data (an older size.json) is kept as it is."""
+    out = []
+    for r in rows:
+        files = r.get("Files")
+        if files is None:
+            out.append(r)
+            continue
+        kept = [f for f in files if filetypes.matches(_rel(f.get("Location", "")), types)]
+        if kept:
+            out.append({**r, "Count": len(kept), "Files": kept,
+                        **{k: sum(f.get(k, 0) for f in kept) for k in ("Code", "Comment", "Blank", "Complexity")}})
+    return out
+
+
+def parse_scc(text: str, types=None) -> dict:
+    """scc --by-file JSON as languages and per-file rows. `types` (as filetypes.parse gives it: a
+    set, or None for everything) keeps only the code files, so the size matches the other tables."""
+    rows = json.loads(text) if text.strip() else []
+    if types is not None:
+        rows = _only(rows, types)
+    languages = sorted(
+        (
+            {
+                "name": r["Name"],
+                "files": r["Count"],
+                "code": r["Code"],
+                "comment": r["Comment"],
+                "blank": r["Blank"],
+                "complexity": r["Complexity"],
+            }
+            for r in rows
+        ),
+        key=lambda r: -r["code"],
+    )
+    files = {}
+    for r in rows:
+        for f in r.get("Files", []) or []:
+            files[_rel(f.get("Location", ""))] = {"code": f.get("Code", 0), "complexity": f.get("Complexity", 0)}
+    return {
+        "languages": languages,
+        "total_code": sum(r["code"] for r in languages),
+        "total_files": sum(r["files"] for r in languages),
+        "files": files,
+    }
+
+
+NUMERIC_COLUMNS = {"n-revs", "degree", "average-revs", "n-authors", "age-months", "added", "deleted", "n-fixes", "recent-fixes"}
+
+
+def parse_maat_csv(text: str) -> list:
+    """Rows as dicts. Only known numeric columns become ints; a file or author named 2024 stays a string."""
+    if not text.strip():
+        return []
+    out = []
+    for row in csv.DictReader(io.StringIO(text)):
+        out.append({k: (_num(v) if k in NUMERIC_COLUMNS else v) for k, v in row.items()})
+    return out
+
+
+def _num(v):
+    """An int for a numeric cell; 0 for a missing, empty or garbage one (a row cut short by a killed step)."""
+    try:
+        return int(v)
+    except (TypeError, ValueError):
+        return 0
+
+
+_SIZER_ROW = re.compile(r"^\|(?P<pad> *)(?P<name>.*?)\s*(?:\[(?P<ref>\d+)\])?\s*\|\s*(?P<value>.*?)\s*\|\s*(?P<concern>\**)\s*\|$")
+_SIZER_NOTE = re.compile(r"^\[(?P<ref>\d+)\]\s+\S+\s+\((?:[^:]+:)?(?P<path>[^)]*)\)")
+
+
+def parse_git_sizer(text: str) -> list:
+    notes = {}
+    for line in text.splitlines():
+        m = _SIZER_NOTE.match(line)
+        if m:
+            notes[m.group("ref")] = m.group("path")
+
+    # Two shapes of section: "Overall repository size" and "Biggest objects" have sub-headers
+    # ("* Blobs") with their metrics indented under them; "History structure" and "Biggest
+    # checkouts" list their metrics directly ("* Number of files"). A starred line with a value
+    # is a metric, a starred line without one is a sub-header, an unstarred line is a section.
+    rows, section, sub = [], "", ""
+    for line in text.splitlines():
+        m = _SIZER_ROW.match(line)
+        if not m:
+            continue
+        indent = len(m.group("pad")) - 1
+        raw = m.group("name").strip()
+        name = raw.lstrip("* ").strip()
+        if not name or name == "Name" or name.startswith("---"):
+            continue
+        if indent == 0 and not raw.startswith("*"):
+            section = sub = name
+            continue
+        if indent == 0 and not m.group("value"):
+            sub = name
+            continue
+        if not m.group("concern"):
+            continue
+        rows.append({
+            "name": f"{sub if indent else section}: {name}",
+            "value": m.group("value"),
+            "concern": len(m.group("concern")),
+            "ref": notes.get(m.group("ref") or "", ""),
+        })
+    return rows
+
+
+def parse_theseus(text: str) -> dict:
+    d = json.loads(text)
+    return OrderedDict((label, d["y"][i][-1]) for i, label in enumerate(d["labels"]))
+
+
+def parse_authors_log(text: str) -> list:
+    counts = Counter()
+    for line in text.splitlines():
+        if "\t" not in line:
+            continue
+        name, email = line.split("\t", 1)
+        counts[(name, email)] += 1
+    return [
+        {"name": n, "email": e, "commits": c}
+        for (n, e), c in sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))
+    ]
+
+
+def parse_functions(text: str) -> list:
+    """lizard --csv rows: nloc, ccn, tokens, params, length, location, file, function, long name, start, end."""
+    rows = []
+    for r in csv.reader(io.StringIO(text)):
+        if len(r) < 11:
+            continue
+        rows.append({"file": _rel(r[6]), "function": r[7], "ccn": _num(r[1]), "nloc": _num(r[0]), "params": _num(r[3]),
+                     "start": _num(r[9]), "end": _num(r[10])})
+    return rows
+
+
+_DUP_PLACE = re.compile(r"^(.+?):(\d+) ~ (\d+)$")
+_DUP_RATE = re.compile(r"Total duplicate rate:\s*([\d.]+)%")
+
+
+def parse_duplicates(text: str) -> dict:
+    """lizard -Eduplicate output: blocks of 'path:start ~ end' lines and the overall rate."""
+    blocks, current = [], None
+    for line in text.splitlines():
+        line = line.rstrip()
+        if line == "Duplicate block:":
+            current = []
+        elif current is not None:
+            m = _DUP_PLACE.match(line)
+            if m:
+                current.append((_rel(m.group(1)), int(m.group(2)), int(m.group(3))))
+            elif line.startswith("^^^"):
+                if current:
+                    blocks.append({"lines": current[0][2] - current[0][1] + 1, "places": sorted(current)})
+                current = None
+    m = _DUP_RATE.search(text)
+    return {"rate": float(m.group(1)) if m else None, "blocks": blocks}
+
+
+def parse_secrets(text: str) -> list:
+    """gitleaks rows as rule, file, short commit, line, fingerprint, the hashed value and the placeholder
+    flag. A report written before values were hashed still has them: hash them here, keep nothing raw."""
+    rows = json.loads(text) if text.strip() else []
+    key = leaks.new_key()   # for an older report with raw values: one key per read, as the wrapper does per run
+    out = []
+    for r in rows:
+        if "SecretHash" in r:
+            value, placeholder = r["SecretHash"], bool(r.get("Placeholder"))
+        elif r.get("Secret"):
+            value, placeholder = leaks.digest(r["Secret"], key), leaks.is_placeholder(r["Secret"])
+        else:
+            value, placeholder = None, False
+        out.append({"rule": r.get("RuleID", ""), "file": r.get("File", ""), "commit": r.get("Commit", "")[:7], "line": r.get("StartLine"),
+                    "fingerprint": r.get("Fingerprint", ""), "value": value, "placeholder": placeholder})
+    return out
+
+
+def _read(out_dir: str, name: str) -> str:
+    path = os.path.join(out_dir, name)
+    if not os.path.exists(path):
+        return ""
+    with open(path, encoding="utf-8", errors="replace") as fh:
+        return fh.read()
+
+
+def _read_json(out_dir: str, name: str, default):
+    """`default` for a missing file or one a killed step left truncated or malformed."""
+    text = _read(out_dir, name)
+    if not text:
+        return default
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        return default
+
+
+def load_report(out_dir: str, nested: bool = True) -> dict:
+    """Read every output file gitmole writes. Missing optional files become empty values.
+
+    `nested`: also load the backtest sub-report (out_dir/backtest), one level deep only."""
+    meta = json.loads(_read(out_dir, "meta.json") or "{}")
+    cohorts = _read(out_dir, "theseus/cohorts.json")
+    authors = _read(out_dir, "theseus/authors.json")
+    canonical = dict(meta["aliases"]) if "aliases" in meta else identity.canonical_names(meta.get("identities") or [])
+    surviving = OrderedDict()
+    for name, lines in (parse_theseus(authors) if authors else {}).items():
+        key = canonical.get(name, name)
+        surviving[key] = surviving.get(key, 0) + lines
+    return {
+        "out_dir": out_dir,
+        "meta": meta,
+        # a run records its --file-types spec (None for the default list); a run from before that record
+        # was measured unfiltered, so it is re-rendered unfiltered rather than with a guessed list
+        "size": parse_scc(_read(out_dir, "size.json"), filetypes.parse(meta["file_types"]) if "file_types" in meta else None),
+        "revisions": parse_maat_csv(_read(out_dir, "maat-revisions.csv")),
+        "coupling": parse_maat_csv(_read(out_dir, "maat-coupling.csv")),
+        "authors": parse_maat_csv(_read(out_dir, "maat-authors.csv")),
+        "age": parse_maat_csv(_read(out_dir, "maat-age.csv")),
+        "ownership": parse_maat_csv(_read(out_dir, "maat-entity-ownership.csv")),
+        "fixes": parse_maat_csv(_read(out_dir, "maat-fixes.csv")),
+        "sizer": parse_git_sizer(_read(out_dir, "repo-health.txt")),
+        "cohorts": parse_theseus(cohorts) if cohorts else {},
+        "theseus_authors": surviving,
+        "secrets": parse_secrets(_read(out_dir, "secrets.json")),
+        "activity": _read_json(out_dir, "activity.json", {}),
+        "functions": parse_functions(_read(out_dir, "functions.csv")),
+        "duplicates": parse_duplicates(_read(out_dir, "duplicates.txt")),
+        "trend": _read_json(out_dir, "trend.json", {"samples": [], "files": {}}),
+        "backtest": load_report(os.path.join(out_dir, "backtest"), nested=False)
+                    if nested and os.path.isfile(os.path.join(out_dir, "backtest", "meta.json")) else None,
+    }

gitmole/loss.py

@@ -0,0 +1,44 @@
+"""Knowledge loss: who has stopped committing, and how much of the code is theirs.
+
+"Gone" is measured against the repository's last commit, not today's date, so a clone that was
+last fetched a year ago does not mark everyone as gone. It is also measured over the whole
+history: a window (--since) that hides someone's last commit must not turn them into a person
+who never existed."""
+from __future__ import annotations
+
+from . import identity, knowledge, maat
+
+DEFAULT_MONTHS = 12
+
+
+def cutoff(report: dict, months: int = DEFAULT_MONTHS):
+    last = report["meta"].get("last_date") or ""
+    return maat.months_before(last, months) if last else None
+
+
+def gone(report: dict, months: int = DEFAULT_MONTHS) -> list:
+    """People whose last commit is before the cut-off, by name. Bots are never people."""
+    cut = cutoff(report, months)
+    act = report.get("activity") or {}
+    authors = act.get("authors_all") or act.get("authors") or {}   # authors_all is absent in older output directories
+    if not cut or not authors:
+        return []
+    bots = {b["name"] for b in report["meta"].get("bots") or []}
+    out = [{"name": name, "last": a.get("last", "")} for name, a in authors.items()
+           if name not in bots and not identity.is_bot(name) and a.get("last", "") < cut]
+    return sorted(out, key=lambda g: g["name"])
+
+
+def surviving(report: dict, gone_names) -> tuple:
+    """(surviving lines written by gone people, all surviving lines); (0, 0) without a blame pass."""
+    shares = report.get("theseus_authors") or {}
+    return sum(n for name, n in shares.items() if name in gone_names), sum(shares.values())
+
+
+def areas(rows: list, gone_names) -> list:
+    """knowledge.areas over the given ownership rows, each row with `lost` lines and `lost_share`."""
+    out = []
+    for a in knowledge.areas(rows):
+        lost = sum(n for name, n in a["owners"] if name in gone_names)
+        out.append({**a, "lost": lost, "lost_share": lost / a["lines"] if a["lines"] else 0.0})
+    return out