crapkit 0.2.0__py3-none-any.whl

crapkit/ratchet.py

@@ -0,0 +1,236 @@
+"""The committed ratchet: per-function high-water CRAP marks for functions above target.
+
+Pure text <-> entries. Marks only ever tighten: an improvement lowers or drops
+an entry on update; a regression NEVER raises one (it surfaces as a verdict
+failure instead); brand-new debt enters only through the audited override.
+Identity is (path, long_name): spans drift with every edit, names survive.
+
+The file opens with a metric stamp comment naming the analysis version and the
+lizard that produced the numbers. Marks measured under different rules are not
+comparable, and without the stamp that reads as a clean run.
+"""
+from __future__ import annotations
+
+from typing import NamedTuple
+
+from .score import ScoredRow
+
+_HEADER = "path\tlong_name\tcrap"
+
+
+class RatchetEntry(NamedTuple):
+    path: str
+    long_name: str
+    crap: float
+
+
+def stamp_text(analysis_version: int, lizard_version: str) -> str:
+    """The metric identity a set of marks was measured under, as one line."""
+    return f"crapkit-analysis={analysis_version} lizard={lizard_version}"
+
+
+def metric_version() -> str:
+    # Imported here, not at module scope: the git merge driver runs this module
+    # in a temp dir with no analysis stack, and it passes its own stamp through.
+    import lizard
+
+    from .analyze import ANALYSIS_VERSION
+    return stamp_text(ANALYSIS_VERSION, lizard.version)
+
+
+def read_stamp(text: str) -> str:
+    """The metric a marks file was written under; "" for one written before stamping."""
+    for line in text.splitlines():
+        if line.startswith("#"):
+            return line[1:].strip()
+        if line.strip():
+            return ""
+    return ""
+
+
+def stamp_conflict(recorded: str, current: str) -> str | None:
+    """The refusal when marks and the running metric disagree; None when they compare.
+
+    An unstamped file has nothing to disagree with — the caller warns instead.
+    """
+    if not recorded or recorded == current:
+        return None
+    return (f"ratchet marks were recorded under [{recorded}] but this run measures "
+            f"[{current}] — CRAP scores are not comparable across metric versions; "
+            "re-baseline with `crapkit ratchet seed`")
+
+
+def _is_skippable(line: str) -> bool:
+    """Blank lines, the header and comment lines (the metric stamp) carry no mark."""
+    return not line.strip() or line == _HEADER or line.startswith("#")
+
+
+def load_ratchet(text: str) -> list[RatchetEntry]:
+    entries = []
+    for i, line in enumerate(text.splitlines()):
+        if _is_skippable(line):
+            continue
+        parts = line.split("\t")
+        if len(parts) != 3:
+            raise ValueError(f"ratchet line {i + 1} has {len(parts)} fields, expected 3: {line!r}")
+        entries.append(RatchetEntry(parts[0], parts[1], float(parts[2])))
+    return entries
+
+
+def mark_for(entries: list[RatchetEntry], path: str, long_name: str) -> float | None:
+    """One function's recorded high-water mark, or None when it carries no mark."""
+    for e in entries:
+        if e.path == path and e.long_name == long_name:
+            return e.crap
+    return None
+
+
+def dump_ratchet(entries: list[RatchetEntry], *, stamp: str | None = None) -> str:
+    """`stamp` None takes the running metric; a version string is written verbatim
+    and "" writes none, which is how the merge driver keeps two legacy sides legacy."""
+    version = metric_version() if stamp is None else stamp
+    lines = [f"# {version}"] if version else []
+    lines.append(_HEADER)
+    for e in sorted(entries, key=lambda e: (e.path, e.long_name)):
+        lines.append(f"{e.path}\t{e.long_name}\t{e.crap:.4f}")
+    return "\n".join(lines) + "\n"
+
+
+def _merge_key(b: float | None, o: float | None, t: float | None) -> float | None:
+    """git 3-way semantics per key: the changed side wins over the unchanged one.
+    Both changed: keep the mark (it can only fall; prune is re-runnable) at min."""
+    if o == t:
+        return o
+    if o == b:
+        return t
+    if t == b:
+        return o
+    # both changed differently; both-None is impossible past the o == t check
+    return min(x for x in (o, t) if x is not None)
+
+
+def merge_ratchets(base: list[RatchetEntry], ours: list[RatchetEntry],
+                   theirs: list[RatchetEntry]) -> list[RatchetEntry]:
+    b = {(e.path, e.long_name): e.crap for e in base}
+    o = {(e.path, e.long_name): e.crap for e in ours}
+    t = {(e.path, e.long_name): e.crap for e in theirs}
+    merged = []
+    for key in sorted(set(b) | set(o) | set(t)):
+        crap = _merge_key(b.get(key), o.get(key), t.get(key))
+        if crap is not None:
+            merged.append(RatchetEntry(key[0], key[1], crap))
+    return merged
+
+
+def seed_ratchet(prior: list[RatchetEntry], fresh: list[ScoredRow], *, target: int,
+                 scope_targets: dict[str, int] | None = None) -> tuple[list[RatchetEntry], int, int]:
+    """First-class mark entry: record every over-ceiling function at its current
+    CRAP. A mark never rises, seeding included — an existing lower mark stays."""
+    from .verify import worst_twins
+
+    ceilings = scope_targets or {}
+    marks = {(e.path, e.long_name): e for e in prior}
+    added = tightened = 0
+    for key, row in worst_twins(fresh).items():
+        if row.crap <= ceilings.get(row.scope, target):
+            continue
+        a, t = _seed_mark(marks, key, row.crap)
+        added += a
+        tightened += t
+    return sorted(marks.values(), key=lambda e: (e.path, e.long_name)), added, tightened
+
+
+def _seed_mark(marks: dict, key: tuple, crap: float) -> tuple[int, int]:
+    """Place one mark; returns (added, tightened) deltas. A mark never rises."""
+    old = marks.get(key)
+    mark = round(crap, 4)
+    if old is not None and mark >= old.crap:
+        return 0, 0
+    marks[key] = RatchetEntry(key[0], key[1], mark)
+    return (1, 0) if old is None else (0, 1)
+
+
+def _repath(entries: list[RatchetEntry], dest_of) -> tuple[list[RatchetEntry], int]:
+    """Rewrite paths by a per-mark chooser (None leaves a mark alone); values never move."""
+    out = []
+    moved = 0
+    for e in entries:
+        dest = dest_of(e)
+        if dest is None:
+            out.append(e)
+            continue
+        out.append(e._replace(path=dest))
+        moved += 1
+    return sorted(out, key=lambda e: (e.path, e.long_name)), moved
+
+
+def _moved_path(path: str, old: str, new: str) -> str | None:
+    """One mark's destination under an explicit move, or None when `old` misses it."""
+    if old.endswith("/"):
+        return f"{new.rstrip('/')}/{path[len(old):]}" if path.startswith(old) else None
+    return new if path == old else None
+
+
+def move_marks(entries: list[RatchetEntry], old: str,
+               new: str) -> tuple[list[RatchetEntry], int]:
+    """Re-path marks at their recorded values. A trailing "/" on `old` moves a whole
+    directory; anything else matches one exact path."""
+    return _repath(entries, lambda e: _moved_path(e.path, old, new))
+
+
+def _rename_target(entry: RatchetEntry, present: set, renames: dict[str, str]) -> str | None:
+    """Where a mark should follow a renamed file, or None to leave it alone.
+
+    Three conditions, all required: the function is gone from its recorded path,
+    git calls that path renamed, and the SAME long_name exists at the new one. A
+    copy fails the first (the source survives), so its mark never travels.
+    """
+    if (entry.path, entry.long_name) in present:
+        return None
+    dest = renames.get(entry.path)
+    if dest is None or (dest, entry.long_name) not in present:
+        return None
+    return dest
+
+
+def follow_renames(prior: list[RatchetEntry], fresh: list[ScoredRow],
+                   renames: dict[str, str]) -> tuple[list[RatchetEntry], int]:
+    """Marks for renamed files, re-pathed. Runs BEFORE prune so a rename reads as a
+    move, not as code that left the repo and forfeits its high-water mark."""
+    from .verify import worst_twins
+
+    present = set(worst_twins(fresh))
+    return _repath(prior, lambda e: _rename_target(e, present, renames))
+
+
+def prune_ratchet(prior: list[RatchetEntry],
+                  fresh: list[ScoredRow]) -> tuple[list[RatchetEntry], int]:
+    """Deliberate mark exit: drop entries whose function is absent from the run.
+    The automatic update keeps them (an exclude glob or a lane outage also removes
+    rows); prune is the human confirming the code is really gone."""
+    from .verify import worst_twins
+
+    present = set(worst_twins(fresh))
+    kept = [e for e in prior if (e.path, e.long_name) in present]
+    return kept, len(prior) - len(kept)
+
+
+def update_ratchet(prior: list[RatchetEntry], fresh: list[ScoredRow], *, target: int,
+                   scope_targets: dict[str, int] | None = None) -> list[RatchetEntry]:
+    from .verify import worst_twins
+
+    fresh_by_key = worst_twins(fresh)
+    updated = []
+    for entry in prior:
+        row = fresh_by_key.get((entry.path, entry.long_name))
+        if row is None:
+            # Absent from the scored rows is NOT proof the code is gone — an
+            # exclude glob or a lane outage also removes it, and dropping the
+            # entry would erase an audited override's only diff-visible record.
+            # Stale entries are inert (verify checks only present functions).
+            updated.append(entry)
+            continue
+        if row.crap <= (scope_targets or {}).get(row.scope, target):
+            continue  # fixed for real: below the scope's ceiling needs no mark
+        updated.append(RatchetEntry(entry.path, entry.long_name, min(entry.crap, round(row.crap, 4))))
+    return sorted(updated, key=lambda e: (e.path, e.long_name))

crapkit/ratchet_report.py

@@ -0,0 +1,135 @@
+"""Burn-down analytics from the ratchet file's own git history. Pure.
+
+No timestamps live in the TSV; the commits that changed it carry them, so a
+fixed history reports deterministically. Ages and velocity anchor on the
+NEWEST commit in the history, never the wall clock.
+"""
+from __future__ import annotations
+
+DAY = 86400
+
+
+def _mark_line(line: str) -> tuple | None:
+    """((path, long_name), crap) when a +/- patch line is a mark row; else None."""
+    body = line[1:]
+    if body.startswith(("++ ", "-- ", "path\t")):
+        return None
+    parts = body.split("\t")
+    if len(parts) != 3:
+        return None
+    try:
+        return (parts[0], parts[1]), float(parts[2])
+    except ValueError:
+        return None
+
+
+def _commit_delta(patch: str) -> tuple[dict, dict]:
+    """The marks one commit's patch added and removed, keyed."""
+    added: dict = {}
+    removed: dict = {}
+    for line in patch.splitlines():
+        if not line or line[0] not in "+-":
+            continue
+        mark = _mark_line(line)
+        if mark is None:
+            continue
+        (added if line[0] == "+" else removed)[mark[0]] = mark[1]
+    return added, removed
+
+
+def mark_events(patches: list[tuple[int, str]]) -> list[tuple]:
+    """(ts, key, 'added'|'dropped', crap) per commit. A key removed and re-added
+    in the same commit is a tightening and emits nothing — only entry and
+    repayment count."""
+    events = []
+    for ts, patch in patches:
+        added, removed = _commit_delta(patch)
+        for key in sorted(set(added) - set(removed)):
+            events.append((ts, key, "added", added[key]))
+        for key in sorted(set(removed) - set(added)):
+            events.append((ts, key, "dropped", removed[key]))
+    return events
+
+
+def _drop_velocity(dropped: list[int], anchor: int) -> dict:
+    return {"dropped_last_30d": sum(1 for ts in dropped if anchor - ts <= 30 * DAY),
+            "dropped_last_90d": sum(1 for ts in dropped if anchor - ts <= 90 * DAY)}
+
+
+def _expired_marks(report: dict, max_age_months: int | None) -> list[str]:
+    if max_age_months is None:
+        return []
+    limit = max_age_months * 30
+    # "oldest" is age-sorted, so any expired mark appears in it; a backlog of
+    # more than its cap still reports the worst offenders
+    return [f"mark {e['long_name']} in {e['path']} is {e['age_days']}d old (limit {limit}d)"
+            for e in report["oldest"] if e["age_days"] > limit]
+
+
+def _stalled_repayment(report: dict, min_repaid_30d: int | None) -> list[str]:
+    if min_repaid_30d is None or not report["open"]:
+        return []
+    if report["dropped_last_30d"] >= min_repaid_30d:
+        return []
+    return [f"repayment stalled: {report['dropped_last_30d']} mark(s) repaid in 30d "
+            f"(policy wants {min_repaid_30d})"]
+
+
+def policy_violations(report: dict, max_age_months: int | None,
+                      min_repaid_30d: int | None) -> list[str]:
+    """Findings against the debt policy knobs; no knobs, no findings."""
+    return _expired_marks(report, max_age_months) + _stalled_repayment(report, min_repaid_30d)
+
+
+def _replay(events: list[tuple]) -> tuple[dict, dict, list[int]]:
+    """The committed state: when each surviving mark entered, what it is worth,
+    and the timestamp of every repayment."""
+    entered: dict = {}
+    crap: dict = {}
+    dropped: list[int] = []
+    for ts, key, kind, value in events:
+        if kind == "added":
+            entered[key] = ts
+            crap[key] = value
+        else:
+            entered.pop(key, None)
+            crap.pop(key, None)
+            dropped.append(ts)
+    return entered, crap, dropped
+
+
+def _open_marks(entered: dict, working: dict | None, anchor: int) -> dict:
+    """Open marks keyed to when they entered. The working tree says WHICH marks
+    are open, history says how old each one is; a mark with no commit behind it
+    entered at the anchor and reports 0d."""
+    if working is None:
+        return entered
+    return {key: entered.get(key, anchor) for key in working}
+
+
+def _uncommitted(crap: dict, working: dict | None) -> int:
+    """Marks the working tree and the newest committed version disagree on:
+    added, repaid or tightened on disk and not committed yet."""
+    if working is None:
+        return 0
+    return sum(1 for key in set(crap) | set(working) if crap.get(key) != working.get(key))
+
+
+def _age_rows(opened: dict, anchor: int) -> list[dict]:
+    return sorted(({"path": k[0], "long_name": k[1], "age_days": (anchor - ts) // DAY}
+                   for k, ts in opened.items()),
+                  key=lambda e: (-e["age_days"], e["path"], e["long_name"]))
+
+
+def report_from_events(events: list[tuple], working: dict | None = None) -> dict:
+    """`working` is the marks on disk, keyed (path, long_name) -> crap. It decides
+    which marks are open, because a seed that has not been committed is still debt
+    somebody owes. Repayment counts and velocity stay on committed history, which
+    is the only place a timestamp exists. None reports the committed state alone.
+    """
+    entered, crap, dropped = _replay(events)
+    anchor = max((ts for ts, *_ in events), default=0)
+    opened = _open_marks(entered, working, anchor)
+    return {"open": len(opened), "dropped_total": len(dropped), "anchor_ts": anchor,
+            "uncommitted": _uncommitted(crap, working),
+            "oldest": _age_rows(opened, anchor)[:20], **_drop_velocity(dropped, anchor)}

crapkit/sarif.py

@@ -0,0 +1,82 @@
+"""SARIF 2.1.0 and GitHub workflow-command emission. Pure builders.
+
+Code-scanning UIs and PR annotation bots consume this; ruleIds, levels, and
+locations are contract. Every uri is repo-relative with forward slashes.
+"""
+from __future__ import annotations
+
+from . import __version__
+
+_RULES = (
+    {"id": "crapkit/over-target",
+     "shortDescription": {"text": "CRAP score above the scope ceiling"}},
+    {"id": "crapkit/gate",
+     "shortDescription": {"text": "touched function over the complexity gate"}},
+    {"id": "crapkit/ratchet-regression",
+     "shortDescription": {"text": "a recorded CRAP mark got worse"}},
+)
+
+
+def _result(rule_id: str, level: str, path: str, line: int, text: str) -> dict:
+    return {
+        "ruleId": rule_id, "level": level,
+        "message": {"text": text},
+        "locations": [{"physicalLocation": {
+            "artifactLocation": {"uri": path.replace("\\", "/")},
+            "region": {"startLine": line},
+        }}],
+    }
+
+
+def over_target_results(scored, scope_targets: dict, target: int) -> list[dict]:
+    out = []
+    for r in scored:
+        ceiling = scope_targets.get(r.scope, target)
+        if r.crap <= ceiling:
+            continue
+        out.append(_result(
+            "crapkit/over-target", "warning", r.path, r.start,
+            f"{r.long_name}: CRAP {r.crap:.1f} over ceiling {ceiling} "
+            f"(ccn {r.ccn}, cov {r.cov:.0%}) -> {r.remedy}"))
+    return out
+
+
+def gate_results(violations) -> list[dict]:
+    return [_result("crapkit/gate", "error", v.path, v.start,
+                    f"{v.long_name}: CRAP {v.crap:.1f} (ccn {v.ccn}, cov {v.cov:.0%}) -> {v.remedy}")
+            for v in violations]
+
+
+def regression_results(regressions) -> list[dict]:
+    # the ratchet stores no line numbers; line 1 anchors the file-level finding
+    return [_result("crapkit/ratchet-regression", "error", r.path, 1,
+                    f"{r.long_name}: recorded {r.recorded} -> fresh {r.fresh_crap}")
+            for r in regressions]
+
+
+def sarif_document(results: list[dict]) -> dict:
+    return {
+        "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/"
+                   "master/Schemata/sarif-schema-2.1.0.json",
+        "version": "2.1.0",
+        "runs": [{
+            "tool": {"driver": {
+                "name": "crapkit",
+                "version": __version__,
+                "informationUri": "https://github.com/JeanFrancoisGagne/crapkit",
+                "rules": [dict(r) for r in _RULES],
+            }},
+            "results": results,
+        }],
+    }
+
+
+def _esc(text: str) -> str:
+    return text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A")
+
+
+def github_annotation(result: dict) -> str:
+    loc = result["locations"][0]["physicalLocation"]
+    return (f"::{result['level']} file={loc['artifactLocation']['uri']},"
+            f"line={loc['region']['startLine']},title={result['ruleId']}"
+            f"::{_esc(result['message']['text'])}")

crapkit/sarifio.py

@@ -0,0 +1,49 @@
+"""Writing a SARIF document to disk, one finding at a time.
+
+json.dump reaches CPython's pure-Python encoder whenever indent is set, so a
+36,767-finding report was serialized character by character in Python: 718 ms,
+against 171 ms for the same findings handed to the C encoder. Nothing in the
+SARIF spec asks for indentation and no consumer of this file is a human
+scrolling 17 MB, so the document is written compact and streamed.
+
+The skeleton comes from the builder itself, encoded with an empty results
+array and split at that array. Findings are then C-encoded one at a time into
+the gap, which lands byte for byte where json.dumps of the whole document
+would have put them — and never builds that document as a string.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from .sarif import sarif_document
+
+_EMPTY_RESULTS = '"results": []'
+
+
+def _frame(document: dict) -> tuple[str, str]:
+    """The document either side of its results array."""
+    text = json.dumps(document, sort_keys=True)
+    head, found, tail = text.partition(_EMPTY_RESULTS)
+    if not found:
+        raise ValueError("sarif document has no results array to stream into")
+    return head + '"results": [', "]" + tail
+
+
+def _write_results(handle, results) -> None:
+    # ", " is json.dumps' own item separator, which is what keeps the spliced
+    # bytes identical to a whole-document encode.
+    separator = ""
+    for result in results:
+        handle.write(separator + json.dumps(result, sort_keys=True))
+        separator = ", "
+
+
+def write_sarif(path: Path | str, results: list[dict]) -> None:
+    """Write the SARIF report for these findings. newline="\\n" is the
+    determinism contract: the bytes must not pick up the host's separator."""
+    head, tail = _frame(sarif_document([]))
+    with open(path, "w", encoding="utf-8", newline="\n") as handle:
+        handle.write(head)
+        _write_results(handle, results)
+        handle.write(tail)