crapkit 0.2.0__py3-none-any.whl

crapkit/uncovered.py

@@ -0,0 +1,131 @@
+"""Dark lines per function: which lines inside a span no lane ever ran.
+
+The line-level truth is the one the diff-coverage check already reads — every
+lane's missing-line set, intersected, so a line stays dark only when NO lane
+ran it. What this module adds is a verdict on whether the artifacts on disk
+still describe the working tree. Line numbers from an artifact built before the
+last edit point at code that has moved, which is worse than no numbers at all:
+then the lines are [] and a note names the lane to rerun.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+from typing import NamedTuple
+
+
+class MissingLines(NamedTuple):
+    """Per-file dead lines, plus the reason there are none to report.
+
+    A populated `note` overrides everything: no span gets lines, because the
+    only lines available would be the wrong ones.
+    """
+    by_path: dict[str, set[int]]
+    note: str
+
+    def in_span(self, path: str, start: int, end: int) -> list[int]:
+        if self.note:
+            return []
+        return sorted(n for n in self.by_path.get(path, ()) if start <= n <= end)
+
+    def note_for(self, path: str, flag: str = "", scope: str = "") -> str:
+        """Why this path has no dark lines, or "" when the artifacts answered.
+
+        A file no artifact mentioned is not a file with full coverage, and an
+        empty list with no note is exactly how that lie would read.
+
+        Three causes read the same on the surface and want different moves, so
+        the note names which one it is. cc-only is decided first and outranks
+        everything: the scope asked for no coverage, so no artifact was ever
+        going to speak for it and no lane is worth naming. A stale or missing
+        artifact is `self.note`, and rerunning coverage on a settled tree clears
+        it. A file absent from every artifact is `flag: untested`: nothing
+        imports it, so coverage never emitted a record for it.
+        """
+        if flag == "cc-only":
+            return _cc_only_note(path, scope)
+        if self.note:
+            return self.note
+        if path in self.by_path:
+            return ""
+        return _absent_note(path, flag)
+
+
+def _cc_only_note(path: str, scope: str) -> str:
+    """The note for a scope that declared coverage_optional.
+
+    It names that setting rather than a lane: the stale-artifact note used to
+    win here and sent readers to commit and rerun coverage for a scope no lane
+    covers, which changes nothing.
+    """
+    return (f"scope {scope!r} sets coverage_optional = true, so no artifact "
+            f"can name uncovered lines for {path}")
+
+
+def _absent_note(path: str, flag: str) -> str:
+    """The note for a file no artifact mentioned, told apart by the score's flag."""
+    absent = f"no lane artifact measured {path}"
+    if flag != "untested":
+        return absent
+    return (f"{absent} (flag untested: no test imports it, so coverage records "
+            f"nothing for it; write the first test that imports {path})")
+
+
+def missing_by_path(root: Path, cfg) -> dict[str, set[int]]:
+    """Union of the lanes' line-level truth; a file two lanes measured keeps a
+    line dead only when NO lane ran it."""
+    from . import covstream
+
+    missing: dict[str, set[int]] = {}
+    for lane in cfg.lanes:
+        artifact = root / lane.artifact
+        if not artifact.is_file():
+            continue
+        # Off the file, not out of a string: every declared lane's artifact
+        # would otherwise be decoded whole, one after another, on one heap.
+        parsed = (covstream.parse_istanbul_missing_file(artifact, repo_root=str(root))
+                  if lane.parser == "istanbul"
+                  else covstream.parse_coveragepy_missing_file(
+                      artifact, path_prefix=lane.path_prefix))
+        for path, lines in parsed.items():
+            missing[path] = missing[path] & lines if path in missing else set(lines)
+    return missing
+
+
+def _artifact_state(root: Path, lane, scope_paths: dict, git) -> str:
+    """What stops this lane's artifact from naming line numbers, or "" when nothing does."""
+    from .lanes import lane_unchanged
+
+    if not (root / lane.artifact).is_file():
+        return f"lane {lane.name!r}: no artifact at {lane.artifact}"
+    if not lane_unchanged(root, lane, scope_paths, git):
+        return (f"lane {lane.name!r}: files in its scopes changed since {lane.artifact} "
+                "was written (uncommitted edits count), so its line numbers are stale — "
+                "commit or revert them, then rerun `crapkit coverage`")
+    return ""
+
+
+def _staleness_note(root: Path, cfg, git) -> str:
+    return "; ".join(state for state in
+                     (_artifact_state(root, lane, cfg.scope_paths, git) for lane in cfg.lanes)
+                     if state)
+
+
+def load_uncovered(root: Path, cfg, git=None) -> MissingLines:
+    """The dark lines the lane artifacts on disk report, or the note saying why not.
+
+    A half-written artifact degrades to a note here, unlike in verify: naming a
+    function's dark lines is a convenience nothing gates on, and it must never
+    turn a question about the worklist into a tooling exit code.
+    """
+    from .errors import ToolError
+    from .gitio import GitFacts
+
+    if not cfg.lanes:
+        return MissingLines({}, "no [[lane]] declared, so no artifact can say which lines are dark")
+    note = _staleness_note(root, cfg, git or GitFacts(root))
+    if note:
+        return MissingLines({}, note)
+    try:
+        return MissingLines(missing_by_path(root, cfg), "")
+    except ToolError as exc:
+        return MissingLines({}, f"unreadable lane artifact: {exc}")

crapkit/universe.py

@@ -0,0 +1,157 @@
+"""Assign tracked files to scopes, apply exclusions, name what fell through. Pure.
+
+The file universe itself comes from `git ls-files` in the shell layer; lizard is
+always fed these explicit lists because its own directory walking descends
+nested node_modules (measured hang).
+"""
+from __future__ import annotations
+
+import fnmatch
+import re
+from collections.abc import Callable
+from typing import NamedTuple
+
+from .config import Config, Scope
+
+LANGUAGE_EXTENSIONS = {
+    "typescript": (".ts",),
+    "tsx": (".tsx",),
+    "javascript": (".js", ".jsx", ".mjs", ".cjs"),
+    "python": (".py",),
+    "swift": (".swift",),
+}
+
+# Test directories are excluded case-insensitively: Swift convention capitalizes Tests/.
+_TEST_DIR = re.compile(r"(^|/)(tests?|__tests__)(/|$)", re.IGNORECASE)
+
+_NEVER = re.compile(r"(?!x)x").match  # an empty glob list must exclude NOTHING
+
+
+def exclude_matcher(globs: tuple[str, ...]) -> Callable[[str], re.Match[str] | None]:
+    """The exclude globs as ONE compiled alternation, matched against a lowered path.
+
+    `fnmatch.fnmatch` normcases BOTH arguments on every call, and on Windows
+    that is an LCMapStringEx syscall per path per glob (measured: 593k calls,
+    0.44s on a 31.6k-file repo) for a lowering the caller already did. Compile
+    once, match once. Build it OUTSIDE any per-file loop.
+    """
+    if not globs:
+        return _NEVER
+    return re.compile("|".join(fnmatch.translate(g.lower()) for g in globs)).match
+
+
+def excluded(path: str, match_glob: Callable[[str], re.Match[str] | None]) -> bool:
+    return bool(_TEST_DIR.search(path) or match_glob(path.lower()))
+
+
+def _source_extensions(languages: tuple[str, ...]) -> tuple[str, ...]:
+    return tuple(e for lang in languages for e in LANGUAGE_EXTENSIONS[lang])
+
+
+class _ScopeMatch(NamedTuple):
+    """One scope's prefix and extension tests, precomputed once per run."""
+    name: str
+    exact: frozenset[str]
+    prefixes: tuple[str, ...]
+    extensions: tuple[str, ...]
+
+
+def _scope_matchers(scopes: tuple[Scope, ...]) -> tuple[_ScopeMatch, ...]:
+    return tuple(
+        _ScopeMatch(s.name, frozenset(s.paths),
+                    tuple(p.rstrip("/") + "/" for p in s.paths),
+                    _source_extensions(s.languages))
+        for s in scopes
+    )
+
+
+def _owning_scope(path: str, matchers: tuple[_ScopeMatch, ...]) -> str | None:
+    """Name of the scope that claims path, or None when no scope does."""
+    # First scope whose path prefix AND language extensions both match wins;
+    # a prefix-only match must not stop the search or shared-prefix scopes
+    # silently black-hole each other's files.
+    for m in matchers:
+        if path in m.exact or path.startswith(m.prefixes):
+            if path.endswith(m.extensions):
+                return m.name
+    return None
+
+
+class Universe(NamedTuple):
+    """Every candidate file's verdict.
+
+    A candidate is a non-excluded path some scope's language claims by
+    extension. `unclaimed` is the set that used to vanish here: source in a
+    declared language that no scope PATH owns, which then commits with zero
+    gating. `oversized` names what the byte ceiling cut, with the sizes, so
+    the skip is reported rather than silent.
+    """
+    by_scope: dict[str, list[str]]
+    unclaimed: tuple[str, ...]
+    oversized: tuple[tuple[str, int], ...]
+
+
+def _candidate(path: str, matchers: tuple[_ScopeMatch, ...]) -> tuple[str | None, bool]:
+    """(owning scope or None, whether any scope's language claims the extension)."""
+    owner = _owning_scope(path, matchers)
+    if owner is not None:
+        return owner, True
+    return None, any(path.endswith(m.extensions) for m in matchers)
+
+
+def _candidates(files: list[str], cfg: Config,
+                matchers: tuple[_ScopeMatch, ...]) -> list[tuple[str, str | None]]:
+    match_glob = exclude_matcher(cfg.exclude_globs)
+    out = []
+    for raw_path in files:
+        path = raw_path.replace("\\", "/")
+        if excluded(path, match_glob):
+            continue
+        owner, known_language = _candidate(path, matchers)
+        if known_language:
+            out.append((path, owner))
+    return out
+
+
+def _oversize(path: str, max_bytes: int | None, size_of) -> int | None:
+    """The file's size when the byte ceiling puts it out of reach, else None.
+
+    No limit means no stat: the size lookup is a syscall per file, and a repo
+    without max_file_bytes must not pay for a rule it never set.
+    """
+    if max_bytes is None or size_of is None:
+        return None
+    size = size_of(path)
+    return size if size > max_bytes else None
+
+
+def _partition(candidates: list[tuple[str, str | None]], scopes: tuple[Scope, ...],
+               max_bytes: int | None, size_of):
+    assigned: dict[str, list[str]] = {s.name: [] for s in scopes}
+    unclaimed, oversized = [], []
+    for path, owner in candidates:
+        size = _oversize(path, max_bytes, size_of)
+        if size is not None:
+            oversized.append((path, size))
+        elif owner is not None:
+            assigned[owner].append(path)
+        else:
+            unclaimed.append(path)
+    return assigned, unclaimed, oversized
+
+
+def scan_files(files: list[str], cfg: Config, *,
+               size_of: Callable[[str], int] | None = None) -> Universe:
+    """The whole verdict. `size_of` is injected so this stays pure; the shell
+    layer passes a working-tree stat, and callers with no tree pass nothing."""
+    assigned, unclaimed, oversized = _partition(
+        _candidates(files, cfg, _scope_matchers(cfg.scopes)),
+        cfg.scopes, cfg.max_file_bytes, size_of)
+    return Universe({name: sorted(paths) for name, paths in assigned.items()},
+                    tuple(sorted(unclaimed)), tuple(sorted(oversized)))
+
+
+def assign_files(files: list[str], cfg: Config, *,
+                 size_of: Callable[[str], int] | None = None) -> dict[str, list[str]]:
+    """Just the per-scope mapping, for callers with no use for the dropped sets."""
+    return scan_files(files, cfg, size_of=size_of).by_scope

crapkit/verify.py

@@ -0,0 +1,194 @@
+"""The verdict. Pure: fresh scored rows + changed ranges + ratchet + failure sets in, Verdict out.
+
+Three independent checks, all must hold:
+- Gate: every function a change touched sits at CRAP <= target (coverage cannot
+  save cc > target; that is the target's design).
+- Ratchet: no function above target scores worse than its recorded high-water
+  mark, touched or not (coverage rot regresses functions nobody edited).
+- Failures: the fresh failure set adds nothing over the baseline's (the suite
+  is never assumed green; 98 pre-existing failures measured on day one).
+"""
+from __future__ import annotations
+
+from bisect import bisect_left, bisect_right
+from collections.abc import Iterator
+from typing import NamedTuple
+
+from .ratchet import RatchetEntry
+from .score import ScoredRow, parse_scored_tsv, scored_tsv_lines
+
+
+def diff_uncovered(changed_ranges: dict, missing: dict) -> list[tuple[str, int]]:
+    """Changed lines (new-file coordinates) whose statement never ran — where
+    the next bug ships. Files no lane measured stay silent (absent from missing)."""
+    out = []
+    for path, ranges in sorted(changed_ranges.items()):
+        dead = missing.get(path)
+        if not dead:
+            continue
+        # Sort the file's dead lines ONCE: sorting (and linearly scanning) them
+        # per hunk was O(hunks x dead log dead) for a file whose dead set never
+        # changes. Sorted, each hunk is two bisects and a slice.
+        ordered = sorted(dead)
+        for start, end in ranges:
+            out.extend((path, line)
+                       for line in ordered[bisect_left(ordered, start):bisect_right(ordered, end)])
+    return out
+
+
+class PortableBaseline(NamedTuple):
+    commit: str
+    kind: str
+    rows: list[ScoredRow]
+
+
+def baseline_tsv_lines(commit: str, kind: str, rows: list[ScoredRow]) -> Iterator[str]:
+    """A baseline run as a file the repo can carry: a commit stamp, then the
+    run's scored export. The store lives in a gitignored .crapkit/, so a fresh
+    clone has nothing else to name what it is being measured against."""
+    yield f"# commit={commit} run_kind={kind}\n"
+    yield from scored_tsv_lines(rows)
+
+
+def _stamp_fields(stamp: str) -> dict[str, str]:
+    return dict(part.split("=", 1) for part in stamp.removeprefix("# ").split() if "=" in part)
+
+
+def parse_baseline_tsv(text: str) -> PortableBaseline:
+    stamp, _, body = text.partition("\n")
+    fields = _stamp_fields(stamp)
+    if "commit" not in fields or "run_kind" not in fields:
+        raise ValueError(
+            f"a baseline file starts with `# commit=<sha> run_kind=<kind>`, got {stamp!r}")
+    return PortableBaseline(fields["commit"], fields["run_kind"], parse_scored_tsv(body))
+
+
+class GateViolation(NamedTuple):
+    path: str
+    long_name: str
+    start: int
+    ccn: int
+    cov: float
+    crap: float
+    remedy: str
+    dirty: bool = False
+
+
+class RatchetRegression(NamedTuple):
+    path: str
+    long_name: str
+    recorded: float
+    fresh_crap: float
+    dirty: bool = False
+
+
+class Verdict(NamedTuple):
+    ok: bool
+    gate_violations: list[GateViolation]
+    ratchet_regressions: list[RatchetRegression]
+    new_failures: list[str]
+    dirty_failures: list[str]
+
+
+def _id_forms(path: str) -> tuple[str, str]:
+    """The two shapes a junit classname takes for one file: the repo-relative
+    path (vitest, and pytest's `file` fallback) and pytest's dotted module."""
+    stem = path[:-3] if path.endswith(".py") else path
+    return path, stem.replace("/", ".")
+
+
+def dirty_failure_ids(new_failures: list[str], dirty_paths: set[str]) -> list[str]:
+    """New failures whose test id names a file with uncommitted edits."""
+    forms = {form for path in dirty_paths for form in _id_forms(path)}
+    return [f for f in new_failures if f.split("::")[0] in forms]
+
+
+def dirty_counts(verdict: Verdict) -> tuple[int, int]:
+    """(committed, dirty) over every finding kind, so one line says how much of
+    a verdict belongs to the tree as committed and how much to somebody's edits."""
+    dirty_ids = set(verdict.dirty_failures)
+    flags = ([v.dirty for v in verdict.gate_violations]
+             + [r.dirty for r in verdict.ratchet_regressions]
+             + [f in dirty_ids for f in verdict.new_failures])
+    dirty = sum(flags)
+    return len(flags) - dirty, dirty
+
+
+def _touched(row: ScoredRow, ranges: dict[str, list[tuple[int, int]]]) -> bool:
+    spans = ranges.get(row.path)
+    if not spans:
+        return False
+    return any(not (hi < row.start or lo > row.end) for lo, hi in spans)
+
+
+def touched_rows(rows: list[ScoredRow],
+                 changed_ranges: dict[str, list[tuple[int, int]]]) -> list[ScoredRow]:
+    """The gate's selection without its policy: rows whose span a change overlaps.
+
+    Every gate in crapkit judges touched functions only — untouched debt is the
+    ratchet's business. `rescore --gate` reuses this so its verdict and the
+    pre-commit hook's cannot disagree about which functions were even in scope.
+    """
+    return [r for r in rows if _touched(r, changed_ranges)]
+
+
+def worst_twins(fresh: list[ScoredRow]) -> dict[tuple[str, str], ScoredRow]:
+    """Twins share (path, long_name); the WORST twin represents the key, so a
+    regression can never hide behind (nor a clean sibling tighten past) it."""
+    worst: dict[tuple[str, str], ScoredRow] = {}
+    for r in fresh:
+        key = (r.path, r.long_name)
+        if key not in worst or r.crap > worst[key].crap:
+            worst[key] = r
+    return worst
+
+
+def _gate_violations(fresh, changed_ranges, target, scope_targets, dirty) -> list[GateViolation]:
+    gate = [
+        GateViolation(r.path, r.long_name, r.start, r.ccn, r.cov, r.crap, r.remedy,
+                      r.path in dirty)
+        for r in fresh
+        if r.crap > (scope_targets or {}).get(r.scope, target) and _touched(r, changed_ranges)
+    ]
+    gate.sort(key=lambda v: (-v.crap, v.path, v.start))
+    return gate
+
+
+def _ratchet_regressions(fresh, ratchet, dirty) -> list[RatchetRegression]:
+    worst_by_key = worst_twins(fresh)
+    regressions = []
+    for entry in ratchet:
+        row = worst_by_key.get((entry.path, entry.long_name))
+        # Compare at the precision the mark is STORED at: marks live as 4dp
+        # strings, and cov = covered/total makes longer decimals routine — an
+        # unrounded compare wedges an unchanged tree against its own mark.
+        if row is not None and round(row.crap, 4) > entry.crap:
+            regressions.append(RatchetRegression(entry.path, entry.long_name, entry.crap,
+                                                 round(row.crap, 4), entry.path in dirty))
+    regressions.sort(key=lambda r: (-(r.fresh_crap - r.recorded), r.path))
+    return regressions
+
+
+def evaluate(
+    *,
+    fresh: list[ScoredRow],
+    changed_ranges: dict[str, list[tuple[int, int]]],
+    ratchet: list[RatchetEntry],
+    baseline_failures: set[str],
+    fresh_failures: set[str],
+    target: int,
+    scope_targets: dict[str, int] | None = None,
+    dirty_paths: set[str] | None = None,
+) -> Verdict:
+    dirty = dirty_paths or set()
+    gate = _gate_violations(fresh, changed_ranges, target, scope_targets, dirty)
+    regressions = _ratchet_regressions(fresh, ratchet, dirty)
+    new_failures = sorted(fresh_failures - baseline_failures)
+
+    return Verdict(
+        ok=not gate and not regressions and not new_failures,
+        gate_violations=gate,
+        ratchet_regressions=regressions,
+        new_failures=new_failures,
+        dirty_failures=dirty_failure_ids(new_failures, dirty),
+    )

crapkit/watch.py

@@ -0,0 +1,112 @@
+"""Watch-mode core. Pure: mtime snapshots in, changed paths out.
+
+The polling loop in the CLI stays a thin shell around this; stdlib mtimes,
+no filesystem-event dependency, works the same on every host.
+"""
+from __future__ import annotations
+
+import os
+import stat
+from pathlib import Path
+
+
+def _stat_mtime(path: Path) -> float | None:
+    """One stat, or None for anything that is not a readable regular file.
+
+    `is_file()` followed by `stat()` paid the syscall TWICE per tracked file,
+    every poll interval. The S_ISREG check keeps directories out, which is the
+    only thing is_file() was buying.
+    """
+    try:
+        st = os.stat(path)
+    except OSError:
+        return None
+    return st.st_mtime if stat.S_ISREG(st.st_mode) else None
+
+
+def _entry_mtime(entry: os.DirEntry) -> float | None:
+    """The same answer for an already-listed name. On Windows the times came
+    with the listing, so this costs no syscall at all; the guard is for the
+    hosts where it does one, and for a name that died between the two."""
+    try:
+        st = entry.stat()
+    except OSError:
+        return None
+    return st.st_mtime if stat.S_ISREG(st.st_mode) else None
+
+
+def _by_directory(files: list[str]) -> dict[str, dict[str, str]]:
+    """{directory: {basename: path}} — the grouping one listing can answer.
+
+    Paths are the repo-relative, slash-separated ones git reports. Anything
+    shaped otherwise simply groups under the root and misses its listing, which
+    costs it a stat and nothing else.
+    """
+    grouped: dict[str, dict[str, str]] = {}
+    for rel in files:
+        parent, _, base = rel.rpartition("/")
+        grouped.setdefault(parent, {})[base] = rel
+    return grouped
+
+
+def _keep_wanted(entry: os.DirEntry, names: dict[str, str], out: dict[str, float]) -> None:
+    """A listing walks a whole directory; only the tracked names are wanted,
+    and only they are worth a stat on the hosts where one is charged."""
+    rel = names.get(entry.name)
+    if rel is None:
+        return
+    mtime = _entry_mtime(entry)
+    if mtime is not None:
+        out[rel] = mtime
+
+
+def _list_directory(directory: Path, names: dict[str, str], out: dict[str, float]) -> None:
+    """Everything one listing can answer. A directory that cannot be read (gone,
+    refused, replaced by a file) answers nothing and leaves every name in it to
+    its own stat, so it costs speed and never an entry."""
+    try:
+        with os.scandir(directory) as entries:
+            for entry in entries:
+                _keep_wanted(entry, names, out)
+    except OSError:
+        return
+
+
+def _listed_mtimes(root: Path, grouped: dict[str, dict[str, str]]) -> dict[str, float]:
+    out: dict[str, float] = {}
+    for parent, names in grouped.items():
+        _list_directory(root / parent if parent else root, names, out)
+    return out
+
+
+def snapshot_mtimes(root: Path, files: list[str]) -> dict[str, float]:
+    """The mtime of every tracked file that is one; the rest simply absent.
+
+    One os.scandir per DIRECTORY, not one os.stat per FILE. On Windows the
+    times ride in the listing itself, so a 14,152-file tree polled every two
+    seconds dropped from 275 ms of stats to 46 ms of listings; elsewhere the
+    listing at least answers from a directory handle instead of walking the
+    whole path again once per file. Whatever the listing did not answer for
+    still gets its own stat, so a newborn file, a name the filesystem spells
+    with different case, and an unreadable directory all behave exactly as they
+    did when every file was stat-ed.
+
+    The result is built in `files` order, not listing order: two polls of one
+    unchanged tree have to produce the same mapping, and no filesystem promises
+    the order it enumerates in.
+    """
+    listed = _listed_mtimes(root, _by_directory(files))
+    out: dict[str, float] = {}
+    for rel in files:
+        mtime = listed.get(rel)
+        if mtime is None:
+            mtime = _stat_mtime(root / rel)
+        if mtime is not None:
+            out[rel] = mtime
+    return out
+
+
+def changed_paths(before: dict[str, float], after: dict[str, float]) -> list[str]:
+    """New, modified, and deleted paths between two snapshots, sorted."""
+    moved = {p for p, m in after.items() if before.get(p) != m}
+    return sorted(moved | (set(before) - set(after)))