gitfence 0.1.0__py3-none-any.whl

fence/__init__.py

@@ -0,0 +1,3 @@
+"""Fence: records why code exists and warns before it is removed."""
+
+__version__ = "0.1.0"

fence/anchors/fingerprint.py

@@ -0,0 +1,69 @@
+"""Content fingerprints: find the same code again after it moves or is reformatted.
+
+`exact` is a hash of the normalized token stream: equal means "same code,
+modulo layout, comments and quote style". `shingles` are hashed k-token windows:
+their Jaccard similarity says how much code two snippets share, which survives
+renames of the surrounding file, moves between files and small edits.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+SHINGLE_K = 5
+SAME_CODE_JACCARD = 0.7
+
+
+@dataclass(frozen=True)
+class Fingerprint:
+    exact: str
+    shingles: frozenset[int]
+    token_count: int
+
+    def to_json(self) -> dict:
+        return {"exact": self.exact, "shingles": sorted(self.shingles), "n": self.token_count}
+
+    @classmethod
+    def from_json(cls, d: dict) -> Fingerprint:
+        return cls(d["exact"], frozenset(d["shingles"]), d["n"])
+
+
+def fingerprint(tokens: Sequence[str], k: int = SHINGLE_K) -> Fingerprint:
+    joined = "\x00".join(tokens).encode("utf-8", errors="replace")
+    exact = hashlib.sha256(joined).hexdigest()[:24]
+    return Fingerprint(exact, shingles(tokens, k), len(tokens))
+
+
+def shingles(tokens: Sequence[str], k: int = SHINGLE_K) -> frozenset[int]:
+    if not tokens:
+        return frozenset()
+    if len(tokens) < k:
+        windows = [tuple(tokens)]
+    else:
+        windows = [tuple(tokens[i : i + k]) for i in range(len(tokens) - k + 1)]
+    out = set()
+    for w in windows:
+        h = hashlib.blake2b("\x00".join(w).encode("utf-8", "replace"), digest_size=8)
+        out.add(int.from_bytes(h.digest(), "big") >> 1)  # fits in a signed 64-bit int
+    return frozenset(out)
+
+
+def jaccard(a: frozenset[int], b: frozenset[int]) -> float:
+    if not a and not b:
+        return 1.0
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def containment(needle: frozenset[int], haystack: frozenset[int]) -> float:
+    """Share of `needle`'s shingles found in `haystack` (moved code inside a bigger file)."""
+    if not needle:
+        return 0.0
+    return len(needle & haystack) / len(needle)
+
+
+def same_code(a: Fingerprint, b: Fingerprint, threshold: float = SAME_CODE_JACCARD) -> bool:
+    return a.exact == b.exact or jaccard(a.shingles, b.shingles) >= threshold

fence/check/classify.py

@@ -0,0 +1,84 @@
+"""What kind of change is a block? delete > loosen > alter > move (by risk).
+
+- move:   the removed code reappears elsewhere in the diff with its guards intact.
+- loosen: a guard, lock, retry, timeout, assertion or error handler is removed or
+          weakened (its code signal disappears, or a number on such a line changes),
+          including a move that silently drops one.
+- delete: nothing meaningful replaces the removed lines.
+- alter:  anything else (expressions or constants changed).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from fence.anchors.fingerprint import containment, jaccard, shingles
+from fence.check.diff import Block
+from fence.lang import tokens_for
+from fence.signals.code import match_code
+
+GUARD_SIGNALS = {
+    "code.guard", "code.lock", "code.retry_timeout", "code.exception_handling",
+    "code.assertion", "code.feature_flag",
+}
+_NUMBER = re.compile(r"(?<![\w.])\d+(?:\.\d+)?(?![\w.])")
+MOVE_CONTAINMENT = 0.8
+DELETE_SIMILARITY = 0.3
+
+
+@dataclass(frozen=True)
+class Classification:
+    kind: str  # delete | loosen | alter | move
+    reason: str
+    lost_signals: tuple[str, ...] = ()
+
+
+def classify(path: str, block: Block, diff_added: frozenset[int]) -> Classification:
+    """`diff_added` is the shingle set of every added line in the whole diff."""
+    removed = [t for _, t in block.removed]
+    added = [t for _, t in block.added]
+    r_tokens = tokens_for(path, "\n".join(removed))
+    a_tokens = tokens_for(path, "\n".join(added))
+    if not r_tokens:
+        return Classification("alter", "only blank lines or comments removed")
+    r_sigs = {s for s, _ in match_code(removed)} & GUARD_SIGNALS
+    a_sigs = {s for s, _ in match_code(added)} & GUARD_SIGNALS
+
+    r_sh = shingles(r_tokens)
+    if len(r_tokens) >= 4 and containment(r_sh, diff_added) >= MOVE_CONTAINMENT and \
+            jaccard(r_sh, shingles(a_tokens)) < MOVE_CONTAINMENT:
+        moved_sigs = r_sigs - a_sigs
+        dest_ok = all(_signal_survives(s, diff_added, path, removed) for s in moved_sigs)
+        if dest_ok:
+            return Classification("move", "the removed code reappears elsewhere in this change")
+
+    lost = tuple(sorted(r_sigs - a_sigs))
+    if not a_tokens:
+        return Classification("delete", "lines removed with no replacement", lost)
+    if jaccard(r_sh, shingles(a_tokens)) < DELETE_SIMILARITY and len(r_tokens) >= 8:
+        return Classification("delete", "replaced by unrelated code", lost)
+    if lost:
+        names = ", ".join(s.split(".", 1)[1].replace("_", " ") for s in lost)
+        return Classification("loosen", f"removes or weakens: {names}", lost)
+    if _guard_number_changed(removed, added):
+        return Classification("loosen", "a retry, timeout, lock or guard value changed")
+    return Classification("alter", "code changed in place")
+
+
+def _signal_survives(sid: str, diff_added: frozenset[int], path: str, removed: list[str]) -> bool:
+    # The guard line itself must be among the moved shingles.
+    guard_lines = [t for t in removed if sid in {s for s, _ in match_code([t])}]
+    if not guard_lines:
+        return True
+    sh = shingles(tokens_for(path, "\n".join(guard_lines)))
+    return containment(sh, diff_added) >= MOVE_CONTAINMENT
+
+
+def _guard_number_changed(removed: list[str], added: list[str]) -> bool:
+    guarded = [t for t in removed if {s for s, _ in match_code([t])} & GUARD_SIGNALS]
+    if not guarded:
+        return False
+    before = [_NUMBER.findall(t) for t in guarded]
+    after_all = [_NUMBER.findall(t) for t in added]
+    return any(nums and nums not in after_all for nums in before)

fence/check/diff.py

@@ -0,0 +1,127 @@
+"""Unified diff -> change blocks: runs of removed lines and the lines that replaced them.
+
+A block is the unit Fence judges: "these base lines were removed, and these lines
+took their place" (possibly none). Context lines end a block.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from unidiff import PatchSet
+from unidiff.errors import UnidiffParseError
+
+from fence.git.runner import GitError, Repo
+
+MAX_DIFF_BYTES = 20_000_000
+MAX_FILES = 2_000
+
+
+class DiffError(Exception):
+    """The diff could not be produced or parsed."""
+
+
+@dataclass
+class Block:
+    removed: list[tuple[int, str]]  # (base line number, text)
+    added: list[tuple[int, str]]  # (new line number, text)
+
+    @property
+    def start(self) -> int:
+        return self.removed[0][0]
+
+    @property
+    def end(self) -> int:
+        return self.removed[-1][0]
+
+
+@dataclass
+class FileDiff:
+    old_path: str | None  # None for added files
+    new_path: str | None  # None for deleted files
+    status: str  # added | deleted | modified | renamed
+    blocks: list[Block] = field(default_factory=list)
+    added_lines: list[str] = field(default_factory=list)  # every added line, for move detection
+    binary: bool = False
+
+    @property
+    def path(self) -> str:
+        return self.new_path or self.old_path or ""
+
+
+def _strip(p: str | None, prefix: str) -> str | None:
+    if p is None or p == "/dev/null":
+        return None
+    return p[len(prefix):] if p.startswith(prefix) else p
+
+
+def parse_diff(text: str) -> list[FileDiff]:
+    if len(text.encode("utf-8", "replace")) > MAX_DIFF_BYTES:
+        raise DiffError(f"diff is larger than {MAX_DIFF_BYTES // 1_000_000} MB")
+    try:
+        patch = PatchSet(text)
+    except UnidiffParseError as e:
+        raise DiffError(f"could not parse diff: {e}") from None
+    if len(patch) > MAX_FILES:
+        raise DiffError(f"diff touches more than {MAX_FILES} files")
+    out: list[FileDiff] = []
+    for pf in patch:
+        old = _strip(pf.source_file, "a/")
+        new = _strip(pf.target_file, "b/")
+        status = ("added" if pf.is_added_file else "deleted" if pf.is_removed_file
+                  else "renamed" if pf.is_rename else "modified")
+        fd = FileDiff(old, new, status, binary=pf.is_binary_file)
+        for hunk in pf:
+            removed: list[tuple[int, str]] = []
+            added: list[tuple[int, str]] = []
+            for line in hunk:
+                value = line.value[:-1] if line.value.endswith("\n") else line.value
+                if line.is_removed:
+                    if added:  # a '-' after '+' starts a new block
+                        _flush(fd, removed, added)
+                        removed, added = [], []
+                    removed.append((line.source_line_no, value))
+                elif line.is_added:
+                    added.append((line.target_line_no, value))
+                    fd.added_lines.append(value)
+                else:
+                    _flush(fd, removed, added)
+                    removed, added = [], []
+            _flush(fd, removed, added)
+        out.append(fd)
+    return out
+
+
+def _flush(fd: FileDiff, removed: list, added: list) -> None:
+    if removed:
+        fd.blocks.append(Block(list(removed), list(added)))
+
+
+@dataclass(frozen=True)
+class DiffSpec:
+    """What to check. `base` is the revision removed lines are traced at."""
+
+    base: str
+    head: str | None = None  # None: working tree (or index with staged=True)
+    staged: bool = False
+
+
+def produce_diff(repo: Repo, spec: DiffSpec, *, rename_threshold: int = 50) -> tuple[str, str]:
+    """Run git diff for `spec`. Returns (base_sha, diff_text)."""
+    base = repo.verify_ref(spec.base)
+    args = ["diff", "--no-color", "--no-ext-diff", "--no-textconv", f"-M{rename_threshold}%",
+            "--src-prefix=a/", "--dst-prefix=b/", "--unified=3"]
+    if spec.staged:
+        args.append("--cached")
+    args.append(base)
+    if spec.head:
+        args.append(repo.verify_ref(spec.head))
+    args.append("--")
+    try:
+        return base, "".join(repo.stream(args, timeout=max(repo.timeout, 60)))
+    except GitError as e:
+        raise DiffError(str(e)) from None
+
+
+def merge_base(repo: Repo, a: str, b: str) -> str:
+    return repo.run(["merge-base", repo.verify_ref(a), repo.verify_ref(b)]).strip()

fence/check/engine.py

@@ -0,0 +1,263 @@
+"""The fence check: a diff in, a justified verdict per change block out.
+
+Verdicts:
+  KEEP     strong, active decision contradicted: stop and verify (can fail CI)
+  VERIFY   a decision is touched and its reason is only partly known: review it
+  CLEAR    nothing meaningful affected
+  SAFE-TO-REMOVE-CANDIDATE  the reason is known and evidence says it is obsolete
+
+Only KEEP can fail a run (exit 2 or 3), and only for grades at or above
+scoring.grade_floor_for_block: "reason unknown" never blocks anyone.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+from fence.anchors.fingerprint import fingerprint, shingles
+from fence.check.classify import Classification, classify
+from fence.check.diff import DiffError, DiffSpec, FileDiff, parse_diff, produce_diff
+from fence.config import BANDS, Config, PathOverride
+from fence.context import Context
+from fence.decisions.build import TEST_PATH, Target, decisions_for_lines, format_only_shas
+from fence.decisions.model import DecisionRecord, Score
+from fence.diffing.ranges import RemovedRange
+from fence.git.blame import MAX_LINES
+from fence.git.runner import GitError
+from fence.globs import match_any
+from fence.index.ghosts import find_ghosts
+from fence.lang import tokens_for
+from fence.policy import acks as ackmod
+from fence.scoring.strength import change_risk
+from fence.text import clean
+from fence.trace import PreimageMismatch, trace
+
+KEEP, VERIFY, CLEAR, SAFE = "KEEP", "VERIFY", "CLEAR", "SAFE-TO-REMOVE-CANDIDATE"
+VERDICT_ORDER = (CLEAR, SAFE, VERIFY, KEEP)
+GRADE_ORDER = "ABCD"
+
+EXIT_OK, EXIT_ERROR, EXIT_POLICY, EXIT_ACK, EXIT_NO_INDEX = 0, 1, 2, 3, 4
+
+
+@dataclass
+class Finding:
+    path: str
+    start: int  # base line numbers of the removed block
+    end: int
+    kind: str
+    kind_reason: str
+    verdict: str
+    risk: Score
+    decision: DecisionRecord
+    decision_fingerprint: str
+    removed_preview: str
+    lost_signals: tuple[str, ...] = ()
+    ack: ackmod.Ack | None = None
+
+    @property
+    def blocking(self) -> bool:
+        return self.verdict == KEEP and self.ack is None
+
+
+@dataclass
+class CheckResult:
+    base: str
+    head: str | None
+    findings: list[Finding] = field(default_factory=list)
+    verdict: str = CLEAR
+    exit_code: int = EXIT_OK
+    files_checked: int = 0
+    blocks_checked: int = 0
+    notes: list[str] = field(default_factory=list)
+    bypassed: bool = False
+    ghosts: list = field(default_factory=list)  # (path, Ghost)
+    seconds: float = 0.0
+
+
+def effective_policy(cfg: Config, path: str) -> tuple[str, str, int, str]:
+    """(fail_on_band, require_ack_for, risk_bonus, reason) after path overrides.
+    When several overrides match, the strictest value of each wins."""
+    fail_on, ack_for, bonus, why = cfg.policy.fail_on_band, cfg.policy.require_ack_for, 0, ""
+    for pattern, ov in cfg.policy.path_overrides.items():
+        if not match_any(path, [pattern]):
+            continue
+        ov: PathOverride
+        if ov.fail_on_band and BANDS.index(ov.fail_on_band) < BANDS.index(fail_on):
+            fail_on = ov.fail_on_band
+        if ov.require_ack_for and BANDS.index(ov.require_ack_for) < BANDS.index(ack_for):
+            ack_for = ov.require_ack_for
+        if ov.risk_bonus > bonus:
+            bonus, why = ov.risk_bonus, f"sensitive path ({pattern})"
+    return fail_on, ack_for, bonus, why
+
+
+def check(
+    ctx: Context, spec: DiffSpec | None = None, *, diff_text: str | None = None,
+    base: str | None = None, deep: bool = True, include_clear: bool = False,
+) -> CheckResult:
+    started = time.monotonic()
+    if diff_text is None:
+        spec = spec or DiffSpec(base="HEAD")
+        base_sha, diff_text = produce_diff(ctx.repo, spec,
+                                           rename_threshold=ctx.cfg.index.follow_renames_sensitivity)
+        head = spec.head
+    else:
+        base_sha = ctx.repo.verify_ref(base or "HEAD")
+        head = None
+    result = CheckResult(base=base_sha, head=head)
+    files = parse_diff(diff_text)
+    ledger = ctx.repo.root / ctx.cfg.policy.ack_ledger
+    acks, problems = ackmod.load(ledger)
+    result.notes += problems
+    if ctx.store.commit_count() == 0:
+        result.notes.append("index is empty: run `fence index` for full evidence")
+
+    diff_added = frozenset().union(*(
+        shingles(tokens_for(f.path, "\n".join(f.added_lines))) for f in files if f.added_lines
+    )) if files else frozenset()
+    tests_removed = any(
+        (f.status == "deleted" or f.blocks) and TEST_PATH.search(f.old_path or "")
+        for f in files
+    )
+    now = datetime.now(UTC)
+    for fd in files:
+        if not fd.blocks or fd.old_path is None or fd.binary:
+            continue
+        if match_any(fd.old_path, ctx.cfg.policy.ignore_paths):
+            continue
+        result.files_checked += 1
+        result.blocks_checked += len(fd.blocks)
+        try:
+            result.findings += _check_file(ctx, fd, base_sha, diff_added, tests_removed,
+                                           acks, now, result.notes, deep=deep,
+                                           include_clear=include_clear)
+        except PreimageMismatch as e:
+            raise DiffError(str(e)) from None
+    for fd in files:
+        if fd.added_lines and not match_any(fd.path, ctx.cfg.policy.ignore_paths):
+            result.ghosts += [(fd.path, g)
+                              for g in find_ghosts(ctx.store, fd.path, fd.added_lines)]
+
+    # Within equal risk, blocks that drop a guard/retry/lock come first: they are the
+    # lines a reviewer most needs to see.
+    result.findings.sort(key=lambda f: (-VERDICT_ORDER.index(f.verdict), -f.risk.value,
+                                        -len(f.lost_signals), f.path, f.start))
+    if result.findings:
+        result.verdict = result.findings[0].verdict
+    bypass_var = ctx.cfg.policy.allow_bypass_env
+    result.bypassed = bool(bypass_var and os.environ.get(bypass_var))
+    result.exit_code = _exit_code(ctx.cfg, result)
+    result.seconds = round(time.monotonic() - started, 3)
+    return result
+
+
+def _exit_code(cfg: Config, result: CheckResult) -> int:
+    if result.bypassed:
+        return EXIT_OK
+    code = EXIT_OK
+    for f in result.findings:
+        if not f.blocking:
+            continue
+        _, ack_for, _, _ = effective_policy(cfg, f.path)
+        if BANDS.index(f.risk.band) >= BANDS.index(ack_for):
+            return EXIT_ACK
+        code = EXIT_POLICY
+    return code
+
+
+def _check_file(
+    ctx: Context, fd: FileDiff, base_sha: str, diff_added: frozenset[int], tests_removed: bool,
+    acks: list[ackmod.Ack], now: datetime, notes: list[str], *, deep: bool,
+    include_clear: bool = False,
+) -> list[Finding]:
+    path = fd.old_path or ""
+    try:
+        text = ctx.repo.read_blob(base_sha, path, max_bytes=ctx.cfg.index.max_file_size)
+    except GitError as e:
+        notes.append(f"{path}: skipped ({e})")
+        return []
+    base_lines = text.split("\n")
+    blocks = fd.blocks
+    total = sum(b.end - b.start + 1 for b in blocks)
+    if total > MAX_LINES:
+        kept, n = [], 0
+        for b in blocks:
+            if n + (b.end - b.start + 1) > MAX_LINES:
+                break
+            kept.append(b)
+            n += b.end - b.start + 1
+        notes.append(f"{path}: {total} removed lines; only the first {n} were traced")
+        blocks = kept
+    ranges = [RemovedRange(path, b.start, b.end, tuple(t for _, t in b.removed)) for b in blocks]
+    history = ctx.store.path_history(path)
+    traced = trace(ctx.repo, base_sha, ranges, extra_ignore=format_only_shas(ctx, history))
+    blamed = [bl for t in traced for bl in t.lines]
+    target = Target(path, blocks[0].start, blocks[-1].end, None, base_sha)
+    decisions = decisions_for_lines(ctx, target, base_lines, blamed, history, deep=deep)
+    owner: dict[int, DecisionRecord] = {}
+    for d in decisions:
+        for a in d.anchors:
+            for n in range(a.start_line, a.end_line + 1):
+                owner[n] = d
+
+    fail_on, _, bonus, bonus_reason = effective_policy(ctx.cfg, path)
+    findings: list[Finding] = []
+    for b in blocks:
+        cls = classify(path, b, diff_added)
+        touched = {owner[n].id: owner[n] for n, _ in b.removed if n in owner}
+        best: Finding | None = None
+        for d in touched.values():
+            risk = change_risk(d.strength, cls.kind, ctx.cfg, tests_removed=tests_removed,
+                               path_bonus=bonus, path_bonus_reason=bonus_reason,
+                               low_signal=match_any(path, ctx.cfg.scoring.low_signal_paths))
+            f = Finding(
+                path=path, start=b.start, end=b.end, kind=cls.kind, kind_reason=cls.reason,
+                verdict=_verdict(ctx.cfg, d, risk, fail_on, cls), risk=risk, decision=d,
+                decision_fingerprint=fingerprint(tokens_for(path, "\n".join(
+                    t for n, t in b.removed if owner.get(n) is d))).exact,
+                removed_preview=clean("\n".join(t for _, t in b.removed[:6]), max_len=400),
+                lost_signals=cls.lost_signals,
+            )
+            if best is None or (VERDICT_ORDER.index(f.verdict), f.risk.value) > (
+                    VERDICT_ORDER.index(best.verdict), best.risk.value):
+                best = f
+        if best is None or (best.verdict == CLEAR and not include_clear):
+            continue
+        best.ack = ackmod.find_valid(acks, best.decision.id, best.decision_fingerprint, now)
+        findings.append(best)
+    return findings
+
+
+def _verdict(cfg: Config, d: DecisionRecord, risk: Score, fail_on: str,
+             cls: Classification) -> str:
+    if d.status == "stale-suspected" and d.grade in ("A", "B"):
+        return SAFE
+    can_block = (
+        GRADE_ORDER.index(d.grade) <= GRADE_ORDER.index(cfg.scoring.grade_floor_for_block)
+        and d.status == "active"
+        and risk.confidence >= cfg.scoring.min_confidence_to_warn
+    )
+    if can_block and BANDS.index(risk.band) >= BANDS.index(fail_on):
+        return KEEP
+    # VERIFY needs scoring.verify_min_risk, except a stated reason (A/B) of a high-stakes
+    # type, which warns from the medium band. Backtest: other A/B warnings below 50 were
+    # 60 false alarms for 2 regrets (DECISIONS.md).
+    stated_high = d.grade in ("A", "B") and d.type in cfg.scoring.high_stakes_types
+    floor = cfg.scoring.bands.medium if stated_high else cfg.scoring.verify_min_risk
+    if risk.value >= floor and cls.kind != "move":
+        return VERIFY
+    return CLEAR
+
+
+def _decision_fingerprint(path: str, base_lines: list[str], d: DecisionRecord) -> str:
+    owned = [base_lines[n - 1] for a in d.anchors for n in range(a.start_line, a.end_line + 1)
+             if n - 1 < len(base_lines)]
+    return fingerprint(tokens_for(path, "\n".join(owned))).exact
+
+
+def ledger_path(ctx: Context) -> Path:
+    return ctx.repo.root / ctx.cfg.policy.ack_ledger