gitfence 0.1.0__py3-none-any.whl

fence/index/formatting.py

@@ -0,0 +1,53 @@
+"""Detect formatting-only commits: every touched file has identical normalized tokens
+before and after. Such commits are skipped by blame (like .git-blame-ignore-revs)
+and never count as evidence.
+
+Checking every commit would cost two blob reads per file across the whole history,
+so only candidates are verified: commits whose message reads like a formatting
+change, or whose every file kept its line count (re-indents, quote swaps). A
+formatter commit that is neither is missed; listing it in .git-blame-ignore-revs
+still works.
+"""
+
+from __future__ import annotations
+
+from fence.git.history import LogEntry
+from fence.git.runner import GitError, Repo
+from fence.lang import tokens_for
+from fence.signals.catalog import signal
+
+MAX_FILES = 200
+_STYLE = signal("text.formatting").pattern
+
+
+def is_candidate(entry: LogEntry) -> bool:
+    if len(entry.parents) != 1 or not entry.changes or len(entry.changes) > MAX_FILES:
+        return False
+    if any(c.status != "M" or c.added is None for c in entry.changes):
+        return False  # adds, deletes, renames and binaries are not formatting
+    worded = bool(_STYLE and _STYLE.search(f"{entry.subject}\n{entry.body}"))
+    return worded or all(c.added == c.deleted for c in entry.changes)
+
+
+def detect_format_only(repo: Repo, entries: list[LogEntry]) -> set[str]:
+    """SHAs of formatting-only commits among `entries`. Two git processes per call."""
+    cands = [e for e in entries if is_candidate(e)]
+    if not cands:
+        return set()
+    specs = [(sha, c.path) for e in cands for c in e.changes for sha in (e.parents[0], e.sha)]
+    try:
+        blobs = repo.read_blobs(specs)
+    except GitError:
+        return set()
+    out = set()
+    for e in cands:
+        parent = e.parents[0]
+        for c in e.changes:
+            old, new = blobs.get((parent, c.path)), blobs.get((e.sha, c.path))
+            if old is None or new is None or old == new:
+                break
+            if tokens_for(c.path, old) != tokens_for(c.path, new):
+                break
+        else:
+            out.add(e.sha)
+    return out

fence/index/ghosts.py

@@ -0,0 +1,143 @@
+"""Ghost Fences (D2): tombstones for deleted guards, and matching code that comes back.
+
+At index time, every first-parent commit that deletes a guard-like block (guard,
+lock, retry/timeout, error handling, assertion) leaves a tombstone: fingerprint,
+removing commit, and the deleted lines. When a later change adds code that
+matches a tombstone, Fence links the two: "this re-adds a guard removed in
+<commit>". The decision the guard carried is resolved lazily, with blame at
+the removing commit's parent, only when a match is shown.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import secrets
+from collections.abc import Iterator
+from dataclasses import dataclass
+
+from fence.anchors.fingerprint import containment, shingles
+from fence.check.classify import GUARD_SIGNALS
+from fence.check.diff import DiffError, parse_diff
+from fence.config import Config
+from fence.git.runner import GitError, Repo
+from fence.index.store import Store
+from fence.lang import plugin_for, tokens_for
+from fence.signals.code import match_code
+from fence.text import clean
+
+MIN_TOKENS = 8
+MATCH = 0.7
+MAX_PER_RUN = 5000
+
+
+@dataclass(frozen=True)
+class Ghost:
+    tombstone_id: str
+    removed_in: str
+    path: str
+    subject: str
+    preview: str
+    similarity: float
+
+
+def _commits(repo: Repo, revs: list[str], timeout: float) -> Iterator[tuple[str, str, str]]:
+    """(sha, subject, diff text) for first-parent commits, streamed one at a time."""
+    nonce = secrets.token_hex(6)
+    marker = f"\x1e{nonce}"
+    args = ["log", "--first-parent", "--diff-merges=first-parent", "-p", "--unified=0",
+            "--no-color", "--no-ext-diff", "--no-textconv", "-M50%", "--src-prefix=a/",
+            "--dst-prefix=b/", f"--format=%x1e{nonce}%H%n%s", *revs, "--"]
+    buf = ""
+    for chunk in repo.stream(args, timeout=timeout):
+        buf += chunk
+        parts = buf.split(marker)
+        buf = parts.pop()
+        for rec in parts:
+            if rec.strip():
+                yield _split(rec)
+    if buf.strip():
+        yield _split(buf)
+
+
+def _split(rec: str) -> tuple[str, str, str]:
+    sha, _, rest = rec.partition("\n")
+    subject, _, diff = rest.partition("\n")
+    return sha.strip(), subject, diff
+
+
+def scan_tombstones(repo: Repo, store: Store, cfg: Config) -> int:
+    """Record tombstones for commits not yet scanned. Returns how many were added."""
+    tip = repo.verify_ref(cfg.index.refs[0]) if cfg.index.refs else repo.verify_ref("HEAD")
+    done = store.get_meta("ghost_tip")
+    revs = [tip] + (["--not", done] if done and _exists(repo, done) else [])
+    if done == tip:
+        return 0
+    added = 0
+    rows = []
+    for sha, subject, diff in _commits(repo, revs, cfg.performance.log_timeout_seconds):
+        if len(diff) > 2_000_000:
+            continue
+        try:
+            files = parse_diff(diff)
+        except DiffError:
+            continue
+        readded = frozenset().union(*(shingles(tokens_for(f.path, "\n".join(f.added_lines)))
+                                      for f in files if f.added_lines)) if files else frozenset()
+        for f in files:
+            if f.old_path is None or plugin_for(f.old_path) is None:
+                continue
+            for b in f.blocks:
+                lines = [t for _, t in b.removed]
+                if not {s for s, _ in match_code(lines)} & GUARD_SIGNALS:
+                    continue
+                toks = tokens_for(f.old_path, "\n".join(lines))
+                if len(toks) < MIN_TOKENS:
+                    continue
+                sh = shingles(toks)
+                if containment(sh, readded) >= MATCH:
+                    continue  # moved, not deleted
+                key = f"{sha}\x00{f.old_path}\x00{b.start}"
+                tid = hashlib.sha256(key.encode()).hexdigest()[:16]
+                rows.append((tid, sha, f.old_path, None, hashlib.sha256(
+                    "\x00".join(toks).encode()).hexdigest()[:24], json.dumps(sorted(sh)),
+                    json.dumps({"subject": clean(subject, max_len=200), "start": b.start,
+                                "preview": clean("\n".join(lines[:6]), max_len=400)})))
+                added += 1
+        if added >= MAX_PER_RUN:
+            break
+    with store.transaction():
+        store.conn.executemany("INSERT OR IGNORE INTO tombstones VALUES(?,?,?,?,?,?,?)", rows)
+        if added < MAX_PER_RUN:
+            store.set_meta("ghost_tip", tip)
+    return added
+
+
+def _exists(repo: Repo, sha: str) -> bool:
+    try:
+        repo.run(["cat-file", "-e", f"{sha}^{{commit}}"])
+        return True
+    except GitError:
+        return False
+
+
+def find_ghosts(store: Store, path: str, added_lines: list[str], *,
+                exclude_shas: frozenset[str] = frozenset()) -> list[Ghost]:
+    """Tombstones whose deleted code is (mostly) re-added by `added_lines`."""
+    if not added_lines:
+        return []
+    added = shingles(tokens_for(path, "\n".join(added_lines)))
+    if not added:
+        return []
+    out = []
+    for row in store.conn.execute("SELECT * FROM tombstones"):
+        if row["sha"] in exclude_shas:
+            continue
+        sh = frozenset(json.loads(row["shingles"]))
+        sim = containment(sh, added)
+        if sim >= MATCH:
+            rec = json.loads(row["record"])
+            out.append(Ghost(row["id"], row["sha"], row["path"], rec["subject"], rec["preview"],
+                             round(sim, 2)))
+    out.sort(key=lambda g: (-g.similarity, g.removed_in))
+    return out[:5]

fence/index/indexer.py

@@ -0,0 +1,219 @@
+"""Incremental history indexing.
+
+Each run indexes only commits not yet stored. If a stored ref tip is no longer an
+ancestor of the current tip (force-push, rebase), the reachable set is recomputed
+and commits that fell out of history are marked orphaned, not deleted: decisions
+that cite them must say their evidence is unreachable.
+
+Refs are saved only after every batch has landed, so an interrupted run simply
+resumes: the next run walks the same range and skips stored commits.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+from fence.config import Config
+from fence.git.history import LogEntry, iter_log
+from fence.git.runner import GitError, Repo
+from fence.globs import included
+from fence.index.store import CommitRow, FileChange, Store
+from fence.signals.catalog import extract_text
+
+_MERGE_PR = re.compile(r"^Merge pull request #(\d+) from ")
+_SQUASH_PR = re.compile(r"\(#(\d+)\)\s*$")
+_REVERT = re.compile(r"This reverts commit ([0-9a-f]{40})")
+
+FormatDetector = Callable[[Repo, list[LogEntry]], set[str]]  # -> format-only SHAs
+Progress = Callable[[int], None]
+
+
+@dataclass
+class IndexReport:
+    indexed: int = 0
+    skipped_existing: int = 0
+    orphaned: int = 0
+    rewrite_detected: bool = False
+    shallow: bool = False
+    empty: bool = False
+    total: int = 0
+    seconds: float = 0.0
+    notes: list[str] = field(default_factory=list)
+
+
+def pr_from_subject(subject: str) -> tuple[int | None, str | None]:
+    if m := _MERGE_PR.match(subject):
+        return int(m.group(1)), "merge-subject"
+    if m := _SQUASH_PR.search(subject):
+        return int(m.group(1)), "squash-subject"
+    return None, None
+
+
+def reverted_sha(body: str) -> str | None:
+    m = _REVERT.search(body)
+    return m.group(1) if m else None
+
+
+def resolve_tips(repo: Repo, refs: list[str]) -> dict[str, str]:
+    tips: dict[str, str] = {}
+    for ref in refs:
+        try:
+            tips[ref] = repo.verify_ref(ref)
+        except GitError:
+            continue  # a missing ref (e.g. empty repo) is reported by the caller
+    return tips
+
+
+def needs_index(repo: Repo, store: Store, cfg: Config) -> bool:
+    """True when the index is empty or a configured tip has commits it has not seen,
+    such as new, amended or squashed commits since the last run."""
+    if store.commit_count(include_orphaned=True) == 0:
+        return True
+    return any(not store.has_commit(sha) for sha in resolve_tips(repo, cfg.index.refs).values())
+
+
+def is_ancestor(repo: Repo, old: str, new: str) -> bool:
+    try:
+        repo.run(["merge-base", "--is-ancestor", old, new])
+        return True
+    except GitError:
+        return False
+
+
+def commit_exists(repo: Repo, sha: str) -> bool:
+    try:
+        repo.run(["cat-file", "-e", f"{sha}^{{commit}}"])
+        return True
+    except GitError:
+        return False
+
+
+def index_repo(
+    repo: Repo,
+    store: Store,
+    cfg: Config,
+    *,
+    detect_format: FormatDetector | None = None,
+    progress: Progress | None = None,
+) -> IndexReport:
+    started = time.monotonic()
+    report = IndexReport()
+    report.shallow = repo.run(["rev-parse", "--is-shallow-repository"]).strip() == "true"
+    if report.shallow:
+        report.notes.append("shallow clone: history before the shallow boundary is not visible")
+
+    tips = resolve_tips(repo, cfg.index.refs)
+    if not tips:
+        report.empty = True
+        report.notes.append("no commits found for " + ", ".join(cfg.index.refs))
+        return report
+
+    old_refs = store.refs()
+    exclude_old: list[str] = []
+    for name, new in tips.items():
+        old = old_refs.get(name)
+        if not old or old == new:
+            if old:
+                exclude_old.append(old)
+            continue
+        if commit_exists(repo, old) and is_ancestor(repo, old, new):
+            exclude_old.append(old)
+        else:
+            report.rewrite_detected = True
+
+    if report.rewrite_detected:
+        reachable = repo.run(["rev-list", *tips.values()]).split()
+        with store.transaction():
+            report.orphaned = store.mark_orphans(reachable)
+        report.notes.append(
+            f"history was rewritten (force-push or rebase); {report.orphaned} commits orphaned"
+        )
+
+    extra: list[str] = []
+    if cfg.index.history_depth:
+        extra.append(f"--max-count={cfg.index.history_depth}")
+    if cfg.index.since_date:
+        if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", cfg.index.since_date):
+            raise GitError(f"index.since_date must be YYYY-MM-DD, got {cfg.index.since_date!r}")
+        extra.append(f"--since={cfg.index.since_date}")
+    revs = [*dict.fromkeys(tips.values())]
+    if exclude_old:
+        revs += ["--not", *dict.fromkeys(exclude_old)]
+
+    batch: list[LogEntry] = []
+    for entry in iter_log(
+        repo, revs, extra=extra, rename_threshold=cfg.index.follow_renames_sensitivity,
+        timeout=cfg.performance.log_timeout_seconds,
+    ):
+        if store.has_commit(entry.sha):
+            report.skipped_existing += 1
+            continue
+        batch.append(entry)
+        if len(batch) >= cfg.index.batch_size:
+            _write_batch(repo, store, cfg, batch, detect_format)
+            report.indexed += len(batch)
+            batch = []
+            if progress:
+                progress(report.indexed)
+    if batch:
+        _write_batch(repo, store, cfg, batch, detect_format)
+        report.indexed += len(batch)
+        if progress:
+            progress(report.indexed)
+
+    with store.transaction():
+        for name, sha in tips.items():
+            store.set_ref(name, sha)
+        store.set_meta("last_indexed_at", str(int(time.time())))
+        store.set_meta("shallow", "1" if report.shallow else "0")
+        if report.indexed or report.rewrite_detected:
+            store.clear_query_cache()
+    report.total = store.commit_count()
+    report.seconds = round(time.monotonic() - started, 3)
+    return report
+
+
+def _write_batch(
+    repo: Repo, store: Store, cfg: Config, batch: list[LogEntry],
+    detect_format: FormatDetector | None,
+) -> None:
+    rows: list[CommitRow] = []
+    changes: list[FileChange] = []
+    signals: list[tuple[str, str, str]] = []
+    reverts: list[tuple[str, str]] = []
+    fmt_shas = detect_format(repo, batch) if detect_format else set()
+    for e in batch:
+        pr, pr_source = pr_from_subject(e.subject)
+        kept = [
+            c for c in e.changes
+            if included(c.path, cfg.index.include, cfg.index.exclude)
+            or (c.old_path and included(c.old_path, cfg.index.include, cfg.index.exclude))
+        ]
+        format_only = bool(kept) and e.sha in fmt_shas
+        rows.append(
+            CommitRow(
+                sha=e.sha, parents=e.parents, author_name=e.author_name,
+                author_email=e.author_email, author_time=e.author_time,
+                committer_time=e.committer_time, subject=e.subject, body=e.body,
+                pr_number=pr, pr_source=pr_source, files_changed=len(e.changes),
+                format_only=format_only, bulk=len(e.changes) >= cfg.index.bulk_commit_files,
+            )
+        )
+        changes.extend(
+            FileChange(e.sha, c.path, c.old_path, c.status, c.similarity, c.added, c.deleted)
+            for c in kept
+        )
+        skip = frozenset({f"#{pr}"}) if pr else frozenset()
+        signals.extend((e.sha, sid, detail) for sid, detail in extract_text(
+            f"{e.subject}\n\n{e.body}", skip_refs=skip
+        ))
+        if target := reverted_sha(e.body):
+            reverts.append((e.sha, target))
+    with store.transaction():
+        store.put_commits(rows, changes)
+        store.put_signals(signals)
+        for sha, target in reverts:
+            store.put_revert(sha, target)