gitfence 0.1.0__py3-none-any.whl

fence/decisions/model.py

@@ -0,0 +1,117 @@
+"""Decision Records and their evidence: the objects every interface returns.
+
+JSON output is versioned (`SCHEMA`). A field is never renamed or retyped within a
+major version; new optional fields may be added.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Literal
+
+SCHEMA = "fence/v1"
+# Shown wherever Strength or Change Risk is output. The pooled backtest's lift
+# interval includes 1 (README, "Measured results"), so the scores claim no accuracy.
+SCORE_NOTE = ("Experimental: Strength and Change Risk are explained sums of evidence "
+              "factors, not probabilities. Backtests have not established that they "
+              "predict which removals are later regretted.")
+
+EvidenceType = Literal[
+    "commit", "pr", "issue", "review_comment", "revert", "code_comment", "test", "ticket",
+    "incident_link", "obsolescence",
+]
+RetrievalStatus = Literal["ok", "not_fetched", "unreachable", "orphaned", "not_indexed"]
+Status = Literal["active", "superseded", "reverted", "orphaned", "stale-suspected"]
+Grade = Literal["A", "B", "C", "D"]
+
+
+@dataclass(frozen=True)
+class Evidence:
+    id: str  # E1, E2 ... unique within one response
+    type: EvidenceType
+    ref: str  # SHA, "#123", "path:line@rev", or a URL
+    url: str | None
+    timestamp: int | None  # unix seconds
+    author: str | None
+    quote: str | None  # verbatim, cleaned for display, length-capped
+    retrieval_status: RetrievalStatus = "ok"
+
+
+@dataclass(frozen=True)
+class Anchor:
+    path: str
+    start_line: int
+    end_line: int
+    symbol: str | None
+    language: str | None
+    path_history: tuple[str, ...] = ()
+    fingerprint: str | None = None
+
+
+@dataclass(frozen=True)
+class Factor:
+    id: str
+    points: float
+    reason: str  # plain English
+
+
+@dataclass(frozen=True)
+class Score:
+    value: int
+    band: str
+    confidence: float
+    factors: tuple[Factor, ...]
+    what_would_change: tuple[str, ...] = ()
+    # Change Risk only: regret rate from a shipped calibration table and its support.
+    # No table ships (the pooled backtest did not support one), so both stay None.
+    regret_rate: float | None = None
+    regret_n: int | None = None
+
+    def top(self, n: int = 3) -> tuple[Factor, ...]:
+        return tuple(sorted(self.factors, key=lambda f: (-abs(f.points), f.id))[:n])
+
+
+@dataclass
+class DecisionRecord:
+    id: str
+    anchors: list[Anchor]
+    type: str
+    grade: Grade
+    stated_reason: str | None  # verbatim quote or None, never paraphrase
+    stated_reason_evidence: str | None  # evidence id the quote comes from
+    inferred_reason: str | None  # always prefixed "Inferred:"
+    constraints: list[str]
+    alternatives_considered: list[str]
+    evidence: list[Evidence]
+    introduced_sha: str
+    introduced_at: int
+    introduced_path: str
+    last_touched_at: int
+    authors: list[str]
+    status: Status
+    strength: Score
+    signals: list[tuple[str, str]] = field(default_factory=list)
+    later_touches: list[str] = field(default_factory=list)  # SHAs, newest first
+    supersedes: list[str] = field(default_factory=list)
+    superseded_by: list[str] = field(default_factory=list)
+    pr_number: int | None = None
+    # Where the reason applies: "code" when the evidence is about these lines (a
+    # focused change, or text naming this code), "change" when only the change as a
+    # whole states it (then it is never stated_reason), None when there is no reason.
+    reason_scope: str | None = None
+    change_reason: str | None = None  # verbatim, change-level; see reason_scope
+
+    @property
+    def reason_text(self) -> str:
+        if self.stated_reason:
+            return self.stated_reason
+        if self.inferred_reason:
+            return self.inferred_reason
+        return "Reason unknown: no stated or inferable reason found in history."
+
+    def to_dict(self) -> dict:
+        d = asdict(self)
+        d["strength"]["top_factors"] = [asdict(f) for f in self.strength.top()]
+        d["signals"] = [{"id": s, "detail": det} for s, det in self.signals]
+        d["reason"] = self.reason_text
+        return d

fence/decisions/obsolete.py

@@ -0,0 +1,121 @@
+"""Obsolescence (F7/D3): concrete evidence that a fence may no longer be needed.
+
+Only concrete, checkable facts mark a decision stale-suspected:
+- a dependency its commit names was removed from the manifests, or upgraded;
+- a feature flag its lines read no longer appears anywhere else in the repo;
+- the function it lives in has no remaining references.
+The output is always "possibly obsolete, verify", never an instruction.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import tomllib
+
+from fence.git.runner import GitError, Repo
+
+MANIFESTS = ("requirements.txt", "pyproject.toml", "package.json", "go.mod", "setup.py")
+_REQ = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*(?:[=<>~!]=?\s*([\w.]+))?")
+_GO = re.compile(r"^\s*(?:require\s+)?([\w./-]+\.[\w./-]+)\s+v([\w.+-]+)", re.M)
+_FLAG = re.compile(r"(?:flags?\.(\w+)|is_?enabled\(\s*['\"]([\w.-]+)['\"])", re.I)
+
+
+def _version_key(v: str | None) -> tuple:
+    return tuple(int(x) if x.isdigit() else 0 for x in re.split(r"[.+-]", v or "")[:4])
+
+
+def manifests(repo: Repo, rev: str) -> dict[str, str | None]:
+    """{package: version or None} from the manifests at `rev` (root directory only)."""
+    deps: dict[str, str | None] = {}
+    for name in MANIFESTS:
+        try:
+            text = repo.read_blob(rev, name, max_bytes=500_000)
+        except GitError:
+            continue
+        if name == "package.json":
+            try:
+                data = json.loads(text)
+            except ValueError:
+                continue
+            for sec in ("dependencies", "devDependencies", "peerDependencies"):
+                for k, v in (data.get(sec) or {}).items():
+                    deps[k.lower()] = re.sub(r"^[^\d]*", "", str(v)) or None
+        elif name == "go.mod":
+            for m in _GO.finditer(text):
+                deps[m.group(1).lower()] = m.group(2)
+        elif name == "pyproject.toml":
+            try:
+                data = tomllib.loads(text)
+            except tomllib.TOMLDecodeError:
+                continue
+            reqs = list((data.get("project") or {}).get("dependencies") or [])
+            for group in ((data.get("project") or {}).get("optional-dependencies") or {}).values():
+                reqs += list(group)
+            for r in reqs:
+                if m := _REQ.match(str(r)):
+                    deps[m.group(1).lower()] = m.group(2)
+        else:
+            for line in text.splitlines():
+                if line.strip().startswith(("#", "-")):
+                    continue
+                if m := _REQ.match(line):
+                    deps[m.group(1).lower()] = m.group(2)
+    return deps
+
+
+def dependency_findings(repo: Repo, intro_sha: str, rev: str, message: str,
+                        cache: dict) -> list[str]:
+    then = cache.setdefault(("m", intro_sha), manifests(repo, intro_sha))
+    now = cache.setdefault(("m", rev), manifests(repo, rev))
+    out = []
+    text = message.lower()
+    for pkg, v_then in sorted(then.items()):
+        if len(pkg) < 3 or not re.search(rf"(?<![\w-]){re.escape(pkg)}(?![\w-])", text):
+            continue
+        if pkg not in now:
+            out.append(f"dependency '{pkg}' named in the commit has since been removed")
+        elif v_then and now[pkg] and _version_key(now[pkg]) > _version_key(v_then):
+            out.append(f"dependency '{pkg}' named in the commit was upgraded "
+                       f"from {v_then} to {now[pkg]}")
+    return out
+
+
+def flag_findings(repo: Repo, rev: str, lines: list[str], path: str) -> list[str]:
+    out = []
+    for line in lines:
+        for m in _FLAG.finditer(line):
+            flag = m.group(1) or m.group(2)
+            if not flag or len(flag) < 4:
+                continue
+            hits = _grep_count(repo, rev, flag)
+            if hits is not None and hits <= 1:
+                out.append(f"feature flag '{flag}' is not referenced anywhere else")
+    return out
+
+
+def caller_findings(repo: Repo, rev: str, symbol: str | None, path: str = "") -> list[str]:
+    if not symbol or symbol == "<module>":
+        return []
+    name = symbol.rsplit(".", 1)[-1]
+    if len(name) < 4 or name.startswith("__") or name in ("main", "init", "setup", "test"):
+        return []
+    # Only private symbols: a public function may be called by code outside this repo.
+    private = (path.endswith((".py", ".pyi")) and name.startswith("_")) or (
+        path.endswith(".go") and name[0].islower())
+    if not private:
+        return []
+    hits = _grep_count(repo, rev, name)
+    if hits is not None and hits <= 1:
+        return [f"'{symbol}' has no remaining references in the repository"]
+    return []
+
+
+def _grep_count(repo: Repo, rev: str, word: str) -> int | None:
+    if not re.fullmatch(r"[\w.-]+", word):
+        return None
+    try:
+        out = repo.run(["grep", "-c", "-w", "-F", "-e", word, rev, "--"])
+    except GitError:
+        return 0  # git grep exits 1 when nothing matches
+    return sum(int(line.rsplit(":", 1)[1]) for line in out.splitlines() if ":" in line)

fence/decisions/reasons.py

@@ -0,0 +1,150 @@
+"""Find an explicitly stated reason in untrusted text, and quote it verbatim.
+
+A "stated reason" is a sentence that explains causation ("because", "otherwise",
+"produced duplicate charges"). Fence never paraphrases it: it returns the exact
+sentence (cleaned of control characters) or nothing. Sentences that try to talk
+to a tool ("ignore previous instructions") are dropped, never quoted.
+"""
+
+from __future__ import annotations
+
+import re
+
+from fence.text import clean
+
+MAX_QUOTE = 300
+
+_REASON = re.compile(
+    r"\b(because|since|so that|otherwise|or else|to (prevent|avoid|stop|ensure|guard|protect)|"
+    r"prevents?|prevented|caus(e|es|ed|ing)|produc(es|ed|ing)|led to|leads to|"
+    r"result(s|ed)? in|in order to|due to|required (by|for)|needed (by|for)|"
+    r"without (this|it)|was (failing|crashing|breaking|timing out)|broke|breaks|"
+    # purpose clauses: "... so a typo cannot weaken policy"
+    r"so (that )?(\w+ ){0,5}(cannot|can't|won't|will not|never|doesn't|does not))\b",
+    re.IGNORECASE,
+)
+_TRAILER = re.compile(r"^[\w-]+(-by)?:\s", re.IGNORECASE)
+_REVERT_LINE = re.compile(r"^This reverts commit [0-9a-f]+", re.IGNORECASE)
+# Text aimed at an AI tool is untrusted input, not evidence.
+_INJECTION = re.compile(
+    r"\b(ignore (all |any )?(previous|prior|above) (instructions|prompts?)|"
+    r"you are (an? )?(ai|assistant|language model)|system prompt|disregard .{0,20}instructions|"
+    r"as an ai)\b",
+    re.IGNORECASE,
+)
+# Reasons that carry no causal word, each narrow enough not to fire on description:
+# - a result clause: "'now' is the commit time, so the same state scores the same."
+_RESULT = re.compile(r",\s+so\s+(?!far\b|on\b|forth\b)\w", re.IGNORECASE)
+# - a hazard: "an escape sequence would be executed by the reader's terminal."
+_HAZARD = re.compile(
+    r"\b(would|could|might)( otherwise| then)?( (be|get))? (executed|lost|corrupted|leaked|"
+    r"exposed|dropped|duplicated|overwritten|truncated|deleted|injected|bypassed|"
+    r"crash(es)?|fail|hang|deadlock|corrupt|leak)\b",
+    re.IGNORECASE,
+)
+# - a consequence of the previous sentence: "... Fence would explain the wrong code. So
+#   every removed range is compared first." It is quoted together with that sentence.
+_CONSEQUENCE = re.compile(
+    r"^(so(?! far\b)|hence|therefore|thus|that is why|this is why|that's why|"
+    r"for (this|that) reason)\b",
+    re.IGNORECASE,
+)
+_PARAGRAPH_SPLIT = re.compile(r"\n\s*\n|\n(?=\s*[-*] )")
+_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+(?=[A-Z0-9`\"'(])")
+
+
+def paragraphs(text: str) -> list[list[str]]:
+    """Sentences grouped by paragraph or bullet item. Neighbours in one paragraph are
+    each other's context; sentences in different paragraphs are unrelated."""
+    out: list[list[str]] = []
+    for para in _PARAGRAPH_SPLIT.split(text or ""):
+        group = []
+        for block in _SENTENCE_SPLIT.split(para):
+            s = " ".join(block.split())  # join wrapped lines
+            s = s.lstrip("-* ").strip()
+            if s and not _TRAILER.match(s) and not _REVERT_LINE.match(s):
+                group.append(s)
+        if group:
+            out.append(group)
+    return out
+
+
+def sentences(text: str) -> list[str]:
+    return [s for para in paragraphs(text) for s in para]
+
+
+def states_reason(sentence: str) -> bool:
+    return bool(_REASON.search(sentence) or _RESULT.search(sentence)
+                or _HAZARD.search(sentence))
+
+
+def is_injection(sentence: str) -> bool:
+    return bool(_INJECTION.search(sentence))
+
+
+def stated_reason(text: str) -> str | None:
+    """The first sentence in `text` that states a reason, verbatim, or None.
+
+    Which text is about which code is decided before this call (see relevance.py);
+    this function only finds the reason sentence.
+    """
+    for para in paragraphs(text):
+        for i, s in enumerate(para):
+            if is_injection(s):
+                continue
+            if states_reason(s):
+                return clean(s, max_len=MAX_QUOTE)
+            if i and _CONSEQUENCE.match(s) and not is_injection(para[i - 1]):
+                # Still verbatim: the two sentences are adjacent in the source.
+                return clean(f"{para[i - 1]} {s}", max_len=MAX_QUOTE)
+    return None
+
+
+_CONSTRAINT = re.compile(
+    r"\b(must( not)?|never|always|do not|don'?t|required|has to|needs? to|only (when|if))\b",
+    re.IGNORECASE,
+)
+
+
+def constraints(text: str, *, limit: int = 3) -> list[str]:
+    """Sentences that state a constraint ("must not", "never"), verbatim."""
+    out: list[str] = []
+    for s in sentences(text):
+        if is_injection(s) or not _CONSTRAINT.search(s):
+            continue
+        q = clean(s, max_len=MAX_QUOTE)
+        if q not in out:
+            out.append(q)
+        if len(out) >= limit:
+            break
+    return out
+
+
+_ALTERNATIVE = re.compile(
+    r"\b(instead of|rather than|alternatively|considered|we could have|an alternative)\b",
+    re.IGNORECASE,
+)
+
+
+def alternatives(text: str, *, limit: int = 2) -> list[str]:
+    out: list[str] = []
+    for s in sentences(text):
+        if not is_injection(s) and _ALTERNATIVE.search(s):
+            out.append(clean(s, max_len=MAX_QUOTE))
+        if len(out) >= limit:
+            break
+    return out
+
+
+_COMMENT = re.compile(r"^\s*(#|//|/\*+|\*|--)\s?(.*?)\s*(\*/)?\s*$")
+
+
+def comment_text(line: str) -> str | None:
+    """The text of a whole-line or trailing comment, or None."""
+    m = _COMMENT.match(line)
+    if m:
+        return m.group(2) or None
+    for marker in ("  # ", " // "):
+        if marker in line:
+            return line.split(marker, 1)[1].strip() or None
+    return None

fence/decisions/relevance.py

@@ -0,0 +1,146 @@
+"""Evidence scope: which commit, PR and issue text is about *this* code.
+
+A Decision Record covers one file's lines from one commit, but commit messages, PR
+bodies and issues describe the whole change. When the change touched only this
+source file, its text is about this code. When it touched many, a sentence counts
+as evidence for this code only if it names the code: the file, or a symbol or
+distinctive identifier defined in or used by the anchored lines. Everything else
+is change-level context: worth showing, but never quoted as this code's reason.
+
+Matching is by explicit mention, not word overlap: generic words such as
+"index", "cache" or "file" appear everywhere and would tie any sentence to any
+file.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+
+from fence.decisions.reasons import paragraphs, sentences
+from fence.globs import match_any
+
+# File stems too generic to identify a file when they appear in prose.
+GENERIC_STEMS = {"index", "main", "init", "__init__", "utils", "util", "base", "common", "core",
+                 "types", "helpers", "lib", "app", "mod", "api", "test", "tests", "setup"}
+_IDENT = re.compile(r"\b[A-Za-z_][A-Za-z0-9_]*\b")
+_TEST_PREFIX = re.compile(r"^(test_|tests?)|(_test|_spec|\.test|\.spec|tests?)$", re.IGNORECASE)
+
+
+@dataclass(frozen=True)
+class Scope:
+    focused: bool  # the change touched no other source file
+    source_files: int  # source files the change touched (tests and docs excluded)
+    names: frozenset[str]  # lower-case names that identify this code in prose
+    other_files: frozenset[str] = frozenset()  # names of the change's other files
+    # Symbol names, matched with their exact case: code identifiers are case-sensitive,
+    # and a class `Repo` or method `load` is not mentioned by the words "repo" or "load".
+    symbols: frozenset[str] = frozenset()
+    # The file's bare stem ("text" for fence/text.py). Usually an ordinary word, so it
+    # names the code only in its own sentence, never the sentences after it, unless it
+    # is written as code (`text`, fence.text, text.clean).
+    stems: frozenset[str] = frozenset()
+
+
+def _distinctive(ident: str) -> bool:
+    """CamelCase or snake_case identifiers long enough not to be ordinary words."""
+    if len(ident) < 6:
+        return False
+    return "_" in ident.strip("_") or bool(re.search(r"[a-z][A-Z]", ident))
+
+
+def build_scope(path: str, changed: list[tuple[str, int]], code_lines: list[str],
+                symbol_names: list[str], *, test_pattern: re.Pattern[str],
+                low_signal: list[str]) -> Scope:
+    """`changed` is (path, lines added + deleted) for every file in the change."""
+    sources = {p for p, n in changed
+               if n > 0 and not test_pattern.search(p) and not match_any(p, low_signal)}
+    focused = sources <= {path}
+    names = _file_names(path)
+    stems = {n for n in names if "." not in n}
+    names -= stems
+    symbols ={part for q in symbol_names for part in q.split(".")
+               if len(part) >= 4 and part != "<module>"}
+    for line in code_lines:
+        for ident in _IDENT.findall(line):
+            if _distinctive(ident):
+                names.add(ident.lower())
+    others: set[str] = set()
+    for p, _ in changed:
+        if p != path:
+            others |= _file_names(p)
+    return Scope(focused, len(sources), frozenset(names), frozenset(others - names - stems),
+                 frozenset(symbols), frozenset(stems))
+
+
+def _file_names(path: str) -> set[str]:
+    pp = PurePosixPath(path)
+    out = {pp.name.lower()}
+    if pp.stem.lower() not in GENERIC_STEMS and len(pp.stem) >= 4:
+        out.add(pp.stem.lower())
+    return out
+
+
+def mentions(sentence: str, names: frozenset[str], exact: frozenset[str] = frozenset()) -> bool:
+    low = sentence.lower()
+    return (any(re.search(rf"(?<!\w){re.escape(n)}(?!\w)", low) for n in names)
+            or any(re.search(rf"(?<!\w){re.escape(n)}(?!\w)", sentence) for n in exact))
+
+
+def _as_code(sentence: str, stems: frozenset[str]) -> bool:
+    """A stem written as code: in backticks, or joined to a path or dotted name."""
+    low = sentence.lower()
+    return any(re.search(rf"`{re.escape(n)}`|[\w/][./]{re.escape(n)}(?!\w)|"
+                         rf"(?<![\w.]){re.escape(n)}\.\w", low) for n in stems)
+
+
+def relevant_text(text: str, scope: Scope) -> str:
+    """The part of `text` that is evidence for this code: all of it for a focused
+    change, otherwise only the sentences that name this code."""
+    if scope.focused:
+        return text
+    # A sentence right after one that names this code, in the same paragraph, continues
+    # it ("fence/text.py strips escapes. Commit messages are attacker-controlled, ...")
+    # unless it names another file in the change. A bare stem ("uses text labels") counts
+    # for its own sentence only. Runs stay joined so a quote built from adjacent
+    # sentences is still verbatim.
+    runs: list[str] = []
+    for para in paragraphs(text):
+        run: list[str] = []
+        for s in para:
+            if mentions(s, scope.names, scope.symbols) or _as_code(s, scope.stems):
+                run.append(s)
+                continue
+            if run and not mentions(s, scope.other_files):
+                run.append(s)
+                continue
+            if run:
+                runs.append(" ".join(run))
+                run = []
+            if mentions(s, scope.stems):
+                runs.append(s)
+        if run:
+            runs.append(" ".join(run))
+    return "\n\n".join(runs)
+
+
+def change_level_text(text: str, scope: Scope) -> str:
+    """Sentences about the change as a whole: those that name no other file in it.
+    A sentence naming another file is that file's reason, not context for this one."""
+    return "\n\n".join(s for s in sentences(text) if not mentions(s, scope.other_files))
+
+
+def related_tests(test_paths: list[str], path: str, scope: Scope) -> list[str]:
+    """Tests from the same change that plausibly exercise this file: any test for a
+    focused change, otherwise tests whose name matches the file's name."""
+    if scope.focused:
+        return test_paths
+    stem = PurePosixPath(path).stem.lower()
+    out = []
+    for t in test_paths:
+        tstem = _TEST_PREFIX.sub("", PurePosixPath(t).stem.lower()).strip("_.-")
+        if tstem and (tstem == stem or (len(tstem) >= 4 and tstem in stem)
+                      or (len(stem) >= 4 and stem in tstem)):
+            out.append(t)
+    return out

fence/diffing/ranges.py

@@ -0,0 +1,25 @@
+"""The unit of work between diff parsing and git tracing."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class RemovedRange:
+    """Consecutive pre-image lines a diff removes or changes.
+
+    `start`..`end` are 1-based, inclusive line numbers in the file at the base revision.
+    `lines` is the removed text, without newlines, exactly as the diff shows it.
+    """
+
+    path: str
+    start: int
+    end: int
+    lines: tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        if self.start < 1 or self.end < self.start:
+            raise ValueError(f"bad range {self.start},{self.end}")
+        if len(self.lines) != self.end - self.start + 1:
+            raise ValueError("line count does not match range")

fence/diffing/symbols.py

@@ -0,0 +1,21 @@
+"""Find the function or class that encloses a line of Python source.
+
+Kept as the stable Day 1 entry point; the work is done by the Python language
+plugin. For any language, use `fence.lang.enclosing_symbol(path, source, line)`.
+"""
+
+from __future__ import annotations
+
+from fence.lang import MODULE, Symbol
+from fence.lang import enclosing_symbol as _enclosing
+
+__all__ = ["MODULE", "Symbol", "enclosing_symbol"]
+
+
+def enclosing_symbol(source: str, line: int) -> Symbol | None:
+    """Return the innermost function/class containing `line` (1-based).
+
+    Returns None if the source does not parse or the line is past the end of the file.
+    Lines outside any def/class return the `<module>` symbol.
+    """
+    return _enclosing("x.py", source, line)