diffcontext 0.5.1__py3-none-any.whl

diffcontext/history.py

@@ -0,0 +1,154 @@
+"""
+history.py — git co-change history as a retrieval signal.
+
+The failure taxonomy (benchmarks/EVAL_V2_REPORT.md) shows a hard ceiling
+for every static signal: cross-subsystem conceptual links (a settings
+flag and the security check that reads it) have no call edge, no lexical
+overlap, and no co-location — graph, BM25, and hybrid all scored 0/20 on
+that bucket. The only signal that CAN see those pairs is the repository's
+own history: files that changed together before tend to change together
+again (Zimmermann et al.'s classic co-change result, here as a retrieval
+signal rather than a recommender).
+
+Design constraints honored:
+  * Zero runtime dependencies — plain `git log` via subprocess.
+  * Graceful degradation — no git repo / no git binary / timeout produce
+    an EMPTY index (scores_for_files returns {}), never an exception.
+  * File-level granularity — symbol-level history is noisy and expensive
+    to mine (rename/move tracking); file-level association is the
+    literature-standard compromise. The blend spreads a file's score to
+    the symbols inside it.
+  * Leakage control for evaluation — `exclude_commits` lets a benchmark
+    mine history WITHOUT the commits it is evaluating on. Scoring a
+    commit's co-change partners with an index that already contains that
+    very commit would be train-on-test leakage; the eval harness passes
+    every mined eval commit here.
+
+Association score: for a changed file c and candidate file f,
+    assoc(f | c) = cochange_count(f, c) / change_count(c)
+i.e. the empirical probability that a commit touching c also touched f,
+maxed over all changed files. Pairs seen fewer than `min_cochanges`
+times are ignored (a single shared commit is usually coincidence).
+Sweeping commits (> max_files_per_commit files) are skipped entirely —
+mass renames and formatting passes assert nothing about relatedness.
+"""
+
+import logging
+import os
+import subprocess
+from collections import Counter
+from typing import Dict, Iterable, Optional, Set
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_MAX_COMMITS = 3000      # how much history to mine
+MAX_FILES_PER_COMMIT = 25       # skip sweeping commits (mechanical churn)
+DEFAULT_MIN_COCHANGES = 2       # pairs must co-occur at least this often
+
+
+class CoChangeIndex:
+    """
+    File-level co-change statistics mined from `git log`.
+
+    Usage:
+        cci = CoChangeIndex("/path/to/repo")
+        scores = cci.scores_for_files(["./src/auth.py"])
+        # {"./src/tokens.py": 0.42, ...}  association in [0, 1]
+    """
+
+    def __init__(
+        self,
+        repo_path: str,
+        max_commits: int = DEFAULT_MAX_COMMITS,
+        max_files_per_commit: int = MAX_FILES_PER_COMMIT,
+        min_cochanges: int = DEFAULT_MIN_COCHANGES,
+        exclude_commits: Optional[Set[str]] = None,
+    ):
+        self.repo_path = os.path.abspath(repo_path)
+        self.min_cochanges = min_cochanges
+        self.pair_counts: Dict[str, Counter] = {}   # "./a.py" -> {"./b.py": n}
+        self.file_counts: Counter = Counter()       # "./a.py" -> n commits touching it
+        self.mined_commits = 0
+
+        # Excluded hashes matched on their first 10 chars so both full and
+        # abbreviated hashes (as eval harnesses store them) work.
+        excluded = {h[:10] for h in (exclude_commits or set())}
+
+        commits = self._read_history(max_commits)
+        for commit_hash, files in commits:
+            if commit_hash[:10] in excluded:
+                continue
+            files = [f for f in files if f.endswith(".py")]
+            if len(files) < 2 or len(files) > max_files_per_commit:
+                # <2: nothing co-changed; >cap: sweeping mechanical commit
+                if files:
+                    self.mined_commits += 1
+                    for f in files:
+                        self.file_counts["./" + f] += 1
+                continue
+            self.mined_commits += 1
+            rels = ["./" + f for f in files]
+            for f in rels:
+                self.file_counts[f] += 1
+            for i, a in enumerate(rels):
+                counter_a = self.pair_counts.setdefault(a, Counter())
+                for b in rels[i + 1:]:
+                    counter_a[b] += 1
+                    self.pair_counts.setdefault(b, Counter())[a] += 1
+
+    def _read_history(self, max_commits: int):
+        """[(full_hash, [file, ...]), ...] — empty list on any failure."""
+        try:
+            res = subprocess.run(
+                ["git", "log", f"--max-count={max_commits}", "--no-merges",
+                 "--name-only", "--format=%x00%H"],
+                cwd=self.repo_path, capture_output=True, text=True, timeout=120,
+            )
+        except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e:
+            logger.warning("co-change mining skipped: git log failed (%s)", e)
+            return []
+        if res.returncode != 0:
+            logger.warning(
+                "co-change mining skipped: git log exited %d", res.returncode
+            )
+            return []
+
+        commits = []
+        for block in res.stdout.split("\x00"):
+            if not block.strip():
+                continue
+            lines = block.strip().split("\n")
+            commit_hash = lines[0].strip()
+            files = [ln.strip() for ln in lines[1:] if ln.strip()]
+            commits.append((commit_hash, files))
+        return commits
+
+    def scores_for_files(self, changed_files: Iterable[str]) -> Dict[str, float]:
+        """
+        Association score in [0, 1] for every file that historically
+        co-changed with any of `changed_files` (max over the changed
+        files). Accepts "./rel/path.py" or "rel/path.py". The changed
+        files themselves are excluded from the result.
+        """
+        changed = {
+            f if f.startswith("./") else "./" + f for f in changed_files
+        }
+        out: Dict[str, float] = {}
+        for c in changed:
+            total = self.file_counts.get(c, 0)
+            if total <= 0:
+                continue
+            for f, n in self.pair_counts.get(c, Counter()).items():
+                if f in changed or n < self.min_cochanges:
+                    continue
+                assoc = n / total
+                if assoc > out.get(f, 0.0):
+                    out[f] = assoc
+        return out
+
+    def scores_for_symbols(self, changed_symbols: Iterable[str]) -> Dict[str, float]:
+        """Convenience: same as scores_for_files, keyed off the file part
+        of changed symbol IDs ("./a.py:fn" -> "./a.py")."""
+        return self.scores_for_files(
+            {s.split(":")[0] for s in changed_symbols}
+        )

diffcontext/impact/__init__.py

@@ -0,0 +1 @@
+"""impact subpackage — blast radius, scoring, traversal."""

diffcontext/impact/blast_radius.py

@@ -0,0 +1,58 @@
+"""
+blast_radius.py — Find all symbols transitively affected by a change.
+
+Given a changed symbol, walks the REVERSE graph (callers) to find
+everything that depends on it.
+
+Performance fix: accept an optional pre-built reverse graph so the caller
+can build it once and reuse it across multiple symbols instead of
+rebuilding it O(N) times.
+"""
+
+from typing import Dict, List, Optional, Set
+
+
+def build_reverse_graph(graph: Dict[str, List[str]]) -> Dict[str, Set[str]]:
+    """Build the reverse (caller) graph once; reuse for many queries."""
+    reverse: Dict[str, Set[str]] = {}
+    for caller, callees in graph.items():
+        for callee in callees:
+            reverse.setdefault(callee, set()).add(caller)
+    return reverse
+
+
+def get_blast_radius(
+    graph: Dict[str, List[str]],
+    changed_symbol: str,
+    reverse: Optional[Dict[str, Set[str]]] = None,
+) -> List[str]:
+    """
+    All functions that (transitively) call changed_symbol.
+
+    Args:
+        graph:          Forward call graph (caller -> [callees]).
+        changed_symbol: The symbol whose blast radius we want.
+        reverse:        Pre-built reverse graph. If None it is built here
+                        (correct but wasteful when called in a loop).
+
+    Uses iterative DFS with cycle detection.
+    """
+    if reverse is None:
+        reverse = build_reverse_graph(graph)
+
+    affected: List[str] = []
+    visited: Set[str] = set()
+
+    # Iterative DFS on reverse graph
+    stack = [changed_symbol]
+    visited.add(changed_symbol)
+
+    while stack:
+        current = stack.pop()
+        for caller in reverse.get(current, ()):
+            if caller not in visited:
+                visited.add(caller)
+                affected.append(caller)
+                stack.append(caller)
+
+    return affected

diffcontext/impact/scoring.py

@@ -0,0 +1,223 @@
+"""
+scoring.py — Impact scoring for prioritizing symbols in context.
+
+Algorithm: Bidirectional decay propagation from changed symbols.
+
+Forward  (callees): score decays by CALLEE_DECAY per hop
+Backward (callers): score decays by CALLER_DECAY per hop
+
+Structural bonuses:
+  - Sibling bonus: shares a caller with a changed symbol (co-change signal),
+    weighted by 1/caller_outdegree so hub callers don’t flood the pool,
+    and log-scaled before capping to dampen compounding across many hubs.
+  - Structural bonus: log2(1 + indegree)*3 + log2(1 + outdegree),
+    hard-capped at STRUCT_MAX so mega-hub nodes (outdegree=400) don’t
+    accumulate +800 and crowd out genuinely co-changed code.
+
+Changes vs v1:
+  1. Structural bonus capped (STRUCT_MAX=15). Uncapped bonus turned hubs
+     into permanent top-scorers regardless of actual co-change signal.
+  2. BFS propagation cutoff raised 5.0 → 8.0. Large repos (transformers)
+     have long paths; 5.0 prematurely cut valid propagation chains.
+  3. Sibling bonus log-scaled before accumulation to dampen compounding.
+"""
+
+import math
+from collections import deque
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set
+
+
+# Tunable constants — the module-level values are the tuned defaults; pass a
+# ScoringConfig to compute_impact_scores() to override per call instead of
+# editing these.
+CHANGED_SCORE   = 100.0
+CALLEE_DECAY    = 0.65   # lowered: callees are less likely to co-change
+CALLER_DECAY    = 0.85   # raised: callers co-change more than callees
+CALLEE_BASE     = 90.0   # direct callee of changed symbol
+CALLER_BASE     = 85.0   # direct caller
+SIBLING_BASE    = 60.0   # base for sibling contribution (divided by caller_outdegree)
+SIBLING_MAX     = 80.0   # cap: a symbol can’t accumulate more than this from siblings
+BLAST_BASE      = 30.0   # in blast_radii but not reached by BFS
+MAX_HOPS        = 8      # propagation depth limit
+BFS_CUTOFF      = 8.0    # stop propagating a path when score drops below this
+STRUCT_MAX      = 15.0   # hard cap on structural bonus (log-scaled)
+
+
+@dataclass(frozen=True)
+class ScoringConfig:
+    """
+    Tunable weights for impact scoring. A harness running different loop
+    kinds (broad refactor check vs. narrow bug fix) can pass its own config
+    instead of editing module constants; benchmark ablations can sweep
+    configs without touching source.
+
+    Defaults are read from the module-level constants at construction time,
+    so existing tuning (and tests that monkeypatch the constants) keep
+    working unchanged.
+    """
+    changed_score: float = None   # type: ignore[assignment]
+    callee_decay: float = None    # type: ignore[assignment]
+    caller_decay: float = None    # type: ignore[assignment]
+    callee_base: float = None     # type: ignore[assignment]
+    caller_base: float = None     # type: ignore[assignment]
+    sibling_base: float = None    # type: ignore[assignment]
+    sibling_max: float = None     # type: ignore[assignment]
+    blast_base: float = None      # type: ignore[assignment]
+    max_hops: int = None          # type: ignore[assignment]
+    bfs_cutoff: float = None      # type: ignore[assignment]
+    struct_max: float = None      # type: ignore[assignment]
+
+    def __post_init__(self):
+        defaults = {
+            "changed_score": CHANGED_SCORE, "callee_decay": CALLEE_DECAY,
+            "caller_decay": CALLER_DECAY, "callee_base": CALLEE_BASE,
+            "caller_base": CALLER_BASE, "sibling_base": SIBLING_BASE,
+            "sibling_max": SIBLING_MAX, "blast_base": BLAST_BASE,
+            "max_hops": MAX_HOPS, "bfs_cutoff": BFS_CUTOFF,
+            "struct_max": STRUCT_MAX,
+        }
+        for name, default in defaults.items():
+            if getattr(self, name) is None:
+                object.__setattr__(self, name, default)
+
+
+def describe_scoring_basis(config: "Optional[ScoringConfig]" = None) -> str:
+    """
+    One-line, human/LLM-readable summary of the live scoring parameters.
+
+    Consumed by the context compiler's meta-header so the description can
+    never drift from the actual algorithm again (it used to be hardcoded
+    prose that went stale when constants changed). Pass the same
+    ScoringConfig used for scoring to describe a non-default run.
+    """
+    cfg = config if config is not None else ScoringConfig()
+    return (
+        f"changed={cfg.changed_score:.0f} "
+        f"| direct_callee={cfg.callee_base:.0f} | direct_caller={cfg.caller_base:.0f} "
+        f"| 2hop_callee={cfg.callee_base * cfg.callee_decay:.1f} "
+        f"| 2hop_caller={cfg.caller_base * cfg.caller_decay:.1f} "
+        f"| struct bonus=log2-scaled, capped at {cfg.struct_max:.0f}"
+    )
+
+
+def compute_impact_scores(
+    graph: Dict[str, List[str]],
+    changed_symbols: List[str],
+    blast_radii: Dict[str, List[str]],
+    expanded_deps: List[str] = None,
+    reverse: Optional[Dict[str, Set[str]]] = None,
+    config: Optional[ScoringConfig] = None,
+) -> Dict[str, float]:
+    """
+    Score every symbol's relevance to understanding the change.
+
+    Args:
+        graph:           Forward call graph.
+        changed_symbols: Symbols that were modified.
+        blast_radii:     Pre-computed blast radii per changed symbol.
+        expanded_deps:   Symbols reachable by forward dependency expansion.
+        reverse:         Pre-built reverse graph (built internally if None).
+        config:          Scoring weights; module-constant defaults if None.
+
+    Returns dict of symbol_id -> score (higher = more important).
+    """
+    cfg = config if config is not None else ScoringConfig()
+    # Build reverse graph once (or reuse caller's)
+    if reverse is None:
+        reverse = {}
+        for caller, callees in graph.items():
+            for callee in callees:
+                reverse.setdefault(callee, set()).add(caller)
+
+    changed_set = set(changed_symbols)
+    scores: Dict[str, float] = {}
+
+    # ── 1. Changed symbols = 100 ──────────────────────────────────────────
+    for sym in changed_symbols:
+        scores[sym] = cfg.changed_score
+
+    # ── 2. Forward BFS (callees) with decay ──────────────────────────────
+    queue: deque = deque()
+    for sym in changed_symbols:
+        for callee in graph.get(sym, []):
+            if callee not in changed_set:
+                queue.append((callee, cfg.callee_base, 1))
+
+    visited_fwd: Set[str] = set(changed_symbols)
+    while queue:
+        node, score, hop = queue.popleft()
+        if node in visited_fwd or hop > cfg.max_hops:
+            continue
+        visited_fwd.add(node)
+        scores[node] = max(scores.get(node, 0.0), score)
+        next_score = score * cfg.callee_decay
+        if next_score >= cfg.bfs_cutoff:
+            for callee in graph.get(node, []):
+                if callee not in visited_fwd:
+                    queue.append((callee, next_score, hop + 1))
+
+    # ── 3. Backward BFS (callers) with decay ─────────────────────────────
+    queue2: deque = deque()
+    for sym in changed_symbols:
+        for caller in reverse.get(sym, set()):
+            if caller not in changed_set:
+                queue2.append((caller, cfg.caller_base, 1))
+
+    visited_bwd: Set[str] = set(changed_symbols)
+    while queue2:
+        node, score, hop = queue2.popleft()
+        if node in visited_bwd or hop > cfg.max_hops:
+            continue
+        visited_bwd.add(node)
+        scores[node] = max(scores.get(node, 0.0), score)
+        next_score = score * cfg.caller_decay
+        if next_score >= cfg.bfs_cutoff:
+            for caller in reverse.get(node, set()):
+                if caller not in visited_bwd:
+                    queue2.append((caller, next_score, hop + 1))
+
+    # ── 4. Specificity-weighted sibling bonus ────────────────────────────
+    sibling_accumulator: Dict[str, float] = {}
+
+    for sym in changed_symbols:
+        for caller in reverse.get(sym, set()):
+            caller_callees = graph.get(caller, [])
+            caller_outdegree = len(caller_callees)
+            if caller_outdegree <= 1:
+                continue
+            raw_contribution = cfg.sibling_base / caller_outdegree
+            contribution = math.log2(1.0 + raw_contribution)
+            for sibling in caller_callees:
+                if sibling not in changed_set and sibling != sym:
+                    sibling_accumulator[sibling] = (
+                        sibling_accumulator.get(sibling, 0.0) + contribution
+                    )
+
+    for sibling, bonus in sibling_accumulator.items():
+        capped_bonus = min(bonus, cfg.sibling_max)
+        scores[sibling] = scores.get(sibling, 0.0) + capped_bonus
+
+    # ── 5. Expanded deps: give them a meaningful score ────────────────────
+    if expanded_deps:
+        for sym in expanded_deps:
+            if sym not in scores:
+                scores[sym] = cfg.blast_base
+
+    # ── 6. Remaining blast radius symbols not yet scored ─────────────────
+    for sym, radius in blast_radii.items():
+        for affected in radius:
+            if affected not in scores:
+                scores[affected] = cfg.blast_base
+
+    # ── 7. Structural bonus: log-scaled, hard-capped ──────────────────────────
+    # Previous formula (indegree*2 + outdegree) was unbounded: a hub with
+    # indegree=50 got +100 structural bonus, drowning all co-change signal.
+    # log2(1+degree) grows slowly and the cfg.struct_max cap prevents run-away.
+    for sym in scores:
+        indegree  = len(reverse.get(sym, set()))
+        outdegree = len(graph.get(sym, []))
+        struct_bonus = math.log2(1 + indegree) * 3 + math.log2(1 + outdegree)
+        scores[sym] += min(struct_bonus, cfg.struct_max)
+
+    return scores

diffcontext/impact/traversal.py

@@ -0,0 +1,58 @@
+"""
+traversal.py — Dependency expansion (forward graph walk).
+
+Given selected symbols, walk forward edges to include their callees.
+Supports both unbounded DFS and bounded BFS.
+"""
+
+from typing import Dict, List, Optional, Set
+
+
+def expand_dependencies(
+    graph: Dict[str, List[str]],
+    selected_symbols: List[str],
+    max_depth: Optional[int] = None,
+) -> List[str]:
+    """
+    Walk forward edges from selected_symbols.
+
+    max_depth=None  -> full transitive closure (iterative DFS)
+    max_depth=N     -> only nodes reachable within N hops (BFS)
+    """
+    visited: Set[str] = set()
+    result: List[str] = []
+
+    if max_depth is None:
+        # Iterative DFS — safe on large repos (no recursion limit)
+        stack = list(selected_symbols)
+        while stack:
+            func = stack.pop()
+            if func in visited:
+                continue
+            visited.add(func)
+            result.append(func)
+            for dep in reversed(graph.get(func, [])):
+                if dep not in visited:
+                    stack.append(dep)
+        return result
+
+    # Bounded BFS
+    frontier = list(selected_symbols)
+    for func in frontier:
+        if func not in visited:
+            visited.add(func)
+            result.append(func)
+
+    depth = 0
+    while frontier and depth < max_depth:
+        next_frontier = []
+        for func in frontier:
+            for dep in graph.get(func, []):
+                if dep not in visited:
+                    visited.add(dep)
+                    result.append(dep)
+                    next_frontier.append(dep)
+        frontier = next_frontier
+        depth += 1
+
+    return result