diffcontext 0.5.1__py3-none-any.whl

diffcontext/__init__.py

@@ -0,0 +1,233 @@
+"""
+DiffContext — static-analysis-powered repository context compiler for LLMs.
+
+Converts code changes into dependency-aware, blast-radius-aware context
+packages, enabling far more accurate code understanding than keyword search
+or traditional RAG.
+
+Usage as a library:
+
+    from diffcontext import blast_radius, index, diff, compile_context
+
+    # Get blast radius for a symbol
+    result = blast_radius("./auth.py:validate_jwt", repo="/path/to/project")
+    print(result.callers)          # who calls this?
+    print(result.dependencies)     # what does this call?
+    print(result.total_affected)   # total transitive impact
+
+    # Auto-detect changes and get blast radius
+    result = blast_radius(ref="HEAD~1", repo="/path/to/project")
+
+    # Index a repository
+    idx = index("/path/to/project")
+    print(idx.symbols)    # all parsed symbols
+    print(idx.graph)      # call graph
+
+    # Find changed symbols from git diff
+    changed = diff(repo="/path/to/project", ref="HEAD~1")
+
+    # Full context compilation for LLMs
+    ctx = compile_context(ref="HEAD~1", repo="/path/to/project")
+    print(ctx.text)             # LLM-ready context
+    print(ctx.reduction_pct)    # how much code was filtered out
+"""
+
+__version__ = "0.5.1"
+
+# Public, semver-covered API. Everything not listed here (graph_builder,
+# resolver, symbols, scanner, parser internals) is importable but carries no
+# stability guarantee across releases.
+__all__ = [
+    "__version__",
+    "BlastResult",
+    "CoChangeIndex",
+    "ContextItem",
+    "ScoringConfig",
+    "blast_radius",
+    "index",
+    "diff",
+    "compile_context",
+]
+
+from dataclasses import dataclass, field
+from typing import Callable, Dict, List, Optional
+
+from .pipeline import index_repository, analyze_impact
+from .pipeline import compile as _compile_pipeline
+from .diff.git_diff import find_changed_symbols
+# Redundant alias: intentional re-export (used by library callers even
+# though nothing in this module calls it).
+from .impact.blast_radius import get_blast_radius as get_blast_radius
+from .history import CoChangeIndex
+from .impact.scoring import ScoringConfig
+from .models import ContextItem
+
+
+# ---------------------------------------------------------------------------
+# Public data classes
+# ---------------------------------------------------------------------------
+
+@dataclass
+class BlastResult:
+    """Result of a blast radius analysis — the public API return type."""
+    changed: List[str]
+    callers: List[str]
+    dependencies: List[str]
+    total_affected: int
+    scores: Dict[str, float] = field(default_factory=dict)
+    graph: Dict[str, List[str]] = field(default_factory=dict)
+
+    @property
+    def affected_files(self) -> List[str]:
+        """Unique files in the blast radius."""
+        files = set()
+        for sym in self.callers:
+            parts = sym.split(":", 1)
+            if len(parts) == 2:
+                files.add(parts[0])
+        return sorted(files)
+
+    def __repr__(self):
+        return (
+            f"BlastResult(changed={len(self.changed)}, "
+            f"callers={len(self.callers)}, "
+            f"dependencies={len(self.dependencies)}, "
+            f"total_affected={self.total_affected})"
+        )
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def blast_radius(
+    symbol: Optional[str] = None,
+    *,
+    ref: Optional[str] = None,
+    repo: str = ".",
+    depth: int = 3,
+) -> BlastResult:
+    """
+    Compute the blast radius of a change.
+
+    Args:
+        symbol: Symbol ID like "./auth.py:validate_jwt". If None, uses `ref`.
+        ref:    Git ref to auto-detect changes (e.g. "HEAD~1").
+        repo:   Path to the repository root.
+        depth:  Max traversal depth.
+
+    Returns:
+        BlastResult with callers, dependencies, scores, etc.
+
+    Examples:
+        >>> from diffcontext import blast_radius
+        >>> r = blast_radius("./auth.py:validate_jwt", repo="/path/to/project")
+        >>> r = blast_radius(ref="HEAD~1", repo="/path/to/project")
+    """
+    idx = index_repository(repo)
+
+    # Determine changed symbols
+    if symbol:
+        changed = [symbol]
+    elif ref:
+        changed = find_changed_symbols(repo, idx.symbols, ref=ref)
+    else:
+        changed = find_changed_symbols(repo, idx.symbols, ref="HEAD~1")
+
+    if not changed:
+        return BlastResult(
+            changed=[], callers=[], dependencies=[],
+            total_affected=0, scores={}, graph=idx.graph,
+        )
+
+    impact = analyze_impact(idx, changed, max_depth=depth)
+
+    return BlastResult(
+        changed=impact.changed,
+        callers=impact.blast_radius,
+        dependencies=impact.dependencies,
+        total_affected=len(impact.all_relevant),
+        scores=impact.scores,
+        graph=idx.graph,
+    )
+
+
+def index(repo: str = "."):
+    """
+    Index a repository: parse all Python files and build the call graph.
+
+    Returns a RepositoryIndex with .symbols and .graph attributes.
+
+    Example:
+        >>> from diffcontext import index
+        >>> idx = index("/path/to/project")
+        >>> len(idx.symbols)
+        354
+    """
+    return index_repository(repo)
+
+
+def diff(repo: str = ".", ref: str = "HEAD~1") -> List[str]:
+    """
+    Find changed symbol IDs from git diff.
+
+    Returns list of symbol IDs that were modified.
+
+    Example:
+        >>> from diffcontext import diff
+        >>> diff(repo="/path/to/project", ref="HEAD~1")
+        ['./auth.py:validate_jwt', './models.py:User.__init__']
+    """
+    idx = index_repository(repo)
+    return find_changed_symbols(repo, idx.symbols, ref=ref)
+
+
+def compile_context(
+    symbol: Optional[str] = None,
+    *,
+    ref: Optional[str] = None,
+    repo: str = ".",
+    depth: int = 2,
+    max_tokens: int = 10000,
+    token_counter: Optional[Callable[[str], int]] = None,
+    scoring_config: Optional[ScoringConfig] = None,
+):
+    """
+    Full pipeline: detect changes → blast radius → compile LLM context.
+
+    Returns a ContextPackage with .text (rendered), .items (structured
+    ContextItem list a harness can re-budget itself), .token_estimate,
+    and .reduction_pct.
+
+    Args:
+        token_counter:  text -> token count callable. Pass your model's
+                        real tokenizer to enforce hard window limits;
+                        defaults to a ~4-chars/token heuristic.
+        scoring_config: Custom impact-scoring weights (ScoringConfig);
+                        tuned defaults when None.
+
+    Example:
+        >>> from diffcontext import compile_context
+        >>> ctx = compile_context(ref="HEAD~1", repo="/path/to/project")
+        >>> print(ctx.text)          # LLM-ready context
+        >>> ctx.items[0].symbol_id   # structured form for harnesses
+        >>> print(ctx.reduction_pct) # e.g. 99.2
+    """
+    idx = index_repository(repo)
+
+    if symbol:
+        changed = [symbol]
+    elif ref:
+        changed = find_changed_symbols(repo, idx.symbols, ref=ref)
+    else:
+        changed = find_changed_symbols(repo, idx.symbols, ref="HEAD~1")
+
+    if not changed:
+        from .models import ContextPackage
+        return ContextPackage(text="", symbol_count=0, token_estimate=0, total_repo_tokens=0)
+
+    impact = analyze_impact(idx, changed, max_depth=depth, scoring_config=scoring_config)
+    return _compile_pipeline(
+        idx, impact, max_tokens=max_tokens,
+        token_counter=token_counter, scoring_config=scoring_config,
+    )

diffcontext/_warn_once.py

@@ -0,0 +1,112 @@
+"""
+_warn_once.py — De-duplicate repeated warnings for the same file.
+
+parser.py, graph_builder.py, and resolver.py each independently call
+ast.parse() on every file. When a file has a syntax error, all three would
+otherwise log their own identical warning. This module tracks which files
+have already been warned about (per process run) so only the first one
+actually prints.
+
+Also covers invalid-UTF-8 source files: these are read with
+errors="ignore", which silently DROPS any byte that isn't valid UTF-8 with
+no warning, anywhere. A dropped byte inside a string literal corrupts that
+literal's contents (e.g. "Café Menu" -> "Caf Menu") without raising any
+error or appearing in any log -- the file still parses fine, the symbol
+still extracts fine, the code text is just silently wrong. warn_encoding_
+issue_once exists so this corruption is at least visible once per file.
+"""
+
+import logging
+import os
+import threading
+from typing import Optional, Set
+
+
+class WarnState:
+    """
+    De-dup state for warn-once semantics, scoped to whoever owns it.
+
+    A long-lived process (an agent harness serving many repos/sessions)
+    should create one WarnState per indexing session so one session's
+    warnings never suppress another's; the module-level default preserves
+    the old process-wide behavior for direct callers.
+    """
+
+    def __init__(self):
+        self.syntax_files: Set[str] = set()
+        self.encoding_files: Set[str] = set()
+        self._lock = threading.Lock()
+
+    def first_syntax(self, key: str) -> bool:
+        with self._lock:
+            if key in self.syntax_files:
+                return False
+            self.syntax_files.add(key)
+            return True
+
+    def first_encoding(self, key: str) -> bool:
+        with self._lock:
+            if key in self.encoding_files:
+                return False
+            self.encoding_files.add(key)
+            return True
+
+    def reset(self) -> None:
+        with self._lock:
+            self.syntax_files.clear()
+            self.encoding_files.clear()
+
+
+_default_state = WarnState()
+
+
+def warn_syntax_error_once(
+    logger: logging.Logger,
+    filename: str,
+    exc: SyntaxError,
+    state: Optional[WarnState] = None,
+) -> None:
+    """Log a SyntaxError warning for `filename`, but only the first time it's
+    seen by `state` (the process-wide default when None)."""
+    key = os.path.abspath(filename)
+    if not (state or _default_state).first_syntax(key):
+        return
+
+    logger.warning(
+        "\033[93mSkipping %s due to SyntaxError: %s (line %s)\033[0m",
+        os.path.basename(filename), exc.msg, exc.lineno,
+    )
+
+
+def check_and_warn_encoding(
+    logger: logging.Logger,
+    filename: str,
+    raw_bytes: bytes,
+    state: Optional[WarnState] = None,
+) -> None:
+    """
+    Check whether `raw_bytes` is valid UTF-8. If not, warn once per file --
+    the caller will go on to decode with errors="ignore", which silently
+    drops the offending bytes (and anything that decoded incorrectly around
+    them). This doesn't stop processing; it just makes the data loss
+    visible instead of completely silent.
+    """
+    try:
+        raw_bytes.decode("utf-8")
+        return
+    except UnicodeDecodeError as e:
+        key = os.path.abspath(filename)
+        if not (state or _default_state).first_encoding(key):
+            return
+
+        logger.warning(
+            "\033[93m%s is not valid UTF-8 (%s at byte %d) -- invalid bytes "
+            "will be silently dropped, which can corrupt string literals "
+            "in the extracted code\033[0m",
+            os.path.basename(filename), e.reason, e.start,
+        )
+
+
+def reset_warned_files() -> None:
+    """Clear the process-wide de-dup caches. Mainly useful for tests."""
+    _default_state.reset()

diffcontext/cache.py

@@ -0,0 +1,216 @@
+"""
+cache.py — SQLite-backed persistent caching for AST parsed symbols and the
+repository call graph.
+"""
+
+import hashlib
+import json
+import sqlite3
+from typing import Dict, Callable, List, Optional, Tuple
+
+from .models import Symbol
+
+
+def get_file_hash(filepath: str) -> str:
+    """Compute SHA-256 hash of a file."""
+    hasher = hashlib.sha256()
+    with open(filepath, "rb") as f:
+        # Python files are small enough to read into memory safely
+        hasher.update(f.read())
+    return hasher.hexdigest()
+
+
+def hash_source(source_bytes: bytes) -> str:
+    """SHA-256 of already-read file contents (avoids a second disk read)."""
+    return hashlib.sha256(source_bytes).hexdigest()
+
+
+def repo_state_hash(file_hashes: Dict[str, str]) -> str:
+    """
+    Single hash summarizing the content state of every Python file in the
+    repo. Keyed on (relative_path, content_hash) pairs, order-independent.
+    Any file added, removed, or edited changes this hash.
+    """
+    hasher = hashlib.sha256()
+    for path in sorted(file_hashes):
+        hasher.update(path.encode("utf-8"))
+        hasher.update(b"\0")
+        hasher.update(file_hashes[path].encode("ascii"))
+        hasher.update(b"\n")
+    return hasher.hexdigest()
+
+
+class SymbolCache:
+    """
+    Persistent SQLite cache for parsed AST symbols and the call graph.
+
+    Safe for concurrent use from multiple threads within one process: the
+    connection is created with check_same_thread=False and every public
+    operation holds an internal lock (SQLite serializes at the file level
+    across processes on its own via WAL).
+    """
+
+    def __init__(self, db_path: str = ".diffcontext_cache.db"):
+        import threading
+        self.db_path = db_path
+        self._conn = None
+        self._lock = threading.RLock()
+        self._connect()
+
+    def _connect(self):
+        if self._conn is None:
+            self._conn = sqlite3.connect(self.db_path, check_same_thread=False)
+            self._conn.execute("PRAGMA journal_mode=WAL")
+            # Cache contents are rebuildable from source; NORMAL skips the
+            # per-commit fsync (a measurable cost at one commit per file on
+            # a cold index) and risks nothing worse than a stale cache row
+            # after power loss, which the content hash then invalidates.
+            self._conn.execute("PRAGMA synchronous=NORMAL")
+            self._conn.execute("PRAGMA foreign_keys=ON")
+            self._init_db()
+
+    def close(self):
+        with self._lock:
+            if self._conn:
+                self._conn.close()
+                self._conn = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def _init_db(self):
+        with self._conn:
+            self._conn.executescript('''
+                CREATE TABLE IF NOT EXISTS files (
+                    file_path TEXT PRIMARY KEY,
+                    file_hash TEXT NOT NULL
+                );
+
+                CREATE TABLE IF NOT EXISTS symbols (
+                    id TEXT PRIMARY KEY,
+                    file_path TEXT NOT NULL,
+                    name TEXT NOT NULL,
+                    code TEXT NOT NULL,
+                    lineno INTEGER,
+                    FOREIGN KEY(file_path) REFERENCES files(file_path) ON DELETE CASCADE
+                );
+
+                CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_path);
+
+                CREATE TABLE IF NOT EXISTS graphs (
+                    state_hash   TEXT PRIMARY KEY,
+                    graph_json   TEXT NOT NULL,
+                    broken_json  TEXT NOT NULL,
+                    created_at   INTEGER
+                );
+            ''')
+
+    # ── Graph caching ─────────────────────────────────────────────────────
+    # The call graph is repo-global (cross-file edges), so it is cached as a
+    # whole, keyed by repo_state_hash(): the combined content hash of every
+    # Python file. Same pattern as symbols — content-addressed, no TTL logic.
+
+    _GRAPH_CACHE_KEEP = 5   # most-recent graph snapshots retained per db
+
+    def get_graph(self, state_hash: str) -> "Optional[Tuple[Dict[str, List[str]], List[str]]]":
+        """Return (graph, broken_files) for this exact repo state, or None."""
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT graph_json, broken_json FROM graphs WHERE state_hash = ?",
+                (state_hash,),
+            ).fetchone()
+        if row is None:
+            return None
+        return json.loads(row[0]), json.loads(row[1])
+
+    def put_graph(
+        self,
+        state_hash: str,
+        graph: Dict[str, List[str]],
+        broken_files: "List[str]",
+    ) -> None:
+        """Persist the graph for this repo state; prune old snapshots."""
+        import time as _time
+        with self._lock, self._conn:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO graphs VALUES (?, ?, ?, ?)",
+                (state_hash, json.dumps(graph), json.dumps(broken_files),
+                 int(_time.time())),
+            )
+            self._conn.execute(
+                """DELETE FROM graphs WHERE state_hash NOT IN (
+                       SELECT state_hash FROM graphs
+                       ORDER BY created_at DESC, rowid DESC LIMIT ?
+                   )""",
+                (self._GRAPH_CACHE_KEEP,),
+            )
+
+    def get_or_parse(
+        self,
+        filepath: str,
+        parse_fn: Callable[[str], Dict[str, Symbol]],
+        known_hash: "Optional[str]" = None,
+    ) -> Dict[str, Symbol]:
+        """
+        Return cached symbols if file hash matches, otherwise parse and persist.
+
+        `known_hash` lets a caller that already read and hashed the file
+        (the pipeline hashes every file for the repo state hash) skip a
+        second full disk read here. It MUST be the hash of the file's
+        current contents.
+        """
+        file_hash = known_hash if known_hash is not None else get_file_hash(filepath)
+
+        with self._lock:
+            cursor = self._conn.execute("SELECT file_hash FROM files WHERE file_path = ?", (filepath,))
+            row = cursor.fetchone()
+
+            if row and row[0] == file_hash:
+                # Cache hit!
+                cursor = self._conn.execute(
+                    "SELECT id, file_path, name, code, lineno FROM symbols WHERE file_path = ?",
+                    (filepath,)
+                )
+                symbols = {}
+                for row in cursor:
+                    sym_id, f_path, name, code, lineno = row
+                    symbols[sym_id] = Symbol(
+                        id=sym_id,
+                        file=f_path,
+                        name=name,
+                        code=code,
+                        lineno=lineno
+                    )
+                return symbols
+
+        # Cache miss or hash mismatch -> parse it (outside the lock; parsing
+        # can be slow and must not serialize other threads' cache hits)
+        symbols = parse_fn(filepath)
+
+        # Persist the new state
+        with self._lock, self._conn:
+            # DELETE CASCADE will drop all existing symbols for this file
+            self._conn.execute("DELETE FROM files WHERE file_path = ?", (filepath,))
+
+            self._conn.execute(
+                "INSERT INTO files (file_path, file_hash) VALUES (?, ?)",
+                (filepath, file_hash)
+            )
+
+            if symbols:
+                # REPLACE, not plain INSERT: rows are keyed by symbols.id but
+                # cleared via ON DELETE CASCADE from files.file_path, and those
+                # two only line up while Symbol.file is byte-identical to the
+                # filepath we just parsed. That holds for the Python parser
+                # (both absolute) but is not guaranteed for a language adapter
+                # that reports relative paths — stale rows would then survive
+                # the DELETE above and collide here on re-index.
+                self._conn.executemany(
+                    "INSERT OR REPLACE INTO symbols (id, file_path, name, code, lineno) VALUES (?, ?, ?, ?, ?)",
+                    [(s.id, s.file, s.name, s.code, s.lineno) for s in symbols.values()]
+                )
+
+        return symbols