codegraph-engine 2.1.1__py3-none-any.whl

codegraph/freshness.py

@@ -0,0 +1,295 @@
+"""Repository Freshness & Dependency-Aware Invalidation Engine.
+
+Tracks:
+  current Git HEAD, indexed Git HEAD, file hashes, parser version,
+  schema version, index generation, timestamps, and parse_failed state.
+
+Provides dependency-aware invalidation so localized file updates only
+invalidate affected symbols, references, graph edges, evidence, and cached
+context packets rather than rebuilding the entire repository.
+"""
+from __future__ import annotations
+
+import hashlib
+import shlex
+import sqlite3
+import subprocess
+from dataclasses import dataclass, field
+from enum import StrEnum
+from pathlib import Path
+
+
+class FreshnessStatus(StrEnum):
+    FRESH = "FRESH"
+    STALE = "STALE"
+    PARTIALLY_STALE = "PARTIALLY_STALE"
+    UNKNOWN = "UNKNOWN"
+
+
+@dataclass
+class FreshnessReport:
+    status: FreshnessStatus
+    indexed_commit: str | None
+    current_commit: str | None
+    index_timestamp: int | None
+    modified_files: list[str] = field(default_factory=list)
+    deleted_files: list[str] = field(default_factory=list)
+    parse_failed_files: list[str] = field(default_factory=list)
+    parser_version: str | None = None
+    index_generation: int = 0
+    detail: str = ""
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "status": self.status.value,
+            "indexed_commit": self.indexed_commit,
+            "current_commit": self.current_commit,
+            "index_timestamp": self.index_timestamp,
+            "modified_files": self.modified_files,
+            "deleted_files": self.deleted_files,
+            "parse_failed_files": self.parse_failed_files,
+            "parser_version": self.parser_version,
+            "index_generation": self.index_generation,
+            "detail": self.detail,
+        }
+
+
+def _run_git(repository: Path, *args: str, timeout: int = 10) -> str | None:
+    """Run an allowlisted read-only git command inside repository."""
+    if not shlex.split("git")[0]:  # pragma: no cover
+        return None
+    try:
+        result = subprocess.run(
+            ["git", *args],
+            cwd=str(repository),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return result.stdout.strip() if result.returncode == 0 else None
+    except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
+        return None
+
+
+def current_commit(repository: Path) -> str | None:
+    """Return the current HEAD commit SHA, or None if not a git repository."""
+    return _run_git(repository, "rev-parse", "HEAD")
+
+
+def current_branch(repository: Path) -> str | None:
+    """Return the current git branch name, or None."""
+    return _run_git(repository, "rev-parse", "--abbrev-ref", "HEAD")
+
+
+def _get_meta(con: sqlite3.Connection, key: str) -> str | None:
+    for table in ("metadata", "codegraph_meta"):
+        try:
+            row = con.execute(
+                f"SELECT value FROM {table} WHERE key=?", (key,)
+            ).fetchone()
+            if row and row[0] != "":
+                return str(row[0])
+        except sqlite3.OperationalError:
+            continue
+    return None
+
+
+def indexed_commit(con: sqlite3.Connection) -> str | None:
+    """Read the commit SHA recorded when the index was last built."""
+    return _get_meta(con, "indexed_commit")
+
+
+def index_timestamp(con: sqlite3.Connection) -> int | None:
+    """Return unix timestamp of the last indexing run."""
+    val = _get_meta(con, "index_timestamp")
+    try:
+        return int(val) if val is not None else None
+    except ValueError:
+        return None
+
+
+def index_generation(con: sqlite3.Connection) -> int:
+    """Return the monotonic index generation counter."""
+    val = _get_meta(con, "index_generation")
+    try:
+        return int(val) if val is not None else 0
+    except ValueError:
+        return 0
+
+
+def save_commit(con: sqlite3.Connection, commit: str | None, timestamp: int) -> None:
+    """Persist commit SHA and timestamp into metadata tables."""
+    _ensure_meta_table(con)
+    for table in ("codegraph_meta", "metadata"):
+        for key, val in [
+            ("indexed_commit", commit or ""),
+            ("index_timestamp", str(timestamp)),
+        ]:
+            try:
+                con.execute(
+                    f"INSERT INTO {table}(key, value) VALUES(?,?) "
+                    "ON CONFLICT(key) DO UPDATE SET value=excluded.value",
+                    (key, val),
+                )
+            except sqlite3.OperationalError:
+                pass
+
+
+def _ensure_meta_table(con: sqlite3.Connection) -> None:
+    con.execute(
+        "CREATE TABLE IF NOT EXISTS codegraph_meta "
+        "(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
+    )
+    con.execute(
+        "CREATE TABLE IF NOT EXISTS metadata "
+        "(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
+    )
+
+
+def compute_invalidated_dependents(
+    con: sqlite3.Connection, changed_paths: set[str]
+) -> dict[str, list[str]]:
+    """Given a set of modified/deleted file paths, compute dependent files and symbols that must be invalidated."""
+    from codegraph.indexing.models import normalize_module
+
+    if not changed_paths:
+        return {
+            "files": [],
+            "modules": [],
+            "symbols": [],
+            "dependent_files": [],
+        }
+
+    changed_modules = {normalize_module(p) for p in changed_paths}
+    invalidated_symbols: set[str] = set()
+    dependent_files: set[str] = set()
+
+    for p in changed_paths:
+        try:
+            rows = con.execute(
+                "SELECT canonical_id, qualified_name FROM symbols WHERE path=?", (p,)
+            ).fetchall()
+            for r in rows:
+                invalidated_symbols.add(str(r["canonical_id"] or r["qualified_name"]))
+        except sqlite3.OperationalError:
+            pass
+
+    for mod in changed_modules:
+        try:
+            imp_rows = con.execute(
+                "SELECT DISTINCT source_path FROM imports "
+                "WHERE resolved_module=? OR imported_module=? OR module=? OR module LIKE ?",
+                (mod, mod, mod, f"%{mod.split('.')[-1]}"),
+            ).fetchall()
+            for r in imp_rows:
+                if r["source_path"] not in changed_paths:
+                    dependent_files.add(str(r["source_path"]))
+        except sqlite3.OperationalError:
+            pass
+
+    return {
+        "files": sorted(changed_paths),
+        "modules": sorted(changed_modules),
+        "symbols": sorted(invalidated_symbols),
+        "dependent_files": sorted(dependent_files),
+    }
+
+
+def check_freshness(
+    repository: Path, con: sqlite3.Connection
+) -> FreshnessReport:
+    """Compare index state against current filesystem state."""
+    _ensure_meta_table(con)
+
+    idx_commit = indexed_commit(con)
+    cur_commit = current_commit(repository)
+    idx_ts = index_timestamp(con)
+    parser_ver = _get_meta(con, "parser_version")
+    gen = index_generation(con)
+
+    try:
+        file_count = con.execute("SELECT count(*) FROM files").fetchone()[0]
+    except sqlite3.OperationalError:
+        return FreshnessReport(
+            status=FreshnessStatus.UNKNOWN,
+            indexed_commit=idx_commit,
+            current_commit=cur_commit,
+            index_timestamp=idx_ts,
+            parser_version=parser_ver,
+            index_generation=gen,
+            detail="No index found. Run `codegraph index` first.",
+        )
+
+    if file_count == 0:
+        return FreshnessReport(
+            status=FreshnessStatus.UNKNOWN,
+            indexed_commit=idx_commit,
+            current_commit=cur_commit,
+            index_timestamp=idx_ts,
+            parser_version=parser_ver,
+            index_generation=gen,
+            detail="Index is empty. Run `codegraph index` first.",
+        )
+
+    modified: list[str] = []
+    deleted: list[str] = []
+    parse_failed: list[str] = []
+
+    try:
+        rows = con.execute("SELECT path, hash, status FROM files").fetchall()
+    except sqlite3.OperationalError:
+        rows = con.execute("SELECT path, hash, 'ok' AS status FROM files").fetchall()
+
+    for row in rows:
+        rel = str(row["path"])
+        if row["status"] == "parse_failed":
+            parse_failed.append(rel)
+            if rel not in modified:
+                modified.append(rel)
+        abs_path = repository / rel
+        if not abs_path.exists():
+            deleted.append(rel)
+        else:
+            try:
+                digest = hashlib.sha256(
+                    abs_path.read_text(encoding="utf-8", errors="replace").encode()
+                ).hexdigest()
+                if digest != row["hash"] and rel not in modified:
+                    modified.append(rel)
+            except OSError:
+                if rel not in modified:
+                    modified.append(rel)
+
+    changed = len(modified) + len(deleted)
+    if changed == 0 and not parse_failed and (idx_commit == cur_commit or cur_commit is None):
+        status = FreshnessStatus.FRESH
+        detail = "Index matches current repository state."
+    elif parse_failed and changed == len(parse_failed):
+        status = FreshnessStatus.PARTIALLY_STALE
+        detail = f"{len(parse_failed)} file(s) had syntax errors; last-known-good index retained (stale)."
+    elif changed == 0:
+        status = FreshnessStatus.PARTIALLY_STALE
+        detail = f"Commit changed ({_short(idx_commit)} → {_short(cur_commit)}) but no file hashes differ."
+    elif changed < file_count:
+        status = FreshnessStatus.PARTIALLY_STALE
+        detail = f"{changed} file(s) changed since indexing."
+    else:
+        status = FreshnessStatus.STALE
+        detail = f"All {file_count} indexed files may have changed. Re-index recommended."
+
+    return FreshnessReport(
+        status=status,
+        indexed_commit=idx_commit,
+        current_commit=cur_commit,
+        index_timestamp=idx_ts,
+        modified_files=modified,
+        deleted_files=deleted,
+        parse_failed_files=parse_failed,
+        parser_version=parser_ver,
+        index_generation=gen,
+        detail=detail,
+    )
+
+
+def _short(sha: str | None) -> str:
+    return sha[:8] if sha else "unknown"

codegraph/git.py

@@ -0,0 +1,278 @@
+"""Read-only Git intelligence.
+
+All Git operations use an explicit allowlist of commands and validated
+arguments.  No arbitrary shell execution.  No network access.
+"""
+from __future__ import annotations
+
+import sqlite3
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class CommitInfo:
+    sha: str
+    author: str
+    date: str
+    message: str
+
+    def as_dict(self) -> dict[str, str]:
+        return {"sha": self.sha, "author": self.author, "date": self.date, "message": self.message}
+
+
+@dataclass
+class FileDiff:
+    path: str
+    status: str   # M=modified, A=added, D=deleted, R=renamed
+    diff_snippet: str = ""
+
+    def as_dict(self) -> dict[str, str]:
+        return {"path": self.path, "status": self.status, "diff_snippet": self.diff_snippet}
+
+
+def _git(repository: Path, *args: str, timeout: int = 15) -> str | None:
+    """Run a git subprocess with validated arguments only."""
+    # Validate: no shell metacharacters, no absolute paths from user input
+    for arg in args:
+        if any(c in arg for c in (";", "|", "&", "`", "$", "\n", "\r")):
+            return None
+    try:
+        result = subprocess.run(
+            ["git", *args],
+            cwd=str(repository),
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return result.stdout if result.returncode == 0 else None
+    except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
+        return None
+
+
+def is_git_repository(repository: Path) -> bool:
+    return _git(repository, "rev-parse", "--git-dir") is not None
+
+
+def current_commit(repository: Path) -> str | None:
+    """Return the current HEAD commit SHA, or None if not a git repository."""
+    raw = _git(repository, "rev-parse", "HEAD")
+    return raw.strip() if raw else None
+
+
+def recent_commits(
+    repository: Path,
+    n: int = 10,
+    path: str | None = None,
+) -> list[CommitInfo]:
+    """Return the last N commits, optionally scoped to a file path."""
+    if not 1 <= n <= 100:
+        n = 10
+    args = ["log", f"-{n}", "--format=%H\x1f%an\x1f%ai\x1f%s"]
+    if path:
+        # Sanitize: path must not be absolute or contain traversal
+        if ".." in path or path.startswith("/"):
+            return []
+        args += ["--", path]
+    raw = _git(repository, *args)
+    if not raw:
+        return []
+    commits = []
+    for line in raw.strip().splitlines():
+        parts = line.split("\x1f", 3)
+        if len(parts) == 4:
+            commits.append(CommitInfo(*parts))
+    return commits
+
+
+def changed_files(
+    repository: Path,
+    since: str = "HEAD~10",
+    until: str = "HEAD",
+) -> list[FileDiff]:
+    """Return files changed between two commits."""
+    # Validate refs — allow only hex SHAs, HEAD, HEAD~N, branch names (alphanumeric + /-_.)
+    import re
+    _REF = re.compile(r"^[A-Za-z0-9_.~^/-]{1,100}$")
+    if not _REF.match(since) or not _REF.match(until):
+        return []
+    raw = _git(repository, "diff", "--name-status", since, until)
+    if not raw:
+        return []
+    diffs: list[FileDiff] = []
+    for line in raw.strip().splitlines():
+        parts = line.split("\t", 1)
+        if len(parts) == 2:
+            status, path = parts[0].strip()[0], parts[1].strip()
+            diffs.append(FileDiff(path=path, status=status))
+    return diffs
+
+
+def file_diff(
+    repository: Path,
+    path: str,
+    since: str = "HEAD~1",
+    until: str = "HEAD",
+    max_lines: int = 50,
+) -> str:
+    """Return a bounded unified diff for a single file."""
+    import re
+    _REF = re.compile(r"^[A-Za-z0-9_.~^/-]{1,100}$")
+    if not _REF.match(since) or not _REF.match(until):
+        return ""
+    if ".." in path or path.startswith("/"):
+        return ""
+    raw = _git(repository, "diff", since, until, "--", path)
+    if not raw:
+        return ""
+    lines = raw.splitlines()[:max_lines]
+    return "\n".join(lines)
+
+
+def blame_line(
+    repository: Path,
+    path: str,
+    line: int,
+) -> dict[str, str] | None:
+    """Return blame info for a single line."""
+    if ".." in path or path.startswith("/"):
+        return None
+    if not 1 <= line <= 100_000:
+        return None
+    raw = _git(
+        repository, "blame", f"-L{line},{line}", "--porcelain", "--", path
+    )
+    if not raw:
+        return None
+    info: dict[str, str] = {}
+    lines = raw.splitlines()
+    if lines:
+        info["sha"] = lines[0].split()[0] if lines else ""
+    for bl in lines[1:]:
+        if bl.startswith("author "):
+            info["author"] = bl[7:]
+        elif bl.startswith("author-time "):
+            info["timestamp"] = bl[12:]
+        elif bl.startswith("summary "):
+            info["summary"] = bl[8:]
+    return info or None
+
+
+def changed_symbols_since(
+    repository: Path,
+    since: str,
+    symbol_names: set[str],
+) -> list[str]:
+    """Return subset of symbol_names that appear in changed lines since `since`.
+
+    This is a heuristic: it checks whether the symbol name appears in the
+    diff text.  Results are LOW confidence.
+    """
+    import re
+    _REF = re.compile(r"^[A-Za-z0-9_.~^/-]{1,100}$")
+    if not _REF.match(since):
+        return []
+    raw = _git(repository, "diff", since, "HEAD")
+    if not raw:
+        return []
+    found = []
+    for name in symbol_names:
+        # Only match if the name appears in added/removed diff lines
+        if re.search(rf"^[+-].*\b{re.escape(name)}\b", raw, re.MULTILINE):
+            found.append(name)
+    return found
+
+
+def get_file_history(
+    repository: Path,
+    path: str,
+    n: int = 10,
+) -> list[dict[str, str]]:
+    """Return commit history for a single file path."""
+    commits = recent_commits(repository, n=n, path=path)
+    return [c.as_dict() for c in commits]
+
+
+def get_symbol_history(
+    repository: Path,
+    path: str,
+    symbol: str,
+    n: int = 10,
+) -> list[dict[str, str]]:
+    """Return commits where a specific symbol name was modified in a file."""
+    all_file_commits = recent_commits(repository, n=n * 2, path=path)
+    matching: list[dict[str, str]] = []
+    import re
+    for c in all_file_commits:
+        diff_text = file_diff(repository, path, since=f"{c.sha}~1", until=c.sha, max_lines=200)
+        if re.search(rf"\b{re.escape(symbol)}\b", diff_text):
+            matching.append(c.as_dict())
+            if len(matching) >= n:
+                break
+    return matching
+
+
+def get_change_context(
+    repository: Path,
+    since: str = "HEAD~1",
+    until: str = "HEAD",
+) -> dict[str, object]:
+    """Return summary of changed files and bounded snippets."""
+    diffs = changed_files(repository, since=since, until=until)
+    file_summaries: list[dict[str, str]] = []
+    for d in diffs[:20]:
+        snippet = file_diff(repository, d.path, since=since, until=until, max_lines=30)
+        file_summaries.append(
+            {
+                "path": d.path,
+                "status": d.status,
+                "diff_snippet": snippet,
+            }
+        )
+    return {
+        "since": since,
+        "until": until,
+        "total_changed": len(diffs),
+        "files": file_summaries,
+    }
+
+
+def analyze_change_impact(
+    con: sqlite3.Connection,
+    repository: Path,
+    since: str = "HEAD~1",
+    until: str = "HEAD",
+) -> dict[str, object]:
+    """Analyze downstream callers and tests affected by changes between since and until."""
+    diffs = changed_files(repository, since=since, until=until)
+    changed_paths = [d.path for d in diffs]
+    changed_symbols: list[str] = []
+    affected_callers: list[dict[str, object]] = []
+    affected_tests: list[dict[str, object]] = []
+
+    from codegraph.graph.traversal import find_callers, find_related_tests
+
+    for p in changed_paths:
+        rows = con.execute("SELECT name, qualified_name FROM symbols WHERE path=?", (p,)).fetchall()
+        sym_names = {r["name"] for r in rows}
+        matched = changed_symbols_since(repository, since, sym_names)
+        for s in matched:
+            changed_symbols.append(s)
+            callers = find_callers(con, s, max_results=5)
+            for c in callers:
+                affected_callers.append(dict(c))
+            tests = find_related_tests(con, s, max_results=5)
+            for t in tests:
+                if "result" not in t:
+                    affected_tests.append(dict(t))
+
+    return {
+        "since": since,
+        "until": until,
+        "changed_files": [d.as_dict() for d in diffs],
+        "changed_symbols": changed_symbols,
+        "affected_callers": affected_callers,
+        "affected_tests": affected_tests,
+    }
+

codegraph/graph/__init__.py

@@ -0,0 +1,46 @@
+from .models import GraphEdge, GraphSeedPolicy
+from .traversal import (
+    analyze_impact,
+    definition_edges,
+    find_affected_tests,
+    find_callees,
+    find_callers,
+    find_implementations,
+    find_importers,
+    find_references,
+    find_related_tests,
+    find_symbol,
+    find_symbol_exact,
+    find_symbols,
+    get_call_graph,
+    get_callees,
+    get_callers,
+    get_dependency_graph,
+    get_symbol,
+    import_edges,
+    trace_call,
+)
+
+__all__ = [
+    "GraphEdge",
+    "analyze_impact",
+    "definition_edges",
+    "find_affected_tests",
+    "find_callees",
+    "find_callers",
+    "find_implementations",
+    "find_importers",
+    "find_references",
+    "find_related_tests",
+    "find_symbol",
+    "find_symbol_exact",
+    "find_symbols",
+    "get_call_graph",
+    "get_callees",
+    "get_callers",
+    "get_dependency_graph",
+    "get_symbol",
+    "import_edges",
+    "trace_call",
+    "GraphSeedPolicy",
+]

codegraph/graph/models.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+
+
+@dataclass(frozen=True)
+class GraphEdge:
+    source: str
+    target: str
+    relationship: str
+    confidence: str
+    file: str
+    start_line: int
+    end_line: int
+    evidence: str
+
+    def as_dict(self) -> dict[str, object]:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class GraphSeedPolicy:
+    """Policy for selecting grounded roots for graph traversal."""
+
+    source_type: str = "EXPLICIT_TARGET"  # EXPLICIT_TARGET | ROUTE_HANDLER | DIRECT_MATCH | USER_REQUESTED
+    min_confidence: str = "HIGH"  # HIGH | MEDIUM | LOW
+    max_seeds: int = 6
+    max_depth: int = 3
+    max_nodes_budget: int = 50
+    allowed_edges: frozenset[str] = frozenset({
+        "CALLS",
+        "CALLED_BY",
+        "HANDLED_BY",
+        "ROUTES_TO",
+        "DEPENDS_ON",
+        "IMPORTS",
+        "TESTS",
+        "DEFINES",
+        "CONTAINS",
+    })
+