superlocalmemory 3.3.20 → 3.3.21

package/src/superlocalmemory/code_graph/graph_engine.py

@@ -0,0 +1,295 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory v3.4 — CodeGraph Module
+
+"""GraphEngine — rustworkx in-memory directed graph.
+
+Loads from SQLite via GraphStore, caches in a rustworkx PyDiGraph,
+and provides O(1) node lookup + O(degree) traversals.
+
+Cache invalidation: rebuild when store.version changes.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+from superlocalmemory.code_graph.graph_store import GraphStore
+from superlocalmemory.code_graph.models import EdgeKind, GraphNode
+
+logger = logging.getLogger(__name__)
+
+try:
+    import rustworkx as rx  # type: ignore[import-untyped]
+except ImportError as _rx_err:
+    rx = None  # type: ignore[assignment]
+    _RX_IMPORT_ERROR = _rx_err
+else:
+    _RX_IMPORT_ERROR = None
+
+
+class RustworkxNotInstalledError(ImportError):
+    """Raised when rustworkx is required but not installed."""
+
+
+class NodeNotFoundError(KeyError):
+    """Raised when a node_id is not present in the graph."""
+
+
+def _require_rustworkx() -> None:
+    """Guard: raise if rustworkx is not available."""
+    if rx is None:
+        raise RustworkxNotInstalledError(
+            "rustworkx is required for GraphEngine. "
+            "Install it with: pip install rustworkx"
+        ) from _RX_IMPORT_ERROR
+
+
+# ---------------------------------------------------------------------------
+# GraphIndex — bidirectional ID mapping
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class GraphIndex:
+    """Bidirectional mapping between SQLite TEXT node_id and rustworkx int index."""
+
+    id_to_rx: dict[str, int] = field(default_factory=dict)
+    rx_to_id: dict[int, str] = field(default_factory=dict)
+    qname_to_id: dict[str, str] = field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# GraphEngine
+# ---------------------------------------------------------------------------
+
+class GraphEngine:
+    """In-memory directed graph backed by rustworkx PyDiGraph.
+
+    Lazily built from SQLite on first query. Automatically rebuilds
+    when the underlying GraphStore version changes (writes detected).
+    """
+
+    def __init__(self, store: GraphStore) -> None:
+        _require_rustworkx()
+        self._store = store
+        self._graph: rx.PyDiGraph | None = None  # type: ignore[name-defined]
+        self._index: GraphIndex = GraphIndex()
+        self._graph_version: int = -1
+
+    # ------------------------------------------------------------------
+    # Graph lifecycle
+    # ------------------------------------------------------------------
+
+    def build_graph(self) -> rx.PyDiGraph:  # type: ignore[name-defined]
+        """Load all nodes and edges from SQLite into a rustworkx PyDiGraph.
+
+        Returns the cached graph if the store version hasn't changed.
+        """
+        if (
+            self._graph is not None
+            and self._graph_version == self._store.version
+        ):
+            return self._graph
+
+        nodes, edges = self._store.get_all_nodes_and_edges()
+
+        graph = rx.PyDiGraph(multigraph=True)
+
+        id_to_rx: dict[str, int] = {}
+        rx_to_id: dict[int, str] = {}
+        qname_to_id: dict[str, str] = {}
+
+        # Add nodes
+        for node in nodes:
+            node_data: dict[str, Any] = {
+                "node_id": node.node_id,
+                "kind": node.kind.value,
+                "name": node.name,
+                "qualified_name": node.qualified_name,
+                "file_path": node.file_path,
+                "line_start": node.line_start,
+                "line_end": node.line_end,
+                "language": node.language,
+                "parent_name": node.parent_name,
+                "is_test": node.is_test,
+                "community_id": node.community_id,
+            }
+            rx_idx = graph.add_node(node_data)
+            id_to_rx[node.node_id] = rx_idx
+            rx_to_id[rx_idx] = node.node_id
+            qname_to_id[node.qualified_name] = node.node_id
+
+        # Add edges
+        for edge in edges:
+            src_rx = id_to_rx.get(edge.source_node_id)
+            tgt_rx = id_to_rx.get(edge.target_node_id)
+            if src_rx is None or tgt_rx is None:
+                logger.warning(
+                    "Skipping dangling edge %s -> %s (kind=%s)",
+                    edge.source_node_id, edge.target_node_id, edge.kind.value,
+                )
+                continue
+            edge_data: dict[str, Any] = {
+                "edge_id": edge.edge_id,
+                "kind": edge.kind.value,
+                "file_path": edge.file_path,
+                "line": edge.line,
+                "confidence": edge.confidence,
+            }
+            graph.add_edge(src_rx, tgt_rx, edge_data)
+
+        # Cache
+        self._graph = graph
+        self._index = GraphIndex(
+            id_to_rx=id_to_rx,
+            rx_to_id=rx_to_id,
+            qname_to_id=qname_to_id,
+        )
+        self._graph_version = self._store.version
+        logger.debug(
+            "Built graph: %d nodes, %d edges",
+            graph.num_nodes(), graph.num_edges(),
+        )
+        return graph
+
+    def invalidate(self) -> None:
+        """Force a graph rebuild on next access."""
+        self._graph = None
+        self._graph_version = -1
+
+    @property
+    def index(self) -> GraphIndex:
+        """Current graph index (builds graph if needed)."""
+        self._ensure_graph()
+        return self._index
+
+    @property
+    def graph(self) -> rx.PyDiGraph:  # type: ignore[name-defined]
+        """Current rustworkx graph (builds if needed)."""
+        return self._ensure_graph()
+
+    # ------------------------------------------------------------------
+    # Query operations
+    # ------------------------------------------------------------------
+
+    def get_callers(
+        self,
+        node_id: str,
+        edge_kinds: set[str] | None = None,
+    ) -> list[dict[str, Any]]:
+        """Return all nodes that have edges pointing *to* node_id.
+
+        Each result is ``{"node": <node_data>, "edge": <edge_data>}``.
+        """
+        graph = self._ensure_graph()
+        rx_idx = self._resolve_rx(node_id)
+
+        results: list[dict[str, Any]] = []
+        for src_rx, _tgt_rx, edge_data in graph.in_edges(rx_idx):
+            if edge_kinds is not None and edge_data["kind"] not in edge_kinds:
+                continue
+            results.append({
+                "node": dict(graph[src_rx]),
+                "edge": dict(edge_data),
+            })
+        return results
+
+    def get_callees(
+        self,
+        node_id: str,
+        edge_kinds: set[str] | None = None,
+    ) -> list[dict[str, Any]]:
+        """Return all nodes that node_id has edges pointing *to*.
+
+        Each result is ``{"node": <node_data>, "edge": <edge_data>}``.
+        """
+        graph = self._ensure_graph()
+        rx_idx = self._resolve_rx(node_id)
+
+        results: list[dict[str, Any]] = []
+        for _src_rx, tgt_rx, edge_data in graph.out_edges(rx_idx):
+            if edge_kinds is not None and edge_data["kind"] not in edge_kinds:
+                continue
+            results.append({
+                "node": dict(graph[tgt_rx]),
+                "edge": dict(edge_data),
+            })
+        return results
+
+    def get_tests_for(self, node_id: str) -> list[dict[str, Any]]:
+        """Return test nodes associated with *node_id*.
+
+        Checks:
+        1. Outgoing TESTED_BY edges from node_id
+        2. Incoming CALLS edges from nodes where is_test=True
+        """
+        graph = self._ensure_graph()
+        rx_idx = self._resolve_rx(node_id)
+
+        seen_ids: set[str] = set()
+        results: list[dict[str, Any]] = []
+
+        # Outgoing TESTED_BY
+        for _src, tgt_rx, edge_data in graph.out_edges(rx_idx):
+            if edge_data["kind"] == EdgeKind.TESTED_BY.value:
+                tgt_data = dict(graph[tgt_rx])
+                tid = tgt_data["node_id"]
+                if tid not in seen_ids:
+                    seen_ids.add(tid)
+                    results.append(tgt_data)
+
+        # Incoming CALLS from test nodes
+        for src_rx, _tgt, edge_data in graph.in_edges(rx_idx):
+            if edge_data["kind"] == EdgeKind.CALLS.value:
+                src_data = dict(graph[src_rx])
+                if src_data.get("is_test") and src_data["node_id"] not in seen_ids:
+                    seen_ids.add(src_data["node_id"])
+                    results.append(src_data)
+
+        return results
+
+    def get_connected_component(self, node_id: str) -> list[str]:
+        """Return all node_ids in the same weakly-connected component.
+
+        Uses rustworkx.weakly_connected_components.
+        """
+        graph = self._ensure_graph()
+        rx_idx = self._resolve_rx(node_id)
+
+        components = rx.weakly_connected_components(graph)
+        for component in components:
+            if rx_idx in component:
+                return [self._index.rx_to_id[i] for i in component]
+
+        # Shouldn't reach here if node exists, but return just the node
+        return [node_id]
+
+    def get_node_data(self, node_id: str) -> dict[str, Any]:
+        """Return the node data dict for a node_id."""
+        graph = self._ensure_graph()
+        rx_idx = self._resolve_rx(node_id)
+        return dict(graph[rx_idx])
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _ensure_graph(self) -> rx.PyDiGraph:  # type: ignore[name-defined]
+        """Build graph if needed and return it."""
+        if (
+            self._graph is None
+            or self._graph_version != self._store.version
+        ):
+            return self.build_graph()
+        return self._graph
+
+    def _resolve_rx(self, node_id: str) -> int:
+        """Resolve a TEXT node_id to a rustworkx index. Raises NodeNotFoundError."""
+        rx_idx = self._index.id_to_rx.get(node_id)
+        if rx_idx is None:
+            raise NodeNotFoundError(
+                f"Node '{node_id}' not found in graph"
+            )
+        return rx_idx

package/src/superlocalmemory/code_graph/graph_store.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory v3.4 — CodeGraph Module
+
+"""GraphStore — thin graph-specific layer over CodeGraphDatabase.
+
+All graph writes go through this layer.  Provides:
+- Atomic file replacement  (store_file_nodes_edges)
+- Bulk read for in-memory graph building  (get_all_nodes_and_edges)
+- File removal  (remove_file)
+- Version tracking for cache invalidation
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Sequence
+
+from superlocalmemory.code_graph.database import CodeGraphDatabase
+from superlocalmemory.code_graph.models import (
+    FileRecord,
+    GraphEdge,
+    GraphNode,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GraphStore:
+    """SQLite persistence layer for graph nodes, edges, and file records.
+
+    Delegates to CodeGraphDatabase but adds higher-level operations
+    that Phase 2+ modules depend on (bulk load, atomic replace, version).
+    """
+
+    def __init__(self, db: CodeGraphDatabase) -> None:
+        self._db = db
+
+    # ------------------------------------------------------------------
+    # Properties
+    # ------------------------------------------------------------------
+
+    @property
+    def db(self) -> CodeGraphDatabase:
+        """Underlying database instance."""
+        return self._db
+
+    @property
+    def version(self) -> int:
+        """Monotonic write-version for cache invalidation."""
+        return self._db.version
+
+    # ------------------------------------------------------------------
+    # Write operations
+    # ------------------------------------------------------------------
+
+    def store_file_nodes_edges(
+        self,
+        file_path: str,
+        nodes: Sequence[GraphNode],
+        edges: Sequence[GraphEdge],
+        file_record: FileRecord,
+    ) -> None:
+        """Atomically replace all data for *file_path*.
+
+        Within a single transaction:
+        1. Delete old edges for this file
+        2. Delete old nodes for this file
+        3. Insert new nodes
+        4. Insert new edges
+        5. Upsert file record
+
+        The database's ``store_file_parse_results`` already does this.
+        """
+        self._db.store_file_parse_results(
+            file_path,
+            list(nodes),
+            list(edges),
+            file_record,
+        )
+        logger.debug(
+            "Stored %d nodes, %d edges for %s",
+            len(nodes), len(edges), file_path,
+        )
+
+    def remove_file(self, file_path: str) -> None:
+        """Remove all graph data for *file_path*.
+
+        Deletes nodes (cascade → edges via FK), edges sourced from this
+        file, and the file record.  All within a transaction.
+        """
+        with self._db.transaction():
+            self._db.delete_edges_by_file(file_path)
+            self._db.delete_nodes_by_file(file_path)
+            self._db.delete_file_record(file_path)
+        logger.debug("Removed all data for %s", file_path)
+
+    # ------------------------------------------------------------------
+    # Read operations
+    # ------------------------------------------------------------------
+
+    def get_all_nodes_and_edges(
+        self,
+    ) -> tuple[list[GraphNode], list[GraphEdge]]:
+        """Load every node and edge — used by GraphEngine.build_graph().
+
+        Returns (nodes, edges) as plain lists.
+        """
+        nodes = self._db.get_all_nodes()
+        edges = self._db.get_all_edges()
+        return nodes, edges
+
+    def get_nodes_by_file(self, file_path: str) -> list[GraphNode]:
+        """All nodes in *file_path*, ordered by line_start."""
+        return self._db.get_nodes_by_file(file_path)
+
+    def get_node(self, node_id: str) -> GraphNode | None:
+        """Single node by ID."""
+        return self._db.get_node(node_id)
+
+    def get_file_record(self, file_path: str) -> FileRecord | None:
+        """File record by path."""
+        return self._db.get_file_record(file_path)
+
+    def get_all_file_records(self) -> list[FileRecord]:
+        """All tracked file records."""
+        return self._db.get_all_file_records()
+
+    # ------------------------------------------------------------------
+    # Dependent tracing (used by IncrementalUpdater)
+    # ------------------------------------------------------------------
+
+    def find_dependents(self, file_path: str) -> set[str]:
+        """Return file paths that have edges *targeting* nodes in *file_path*.
+
+        Looks for IMPORTS, CALLS, INHERITS, DEPENDS_ON edges whose
+        target lives in *file_path* but whose source is in a *different* file.
+        """
+        rows = self._db.execute(
+            """
+            SELECT DISTINCT ge.file_path
+            FROM graph_edges ge
+            JOIN graph_nodes gn_target
+                ON ge.target_node_id = gn_target.node_id
+            WHERE gn_target.file_path = ?
+              AND ge.file_path != ?
+            """,
+            (file_path, file_path),
+        )
+        return {row["file_path"] for row in rows}
+
+    # ------------------------------------------------------------------
+    # Stats
+    # ------------------------------------------------------------------
+
+    def get_stats(self) -> dict[str, int]:
+        """Delegate to DB stats."""
+        return self._db.get_stats()

package/src/superlocalmemory/code_graph/incremental.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory v3.4 — CodeGraph Module
+
+"""IncrementalUpdater — hash-based change detection + dependent tracing.
+
+Given a list of changed file paths:
+1. SHA-256 hash check to skip unchanged files
+2. Re-parse changed files
+3. Trace dependents via IMPORTS edges
+4. Re-parse dependent files (their edges may be stale)
+5. Invalidate the in-memory graph cache
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Protocol
+
+from superlocalmemory.code_graph.graph_engine import GraphEngine
+from superlocalmemory.code_graph.graph_store import GraphStore
+from superlocalmemory.code_graph.models import (
+    FileRecord,
+    GraphEdge,
+    GraphNode,
+    ParseResult,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Parser protocol (dependency inversion — no import of Phase 1 parser)
+# ---------------------------------------------------------------------------
+
+class ParserProtocol(Protocol):
+    """Minimal interface the IncrementalUpdater needs from a parser."""
+
+    def parse_file(self, file_path: str, repo_root: Path) -> ParseResult:
+        """Parse a single file and return nodes, edges, file record."""
+        ...
+
+
+# ---------------------------------------------------------------------------
+# Result dataclass
+# ---------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class UpdateResult:
+    """Result of an incremental update run."""
+
+    parsed: int = 0
+    skipped: int = 0
+    deleted: int = 0
+    dependents_parsed: int = 0
+    errors: tuple[str, ...] = ()
+
+
+# ---------------------------------------------------------------------------
+# IncrementalUpdater
+# ---------------------------------------------------------------------------
+
+class IncrementalUpdater:
+    """Hash-based incremental graph updater.
+
+    Usage::
+
+        updater = IncrementalUpdater(store, engine)
+        result = updater.update(
+            changed_files=["src/foo.py", "src/bar.py"],
+            parser=my_parser,
+            repo_root=Path("/my/repo"),
+        )
+    """
+
+    def __init__(self, store: GraphStore, engine: GraphEngine) -> None:
+        self._store = store
+        self._engine = engine
+
+    def update(
+        self,
+        changed_files: list[str],
+        parser: ParserProtocol,
+        repo_root: Path,
+    ) -> UpdateResult:
+        """Run incremental update for *changed_files*.
+
+        Parameters
+        ----------
+        changed_files : relative file paths (relative to repo_root)
+        parser : object implementing ParserProtocol
+        repo_root : absolute path to repository root
+
+        Returns
+        -------
+        UpdateResult with counts of parsed/skipped/deleted/dependents_parsed.
+        """
+        if not changed_files:
+            return UpdateResult()
+
+        # Load stored hashes
+        stored_hashes: dict[str, str] = {}
+        for rec in self._store.get_all_file_records():
+            stored_hashes[rec.file_path] = rec.content_hash
+
+        # Partition files
+        files_to_parse: list[tuple[str, str]] = []  # (rel_path, new_hash)
+        files_to_delete: list[str] = []
+        skipped = 0
+        errors: list[str] = []
+
+        for rel_path in changed_files:
+            abs_path = repo_root / rel_path
+            if not abs_path.exists():
+                files_to_delete.append(rel_path)
+                continue
+
+            try:
+                content_bytes = abs_path.read_bytes()
+            except OSError as exc:
+                errors.append(f"Cannot read {rel_path}: {exc}")
+                continue
+
+            new_hash = hashlib.sha256(content_bytes).hexdigest()
+
+            if stored_hashes.get(rel_path) == new_hash:
+                skipped += 1
+                continue
+
+            files_to_parse.append((rel_path, new_hash))
+
+        # Trace dependents BEFORE modifying the store — FK cascades
+        # would destroy the edges we need for dependency resolution.
+        all_changing = {fp for fp, _ in files_to_parse} | set(files_to_delete)
+        dependent_files: set[str] = set()
+        for fp in all_changing:
+            deps = self._store.find_dependents(fp)
+            dependent_files.update(deps)
+
+        # Delete removed files
+        deleted = 0
+        for fp in files_to_delete:
+            self._store.remove_file(fp)
+            deleted += 1
+
+        # Parse changed files
+        parsed = 0
+        parsed_paths: set[str] = set()
+        for rel_path, _new_hash in files_to_parse:
+            try:
+                result = parser.parse_file(rel_path, repo_root)
+                self._store.store_file_nodes_edges(
+                    result.file_path,
+                    list(result.nodes),
+                    list(result.edges),
+                    result.file_record,
+                )
+                parsed += 1
+                parsed_paths.add(rel_path)
+            except Exception as exc:
+                errors.append(f"Parse error {rel_path}: {exc}")
+                logger.warning("Failed to parse %s: %s", rel_path, exc)
+
+        # Remove already-handled files
+        dependent_files -= parsed_paths
+        dependent_files -= set(files_to_delete)
+
+        # Re-parse dependents
+        dependents_parsed = 0
+        for dep_path in dependent_files:
+            abs_dep = repo_root / dep_path
+            if not abs_dep.exists():
+                continue
+            try:
+                result = parser.parse_file(dep_path, repo_root)
+                self._store.store_file_nodes_edges(
+                    result.file_path,
+                    list(result.nodes),
+                    list(result.edges),
+                    result.file_record,
+                )
+                dependents_parsed += 1
+            except Exception as exc:
+                errors.append(f"Dependent parse error {dep_path}: {exc}")
+                logger.warning("Failed to parse dependent %s: %s", dep_path, exc)
+
+        # Invalidate engine cache
+        self._engine.invalidate()
+
+        return UpdateResult(
+            parsed=parsed,
+            skipped=skipped,
+            deleted=deleted,
+            dependents_parsed=dependents_parsed,
+            errors=tuple(errors),
+        )