superlocalmemory 4.0.5 → 4.0.7

package/src/superlocalmemory/code_graph/bridge/maintenance.py

@@ -0,0 +1,206 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory | https://qualixar.com
+
+"""Bridge pass — runs the code↔memory bridge as background maintenance.
+
+WHY THIS MODULE EXISTS
+----------------------
+The bridge was authored to run from ``BridgeEventListeners.on_memory_stored``,
+i.e. once per ``memory.stored`` event. ``EventBus._notify_listeners`` calls
+listeners **synchronously on the emitting thread**, so that design puts entity
+resolution, enrichment and Hebbian linking inside every single ``remember``.
+The owner's constraint for this release is explicit: remember and recall timing
+must not move. So the memory-stored listener is gone and the work happens here,
+in the same background pass that already runs ``consolidate_facts``.
+
+WHAT THIS PASS TOUCHES
+----------------------
+Writes to ``code_graph.db`` only:
+  * ``code_memory_links``            — EntityResolver output
+  * ``code_memory_links.enriched_description`` — FactEnricher output
+
+Recall never opens ``code_graph.db`` — verified: nothing under ``retrieval/`` or
+``core/`` references ``code_memory_links`` or ``CodeGraphDatabase``. That makes
+this half of the bridge recall-neutral by construction rather than by
+measurement.
+
+DELIBERATELY NOT HERE: Hebbian association edges. ``HebbianLinker`` produces
+edges for ``association_edges`` in **memory.db**, which
+``retrieval/spreading_activation.py`` reads via a UNION with ``graph_edges``.
+Every such edge is an extra neighbour recall must traverse and changes which
+memories come back — not just how fast. That cannot be made safe by moving it
+into this pass, so it is deferred to its own release, where the edge volume at
+production scale can be measured against the recall baseline first. No writer for
+it ships here; ``bridge/hebbian_linker.py`` remains unwired on purpose.
+
+IDEMPOTENCE
+-----------
+A watermark in ``graph_metadata`` records the ``created_at`` of the newest fact
+processed. Re-running the pass processes only facts newer than that, so a
+maintenance cycle every few minutes does not rescan the whole store. Links use
+``INSERT OR REPLACE`` on a deterministic key, so reprocessing a fact cannot
+duplicate its links.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:  # pragma: no cover - typing only
+    from superlocalmemory.code_graph.database import CodeGraphDatabase
+
+logger = logging.getLogger(__name__)
+
+#: Watermark key in graph_metadata.
+_WATERMARK_KEY = "bridge.last_fact_created_at"
+
+#: Facts examined in a single pass. Bounds the pass so a first run on a large
+#: store cannot occupy the maintenance thread indefinitely; the watermark means
+#: the next cycle resumes where this one stopped.
+MAX_FACTS_PER_PASS = 500
+
+#: Links kept for one fact, highest confidence first. A single file-path mention
+#: matches every node in that file — "the parser in code_graph/parser.py was
+#: dropping edges" produced 17 links at confidence 0.6-0.8, against one link at
+#: 0.95 for a backticked function name. Without a bound the broad matches bury
+#: the precise ones in the UI and hand a large node set to the Hebbian pass.
+MAX_LINKS_PER_FACT = 10
+
+
+def _fact_rows(
+    memory_db: Any,
+    profile_id: str,
+    since: str | None,
+    limit: int,
+) -> list[tuple[str, str, str]]:
+    """Return (fact_id, content, created_at) for facts newer than *since*.
+
+    Ordered by ``created_at`` so the watermark advances monotonically even when
+    the pass stops at ``limit``.
+    """
+    sql = (
+        "SELECT fact_id, content, created_at FROM atomic_facts "
+        "WHERE profile_id = ? AND content IS NOT NULL AND content != '' "
+    )
+    # M011 (archive_status) is a DEFERRED migration, so the column is absent on
+    # a database where it has not run yet. DatabaseManager._has_archive_status
+    # exists for exactly this and its docstring is explicit: "callers must not
+    # filter on a column that may not exist." Filtering unconditionally made
+    # this query raise "no such column" on any fresh install, which the caller's
+    # except swallowed into a warning and zero links — the bridge would simply
+    # never have run for a new user.
+    try:
+        has_archive = memory_db._has_archive_status()
+    except Exception:  # pragma: no cover - helper absent on an unusual manager
+        has_archive = False
+    if has_archive:
+        sql += "AND (archive_status IS NULL OR archive_status = '') "
+
+    params: list[Any] = [profile_id]
+    if since:
+        sql += "AND created_at > ? "
+        params.append(since)
+    sql += "ORDER BY created_at ASC LIMIT ?"
+    params.append(limit)
+
+    # DatabaseManager.execute serves both reads and writes (see
+    # core/maintenance.py, which uses it for each). It returns sqlite3.Row;
+    # index by position so a plain-tuple factory also works.
+    rows = memory_db.execute(sql, tuple(params))
+    return [(r[0], r[1], r[2]) for r in rows]
+
+
+def run_bridge_pass(
+    memory_db: Any,
+    code_graph_db: CodeGraphDatabase,
+    profile_id: str,
+    *,
+    max_facts: int = MAX_FACTS_PER_PASS,
+) -> dict[str, int]:
+    """Resolve code mentions in new facts and enrich the resulting links.
+
+    Returns counts. Never raises — the caller is background maintenance and a
+    bridge failure must not abort the rest of the cycle.
+    """
+    counts = {"facts_scanned": 0, "links_created": 0, "enriched": 0}
+
+    try:
+        from superlocalmemory.code_graph.bridge.entity_resolver import EntityResolver
+        from superlocalmemory.code_graph.bridge.fact_enricher import FactEnricher
+    except Exception as exc:  # pragma: no cover - import guard
+        logger.debug("bridge pass unavailable: %s", exc)
+        return counts
+
+    # Nothing to match against — skip before touching memory.db at all.
+    stats = code_graph_db.get_stats()
+    if not stats.get("nodes"):
+        logger.debug("bridge pass: code graph is empty, nothing to resolve against")
+        return counts
+
+    watermark = code_graph_db.get_metadata(_WATERMARK_KEY)
+    try:
+        rows = _fact_rows(memory_db, profile_id, watermark, max_facts)
+    except Exception as exc:
+        logger.warning("bridge pass could not read facts: %s", exc)
+        return counts
+
+    if not rows:
+        return counts
+
+    resolver = EntityResolver(code_graph_db)
+    enricher = FactEnricher(code_graph_db)
+    newest = watermark
+
+    for fact_id, content, created_at in rows:
+        counts["facts_scanned"] += 1
+        newest = created_at if newest is None or created_at > newest else newest
+        try:
+            links = resolver.resolve(content, fact_id, max_links=MAX_LINKS_PER_FACT)
+        except Exception as exc:
+            logger.debug("bridge resolve failed for %s: %s", fact_id, exc)
+            continue
+        if not links:
+            continue
+        counts["links_created"] += len(links)
+
+        # Enrichment is derived from (fact text, matched nodes) and is stored
+        # beside the link in code_graph.db. The user's own fact wording in
+        # memory.db is never rewritten: doing that would invalidate the fact's
+        # embedding, and would compound a suffix on every maintenance cycle.
+        try:
+            matched = resolver.get_matched_nodes(content)
+            if not matched:
+                continue
+            enriched = enricher.enrich(fact_id, matched, content)
+            if enriched and enriched != content:
+                _store_enrichment(code_graph_db, fact_id, enriched)
+                counts["enriched"] += 1
+        except Exception as exc:
+            logger.debug("bridge enrichment failed for %s: %s", fact_id, exc)
+
+    if newest and newest != watermark:
+        try:
+            code_graph_db.set_metadata(_WATERMARK_KEY, newest)
+        except Exception as exc:
+            logger.warning("bridge watermark not advanced: %s", exc)
+
+    # One summary line per pass, never one per fact. A 3,527-fact store must not
+    # produce 3,527 log lines; per-fact detail stays at debug.
+    if counts["links_created"]:
+        logger.info(
+            "Code bridge: %d facts scanned, %d links, %d enriched",
+            counts["facts_scanned"], counts["links_created"], counts["enriched"],
+        )
+    return counts
+
+
+def _store_enrichment(
+    code_graph_db: CodeGraphDatabase, fact_id: str, enriched: str
+) -> None:
+    """Persist enrichment text onto every link for *fact_id*."""
+    code_graph_db.execute_write(
+        "UPDATE code_memory_links SET enriched_description = ? WHERE slm_fact_id = ?",
+        (enriched, fact_id),
+    )

package/src/superlocalmemory/code_graph/config.py

@@ -9,7 +9,7 @@ Frozen dataclass with all tunables. Sensible defaults for typical repos.
 
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, fields
 from pathlib import Path
 
 from superlocalmemory.infra.data_root import state_path
@@ -88,3 +88,67 @@ class CodeGraphConfig:
         if slm_base_dir is not None:
             return slm_base_dir / "code_graph.db"
         return state_path("code_graph.db")
+
+    @classmethod
+    def load(cls, **overrides: object) -> CodeGraphConfig:
+        """Build a config from ``code_graph_config.json``, then apply overrides.
+
+        WHY THIS EXISTS (4.0.7). ``cli/setup_wizard.py`` has always written
+        ``code_graph_config.json`` with ``enabled`` and ``bridge_enabled``, and
+        until now **nothing read it**. There was no loader on this class at all,
+        and every call site constructed ``CodeGraphConfig(enabled=True)`` with
+        hardcoded defaults. So a user could answer "yes, enable the code graph"
+        in setup, get ``bridge_enabled: true`` written to disk, and have it
+        affect nothing — silently, with no way to tell from the outside.
+
+        Unknown keys in the file are ignored rather than raising: the file is
+        user-editable, and a stray key should not stop the code graph from
+        loading. Malformed JSON falls back to defaults with a warning, because
+        failing closed here would disable a working code graph over a typo.
+        """
+        import json
+        import logging
+
+        data: dict[str, object] = {}
+        path = state_path("code_graph_config.json")
+        try:
+            if path.exists():
+                loaded = json.loads(path.read_text(encoding="utf-8"))
+                if isinstance(loaded, dict):
+                    data = loaded
+                else:
+                    logging.getLogger(__name__).warning(
+                        "%s does not contain a JSON object; using defaults", path,
+                    )
+        except (OSError, json.JSONDecodeError) as exc:
+            logging.getLogger(__name__).warning(
+                "could not read %s (%s); using defaults", path, exc,
+            )
+
+        data.update(overrides)
+
+        valid = {f.name for f in fields(cls)}
+        unknown = sorted(set(data) - valid)
+        if unknown:
+            logging.getLogger(__name__).debug(
+                "ignoring unknown code_graph config keys: %s", ", ".join(unknown),
+            )
+
+        kwargs = {k: v for k, v in data.items() if k in valid}
+
+        # JSON has no frozenset/Path; coerce the fields that need it.
+        if isinstance(kwargs.get("languages"), list):
+            kwargs["languages"] = frozenset(kwargs["languages"])
+        if isinstance(kwargs.get("exclude_dirs"), list):
+            kwargs["exclude_dirs"] = frozenset(kwargs["exclude_dirs"])
+        for key in ("repo_root", "db_path"):
+            if isinstance(kwargs.get(key), str):
+                kwargs[key] = Path(kwargs[key])
+
+        try:
+            return cls(**kwargs)  # type: ignore[arg-type]
+        except TypeError as exc:
+            logging.getLogger(__name__).warning(
+                "code_graph config rejected (%s); using defaults", exc,
+            )
+            return cls()

package/src/superlocalmemory/code_graph/extractors/__init__.py

@@ -76,3 +76,20 @@ class BaseExtractor(ABC):
         import_edges, import_map = self.extract_imports()
         call_edges = self.extract_calls(import_map)
         return (classes + functions, import_edges + call_edges)
+
+    def extract_with_import_map(
+        self,
+    ) -> tuple[list[GraphNode], list[GraphEdge], dict[str, tuple[str, str]]]:
+        """Like extract() but also returns the per-file import map.
+
+        The import map has the form {local_name: (module_path, imported_name)}
+        and is needed by ImportResolver.resolve_call_targets (Strategy 1).
+
+        Returns:
+            (nodes, edges, import_map)
+        """
+        classes = self.extract_classes()
+        functions = self.extract_functions()
+        import_edges, import_map = self.extract_imports()
+        call_edges = self.extract_calls(import_map)
+        return (classes + functions, import_edges + call_edges, import_map)

package/src/superlocalmemory/code_graph/graph_store.py

@@ -70,19 +70,196 @@ class GraphStore:
         4. Insert new edges
         5. Upsert file record
 
-        The database's ``store_file_parse_results`` already does this.
+        Fix B — Defensive pre-filter:
+        Before delegating to the DB, drop any edge whose source_node_id or
+        target_node_id is absent from *both* the local node set AND the
+        database.  This guards the incremental update path (update_code_graph
+        → parse_file → here) which does not run the parse_all resolver
+        pipeline.  After a full build, parse_all's resolution pass should make
+        this a no-op; the filter is a belt-and-braces against partial runs.
+
+        DO NOT fix by disabling the FK — the FK is correct and protects graph
+        integrity.
         """
+        # ── build valid-id set: local nodes ──────────────────────────────
+        #
+        # REPLACE semantics: `store_file_parse_results` uses INSERT OR REPLACE.
+        # graph_nodes has a UNIQUE constraint on `qualified_name`.  When two
+        # nodes share a qualified_name (e.g. a @property getter AND setter, or
+        # a function defined twice in source), the LATER insert wins — the
+        # EARLIER node is deleted via REPLACE, and ON DELETE CASCADE removes
+        # any edges that already referenced it.  We simulate this here: only
+        # the LAST node per qualified_name survives; edges referencing the
+        # earlier "losers" must be dropped before we reach the DB.
+        qn_last: dict[str, GraphNode] = {}
+        for n in nodes:
+            qn_last[n.qualified_name] = n          # last occurrence wins
+        surviving_ids = {n.node_id for n in qn_last.values()}
+
+        # Collect foreign endpoints (not satisfied locally) for a single
+        # batch DB check — avoids N+1 queries.
+        foreign_ids: set[str] = set()
+        for edge in edges:
+            if edge.source_node_id not in surviving_ids:
+                foreign_ids.add(edge.source_node_id)
+            if edge.target_node_id not in surviving_ids:
+                foreign_ids.add(edge.target_node_id)
+
+        db_ids: set[str] = set()
+        if foreign_ids:
+            placeholders = ",".join("?" * len(foreign_ids))
+            rows = self._db.execute(
+                f"SELECT node_id FROM graph_nodes WHERE node_id IN ({placeholders})",
+                tuple(foreign_ids),
+            )
+            db_ids = {row["node_id"] for row in rows}
+
+        valid_ids = surviving_ids | db_ids
+
+        safe_edges: list[GraphEdge] = []
+        dropped = 0
+        for edge in edges:
+            if edge.source_node_id in valid_ids and edge.target_node_id in valid_ids:
+                safe_edges.append(edge)
+            else:
+                dropped += 1
+                logger.debug(
+                    "store_file_nodes_edges: dropped dangling edge %s→%s "
+                    "for file %s (resolver may not have run on this path)",
+                    edge.source_node_id, edge.target_node_id, file_path,
+                )
+        if dropped:
+            logger.debug(
+                "store_file_nodes_edges: total %d dangling edge(s) dropped for %s",
+                dropped, file_path,
+            )
+
         self._db.store_file_parse_results(
             file_path,
             list(nodes),
-            list(edges),
+            safe_edges,
             file_record,
         )
         logger.debug(
             "Stored %d nodes, %d edges for %s",
-            len(nodes), len(edges), file_path,
+            len(nodes), len(safe_edges), file_path,
         )
 
+    def commit_build_batch(
+        self,
+        batch: list[tuple[str, list[GraphNode], list[GraphEdge], FileRecord]],
+    ) -> None:
+        """Two-phase bulk commit — order-independent storage for full builds.
+
+        The single-file ``store_file_nodes_edges`` is insertion-order-
+        dependent: when a caller file (a.py, has CALLS foo→bar) is stored
+        before the callee file (b.py, defines bar), the cross-file CALLS
+        edge is dropped because Fix-B's DB existence check for bar.node_id
+        fails (b.py has not been stored yet).
+
+        This method fixes the root cause by separating commits into two
+        phases, both executed in a single atomic transaction:
+
+        **Phase 1 — all nodes**: for every file in the batch, delete old
+        data and insert new nodes.  After Phase 1, every node_id from every
+        file in the batch is present in ``graph_nodes``.
+
+        **Phase 2 — all edges**: for every file, validate endpoints (the DB
+        existence check now finds callee nodes regardless of file order) and
+        insert qualifying edges.
+
+        Design pattern: *Separated Phases*.  Interleaved per-file commits
+        (the existing loop) are O(1) transaction boundaries but
+        insertion-order-dependent.  Separated phases add one extra pass but
+        are fully order-independent.
+
+        Use this for full builds (``build_code_graph``).  Single-file
+        updates (``update_code_graph``) continue to use
+        ``store_file_nodes_edges`` — the callee nodes from other files are
+        already in the DB from the previous build, so no ordering issue.
+        """
+        if not batch:
+            return
+
+        with self._db.transaction():
+            # ── Phase 1: delete old data + insert all new nodes ───────────
+            #
+            # Edges must be deleted before nodes (FK constraint prevents
+            # deleting a node that an edge still references).  We delete ALL
+            # file edges across the whole batch first, then delete nodes.
+            # This avoids cascade surprises when file A's nodes are deleted
+            # before file B's edges that target those nodes are cleaned up.
+            for fp, _, _, _ in batch:
+                self._db.delete_edges_by_file(fp)
+            for fp, nodes, _, fr in batch:
+                self._db.delete_nodes_by_file(fp)
+                for node in nodes:
+                    self._db.upsert_node(node)
+                self._db.upsert_file_record(fr)
+
+            # ── Phase 2: validate + insert all edges ──────────────────────
+            #
+            # All node_ids from Phase 1 are now in graph_nodes, so the
+            # batch-DB check for cross-file foreign endpoints succeeds
+            # regardless of which file was stored first.
+            total_dropped = 0
+            for fp, nodes, edges, _ in batch:
+                # Simulate INSERT OR REPLACE dedup: only the last node per
+                # qualified_name survives; edges referencing earlier losers
+                # must be pre-dropped (mirrors store_file_nodes_edges logic).
+                qn_last: dict[str, GraphNode] = {}
+                for n in nodes:
+                    qn_last[n.qualified_name] = n
+                surviving_ids = {n.node_id for n in qn_last.values()}
+
+                # Batch-check foreign endpoints against DB (avoids N+1).
+                foreign_ids: set[str] = set()
+                for edge in edges:
+                    if edge.source_node_id not in surviving_ids:
+                        foreign_ids.add(edge.source_node_id)
+                    if edge.target_node_id not in surviving_ids:
+                        foreign_ids.add(edge.target_node_id)
+
+                db_ids: set[str] = set()
+                if foreign_ids:
+                    placeholders = ",".join("?" * len(foreign_ids))
+                    rows = self._db.execute(
+                        f"SELECT node_id FROM graph_nodes "
+                        f"WHERE node_id IN ({placeholders})",
+                        tuple(foreign_ids),
+                    )
+                    db_ids = {row["node_id"] for row in rows}
+
+                valid_ids = surviving_ids | db_ids
+                dropped = 0
+                for edge in edges:
+                    if (
+                        edge.source_node_id in valid_ids
+                        and edge.target_node_id in valid_ids
+                    ):
+                        self._db.upsert_edge(edge)
+                    else:
+                        dropped += 1
+                        logger.debug(
+                            "commit_build_batch: dropped dangling edge "
+                            "%s→%s for %s",
+                            edge.source_node_id, edge.target_node_id, fp,
+                        )
+                if dropped:
+                    total_dropped += dropped
+                    logger.debug(
+                        "commit_build_batch: %d dangling edge(s) dropped for %s",
+                        dropped, fp,
+                    )
+
+        if total_dropped:
+            logger.debug(
+                "commit_build_batch: total %d dangling edges dropped across "
+                "batch of %d files",
+                total_dropped, len(batch),
+            )
+        logger.debug("commit_build_batch: committed %d files", len(batch))
+
     def remove_file(self, file_path: str) -> None:
         """Remove all graph data for *file_path*.