superlocalmemory 4.0.9 → 4.1.0

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -21,6 +21,7 @@ import threading
 from collections import defaultdict
 from typing import TYPE_CHECKING, Any
 
+from superlocalmemory.retrieval import spreading
 from superlocalmemory.retrieval.scope_policy import (
     authorized_fact_ids,
     filter_authorized_results,
@@ -295,23 +296,58 @@ class EntityGraphChannel:
         ):
             return
         adj: dict[str, list[tuple[str, float]]] = defaultdict(list)
+        # The graph projection, when there is one, answers this in 395 ms where
+        # SQLite takes 2,477 ms on the same 208k-edge store (hand-measured, see
+        # graph/cozo_adjacency for the run; no test reproduces it) -- and this rebuild
+        # sits on the recall path, triggered by any edge-count change or a TTL
+        # expiry. It declines global and shared scope, because it stores one
+        # profile per edge and a short answer here would silently shrink the
+        # graph around a candidate. That decline is correct, not a degradation.
+        #
+        # This is the projection's only reader. Before it, the projection was
+        # maintained by the outbox, held at parity, purged on erasure, and
+        # queried by nothing.
+        triples: list[tuple[str, str, float]] | None = None
+        source_name = "sqlite"
         try:
-            where, params = _scope_where(
-                profile_id,
-                include_global=include_global,
-                include_shared=include_shared,
-            )
-            rows = self._db.execute(
-                f"SELECT source_id, target_id, weight FROM graph_edges WHERE {where}",
-                (*params,),
+            from superlocalmemory.graph.cozo_adjacency import adjacency_source
+
+            projection = (
+                adjacency_source() if self._projection_is_caught_up(profile_id) else None
             )
-        except Exception:
-            rows = []
-        for r in rows:
-            d = dict(r)
-            s, t, w = d["source_id"], d["target_id"], float(d["weight"])
-            adj[s].append((t, w))
-            adj[t].append((s, w))
+            if projection is not None:
+                triples = projection.edges(
+                    profile_id,
+                    include_global=include_global,
+                    include_shared=include_shared,
+                )
+                if triples is not None:
+                    source_name = projection.name
+        except Exception:  # noqa: BLE001 -- SQLite answers this unconditionally
+            triples = None
+        if triples is None:
+            try:
+                where, params = _scope_where(
+                    profile_id,
+                    include_global=include_global,
+                    include_shared=include_shared,
+                )
+                rows = self._db.execute(
+                    f"SELECT source_id, target_id, weight FROM graph_edges WHERE {where}",
+                    (*params,),
+                )
+            except Exception:
+                rows = []
+            triples = []
+            for r in rows:
+                d = dict(r)
+                triples.append(
+                    (d["source_id"], d["target_id"], float(d["weight"])),
+                )
+        self._adjacency_source_name = source_name
+        for edge_source, edge_target, edge_weight in triples:
+            adj[edge_source].append((edge_target, edge_weight))
+            adj[edge_target].append((edge_source, edge_weight))
         # Also load entity maps (same staleness lifecycle)
         self._load_entity_maps(
             profile_id,
@@ -338,6 +374,32 @@ class EntityGraphChannel:
         # v3.4.1: Load graph intelligence metrics (P0)
         self._load_graph_metrics(profile_id)
 
+        # One array-shaped view of the same graph, for the walk to run over.
+        # Built here rather than per query because it is derived entirely from
+        # the maps above and shares their staleness lifecycle exactly.
+        from superlocalmemory.retrieval.graph_adjacency import snapshot_from_maps
+
+        self._snapshot = snapshot_from_maps(
+            self._adj,
+            self._entity_to_facts,
+            self._fact_to_entities,
+            self._graph_metrics,
+            source=getattr(self, "_adjacency_source_name", "sqlite"),
+            profile_id=profile_id,
+            # Every visible fact is a node, including the ones with no edges
+            # yet. Ingestion is queryable-first, so a memory stored a moment ago
+            # has entities and no edges, and it must still be reachable.
+            nodes=self._visible_fact_ids,
+            # Only a real count. The staleness check above compares this value
+            # with ``==``, which a MagicMock tolerates; an ordering comparison
+            # does not, and the mock DBs in the test suite reach here.
+            fact_count=(
+                current_fact_count
+                if isinstance(current_fact_count, int) and current_fact_count >= 0
+                else 0
+            ),
+        )
+
         logger.info(
             "Loaded adjacency cache: %d nodes, %d edges, %d entity mappings for profile %s",
             len(self._adj),
@@ -346,6 +408,42 @@ class EntityGraphChannel:
             profile_id,
         )
 
+    def _projection_is_caught_up(self, profile_id: str) -> bool:
+        """Whether the second graph store has seen every change SQLite has.
+
+        The graph lives in two stores and no transaction spans them, so the
+        durable record of "this fact still needs projecting" is a queue row
+        written in the same transaction as the change. A row outstanding for
+        this profile is that store telling us, in its own words, that it is
+        behind -- and reading a graph that is behind means walking a link the
+        store has already removed.
+
+        This is a primary-key count on a table whose steady state is empty and
+        whose size is bounded by the fact count, so it is microseconds. The
+        alternative -- comparing the two edge sets -- costs 1.9 s on the
+        author's store and 7.7 s on the larger one, which is the whole recall
+        budget spent proving a cache is warm.
+        """
+        try:
+            from superlocalmemory.storage import projection_outbox
+
+            if not projection_outbox.is_available(self._db):
+                return True
+            rows = self._db.execute(
+                "SELECT COUNT(*) AS cnt FROM projection_outbox WHERE profile_id = ?",
+                (profile_id,),
+            )
+            pending = int(dict(rows[0]).get("cnt", 0)) if rows else 0
+        except Exception:  # noqa: BLE001 -- an unreadable queue means read SQLite
+            return False
+        if pending:
+            logger.debug(
+                "adjacency: %d change(s) not yet in the graph projection for "
+                "profile %s; reading the store directly", pending, profile_id,
+            )
+            return False
+        return True
+
     def _get_edge_count(
         self,
         profile_id: str,
@@ -398,9 +496,15 @@ class EntityGraphChannel:
                 include_global=include_global,
                 include_shared=include_shared,
             )
+            # Withheld rows must not enter the entity map at all. They carry
+            # their whole cluster's pooled entity list, which is exactly why
+            # they out-ranked real memories here in the first place — leaving
+            # them in the map would keep spending this channel's budget on
+            # candidates that hydration then discards.
             rows = self._db.execute(
                 "SELECT fact_id, canonical_entities_json "
-                f"FROM atomic_facts WHERE {where} "
+                f"FROM atomic_facts WHERE {where}"
+                f"{self._db.visible_fact_clause()} "
                 "ORDER BY created_at DESC LIMIT ?",
                 (*params, _unbounded_facts_ceiling()),
             )
@@ -507,113 +611,93 @@ class EntityGraphChannel:
             include_shared=include_shared,
         )
 
-        # v3.4.5: Route to CozoDB if active
-        if self._cozo is not None:
-            return self._search_via_cozo(
-                query,
-                raw_entities,
-                profile_id,
-                top_k,
-                include_global=include_global,
-                include_shared=include_shared,
-            )
-
         canonical_ids = self._resolve_entities(raw_entities, profile_id)
         if not canonical_ids:
             return []
 
-        # Seed activation from direct entity-linked facts
-        # Use in-memory map when available, fall back to SQL for mock/test DBs
+        # One walk, over the array-shaped snapshot when there is one. The
+        # dict form below is kept for the mock/lightweight DBs that never build
+        # an adjacency cache, and it is the only path that still pays a Python
+        # loop per edge.
+        snapshot = getattr(self, "_snapshot", None)
+        if snapshot is not None and snapshot.node_count:
+            activation_result = spreading.activate(
+                snapshot,
+                canonical_ids,
+                decay=self._decay,
+                threshold=self._threshold,
+                max_hops=self._max_hops,
+            )
+            spreading.apply_community_bias(
+                activation_result.scores, snapshot, canonical_ids,
+            )
+            activation = activation_result.as_mapping(snapshot, threshold=-1.0)
+            if activation:
+                self._suppress_contradictions(activation, profile_id)
+            results = [
+                (fid, sc) for fid, sc in activation.items() if sc >= self._threshold
+            ]
+            if not results:
+                return []
+            max_score = max(sc for _, sc in results)
+            if max_score > 0:
+                results = [(fid, sc / max_score) for fid, sc in results]
+            results.sort(key=lambda x: (-x[1], x[0]))
+            return filter_authorized_results(
+                self._db,
+                results,
+                profile_id,
+                include_global=include_global,
+                include_shared=include_shared,
+            )[:top_k]
+
+        # Seed activation from direct entity-linked facts (no adjacency cache:
+        # mock and lightweight DBs only). Graph intelligence is unavailable on
+        # this path by design -- see Phase 7 LLD H-01.
         activation: dict[str, float] = defaultdict(float)
         visited_entities: set[str] = set(canonical_ids)
-
-        use_cache = bool(self._entity_to_facts)
+        use_cache = False
         for eid in canonical_ids:
-            if use_cache:
-                for fid in self._entity_to_facts.get(eid, ()):
-                    activation[fid] = max(activation[fid], 1.0)
-            else:
-                for fact in self._db.get_facts_by_entity(
-                    eid,
-                    profile_id,
-                    include_global=include_global,
-                    include_shared=include_shared,
-                ):
-                    activation[fact.fact_id] = max(activation[fact.fact_id], 1.0)
+            for fact in self._db.get_facts_by_entity(
+                eid,
+                profile_id,
+                include_global=include_global,
+                include_shared=include_shared,
+            ):
+                activation[fact.fact_id] = max(activation[fact.fact_id], 1.0)
 
-        # Spreading activation through graph edges (all in-memory O(1) lookups)
         frontier = set(activation.keys())
         for hop in range(1, self._max_hops):
             hop_decay = self._decay**hop
             if hop_decay < self._threshold:
                 break
             next_frontier: set[str] = set()
-
             for fid in frontier:
-                if use_cache:
-                    neighbors = self._adj.get(fid, ())
-                    for neighbor, edge_weight in neighbors:
-                        # v3.4.2: Only apply edge_weight and PageRank bias when
-                        # graph metrics are available. Without metrics, edge_weight
-                        # dampens propagation by ~14% with no compensating boost,
-                        # causing retrieval regression (68.4% vs 70.4% on LoCoMo).
-                        if self._graph_metrics:
-                            weighted = activation[fid] * self._decay * edge_weight
-                            if neighbor in self._graph_metrics:
-                                target_pr = self._graph_metrics[neighbor].get("pagerank_score", 0.0)
-                                pr_boost = min(1.0 + target_pr * 2.0, 2.0)
-                                weighted *= pr_boost
-                        else:
-                            weighted = activation[fid] * self._decay
-                        if weighted >= self._threshold and weighted > activation.get(neighbor, 0.0):
-                            activation[neighbor] = weighted
-                            next_frontier.add(neighbor)
-                else:
-                    # NOTE: SQL fallback path does NOT use graph intelligence (P1/P2/P3).
-                    # Graph intelligence is only available on the in-memory cache path.
-                    # This fallback exists for mock/test DBs. See Phase 7 LLD H-01.
-                    for edge in self._db.get_edges_for_node(
-                        fid,
-                        profile_id,
-                        include_global=include_global,
-                        include_shared=include_shared,
-                    ):
-                        neighbor = edge.target_id if edge.source_id == fid else edge.source_id
-                        propagated = activation[fid] * self._decay
-                        if propagated >= self._threshold and propagated > activation.get(
-                            neighbor, 0.0
-                        ):
-                            activation[neighbor] = propagated
-                            next_frontier.add(neighbor)
-
-            # Discover new entities from activated facts
-            if use_cache:
-                new_eids: list[str] = []
-                for fid in frontier:
-                    for eid in self._fact_to_entities.get(fid, ()):
-                        if eid not in visited_entities:
-                            visited_entities.add(eid)
-                            new_eids.append(eid)
-                for eid in new_eids:
-                    for fid in self._entity_to_facts.get(eid, ()):
-                        if hop_decay > activation.get(fid, 0.0):
-                            activation[fid] = hop_decay
-                            next_frontier.add(fid)
-            else:
-                # SQL fallback (mock/test DBs)
-                new_eids_sql = self._discover_entities(frontier, profile_id, visited_entities)
-                for eid in new_eids_sql:
-                    visited_entities.add(eid)
-                    for fact in self._db.get_facts_by_entity(
-                        eid,
-                        profile_id,
-                        include_global=include_global,
-                        include_shared=include_shared,
+                for edge in self._db.get_edges_for_node(
+                    fid,
+                    profile_id,
+                    include_global=include_global,
+                    include_shared=include_shared,
+                ):
+                    neighbor = edge.target_id if edge.source_id == fid else edge.source_id
+                    propagated = activation[fid] * self._decay
+                    if propagated >= self._threshold and propagated > activation.get(
+                        neighbor, 0.0
                     ):
-                        if hop_decay > activation.get(fact.fact_id, 0.0):
-                            activation[fact.fact_id] = hop_decay
-                            next_frontier.add(fact.fact_id)
-
+                        activation[neighbor] = propagated
+                        next_frontier.add(neighbor)
+            new_eids_sql = self._discover_entities(frontier, profile_id, visited_entities)
+            for eid in new_eids_sql:
+                visited_entities.add(eid)
+                for fact in self._db.get_facts_by_entity(
+                    eid,
+                    profile_id,
+                    include_global=include_global,
+                    include_shared=include_shared,
+                ):
+                    if hop_decay > activation.get(fact.fact_id, 0.0):
+                        activation[fact.fact_id] = hop_decay
+                        next_frontier.add(fact.fact_id)
             frontier = next_frontier
             if not frontier:
                 break
@@ -741,77 +825,32 @@ class EntityGraphChannel:
         if not canonical_ids:
             return {}
 
-        # Run full spreading activation (same as search())
-        activation: dict[str, float] = defaultdict(float)
-        visited_entities: set[str] = set(canonical_ids)
-        use_cache = bool(self._entity_to_facts)
-
-        for eid in canonical_ids:
-            if use_cache:
-                for fid in self._entity_to_facts.get(eid, ()):
-                    activation[fid] = max(activation[fid], 1.0)
-            else:
-                for fact in self._db.get_facts_by_entity(
-                    eid,
-                    profile_id,
-                    include_global=include_global,
-                    include_shared=include_shared,
-                ):
-                    activation[fact.fact_id] = max(activation[fact.fact_id], 1.0)
-
-        frontier = set(activation.keys())
-        for hop in range(1, self._max_hops):
-            hop_decay = self._decay**hop
-            if hop_decay < self._threshold:
-                break
-            next_frontier: set[str] = set()
-            for fid in frontier:
-                if use_cache:
-                    for neighbor, edge_weight in self._adj.get(fid, ()):
-                        if self._graph_metrics:
-                            weighted = activation[fid] * self._decay * edge_weight
-                            if neighbor in self._graph_metrics:
-                                pr = self._graph_metrics[neighbor].get("pagerank_score", 0.0)
-                                weighted *= min(1.0 + pr * 2.0, 2.0)
-                        else:
-                            weighted = activation[fid] * self._decay
-                        if weighted >= self._threshold and weighted > activation.get(neighbor, 0.0):
-                            activation[neighbor] = weighted
-                            next_frontier.add(neighbor)
-
-            if use_cache:
-                for fid in frontier:
-                    for eid in self._fact_to_entities.get(fid, ()):
-                        if eid not in visited_entities:
-                            visited_entities.add(eid)
-                            for linked_fid in self._entity_to_facts.get(eid, ()):
-                                if hop_decay > activation.get(linked_fid, 0.0):
-                                    activation[linked_fid] = hop_decay
-                                    next_frontier.add(linked_fid)
-
-            frontier = next_frontier
-            if not frontier:
-                break
-
-        # Community-aware boosting (same as search)
-        if self._graph_metrics and use_cache:
-            from collections import Counter as _Counter
-
-            seed_communities: _Counter = _Counter()
-            for eid in canonical_ids:
-                for fid in self._entity_to_facts.get(eid, ()):
-                    m = self._graph_metrics.get(fid, {})
-                    comm = m.get("community_id")
-                    if comm is not None:
-                        seed_communities[comm] += 1
-            if seed_communities:
-                total_seeds = sum(seed_communities.values())
-                for fid in list(activation.keys()):
-                    m = self._graph_metrics.get(fid, {})
-                    fact_comm = m.get("community_id")
-                    if fact_comm is not None and fact_comm in seed_communities:
-                        boost = min(1.0 + 0.15 * (seed_communities[fact_comm] / total_seeds), 1.3)
-                        activation[fid] *= boost
+        # The same walk as search(), over the same snapshot. This method used
+        # to carry its own copy of the loop, which is how the two drifted: the
+        # community bias here has never applied search()'s outsider penalty, and
+        # the only record of that was the absence of six lines.
+        snapshot = getattr(self, "_snapshot", None)
+        if snapshot is None or not snapshot.node_count:
+            return {}
+        activation_result = spreading.activate(
+            snapshot,
+            canonical_ids,
+            decay=self._decay,
+            threshold=self._threshold,
+            max_hops=self._max_hops,
+        )
+        spreading.apply_community_bias(
+            activation_result.scores,
+            snapshot,
+            canonical_ids,
+            # Re-scoring another channel's candidates, so a fact outside every
+            # seed community is not damped -- see apply_community_bias.
+            penalise_outsiders=False,
+        )
+        activation = {
+            fid: float(activation_result.scores[idx])
+            for fid, idx in snapshot.node_index.items()
+        }
 
         # Extract scores ONLY for the candidate set, normalize to [0, 1]
         candidate_set = allowed_candidates
@@ -953,82 +992,3 @@ class EntityGraphChannel:
         return new
 
     # v3.4.5: CozoDB-backed search (Sprint 2)
-    def _search_via_cozo(
-        self,
-        query: str,
-        raw_entities: list[str],
-        profile_id: str,
-        top_k: int,
-        *,
-        include_global: bool = False,
-        include_shared: bool = False,
-    ) -> list[tuple[str, float]]:
-        """Entity graph search routed through CozoDB.
-
-        Uses CozoDB for spreading activation — avoids loading
-        the full adjacency graph into memory.
-        Falls back to in-memory adjacency if CozoDB fails.
-        """
-        if not raw_entities:
-            return []
-
-        canonical_ids = self._resolve_entities(raw_entities, profile_id)
-        if not canonical_ids:
-            return []
-
-        # Scoped/global recall has deliberately more complex authorization
-        # semantics than the promoted default-profile projection.  Never let
-        # a projection broaden that boundary: SQLite remains authoritative.
-        if include_global or include_shared:
-            return self._search_without_cozo(query, profile_id, top_k)
-
-        try:
-            scored = self._cozo.recall_facts(
-                canonical_ids,
-                profile_id=profile_id,
-                depth=self._max_hops,
-                decay=self._decay,
-                threshold=self._threshold,
-                top_k=top_k * 2,
-            )
-
-            cozo_results = filter_authorized_results(
-                self._db,
-                scored,
-                profile_id,
-                include_global=include_global,
-                include_shared=include_shared,
-            )[:top_k]
-            # Shadow SQLite before accepting a projected answer.  The graph
-            # channel has optional PageRank/community enrichments, so exact
-            # Score equality is neither required nor useful; result *membership*
-            # is the correctness contract. Order within the same fact set is
-            # tolerated — requiring identical ordering would fail closed on
-            # every query with score ties, leaving Cozo permanently unused.
-            # Any membership divergence is recorded and fails closed to SQLite.
-            sqlite_results = self._search_without_cozo(query, profile_id, top_k)
-            matches = {fact_id for fact_id, _ in cozo_results} == {
-                fact_id for fact_id, _ in sqlite_results
-            }
-            record = getattr(self._cozo, "record_shadow_comparison", None)
-            if callable(record):
-                record(matches=matches, projected=cozo_results, canonical=sqlite_results)
-            return cozo_results if matches else sqlite_results
-        except Exception as exc:
-            record = getattr(self._cozo, "record_shadow_error", None)
-            if callable(record):
-                record(str(exc))
-            return self._search_without_cozo(query, profile_id, top_k)
-
-    def _search_without_cozo(
-        self,
-        query: str,
-        profile_id: str,
-        top_k: int,
-    ) -> list[tuple[str, float]]:
-        """Run canonical SQLite entity recall without recursive projection use."""
-        cozo, self._cozo = self._cozo, None
-        try:
-            return self._search_locked(query, profile_id, top_k)
-        finally:
-            self._cozo = cozo