superlocalmemory 3.8.6 → 3.8.8

package/src/superlocalmemory/encoding/scene_builder.py

@@ -38,6 +38,8 @@ class SceneBuilder:
     def __init__(self, db, embedder=None) -> None:
         self._db = db
         self._embedder = embedder
+        # Key by scene ID, never theme. Themes are deliberately non-unique,
+        # while eligibility and durable anchor membership are scene-specific.
         self._scene_embeddings_cache: dict[str, list[float]] = {}
 
     def assign_to_scene(
@@ -53,8 +55,12 @@ class SceneBuilder:
         if self._embedder is None:
             return self._create_scene(new_fact, profile_id)
 
-        # Always compute fact embedding first — needed for comparisons
-        fact_emb = self._embedder.embed(new_fact.content)
+        # Canonical ingestion already embeds the fact before scene assignment.
+        # Reuse that vector so scene clustering does not issue a duplicate model
+        # request for every remembered fact.
+        fact_emb = new_fact.embedding
+        if fact_emb is None:
+            fact_emb = self._embedder.embed(new_fact.content)
 
         # v3.4.38: Defensive None guard. embedder.embed() returns None when
         # the embedding worker is unavailable (timeout, crash). Without this
@@ -69,30 +75,58 @@ class SceneBuilder:
         if not scenes:
             return self._create_scene(new_fact, profile_id)
 
+        live_scene_embeddings = self._load_live_scene_embeddings(profile_id)
+        live_scene_ids = set(live_scene_embeddings)
+        self._scene_embeddings_cache.update({
+            scene_id: embedding
+            for scene_id, embedding in live_scene_embeddings.items()
+            if embedding is not None
+        })
+        # Old consolidation/deletion paths left scene rows whose fact IDs no
+        # longer exist. They are not evidence and must not trigger thousands of
+        # replacement model calls after restart. A cache hit cannot prove that
+        # a scene still has a surviving fact, so eligibility is always derived
+        # from the current database state.
+        scenes = [
+            scene for scene in scenes if scene.scene_id in live_scene_ids
+        ]
+        if not scenes:
+            return self._create_scene(new_fact, profile_id)
+
         # Find best matching scene
         best_scene: MemoryScene | None = None
         best_sim = -1.0
 
-        # V3.3.27: Batch-embed all uncached scene themes in ONE call.
+        # A scene's theme is derived from its first (anchor) fact. On daemon
+        # restart the in-memory cache is empty, but the anchor embeddings remain
+        # durable in atomic_facts. Prime from those vectors before calling the
+        # model; otherwise a mature database re-embeds thousands of themes and
+        # repeatedly recycles the shared foreground worker.
+        # V3.3.27: Batch-embed all still-uncached scene themes in ONE call.
         # Previously: 200+ individual embed() calls per fact (30s on Mode B).
         # Now: 1 batch call for all uncached themes, then cache hits for the rest.
-        uncached_themes = [s.theme for s in scenes if s.theme not in self._scene_embeddings_cache]
-        if uncached_themes and hasattr(self._embedder, 'embed_batch'):
+        uncached_scenes = [
+            scene for scene in scenes
+            if scene.scene_id not in self._scene_embeddings_cache
+        ]
+        if uncached_scenes and hasattr(self._embedder, 'embed_batch'):
             try:
-                batch_embs = self._embedder.embed_batch(uncached_themes)
-                for theme, emb in zip(uncached_themes, batch_embs):
+                batch_embs = self._embedder.embed_batch(
+                    [scene.theme for scene in uncached_scenes]
+                )
+                for scene, emb in zip(uncached_scenes, batch_embs):
                     if emb is not None:
-                        self._scene_embeddings_cache[theme] = emb
+                        self._scene_embeddings_cache[scene.scene_id] = emb
             except Exception:
                 pass  # Fall through to individual embeds below
 
         for scene in scenes:
-            if scene.theme in self._scene_embeddings_cache:
-                theme_emb = self._scene_embeddings_cache[scene.theme]
+            if scene.scene_id in self._scene_embeddings_cache:
+                theme_emb = self._scene_embeddings_cache[scene.scene_id]
             else:
                 theme_emb = self._embedder.embed(scene.theme)
                 if theme_emb is not None:
-                    self._scene_embeddings_cache[scene.theme] = theme_emb
+                    self._scene_embeddings_cache[scene.scene_id] = theme_emb
             if theme_emb is None:
                 continue
             sim = _cosine(fact_emb, theme_emb)
@@ -130,10 +164,6 @@ class SceneBuilder:
         comparisons in assign_to_scene.
         """
         theme = fact.content[:200]
-        # Pre-compute theme embedding for future comparisons
-        if self._embedder is not None:
-            self._scene_embeddings_cache[theme] = self._embedder.embed(theme)
-
         scene = MemoryScene(
             profile_id=profile_id,
             theme=theme,
@@ -142,6 +172,14 @@ class SceneBuilder:
             created_at=datetime.now(UTC).isoformat(),
             last_updated=datetime.now(UTC).isoformat(),
         )
+        # Pre-compute theme embedding for future comparisons. The canonical fact
+        # vector represents this exact theme and avoids a duplicate model call.
+        if self._embedder is not None:
+            theme_embedding = fact.embedding
+            if theme_embedding is None:
+                theme_embedding = self._embedder.embed(theme)
+            if theme_embedding is not None:
+                self._scene_embeddings_cache[scene.scene_id] = theme_embedding
         self._save_scene(scene)
         return scene
 
@@ -171,6 +209,58 @@ class SceneBuilder:
         )
         return [self._row_to_scene(dict(r)) for r in rows]
 
+    def _load_live_scene_embeddings(
+        self,
+        profile_id: str,
+    ) -> dict[str, list[float] | None]:
+        """Load one durable anchor embedding for every live scene.
+
+        ``json_each`` resolves the first still-existing fact in each scene, so
+        scenes whose original anchor was consolidated away can still reuse a
+        surviving member. The result also identifies fully stale scene rows,
+        which are ignored by assignment instead of being re-embedded.
+        """
+        try:
+            rows = self._db.execute(
+                """
+                WITH live_scene_facts AS (
+                    SELECT
+                        ms.scene_id,
+                        ms.theme,
+                        af.embedding,
+                        ROW_NUMBER() OVER (
+                            PARTITION BY ms.scene_id
+                            ORDER BY CAST(member.key AS INTEGER)
+                        ) AS member_rank
+                    FROM memory_scenes AS ms
+                    JOIN json_each(ms.fact_ids_json) AS member
+                    JOIN atomic_facts AS af
+                      ON af.fact_id = member.value
+                     AND af.profile_id = ms.profile_id
+                    WHERE ms.profile_id = ?
+                )
+                SELECT scene_id, embedding
+                FROM live_scene_facts
+                WHERE member_rank = 1
+                """,
+                (profile_id,),
+            )
+        except Exception:
+            return {}
+
+        result: dict[str, list[float] | None] = {}
+        for row in rows:
+            data = dict(row)
+            raw_embedding = data.get("embedding")
+            embedding = None
+            if raw_embedding:
+                try:
+                    embedding = json.loads(raw_embedding)
+                except (TypeError, ValueError, json.JSONDecodeError):
+                    embedding = None
+            result[str(data["scene_id"])] = embedding
+        return result
+
     def _save_scene(self, scene: MemoryScene) -> None:
         """Upsert scene to DB."""
         self._db.execute(

package/src/superlocalmemory/graph/cozo_backend.py

@@ -75,12 +75,12 @@ class _CozoResult:
 
 
 class _CozoClientAdapter:
-    """Bridge legacy PyCozo responses and the pinned 0.7.6 client surface.
+    """Bridge legacy PyCozo responses and the store-compatible client surface.
 
-    PyCozo 0.7.6 publishes embedded wheels for macOS, Linux x86_64/aarch64,
-    and Windows. Older installations can still return dictionaries and expose
-    ``import_relations`` rather than ``put``. SLM only needs relation upserts
-    and row results, so normalize both forms here.
+    SLM graph stores were established with PyCozo 0.3.0. That client returns
+    dictionaries and exposes ``import_relations`` rather than ``put``; later
+    clients return dataframe-like values and add ``put``. SLM only needs
+    relation upserts and row results, so normalize both forms here.
     """
 
     def __init__(self, client: Any) -> None:

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -10,6 +10,7 @@ with decay. Handles BOTH uppercase and lowercase entity mentions.
 Part of Qualixar | Author: Varun Pratap Bhardwaj
 License: AGPL-3.0-or-later
 """
+
 from __future__ import annotations
 
 import json
@@ -24,7 +25,10 @@ from superlocalmemory.retrieval.scope_policy import (
     authorized_fact_ids,
     filter_authorized_results,
 )
-from superlocalmemory.storage.database import _scope_where
+from superlocalmemory.storage.database import (
+    _scope_where,
+    _unbounded_facts_ceiling,
+)
 
 if TYPE_CHECKING:
     from superlocalmemory.encoding.entity_resolver import EntityResolver
@@ -50,26 +54,126 @@ def _adj_ttl_seconds() -> float:
     except (TypeError, ValueError):
         return 3600.0
 
+
 _PROPER_NOUN_RE = re.compile(r"\b[A-Z][a-z]{1,}\b")
 
-_ENTITY_STOP: frozenset[str] = frozenset({
-    # Expanded stop list for query entity extraction
-    "what", "when", "where", "who", "which", "how", "does", "did",
-    "the", "that", "this", "there", "then", "than", "they", "them",
-    "have", "has", "had", "been", "being", "about", "after", "before",
-    "from", "into", "with", "some", "other", "would", "could", "should",
-    "will", "because", "also", "just", "like", "know", "think",
-    "feel", "want", "need", "make", "take", "give", "tell", "said",
-    "wow", "gonna", "got", "by", "thanks", "thank", "hey", "hi",
-    "hello", "bye", "good", "great", "nice", "cool", "right",
-    "let", "can", "might", "much", "many", "more", "most",
-    "something", "anything", "everything", "nothing", "someone",
-    "it", "my", "your", "our", "their", "me", "you", "we", "us",
-    "do", "if", "or", "no", "to", "at", "on", "in", "so",
-    "go", "come", "see", "look", "say", "ask", "try", "keep",
-    "yes", "yeah", "sure", "okay", "ok", "really", "actually",
-    "maybe", "well", "still", "even", "very",
-})
+_ENTITY_STOP: frozenset[str] = frozenset(
+    {
+        # Expanded stop list for query entity extraction
+        "what",
+        "when",
+        "where",
+        "who",
+        "which",
+        "how",
+        "does",
+        "did",
+        "the",
+        "that",
+        "this",
+        "there",
+        "then",
+        "than",
+        "they",
+        "them",
+        "have",
+        "has",
+        "had",
+        "been",
+        "being",
+        "about",
+        "after",
+        "before",
+        "from",
+        "into",
+        "with",
+        "some",
+        "other",
+        "would",
+        "could",
+        "should",
+        "will",
+        "because",
+        "also",
+        "just",
+        "like",
+        "know",
+        "think",
+        "feel",
+        "want",
+        "need",
+        "make",
+        "take",
+        "give",
+        "tell",
+        "said",
+        "wow",
+        "gonna",
+        "got",
+        "by",
+        "thanks",
+        "thank",
+        "hey",
+        "hi",
+        "hello",
+        "bye",
+        "good",
+        "great",
+        "nice",
+        "cool",
+        "right",
+        "let",
+        "can",
+        "might",
+        "much",
+        "many",
+        "more",
+        "most",
+        "something",
+        "anything",
+        "everything",
+        "nothing",
+        "someone",
+        "it",
+        "my",
+        "your",
+        "our",
+        "their",
+        "me",
+        "you",
+        "we",
+        "us",
+        "do",
+        "if",
+        "or",
+        "no",
+        "to",
+        "at",
+        "on",
+        "in",
+        "so",
+        "go",
+        "come",
+        "see",
+        "look",
+        "say",
+        "ask",
+        "try",
+        "keep",
+        "yes",
+        "yeah",
+        "sure",
+        "okay",
+        "ok",
+        "really",
+        "actually",
+        "maybe",
+        "well",
+        "still",
+        "even",
+        "very",
+    }
+)
 
 
 def extract_query_entities(query: str) -> list[str]:
@@ -94,10 +198,10 @@ def extract_query_entities(query: str) -> list[str]:
     for m in re.finditer(r'"([^"]+)"', query):
         _add(m.group(1).strip())
     # Also extract multi-word capitalized sequences (e.g. "New York", "San Francisco")
-    for m in re.finditer(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b', query):
+    for m in re.finditer(r"\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b", query):
         _add(m.group(1))
     # Extract all-caps abbreviations (e.g. NYU, MIT, UCLA) — min 2 chars
-    for m in re.finditer(r'\b([A-Z]{2,})\b', query):
+    for m in re.finditer(r"\b([A-Z]{2,})\b", query):
         _add(m.group(1))
 
     return candidates
@@ -113,9 +217,11 @@ class EntityGraphChannel:
     """
 
     def __init__(
-        self, db: DatabaseManager,
+        self,
+        db: DatabaseManager,
         entity_resolver: EntityResolver | None = None,
-        decay: float = 0.7, activation_threshold: float = 0.05,
+        decay: float = 0.7,
+        activation_threshold: float = 0.05,
         max_hops: int = 4,
         graph_metrics: dict[str, dict] | None = None,
         cozo_backend: Any = None,  # v3.4.5: optional CozoDB backend
@@ -174,18 +280,19 @@ class EntityGraphChannel:
         # the graph without changing the count (e.g. store_edge MAX-merge), and a
         # count-stable window would otherwise serve a stale adjacency map.
         import time as _t_ec
+
         _now_ec = _t_ec.monotonic()
         _ttl = _adj_ttl_seconds()
         # TTL=0 disables the time-based reload entirely (count-based correctness
         # reload still applies); otherwise the cache is fresh within the TTL.
-        _fresh = _ttl <= 0.0 or (
-            (_now_ec - getattr(self, "_adj_loaded_at", 0.0)) < _ttl
-        )
-        if (self._adj_scope_key == scope_key
-                and (self._adj or self._visible_fact_ids)
-                and self._adj_edge_count == current_count
-                and self._adj_fact_count == current_fact_count
-                and _fresh):
+        _fresh = _ttl <= 0.0 or ((_now_ec - getattr(self, "_adj_loaded_at", 0.0)) < _ttl)
+        if (
+            self._adj_scope_key == scope_key
+            and (self._adj or self._visible_fact_ids)
+            and self._adj_edge_count == current_count
+            and self._adj_fact_count == current_fact_count
+            and _fresh
+        ):
             return
         adj: dict[str, list[tuple[str, float]]] = defaultdict(list)
         try:
@@ -195,8 +302,7 @@ class EntityGraphChannel:
                 include_shared=include_shared,
             )
             rows = self._db.execute(
-                "SELECT source_id, target_id, weight FROM graph_edges "
-                f"WHERE {where}",
+                f"SELECT source_id, target_id, weight FROM graph_edges WHERE {where}",
                 (*params,),
             )
         except Exception:
@@ -234,8 +340,10 @@ class EntityGraphChannel:
 
         logger.info(
             "Loaded adjacency cache: %d nodes, %d edges, %d entity mappings for profile %s",
-            len(self._adj), sum(len(v) for v in self._adj.values()) // 2,
-            len(self._entity_to_facts), profile_id,
+            len(self._adj),
+            sum(len(v) for v in self._adj.values()) // 2,
+            len(self._entity_to_facts),
+            profile_id,
         )
 
     def _get_edge_count(
@@ -272,7 +380,11 @@ class EntityGraphChannel:
         """Pre-load entity→fact and fact→entity maps into memory.
 
         Eliminates per-entity and per-fact SQL in the spreading activation loop.
-        Same data, same algorithm — zero quality change.
+        Fetch only the two columns this index consumes. Loading full AtomicFact
+        objects also deserializes every 768-d embedding and Fisher vector; on a
+        mature database that turned one new fact into a 5-second recall stall.
+        The scope predicate and configurable 50k safety ceiling are identical
+        to ``get_all_facts``; only heavyweight unused columns are omitted.
         """
         # entity_id -> [fact_id, ...]
         self._entity_to_facts: dict[str, list[str]] = defaultdict(list)
@@ -281,22 +393,41 @@ class EntityGraphChannel:
         self._visible_fact_ids = set()
 
         try:
-            facts = self._db.get_all_facts(
+            where, params = _scope_where(
                 profile_id,
                 include_global=include_global,
                 include_shared=include_shared,
             )
+            rows = self._db.execute(
+                "SELECT fact_id, canonical_entities_json "
+                f"FROM atomic_facts WHERE {where} "
+                "ORDER BY created_at DESC LIMIT ?",
+                (*params, _unbounded_facts_ceiling()),
+            )
         except Exception:
-            facts = []
-        for fact in facts:
-            self._visible_fact_ids.add(fact.fact_id)
-            for eid in fact.canonical_entities:
-                self._entity_to_facts[eid].append(fact.fact_id)
-                self._fact_to_entities[fact.fact_id].append(eid)
+            rows = []
+        for row in rows:
+            data = dict(row)
+            fact_id = str(data.get("fact_id") or "")
+            if not fact_id:
+                continue
+            self._visible_fact_ids.add(fact_id)
+            try:
+                entity_ids = json.loads(
+                    data.get("canonical_entities_json") or "[]",
+                )
+            except (TypeError, ValueError):
+                entity_ids = []
+            for entity_id in entity_ids:
+                if not isinstance(entity_id, str) or not entity_id:
+                    continue
+                self._entity_to_facts[entity_id].append(fact_id)
+                self._fact_to_entities[fact_id].append(entity_id)
 
         logger.info(
             "Loaded entity maps: %d entities, %d facts with entities",
-            len(self._entity_to_facts), len(self._fact_to_entities),
+            len(self._entity_to_facts),
+            len(self._fact_to_entities),
         )
 
     def _load_graph_metrics(self, profile_id: str) -> None:
@@ -324,7 +455,8 @@ class EntityGraphChannel:
                 }
             logger.info(
                 "Loaded graph metrics: %d facts for profile %s",
-                len(self._graph_metrics), profile_id,
+                len(self._graph_metrics),
+                profile_id,
             )
         except Exception as exc:
             logger.debug("Graph metrics load failed (graceful degradation): %s", exc)
@@ -412,7 +544,7 @@ class EntityGraphChannel:
         # Spreading activation through graph edges (all in-memory O(1) lookups)
         frontier = set(activation.keys())
         for hop in range(1, self._max_hops):
-            hop_decay = self._decay ** hop
+            hop_decay = self._decay**hop
             if hop_decay < self._threshold:
                 break
             next_frontier: set[str] = set()
@@ -448,9 +580,8 @@ class EntityGraphChannel:
                     ):
                         neighbor = edge.target_id if edge.source_id == fid else edge.source_id
                         propagated = activation[fid] * self._decay
-                        if (
-                            propagated >= self._threshold
-                            and propagated > activation.get(neighbor, 0.0)
+                        if propagated >= self._threshold and propagated > activation.get(
+                            neighbor, 0.0
                         ):
                             activation[neighbor] = propagated
                             next_frontier.add(neighbor)
@@ -490,6 +621,7 @@ class EntityGraphChannel:
         # v3.4.1 P2: Community-aware boosting
         if self._graph_metrics and use_cache:
             from collections import Counter as _Counter
+
             seed_communities: _Counter = _Counter()
             for eid in canonical_ids:
                 for fid in self._entity_to_facts.get(eid, ()):
@@ -629,7 +761,7 @@ class EntityGraphChannel:
 
         frontier = set(activation.keys())
         for hop in range(1, self._max_hops):
-            hop_decay = self._decay ** hop
+            hop_decay = self._decay**hop
             if hop_decay < self._threshold:
                 break
             next_frontier: set[str] = set()
@@ -664,6 +796,7 @@ class EntityGraphChannel:
         # Community-aware boosting (same as search)
         if self._graph_metrics and use_cache:
             from collections import Counter as _Counter
+
             seed_communities: _Counter = _Counter()
             for eid in canonical_ids:
                 for fid in self._entity_to_facts.get(eid, ()):
@@ -691,7 +824,9 @@ class EntityGraphChannel:
         return scored
 
     def _suppress_contradictions(
-        self, activation: dict[str, float], profile_id: str,
+        self,
+        activation: dict[str, float],
+        profile_id: str,
     ) -> None:
         """P3: Penalize older fact in contradiction pairs, heavy-penalize superseded.
 
@@ -783,7 +918,10 @@ class EntityGraphChannel:
         return ids
 
     def _discover_entities(
-        self, fact_ids: set[str], profile_id: str, visited: set[str],
+        self,
+        fact_ids: set[str],
+        profile_id: str,
+        visited: set[str],
     ) -> list[str]:
         """Find new canonical entity IDs referenced by a set of facts."""
         new: list[str] = []
@@ -797,7 +935,8 @@ class EntityGraphChannel:
         )
         for fid in allowed_fact_ids:
             rows = self._db.execute(
-                "SELECT canonical_entities_json FROM atomic_facts WHERE fact_id = ?", (fid,),
+                "SELECT canonical_entities_json FROM atomic_facts WHERE fact_id = ?",
+                (fid,),
             )
             if not rows:
                 continue
@@ -815,8 +954,11 @@ class EntityGraphChannel:
 
     # v3.4.5: CozoDB-backed search (Sprint 2)
     def _search_via_cozo(
-        self, query: str, raw_entities: list[str],
-        profile_id: str, top_k: int,
+        self,
+        query: str,
+        raw_entities: list[str],
+        profile_id: str,
+        top_k: int,
         *,
         include_global: bool = False,
         include_shared: bool = False,
@@ -879,7 +1021,10 @@ class EntityGraphChannel:
             return self._search_without_cozo(query, profile_id, top_k)
 
     def _search_without_cozo(
-        self, query: str, profile_id: str, top_k: int,
+        self,
+        query: str,
+        profile_id: str,
+        top_k: int,
     ) -> list[tuple[str, float]]:
         """Run canonical SQLite entity recall without recursive projection use."""
         cozo, self._cozo = self._cozo, None