superlocalmemory 3.4.63 → 3.5.0

package/src/superlocalmemory/server/unified_daemon.py

@@ -469,6 +469,36 @@ async def lifespan(application: FastAPI):
         _engine = engine
         logger.info("Unified daemon: MemoryEngine initialized (mode=%s)", config.mode.value)
 
+        # v3.5.0: Backend Orchestrator — CozoDB (graph) + LanceDB (vector) backends.
+        # Initialise AFTER engine so the retrieval channels exist to receive backends.
+        # Migrates edges/embeddings automatically; fail-soft (non-blocking).
+        _cozo_backend = None
+        _lancedb_backend = None
+        try:
+            from superlocalmemory.core.backend_orchestrator import (
+                BackendOrchestrator, set_orchestrator,
+            )
+            orch = BackendOrchestrator(config=config, db=engine._db)
+            orch.on_daemon_start()
+            set_orchestrator(orch)
+            _cozo_backend = orch.get_graph_backend()
+            _lancedb_backend = orch.get_vector_backend()
+            # Inject CozoDB into entity_graph channel (already has the param).
+            re = getattr(engine, '_retrieval_engine', None)
+            if re is not None:
+                eg = getattr(re, '_entity', None)
+                if eg is not None and _cozo_backend is not None:
+                    try:
+                        eg._cozo = _cozo_backend
+                        logger.info("CozoDB backend wired into entity_graph channel")
+                    except Exception as exc:
+                        logger.warning("CozoDB channel injection failed: %s", exc)
+            logger.info("BackendOrchestrator: ready (cozo=%s, lancedb=%s)",
+                         "active" if _cozo_backend else "off",
+                         "active" if _lancedb_backend else "off")
+        except Exception as exc:
+            logger.warning("BackendOrchestrator init failed (non-fatal): %s", exc)
+
         # LLD-07 §4 — deferred migrations (e.g. M006 reward column) need to
         # run AFTER MemoryEngine.initialize() has bootstrapped runtime tables
         # like action_outcomes. Non-fatal by contract.
@@ -603,8 +633,58 @@ async def lifespan(application: FastAPI):
             except Exception as exc:
                 logger.warning("Recall warmup failed (non-fatal): %s", exc)
 
+        def _backfill_vector_store():
+            """v3.5.0: index facts whose embeddings exist in atomic_facts but
+            are missing from the sqlite-vec store. Facts stored before dual-write
+            was complete were never indexed, so semantic + hopfield only saw a
+            fraction of the corpus (observed: 5.8k indexed of 17.2k embedded).
+            Idempotent (skips when the store is already complete), non-blocking,
+            fail-soft. Runs once after a 3.5.0 upgrade, then no-ops every restart.
+            """
+            import time as _t
+            from pathlib import Path as _P
+            for _ in range(120):
+                if _embedding_warm:
+                    break
+                _t.sleep(0.5)
+            try:
+                from superlocalmemory.retrieval.vector_store import (
+                    VectorStore, VectorStoreConfig,
+                )
+                db = engine._db
+                db_path = getattr(db, "db_path", None) or getattr(db, "_db_path", None)
+                if db_path is None:
+                    return
+                dim = getattr(getattr(config, "embedding", None), "dimension", 768) or 768
+                vs = VectorStore(_P(db_path), VectorStoreConfig(dimension=dim))
+                if not vs.available:
+                    return
+                try:
+                    profiles = list(db.list_profiles()) or ["default"]
+                except Exception:
+                    profiles = ["default"]
+                for pid in profiles:
+                    facts = db.get_all_facts(pid)
+                    with_emb = [
+                        (f.fact_id, getattr(f, "profile_id", pid) or pid, f.embedding)
+                        for f in facts
+                        if getattr(f, "embedding", None) and len(f.embedding) == dim
+                    ]
+                    if not with_emb:
+                        continue
+                    if vs.count(pid) >= int(len(with_emb) * 0.98):
+                        continue  # already complete — no-op
+                    n = vs.rebuild_from_facts(with_emb)
+                    logger.info(
+                        "VS backfill[%s]: indexed %d of %d embedded facts",
+                        pid, n, len(with_emb),
+                    )
+            except Exception as exc:
+                logger.warning("Vector store backfill failed (non-fatal): %s", exc)
+
         threading.Thread(target=_warmup_embedder, daemon=True, name="embed-warmup").start()
         threading.Thread(target=_warmup_recall, daemon=True, name="recall-warmup").start()
+        threading.Thread(target=_backfill_vector_store, daemon=True, name="vs-backfill").start()
 
         # v3.4.37: QueueConsumer uses daemon's engine directly via adapter.
         # Previously routed through WorkerPool → recall_worker subprocess,

package/src/superlocalmemory/storage/database.py

@@ -239,9 +239,26 @@ class DatabaseManager:
             emotional_valence=d.get("emotional_valence", 0.0),
             emotional_arousal=d.get("emotional_arousal", 0.0),
             signal_type=SignalType(d["signal_type"]) if d.get("signal_type") else SignalType.FACTUAL,
+            pinned=bool(d.get("pinned", 0)),
             created_at=d["created_at"],
         )
 
+    def set_pinned(self, fact_id: str, pinned: bool) -> None:
+        """Set or clear the pinned flag on a fact (v3.4.65 core-memory)."""
+        self.execute(
+            "UPDATE atomic_facts SET pinned = ? WHERE fact_id = ?",
+            (1 if pinned else 0, fact_id),
+        )
+
+    def get_pinned(self, profile_id: str) -> list[AtomicFact]:
+        """Return all pinned facts for a profile, highest-importance first."""
+        rows = self.execute(
+            "SELECT * FROM atomic_facts WHERE profile_id = ? AND pinned = 1 "
+            "ORDER BY importance DESC",
+            (profile_id,),
+        )
+        return [self._row_to_fact(r) for r in rows]
+
     def get_all_facts(self, profile_id: str) -> list[AtomicFact]:
         """All facts for a profile, newest first."""
         rows = self.execute(
@@ -606,6 +623,36 @@ class DatabaseManager:
         )
         return [self._row_to_scene(r) for r in rows]
 
+    def get_scenes_for_facts_batch(
+        self, fact_ids: list[str], profile_id: str,
+    ) -> dict[str, list[MemoryScene]]:
+        """v3.5.0: batch scene lookup — one query replaces N individual LIKE scans.
+
+        The 20 individual ``get_scenes_for_fact`` calls in the retrieval engine's
+        scene expansion path were the single largest recall latency source (~5.7s).
+        This replaces them with a single multi-LIKE OR query. Returns
+        ``{fact_id: [scenes]}`` for facts that belong to at least one scene.
+        """
+        if not fact_ids:
+            return {}
+        clauses = " OR ".join(
+            '(profile_id = ? AND fact_ids_json LIKE ?)' for _ in fact_ids
+        )
+        params: list[str] = []
+        for fid in fact_ids:
+            params.extend((profile_id, f'%"{fid}"%'))
+        rows = self.execute(
+            f"SELECT * FROM memory_scenes WHERE {clauses} ORDER BY last_updated DESC",
+            tuple(params),
+        )
+        out: dict[str, list[MemoryScene]] = {}
+        for r in rows:
+            scene = self._row_to_scene(r)
+            for fid in fact_ids:
+                if fid in (scene.fact_ids or []):
+                    out.setdefault(fid, []).append(scene)
+        return out
+
     def increment_entity_fact_count(self, entity_id: str) -> None:
         """Atomically increment fact_count for a canonical entity."""
         self.execute(

package/src/superlocalmemory/storage/migration_runner.py

@@ -50,6 +50,7 @@ from superlocalmemory.storage.migrations import (
     M012_shadow_observations as _M012,
     M013_bi_temporal_columns as _M013,
     M014_v345_scale_ready as _M014,
+    M015_add_pinned_column as _M015,
 )
 
 # Map migration name → module (used for the optional ``verify(conn)`` hook
@@ -69,6 +70,7 @@ _MODULES = {
     _M012.NAME: _M012,
     _M013.NAME: _M013,
     _M014.NAME: _M014,
+    _M015.NAME: _M015,
 }
 
 logger = logging.getLogger(__name__)
@@ -130,6 +132,8 @@ DEFERRED_MIGRATIONS: list[Migration] = [
     # as M011.
     Migration(name=_M013.NAME, db_target="memory", ddl=_M013.DDL),
     Migration(name=_M014.NAME, db_target="memory", ddl=_M014.DDL),
+    # M015 adds pinned column to atomic_facts (v3.4.65 core-memory pins).
+    Migration(name=_M015.NAME, db_target="memory", ddl=_M015.DDL),
 ]
 
 

package/src/superlocalmemory/storage/migrations/M015_add_pinned_column.py

@@ -0,0 +1,40 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.65 — Context Injection v2
+
+"""M015 — add `pinned` column to atomic_facts (v3.4.65, core-memory explicit pins).
+
+Additive only — ALTER TABLE ADD COLUMN, default 0. No data loss, no type
+changes. Idempotent via verify() + migration_log. Mirrors M001 pattern.
+
+Backward-compat: old code that doesn't know about the column simply gets
+DEFAULT 0 on SELECT *, which is the safe "not-pinned" state.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+NAME = "M015_add_pinned_column"
+DB_TARGET = "memory"
+
+_REQUIRED_COLS = frozenset({"pinned"})
+
+
+def verify(conn: sqlite3.Connection) -> bool:
+    try:
+        cols = {r[1] for r in conn.execute(
+            "PRAGMA table_info(atomic_facts)"
+        ).fetchall()}
+    except sqlite3.Error:
+        return False
+    return _REQUIRED_COLS <= cols
+
+
+DDL = """
+BEGIN IMMEDIATE;
+ALTER TABLE atomic_facts ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0;
+CREATE INDEX IF NOT EXISTS idx_facts_pinned
+    ON atomic_facts(profile_id, pinned);
+COMMIT;
+"""

package/src/superlocalmemory/storage/migrations/__init__.py

@@ -23,6 +23,7 @@ from . import (
     M003_migration_log,
     M004_cross_platform_sync_log,
     M005_bandit_tables,
+    M015_add_pinned_column,
 )
 
 # ---------------------------------------------------------------------------

package/src/superlocalmemory/storage/models.py

@@ -178,6 +178,9 @@ class AtomicFact:
     # Signal type (V2 compatible)
     signal_type: SignalType = SignalType.FACTUAL
 
+    # v3.4.65: Core Memory Block explicit pin (M015)
+    pinned: bool = False
+
     created_at: str = field(default_factory=_now)
 
 

package/src/superlocalmemory.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: superlocalmemory
-Version: 3.4.63
+Version: 3.5.0
 Summary: Information-geometric agent memory with mathematical guarantees
 Author-email: Varun Pratap Bhardwaj <admin@superlocalmemory.com>
 License: AGPL-3.0-or-later
@@ -69,6 +69,8 @@ Provides-Extra: ui
 Requires-Dist: fastapi[all]>=0.135.1; extra == "ui"
 Requires-Dist: uvicorn>=0.42.0; extra == "ui"
 Requires-Dist: python-multipart<1.0.0,>=0.0.6; extra == "ui"
+Provides-Extra: injection
+Requires-Dist: tiktoken>=0.8.0; extra == "injection"
 Provides-Extra: learning
 Requires-Dist: lightgbm>=4.0.0; extra == "learning"
 Provides-Extra: performance
@@ -79,7 +81,7 @@ Requires-Dist: google-auth-oauthlib>=1.2.0; extra == "ingestion"
 Requires-Dist: google-api-python-client>=2.100.0; extra == "ingestion"
 Requires-Dist: icalendar>=6.0.0; extra == "ingestion"
 Provides-Extra: full
-Requires-Dist: superlocalmemory[ingestion,learning,performance,search,ui]; extra == "full"
+Requires-Dist: superlocalmemory[ingestion,injection,learning,performance,search,ui]; extra == "full"
 Provides-Extra: dev
 Requires-Dist: pytest>=8.0; extra == "dev"
 Requires-Dist: pytest-cov>=4.1; extra == "dev"

package/src/superlocalmemory.egg-info/SOURCES.txt

@@ -86,6 +86,7 @@ src/superlocalmemory/core/graph_analyzer.py
 src/superlocalmemory/core/graph_pruner.py
 src/superlocalmemory/core/health_monitor.py
 src/superlocalmemory/core/hooks.py
+src/superlocalmemory/core/injection.py
 src/superlocalmemory/core/loop_watchdog.py
 src/superlocalmemory/core/maintenance.py
 src/superlocalmemory/core/maintenance_scheduler.py
@@ -377,6 +378,7 @@ src/superlocalmemory/storage/migrations/M011_archive_and_merge.py
 src/superlocalmemory/storage/migrations/M012_shadow_observations.py
 src/superlocalmemory/storage/migrations/M013_bi_temporal_columns.py
 src/superlocalmemory/storage/migrations/M014_v345_scale_ready.py
+src/superlocalmemory/storage/migrations/M015_add_pinned_column.py
 src/superlocalmemory/storage/migrations/__init__.py
 src/superlocalmemory/trust/__init__.py
 src/superlocalmemory/trust/gate.py
@@ -452,6 +454,7 @@ tests/test_final_locomo_mini.py
 tests/test_hook_handlers.py
 tests/test_ide_connector.py
 tests/test_infra.py
+tests/test_injection.py
 tests/test_learning_advanced.py
 tests/test_learning_collectors.py
 tests/test_llm_provider.py

package/src/superlocalmemory.egg-info/requires.txt

@@ -36,7 +36,7 @@ pytest-asyncio>=0.21
 sqlite-vec>=0.1.6
 
 [full]
-superlocalmemory[ingestion,learning,performance,search,ui]
+superlocalmemory[ingestion,injection,learning,performance,search,ui]
 
 [ingestion]
 keyring>=25.0.0
@@ -44,6 +44,9 @@ google-auth-oauthlib>=1.2.0
 google-api-python-client>=2.100.0
 icalendar>=6.0.0
 
+[injection]
+tiktoken>=0.8.0
+
 [learning]
 lightgbm>=4.0.0