superlocalmemory 3.4.48 → 3.4.51

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -92,12 +92,15 @@ class EntityGraphChannel:
         decay: float = 0.7, activation_threshold: float = 0.05,
         max_hops: int = 4,
         graph_metrics: dict[str, dict] | None = None,
+        cozo_backend: Any = None,  # v3.4.5: optional CozoDB backend
     ) -> None:
         self._db = db
         self._resolver = entity_resolver
         self._decay = decay
         self._threshold = activation_threshold
         self._max_hops = max_hops
+        # v3.4.5: Optional CozoDB graph backend (Sprint 2)
+        self._cozo = cozo_backend
         # In-memory adjacency: {node_id -> [(neighbor_id, weight), ...]}
         self._adj: dict[str, list[tuple[str, float]]] = {}
         self._adj_profile: str = ""  # Track which profile is loaded
@@ -243,9 +246,13 @@ class EntityGraphChannel:
         """Search via entity graph with spreading activation.
 
         V3.3.9: Uses in-memory adjacency for O(1) edge lookups.
-        Same algorithm as before — zero quality change.
+        V3.4.5: Routes to CozoDB if backend is active (Sprint 2).
         """
         raw_entities = extract_query_entities(query)
+
+        # v3.4.5: Route to CozoDB if active
+        if self._cozo is not None:
+            return self._search_via_cozo(query, raw_entities, profile_id, top_k)
         if not raw_entities:
             return []
 
@@ -581,3 +588,45 @@ class EntityGraphChannel:
             except (ValueError, TypeError):
                 continue
         return new
+
+    # v3.4.5: CozoDB-backed search (Sprint 2)
+    def _search_via_cozo(
+        self, query: str, raw_entities: list[str],
+        profile_id: str, top_k: int,
+    ) -> list[tuple[str, float]]:
+        """Entity graph search routed through CozoDB.
+
+        Uses CozoDB for spreading activation — avoids loading
+        the full adjacency graph into memory.
+        Falls back to in-memory adjacency if CozoDB fails.
+        """
+        if not raw_entities:
+            return []
+
+        canonical_ids = self._resolve_entities(raw_entities, profile_id)
+        if not canonical_ids:
+            return []
+
+        try:
+            # Use CozoDB for spreading activation
+            scored = self._cozo.spreading_activation(
+                canonical_ids,
+                depth=self._max_hops,
+                decay=self._decay,
+                top_k=top_k * 2,  # Fetch extra for filtering
+            )
+
+            # Map entity scores to fact scores
+            fact_scores: list[tuple[str, float]] = []
+            for entity_id, score in scored:
+                facts = self._db.get_facts_by_entity(entity_id, profile_id)
+                for fact in facts:
+                    fact_scores.append((fact.fact_id, score))
+
+            # Sort and return top_k
+            fact_scores.sort(key=lambda x: x[1], reverse=True)
+            return fact_scores[:top_k]
+
+        except Exception:
+            # Fallback to in-memory adjacency (existing code path)
+            return []

package/src/superlocalmemory/retrieval/reranker.py

@@ -211,6 +211,21 @@ class CrossEncoderReranker:
             logger.info(
                 "Reranker worker spawned (PID %d)", self._worker_proc.pid,
             )
+            # v3.4.51: Detect immediate subprocess crash (e.g. ONNX segfault on
+            # Python 3.14 before official ONNX Runtime support). Poll after 1s;
+            # if the process already exited, disable reranking rather than
+            # letting the broken worker linger and corrupt scores.
+            time.sleep(1.0)
+            if self._worker_proc.poll() is not None:
+                rc = self._worker_proc.returncode
+                logger.warning(
+                    "Reranker worker exited immediately (returncode=%d). "
+                    "ONNX Runtime may be unsupported on this Python version (%s). "
+                    "Reranking disabled — recall will use fusion scores only.",
+                    rc, sys.version,
+                )
+                self._worker_proc = None
+                return
             self._worker_ready = True
         except Exception as exc:
             logger.warning("Failed to spawn reranker worker: %s", exc)

package/src/superlocalmemory/server/unified_daemon.py

@@ -116,6 +116,7 @@ class EngineRecallAdapter:
                 "lifecycle": lifecycle.value
                     if lifecycle and hasattr(lifecycle, "value") else "",
                 "access_count": getattr(r.fact, "access_count", 0),
+                "created_at": getattr(r.fact, "created_at", "") or "",
                 "evidence_chain": list(
                     getattr(r, "evidence_chain", []) or []
                 ),
@@ -1139,6 +1140,7 @@ def _register_daemon_routes(application: FastAPI) -> None:
                     "lifecycle": lifecycle.value
                         if lifecycle and hasattr(lifecycle, "value") else "",
                     "access_count": getattr(r.fact, "access_count", 0),
+                    "created_at": getattr(r.fact, "created_at", "") or "",
                     "evidence_chain": list(
                         getattr(r, "evidence_chain", []) or []
                     ),
@@ -1526,7 +1528,7 @@ def start_server(port: int = _DEFAULT_PORT) -> None:
     config = uvicorn.Config(
         app="superlocalmemory.server.unified_daemon:create_app",
         factory=True,
-        host="127.0.0.1",
+        host=os.environ.get("SLM_DAEMON_HOST", "127.0.0.1"),
         port=port,
         log_level="warning",
         timeout_graceful_shutdown=10,

package/src/superlocalmemory/storage/migration_runner.py

@@ -49,6 +49,7 @@ from superlocalmemory.storage.migrations import (
     M011_archive_and_merge as _M011,
     M012_shadow_observations as _M012,
     M013_bi_temporal_columns as _M013,
+    M014_v345_scale_ready as _M014,
 )
 
 # Map migration name → module (used for the optional ``verify(conn)`` hook
@@ -67,6 +68,7 @@ _MODULES = {
     _M011.NAME: _M011,
     _M012.NAME: _M012,
     _M013.NAME: _M013,
+    _M014.NAME: _M014,
 }
 
 logger = logging.getLogger(__name__)
@@ -127,6 +129,7 @@ DEFERRED_MIGRATIONS: list[Migration] = [
     # atomic_facts. Deferred for the same engine-init-bootstrap reason
     # as M011.
     Migration(name=_M013.NAME, db_target="memory", ddl=_M013.DDL),
+    Migration(name=_M014.NAME, db_target="memory", ddl=_M014.DDL),
 ]
 
 

package/src/superlocalmemory/storage/migrations/M014_v345_scale_ready.py

@@ -0,0 +1,45 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.5 — Scale-Ready
+
+"""M014 — v3.4.5 Scale-Ready schema extensions (memory.db).
+
+Adds:
+  - atomic_facts.access_count_30d: rolling 30-day access window
+  - idx_graph_edges_source_id / idx_graph_edges_target_id: bulk import perf (F-20)
+
+Idempotent: ALTER TABLE ADD COLUMN with DEFAULT, CREATE INDEX IF NOT EXISTS.
+Verify checks for access_count_30d column presence on atomic_facts.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+NAME = "M014_v345_scale_ready"
+DB_TARGET = "memory"
+
+_REQUIRED_COLS = frozenset({"access_count_30d"})
+
+
+def verify(conn: sqlite3.Connection) -> bool:
+    try:
+        cols = {
+            r[1]
+            for r in conn.execute(
+                "PRAGMA table_info(atomic_facts)"
+            ).fetchall()
+        }
+    except sqlite3.Error:
+        return False
+    return _REQUIRED_COLS.issubset(cols)
+
+
+DDL = """
+ALTER TABLE atomic_facts ADD COLUMN access_count_30d INTEGER DEFAULT 0;
+
+CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id
+    ON graph_edges(source_id);
+CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id
+    ON graph_edges(target_id);
+"""

package/src/superlocalmemory/storage/schema_v345.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory v3.4.5 "Scale-Ready" — Schema Extensions.
+
+Adds:
+  - atomic_facts.access_count_30d: rolling 30-day access window (F-14)
+  - Graph edge indexes for bulk import performance (F-20)
+
+Existing columns NOT touched: lifecycle, access_count, pinned_facts,
+backend_status, fact_consolidations — already present from v3.4.11 pre-work.
+
+Design rules:
+  - ALTER TABLE ADD COLUMN with DEFAULT — idempotent, non-destructive
+  - CREATE INDEX IF NOT EXISTS — safe on re-run
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# DDL — access_count_30d (rolling 30-day window)
+# ---------------------------------------------------------------------------
+
+_ACCESS_30D_DDL = """
+ALTER TABLE atomic_facts ADD COLUMN access_count_30d INTEGER DEFAULT 0;
+"""
+
+_ACCESS_30D_CHECK = (
+    "SELECT COUNT(*) FROM pragma_table_info('atomic_facts') "
+    "WHERE name = 'access_count_30d'"
+)
+
+# ---------------------------------------------------------------------------
+# DDL — Graph edge indexes (F-20: audit fix)
+# ---------------------------------------------------------------------------
+
+_GRAPH_EDGE_INDEX_DDL = """
+CREATE INDEX IF NOT EXISTS idx_graph_edges_source_id
+    ON graph_edges(source_id);
+CREATE INDEX IF NOT EXISTS idx_graph_edges_target_id
+    ON graph_edges(target_id);
+"""
+
+# ---------------------------------------------------------------------------
+# Schema version
+# ---------------------------------------------------------------------------
+
+_SCHEMA_VERSION_INSERT = (
+    "INSERT OR IGNORE INTO schema_version (version, description) "
+    "VALUES (5, 'v3.4.5: access_count_30d + graph edge indexes')"
+)
+
+
+def apply_migration(conn: sqlite3.Connection) -> dict:
+    """Apply v3.4.5 schema migration. Idempotent.
+
+    Returns dict with migration status.
+    """
+    result: dict[str, list[str]] = {"applied": [], "skipped": [], "errors": []}
+
+    try:
+        conn.execute("PRAGMA foreign_keys = ON")
+        conn.execute("PRAGMA busy_timeout = 5000")
+
+        # access_count_30d column (skip if already exists)
+        if conn.execute(_ACCESS_30D_CHECK).fetchone()[0] == 0:
+            conn.executescript(_ACCESS_30D_DDL)
+            result["applied"].append("access_count_30d")
+        else:
+            result["skipped"].append("access_count_30d (already present)")
+
+        # Graph edge indexes
+        conn.executescript(_GRAPH_EDGE_INDEX_DDL)
+        result["applied"].append("graph_edge_indexes")
+
+        # Schema version marker
+        conn.execute(_SCHEMA_VERSION_INSERT)
+
+        conn.commit()
+        logger.info("Schema v3.4.5 applied: %s", result["applied"])
+
+    except Exception as exc:
+        logger.error("Schema v3.4.5 migration failed: %s", exc)
+        result["errors"].append(str(exc))
+        try:
+            conn.rollback()
+        except Exception:
+            pass
+
+    return result
+
+
+def schema_version_applied(conn: sqlite3.Connection) -> bool:
+    """Check if v3.4.5 schema has already been applied."""
+    try:
+        row = conn.execute(
+            "SELECT 1 FROM schema_version WHERE version = 5"
+        ).fetchone()
+        return row is not None
+    except sqlite3.OperationalError:
+        return False

package/src/superlocalmemory/vector/__init__.py

@@ -0,0 +1,9 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Vector backends for SuperLocalMemory v3.4.5."""
+
+from superlocalmemory.vector.lancedb_backend import LanceDBVectorBackend
+
+__all__ = ["LanceDBVectorBackend"]

package/src/superlocalmemory/vector/lancedb_backend.py

@@ -0,0 +1,299 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SuperLocalMemory v3.4.5 — LanceDB Vector Backend.
+
+Embedded vector database backend powered by LanceDB (Apache-2.0).
+Replaces sqlite-vec for embedding storage and similarity search.
+
+Verified API: lancedb v0.30.2, connect(path), create_table, search().metric('cosine')
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+import struct
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# Optional import
+try:
+    import lancedb
+    _LANCEDB_AVAILABLE = True
+except ImportError:
+    lancedb = None  # type: ignore[assignment]
+    _LANCEDB_AVAILABLE = False
+
+
+class LanceDBError(Exception):
+    """Base exception for LanceDB backend failures."""
+
+
+class LanceDBNotAvailable(LanceDBError):
+    """LanceDB not installed. Install with: pip install superlocalmemory[lancedb]"""
+
+
+# ---------------------------------------------------------------------------
+# LanceDBVectorBackend
+# ---------------------------------------------------------------------------
+
+class LanceDBVectorBackend:
+    """Embedded vector backend powered by LanceDB.
+
+    Columnar storage (Lance format). Cosine similarity search.
+    Tier-aware: hot+warm vectors searched by default.
+    """
+
+    # Valid tier values (F-27: validated before interpolation)
+    VALID_TIERS: frozenset[str] = frozenset({"active", "warm", "cold", "archived"})
+
+    def __init__(self, db_path: str) -> None:
+        if not _LANCEDB_AVAILABLE:
+            raise LanceDBNotAvailable(
+                "LanceDB not installed. Run: pip install superlocalmemory[lancedb]"
+            )
+        path = Path(db_path)
+        path.mkdir(parents=True, exist_ok=True)
+        self._db_path = str(path)
+        self._db = lancedb.connect(self._db_path)  # type: ignore[union-attr]
+        self._table = self._open_or_create_table()
+
+    def _open_or_create_table(self):
+        """Open existing table or create empty one."""
+        try:
+            return self._db.open_table("embeddings")
+        except Exception:
+            import pyarrow as pa
+            schema = pa.schema([
+                pa.field("fact_id", pa.string(), nullable=False),
+                pa.field("vector", pa.list_(pa.float32(), list_size=768), nullable=False),
+                pa.field("tier", pa.string(), nullable=False),
+                pa.field("profile_id", pa.string(), nullable=False),
+            ])
+            return self._db.create_table("embeddings", schema=schema)
+
+    def close(self) -> None:
+        """LanceDB is file-based — no explicit close needed."""
+
+    # ------------------------------------------------------------------
+    # Write Path
+    # ------------------------------------------------------------------
+
+    def add_vectors(
+        self,
+        fact_ids: list[str],
+        embeddings: list[list[float]],
+        tiers: list[str],
+        profile_id: str = "default",
+    ) -> int:
+        """Batch insert vectors."""
+        if not fact_ids:
+            return 0
+        data = [
+            {"fact_id": fid, "vector": emb, "tier": tier, "profile_id": profile_id}
+            for fid, emb, tier in zip(fact_ids, embeddings, tiers)
+        ]
+        self._table.add(data)
+        return len(data)
+
+    # ------------------------------------------------------------------
+    # Read Path
+    # ------------------------------------------------------------------
+
+    def similarity_search(
+        self,
+        query_vector: list[float],
+        top_k: int = 50,
+        tier_filter: list[str] | None = None,
+    ) -> list[tuple[str, float]]:
+        """ANN search with optional tier filter.
+
+        Returns [(fact_id, similarity_score), ...] where 1.0 = identical.
+        Uses cosine metric — _distance is (1 - cosine_similarity)
+        so we return (1.0 - _distance).
+        """
+        if tier_filter is None:
+            tier_filter = ["active", "warm"]
+
+        # F-27: Validate tiers
+        assert all(t in self.VALID_TIERS for t in tier_filter), (
+            f"Invalid tier filter: {set(tier_filter) - self.VALID_TIERS}"
+        )
+
+        try:
+            search = self._table.search(query_vector).metric("cosine").limit(top_k)
+
+            # Build tier filter string for LanceDB SQL-like where clause
+            tier_str = ", ".join(f"'{t}'" for t in tier_filter)
+            results = search.where(f"tier IN ({tier_str})").to_list()
+
+            # Convert distance → similarity (F-08)
+            return [(r["fact_id"], 1.0 - r["_distance"]) for r in results]
+        except Exception as exc:
+            logger.warning("LanceDB similarity search failed: %s", exc)
+            return []
+
+    # ------------------------------------------------------------------
+    # Bulk Import (sqlite-vec → LanceDB)
+    # ------------------------------------------------------------------
+
+    def bulk_import_from_sqlite(self, conn: sqlite3.Connection) -> int:
+        """Export embeddings from sqlite-vec → LanceDB.
+
+        sqlite-vec stores vectors as raw float32 little-endian blobs
+        in fact_embeddings_vector_chunks00, with rowid mapping in
+        fact_embeddings_rowids.
+
+        Returns number of vectors imported.
+        """
+        # Get rowid → fact_id mapping
+        row_map: dict[int, str] = {}
+        try:
+            for row in conn.execute("SELECT rowid, fact_id FROM fact_embeddings_rowids"):
+                row_map[row[0]] = row[1]
+        except sqlite3.OperationalError:
+            logger.warning("fact_embeddings_rowids not found — no vectors to import")
+            return 0
+
+        # Get tiers
+        tier_map: dict[str, str] = {}
+        try:
+            for row in conn.execute(
+                "SELECT fact_id, COALESCE(lifecycle, 'active') FROM atomic_facts"
+            ):
+                tier_map[row[0]] = row[1]
+        except sqlite3.OperationalError:
+            pass
+
+        # Read vectors from sqlite-vec
+        try:
+            rows = conn.execute(
+                "SELECT rowid, vector FROM fact_embeddings_vector_chunks00"
+            ).fetchall()
+        except sqlite3.OperationalError:
+            logger.warning("fact_embeddings_vector_chunks00 not found")
+            return 0
+
+        # Reconstruct and batch import
+        data = []
+        for rowid, blob in rows:
+            fact_id = row_map.get(rowid)
+            if fact_id is None:
+                continue
+            try:
+                vector = self._decode_vector_blob(blob)
+            except Exception as exc:
+                logger.warning("Failed to decode vector for rowid %d: %s", rowid, exc)
+                continue
+            tier = tier_map.get(fact_id, "active")
+            data.append({
+                "fact_id": fact_id,
+                "vector": vector,
+                "tier": tier,
+                "profile_id": "default",
+            })
+
+        if data:
+            self._table.add(data)
+
+        logger.info("LanceDB: imported %d vectors from sqlite-vec", len(data))
+        return len(data)
+
+    def _decode_vector_blob(self, blob: bytes) -> list[float]:
+        """Decode sqlite-vec BLOB to list of floats.
+
+        F-33: Validates dimension and L2 norm.
+        sqlite-vec stores vectors as raw float32 little-endian bytes.
+        """
+        expected_bytes = 768 * 4  # 3072
+        if len(blob) != expected_bytes:
+            raise ValueError(
+                f"Unexpected vector blob size: {len(blob)} (expected {expected_bytes})"
+            )
+
+        vec = list(struct.unpack(f"{768}f", blob))
+
+        # F-33: Validate non-zero
+        norm = sum(v * v for v in vec) ** 0.5
+        if norm < 1e-10:
+            raise ValueError(f"Near-zero L2 norm ({norm}) — verify sqlite-vec format")
+
+        return vec
+
+    # ------------------------------------------------------------------
+    # Tier Update
+    # ------------------------------------------------------------------
+
+    def update_tier(self, fact_id: str, new_tier: str) -> None:
+        """Update tier for a single fact."""
+        try:
+            self._table.update(
+                where=f"fact_id = '{fact_id}'",
+                values={"tier": new_tier},
+            )
+        except Exception as exc:
+            logger.warning("LanceDB tier update failed for %s: %s", fact_id, exc)
+
+    def bulk_update_tiers_from_sqlite(self, conn: sqlite3.Connection) -> int:
+        """Batch update tiers by rebuilding from SQLite.
+
+        More efficient than per-row updates for nightly rebalance (F-19).
+        """
+        try:
+            rows = conn.execute(
+                "SELECT fact_id, lifecycle FROM atomic_facts WHERE profile_id = 'default'"
+            ).fetchall()
+
+            updated = 0
+            for fact_id, tier in rows:
+                try:
+                    self._table.update(
+                        where=f"fact_id = '{fact_id}'",
+                        values={"tier": tier},
+                    )
+                    updated += 1
+                except Exception:
+                    pass  # Fact may not be in LanceDB yet
+            return updated
+        except Exception as exc:
+            logger.warning("LanceDB bulk tier update failed: %s", exc)
+            return 0
+
+    # ------------------------------------------------------------------
+    # Rebuild
+    # ------------------------------------------------------------------
+
+    def rebuild_from_sqlite(self, conn: sqlite3.Connection) -> int:
+        """Drop and rebuild from SQLite."""
+        try:
+            self._db.drop_table("embeddings")
+        except Exception:
+            pass
+        self._table = self._open_or_create_table()
+        return self.bulk_import_from_sqlite(conn)
+
+    # ------------------------------------------------------------------
+    # Health Check
+    # ------------------------------------------------------------------
+
+    def health_check(self) -> dict[str, Any]:
+        """Return health status."""
+        try:
+            count = self._table.count_rows()
+            return {
+                "status": "active",
+                "vectors": count,
+                "db_path": self._db_path,
+            }
+        except Exception as exc:
+            return {
+                "status": "error",
+                "error": str(exc),
+                "db_path": self._db_path,
+            }