superlocalmemory 3.0.31 → 3.0.33

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "superlocalmemory",
-  "version": "3.0.31",
+  "version": "3.0.33",
   "description": "Information-geometric agent memory with mathematical guarantees. 4-channel retrieval, Fisher-Rao similarity, zero-LLM mode, EU AI Act compliant. Works with Claude, Cursor, Windsurf, and 17+ AI tools.",
   "keywords": [
     "ai-memory",

package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "superlocalmemory"
-version = "3.0.31"
+version = "3.0.33"
 description = "Information-geometric agent memory with mathematical guarantees"
 readme = "README.md"
 license = {text = "MIT"}

package/src/superlocalmemory/core/config.py

@@ -37,6 +37,11 @@ class EmbeddingConfig:
 
     model_name: str = "nomic-ai/nomic-embed-text-v1.5"
     dimension: int = 768
+    # Provider: "" = auto-detect, "sentence-transformers", "ollama", "cloud"
+    provider: str = ""
+    # Ollama settings (used when provider="ollama" or auto-detected)
+    ollama_model: str = "nomic-embed-text"
+    ollama_base_url: str = "http://localhost:11434"
     # Azure / cloud settings (Mode C only)
     api_endpoint: str = ""
     api_key: str = ""
@@ -45,7 +50,11 @@ class EmbeddingConfig:
 
     @property
     def is_cloud(self) -> bool:
-        return bool(self.api_endpoint)
+        return bool(self.api_endpoint) or self.provider == "cloud"
+
+    @property
+    def is_ollama(self) -> bool:
+        return self.provider == "ollama"
 
 
 # ---------------------------------------------------------------------------
@@ -235,15 +244,17 @@ class SLMConfig:
         data = json.loads(path.read_text())
         mode = Mode(data.get("mode", "a"))
         llm_data = data.get("llm", {})
+        emb_data = data.get("embedding", {})
         config = cls.for_mode(
             mode,
             llm_provider=llm_data.get("provider", ""),
             llm_model=llm_data.get("model", ""),
             llm_api_key=llm_data.get("api_key", ""),
             llm_api_base=llm_data.get("base_url", ""),
-            embedding_endpoint=data.get("embedding", {}).get("api_endpoint", ""),
-            embedding_key=data.get("embedding", {}).get("api_key", ""),
-            embedding_deployment=data.get("embedding", {}).get("deployment_name", ""),
+            embedding_provider=emb_data.get("provider", ""),
+            embedding_endpoint=emb_data.get("api_endpoint", ""),
+            embedding_key=emb_data.get("api_key", ""),
+            embedding_deployment=emb_data.get("deployment_name", ""),
         )
         config.active_profile = data.get("active_profile", "default")
         return config
@@ -265,6 +276,7 @@ class SLMConfig:
             "embedding": {
                 "model_name": self.embedding.model_name,
                 "dimension": self.embedding.dimension,
+                "provider": self.embedding.provider,
                 "api_endpoint": self.embedding.api_endpoint,
                 "api_key": self.embedding.api_key,
                 "deployment_name": self.embedding.deployment_name,
@@ -317,6 +329,7 @@ class SLMConfig:
         llm_model: str = "",
         llm_api_key: str = "",
         llm_api_base: str = "",
+        embedding_provider: str = "",
         embedding_endpoint: str = "",
         embedding_key: str = "",
         embedding_deployment: str = "",
@@ -331,6 +344,7 @@ class SLMConfig:
                 embedding=EmbeddingConfig(
                     model_name="nomic-ai/nomic-embed-text-v1.5",
                     dimension=768,
+                    provider=embedding_provider,
                 ),
                 llm=LLMConfig(),  # No LLM
                 retrieval=RetrievalConfig(
@@ -348,6 +362,7 @@ class SLMConfig:
                 embedding=EmbeddingConfig(
                     model_name="nomic-ai/nomic-embed-text-v1.5",
                     dimension=768,
+                    provider=embedding_provider,
                 ),
                 llm=LLMConfig(
                     provider=llm_provider or "ollama",

package/src/superlocalmemory/core/embeddings.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 import json
 import logging
 import os
+import select
 import subprocess
 import sys
 import threading
@@ -45,6 +46,7 @@ class DimensionMismatchError(RuntimeError):
 
 
 _IDLE_TIMEOUT_SECONDS = 120  # 2 minutes — kill worker after idle
+_SUBPROCESS_RESPONSE_TIMEOUT = 60  # seconds — max wait for worker response
 
 
 class EmbeddingService:
@@ -137,7 +139,11 @@ class EmbeddingService:
     # ------------------------------------------------------------------
 
     def _subprocess_embed(self, texts: list[str]) -> list[list[float]] | None:
-        """Send texts to worker subprocess, get embeddings back."""
+        """Send texts to worker subprocess, get embeddings back.
+
+        Includes a timeout (_SUBPROCESS_RESPONSE_TIMEOUT seconds) so the CLI
+        never hangs indefinitely on cold model loads or network issues.
+        """
         with self._lock:
             self._ensure_worker()
             if self._worker_proc is None:
@@ -153,9 +159,12 @@ class EmbeddingService:
             try:
                 self._worker_proc.stdin.write(req)
                 self._worker_proc.stdin.flush()
-                resp_line = self._worker_proc.stdout.readline()
+                resp_line = self._readline_with_timeout(
+                    self._worker_proc.stdout,
+                    _SUBPROCESS_RESPONSE_TIMEOUT,
+                )
                 if not resp_line:
-                    logger.warning("Worker returned empty response, restarting")
+                    logger.warning("Worker returned empty or timed out, restarting")
                     self._kill_worker()
                     return None
                 resp = json.loads(resp_line)
@@ -169,6 +178,31 @@ class EmbeddingService:
                 self._kill_worker()
                 return None
 
+    @staticmethod
+    def _readline_with_timeout(stream, timeout_seconds: float) -> str:
+        """Read a line from stream with a timeout. Returns '' on timeout."""
+        result_container: list[str] = []
+        error_container: list[Exception] = []
+
+        def _read() -> None:
+            try:
+                result_container.append(stream.readline())
+            except Exception as exc:
+                error_container.append(exc)
+
+        reader = threading.Thread(target=_read, daemon=True)
+        reader.start()
+        reader.join(timeout=timeout_seconds)
+
+        if reader.is_alive():
+            logger.warning(
+                "Embedding worker did not respond within %ds", timeout_seconds,
+            )
+            return ""
+        if error_container:
+            raise error_container[0]
+        return result_container[0] if result_container else ""
+
     def _ensure_worker(self) -> None:
         """Spawn worker subprocess if not running."""
         if self._worker_proc is not None and self._worker_proc.poll() is None:

package/src/superlocalmemory/core/engine.py

@@ -81,12 +81,7 @@ class MemoryEngine:
 
         self._db = DatabaseManager(self._config.db_path)
         self._db.initialize(schema)
-        try:
-            emb = EmbeddingService(self._config.embedding)
-            self._embedder = emb if emb.is_available else None
-        except Exception as exc:
-            logger.warning("Embeddings unavailable (%s). BM25-only mode.", exc)
-            self._embedder = None
+        self._embedder = self._init_embedder()
 
         if self._caps.llm_fact_extraction:
             self._llm = LLMBackbone(self._config.llm)
@@ -115,6 +110,76 @@ class MemoryEngine:
         logger.info("MemoryEngine initialized: mode=%s profile=%s",
                      self._config.mode.value, self._profile_id)
 
+    def _init_embedder(self):
+        """Initialize the best available embedding provider.
+
+        Priority order:
+        1. Explicit provider in config (ollama / cloud / sentence-transformers)
+        2. Auto-detect: if LLM provider=ollama and Ollama has embedding model → use it
+        3. Fallback to sentence-transformers subprocess
+        4. If nothing works → None (BM25-only mode)
+        """
+        from superlocalmemory.core.embeddings import EmbeddingService
+
+        emb_cfg = self._config.embedding
+        provider = emb_cfg.provider
+
+        # --- Explicit ollama provider ---
+        if provider == "ollama":
+            return self._try_ollama_embedder(emb_cfg)
+
+        # --- Explicit cloud provider ---
+        if provider == "cloud" or emb_cfg.is_cloud:
+            return self._try_service_embedder(EmbeddingService, emb_cfg)
+
+        # --- Explicit sentence-transformers ---
+        if provider == "sentence-transformers":
+            return self._try_service_embedder(EmbeddingService, emb_cfg)
+
+        # --- Auto-detect: try Ollama first (fast path, <1s) ---
+        # Check regardless of LLM provider — if Ollama is running and has
+        # the embedding model, use it. This avoids the 30s cold start of
+        # sentence-transformers subprocess.
+        ollama_emb = self._try_ollama_embedder(emb_cfg)
+        if ollama_emb is not None:
+            logger.info("Auto-detected Ollama embeddings (fast path)")
+            return ollama_emb
+
+        # --- Fallback: sentence-transformers subprocess ---
+        return self._try_service_embedder(EmbeddingService, emb_cfg)
+
+    def _try_ollama_embedder(self, emb_cfg):
+        """Try to create an OllamaEmbedder. Returns it or None."""
+        try:
+            from superlocalmemory.core.ollama_embedder import OllamaEmbedder
+            emb = OllamaEmbedder(
+                model=emb_cfg.ollama_model,
+                base_url=emb_cfg.ollama_base_url,
+                dimension=emb_cfg.dimension,
+            )
+            if emb.is_available:
+                logger.info("Using Ollama embeddings (%s)", emb_cfg.ollama_model)
+                return emb
+            logger.warning(
+                "Ollama embedder not available (model=%s). Falling back.",
+                emb_cfg.ollama_model,
+            )
+        except Exception as exc:
+            logger.warning("OllamaEmbedder init failed: %s", exc)
+        return None
+
+    @staticmethod
+    def _try_service_embedder(cls, emb_cfg):
+        """Try to create an EmbeddingService. Returns it or None."""
+        try:
+            emb = cls(emb_cfg)
+            if emb.is_available:
+                return emb
+            logger.warning("EmbeddingService not available. BM25-only mode.")
+        except Exception as exc:
+            logger.warning("Embeddings unavailable (%s). BM25-only mode.", exc)
+        return None
+
     def store(
         self,
         content: str,

package/src/superlocalmemory/core/ollama_embedder.py

@@ -0,0 +1,179 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the MIT License - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Ollama Embedding Provider — lightweight HTTP-based embeddings.
+
+Uses Ollama's /api/embed endpoint for fast local embeddings without
+loading PyTorch or sentence-transformers into the process.
+
+Typical latency: <1 second (vs 30s cold start for sentence-transformers).
+Memory: ~0 MB in the SLM process (Ollama manages its own memory).
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+import numpy as np
+
+if TYPE_CHECKING:
+    pass
+
+logger = logging.getLogger(__name__)
+
+# Fisher variance constants (must match EmbeddingService)
+_FISHER_VAR_MIN = 0.05
+_FISHER_VAR_MAX = 2.0
+_FISHER_VAR_RANGE = _FISHER_VAR_MAX - _FISHER_VAR_MIN
+
+# Ollama connect/response timeouts
+_CONNECT_TIMEOUT = 5.0
+_RESPONSE_TIMEOUT = 30.0
+
+
+class OllamaEmbedder:
+    """Embedding service backed by a local Ollama instance.
+
+    Drop-in replacement for EmbeddingService. Implements the same
+    public interface (embed, embed_batch, compute_fisher_params,
+    is_available, dimension) so the engine can swap transparently.
+    """
+
+    def __init__(
+        self,
+        model: str = "nomic-embed-text",
+        base_url: str = "http://localhost:11434",
+        dimension: int = 768,
+    ) -> None:
+        self._model = model
+        self._base_url = base_url.rstrip("/")
+        self._dimension = dimension
+        self._available: bool | None = None  # lazy-checked
+
+    # ------------------------------------------------------------------
+    # Public interface (matches EmbeddingService)
+    # ------------------------------------------------------------------
+
+    @property
+    def is_available(self) -> bool:
+        """Check if Ollama is reachable and the model is pulled."""
+        if self._available is not None:
+            return self._available
+        self._available = self._check_availability()
+        return self._available
+
+    @property
+    def dimension(self) -> int:
+        return self._dimension
+
+    def embed(self, text: str) -> list[float] | None:
+        """Embed a single text. Returns normalized vector or None on failure."""
+        if not text or not text.strip():
+            raise ValueError("Cannot embed empty text")
+        try:
+            return self._call_ollama_embed(text)
+        except Exception as exc:
+            logger.warning("Ollama embed failed: %s", exc)
+            return None
+
+    def embed_batch(self, texts: list[str]) -> list[list[float] | None]:
+        """Embed a batch of texts. Uses the batch API when available."""
+        if not texts:
+            raise ValueError("Cannot embed empty batch")
+        try:
+            return self._call_ollama_embed_batch(texts)
+        except Exception as exc:
+            logger.warning("Ollama batch embed failed: %s", exc)
+            return [None] * len(texts)
+
+    def compute_fisher_params(
+        self, embedding: list[float],
+    ) -> tuple[list[float], list[float]]:
+        """Compute Fisher-Rao parameters from a raw embedding."""
+        arr = np.asarray(embedding, dtype=np.float64)
+        norm = float(np.linalg.norm(arr))
+        if norm < 1e-10:
+            mean = np.zeros(len(arr), dtype=np.float64)
+            variance = np.full(len(arr), _FISHER_VAR_MAX, dtype=np.float64)
+            return mean.tolist(), variance.tolist()
+        mean = arr / norm
+        abs_mean = np.abs(mean)
+        max_val = float(np.max(abs_mean)) + 1e-10
+        signal_strength = abs_mean / max_val
+        variance = _FISHER_VAR_MAX - _FISHER_VAR_RANGE * signal_strength
+        variance = np.clip(variance, _FISHER_VAR_MIN, _FISHER_VAR_MAX)
+        return mean.tolist(), variance.tolist()
+
+    def unload(self) -> None:
+        """No-op for Ollama (Ollama manages its own model lifecycle)."""
+
+    # ------------------------------------------------------------------
+    # Internal
+    # ------------------------------------------------------------------
+
+    def _check_availability(self) -> bool:
+        """Verify Ollama is running and has the embedding model."""
+        import httpx
+
+        try:
+            resp = httpx.get(
+                f"{self._base_url}/api/tags",
+                timeout=_CONNECT_TIMEOUT,
+            )
+            if resp.status_code != 200:
+                return False
+            models = resp.json().get("models", [])
+            model_names = [m.get("name", "").split(":")[0] for m in models]
+            model_base = self._model.split(":")[0]
+            if model_base not in model_names:
+                logger.info(
+                    "Ollama running but model '%s' not found (have: %s)",
+                    self._model, ", ".join(model_names),
+                )
+                return False
+            return True
+        except Exception as exc:
+            logger.debug("Ollama not reachable: %s", exc)
+            return False
+
+    def _call_ollama_embed(self, text: str) -> list[float]:
+        """Call Ollama embed endpoint for a single text."""
+        import httpx
+
+        resp = httpx.post(
+            f"{self._base_url}/api/embed",
+            json={"model": self._model, "input": [text]},
+            timeout=httpx.Timeout(_RESPONSE_TIMEOUT, connect=_CONNECT_TIMEOUT),
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        # Ollama /api/embed returns {"embeddings": [[...]]}
+        vec = data["embeddings"][0]
+        return self._normalize(vec)
+
+    def _call_ollama_embed_batch(self, texts: list[str]) -> list[list[float] | None]:
+        """Call Ollama embed endpoint with batch input."""
+        import httpx
+
+        resp = httpx.post(
+            f"{self._base_url}/api/embed",
+            json={"model": self._model, "input": texts},
+            timeout=httpx.Timeout(_RESPONSE_TIMEOUT, connect=_CONNECT_TIMEOUT),
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        vectors = data.get("embeddings", [])
+        return [self._normalize(v) for v in vectors]
+
+    @staticmethod
+    def _normalize(vec: list[float]) -> list[float]:
+        """L2-normalize embedding vector."""
+        arr = np.asarray(vec, dtype=np.float32)
+        norm = float(np.linalg.norm(arr))
+        if norm > 1e-10:
+            arr = arr / norm
+        return arr.tolist()

package/src/superlocalmemory/encoding/type_router.py

@@ -171,6 +171,8 @@ class TypeRouter:
 
         assert self._template_embeddings is not None
         fact_emb = self._embedder.embed(fact.content)
+        if fact_emb is None:
+            return self._classify_keywords(fact)
 
         best_type = FactType.SEMANTIC
         best_score = -1.0

package/src/superlocalmemory/storage/database.py

@@ -5,13 +5,14 @@
 """SuperLocalMemory V3 — Database Manager.
 
 SQLite with WAL, profile-scoped CRUD, FTS5 search, BM25 persistence.
-All connections use try/finally. Only ``except sqlite3.Error``.
+Concurrent-safe: WAL mode + busy_timeout + retry on SQLITE_BUSY.
+Multiple processes (MCP, CLI, integrations) can read/write safely.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
 """
 from __future__ import annotations
 
-import json, logging, sqlite3, threading
+import json, logging, sqlite3, threading, time
 from contextlib import contextmanager
 from pathlib import Path
 from types import ModuleType
@@ -37,11 +38,22 @@ def _jd(val: Any) -> str | None:
     return json.dumps(val) if val is not None else None
 
 
+_BUSY_TIMEOUT_MS = 10_000   # 10 seconds — wait for other writers
+_MAX_RETRIES = 5            # retry on transient SQLITE_BUSY
+_RETRY_BASE_DELAY = 0.1    # seconds — exponential backoff base
+
+
 class DatabaseManager:
-    """Thread-safe SQLite manager with WAL, profile isolation, and FTS5.
+    """Concurrent-safe SQLite manager with WAL, profile isolation, and FTS5.
+
+    Designed for multi-process access: MCP server, CLI, LangChain, CrewAI,
+    and other integrations can all read/write the same database safely.
 
-    Per-call connections outside transactions; shared connection inside
-    a ``transaction()`` block. Thread-safe via threading.Lock.
+    Concurrency model:
+    - WAL mode: readers never block writers, writers never block readers
+    - busy_timeout: writers wait up to 10s for other writers instead of failing
+    - Retry with backoff: transient SQLITE_BUSY errors are retried automatically
+    - Per-call connections: no shared state between processes
     """
 
     def __init__(self, db_path: str | Path) -> None:
@@ -55,6 +67,7 @@ class DatabaseManager:
         conn = sqlite3.connect(str(self.db_path))
         try:
             conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute(f"PRAGMA busy_timeout={_BUSY_TIMEOUT_MS}")
             conn.execute("PRAGMA foreign_keys=ON")
             conn.commit()
         finally:
@@ -62,9 +75,8 @@ class DatabaseManager:
 
     def initialize(self, schema_module: ModuleType) -> None:
         """Create all tables. *schema_module* must expose ``create_all_tables(conn)``."""
-        conn = sqlite3.connect(str(self.db_path))
+        conn = self._connect()
         try:
-            conn.execute("PRAGMA foreign_keys=ON")
             schema_module.create_all_tables(conn)
             conn.commit()
             logger.info("Schema initialized at %s", self.db_path)
@@ -81,8 +93,9 @@ class DatabaseManager:
         self.close()
 
     def _connect(self) -> sqlite3.Connection:
-        conn = sqlite3.connect(str(self.db_path))
+        conn = sqlite3.connect(str(self.db_path), timeout=_BUSY_TIMEOUT_MS / 1000)
         conn.row_factory = sqlite3.Row
+        conn.execute(f"PRAGMA busy_timeout={_BUSY_TIMEOUT_MS}")
         conn.execute("PRAGMA foreign_keys=ON")
         return conn
 
@@ -103,16 +116,36 @@ class DatabaseManager:
                 conn.close()
 
     def execute(self, sql: str, params: tuple[Any, ...] = ()) -> list[sqlite3.Row]:
-        """Execute SQL. Uses shared conn inside transaction, else per-call."""
+        """Execute SQL with automatic retry on SQLITE_BUSY.
+
+        Uses shared conn inside transaction, else per-call with retry.
+        """
         if self._txn_conn is not None:
             return self._txn_conn.execute(sql, params).fetchall()
-        conn = self._connect()
-        try:
-            rows = conn.execute(sql, params).fetchall()
-            conn.commit()
-            return rows
-        finally:
-            conn.close()
+
+        last_error: Exception | None = None
+        for attempt in range(_MAX_RETRIES):
+            conn = self._connect()
+            try:
+                rows = conn.execute(sql, params).fetchall()
+                conn.commit()
+                return rows
+            except sqlite3.OperationalError as exc:
+                last_error = exc
+                if "locked" in str(exc).lower() or "busy" in str(exc).lower():
+                    delay = _RETRY_BASE_DELAY * (2 ** attempt)
+                    logger.debug(
+                        "DB busy (attempt %d/%d), retrying in %.1fs: %s",
+                        attempt + 1, _MAX_RETRIES, delay, exc,
+                    )
+                    time.sleep(delay)
+                    continue
+                raise
+            finally:
+                conn.close()
+
+        logger.warning("DB operation failed after %d retries: %s", _MAX_RETRIES, last_error)
+        raise last_error  # type: ignore[misc]
 
     def store_memory(self, record: MemoryRecord) -> str:
         """Persist a raw memory record. Returns memory_id."""