superlocalmemory 3.8.2 → 3.8.5

package/src/superlocalmemory/core/engine.py

@@ -18,6 +18,8 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
+import os
+import threading
 from pathlib import Path
 from typing import Any
 
@@ -26,7 +28,7 @@ from superlocalmemory.core.engine_capabilities import Capabilities, CapabilityEr
 from superlocalmemory.core.modes import get_capabilities
 from superlocalmemory.learning.outcome_queue import RecallEvent, enqueue_recall
 from superlocalmemory.storage.models import (
-    AtomicFact, MemoryRecord, Mode, RecallResponse,
+    AtomicFact, FactType, MemoryRecord, Mode, RecallResponse,
 )
 
 logger = logging.getLogger(__name__)
@@ -47,6 +49,34 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
         connection.close()
 
 
+# ---------------------------------------------------------------------------
+# Workstream D (3.8.4) — warm-guard sync embed helpers
+# ---------------------------------------------------------------------------
+
+def _is_remote_embedder(embedder: object) -> bool:
+    """Return True if *embedder* makes remote HTTP calls (cloud / OpenAI-compatible).
+
+    Remote embedders (100–400ms round-trip) must never block the store_fast()
+    write path synchronously — they stay on the async materializer path.
+
+    Local embedders:
+      - EmbeddingService (subprocess, local ONNX/sentence-transformers): has
+        ``_config`` with ``is_cloud=False`` and ``is_openai_compatible=False``.
+      - OllamaEmbedder (localhost HTTP, ~73ms): has NO ``_config`` attribute.
+
+    Remote embedders:
+      - EmbeddingService with ``_config.is_cloud=True`` (Azure, etc.)
+      - EmbeddingService with ``_config.is_openai_compatible=True``
+    """
+    cfg = getattr(embedder, "_config", None)
+    if cfg is None:
+        return False  # OllamaEmbedder — local
+    return bool(
+        getattr(cfg, "is_cloud", False)
+        or getattr(cfg, "is_openai_compatible", False)
+    )
+
+
 class MemoryEngine:
     """Main orchestrator for the SuperLocalMemory V3 memory system.
 
@@ -103,6 +133,11 @@ class MemoryEngine:
         self._consolidation_engine = None
         self._maintenance_scheduler = None
         self._hooks = HookRegistry()
+        # Workstream D (3.8.4): single-worker pool reused across store_fast() calls.
+        # Lazy-created on first warm-guard attempt; avoids per-call thread churn.
+        self._store_fast_embed_pool: object | None = None
+        # Lock guards the lazy-init to prevent TOCTOU race on concurrent first calls.
+        self._store_fast_embed_pool_lock = threading.Lock()
 
     # -- Public properties (Phase 2+ access) --------------------------------
 
@@ -525,9 +560,6 @@ class MemoryEngine:
         import re as _re
         import uuid as _uuid
         from datetime import datetime, timezone
-        from superlocalmemory.storage.models import (
-            AtomicFact, FactType, MemoryRecord,
-        )
         from superlocalmemory.core.engine_ingestion import content_passes_admission
         if not content_passes_admission(content):
             return []
@@ -568,15 +600,64 @@ class MemoryEngine:
                 r"\b([A-Z][a-z]+(?:\s[A-Z][a-z]+){0,3})\b", fact_text)}
             | {m.group(1) for m in _re.finditer(r"\b([A-Z]{2,})\b", fact_text)}
         )
-        # Queryable admission must never acquire the embedding worker lock.
-        # On a warm daemon that looked cheap, but on a clean Mode A install the
-        # background model load owns that lock for up to 180s and turned the
-        # receipt-first path into a hidden synchronous wait.  The canonical
-        # materializer below runs the complete pipeline and promotes this same
-        # fact with its embedding, Fisher parameters, entities and graph edges.
-        # Until then it is deliberately BM25/entity/date recallable.
+        # Workstream D (3.8.4) — warm-guard synchronous embed.
+        #
+        # Original contract (3.8.2): queryable admission NEVER acquires the
+        # embedding worker lock. On a clean Mode-A install the background model
+        # load owns that lock for up to 180s, turning the receipt-first path into
+        # a hidden synchronous wait. The canonical materializer promotes this
+        # fact with its full pipeline (embedding, Fisher, entities, graph edges).
+        #
+        # 3.8.4 extension: when the embedder is PROVABLY warm (_available is True)
+        # AND is a local embedder (not a remote cloud/OpenAI endpoint), compute the
+        # embedding synchronously with a hard 500ms cap. On timeout or any
+        # exception, fall through to emb=None — the materializer fills it async.
+        # This preserves the 3.8.2 invariant for cold start while eliminating the
+        # semantic-channel blind spot on warm daemons (the top UX complaint).
         emb = None
         fmean = fvar = None
+        _embedder_ref = self._embedder
+        if (
+            _embedder_ref is not None
+            and getattr(_embedder_ref, "_available", None) is True
+            and not _is_remote_embedder(_embedder_ref)
+        ):
+            import concurrent.futures as _cf
+            # Lazy-init the pool once per engine instance — avoids per-call
+            # thread churn and the associated resource leak from discard-on-exit.
+            # Double-checked locking guards against TOCTOU on concurrent first calls.
+            if self._store_fast_embed_pool is None:
+                with self._store_fast_embed_pool_lock:
+                    if self._store_fast_embed_pool is None:
+                        self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
+                            max_workers=1,
+                            thread_name_prefix="slm-sg-embed",
+                        )
+            try:
+                _timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 500)) / 1000.0
+            except (ValueError, TypeError):
+                _timeout_s = 0.5  # default 500 ms
+            try:
+                _future = self._store_fast_embed_pool.submit(_embedder_ref.embed, fact_text)
+                try:
+                    emb = _future.result(timeout=_timeout_s)
+                    if emb:
+                        fmean, fvar = _embedder_ref.compute_fisher_params(emb)
+                except _cf.TimeoutError:
+                    logger.debug(
+                        "store_fast: warm-guard embed timed out (>%.0fms) — deferring to materializer",
+                        _timeout_s * 1000,
+                    )
+                    emb = None
+                except Exception as _exc:
+                    logger.debug(
+                        "store_fast: warm-guard embed failed (%s) — deferring to materializer",
+                        _exc,
+                    )
+                    emb = None
+            except Exception as _exc:
+                logger.debug("store_fast: warm-guard pool submit failed (%s)", _exc)
+                emb = None
         fact = AtomicFact(
             fact_id=_uuid.uuid4().hex[:16], memory_id=record.memory_id,
             profile_id=self._profile_id, content=fact_text,

package/src/superlocalmemory/core/fact_consolidator.py

@@ -32,6 +32,10 @@ import sqlite3
 import uuid
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import TYPE_CHECKING, Union
+
+if TYPE_CHECKING:
+    from superlocalmemory.storage.database import DatabaseManager
 
 logger = logging.getLogger("superlocalmemory.fact_consolidator")
 
@@ -41,7 +45,7 @@ _MAX_CONSOLIDATED_CHARS = 2000
 
 
 def consolidate_facts(
-    db_path: str | Path,
+    db_or_path: "Union[DatabaseManager, str, Path]",
     profile_id: str = "default",
     max_clusters: int = 20,
     dry_run: bool = False,
@@ -49,6 +53,20 @@ def consolidate_facts(
 ) -> dict:
     """Find and consolidate clusters of related facts.
 
+    Concurrency fix (v3.8.4 — implements TODO from Fix-A comment):
+    When a DatabaseManager is provided the consolidation now uses per-cluster
+    short write transactions instead of one long raw_connection() that held
+    the write lock across Ollama / Cloud-LLM calls for every cluster after
+    the first.  The fix:
+
+      1. Discover clusters with a short memory_read() (no write lock).
+      2. For each cluster: fetch fact content (memory_read()), generate the
+         summary OUTSIDE any lock (Ollama / Cloud LLM may take 30 s), then
+         do the SAVEPOINT write inside a short memory_write() block.
+
+    SAVEPOINT atomicity per cluster is preserved: _consolidate_cluster still
+    uses SAVEPOINT / RELEASE / ROLLBACK TO internally.
+
     Mode behavior:
       - Mode A: Extractive only (no LLM). Always available.
       - Mode B: Ollama LLM summarization. Falls back to extractive if Ollama down.
@@ -56,7 +74,10 @@ def consolidate_facts(
 
     Returns stats: consolidated, clusters_found, facts_archived, errors.
     """
-    stats = {
+    from superlocalmemory.storage.database import DatabaseManager
+    from superlocalmemory.storage.memory_write import memory_read, memory_write
+
+    stats: dict = {
         "clusters_found": 0,
         "consolidated": 0,
         "facts_archived": 0,
@@ -71,7 +92,79 @@ def consolidate_facts(
             mode_str = getattr(mode, 'value', str(mode)).lower()
             stats["mode"] = mode_str
 
-    conn = sqlite3.connect(str(db_path))
+    if isinstance(db_or_path, DatabaseManager):
+        db_path = db_or_path.db_path
+        try:
+            # Step 1: discover clusters — short read, no write lock.
+            with memory_read(db_path) as rconn:
+                rconn.row_factory = sqlite3.Row
+                clusters = _find_consolidation_clusters(rconn, profile_id, max_clusters)
+            stats["clusters_found"] = len(clusters)
+
+            for entity_id, entity_name, fact_ids in clusters:
+                try:
+                    # Step 2: load fact content for this cluster (read, no write lock).
+                    placeholders = ",".join("?" * len(fact_ids))
+                    with memory_read(db_path) as rconn:
+                        rconn.row_factory = sqlite3.Row
+                        facts = rconn.execute(
+                            f"SELECT fact_id, content, confidence, created_at, "
+                            f"canonical_entities_json, scope, shared_with "
+                            f"FROM atomic_facts "
+                            f"WHERE fact_id IN ({placeholders}) ORDER BY created_at",
+                            fact_ids,
+                        ).fetchall()
+                        facts = [dict(f) for f in facts]
+
+                    if len(facts) < _MIN_CLUSTER_SIZE:
+                        continue
+
+                    # Step 3: generate summary OUTSIDE any write lock.
+                    # Ollama (Mode B) or Cloud LLM (Mode C) may take 30 s here.
+                    summary = _generate_summary(entity_name, facts, config)
+                    if not summary:
+                        continue
+
+                    if dry_run:
+                        stats["consolidated"] += 1
+                        stats["facts_archived"] += len(fact_ids)
+                        continue
+
+                    # Step 4: short per-cluster write — hold lock only for SQL.
+                    with memory_write(db_path) as conn:
+                        conn.row_factory = sqlite3.Row
+                        result = _consolidate_cluster(
+                            conn, profile_id, entity_id, entity_name,
+                            fact_ids, dry_run=False, config=None,
+                            _presummary=summary,
+                        )
+                    if result:
+                        stats["consolidated"] += 1
+                        stats["facts_archived"] += len(fact_ids)
+                except Exception as exc:
+                    logger.warning(
+                        "Consolidation failed for %s: %s",
+                        entity_name, exc, exc_info=True,
+                    )
+                    stats["errors"] += 1
+
+            if stats["consolidated"] > 0:
+                logger.info(
+                    "Fact consolidation: %d clusters merged, %d facts archived",
+                    stats["consolidated"], stats["facts_archived"],
+                )
+        except Exception as exc:
+            logger.error("Fact consolidation failed: %s", exc, exc_info=True)
+            stats["errors"] += 1
+            stats["error_detail"] = str(exc)
+        return stats
+
+    # Backward-compat: str | Path — open own connection.
+    logger.warning(
+        "consolidate_facts: passing a db_path is deprecated — pass a "
+        "DatabaseManager instead (Fix A backward-compat shim active)"
+    )
+    conn = sqlite3.connect(str(db_or_path))
     wal_mode = conn.execute("PRAGMA journal_mode=WAL").fetchone()
     if wal_mode and wal_mode[0] != "wal":
         logger.warning("WAL mode not active, got: %s", wal_mode[0])
@@ -79,33 +172,9 @@ def consolidate_facts(
     conn.row_factory = sqlite3.Row
 
     try:
-        clusters = _find_consolidation_clusters(conn, profile_id, max_clusters)
-        stats["clusters_found"] = len(clusters)
-
-        for entity_id, entity_name, fact_ids in clusters:
-            try:
-                result = _consolidate_cluster(
-                    conn, profile_id, entity_id, entity_name,
-                    fact_ids, dry_run, config,
-                )
-                if result:
-                    stats["consolidated"] += 1
-                    stats["facts_archived"] += len(fact_ids)
-            except Exception as exc:
-                logger.warning(
-                    "Consolidation failed for %s: %s",
-                    entity_name, exc, exc_info=True,
-                )
-                stats["errors"] += 1
-
+        _run_consolidation(conn, profile_id, max_clusters, dry_run, config, stats)
         if not dry_run:
             conn.commit()
-
-        if stats["consolidated"] > 0:
-            logger.info(
-                "Fact consolidation: %d clusters merged, %d facts archived",
-                stats["consolidated"], stats["facts_archived"],
-            )
     except Exception as exc:
         logger.error("Fact consolidation failed: %s", exc, exc_info=True)
         stats["errors"] += 1
@@ -116,6 +185,41 @@ def consolidate_facts(
     return stats
 
 
+def _run_consolidation(
+    conn: sqlite3.Connection,
+    profile_id: str,
+    max_clusters: int,
+    dry_run: bool,
+    config: object | None,
+    stats: dict,
+) -> None:
+    """Core consolidation logic — connection-agnostic inner function."""
+    clusters = _find_consolidation_clusters(conn, profile_id, max_clusters)
+    stats["clusters_found"] = len(clusters)
+
+    for entity_id, entity_name, fact_ids in clusters:
+        try:
+            result = _consolidate_cluster(
+                conn, profile_id, entity_id, entity_name,
+                fact_ids, dry_run, config,
+            )
+            if result:
+                stats["consolidated"] += 1
+                stats["facts_archived"] += len(fact_ids)
+        except Exception as exc:
+            logger.warning(
+                "Consolidation failed for %s: %s",
+                entity_name, exc, exc_info=True,
+            )
+            stats["errors"] += 1
+
+    if stats["consolidated"] > 0:
+        logger.info(
+            "Fact consolidation: %d clusters merged, %d facts archived",
+            stats["consolidated"], stats["facts_archived"],
+        )
+
+
 def _find_consolidation_clusters(
     conn: sqlite3.Connection,
     profile_id: str,
@@ -179,15 +283,25 @@ def _consolidate_cluster(
     fact_ids: list[str],
     dry_run: bool,
     config: object | None = None,
+    *,
+    _presummary: str | None = None,
 ) -> dict | None:
     """Merge a cluster of facts into one consolidated fact.
 
     All writes are wrapped in a SAVEPOINT for atomicity — if any step fails,
     the entire cluster consolidation is rolled back.
+
+    _presummary (v3.8.4): when the caller supplies a pre-computed summary
+    (generated OUTSIDE the write lock), skip the _generate_summary() call.
+    This is the short-lock path used by the DatabaseManager branch of
+    consolidate_facts().  The str | Path backward-compat path still calls
+    _generate_summary() inline (legacy behaviour, no regression).
     """
     c = conn.cursor()
 
-    # Load fact contents including canonical_entities_json
+    # Load fact contents including canonical_entities_json.
+    # Even when _presummary is provided we still re-read from the DB inside
+    # the write lock so the SAVEPOINT has a fresh, authoritative facts list.
     placeholders = ",".join("?" * len(fact_ids))
     facts = c.execute(
         f"SELECT fact_id, content, confidence, created_at, canonical_entities_json, "
@@ -200,7 +314,11 @@ def _consolidate_cluster(
     if len(facts) < _MIN_CLUSTER_SIZE:
         return None
 
-    summary = _generate_summary(entity_name, facts, config)
+    if _presummary is not None:
+        summary = _presummary
+    else:
+        # Legacy path (str | Path caller) — may call Ollama with write lock held.
+        summary = _generate_summary(entity_name, facts, config)
     if not summary:
         return None