superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/core/engine.py

@@ -18,15 +18,16 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
+import os
+import threading
 from pathlib import Path
 from typing import Any
 
 from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT, SLMConfig
 from superlocalmemory.core.engine_capabilities import Capabilities, CapabilityError
 from superlocalmemory.core.modes import get_capabilities
-from superlocalmemory.learning.outcome_queue import RecallEvent, enqueue_recall
 from superlocalmemory.storage.models import (
-    AtomicFact, MemoryRecord, Mode, RecallResponse,
+    AtomicFact, FactType, MemoryRecord, Mode, RecallResponse,
 )
 
 logger = logging.getLogger(__name__)
@@ -47,6 +48,34 @@ def _verify_ingestion_schema(memory_db: Path) -> bool:
         connection.close()
 
 
+# ---------------------------------------------------------------------------
+# Workstream D (3.8.4) — warm-guard sync embed helpers
+# ---------------------------------------------------------------------------
+
+def _is_remote_embedder(embedder: object) -> bool:
+    """Return True if *embedder* makes remote HTTP calls (cloud / OpenAI-compatible).
+
+    Remote embedders (100–400ms round-trip) must never block the store_fast()
+    write path synchronously — they stay on the async materializer path.
+
+    Local embedders:
+      - EmbeddingService (subprocess, local ONNX/sentence-transformers): has
+        ``_config`` with ``is_cloud=False`` and ``is_openai_compatible=False``.
+      - OllamaEmbedder (localhost HTTP, ~73ms): has NO ``_config`` attribute.
+
+    Remote embedders:
+      - EmbeddingService with ``_config.is_cloud=True`` (Azure, etc.)
+      - EmbeddingService with ``_config.is_openai_compatible=True``
+    """
+    cfg = getattr(embedder, "_config", None)
+    if cfg is None:
+        return False  # OllamaEmbedder — local
+    return bool(
+        getattr(cfg, "is_cloud", False)
+        or getattr(cfg, "is_openai_compatible", False)
+    )
+
+
 class MemoryEngine:
     """Main orchestrator for the SuperLocalMemory V3 memory system.
 
@@ -103,6 +132,11 @@ class MemoryEngine:
         self._consolidation_engine = None
         self._maintenance_scheduler = None
         self._hooks = HookRegistry()
+        # Workstream D (3.8.4): single-worker pool reused across store_fast() calls.
+        # Lazy-created on first warm-guard attempt; avoids per-call thread churn.
+        self._store_fast_embed_pool: object | None = None
+        # Lock guards the lazy-init to prevent TOCTOU race on concurrent first calls.
+        self._store_fast_embed_pool_lock = threading.Lock()
 
     # -- Public properties (Phase 2+ access) --------------------------------
 
@@ -525,9 +559,6 @@ class MemoryEngine:
         import re as _re
         import uuid as _uuid
         from datetime import datetime, timezone
-        from superlocalmemory.storage.models import (
-            AtomicFact, FactType, MemoryRecord,
-        )
         from superlocalmemory.core.engine_ingestion import content_passes_admission
         if not content_passes_admission(content):
             return []
@@ -568,15 +599,64 @@ class MemoryEngine:
                 r"\b([A-Z][a-z]+(?:\s[A-Z][a-z]+){0,3})\b", fact_text)}
             | {m.group(1) for m in _re.finditer(r"\b([A-Z]{2,})\b", fact_text)}
         )
-        # Queryable admission must never acquire the embedding worker lock.
-        # On a warm daemon that looked cheap, but on a clean Mode A install the
-        # background model load owns that lock for up to 180s and turned the
-        # receipt-first path into a hidden synchronous wait.  The canonical
-        # materializer below runs the complete pipeline and promotes this same
-        # fact with its embedding, Fisher parameters, entities and graph edges.
-        # Until then it is deliberately BM25/entity/date recallable.
+        # Workstream D (3.8.4) — warm-guard synchronous embed.
+        #
+        # Original contract (3.8.2): queryable admission NEVER acquires the
+        # embedding worker lock. On a clean Mode-A install the background model
+        # load owns that lock for up to 180s, turning the receipt-first path into
+        # a hidden synchronous wait. The canonical materializer promotes this
+        # fact with its full pipeline (embedding, Fisher, entities, graph edges).
+        #
+        # 3.8.4 extension: when the embedder is PROVABLY warm (_available is True)
+        # AND is a local embedder (not a remote cloud/OpenAI endpoint), compute the
+        # embedding synchronously with a hard 500ms cap. On timeout or any
+        # exception, fall through to emb=None — the materializer fills it async.
+        # This preserves the 3.8.2 invariant for cold start while eliminating the
+        # semantic-channel blind spot on warm daemons (the top UX complaint).
         emb = None
         fmean = fvar = None
+        _embedder_ref = self._embedder
+        if (
+            _embedder_ref is not None
+            and getattr(_embedder_ref, "_available", None) is True
+            and not _is_remote_embedder(_embedder_ref)
+        ):
+            import concurrent.futures as _cf
+            # Lazy-init the pool once per engine instance — avoids per-call
+            # thread churn and the associated resource leak from discard-on-exit.
+            # Double-checked locking guards against TOCTOU on concurrent first calls.
+            if self._store_fast_embed_pool is None:
+                with self._store_fast_embed_pool_lock:
+                    if self._store_fast_embed_pool is None:
+                        self._store_fast_embed_pool = _cf.ThreadPoolExecutor(
+                            max_workers=1,
+                            thread_name_prefix="slm-sg-embed",
+                        )
+            try:
+                _timeout_s = int(os.environ.get("SLM_STORE_FAST_EMBED_TIMEOUT_MS", 500)) / 1000.0
+            except (ValueError, TypeError):
+                _timeout_s = 0.5  # default 500 ms
+            try:
+                _future = self._store_fast_embed_pool.submit(_embedder_ref.embed, fact_text)
+                try:
+                    emb = _future.result(timeout=_timeout_s)
+                    if emb:
+                        fmean, fvar = _embedder_ref.compute_fisher_params(emb)
+                except _cf.TimeoutError:
+                    logger.debug(
+                        "store_fast: warm-guard embed timed out (>%.0fms) — deferring to materializer",
+                        _timeout_s * 1000,
+                    )
+                    emb = None
+                except Exception as _exc:
+                    logger.debug(
+                        "store_fast: warm-guard embed failed (%s) — deferring to materializer",
+                        _exc,
+                    )
+                    emb = None
+            except Exception as _exc:
+                logger.debug("store_fast: warm-guard pool submit failed (%s)", _exc)
+                emb = None
         fact = AtomicFact(
             fact_id=_uuid.uuid4().hex[:16], memory_id=record.memory_id,
             profile_id=self._profile_id, content=fact_text,
@@ -674,44 +754,12 @@ class MemoryEngine:
                 include_shared=include_shared,
                 window=window,
             )
-        except Exception as exc:
-            from superlocalmemory.infra.local_diagnostics import record_operation
-
-            record_operation("recall", client=agent_id, error=exc)
+        except Exception:
+            # Diagnostics are intentionally not recorded here.  A recall is a
+            # read command; diagnostics, outcomes, and implicit feedback must
+            # be submitted through explicit write commands.
             raise
 
-        from superlocalmemory.infra.local_diagnostics import record_recall
-
-        record_recall(self._db, response, client=agent_id)
-
-        # S9-DASH-02: enqueue for pending_outcomes. Non-blocking; errors
-        # swallowed because signal capture is never load-bearing on
-        # recall correctness (LLD-02 §4.9, LLD-08 §4.1).
-        if session_id:
-            try:
-                fact_ids = tuple(
-                    getattr(r.fact, "fact_id", "") or ""
-                    for r in getattr(response, "results", [])
-                    if getattr(r, "fact", None) is not None
-                )
-                fact_ids = tuple(f for f in fact_ids if f)
-                if fact_ids:
-                    enqueue_recall(RecallEvent(
-                        session_id=session_id,
-                        profile_id=pid,
-                        query=query,
-                        fact_ids=fact_ids,
-                        query_id=getattr(response, "query_id", "") or "",
-                    ))
-            except Exception as _outcome_exc:
-                # Engagement-signal enqueue is non-blocking; recall
-                # correctness does not depend on it. Log so the failure
-                # is visible instead of silently losing learning signals.
-                logger.warning(
-                    "outcome-queue enqueue failed (engagement signal lost): %s",
-                    _outcome_exc,
-                )
-
         return response
 
     # -- Session operations -------------------------------------------------
@@ -739,25 +787,103 @@ class MemoryEngine:
     # -- Lifecycle ----------------------------------------------------------
 
     def close(self) -> None:
-        if self._maintenance_scheduler is not None:
-            self._maintenance_scheduler.stop()
-        if self._retrieval_engine is not None:
+        """Release engine-owned resources without waiting for model workers.
+
+        Daemon shutdown must be a bounded operation.  In particular, a
+        ``store_fast`` embed submitted just before shutdown may be blocked in a
+        model runtime forever; waiting for its executor here used to make the
+        service manager SIGKILL the daemon and leave its children behind.
+        References are cleared before invoking each cleanup hook, so a second
+        close is safe even when one optional cleanup hook fails.
+        """
+        scheduler = getattr(self, "_maintenance_scheduler", None)
+        self._maintenance_scheduler = None
+        if scheduler is not None:
             try:
-                self._retrieval_engine.close()
+                scheduler.stop()
             except Exception:
-                pass
-        if self._db is not None:
+                logger.warning("engine cleanup: maintenance scheduler stop failed", exc_info=True)
+
+        embed_pool = getattr(self, "_store_fast_embed_pool", None)
+        embed_pool_lock = getattr(self, "_store_fast_embed_pool_lock", None)
+        if embed_pool_lock is not None:
+            with embed_pool_lock:
+                embed_pool, self._store_fast_embed_pool = self._store_fast_embed_pool, None
+        else:
+            self._store_fast_embed_pool = None
+        if embed_pool is not None:
             try:
-                from superlocalmemory.core.recall_pipeline import (
-                    release_recall_resources,
+                embed_pool.shutdown(wait=False, cancel_futures=True)
+            except Exception:
+                logger.warning(
+                    "engine cleanup: store-fast embed pool shutdown failed",
+                    exc_info=True,
                 )
-                release_recall_resources(self._db)
+
+        retrieval = getattr(self, "_retrieval_engine", None)
+        self._retrieval_engine = None
+        if retrieval is not None:
+            reranker = getattr(retrieval, "_reranker", None)
+            try:
+                if reranker is not None:
+                    shutdown = getattr(reranker, "shutdown", None)
+                    if callable(shutdown):
+                        shutdown(timeout=1.0)
+                    else:
+                        unload = getattr(reranker, "unload", None)
+                        if callable(unload):
+                            unload()
             except Exception:
-                pass
+                logger.warning("engine cleanup: reranker shutdown failed", exc_info=True)
             try:
-                self._db.close()
+                retrieval.close(wait=False)
+            except TypeError:
+                # Compatibility for plugins with the legacy no-argument
+                # close hook.  Built-in RetrievalEngine accepts ``wait``.
+                try:
+                    retrieval.close()
+                except Exception:
+                    logger.warning(
+                        "engine cleanup: legacy retrieval shutdown failed",
+                        exc_info=True,
+                    )
             except Exception:
-                pass
+                logger.warning("engine cleanup: retrieval executor shutdown failed", exc_info=True)
+
+        embedder = getattr(self, "_embedder", None)
+        self._embedder = None
+        if embedder is not None:
+            try:
+                shutdown = getattr(embedder, "shutdown", None)
+                if callable(shutdown):
+                    shutdown(timeout=1.0)
+                else:
+                    unload = getattr(embedder, "unload", None)
+                    if not callable(unload):
+                        unload = None
+                if not callable(shutdown) and callable(unload):
+                    try:
+                        unload(timeout=1.0)
+                    except TypeError:
+                        # Ollama and third-party embedders may still expose
+                        # the legacy no-argument unload hook.
+                        unload()
+            except Exception:
+                logger.warning("engine cleanup: embedder unload failed", exc_info=True)
+
+        db = getattr(self, "_db", None)
+        self._db = None
+        if db is not None:
+            try:
+                from superlocalmemory.core.recall_pipeline import release_recall_resources
+
+                release_recall_resources(db)
+            except Exception:
+                logger.warning("engine cleanup: recall resources release failed", exc_info=True)
+            try:
+                db.close()
+            except Exception:
+                logger.warning("engine cleanup: database close failed", exc_info=True)
         self._initialized = False
 
     @property

package/src/superlocalmemory/core/engine_ingestion.py

@@ -14,7 +14,7 @@ import hashlib
 import logging
 import os
 import uuid
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
 
 from superlocalmemory.core.ingestion_command import (
     IngestionCommand,
@@ -26,7 +26,7 @@ from superlocalmemory.core.ingestion_command import (
 
 if TYPE_CHECKING:
     from superlocalmemory.core.engine import MemoryEngine
-    from superlocalmemory.storage.models import AtomicFact
+    from superlocalmemory.storage.models import AtomicFact, MemoryRecord
 
 
 logger = logging.getLogger(__name__)
@@ -36,6 +36,22 @@ _PREBUILT_FACT_KEY = "_slm_prebuilt_fact_v1"
 _DERIVATION_VERSION = "v3.7-ingestion-1"
 
 
+class _ImmediateAdmissionDatabase(Protocol):
+    """The deliberately tiny persistence surface used by receipt admission.
+
+    The coordinator will bind this to its sole writer connection.  Keeping the
+    seam this narrow prevents the immediate path from accidentally acquiring a
+    model, hook, graph, or secondary-store dependency while it owns SQLite's
+    write transaction.
+    """
+
+    def store_memory(self, record: "MemoryRecord") -> str: ...
+
+    def store_fact(self, fact: "AtomicFact") -> str: ...
+
+    def execute(self, sql: str, params: tuple = ()) -> list: ...
+
+
 def _pii_redaction_enabled(engine: "MemoryEngine") -> bool:
     """C4: opt-in PII redaction on ingest.
 
@@ -106,6 +122,124 @@ def _prebuilt_fact_from_payload(payload: dict):
     return AtomicFact(**values)
 
 
+def build_immediate_admission_handler(
+    db: _ImmediateAdmissionDatabase,
+    *,
+    profile_id: str,
+    max_verbatim_chars: int = 24_000,
+    max_ingest_bytes: int = 1_048_576,
+):
+    """Build the deterministic queryable projection for one engine profile.
+
+    ``IngestionCommand`` wraps this callback with operation creation and the
+    RAW -> QUERYABLE receipt transition in one transaction.  This function is
+    intentionally restricted to constructing a raw ``MemoryRecord`` and one
+    embedding-free ``AtomicFact`` then persisting those two records.  All
+    authorization hooks, embedding, extraction, FTS-adjacent enrichment,
+    graph, provenance, and external index work run only after this receipt has
+    committed through the durable materializer.
+    """
+    def write_queryable(request: IngestionRequest, operation_id: str) -> list[str]:
+        if request.profile_id != profile_id:
+            raise ValueError("ingestion request profile does not match engine")
+        if not request.trusted_actor_id:
+            raise ValueError("trusted actor identity is required")
+        if not content_passes_admission(request.content):
+            return []
+
+        import re
+        from datetime import UTC, datetime
+
+        from superlocalmemory.core.ingest_gate import apply_ingest_gate
+        from superlocalmemory.storage.models import AtomicFact, FactType, MemoryRecord
+
+        metadata = dict(request.metadata)
+        metadata["ingestion_operation_id"] = operation_id
+        if request.session_id:
+            metadata.setdefault("session_id", request.session_id)
+        gate = apply_ingest_gate(
+            request.content,
+            max_verbatim_chars=max_verbatim_chars,
+            max_ingest_bytes=max_ingest_bytes,
+        )
+        if gate.rejected:
+            return []
+        fact_content = gate.fact_content
+        now = datetime.now(UTC).isoformat()
+        observation_date = request.session_date or now[:10]
+
+        prebuilt_payload = metadata.get(_PREBUILT_FACT_KEY)
+        if isinstance(prebuilt_payload, dict):
+            fact = _prebuilt_fact_from_payload(prebuilt_payload)
+            fact.profile_id = request.profile_id
+            fact.scope = request.scope
+            fact.shared_with = list(request.shared_with) or None
+            fact.session_id = request.session_id or fact.session_id
+            if request.session_date:
+                fact.observation_date = request.session_date
+        else:
+            entities = sorted(
+                {match.group(1) for match in re.finditer(
+                    r"\b([A-Z][a-z]+(?:\s[A-Z][a-z]+){0,3})\b", fact_content,
+                )}
+                | {match.group(1) for match in re.finditer(
+                    r"\b([A-Z]{2,})\b", fact_content,
+                )}
+            )
+            fact = AtomicFact(
+                fact_id=uuid.uuid4().hex[:16],
+                profile_id=request.profile_id,
+                scope=request.scope,
+                shared_with=list(request.shared_with) or None,
+                content=fact_content,
+                fact_type=FactType.EPISODIC,
+                entities=entities,
+                observation_date=observation_date,
+                session_id=request.session_id,
+                confidence=0.7,
+                importance=0.5,
+                created_at=now,
+            )
+
+        # A receipt is queryable through FTS immediately, but model-derived
+        # values cannot participate in its write transaction.  The materializer
+        # owns their later, retryable population.
+        fact.embedding = None
+        fact.fisher_mean = None
+        fact.fisher_variance = None
+        memory_id = fact.memory_id or uuid.uuid4().hex
+        existing = db.execute(
+            "SELECT profile_id FROM memories WHERE memory_id = ?",
+            (memory_id,),
+        )
+        if existing:
+            existing_profile = str(existing[0]["profile_id"])
+            if existing_profile != request.profile_id:
+                raise ValueError("prebuilt fact memory belongs to a different profile")
+            # Reuse the existing source row. DatabaseManager.store_memory uses
+            # INSERT OR REPLACE; calling it here would delete every child fact
+            # through the memory_id foreign-key cascade before recreating the
+            # parent.
+            fact.memory_id = memory_id
+        else:
+            record = MemoryRecord(
+                memory_id=memory_id,
+                profile_id=request.profile_id,
+                content=request.content,
+                session_id=request.session_id,
+                session_date=observation_date,
+                speaker=request.speaker,
+                role=request.role,
+                metadata=metadata,
+                scope=request.scope,
+                shared_with=list(request.shared_with) or None,
+            )
+            fact.memory_id = db.store_memory(record)
+        return [db.store_fact(fact)]
+
+    return write_queryable
+
+
 def local_trusted_actor_id(actor_kind: str) -> str:
     """Derive a stable local actor from the private install capability."""
     from superlocalmemory.core.security_primitives import ensure_install_token
@@ -283,71 +417,22 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
     engine._require_full("canonical_ingestion")
     engine._ensure_init()
     repository = IngestionOperationRepository(engine._db)
+    store_config = getattr(engine._config, "store", None)
+    write_queryable = build_immediate_admission_handler(
+        engine._db,
+        profile_id=engine._profile_id,
+        max_verbatim_chars=getattr(store_config, "max_verbatim_chars", 24_000),
+        max_ingest_bytes=getattr(store_config, "max_ingest_bytes", 1_048_576),
+    )
 
-    def write_queryable(request: IngestionRequest, operation_id: str) -> list[str]:
-        if request.profile_id != engine._profile_id:
-            raise ValueError("ingestion request profile does not match engine")
-        if not request.trusted_actor_id:
-            raise ValueError("trusted actor identity is required")
-        hook_context = {
+    def validate_admission(request: IngestionRequest) -> None:
+        """Apply trust policy before the journal or canonical transaction."""
+        engine._hooks.run_pre("store", {
             "operation": "store",
-            "agent_id": request.trusted_actor_id or "unknown",
+            "agent_id": request.trusted_actor_id,
             "profile_id": request.profile_id,
             "content_preview": request.content[:100],
-            "ingestion_operation_id": operation_id,
-        }
-        # Authorization and trust policy must run before raw evidence reaches
-        # durable storage.  Materialization reuses this authorization decision.
-        engine._hooks.run_pre("store", hook_context)
-        metadata = dict(request.metadata)
-        metadata["ingestion_operation_id"] = operation_id
-        if request.session_id:
-            metadata.setdefault("session_id", request.session_id)
-        prebuilt_payload = metadata.get(_PREBUILT_FACT_KEY)
-        if isinstance(prebuilt_payload, dict):
-            from superlocalmemory.storage.models import MemoryRecord
-
-            fact = _prebuilt_fact_from_payload(prebuilt_payload)
-            fact.profile_id = request.profile_id
-            fact.scope = request.scope
-            fact.shared_with = list(request.shared_with) or None
-            fact.session_id = request.session_id or fact.session_id
-            if request.session_date:
-                fact.observation_date = request.session_date
-            memory_id = fact.memory_id
-            memory_rows = (
-                engine._db.execute(
-                    "SELECT memory_id FROM memories WHERE memory_id=? AND profile_id=?",
-                    (memory_id, request.profile_id),
-                )
-                if memory_id else []
-            )
-            if not memory_rows:
-                record = MemoryRecord(
-                    memory_id=memory_id or uuid.uuid4().hex,
-                    profile_id=request.profile_id,
-                    content=request.content,
-                    session_id=request.session_id,
-                    session_date=request.session_date,
-                    metadata=metadata,
-                    scope=request.scope,
-                    shared_with=list(request.shared_with) or None,
-                )
-                engine._db.store_memory(record)
-                memory_id = record.memory_id
-            fact.memory_id = memory_id
-            engine._db.store_fact(fact)
-            return [fact.fact_id]
-        return engine.store_fast(
-            request.content,
-            metadata=metadata,
-            scope=request.scope,
-            shared_with=list(request.shared_with) or None,
-            session_date=request.session_date or None,
-            speaker=request.speaker,
-            role=request.role,
-            index_external=False,
-        )
+        })
 
     def resume_checkpoint(operation: IngestionOperation) -> MaterializationResult:
         """Repair only stages whose writes have an idempotent natural key."""
@@ -816,12 +901,14 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
         repository,
         write_queryable=write_queryable,
         materialize=materialize,
+        validate_admission=validate_admission,
         project=project,
         derivation_version=_DERIVATION_VERSION,
     )
 
 
 __all__ = [
+    "build_immediate_admission_handler",
     "build_engine_ingestion_command",
     "canonical_store",
     "canonical_store_fn",