superlocalmemory 3.8.5 → 3.8.7

package/src/superlocalmemory/core/backend_orchestrator.py

@@ -438,7 +438,13 @@ class BackendOrchestrator:
             self._cozo = CozoDBGraphBackend(str(cozo_path / "graph"))
             self._update_status("cozo", "not_initialized")
             logger.info("CozoDB initialized at %s", cozo_path)
-        except Exception as exc:
+        except BaseException as exc:
+            # PyO3 exposes Rust panics as PanicException(BaseException), not
+            # Exception. An incompatible optional projection must never abort
+            # daemon startup or hide canonical SQLite memory. Re-raise genuine
+            # process-control exceptions; preserve the graph and degrade Cozo.
+            if not isinstance(exc, Exception) and type(exc).__name__ != "PanicException":
+                raise
             logger.warning("CozoDB init failed: %s", exc)
             self._cozo = None
 

package/src/superlocalmemory/core/component_registry.py

@@ -265,8 +265,10 @@ def probe_reranker_model(config: Any = None) -> Component:
         auto_fixable=enabled,
         fix_cmd="slm doctor --fix",
     )
-    if not enabled and comp.status == STATUS_MISSING:
-        # Not enabled → absent is expected, not a problem.
+    if not enabled:
+        # A cached reranker is still inactive when the operator disabled the
+        # channel. The dashboard must describe configured runtime state, not
+        # machine-specific HuggingFace-cache state.
         return replace(comp, status=STATUS_OK,
                        detail="disabled (retrieval.use_cross_encoder=false)",
                        fix_cmd="", auto_fixable=False)

package/src/superlocalmemory/core/embeddings.py

@@ -250,11 +250,38 @@ class EmbeddingService:
     def dimension(self) -> int:
         return self._config.dimension
 
-    def unload(self) -> None:
-        """Kill the worker subprocess to free all memory."""
-        with self._lock:
+    def unload(self, timeout: float = 1.0) -> bool:
+        """Release the worker without blocking daemon shutdown on an embed call.
+
+        An in-flight request owns ``_lock`` while it waits for the worker's
+        response.  Shutdown must not wait behind a wedged response: callers
+        can continue teardown and the worker process will be handled by the
+        process supervisor if necessary.
+        """
+        if not self._lock.acquire(timeout=max(0.0, timeout)):
+            logger.warning("EmbeddingService: unload skipped; embed worker is busy")
+            return False
+        try:
             self._kill_worker()
             logger.info("EmbeddingService: worker killed (idle timeout)")
+            return True
+        finally:
+            self._lock.release()
+
+    def shutdown(self, timeout: float = 1.0) -> None:
+        """Force bounded process teardown even when an embed call owns the lock.
+
+        Shutdown is stronger than the idle-time ``unload`` operation.  Once
+        the engine is closing, no new request may use this service, so it is
+        safe to detach and terminate a wedged child without waiting behind the
+        request lock.
+        """
+        acquired = self._lock.acquire(timeout=max(0.0, timeout))
+        try:
+            self._kill_worker(timeout=min(max(0.0, timeout), 1.0))
+        finally:
+            if acquired:
+                self._lock.release()
 
     # ------------------------------------------------------------------
     # Public API
@@ -596,7 +623,7 @@ class EmbeddingService:
             self._available = False
             self._worker_proc = None
 
-    def _kill_worker(self) -> None:
+    def _kill_worker(self, timeout: float = 3.0) -> None:
         """Terminate the worker and close every owned pipe exactly once."""
         if self._idle_timer is not None:
             self._idle_timer.cancel()
@@ -610,7 +637,7 @@ class EmbeddingService:
             try:
                 proc.stdin.write('{"cmd":"quit"}\n')
                 proc.stdin.flush()
-                proc.wait(timeout=3)
+                proc.wait(timeout=max(0.0, timeout))
             except Exception:
                 try:
                     returncode = proc.poll()
@@ -621,7 +648,7 @@ class EmbeddingService:
                 if returncode is None or not isinstance(returncode, int):
                     try:
                         proc.kill()
-                        proc.wait(timeout=3)
+                        proc.wait(timeout=max(0.0, timeout))
                     except Exception:
                         pass
             finally:

package/src/superlocalmemory/core/engine.py

@@ -26,7 +26,6 @@ from typing import Any
 from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT, SLMConfig
 from superlocalmemory.core.engine_capabilities import Capabilities, CapabilityError
 from superlocalmemory.core.modes import get_capabilities
-from superlocalmemory.learning.outcome_queue import RecallEvent, enqueue_recall
 from superlocalmemory.storage.models import (
     AtomicFact, FactType, MemoryRecord, Mode, RecallResponse,
 )
@@ -755,44 +754,12 @@ class MemoryEngine:
                 include_shared=include_shared,
                 window=window,
             )
-        except Exception as exc:
-            from superlocalmemory.infra.local_diagnostics import record_operation
-
-            record_operation("recall", client=agent_id, error=exc)
+        except Exception:
+            # Diagnostics are intentionally not recorded here.  A recall is a
+            # read command; diagnostics, outcomes, and implicit feedback must
+            # be submitted through explicit write commands.
             raise
 
-        from superlocalmemory.infra.local_diagnostics import record_recall
-
-        record_recall(self._db, response, client=agent_id)
-
-        # S9-DASH-02: enqueue for pending_outcomes. Non-blocking; errors
-        # swallowed because signal capture is never load-bearing on
-        # recall correctness (LLD-02 §4.9, LLD-08 §4.1).
-        if session_id:
-            try:
-                fact_ids = tuple(
-                    getattr(r.fact, "fact_id", "") or ""
-                    for r in getattr(response, "results", [])
-                    if getattr(r, "fact", None) is not None
-                )
-                fact_ids = tuple(f for f in fact_ids if f)
-                if fact_ids:
-                    enqueue_recall(RecallEvent(
-                        session_id=session_id,
-                        profile_id=pid,
-                        query=query,
-                        fact_ids=fact_ids,
-                        query_id=getattr(response, "query_id", "") or "",
-                    ))
-            except Exception as _outcome_exc:
-                # Engagement-signal enqueue is non-blocking; recall
-                # correctness does not depend on it. Log so the failure
-                # is visible instead of silently losing learning signals.
-                logger.warning(
-                    "outcome-queue enqueue failed (engagement signal lost): %s",
-                    _outcome_exc,
-                )
-
         return response
 
     # -- Session operations -------------------------------------------------
@@ -820,25 +787,103 @@ class MemoryEngine:
     # -- Lifecycle ----------------------------------------------------------
 
     def close(self) -> None:
-        if self._maintenance_scheduler is not None:
-            self._maintenance_scheduler.stop()
-        if self._retrieval_engine is not None:
+        """Release engine-owned resources without waiting for model workers.
+
+        Daemon shutdown must be a bounded operation.  In particular, a
+        ``store_fast`` embed submitted just before shutdown may be blocked in a
+        model runtime forever; waiting for its executor here used to make the
+        service manager SIGKILL the daemon and leave its children behind.
+        References are cleared before invoking each cleanup hook, so a second
+        close is safe even when one optional cleanup hook fails.
+        """
+        scheduler = getattr(self, "_maintenance_scheduler", None)
+        self._maintenance_scheduler = None
+        if scheduler is not None:
             try:
-                self._retrieval_engine.close()
+                scheduler.stop()
             except Exception:
-                pass
-        if self._db is not None:
+                logger.warning("engine cleanup: maintenance scheduler stop failed", exc_info=True)
+
+        embed_pool = getattr(self, "_store_fast_embed_pool", None)
+        embed_pool_lock = getattr(self, "_store_fast_embed_pool_lock", None)
+        if embed_pool_lock is not None:
+            with embed_pool_lock:
+                embed_pool, self._store_fast_embed_pool = self._store_fast_embed_pool, None
+        else:
+            self._store_fast_embed_pool = None
+        if embed_pool is not None:
             try:
-                from superlocalmemory.core.recall_pipeline import (
-                    release_recall_resources,
+                embed_pool.shutdown(wait=False, cancel_futures=True)
+            except Exception:
+                logger.warning(
+                    "engine cleanup: store-fast embed pool shutdown failed",
+                    exc_info=True,
                 )
-                release_recall_resources(self._db)
+
+        retrieval = getattr(self, "_retrieval_engine", None)
+        self._retrieval_engine = None
+        if retrieval is not None:
+            reranker = getattr(retrieval, "_reranker", None)
+            try:
+                if reranker is not None:
+                    shutdown = getattr(reranker, "shutdown", None)
+                    if callable(shutdown):
+                        shutdown(timeout=1.0)
+                    else:
+                        unload = getattr(reranker, "unload", None)
+                        if callable(unload):
+                            unload()
             except Exception:
-                pass
+                logger.warning("engine cleanup: reranker shutdown failed", exc_info=True)
             try:
-                self._db.close()
+                retrieval.close(wait=False)
+            except TypeError:
+                # Compatibility for plugins with the legacy no-argument
+                # close hook.  Built-in RetrievalEngine accepts ``wait``.
+                try:
+                    retrieval.close()
+                except Exception:
+                    logger.warning(
+                        "engine cleanup: legacy retrieval shutdown failed",
+                        exc_info=True,
+                    )
             except Exception:
-                pass
+                logger.warning("engine cleanup: retrieval executor shutdown failed", exc_info=True)
+
+        embedder = getattr(self, "_embedder", None)
+        self._embedder = None
+        if embedder is not None:
+            try:
+                shutdown = getattr(embedder, "shutdown", None)
+                if callable(shutdown):
+                    shutdown(timeout=1.0)
+                else:
+                    unload = getattr(embedder, "unload", None)
+                    if not callable(unload):
+                        unload = None
+                if not callable(shutdown) and callable(unload):
+                    try:
+                        unload(timeout=1.0)
+                    except TypeError:
+                        # Ollama and third-party embedders may still expose
+                        # the legacy no-argument unload hook.
+                        unload()
+            except Exception:
+                logger.warning("engine cleanup: embedder unload failed", exc_info=True)
+
+        db = getattr(self, "_db", None)
+        self._db = None
+        if db is not None:
+            try:
+                from superlocalmemory.core.recall_pipeline import release_recall_resources
+
+                release_recall_resources(db)
+            except Exception:
+                logger.warning("engine cleanup: recall resources release failed", exc_info=True)
+            try:
+                db.close()
+            except Exception:
+                logger.warning("engine cleanup: database close failed", exc_info=True)
         self._initialized = False
 
     @property

package/src/superlocalmemory/core/engine_ingestion.py

@@ -14,7 +14,7 @@ import hashlib
 import logging
 import os
 import uuid
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
 
 from superlocalmemory.core.ingestion_command import (
     IngestionCommand,
@@ -26,7 +26,7 @@ from superlocalmemory.core.ingestion_command import (
 
 if TYPE_CHECKING:
     from superlocalmemory.core.engine import MemoryEngine
-    from superlocalmemory.storage.models import AtomicFact
+    from superlocalmemory.storage.models import AtomicFact, MemoryRecord
 
 
 logger = logging.getLogger(__name__)
@@ -36,6 +36,22 @@ _PREBUILT_FACT_KEY = "_slm_prebuilt_fact_v1"
 _DERIVATION_VERSION = "v3.7-ingestion-1"
 
 
+class _ImmediateAdmissionDatabase(Protocol):
+    """The deliberately tiny persistence surface used by receipt admission.
+
+    The coordinator will bind this to its sole writer connection.  Keeping the
+    seam this narrow prevents the immediate path from accidentally acquiring a
+    model, hook, graph, or secondary-store dependency while it owns SQLite's
+    write transaction.
+    """
+
+    def store_memory(self, record: "MemoryRecord") -> str: ...
+
+    def store_fact(self, fact: "AtomicFact") -> str: ...
+
+    def execute(self, sql: str, params: tuple = ()) -> list: ...
+
+
 def _pii_redaction_enabled(engine: "MemoryEngine") -> bool:
     """C4: opt-in PII redaction on ingest.
 
@@ -106,6 +122,124 @@ def _prebuilt_fact_from_payload(payload: dict):
     return AtomicFact(**values)
 
 
+def build_immediate_admission_handler(
+    db: _ImmediateAdmissionDatabase,
+    *,
+    profile_id: str,
+    max_verbatim_chars: int = 24_000,
+    max_ingest_bytes: int = 1_048_576,
+):
+    """Build the deterministic queryable projection for one engine profile.
+
+    ``IngestionCommand`` wraps this callback with operation creation and the
+    RAW -> QUERYABLE receipt transition in one transaction.  This function is
+    intentionally restricted to constructing a raw ``MemoryRecord`` and one
+    embedding-free ``AtomicFact`` then persisting those two records.  All
+    authorization hooks, embedding, extraction, FTS-adjacent enrichment,
+    graph, provenance, and external index work run only after this receipt has
+    committed through the durable materializer.
+    """
+    def write_queryable(request: IngestionRequest, operation_id: str) -> list[str]:
+        if request.profile_id != profile_id:
+            raise ValueError("ingestion request profile does not match engine")
+        if not request.trusted_actor_id:
+            raise ValueError("trusted actor identity is required")
+        if not content_passes_admission(request.content):
+            return []
+
+        import re
+        from datetime import UTC, datetime
+
+        from superlocalmemory.core.ingest_gate import apply_ingest_gate
+        from superlocalmemory.storage.models import AtomicFact, FactType, MemoryRecord
+
+        metadata = dict(request.metadata)
+        metadata["ingestion_operation_id"] = operation_id
+        if request.session_id:
+            metadata.setdefault("session_id", request.session_id)
+        gate = apply_ingest_gate(
+            request.content,
+            max_verbatim_chars=max_verbatim_chars,
+            max_ingest_bytes=max_ingest_bytes,
+        )
+        if gate.rejected:
+            return []
+        fact_content = gate.fact_content
+        now = datetime.now(UTC).isoformat()
+        observation_date = request.session_date or now[:10]
+
+        prebuilt_payload = metadata.get(_PREBUILT_FACT_KEY)
+        if isinstance(prebuilt_payload, dict):
+            fact = _prebuilt_fact_from_payload(prebuilt_payload)
+            fact.profile_id = request.profile_id
+            fact.scope = request.scope
+            fact.shared_with = list(request.shared_with) or None
+            fact.session_id = request.session_id or fact.session_id
+            if request.session_date:
+                fact.observation_date = request.session_date
+        else:
+            entities = sorted(
+                {match.group(1) for match in re.finditer(
+                    r"\b([A-Z][a-z]+(?:\s[A-Z][a-z]+){0,3})\b", fact_content,
+                )}
+                | {match.group(1) for match in re.finditer(
+                    r"\b([A-Z]{2,})\b", fact_content,
+                )}
+            )
+            fact = AtomicFact(
+                fact_id=uuid.uuid4().hex[:16],
+                profile_id=request.profile_id,
+                scope=request.scope,
+                shared_with=list(request.shared_with) or None,
+                content=fact_content,
+                fact_type=FactType.EPISODIC,
+                entities=entities,
+                observation_date=observation_date,
+                session_id=request.session_id,
+                confidence=0.7,
+                importance=0.5,
+                created_at=now,
+            )
+
+        # A receipt is queryable through FTS immediately, but model-derived
+        # values cannot participate in its write transaction.  The materializer
+        # owns their later, retryable population.
+        fact.embedding = None
+        fact.fisher_mean = None
+        fact.fisher_variance = None
+        memory_id = fact.memory_id or uuid.uuid4().hex
+        existing = db.execute(
+            "SELECT profile_id FROM memories WHERE memory_id = ?",
+            (memory_id,),
+        )
+        if existing:
+            existing_profile = str(existing[0]["profile_id"])
+            if existing_profile != request.profile_id:
+                raise ValueError("prebuilt fact memory belongs to a different profile")
+            # Reuse the existing source row. DatabaseManager.store_memory uses
+            # INSERT OR REPLACE; calling it here would delete every child fact
+            # through the memory_id foreign-key cascade before recreating the
+            # parent.
+            fact.memory_id = memory_id
+        else:
+            record = MemoryRecord(
+                memory_id=memory_id,
+                profile_id=request.profile_id,
+                content=request.content,
+                session_id=request.session_id,
+                session_date=observation_date,
+                speaker=request.speaker,
+                role=request.role,
+                metadata=metadata,
+                scope=request.scope,
+                shared_with=list(request.shared_with) or None,
+            )
+            fact.memory_id = db.store_memory(record)
+        return [db.store_fact(fact)]
+
+    return write_queryable
+
+
 def local_trusted_actor_id(actor_kind: str) -> str:
     """Derive a stable local actor from the private install capability."""
     from superlocalmemory.core.security_primitives import ensure_install_token
@@ -283,71 +417,22 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
     engine._require_full("canonical_ingestion")
     engine._ensure_init()
     repository = IngestionOperationRepository(engine._db)
+    store_config = getattr(engine._config, "store", None)
+    write_queryable = build_immediate_admission_handler(
+        engine._db,
+        profile_id=engine._profile_id,
+        max_verbatim_chars=getattr(store_config, "max_verbatim_chars", 24_000),
+        max_ingest_bytes=getattr(store_config, "max_ingest_bytes", 1_048_576),
+    )
 
-    def write_queryable(request: IngestionRequest, operation_id: str) -> list[str]:
-        if request.profile_id != engine._profile_id:
-            raise ValueError("ingestion request profile does not match engine")
-        if not request.trusted_actor_id:
-            raise ValueError("trusted actor identity is required")
-        hook_context = {
+    def validate_admission(request: IngestionRequest) -> None:
+        """Apply trust policy before the journal or canonical transaction."""
+        engine._hooks.run_pre("store", {
             "operation": "store",
-            "agent_id": request.trusted_actor_id or "unknown",
+            "agent_id": request.trusted_actor_id,
             "profile_id": request.profile_id,
             "content_preview": request.content[:100],
-            "ingestion_operation_id": operation_id,
-        }
-        # Authorization and trust policy must run before raw evidence reaches
-        # durable storage.  Materialization reuses this authorization decision.
-        engine._hooks.run_pre("store", hook_context)
-        metadata = dict(request.metadata)
-        metadata["ingestion_operation_id"] = operation_id
-        if request.session_id:
-            metadata.setdefault("session_id", request.session_id)
-        prebuilt_payload = metadata.get(_PREBUILT_FACT_KEY)
-        if isinstance(prebuilt_payload, dict):
-            from superlocalmemory.storage.models import MemoryRecord
-
-            fact = _prebuilt_fact_from_payload(prebuilt_payload)
-            fact.profile_id = request.profile_id
-            fact.scope = request.scope
-            fact.shared_with = list(request.shared_with) or None
-            fact.session_id = request.session_id or fact.session_id
-            if request.session_date:
-                fact.observation_date = request.session_date
-            memory_id = fact.memory_id
-            memory_rows = (
-                engine._db.execute(
-                    "SELECT memory_id FROM memories WHERE memory_id=? AND profile_id=?",
-                    (memory_id, request.profile_id),
-                )
-                if memory_id else []
-            )
-            if not memory_rows:
-                record = MemoryRecord(
-                    memory_id=memory_id or uuid.uuid4().hex,
-                    profile_id=request.profile_id,
-                    content=request.content,
-                    session_id=request.session_id,
-                    session_date=request.session_date,
-                    metadata=metadata,
-                    scope=request.scope,
-                    shared_with=list(request.shared_with) or None,
-                )
-                engine._db.store_memory(record)
-                memory_id = record.memory_id
-            fact.memory_id = memory_id
-            engine._db.store_fact(fact)
-            return [fact.fact_id]
-        return engine.store_fast(
-            request.content,
-            metadata=metadata,
-            scope=request.scope,
-            shared_with=list(request.shared_with) or None,
-            session_date=request.session_date or None,
-            speaker=request.speaker,
-            role=request.role,
-            index_external=False,
-        )
+        })
 
     def resume_checkpoint(operation: IngestionOperation) -> MaterializationResult:
         """Repair only stages whose writes have an idempotent natural key."""
@@ -816,12 +901,14 @@ def build_engine_ingestion_command(engine: MemoryEngine) -> IngestionCommand:
         repository,
         write_queryable=write_queryable,
         materialize=materialize,
+        validate_admission=validate_admission,
         project=project,
         derivation_version=_DERIVATION_VERSION,
     )
 
 
 __all__ = [
+    "build_immediate_admission_handler",
     "build_engine_ingestion_command",
     "canonical_store",
     "canonical_store_fn",