superlocalmemory 3.6.22 → 3.7.0

package/src/superlocalmemory/core/engine.py

@@ -18,6 +18,7 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import logging
+from pathlib import Path
 from typing import Any
 
 from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT, SLMConfig
@@ -32,6 +33,19 @@ logger = logging.getLogger(__name__)
 from superlocalmemory.core.hooks import HookRegistry
 
 
+def _verify_ingestion_schema(memory_db: Path) -> bool:
+    """Verify M018 using an explicitly owned SQLite connection."""
+    import sqlite3
+
+    from superlocalmemory.storage.migrations import M018_ingestion_operations
+
+    connection = sqlite3.connect(str(memory_db))
+    try:
+        return bool(M018_ingestion_operations.verify(connection))
+    finally:
+        connection.close()
+
+
 class MemoryEngine:
     """Main orchestrator for the SuperLocalMemory V3 memory system.
 
@@ -170,7 +184,9 @@ class MemoryEngine:
         # Previously only the daemon lifespan ran apply_deferred, so an existing
         # pre-3.6.15 database used WITHOUT the daemon hit
         # "table memories has no column named scope" on the first scoped write.
-        # Idempotent (skips applied migrations) + non-fatal; mirrors the daemon.
+        # Idempotent (skips applied migrations). M018 is now a hard runtime
+        # prerequisite: allowing initialization to continue without it would
+        # make every advertised write path fail after startup.
         try:
             from superlocalmemory.storage.migration_runner import (
                 apply_all, apply_deferred,
@@ -178,10 +194,22 @@ class MemoryEngine:
             _base = self._config.base_dir
             _learning_db = _base / "learning.db"
             _memory_db = self._db.db_path
+            # Forward migrations extend the historical learning schema. A
+            # fresh explicit data root has no learning.db yet, so bootstrap
+            # those base tables before applying M001/M002/M009.
+            from superlocalmemory.learning.database import LearningDatabase
+            LearningDatabase(_learning_db)
             apply_all(_learning_db, _memory_db)
             apply_deferred(_learning_db, _memory_db)
+            if not _verify_ingestion_schema(_memory_db):
+                raise RuntimeError(
+                    "required M018 canonical-ingestion schema is unavailable"
+                )
         except Exception as exc:
-            logger.warning("v3.6.15 deferred migration apply failed: %s", exc)
+            raise RuntimeError(
+                "MemoryEngine initialization stopped because the required "
+                f"ingestion migration failed: {exc}"
+            ) from exc
 
         # V3.4.7: Apply "Learning Brain" schema (tool_events, behavioral_assertions)
         try:
@@ -364,11 +392,30 @@ class MemoryEngine:
                         meta = {}
                 if not isinstance(meta, dict):
                     meta = {}
-                _scope = meta.get("scope") or "personal"
-                _shared = meta.get("shared_with")
-                self.store(
-                    item["content"], metadata=meta or None,
-                    scope=_scope, shared_with=_shared,
+                _scope = meta.pop("scope", None) or "personal"
+                _shared = meta.pop("shared_with", None)
+                _source_type = str(
+                    meta.pop("_slm_source_type", "legacy-pending")
+                )
+                _idempotency_key = str(
+                    meta.pop("_slm_idempotency_key", f"pending:{item['id']}")
+                )
+                if item.get("tags"):
+                    meta.setdefault("tags", item["tags"])
+                from superlocalmemory.core.engine_ingestion import (
+                    canonical_store,
+                    local_trusted_actor_id,
+                )
+                canonical_store(
+                    self,
+                    item["content"],
+                    source_type=_source_type,
+                    trusted_actor_id=local_trusted_actor_id("engine-startup"),
+                    metadata=meta or None,
+                    scope=_scope,
+                    shared_with=_shared,
+                    session_id=str(meta.get("session_id") or ""),
+                    idempotency_key=_idempotency_key,
                 )
                 mark_done(item["id"], base_dir)
             except Exception as exc:
@@ -397,54 +444,45 @@ class MemoryEngine:
         self._require_full("store")
         self._ensure_init()
 
-        from superlocalmemory.core.store_pipeline import run_store
-        return run_store(
-            content, self._profile_id,
-            session_id=session_id, session_date=session_date,
-            speaker=speaker, role=role, metadata=metadata,
-            scope=scope, shared_with=shared_with,
-            config=self._config, db=self._db,
-            embedder=self._embedder,
-            fact_extractor=self._fact_extractor,
-            entity_resolver=self._entity_resolver,
-            temporal_parser=self._temporal_parser,
-            type_router=self._type_router,
-            graph_builder=self._graph_builder,
-            consolidator=self._consolidator,
-            observation_builder=self._observation_builder,
-            scene_builder=self._scene_builder,
-            entropy_gate=self._entropy_gate,
-            ann_index=self._ann_index,
-            sheaf_checker=self._sheaf_checker,
-            retrieval_engine=self._retrieval_engine,
-            provenance=self._provenance,
-            hooks=self._hooks,
-            vector_store=self._vector_store,
-            context_generator=self._context_generator,
-            temporal_validator=self._temporal_validator,
-            auto_linker=self._auto_linker,
-            consolidation_engine=self._consolidation_engine,
+        from superlocalmemory.core.engine_ingestion import (
+            canonical_store,
+            local_trusted_actor_id,
+        )
+        return canonical_store(
+            self,
+            content,
+            source_type="python-api",
+            trusted_actor_id=local_trusted_actor_id("python-api"),
+            metadata=metadata,
+            scope=scope,
+            shared_with=shared_with,
+            session_id=session_id,
+            session_date=session_date,
+            speaker=speaker,
+            role=role,
+            require_complete=False,
         )
 
     def store_fact_direct(self, fact: AtomicFact) -> str:
-        """Store a pre-built fact with full enrichment."""
+        """Durably store a pre-built fact with full enrichment."""
         self._require_full("store_fact_direct")
         self._ensure_init()
 
-        from superlocalmemory.core.store_pipeline import run_store_fact_direct
-        return run_store_fact_direct(
-            fact, self._profile_id,
-            db=self._db, embedder=self._embedder,
-            entity_resolver=self._entity_resolver,
-            ann_index=self._ann_index,
-            graph_builder=self._graph_builder,
-            retrieval_engine=self._retrieval_engine,
-            vector_store=self._vector_store,
+        from superlocalmemory.core.engine_ingestion import (
+            canonical_store_fact,
+            local_trusted_actor_id,
+        )
+        return canonical_store_fact(
+            self,
+            fact,
+            trusted_actor_id=local_trusted_actor_id("python-api-prebuilt"),
         )
 
     def store_fast(
         self, content: str, metadata: dict[str, Any] | None = None,
         *, scope: str = "personal", shared_with: list[str] | None = None,
+        session_date: str | None = None, speaker: str = "", role: str = "user",
+        index_external: bool = True,
     ) -> list[str]:
         """v3.5.5 WRITE-THROUGH: synchronous verbatim insert for IMMEDIATE recall.
 
@@ -470,14 +508,9 @@ class MemoryEngine:
         from superlocalmemory.storage.models import (
             AtomicFact, FactType, MemoryRecord,
         )
-        if not content or not content.strip():
+        from superlocalmemory.core.engine_ingestion import content_passes_admission
+        if not content_passes_admission(content):
             return []
-        try:
-            from superlocalmemory.core.injection import is_low_quality
-            if is_low_quality(content):
-                return []
-        except Exception:
-            pass
         # v3.6.6 ingest gate: reject 1MB monsters + prompt-template pollution;
         # clamp the searchable FACT copy (head+tail) while the memories row keeps
         # the FULL original. Embedding/BM25 use the clamped copy so a 167KB paste
@@ -500,8 +533,10 @@ class MemoryEngine:
         now = datetime.now(timezone.utc).isoformat()
         record = MemoryRecord(
             profile_id=self._profile_id, content=content,
-            session_date=now[:10],
+            session_date=session_date or now[:10],
             session_id=(metadata or {}).get("session_id", ""),
+            speaker=speaker,
+            role=role,
             metadata=metadata or {},
             scope=scope, shared_with=shared_with,
         )
@@ -513,44 +548,42 @@ class MemoryEngine:
                 r"\b([A-Z][a-z]+(?:\s[A-Z][a-z]+){0,3})\b", fact_text)}
             | {m.group(1) for m in _re.finditer(r"\b([A-Z]{2,})\b", fact_text)}
         )
-        # v3.5.5: compute the embedding SYNCHRONOUSLY. A single warm embed is
-        # ~22ms (the 30-180s of full store() was LLM fact-extraction + graph,
-        # NOT embedding). With the embedding present, the semantic channel
-        # scores this fact correctly so it ranks properly IMMEDIATELY — not
-        # just keyword-findable but top-ranked. Graph/entity enrichment stays
-        # async. Embed failure → fact still inserted (keyword-recallable).
+        # Queryable admission must never acquire the embedding worker lock.
+        # On a warm daemon that looked cheap, but on a clean Mode A install the
+        # background model load owns that lock for up to 180s and turned the
+        # receipt-first path into a hidden synchronous wait.  The canonical
+        # materializer below runs the complete pipeline and promotes this same
+        # fact with its embedding, Fisher parameters, entities and graph edges.
+        # Until then it is deliberately BM25/entity/date recallable.
         emb = None
         fmean = fvar = None
-        try:
-            emb = self._embedder.embed(fact_text) if self._embedder else None
-            if emb:
-                fmean, fvar = self._embedder.compute_fisher_params(emb)
-        except Exception:
-            emb = None
         fact = AtomicFact(
             fact_id=_uuid.uuid4().hex[:16], memory_id=record.memory_id,
             profile_id=self._profile_id, content=fact_text,
             fact_type=FactType.EPISODIC, entities=ents,
-            observation_date=now[:10], confidence=0.7, importance=0.5,
+            observation_date=session_date or now[:10],
+            confidence=0.7, importance=0.5,
             embedding=emb, fisher_mean=fmean, fisher_variance=fvar,
             created_at=now,
             scope=scope, shared_with=shared_with,
         )
         self._db.store_fact(fact)  # FTS5 trigger → immediately BM25-recallable
         # Upsert to vector store so the semantic channel finds it now.
-        try:
-            vs = getattr(self, "_vector_store", None)
-            if emb and vs and getattr(vs, "available", False):
-                vs.upsert(fact.fact_id, self._profile_id, emb)
-        except Exception:
-            pass
+        if index_external:
+            try:
+                vs = getattr(self, "_vector_store", None)
+                if emb and vs and getattr(vs, "available", False):
+                    vs.upsert(fact.fact_id, self._profile_id, emb)
+            except Exception:
+                pass
         # Persist BM25 tokens too (covers the in-memory rank_bm25 fallback path).
-        try:
-            bm25 = getattr(self._retrieval_engine, "_bm25", None)
-            if bm25:
-                bm25.add(fact.fact_id, fact_text, self._profile_id)
-        except Exception:
-            pass
+        if index_external:
+            try:
+                bm25 = getattr(self._retrieval_engine, "_bm25", None)
+                if bm25:
+                    bm25.add(fact.fact_id, fact_text, self._profile_id)
+            except Exception:
+                pass
         return [fact.fact_id]
 
     # -- Recall operations --------------------------------------------------
@@ -574,11 +607,10 @@ class MemoryEngine:
         ``put_nowait`` and the actual ``pending_outcomes`` INSERT runs
         on a background worker.
 
-        V3.4.40 (2026-05-09): ``fast=True`` skips the SpreadingActivation
-        channel. Deprecated in v3.6.9 — SA now completes in ~36ms after the
-        neighbor-cache fix; fast=True is slower than fast=False and reduces
-        recall quality. The parameter is accepted for backward compatibility
-        but is silently treated as False.
+        ``fast=True`` is the latency-bounded path: it skips spreading
+        activation and remote agentic verification while retaining semantic,
+        lexical, graph, temporal, and Hopfield retrieval. Use full recall when
+        maximum multi-round quality matters more than response time.
 
         Multi-scope: ``include_global`` / ``include_shared`` control which
         scopes participate in retrieval. ``None`` (the default) means "use the
@@ -599,31 +631,33 @@ class MemoryEngine:
         if include_shared is None:
             include_shared = bool(getattr(_scope_cfg, "recall_include_shared", False))
 
-        if fast:
-            logger.warning(
-                "fast=True is deprecated (v3.6.9): SpreadingActivation now "
-                "completes in ~36ms; fast mode is slower and reduces quality. "
-                "Pass fast=False (the default) to silence this warning."
-            )
-            fast = False
-
         pid = profile_id or self._profile_id
 
         from superlocalmemory.core.recall_pipeline import run_recall
-        response = run_recall(
-            query, pid, mode=mode, limit=limit, agent_id=agent_id,
-            config=self._config,
-            retrieval_engine=self._retrieval_engine,
-            trust_scorer=self._trust_scorer,
-            embedder=self._embedder,
-            db=self._db, llm=self._llm,
-            hooks=self._hooks,
-            access_log=self._access_log,
-            auto_linker=self._auto_linker,
-            fast=fast,
-            include_global=include_global,
-            include_shared=include_shared,
-        )
+        try:
+            response = run_recall(
+                query, pid, mode=mode, limit=limit, agent_id=agent_id,
+                config=self._config,
+                retrieval_engine=self._retrieval_engine,
+                trust_scorer=self._trust_scorer,
+                embedder=self._embedder,
+                db=self._db, llm=self._llm,
+                hooks=self._hooks,
+                access_log=self._access_log,
+                auto_linker=self._auto_linker,
+                fast=fast,
+                include_global=include_global,
+                include_shared=include_shared,
+            )
+        except Exception as exc:
+            from superlocalmemory.infra.local_diagnostics import record_operation
+
+            record_operation("recall", client=agent_id, error=exc)
+            raise
+
+        from superlocalmemory.infra.local_diagnostics import record_recall
+
+        record_recall(self._db, response, client=agent_id)
 
         # S9-DASH-02: enqueue for pending_outcomes. Non-blocking; errors
         # swallowed because signal capture is never load-bearing on
@@ -685,7 +719,19 @@ class MemoryEngine:
     def close(self) -> None:
         if self._maintenance_scheduler is not None:
             self._maintenance_scheduler.stop()
+        if self._retrieval_engine is not None:
+            try:
+                self._retrieval_engine.close()
+            except Exception:
+                pass
         if self._db is not None:
+            try:
+                from superlocalmemory.core.recall_pipeline import (
+                    release_recall_resources,
+                )
+                release_recall_resources(self._db)
+            except Exception:
+                pass
             try:
                 self._db.close()
             except Exception: