superlocalmemory 3.4.18 → 3.4.21

package/src/superlocalmemory/core/context_cache.py

@@ -0,0 +1,397 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.21 — LLD-01 §4.1
+
+"""Context cache — separate SQLite WAL DB, read-path <10 ms.
+
+LLD reference: `.backup/active-brain/lld/LLD-01-context-cache-and-hot-path-hooks.md`
+Section 4.1.
+
+Two concerns in one module:
+  - Writer (``ContextCache``) — used by the daemon only. Owns pragmas,
+    schema bootstrap, install-binding row, LRU sweep.
+  - Reader (``read_entry_fast``) — used by the UserPromptSubmit hook.
+    Read-only SQLite URI, no pragmas, NEVER raises.
+
+Hot-path contract (``read_entry_fast``):
+  - stdlib-only imports.
+  - Never raises: any exception → returns ``None`` (fail-open miss).
+  - Verifies install-binding HMAC to reject a DB at an env-var-hijacked
+    path.
+  - Applies TTL in SQL, not Python, to avoid fetching stale rows.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import json
+import os
+import sqlite3
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from superlocalmemory.core.security_primitives import (
+    PathTraversalError,
+    ensure_install_token,
+    redact_secrets,
+    safe_resolve,
+)
+
+
+CACHE_DB_DEFAULT: Path = Path.home() / ".superlocalmemory" / "active_brain_cache.db"
+INSTALL_TOKEN_DEFAULT: Path = Path.home() / ".superlocalmemory" / ".install_token"
+
+TTL_SECONDS: int = 120
+CLEANUP_HORIZON_SECONDS: int = 600
+MAX_BYTES: int = 50 * 1024 * 1024
+MAX_CONTENT_CHARS: int = 4000
+SCHEMA_VERSION: str = "3.4.21"
+
+_HMAC_MATERIAL: bytes = b"active_brain_cache"
+_HMAC_HEX_LEN: int = 32
+
+
+@dataclass(frozen=True, slots=True)
+class CacheEntry:
+    """Single cache row. ``content`` is expected to be pre-redacted."""
+
+    session_id: str
+    topic_sig: str
+    content: str
+    fact_ids: list[str] = field(default_factory=list)
+    provenance: str = "tool_observation"
+    computed_at: int = 0
+    byte_size: int = 0
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _expected_binding_hmac(token: str) -> str:
+    """Compute the HMAC-SHA256 first-32-hex value for the install token."""
+    return hmac.new(
+        token.encode("utf-8"), _HMAC_MATERIAL, hashlib.sha256,
+    ).hexdigest()[:_HMAC_HEX_LEN]
+
+
+def _read_install_token(home: Path) -> str | None:
+    """Read the install token without creating it. Returns None if missing."""
+    token_path = home / ".install_token"
+    if not token_path.exists():
+        return None
+    try:
+        token = token_path.read_text(encoding="utf-8").strip()
+    except OSError:  # pragma: no cover — disk-IO failure under contention
+        return None
+    return token or None
+
+
+# ---------------------------------------------------------------------------
+# Writer (daemon-side)
+# ---------------------------------------------------------------------------
+
+
+class ContextCache:
+    """Writer-side cache. One instance per daemon process.
+
+    Opens the DB with WAL + NORMAL sync + bounded mmap, bootstraps the
+    schema, and writes an install-bound HMAC into ``slm_meta`` so the
+    reader can reject a foreign DB pointed at via ``SLM_CACHE_DB``.
+    """
+
+    def __init__(
+        self,
+        db_path: Path | None = None,
+        home_dir: Path | None = None,
+    ) -> None:
+        self._home = home_dir or (Path.home() / ".superlocalmemory")
+        self._home.mkdir(parents=True, exist_ok=True)
+
+        raw = db_path or (self._home / "active_brain_cache.db")
+        self._db_path = safe_resolve(self._home, Path(raw).name) \
+            if Path(raw).parent == self._home else \
+            safe_resolve(self._home, raw)
+
+        self._write_conn = self._open_writer()
+        self._bootstrap_schema_and_meta()
+
+    # -- Open / bootstrap ---------------------------------------------------
+
+    def _open_writer(self) -> sqlite3.Connection:
+        # Pre-create the file with 0600 if it's missing, so sqlite inherits
+        # the restrictive mode instead of umask default.
+        if not self._db_path.exists():
+            flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+            if hasattr(os, "O_NOFOLLOW"):
+                flags |= os.O_NOFOLLOW
+            try:
+                fd = os.open(str(self._db_path), flags, 0o600)
+                os.close(fd)
+            except FileExistsError:  # pragma: no cover — race on concurrent start
+                pass
+        # On POSIX, enforce mode in case an earlier process created it.
+        if os.name != "nt":
+            try:
+                os.chmod(self._db_path, 0o600)
+            except OSError:  # pragma: no cover — remote FS without chmod
+                pass
+
+        conn = sqlite3.connect(
+            str(self._db_path), isolation_level=None, timeout=5.0,
+        )
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA synchronous=NORMAL")
+        conn.execute("PRAGMA temp_store=MEMORY")
+        conn.execute("PRAGMA cache_size=-32768")
+        conn.execute("PRAGMA busy_timeout=500")
+        # 64 MB writer mmap — reader opens with defaults to keep budget tight.
+        try:
+            conn.execute("PRAGMA mmap_size=67108864")
+        except sqlite3.Error:  # pragma: no cover — some builds disable mmap
+            pass
+        return conn
+
+    def _bootstrap_schema_and_meta(self) -> None:
+        self._write_conn.executescript(
+            """
+            CREATE TABLE IF NOT EXISTS context_entries (
+                session_id  TEXT NOT NULL,
+                topic_sig   TEXT NOT NULL,
+                content     TEXT NOT NULL,
+                fact_ids    TEXT NOT NULL,
+                provenance  TEXT NOT NULL DEFAULT 'tool_observation',
+                computed_at INTEGER NOT NULL,
+                byte_size   INTEGER NOT NULL,
+                PRIMARY KEY (session_id, topic_sig)
+            ) WITHOUT ROWID;
+
+            CREATE INDEX IF NOT EXISTS idx_ctx_session_time
+                ON context_entries(session_id, computed_at);
+            CREATE INDEX IF NOT EXISTS idx_ctx_time
+                ON context_entries(computed_at);
+
+            CREATE TABLE IF NOT EXISTS slm_meta (
+                key        TEXT PRIMARY KEY,
+                value      TEXT NOT NULL,
+                created_at INTEGER NOT NULL
+            );
+            """
+        )
+        token = ensure_install_token()
+        now = int(time.time())
+        self._write_conn.execute(
+            "INSERT OR IGNORE INTO slm_meta (key, value, created_at) "
+            "VALUES (?, ?, ?)",
+            ("install_token_hmac", _expected_binding_hmac(token), now),
+        )
+        self._write_conn.execute(
+            "INSERT OR IGNORE INTO slm_meta (key, value, created_at) "
+            "VALUES (?, ?, ?)",
+            ("schema_version", SCHEMA_VERSION, now),
+        )
+
+    # -- Write path ---------------------------------------------------------
+
+    def upsert(self, entry: CacheEntry) -> None:
+        """Insert-or-replace a cache row.
+
+        Content is redacted (belt-and-suspenders — caller should have done
+        this already per LLD-07 §6.3) and truncated. Byte size is computed
+        here, not trusted from caller, so LRU accounting stays accurate.
+        Does NOT run LRU sweep inline (PERF-01-07) — that's a background
+        task on the daemon.
+        """
+        content = redact_secrets(entry.content)[:MAX_CONTENT_CHARS]
+        fact_ids_json = json.dumps(list(entry.fact_ids))
+        byte_size = (
+            len(content.encode("utf-8"))
+            + len(fact_ids_json)
+            + len(entry.session_id)
+            + len(entry.topic_sig)
+        )
+        computed_at = entry.computed_at or int(time.time())
+        self._write_conn.execute(
+            """
+            INSERT OR REPLACE INTO context_entries
+                (session_id, topic_sig, content, fact_ids,
+                 provenance, computed_at, byte_size)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+            """,
+            (entry.session_id, entry.topic_sig, content, fact_ids_json,
+             entry.provenance, computed_at, byte_size),
+        )
+
+    # -- Cleanup ------------------------------------------------------------
+
+    def cleanup_session(
+        self, session_id: str, *, older_than: int = CLEANUP_HORIZON_SECONDS,
+    ) -> int:
+        """Delete rows for ``session_id`` older than ``older_than`` seconds."""
+        cutoff = int(time.time()) - older_than
+        cur = self._write_conn.execute(
+            "DELETE FROM context_entries "
+            "WHERE session_id=? AND computed_at < ?",
+            (session_id, cutoff),
+        )
+        return cur.rowcount
+
+    def cleanup_global_lru(self) -> int:
+        """Background sweep — runs every 60s in the daemon.
+
+        Two passes:
+          1. Time-based — delete rows older than ``CLEANUP_HORIZON_SECONDS``.
+          2. Byte-based — if total size still exceeds ``MAX_BYTES``, delete
+             oldest rows (by ``computed_at``) until total is <= 90% of cap.
+        Returns total deletions.
+        """
+        cutoff = int(time.time()) - CLEANUP_HORIZON_SECONDS
+        cur = self._write_conn.execute(
+            "DELETE FROM context_entries WHERE computed_at < ?", (cutoff,),
+        )
+        deleted = cur.rowcount
+
+        total = self._write_conn.execute(
+            "SELECT COALESCE(SUM(byte_size), 0) FROM context_entries",
+        ).fetchone()[0]
+        if total <= MAX_BYTES:
+            return deleted
+
+        target = int(MAX_BYTES * 0.9)
+        while total > target:
+            rows = self._write_conn.execute(
+                "SELECT session_id, topic_sig, byte_size "
+                "FROM context_entries "
+                "ORDER BY computed_at ASC LIMIT 100",
+            ).fetchall()
+            if not rows:  # pragma: no cover — reached only if table empties mid-sweep
+                break
+            for sess, sig, size in rows:
+                self._write_conn.execute(
+                    "DELETE FROM context_entries "
+                    "WHERE session_id=? AND topic_sig=?",
+                    (sess, sig),
+                )
+                deleted += 1
+                total -= size
+                if total <= target:
+                    break
+        return deleted
+
+    def close(self) -> None:
+        try:
+            self._write_conn.close()
+        except sqlite3.Error:  # pragma: no cover — defensive
+            pass
+
+
+# ---------------------------------------------------------------------------
+# Reader (hook-side hot path — NEVER raises)
+# ---------------------------------------------------------------------------
+
+
+def read_entry_fast(
+    session_id: str,
+    topic_sig: str,
+    *,
+    db_path: Path | None = None,
+    home_dir: Path | None = None,
+) -> CacheEntry | None:
+    """Hot-path reader used by the UserPromptSubmit hook.
+
+    Contract (LLD-01 §4.1):
+      - NEVER raises. Any exception → returns ``None``.
+      - Returns ``None`` on miss, stale (TTL exceeded), missing DB, path
+        traversal attempt, or failed install-binding check.
+      - Opens the SQLite file read-only via ``?mode=ro`` URI.
+      - stdlib-only — no heavy imports, no daemon HTTP call.
+    """
+    try:
+        home = home_dir or (Path.home() / ".superlocalmemory")
+        if not home.exists():
+            return None
+
+        requested = db_path or Path(
+            os.environ.get("SLM_CACHE_DB") or (home / "active_brain_cache.db"),
+        )
+        try:
+            resolved = safe_resolve(home, Path(requested).resolve())
+        except (PathTraversalError, OSError, ValueError):
+            return None
+
+        if not resolved.exists():
+            return None
+
+        token = _read_install_token(home)
+        if token is None:
+            return None
+
+        # Read-only URI connection. Short timeout so busy WAL doesn't stall
+        # the hot path.
+        conn = sqlite3.connect(
+            f"file:{resolved}?mode=ro", uri=True, timeout=0.5,
+        )
+        try:
+            # Verify install binding first — cheap SELECT.
+            row = conn.execute(
+                "SELECT value FROM slm_meta WHERE key='install_token_hmac'",
+            ).fetchone()
+            if row is None:
+                return None
+            expected = _expected_binding_hmac(token)
+            if not hmac.compare_digest(row[0], expected):
+                return None
+
+            now = int(time.time())
+            row = conn.execute(
+                """
+                SELECT content, fact_ids, provenance, computed_at, byte_size
+                FROM context_entries
+                WHERE session_id=? AND topic_sig=?
+                  AND computed_at > ?
+                """,
+                (session_id, topic_sig, now - TTL_SECONDS),
+            ).fetchone()
+        finally:
+            try:
+                conn.close()
+            except sqlite3.Error:  # pragma: no cover — defensive
+                pass
+
+        if row is None:
+            return None
+
+        try:
+            fact_ids = json.loads(row[1])
+            if not isinstance(fact_ids, list):
+                fact_ids = []
+        except (ValueError, TypeError):
+            fact_ids = []
+
+        return CacheEntry(
+            session_id=session_id,
+            topic_sig=topic_sig,
+            content=row[0],
+            fact_ids=fact_ids,
+            provenance=row[2],
+            computed_at=int(row[3]),
+            byte_size=int(row[4]),
+        )
+    except Exception:  # pragma: no cover — last-resort fail-open
+        return None
+
+
+__all__ = (
+    "CACHE_DB_DEFAULT",
+    "CLEANUP_HORIZON_SECONDS",
+    "CacheEntry",
+    "ContextCache",
+    "MAX_BYTES",
+    "MAX_CONTENT_CHARS",
+    "SCHEMA_VERSION",
+    "TTL_SECONDS",
+    "read_entry_fast",
+)

package/src/superlocalmemory/core/embeddings.py

@@ -140,8 +140,14 @@ def release_embedding_lock() -> None:
         _embedding_lock_fd = None
 
 
-_IDLE_TIMEOUT_SECONDS = 120  # 2 minutes — kill worker after idle
-# V3.3.12: Configurable via SLM_EMBED_IDLE_TIMEOUT env var (seconds)
+_IDLE_TIMEOUT_SECONDS = 1800  # 30 minutes — keep model warm across bursty use.
+# V3.3.12: Configurable via SLM_EMBED_IDLE_TIMEOUT env var (seconds).
+# V3.4.19: Bumped from 120 → 1800 to eliminate the 30-60s cold-start pain
+# when the embedding worker was killed too aggressively. Safety: the
+# per-embed RSS self-check (SLM_EMBED_WORKER_RSS_LIMIT_MB, 4GB default) and
+# the daemon memory watchdog (unified_daemon.py, 4GB/60s) still cap any
+# runaway. To restore the old aggressive policy without redeploying, set
+# ``SLM_EMBED_IDLE_TIMEOUT=120`` and ``slm restart``.
 _IDLE_TIMEOUT_SECONDS = int(os.environ.get("SLM_EMBED_IDLE_TIMEOUT", _IDLE_TIMEOUT_SECONDS))
 # V3.3.21: Configurable response timeout — 180s default, but batch ingestion
 # (2-turn chunks across 10 conversations) needs 600s+ to survive cold-start

package/src/superlocalmemory/core/engine.py

@@ -346,14 +346,23 @@ class MemoryEngine:
         self, query: str, profile_id: str | None = None,
         mode: Mode | None = None, limit: int = 20,
         agent_id: str = "unknown",
+        session_id: str | None = None,
     ) -> RecallResponse:
-        """Recall relevant facts for a query."""
+        """Recall relevant facts for a query.
+
+        S9-DASH-02: when ``session_id`` is provided, the recall is
+        non-blockingly enqueued to the outcome queue so downstream
+        hooks (PostToolUse, Stop) can attach engagement signals.
+        Zero additional latency on the hot path — enqueue is a
+        ``put_nowait`` and the actual ``pending_outcomes`` INSERT runs
+        on a background worker.
+        """
         self._ensure_init()
 
         pid = profile_id or self._profile_id
 
         from superlocalmemory.core.recall_pipeline import run_recall
-        return run_recall(
+        response = run_recall(
             query, pid, mode=mode, limit=limit, agent_id=agent_id,
             config=self._config,
             retrieval_engine=self._retrieval_engine,
@@ -365,6 +374,33 @@ class MemoryEngine:
             auto_linker=self._auto_linker,
         )
 
+        # S9-DASH-02: enqueue for pending_outcomes. Non-blocking; errors
+        # swallowed because signal capture is never load-bearing on
+        # recall correctness (LLD-02 §4.9, LLD-08 §4.1).
+        if session_id:
+            try:
+                from superlocalmemory.learning.outcome_queue import (
+                    RecallEvent, enqueue_recall,
+                )
+                fact_ids = tuple(
+                    getattr(r.fact, "fact_id", "") or ""
+                    for r in getattr(response, "results", [])
+                    if getattr(r, "fact", None) is not None
+                )
+                fact_ids = tuple(f for f in fact_ids if f)
+                if fact_ids:
+                    enqueue_recall(RecallEvent(
+                        session_id=session_id,
+                        profile_id=pid,
+                        query=query,
+                        fact_ids=fact_ids,
+                        query_id=getattr(response, "query_id", "") or "",
+                    ))
+            except Exception:
+                pass
+
+        return response
+
     # -- Session operations -------------------------------------------------
 
     def create_speaker_entities(

package/src/superlocalmemory/core/engine_wiring.py

@@ -476,7 +476,7 @@ def _init_quantization_aware_search(
             quantized_store=q_store,
             config=config.quantization,
         )
-        logger.info("QuantizationAwareSearch initialized (TurboQuant 3-tier search)")
+        logger.info("QuantizationAwareSearch initialized (3-tier quantization pipeline)")
         return qas
     except Exception as exc:
         logger.debug("QuantizationAwareSearch init failed (non-fatal): %s", exc)

package/src/superlocalmemory/core/ram_lock.py

@@ -0,0 +1,111 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory v3.4.21 — LLD-00 §7 + P0.5
+
+"""Process-wide RAM semaphore for heavyweight consolidation subsystems.
+
+LLD-00 §7: hnswlib index builds (LLD-12) and trigram full rebuilds
+(LLD-13) both spike RAM into the hundreds of MB. Running them
+concurrently in the same consolidation tick can blow past the I2 RAM
+ceiling on Light/Minimal profiles. This semaphore serialises them.
+
+Design notes:
+- One flock per process pair (exclusive, non-blocking with poll loop).
+- Fast-fail if psutil reports less than ``MIN_FREE_MB + required_mb``
+  available before we even try — better to defer than thrash.
+- Lock file lives at ``~/.superlocalmemory/ram_lock.sem`` by default,
+  tests may monkeypatch ``RAM_LOCK_PATH``.
+- POSIX only; Windows build is deferred per LLD-00 §7 and MASTER-PLAN H-01.
+"""
+
+from __future__ import annotations
+
+import fcntl
+import os
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+import psutil
+
+
+RAM_LOCK_PATH: Path = Path.home() / ".superlocalmemory" / "ram_lock.sem"
+MIN_FREE_MB: int = 400
+
+
+@contextmanager
+def ram_reservation(
+    name: str,
+    *,
+    timeout_s: float = 60.0,
+    required_mb: int = 200,
+) -> Iterator[None]:
+    """Process-wide RAM semaphore. Acquire before heavy subsystems.
+
+    Usage::
+
+        with ram_reservation('hnswlib', required_mb=200):
+            build_index(...)
+
+    Guarantees:
+
+    - Fast-fails if ``psutil.virtual_memory().available < (MIN_FREE_MB +
+      required_mb) * 1024 * 1024``. No body execution in that case.
+    - Acquires the exclusive flock on ``RAM_LOCK_PATH`` with a polling
+      wait. Raises ``RuntimeError`` after ``timeout_s``.
+    - Releases the lock on normal exit AND on exception propagation.
+    - Writes a short audit line ``<pid>:<name>\\n`` on acquire so
+      an operator can see which subsystem holds it.
+    """
+    if not isinstance(name, str) or not name:
+        raise ValueError("name must be a non-empty string")
+    if not isinstance(required_mb, int) or required_mb < 0:
+        raise ValueError(f"required_mb must be non-negative int, got {required_mb!r}")
+
+    vm = psutil.virtual_memory()
+    free_mb = vm.available / (1024 * 1024)
+    floor_mb = MIN_FREE_MB + required_mb
+    if free_mb < floor_mb:
+        raise RuntimeError(
+            f"ram_reservation({name}): free {free_mb:.0f}MB < required "
+            f"{floor_mb}MB"
+        )
+
+    RAM_LOCK_PATH.parent.mkdir(parents=True, exist_ok=True)
+    # SEC-M6 — tighten the parent dir so the audit marker (``{pid}:{name}``)
+    # is not readable by other UIDs on shared hosts. Idempotent; POSIX-only.
+    try:
+        if os.name == "posix":
+            os.chmod(RAM_LOCK_PATH.parent, 0o700)
+    except OSError:  # pragma: no cover — perms race
+        pass
+    fd = os.open(str(RAM_LOCK_PATH), os.O_CREAT | os.O_RDWR, 0o600)
+    try:
+        deadline = time.time() + timeout_s
+        while True:
+            try:
+                fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.time() >= deadline:
+                    raise RuntimeError(
+                        f"ram_reservation({name}) timeout after {timeout_s:.1f}s"
+                    )
+                time.sleep(0.05)
+        try:
+            # Truncate + write a fresh audit marker.
+            os.ftruncate(fd, 0)
+            os.lseek(fd, 0, os.SEEK_SET)
+            os.write(fd, f"{os.getpid()}:{name}\n".encode("utf-8"))
+        except OSError:  # pragma: no cover — marker write is best-effort
+            pass
+        yield
+    finally:
+        try:
+            fcntl.flock(fd, fcntl.LOCK_UN)
+        finally:
+            os.close(fd)
+
+
+__all__ = ("RAM_LOCK_PATH", "MIN_FREE_MB", "ram_reservation")