superlocalmemory 3.7.8 → 3.8.0

package/src/superlocalmemory/optimize/config/schema.py

@@ -13,6 +13,81 @@ from typing import Any
 logger = logging.getLogger(__name__)
 
 
+# ---------------------------------------------------------------------------
+# Nested-group compatibility.
+#
+# The canonical serialized form (``OptimizeConfig.as_dict``) is FLAT
+# (``compress_enabled``, ``compress_mode``, ``cache_enabled``, ...). But
+# hand-authored ``optimize.json`` files — and older doc examples — group
+# settings under nested blocks (``{"compress": {"enabled": true, ...}}``).
+# ``from_dict`` accepts BOTH: be liberal in what we read, strict in what we
+# write (Postel's law). Flat keys, when present, always win over a nested
+# value for the same setting.
+# ---------------------------------------------------------------------------
+_NESTED_ALIASES: dict[str, dict[str, str]] = {
+    "compress": {
+        "enabled": "compress_enabled",
+        "mode": "compress_mode",
+        "prose": "compress_prose",
+        "protect_recent": "compress_protect_recent",
+    },
+    "cache": {
+        "enabled": "cache_enabled",
+        "ttl_seconds": "ttl_seconds",
+        "semantic": "semantic_enabled",
+        "semantic_enabled": "semantic_enabled",
+    },
+    "proxy": {
+        "enabled": "proxy_enabled",
+    },
+}
+
+# compress_mode aliases → canonical value. Any unknown mode normalizes to
+# "safe" (fail-open) rather than crashing ``validate()`` at daemon boot.
+_COMPRESS_MODE_ALIASES: dict[str, str] = {
+    "safe": "safe",
+    "aggressive": "aggressive",
+    "fast": "safe",       # legacy alias: "fast" == lightweight == safe
+    "lossless": "safe",
+    "off": "safe",
+}
+
+
+def _flatten_optimize_dict(d: dict[str, Any]) -> dict[str, Any]:
+    """Return a NEW flat dict, merging nested config groups into flat keys.
+
+    Explicit flat keys always take precedence over a nested value for the
+    same setting. The input dict is never mutated (immutable transform).
+    """
+    flat: dict[str, Any] = dict(d)
+    for group, mapping in _NESTED_ALIASES.items():
+        block = d.get(group)
+        if not isinstance(block, dict):
+            continue
+        for nested_key, flat_key in mapping.items():
+            if nested_key in block and flat_key not in d:
+                flat[flat_key] = block[nested_key]
+    return flat
+
+
+def _normalize_compress_mode(raw: Any) -> str:
+    """Map a compress_mode value (including legacy aliases) to a valid mode.
+
+    Unknown values fall back to "safe" with a warning — a single bad enum in
+    a hand-authored config must never crash the optimize subsystem at boot.
+    """
+    key = str(raw).strip().lower()
+    mode = _COMPRESS_MODE_ALIASES.get(key)
+    if mode is None:
+        logger.warning(
+            "optimize.json: unknown compress_mode %r — falling back to 'safe' "
+            "(valid: safe, aggressive)",
+            raw,
+        )
+        return "safe"
+    return mode
+
+
 @dataclass
 class TTLConfig:
     """TTL settings for each cache tier, in seconds."""
@@ -90,8 +165,12 @@ class OptimizeConfig:
     semantic_max_index_entries: int = 10000
     semantic_max_tenants: int = 10000
 
-    # Compress
-    compress_enabled: bool = False
+    # Compress. Safe lossless compression (whitespace normalize + JSON minify) is
+    # ON by default; it runs proxy-free via the slm_compress tool and never
+    # touches code. Lossy prose (LLMLingua) stays gated behind compress_mode=
+    # "aggressive" + compress_prose. Set compress_enabled=False (slm optimize off)
+    # to disable entirely.
+    compress_enabled: bool = True
     compress_mode: str = "safe"
     compress_prose: bool = False
     compress_protect_recent: int = 4
@@ -144,6 +223,10 @@ class OptimizeConfig:
 
     @classmethod
     def from_dict(cls, d: dict[str, Any]) -> "OptimizeConfig":
+        # Accept nested config groups (compress/cache/proxy) in addition to
+        # the canonical flat form. Flat keys win on conflict.
+        d = _flatten_optimize_dict(d)
+
         ttl_raw = d.get("ttl", {})
         ttl = TTLConfig.from_dict(ttl_raw) if isinstance(ttl_raw, dict) else TTLConfig()
 
@@ -181,8 +264,8 @@ class OptimizeConfig:
             ),
             semantic_max_index_entries=int(d.get("semantic_max_index_entries", 10000)),
             semantic_max_tenants=int(d.get("semantic_max_tenants", 10000)),
-            compress_enabled=bool(d.get("compress_enabled", False)),
-            compress_mode=str(d.get("compress_mode", "safe")),
+            compress_enabled=bool(d.get("compress_enabled", True)),
+            compress_mode=_normalize_compress_mode(d.get("compress_mode", "safe")),
             compress_prose=bool(d.get("compress_prose", False)),
             compress_protect_recent=int(d.get("compress_protect_recent", 4)),
             ttl_seconds=int(d.get("ttl_seconds", 86400)),

package/src/superlocalmemory/optimize/metrics/counters.py

@@ -1,9 +1,6 @@
 # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-#
-# ATTRIBUTION: CacheMetrics design pattern adapted from:
-#   omnicache_ai/core/metrics.py (MIT License)
 
 """Thread-safe atomic counters for all SLM Optimize operations."""
 
@@ -43,6 +40,7 @@ class MetricsCollector:
         self._latency_overhead_ms_sum: float = 0.0
         self._latency_samples: int = 0
         self._compress_runs: int = 0
+        self._lossy_compress_runs: int = 0
         self._compress_bytes_original: int = 0
         self._compress_bytes_after: int = 0
         self._cache_size_bytes: int = 0
@@ -69,18 +67,28 @@ class MetricsCollector:
         with self._data_lock:
             self._misses += 1
 
-    def on_compress(self, tokens_before: int, tokens_after: int) -> None:
+    def on_compress(self, tokens_before: int, tokens_after: int, lossy: bool = False) -> None:
         """Record a compression run. Arguments are word-count proxy estimates from _token_estimate().
 
         M-03: consistent naming — these are token estimates, not byte counts.
         Stored in compress_bytes_original/after fields for DB schema compat; unit is word-count.
+
+        ``lossy`` distinguishes a Layer-2 lossy (LLMLingua prose) run from a
+        lossless Layer-1/JSON-minify run; tracked in memory (not yet persisted).
         """
         with self._data_lock:
             self._compress_runs += 1
+            if lossy:
+                self._lossy_compress_runs += 1
             self._compress_bytes_original += max(0, tokens_before)
             self._compress_bytes_after += max(0, tokens_after)
             self._tokens_saved_compress += max(0, tokens_before - tokens_after)
 
+    def lossy_compress_runs(self) -> int:
+        """Session-scoped count of lossy (Layer-2) compression runs."""
+        with self._data_lock:
+            return self._lossy_compress_runs
+
     def on_eviction(self) -> None:
         """Record an eviction."""
         with self._data_lock:
@@ -135,6 +143,7 @@ class MetricsCollector:
             self._latency_overhead_ms_sum = 0.0
             self._latency_samples = 0
             self._compress_runs = 0
+            self._lossy_compress_runs = 0
             self._compress_bytes_original = 0
             self._compress_bytes_after = 0
             self._cache_size_bytes = 0

package/src/superlocalmemory/optimize/metrics/estimator.py

@@ -1,9 +1,6 @@
 # Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-#
-# ATTRIBUTION: PricingRegistry + stale-detection pattern adapted from:
-#   headroom/pricing/registry.py (Apache-2.0)
 
 """Savings estimator — converts MetricsSnapshot to dollar/rupee savings."""
 

package/src/superlocalmemory/optimize/proxy/_helpers.py

@@ -97,6 +97,9 @@ _HOP_BY_HOP = frozenset([
     "te", "trailer", "transfer-encoding", "upgrade", "host",
     "x-forwarded-for", "x-forwarded-host", "x-forwarded-proto",
     "x-real-ip", "x-original-forwarded-for",
+    # ponytail: httpx decompresses gzip automatically; forwarding this header
+    # causes clients to double-decompress → ZlibError. Strip it here.
+    "content-encoding",
 ])
 
 _ANTHROPIC_FORWARD_HEADERS = frozenset([
@@ -152,6 +155,21 @@ def _redact_headers(headers: dict) -> dict:
     }
 
 
+def _active_profile_for_cache() -> str:
+    """Resolve the active memory profile for cache-tenant scoping.
+
+    Lazy import keeps the optimize layer free of a hard server dependency; any
+    failure falls back to 'default'. The profiles.json active_profile cache is
+    kept in sync on every switch, so this is correct for the daemon-mounted
+    proxy even outside an HTTP request context.
+    """
+    try:
+        from superlocalmemory.server.routes.helpers import get_active_profile
+        return get_active_profile() or "default"
+    except Exception:
+        return "default"
+
+
 def _derive_tenant_id(provider: str, raw_credential: "str | None") -> "str | None":
     """Derive a per-tenant isolation key from the raw (un-redacted) credential.
 
@@ -160,18 +178,27 @@ def _derive_tenant_id(provider: str, raw_credential: "str | None") -> "str | Non
     into CacheManager.check() / .store() so that two users sharing the same
     prompt but using different API keys receive independent cache namespaces.
 
+    ISOLATION (I-5): the active memory profile is folded into the key so two
+    profiles sharing the SAME API key never serve each other's cached LLM
+    responses. Cross-profile cache reuse is intentionally sacrificed for
+    tenant isolation (the accepted cost tradeoff for a multi-tenant memory DB).
+
     Returns None when no credential is present — callers must SKIP caching
     (never collapse to the default tenant) to prevent cross-tenant disclosure.
 
-    Output: 64-char lowercase hex SHA-256 of ``f"{provider}:{raw_credential}"``.
-    Provider is folded in so anthropic:K and openai:K are distinct tenants even
-    if the literal key string coincidentally matches.
+    Output: 64-char lowercase hex SHA-256 of
+    ``f"{profile}:{provider}:{raw_credential}"``. Provider is folded in so
+    anthropic:K and openai:K are distinct tenants even if the literal key
+    string coincidentally matches.
     """
     import hashlib as _hashlib
 
     if not raw_credential:
         return None
-    return _hashlib.sha256(f"{provider}:{raw_credential}".encode()).hexdigest()
+    profile = _active_profile_for_cache()
+    return _hashlib.sha256(
+        f"{profile}:{provider}:{raw_credential}".encode()
+    ).hexdigest()
 
 
 def _body_has_tools(body: dict) -> bool:

package/src/superlocalmemory/optimize/storage/db.py

@@ -901,6 +901,8 @@ class CacheDB:
         ccr_id: str,
         original: bytes,
         ttl_expires: float | None = None,
+        *,
+        tenant_id: str = "default",
     ) -> None:
         import hashlib
         try:
@@ -908,10 +910,11 @@ class CacheDB:
             encrypted = self._encrypt(compressed)
             self._db.execute(
                 "INSERT OR REPLACE INTO llmcache_ccr_originals "
-                "(ccr_id, original_blob, compressed_hash, byte_size_orig, byte_size_comp, ttl_expires) "
-                "VALUES (?, ?, ?, ?, ?, ?)",
+                "(ccr_id, tenant_id, original_blob, compressed_hash, "
+                " byte_size_orig, byte_size_comp, ttl_expires) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?)",
                 (
-                    ccr_id, encrypted,
+                    ccr_id, tenant_id, encrypted,
                     hashlib.sha256(original).hexdigest(),
                     len(original), len(compressed),
                     ttl_expires,
@@ -920,13 +923,14 @@ class CacheDB:
         except sqlite3.Error as exc:
             logger.warning("CacheDB.ccr_put failed: %s", exc)
 
-    def ccr_get(self, ccr_id: str) -> bytes | None:
+    def ccr_get(self, ccr_id: str, *, tenant_id: str = "default") -> bytes | None:
         try:
             rows = self._db.execute(
                 "SELECT original_blob FROM llmcache_ccr_originals "
                 "WHERE ccr_id = ? "
+                "AND tenant_id = ? "
                 "AND (ttl_expires IS NULL OR ttl_expires > ?)",
-                (ccr_id, time.time()),
+                (ccr_id, tenant_id, time.time()),
             )
             if not rows:
                 return None
@@ -948,15 +952,16 @@ class CacheDB:
         except sqlite3.Error as exc:
             logger.warning("CacheDB.ccr_update_compressed failed: %s", exc)
 
-    def ccr_delete(self, ccr_id: str) -> None:
-        """Delete a CCR row by ccr_id. Idempotent — warns on sqlite error, never raises.
+    def ccr_delete(self, ccr_id: str, *, tenant_id: str = "default") -> None:
+        """Delete a CCR row by ccr_id scoped to tenant. Idempotent — warns on sqlite error, never raises.
 
         WP-10 D6: defensive infra + sweep parity. Deleting a non-existent row is a no-op.
+        H-02: tenant_id guard prevents a tenant from deleting another tenant's CCR.
         """
         try:
             self._db.execute(
-                "DELETE FROM llmcache_ccr_originals WHERE ccr_id = ?",
-                (ccr_id,),
+                "DELETE FROM llmcache_ccr_originals WHERE ccr_id = ? AND tenant_id = ?",
+                (ccr_id, tenant_id),
             )
         except sqlite3.Error as exc:
             logger.warning("CacheDB.ccr_delete failed (non-fatal): %s", exc)
@@ -1049,6 +1054,30 @@ class CacheDB:
         except sqlite3.Error as exc:
             logger.warning("CacheDB.metrics_flush failed: %s", exc)
 
+    # ---- KV counters (M2 — durable slm_cache_* hit/miss stats) ----
+
+    def kv_counter_incr(self, name: str, delta: int = 1) -> None:
+        """Atomically add ``delta`` to a named KV counter (fail-open)."""
+        try:
+            with self._db.transaction():
+                self._db.execute(
+                    "INSERT INTO llmcache_kv_counters(name, count) "
+                    "VALUES (:name, :delta) "
+                    "ON CONFLICT(name) DO UPDATE SET count = count + :delta",
+                    {"name": name, "delta": delta},
+                )
+        except sqlite3.Error as exc:
+            logger.debug("CacheDB.kv_counter_incr(%s) failed (non-fatal): %s", name, exc)
+
+    def kv_counters_load(self) -> dict[str, int]:
+        """Return all persisted KV counters as {name: count} (fail-open)."""
+        try:
+            rows = self._db.execute("SELECT name, count FROM llmcache_kv_counters")
+            return {r["name"]: r["count"] for r in rows}
+        except sqlite3.Error as exc:
+            logger.debug("CacheDB.kv_counters_load failed: %s", exc)
+            return {}
+
     # ---- convenience / non-contract helpers ----
 
     def entry_exists(self, cache_key: str, tenant_id: str = _DEFAULT_TENANT) -> bool:

package/src/superlocalmemory/optimize/storage/schema.py

@@ -120,6 +120,15 @@ _DDL_STATEMENTS: tuple[str, ...] = (
         updated_at                  REAL    NOT NULL DEFAULT 0
     )
     """,
+    # M2: durable KV hit/miss counters for the slm_cache_* MCP tools. Kept in a
+    # separate table (not llmcache_metrics, whose columns mirror MetricsSnapshot
+    # exactly) so KV stats survive daemon restarts.
+    """
+    CREATE TABLE IF NOT EXISTS llmcache_kv_counters (
+        name  TEXT PRIMARY KEY,
+        count INTEGER NOT NULL DEFAULT 0
+    )
+    """,
 )
 
 
@@ -170,6 +179,7 @@ def get_table_names() -> tuple[str, ...]:
         "llmcache_boundaries",
         "llmcache_centroids",
         "llmcache_metrics",
+        "llmcache_kv_counters",
     )
 
 

package/src/superlocalmemory/parameterization/pattern_extractor.py

@@ -157,7 +157,7 @@ class PatternExtractor:
         all_patterns: list[PatternAssertion] = []
         all_patterns.extend(self._extract_from_core_memory(profile_id))
         all_patterns.extend(self._extract_from_behavioral(profile_id))
-        all_patterns.extend(self._extract_from_cross_project())
+        all_patterns.extend(self._extract_from_cross_project(profile_id))
         all_patterns.extend(self._extract_from_workflows(profile_id))
         # v3.4.7: Extract from behavioral assertions (learned patterns)
         all_patterns.extend(self._extract_from_assertions(profile_id))
@@ -270,9 +270,12 @@ class PatternExtractor:
                 ))
         return patterns
 
-    def _extract_from_cross_project(self) -> list[PatternAssertion]:
-        """Extract patterns from cross-project aggregator."""
+    def _extract_from_cross_project(
+        self, profile_id: str = "default"
+    ) -> list[PatternAssertion]:
+        """Extract patterns from cross-project aggregator, scoped to *profile_id*."""
         preferences = self._cross_project.get_preferences(
+            profile_id=profile_id,
             min_confidence=self._config.min_confidence,
         )
         patterns: list[PatternAssertion] = []

package/src/superlocalmemory/retrieval/agentic.py

@@ -314,7 +314,7 @@ class AgenticRetriever:
                 entity = self._db.get_entity_by_name(name, profile_id)
                 if entity:
                     try:
-                        aliases = self._db.get_aliases_for_entity(entity.entity_id)
+                        aliases = self._db.get_aliases_for_entity(entity.entity_id, profile_id)
                         for a in aliases[:2]:
                             sub_queries.append(f"{a.alias} {' '.join(action_words)}")
                     except Exception:

package/src/superlocalmemory/retrieval/bm25_channel.py

@@ -80,13 +80,26 @@ class BM25Channel:
     def document_count(self) -> int:
         return len(self._corpus)
 
-    def ensure_loaded(self, profile_id: str) -> None:
+    def ensure_loaded(
+        self,
+        profile_id: str,
+        include_global: bool | None = None,
+        include_shared: bool | None = None,
+    ) -> None:
         """Cold-load BM25 tokens from DB for a profile (once).
 
-        Idempotent: subsequent calls for the same profile are no-ops.
+        Idempotent: subsequent calls for the same profile/scope are no-ops.
+
+        Args:
+            profile_id: Profile to load.
+            include_global: Include global-scope facts. Falls back to the
+                instance attribute when not supplied.
+            include_shared: Include shared-scope facts. Same fallback.
         """
-        include_global = bool(getattr(self, "include_global", False))
-        include_shared = bool(getattr(self, "include_shared", False))
+        if include_global is None:
+            include_global = bool(getattr(self, "include_global", False))
+        if include_shared is None:
+            include_shared = bool(getattr(self, "include_shared", False))
         scope_key = (profile_id, include_global, include_shared)
         if scope_key == self._loaded_scope_key:
             return
@@ -174,7 +187,12 @@ class BM25Channel:
         self._db.store_bm25_tokens(fact_id, profile_id, tokens)
 
     def _fts5_search(
-        self, query: str, profile_id: str, top_k: int = 30,
+        self,
+        query: str,
+        profile_id: str,
+        top_k: int = 30,
+        include_global: bool | None = None,
+        include_shared: bool | None = None,
     ) -> list[tuple[str, float]]:
         """v3.5.0: SQLite FTS5 keyword search (C-level indexed, scales to millions).
 
@@ -187,6 +205,10 @@ class BM25Channel:
         Raises (OperationalError) if the FTS5 table is absent — the caller
         then falls back to the legacy in-memory rank_bm25 path.
         """
+        if include_global is None:
+            include_global = bool(getattr(self, "include_global", False))
+        if include_shared is None:
+            include_shared = bool(getattr(self, "include_shared", False))
         tokens = tokenize(query)
         if not tokens:
             return []
@@ -195,8 +217,8 @@ class BM25Channel:
         match_expr = " OR ".join('"' + t.replace('"', "") + '"' for t in tokens)
         where, params = _scope_where(
             profile_id,
-            include_global=bool(getattr(self, "include_global", False)),
-            include_shared=bool(getattr(self, "include_shared", False)),
+            include_global=include_global,
+            include_shared=include_shared,
             prefix="af",
         )
         sql = (
@@ -214,13 +236,39 @@ class BM25Channel:
             if not fid:
                 continue
             out.append((fid, -float(d.get("rank", 0.0))))
-        return out
+
+        # T3b: UNION fact-expansion (alias / paraphrase) matches so a query for
+        # a synonym matches a fact that only used the canonical term. Additive —
+        # direct content hits stay primary; an alias-only hit is added at a
+        # discount so it ranks below content matches. Fail-open if the expansion
+        # FTS is absent (legacy DB).
+        content_ids = {fid for fid, _ in out}
+        try:
+            exp_sql = (
+                "SELECT af.fact_id AS fact_id, bm25(fact_expansion_fts) AS rank "
+                "FROM fact_expansion_fts "
+                "JOIN atomic_facts af ON af.fact_id = fact_expansion_fts.fact_id "
+                f"WHERE fact_expansion_fts MATCH ? AND {where} "
+                "ORDER BY rank LIMIT ?"
+            )
+            for r in self._db.execute(exp_sql, (match_expr, *params, int(top_k))):
+                d = dict(r)
+                fid = d.get("fact_id")
+                if fid and fid not in content_ids:
+                    out.append((fid, -float(d.get("rank", 0.0)) * 0.85))
+        except Exception as exc:  # pragma: no cover — legacy/missing expansion FTS
+            logger.debug("Expansion FTS search skipped: %s", exc)
+
+        out.sort(key=lambda x: x[1], reverse=True)
+        return out[:top_k]
 
     def search(
         self,
         query: str,
         profile_id: str,
         top_k: int = 30,
+        include_global: bool | None = None,
+        include_shared: bool | None = None,
     ) -> list[tuple[str, float]]:
         """Search BM25 index for matching facts.
 
@@ -230,10 +278,17 @@ class BM25Channel:
             query: Search query text.
             profile_id: Scope to this profile.
             top_k: Maximum results.
+            include_global: Include global-scope facts. Falls back to the
+                instance attribute when not supplied.
+            include_shared: Include shared-scope facts. Same fallback.
 
         Returns:
             List of (fact_id, bm25_score) sorted by score descending.
         """
+        if include_global is None:
+            include_global = bool(getattr(self, "include_global", False))
+        if include_shared is None:
+            include_shared = bool(getattr(self, "include_shared", False))
         # v3.5.0: FTS5 fast path — C-level indexed, ~ms, scales to millions.
         # The legacy in-memory rank_bm25 path rebuilt the whole index over the
         # entire corpus on every corpus change (11s+ at 17.5k facts, does not
@@ -241,13 +296,16 @@ class BM25Channel:
         # Falls back to rank_bm25 ONLY if the FTS5 table is genuinely
         # unavailable (raises) — e.g. a pre-FTS legacy DB.
         try:
-            return self._fts5_search(query, profile_id, top_k)
+            return self._fts5_search(
+                query, profile_id, top_k,
+                include_global=include_global, include_shared=include_shared,
+            )
         except Exception as exc:  # pragma: no cover — legacy/missing FTS table
             logger.debug(
                 "BM25 FTS5 path unavailable, using rank_bm25 fallback: %s", exc,
             )
 
-        self.ensure_loaded(profile_id)
+        self.ensure_loaded(profile_id, include_global=include_global, include_shared=include_shared)
 
         if not self._corpus:
             return []