superlocalmemory 3.6.9 → 3.6.11

package/src/superlocalmemory/cli/compress_cmd.py

@@ -2,7 +2,7 @@
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 # Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
 
-"""Handlers for ``slm compress status|mode|code|prose|ccr``."""
+"""Handlers for ``slm compress status|mode|prose``."""
 
 from __future__ import annotations
 
@@ -19,11 +19,6 @@ def _get_store():
     return ConfigStore()
 
 
-def _get_cache_db():
-    from superlocalmemory.optimize.storage.db import CacheDB
-    return CacheDB()
-
-
 def _write_config(**fields) -> None:
     """5-step immutable config-write."""
     store = _get_store()
@@ -44,18 +39,31 @@ def cmd_compress(args: Namespace) -> None:
     sub = getattr(args, "compress_command", None)
     _dispatch = {
         "status": cmd_compress_status,
-        "mode": cmd_compress_mode,
-        "code": cmd_compress_code,
-        "prose": cmd_compress_prose,
-        "ccr": cmd_compress_ccr,
-        "align": cmd_compress_align,
+        "mode":   cmd_compress_mode,
+        "prose":  cmd_compress_prose,
+        # removed in v3.6.10: code, ccr, align (extractive compressors removed)
+        "code":   _cmd_compress_removed("code"),
+        "ccr":    _cmd_compress_removed("ccr"),
+        "align":  _cmd_compress_removed("align"),
     }
     handler = _dispatch.get(sub or "")
     if handler:
         handler(args)
     else:
-        print("Usage: slm compress status|mode|code|prose|ccr [options]")
+        print("Usage: slm compress status|mode|prose [options]")
+        sys.exit(0)
+
+
+def _cmd_compress_removed(name: str):
+    def handler(args: Namespace) -> None:
+        print(
+            f"slm compress {name}: removed in SLM v3.6.10. "
+            f"Extractive {name} compression has been replaced by "
+            f"Layer 1 (lossless whitespace) + Layer 2 (LLMLingua-2 prose). "
+            f"Use 'slm compress prose on' to enable prose compression."
+        )
         sys.exit(0)
+    return handler
 
 
 def cmd_compress_status(args: Namespace) -> None:
@@ -68,21 +76,19 @@ def cmd_compress_status(args: Namespace) -> None:
             "status": "ok",
             "compress_enabled": cfg.compress_enabled,
             "compress_mode": cfg.compress_mode,
-            "compress_code": cfg.compress_code,
             "compress_prose": cfg.compress_prose,
-            "compress_ccr": cfg.compress_ccr,
+            "compress_protect_recent": cfg.compress_protect_recent,
         }
         print(json.dumps(data, indent=2))
         return
 
     print("Compression status:")
-    print(f"  Enabled:  {'yes' if cfg.compress_enabled else 'no'}")
-    print(f"  Mode:     {cfg.compress_mode}")
-    print(f"  Code:     {'ON' if cfg.compress_code else 'OFF'}"
-          "  (extractive JSON/code — lossless structure)")
-    print(f"  Prose:    {'ON' if cfg.compress_prose else 'OFF'}")
-    print(f"  CCR:      {'ON' if cfg.compress_ccr else 'OFF'}"
-          " (reversible context retrieval)")
+    print(f"  Enabled:         {'yes' if cfg.compress_enabled else 'no'}")
+    print(f"  Mode:            {cfg.compress_mode}")
+    print(f"  Prose (Layer 2): {'ON' if cfg.compress_prose else 'OFF'}"
+          "  (LLMLingua-2, aggressive mode only)")
+    print(f"  Protect recent:  {cfg.compress_protect_recent} user turns")
+    print("  Layer 1 (lossless whitespace normalization) is always ON when enabled.")
 
 
 def cmd_compress_mode(args: Namespace) -> None:
@@ -103,23 +109,8 @@ def cmd_compress_mode(args: Namespace) -> None:
     print("Daemon hot-reload: active within 2s. No restart required.")
 
 
-def cmd_compress_code(args: Namespace) -> None:
-    """Enable or disable code/JSON compression."""
-    use_json = getattr(args, "json", False)
-    value = getattr(args, "code_value", "on")
-
-    _write_config(compress_code=(value == "on"))
-
-    if use_json:
-        print(json.dumps({"status": "ok", "compress_code": value == "on"}))
-        return
-
-    print(f"Code compression: {'ENABLED' if value == 'on' else 'DISABLED'}.")
-    print("Daemon hot-reload: active within 2s.")
-
-
 def cmd_compress_prose(args: Namespace) -> None:
-    """Enable or disable prose compression."""
+    """Enable or disable prose compression (Layer 2, LLMLingua-2)."""
     use_json = getattr(args, "json", False)
     value = getattr(args, "prose_value", "off")
 
@@ -141,39 +132,10 @@ def cmd_compress_prose(args: Namespace) -> None:
         print(json.dumps({"status": "ok", "compress_prose": value == "on"}))
         return
 
-    print(f"Prose compression: {'ENABLED' if value == 'on' else 'DISABLED'}.")
+    print(f"Prose compression (LLMLingua-2): {'ENABLED' if value == 'on' else 'DISABLED'}.")
+    if value == "on":
+        print("  Requires: compress_mode=aggressive and llmlingua package installed.")
+        print("  Run: slm compress mode aggressive  (if not already set)")
     if value == "on" and "compress_enabled" in fields:
         print("  (also enabled global compress)")
     print("Daemon hot-reload: active within 2s.")
-
-
-def cmd_compress_align(args: Namespace) -> None:
-    """Enable or disable alignment compression."""
-    use_json = getattr(args, "json", False)
-    value = getattr(args, "align_value", "on")
-
-    _write_config(compress_align=(value == "on"))
-
-    if use_json:
-        print(json.dumps({"status": "ok", "compress_align": value == "on"}))
-        return
-
-    print(f"Alignment compression: {'ENABLED' if value == 'on' else 'DISABLED'}.")
-    print("Daemon hot-reload: active within 2s.")
-
-
-def cmd_compress_ccr(args: Namespace) -> None:
-    """Enable or disable CCR (Compressed Context Retrieval)."""
-    use_json = getattr(args, "json", False)
-    value = getattr(args, "ccr_value", "off")
-
-    _write_config(compress_ccr=(value == "on"))
-
-    if use_json:
-        print(json.dumps({"status": "ok", "compress_ccr": value == "on"}))
-        return
-
-    print(f"CCR (Compressed Context Retrieval): {'ENABLED' if value == 'on' else 'DISABLED'}.")
-    if value == "on":
-        print("Originals stored in llmcache.db for reversible retrieval.")
-    print("Daemon hot-reload: active within 2s.")

package/src/superlocalmemory/cli/optimize_cmd.py

@@ -97,9 +97,7 @@ def cmd_optimize_status(args: Namespace) -> None:
           f" semantic: {'OFF' if not cfg.semantic_enabled else f'{cfg.ttl.semantic_seconds}s'})")
     print(f"  Compress:  {'enabled' if cfg.compress_enabled else 'disabled'}"
           f"  (mode: {cfg.compress_mode},"
-          f" code: {'ON' if cfg.compress_code else 'OFF'},"
-          f" prose: {'ON' if cfg.compress_prose else 'OFF'},"
-          f" CCR: {'ON' if cfg.compress_ccr else 'OFF'})")
+          f" prose/L2: {'ON' if cfg.compress_prose else 'OFF'})")
     proxy_status = f"running on :{OPTIMIZE_DEFAULT_PORT}" if proxy_running else "not running"
     print(f"  Proxy:     {proxy_status}")
     print(f"  Config:    ~/.superlocalmemory/optimize.json  (version {cfg.config_version})")

package/src/superlocalmemory/cli/setup_wizard.py

@@ -32,6 +32,8 @@ _SLM_HOME = Path(os.environ.get("SL_MEMORY_PATH", Path.home() / ".superlocalmemo
 _SETUP_MARKER = _SLM_HOME / ".setup-complete"
 _EMBED_MODEL = "nomic-ai/nomic-embed-text-v1.5"
 _RERANKER_MODEL = "cross-encoder/ms-marco-MiniLM-L-12-v2"
+# v3.6.10: compulsory LLMLingua-2 prose compression model (~560MB, aggressive mode).
+_COMPRESSOR_MODEL = "microsoft/llmlingua-2-xlm-roberta-large-meetingbank"
 
 
 # ---------------------------------------------------------------------------
@@ -178,6 +180,49 @@ def _download_reranker(model_name: str) -> bool:
         return False
 
 
+def _download_compressor(model_name: str) -> bool:
+    """Download the LLMLingua-2 prose compression model (v3.6.10).
+
+    Mirrors _download_reranker: a subprocess forces the HF download with visible
+    progress. Fail-open — a network hiccup must NOT break setup; the model also
+    lazy-downloads on first use in prose_llmlingua.py.
+    """
+    print(f"\n  Downloading compression model: {model_name}")
+    print(f"  (LLMLingua-2 prose compressor, ~560MB — aggressive mode only)\n")
+
+    script = (
+        "from llmlingua import PromptCompressor; "
+        f"PromptCompressor(model_name='{model_name}', use_llmlingua2=True, "
+        "device_map='cpu'); "
+        "print('OK')"
+    )
+
+    try:
+        result = subprocess.run(
+            [sys.executable, "-c", script],
+            timeout=900,  # 560MB on a slow link can exceed 5 min
+            capture_output=False,
+            text=True,
+            env={
+                **os.environ,
+                "CUDA_VISIBLE_DEVICES": "",
+                "TOKENIZERS_PARALLELISM": "false",
+                "TORCH_DEVICE": "cpu",
+            },
+        )
+        if result.returncode == 0:
+            print(f"  ✓ Compression model ready")
+            return True
+        print(f"  ✗ Compression model download failed (will lazy-download on first use)")
+        return False
+    except ImportError:
+        print(f"  ⚠ llmlingua not installed — compression model will download on first use")
+        return False
+    except Exception as exc:
+        print(f"  ✗ Compression model error: {exc}")
+        return False
+
+
 # ---------------------------------------------------------------------------
 # Verification
 # ---------------------------------------------------------------------------
@@ -393,6 +438,10 @@ def run_wizard(auto: bool = False) -> None:
     else:
         _download_reranker(_RERANKER_MODEL)
 
+    print()
+    print("─── Step 4c/10: Download Compression Model (LLMLingua-2) ───")
+    _download_compressor(_COMPRESSOR_MODEL)
+
     # -- Step 5: Daemon Configuration (v3.4.3) --
     print()
     print("─── Step 5/10: Daemon Configuration ───")

package/src/superlocalmemory/mcp/agent_context.py

@@ -0,0 +1,111 @@
+"""Per-HTTP-request agent ID resolution — ContextVar home.
+
+Kept in a standalone module so both tools_core and tools_active can import
+it without creating circular dependencies (server.py → tools_core → here,
+and unified_daemon.py → here independently).
+
+Priority chain (HTTP-first, stdio-fallback):
+  1. ContextVar set by _AgentIDExtractorASGI middleware from /mcp/{agent_id} URL path.
+  2. SLM_AGENT_ID environment variable (stdio transport legacy).
+  3. Hard-coded "mcp_client" sentinel.
+"""
+from __future__ import annotations
+
+import contextvars
+import os
+import re
+
+_current_agent_id: contextvars.ContextVar[str] = contextvars.ContextVar(
+    "slm_agent_id", default="mcp_client"
+)
+
+# Agent ids arrive from an untrusted URL path segment. They are ATTRIBUTION
+# metadata, never an authenticated principal — but they reach loggers, the
+# agent registry, and SQL-bound attribution columns, so we hard-restrict the
+# charset at the single extraction chokepoint. This neutralises log-injection
+# (CRLF / ANSI), oversized ids, and any path-ish characters in one place.
+_AGENT_ID_SANITIZE = re.compile(r"[^A-Za-z0-9._-]")
+_AGENT_ID_MAX_LEN = 64
+
+
+def sanitize_agent_id(raw: str) -> str:
+    """Coerce an untrusted agent-id segment to a safe, bounded token."""
+    return _AGENT_ID_SANITIZE.sub("_", raw)[:_AGENT_ID_MAX_LEN]
+
+
+def get_current_agent_id(env_fallback: bool = True) -> str:
+    """Return the agent_id for the current asyncio task.
+
+    For HTTP transport the ASGI wrapper sets the ContextVar from the URL path
+    before the request reaches any MCP tool, so this returns the URL-derived id.
+    For stdio transport the ContextVar holds its default ("mcp_client") and we
+    fall through to the SLM_AGENT_ID env var instead.
+    """
+    ctx_id = _current_agent_id.get()
+    if ctx_id != "mcp_client":
+        return ctx_id
+    if env_fallback:
+        return os.environ.get("SLM_AGENT_ID", "mcp_client")
+    return "mcp_client"
+
+
+class AgentIDExtractorASGI:
+    """ASGI wrapper that maps ``/mcp/{agent_id}`` → the agent-id ContextVar.
+
+    Mounted at ``/mcp`` in unified_daemon. IMPORTANT: Starlette's ``Mount``
+    (≥0.35 / 1.x) does NOT strip the mount prefix from ``scope["path"]`` — it
+    records the prefix in ``scope["root_path"]`` and leaves ``path`` as the full
+    request path (e.g. ``/mcp/claude`` with ``root_path == "/mcp"``). So we
+    compute the mount-relative sub-path ourselves as ``path[len(root_path):]``.
+
+    Flow for ``POST /mcp/claude``:
+      sub-path ``/claude`` → agent id ``claude`` → set ContextVar → rewrite the
+      scope path to ``{root_path}/`` so the inner FastMCP app (Starlette, route
+      ``/``) sees the same mount-relative ``/`` it sees for a bare ``/mcp/``.
+
+    Backward compatible: bare ``/mcp/`` has sub-path ``/`` → no agent segment →
+    the request passes through untouched and the ContextVar keeps its
+    ``"mcp_client"`` default.
+
+    Per-request isolation is guaranteed by ContextVar + ``reset(token)`` in a
+    ``finally``, so concurrent HTTP sessions never see each other's agent id.
+    """
+
+    __slots__ = ("_app",)
+
+    def __init__(self, inner) -> None:
+        self._app = inner
+
+    async def __call__(self, scope, receive, send):
+        if scope.get("type") == "http":
+            root_path: str = scope.get("root_path", "")
+            full_path: str = scope.get("path", "/")
+            # Mount-relative sub-path (what comes AFTER /mcp). When a root_path
+            # is present the request path MUST start with it (Starlette Mount
+            # guarantees this); if it somehow does not, treat it as no-agent and
+            # pass through untouched rather than mis-parsing the full path.
+            if root_path:
+                if not full_path.startswith(root_path):
+                    await self._app(scope, receive, send)
+                    return
+                subpath = full_path[len(root_path):]
+            else:
+                subpath = full_path
+            first = subpath.lstrip("/").split("/")[0]
+            if first:
+                first = sanitize_agent_id(first)
+                token = _current_agent_id.set(first)
+                # Rewrite the path so the inner app sees the bare mount root,
+                # exactly as it would for a no-agent /mcp/ request.
+                new_full = (root_path + "/") if root_path else "/"
+                new_scope = {
+                    **scope,
+                    "path": new_full,
+                    "raw_path": new_full.encode(),
+                }
+                try:
+                    await self._app(new_scope, receive, send)
+                finally:
+                    _current_agent_id.reset(token)
+                return
+        await self._app(scope, receive, send)

package/src/superlocalmemory/mcp/server.py

@@ -104,6 +104,8 @@ _ESSENTIAL_TOOLS: set[str] = {
     "reinforce_assertion", "contradict_assertion",
     # v3.4.11: Skill evolution (3)
     "evolve_skill", "skill_health", "skill_lineage",
+    # v3.6.11: Surface B Optimize tools (5)
+    "slm_compress", "slm_retrieve", "slm_cache_set", "slm_cache_get", "slm_optimize_stats",
 }
 
 # v3.4.4: Mesh tools — enabled if mesh_enabled in config or SLM_MCP_MESH_TOOLS=1
@@ -189,6 +191,8 @@ register_code_graph_tools(_target, get_engine)  # CodeGraph: filtered like other
 register_mesh_tools(_target, get_engine)  # v3.4.4: Mesh P2P tools — ships with SLM, no separate slm-mesh needed
 register_learning_tools(_target, get_engine)  # v3.4.7: Two-way learning tools
 register_evolution_tools(_target, get_engine)  # v3.4.11: Skill evolution tools
+from superlocalmemory.mcp.tools_optimize import register_optimize_tools
+register_optimize_tools(_target)  # v3.6.11: Surface B Optimize tools (proxy-free)
 
 
 # V3.3.21: Eager engine warmup — start initializing BEFORE first tool call.

package/src/superlocalmemory/mcp/tools_active.py

@@ -109,15 +109,14 @@ def _sqlite_emergency_recall(
 def _get_agent_id(default: str = "mcp_client") -> str:
     """Resolve the calling agent's ID for attribution.
 
-    Each MCP client (Claude Code, Codex, Gemini CLI, Kimi, etc.) can set
-    the ``SLM_AGENT_ID`` env var in its MCP server config so that memories,
-    observations, and registry entries are tagged with the actual source
-    agent — not the legacy ``"mcp_client"`` default.
-
-    v3.4.39+: enables proper per-agent attribution in ``session_init``,
-    ``observe``, and event emissions.
+    Priority chain (v3.6.10+):
+    1. ContextVar set by HTTP URL path (/mcp/{agent_id}) — HTTP transport.
+    2. SLM_AGENT_ID env var — stdio transport per-process identity.
+    3. Provided default (legacy "mcp_client").
     """
-    return os.environ.get("SLM_AGENT_ID", default)
+    from superlocalmemory.mcp.agent_context import get_current_agent_id
+    resolved = get_current_agent_id(env_fallback=True)
+    return resolved if resolved != "mcp_client" else default
 
 
 def _emit_event(event_type: str, payload: dict | None = None,

package/src/superlocalmemory/mcp/tools_core.py

@@ -110,6 +110,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
         Extracts atomic facts, resolves entities, builds graph edges,
         and indexes for 4-channel retrieval.
         """
+        # v3.6.10: resolve "mcp_client" sentinel → URL path (HTTP) or env var (stdio)
+        if agent_id == "mcp_client":
+            from superlocalmemory.mcp.agent_context import get_current_agent_id
+            agent_id = get_current_agent_id()
         meta = {
             "project": project,
             "importance": importance,
@@ -171,6 +175,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
         ``CLAUDE_SESSION_ID``. Omitting it degrades to "no closed-loop
         learning for this recall" — the recall itself always works.
         """
+        # v3.6.10: resolve "mcp_client" sentinel → URL path (HTTP) or env var (stdio)
+        if agent_id == "mcp_client":
+            from superlocalmemory.mcp.agent_context import get_current_agent_id
+            agent_id = get_current_agent_id()
         import asyncio
         try:
             from superlocalmemory.mcp._daemon_proxy import choose_pool
@@ -485,6 +493,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
             fact_id: Exact fact ID to delete (from recall or list_recent results).
             agent_id: Identifier of the calling agent (logged for audit).
         """
+        # v3.6.10: resolve "mcp_client" sentinel → URL path (HTTP) or env var (stdio)
+        if agent_id == "mcp_client":
+            from superlocalmemory.mcp.agent_context import get_current_agent_id
+            agent_id = get_current_agent_id()
         try:
             from superlocalmemory.core.worker_pool import WorkerPool
             pool = WorkerPool.shared()
@@ -519,6 +531,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
             content: New content for the memory (cannot be empty).
             agent_id: Identifier of the calling agent (logged for audit).
         """
+        # v3.6.10: resolve "mcp_client" sentinel → URL path (HTTP) or env var (stdio)
+        if agent_id == "mcp_client":
+            from superlocalmemory.mcp.agent_context import get_current_agent_id
+            agent_id = get_current_agent_id()
         try:
             if not content or not content.strip():
                 return {"success": False, "error": "content cannot be empty"}