superlocalmemory 3.7.5 → 3.7.7

package/src/superlocalmemory/server/profile_runtime.py

@@ -0,0 +1,384 @@
+"""Linearizable active-profile runtime state for the unified daemon."""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+import threading
+from contextlib import contextmanager
+from contextvars import ContextVar
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable, Iterator
+
+from superlocalmemory.infra.data_root import state_path
+
+_RUNTIME_BIND_LOCK = threading.Lock()
+_REQUEST_PROFILE: ContextVar[str | None] = ContextVar(
+    "slm_request_profile", default=None,
+)
+
+
+def current_request_profile() -> str | None:
+    """Return the immutable profile snapshot admitted for this request."""
+    return _REQUEST_PROFILE.get()
+
+
+@dataclass(frozen=True, slots=True)
+class ProfileSnapshot:
+    """One immutable active-profile generation."""
+
+    profile_id: str
+    generation: int
+
+
+class ProfileRuntime:
+    """Coordinate profile-sensitive operations and exclusive transitions."""
+
+    def __init__(self, profile_id: str, *, generation: int = 0) -> None:
+        self._condition = threading.Condition(threading.Lock())
+        self._snapshot = ProfileSnapshot(profile_id, generation)
+        self._active_operations = 0
+        self._transitioning = False
+
+    @property
+    def snapshot(self) -> ProfileSnapshot:
+        with self._condition:
+            return self._snapshot
+
+    @property
+    def transitioning(self) -> bool:
+        with self._condition:
+            return self._transitioning
+
+    def acquire_operation(self) -> ProfileSnapshot:
+        """Admit an operation only when no profile transition is active."""
+        with self._condition:
+            while self._transitioning:
+                self._condition.wait()
+            self._active_operations += 1
+            return self._snapshot
+
+    def release_operation(self) -> None:
+        with self._condition:
+            if self._active_operations <= 0:
+                raise RuntimeError("profile runtime operation lease underflow")
+            self._active_operations -= 1
+            if self._active_operations == 0:
+                self._condition.notify_all()
+
+    @contextmanager
+    def operation(self) -> Iterator[ProfileSnapshot]:
+        snapshot = self.acquire_operation()
+        try:
+            yield snapshot
+        finally:
+            self.release_operation()
+
+    def transition(
+        self,
+        target_profile: str,
+        commit: Callable[[ProfileSnapshot, str], None],
+    ) -> ProfileSnapshot:
+        """Drain admitted operations, commit, then publish a new generation."""
+        with self._condition:
+            while self._transitioning:
+                self._condition.wait()
+            if target_profile == self._snapshot.profile_id:
+                return self._snapshot
+            self._transitioning = True
+            while self._active_operations:
+                self._condition.wait()
+            previous = self._snapshot
+
+        try:
+            commit(previous, target_profile)
+        except BaseException:
+            with self._condition:
+                self._transitioning = False
+                self._condition.notify_all()
+            raise
+
+        with self._condition:
+            self._snapshot = ProfileSnapshot(
+                profile_id=target_profile,
+                generation=previous.generation + 1,
+            )
+            self._transitioning = False
+            self._condition.notify_all()
+            return self._snapshot
+
+    def reconfigure(self, commit: Callable[[ProfileSnapshot], None]) -> ProfileSnapshot:
+        """Run a same-profile engine transition behind the operation barrier."""
+        with self._condition:
+            while self._transitioning:
+                self._condition.wait()
+            self._transitioning = True
+            while self._active_operations:
+                self._condition.wait()
+            snapshot = self._snapshot
+
+        try:
+            commit(snapshot)
+        except BaseException:
+            with self._condition:
+                self._transitioning = False
+                self._condition.notify_all()
+            raise
+
+        with self._condition:
+            self._transitioning = False
+            self._condition.notify_all()
+            return self._snapshot
+
+
+@dataclass(slots=True)
+class ActiveProfilePersistence:
+    """Rollback handle for the two compatibility configuration stores."""
+
+    previous: dict[Path, bytes | None]
+    _rolled_back: bool = False
+
+    def rollback(self) -> None:
+        if self._rolled_back:
+            return
+        for path, content in self.previous.items():
+            if content is None:
+                path.unlink(missing_ok=True)
+            else:
+                _atomic_write_bytes(path, content)
+        self._rolled_back = True
+
+
+def _load_json_object(path: Path, *, default: dict) -> dict:
+    if not path.exists():
+        return dict(default)
+    value = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(value, dict):
+        raise ValueError(f"{path.name} must contain a JSON object")
+    return value
+
+
+def _atomic_write_bytes(path: Path, content: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temporary = tempfile.mkstemp(
+        dir=str(path.parent),
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+    )
+    temporary_path = Path(temporary)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(content)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary_path, path)
+    except BaseException:
+        temporary_path.unlink(missing_ok=True)
+        raise
+
+
+def _json_bytes(payload: dict) -> bytes:
+    return (json.dumps(payload, indent=2) + "\n").encode("utf-8")
+
+
+def persist_active_profile(profile_id: str) -> ActiveProfilePersistence:
+    """Atomically update config plus the legacy profiles cache, with rollback."""
+    config_path = Path(state_path("config.json"))
+    profiles_path = Path(state_path("profiles.json"))
+    paths = (config_path, profiles_path)
+    previous = {
+        path: path.read_bytes() if path.exists() else None
+        for path in paths
+    }
+    config = _load_json_object(config_path, default={})
+    profiles = _load_json_object(
+        profiles_path,
+        default={
+            "profiles": {
+                "default": {
+                    "name": "default",
+                    "description": "Default memory profile",
+                },
+            },
+        },
+    )
+    config["active_profile"] = profile_id
+    profiles["active_profile"] = profile_id
+    profile_catalog = profiles.get("profiles")
+    if isinstance(profile_catalog, dict):
+        selected = profile_catalog.get(profile_id)
+        if isinstance(selected, dict):
+            selected["last_used"] = datetime.now(timezone.utc).isoformat()
+    rollback = ActiveProfilePersistence(previous)
+    try:
+        _atomic_write_bytes(config_path, _json_bytes(config))
+        _atomic_write_bytes(profiles_path, _json_bytes(profiles))
+    except BaseException:
+        rollback.rollback()
+        raise
+    return rollback
+
+
+def get_profile_runtime(app_state) -> ProfileRuntime:
+    """Return the daemon runtime, lazily deriving its initial profile."""
+    runtime = getattr(app_state, "profile_runtime", None)
+    if runtime is not None:
+        return runtime
+    with _RUNTIME_BIND_LOCK:
+        runtime = getattr(app_state, "profile_runtime", None)
+        if runtime is not None:
+            return runtime
+        engine = getattr(app_state, "engine", None)
+        config = getattr(app_state, "config", None)
+        profile_id = (
+            getattr(engine, "profile_id", "")
+            or getattr(config, "active_profile", "")
+            or "default"
+        )
+        runtime = ProfileRuntime(str(profile_id))
+        app_state.profile_runtime = runtime
+        return runtime
+
+
+def bind_profile_runtime(app_state, engine, config) -> ProfileRuntime:
+    """Attach the authoritative runtime after daemon engine initialization."""
+    runtime = ProfileRuntime(str(engine.profile_id))
+    app_state.profile_runtime = runtime
+    app_state.engine = engine
+    app_state.config = config
+    return runtime
+
+
+def commit_daemon_profile_switch(
+    app_state,
+    previous: ProfileSnapshot,
+    target_profile: str,
+) -> None:
+    """Persist and rebind a quiescent resident engine, rolling back on error."""
+    engine = getattr(app_state, "engine", None)
+    if engine is None:
+        raise RuntimeError("resident engine is unavailable")
+    rows = engine._db.execute(
+        "SELECT 1 FROM profiles WHERE profile_id = ?",
+        (target_profile,),
+    )
+    if not rows:
+        raise RuntimeError(
+            f"profile '{target_profile}' no longer exists at commit time"
+        )
+    app_config = getattr(app_state, "config", None)
+    engine_config = getattr(engine, "_config", None)
+    persistence = None
+    try:
+        # Requests are drained and runtime generation is not yet published.
+        # Rebind the in-memory engine first, then make compatibility files the
+        # final commit step so they can never lead daemon runtime truth.
+        engine.profile_id = target_profile
+        if app_config is not None:
+            app_config.active_profile = target_profile
+        if engine_config is not None:
+            engine_config.active_profile = target_profile
+        persistence = persist_active_profile(target_profile)
+    except BaseException:
+        engine.profile_id = previous.profile_id
+        if app_config is not None:
+            app_config.active_profile = previous.profile_id
+        if engine_config is not None:
+            engine_config.active_profile = previous.profile_id
+        if persistence is not None:
+            persistence.rollback()
+        raise
+
+
+def reconfigure_daemon_engine(app_state, new_config, *, mode_change: bool) -> None:
+    """Exclusively rebuild every daemon engine reference for a new config."""
+    callback = getattr(app_state, "reconfigure_engine", None)
+    if not callable(callback):
+        # Direct route/unit usage without a resident daemon retains the
+        # established persistence-only behavior.
+        new_config.save(mode_change=mode_change)
+        return
+    runtime = get_profile_runtime(app_state)
+
+    def _commit(snapshot: ProfileSnapshot) -> None:
+        # The candidate may have been loaded before a concurrent profile
+        # switch. Runtime truth always wins over that stale config snapshot.
+        new_config.active_profile = snapshot.profile_id
+        callback(new_config, mode_change=mode_change)
+
+    runtime.reconfigure(_commit)
+
+
+class ProfileRuntimeMiddleware:
+    """Hold an operation lease for each daemon HTTP request."""
+
+    def __init__(self, app, *, app_state) -> None:
+        self._app = app
+        self._app_state = app_state
+
+    @staticmethod
+    def _is_transition_request(path: str, method: str) -> bool:
+        profile_switch = (
+            path.startswith("/api/profiles/")
+            and path.endswith("/switch")
+        )
+        config_transition = method in {"POST", "PUT", "PATCH"} and path in {
+            "/api/v3/mode",
+            "/api/v3/mode/set",
+            "/api/v3/embedding/config",
+            "/api/v3/scope/config",
+        }
+        return profile_switch or config_transition
+
+    async def __call__(self, scope, receive, send) -> None:
+        if scope.get("type") != "http":
+            await self._app(scope, receive, send)
+            return
+        path = str(scope.get("path", ""))
+        method = str(scope.get("method", "GET")).upper()
+        # Mounted MCP tools proxy profile-sensitive work back through the
+        # canonical daemon routes. Leasing the outer MCP request would make an
+        # embedded switch wait on itself.
+        if path.startswith("/mcp") or self._is_transition_request(path, method):
+            await self._app(scope, receive, send)
+            return
+
+        import asyncio
+
+        runtime = get_profile_runtime(self._app_state)
+        acquire_task = asyncio.create_task(
+            asyncio.to_thread(runtime.acquire_operation)
+        )
+        try:
+            # Shield the worker so cancellation cannot strand a lease that the
+            # thread acquires after this coroutine has already unwound.
+            snapshot = await asyncio.shield(acquire_task)
+        except asyncio.CancelledError:
+            await acquire_task
+            runtime.release_operation()
+            raise
+        scope.setdefault("state", {})["profile_snapshot"] = snapshot
+        token = _REQUEST_PROFILE.set(snapshot.profile_id)
+        try:
+            await self._app(scope, receive, send)
+        finally:
+            _REQUEST_PROFILE.reset(token)
+            # Release is lock-only and must not itself be cancellation-prone.
+            runtime.release_operation()
+
+
+__all__ = [
+    "ActiveProfilePersistence",
+    "ProfileRuntime",
+    "ProfileRuntimeMiddleware",
+    "ProfileSnapshot",
+    "bind_profile_runtime",
+    "commit_daemon_profile_switch",
+    "current_request_profile",
+    "get_profile_runtime",
+    "persist_active_profile",
+    "reconfigure_daemon_engine",
+]

package/src/superlocalmemory/server/recall_health.py

@@ -1,4 +1,4 @@
-"""Runtime recall-health monitor — keep the 6-channel recall path warm,
+"""Runtime recall-health monitor — keep the full recall path warm,
 detect a "warm-but-broken" embedder at runtime, and self-heal it (v3.6.8).
 
 Why this exists
@@ -39,6 +39,7 @@ from __future__ import annotations
 
 import logging
 import threading
+from contextlib import nullcontext
 from dataclasses import dataclass
 
 logger = logging.getLogger(__name__)
@@ -118,7 +119,7 @@ def _heal_embedder(engine, *, log) -> bool:
 
 
 def run_health_tick(engine, state: RecallHealth, *, probe: str = DEFAULT_PROBE,
-                    log=logger) -> RecallHealth:
+                    log=logger, runtime=None) -> RecallHealth:
     """One monitor tick: re-warm (Tier 1), probe (Tier 2), self-heal (Tier 3).
 
     Mutates and returns ``state``. Never raises — a timed-out / failing recall
@@ -129,7 +130,9 @@ def run_health_tick(engine, state: RecallHealth, *, probe: str = DEFAULT_PROBE,
     # Tier 1: re-warm. A real full-fusion recall keeps the graph page cache hot
     # and the embedder resident.
     try:
-        resp = engine.recall(probe, limit=3, fast=False)
+        lease = runtime.operation() if runtime is not None else nullcontext()
+        with lease:
+            resp = engine.recall(probe, limit=3, fast=False)
     except Exception as exc:
         state.healthy = False
         state.consecutive_failures += 1
@@ -183,7 +186,7 @@ def run_health_tick(engine, state: RecallHealth, *, probe: str = DEFAULT_PROBE,
 
 def health_monitor_loop(engine, *, interval_s: int, stop_event: threading.Event,
                         state: RecallHealth, probe: str = DEFAULT_PROBE,
-                        log=logger) -> None:
+                        log=logger, runtime=None) -> None:
     """Background loop. Sleeps ``interval_s`` between ticks; exits promptly when
     ``stop_event`` is set. An initial short delay avoids racing boot warmup."""
     # Initial delay (bounded) so we don't pile onto the boot warmup threads.
@@ -191,7 +194,9 @@ def health_monitor_loop(engine, *, interval_s: int, stop_event: threading.Event,
         return
     while not stop_event.is_set():
         try:
-            run_health_tick(engine, state, probe=probe, log=log)
+            run_health_tick(
+                engine, state, probe=probe, log=log, runtime=runtime,
+            )
         except Exception as exc:  # pragma: no cover - belt & suspenders
             log.warning("recall-health: tick crashed (non-fatal): %s", exc)
         if stop_event.wait(interval_s):
@@ -204,7 +209,8 @@ _GLOBAL_STATE = RecallHealth()
 
 
 def start_recall_health_monitor(engine, *, interval_s: int = DEFAULT_INTERVAL_S,
-                                probe: str = DEFAULT_PROBE, log=None):
+                                probe: str = DEFAULT_PROBE, log=None,
+                                runtime=None):
     """Start the monitor as a daemon thread. Returns ``(thread, stop_event,
     state)``. The state is the shared module-level state read by
     :func:`get_recall_health`."""
@@ -215,7 +221,7 @@ def start_recall_health_monitor(engine, *, interval_s: int = DEFAULT_INTERVAL_S,
         target=health_monitor_loop,
         kwargs=dict(
             engine=engine, interval_s=interval_s, stop_event=stop,
-            state=state, probe=probe, log=log,
+            state=state, probe=probe, log=log, runtime=runtime,
         ),
         daemon=True,
         name="recall-health",

package/src/superlocalmemory/server/routes/chat.py

@@ -4,7 +4,7 @@
 
 """Ask My Memory — SSE chat endpoint.
 
-Flow: query → 6-channel retrieval → format context → LLM stream → SSE
+Flow: query → full recall (five-producer fusion) → format context → LLM stream → SSE
 Mode A: No LLM, returns formatted retrieval results.
 Mode B: Ollama local streaming via /api/chat.
 Mode C: Cloud LLM streaming (OpenAI-compatible).
@@ -315,7 +315,7 @@ async def _stream_openai_compat(
 # ── Retrieval Helper ─────────────────────────────────────────────
 
 def _recall_memories(query: str, limit: int) -> list:
-    """Run 6-channel retrieval via WorkerPool (synchronous, runs in executor)."""
+    """Run full recall via WorkerPool (synchronous, runs in executor)."""
     from superlocalmemory.core.worker_pool import WorkerPool
     pool = WorkerPool.shared()
     result = pool.recall(query, limit=limit)

package/src/superlocalmemory/server/routes/helpers.py

@@ -233,7 +233,12 @@ def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict:
 
 
 def get_active_profile() -> str:
-    """Read the active profile from profiles.json. Falls back to 'default'."""
+    """Read request runtime truth, falling back to the compatibility cache."""
+    from superlocalmemory.server.profile_runtime import current_request_profile
+
+    runtime_profile = current_request_profile()
+    if runtime_profile:
+        return runtime_profile
     config_file = MEMORY_DIR / "profiles.json"
     if config_file.exists():
         try:
@@ -330,22 +335,10 @@ def sync_profiles() -> list[dict]:
 
 
 def set_active_profile_everywhere(name: str) -> None:
-    """Persist the active profile to BOTH profiles.json and config.json."""
-    # profiles.json
-    config = _load_profiles_json()
-    config['active_profile'] = name
-    _save_profiles_json(config)
+    """Persist active profile through the crash-safe compatibility writer."""
+    from superlocalmemory.server.profile_runtime import persist_active_profile
 
-    # config.json (read by Engine/MCP on startup)
-    config_path = MEMORY_DIR / "config.json"
-    cfg = {}
-    if config_path.exists():
-        try:
-            cfg = json.loads(config_path.read_text())
-        except (json.JSONDecodeError, IOError):
-            pass
-    cfg['active_profile'] = name
-    config_path.write_text(json.dumps(cfg, indent=2))
+    persist_active_profile(name)
 
 
 def delete_profile_from_db(name: str) -> None:

package/src/superlocalmemory/server/routes/profiles.py

@@ -10,6 +10,7 @@ Routes: /api/profiles, /api/profiles/{name}/switch,
 SQLite is the single source of truth for profiles. profiles.json
 is kept in sync as a cache for backward compatibility.
 """
+import asyncio
 import logging
 from datetime import datetime, timezone
 
@@ -21,9 +22,13 @@ from .helpers import (
     get_db_connection, validate_profile_name,
     ProfileSwitch, DB_PATH,
     sync_profiles, ensure_profile_in_db, ensure_profile_in_json,
-    set_active_profile_everywhere, delete_profile_from_db,
+    delete_profile_from_db,
     _load_profiles_json, _save_profiles_json,
 )
+from superlocalmemory.server.profile_runtime import (
+    commit_daemon_profile_switch,
+    get_profile_runtime,
+)
 
 logger = logging.getLogger("superlocalmemory.routes.profiles")
 router = APIRouter()
@@ -54,12 +59,11 @@ def _get_memory_count(profile: str) -> int:
 
 
 @router.get("/api/profiles")
-async def list_profiles():
+async def list_profiles(request: Request):
     """List available memory profiles (synced from SQLite + profiles.json)."""
     try:
         merged = sync_profiles()
-        json_config = _load_profiles_json()
-        active = json_config.get('active_profile', 'default')
+        active = get_profile_runtime(request.app.state).snapshot.profile_id
 
         profiles = []
         for p in merged:
@@ -108,14 +112,17 @@ async def switch_profile(name: str, request: Request):
             source_agent_id="http-profile-switch",
             profile_id=name,
         )
-        previous = _load_profiles_json().get('active_profile', 'default')
-        set_active_profile_everywhere(name)
-
-        # Update last_used in profiles.json
-        json_config = _load_profiles_json()
-        if name in json_config.get('profiles', {}):
-            json_config['profiles'][name]['last_used'] = datetime.now(timezone.utc).isoformat()
-            _save_profiles_json(json_config)
+        runtime = get_profile_runtime(request.app.state)
+        previous = runtime.snapshot.profile_id
+        snapshot = await asyncio.to_thread(
+            runtime.transition,
+            name,
+            lambda prior, target: commit_daemon_profile_switch(
+                request.app.state,
+                prior,
+                target,
+            ),
+        )
 
         count = _get_memory_count(name)
 
@@ -130,6 +137,7 @@ async def switch_profile(name: str, request: Request):
         return {
             "success": True, "active_profile": name,
             "previous_profile": previous, "memory_count": count,
+            "generation": snapshot.generation,
             "message": f"Switched to profile '{name}' ({count} memories).",
         }
 
@@ -184,10 +192,12 @@ async def delete_profile(name: str, request: Request):
         if name not in merged_ids:
             raise HTTPException(status_code=404, detail=f"Profile '{name}' not found")
 
-        json_config = _load_profiles_json()
-        if json_config.get('active_profile') == name:
+        runtime = get_profile_runtime(request.app.state)
+        if runtime.snapshot.profile_id == name:
             raise HTTPException(status_code=400, detail="Cannot delete active profile.")
 
+        json_config = _load_profiles_json()
+
         authorization = authorize_route_mutation(
             request,
             operation="delete",