superlocalmemory 3.8.0 → 3.8.2

package/src/superlocalmemory/server/routes/learning_telemetry.py

@@ -0,0 +1,153 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+"""Read-only persisted telemetry used by the Living Brain routes."""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+from pathlib import Path
+
+logger = logging.getLogger("superlocalmemory.routes.learning")
+
+SOURCE_SCORE_LIMIT = 50
+
+
+class ReadOnlyRankerStore:
+    """Minimal model-cache adapter that never creates or migrates tables."""
+
+    def __init__(self, db_path: Path) -> None:
+        self._db_path = Path(db_path)
+
+    def count_signals(self, profile_id: str) -> int:
+        connection = _readonly_connection(self._db_path)
+        try:
+            row = connection.execute(
+                "SELECT COUNT(*) AS count FROM learning_signals "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            ).fetchone()
+            return int(row["count"] if row else 0)
+        finally:
+            connection.close()
+
+    def load_active_model(self, profile_id: str) -> dict | None:
+        connection = _readonly_connection(self._db_path)
+        try:
+            row = connection.execute(
+                "SELECT state_bytes, bytes_sha256, feature_names, trained_at, "
+                "model_version FROM learning_model_state "
+                "WHERE profile_id = ? AND is_active = 1 LIMIT 1",
+                (profile_id,),
+            ).fetchone()
+        finally:
+            connection.close()
+        if row is None:
+            return None
+        return {
+            "state_bytes": bytes(row["state_bytes"]),
+            "bytes_sha256": row["bytes_sha256"],
+            "feature_names": row["feature_names"],
+            "trained_at": row["trained_at"],
+            "model_version": row["model_version"],
+        }
+
+
+def _readonly_connection(db_path: Path) -> sqlite3.Connection:
+    connection = sqlite3.connect(
+        f"file:{db_path}?mode=ro", uri=True, timeout=1.0,
+    )
+    connection.execute("PRAGMA busy_timeout=1000")
+    connection.row_factory = sqlite3.Row
+    return connection
+
+
+def sqlite_status(exc: sqlite3.Error) -> str:
+    """Return a stable, non-sensitive dashboard status for SQLite failures."""
+    message = str(exc).lower()
+    return (
+        "database_busy"
+        if "locked" in message or "busy" in message
+        else "query_error"
+    )
+
+
+def _table_exists(connection: sqlite3.Connection, table: str) -> bool:
+    row = connection.execute(
+        "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
+        (table,),
+    ).fetchone()
+    return row is not None
+
+
+def load_source_quality_state(profile_id: str, db_path: Path) -> dict:
+    """Read bounded source scores plus the profile's full source count."""
+    if not db_path.exists():
+        return {
+            "scores": {}, "tracked_sources": 0,
+            "status": "missing_database",
+        }
+    try:
+        connection = _readonly_connection(db_path)
+        try:
+            if not _table_exists(connection, "source_quality"):
+                return {
+                    "scores": {}, "tracked_sources": 0,
+                    "status": "missing_table",
+                }
+            count_row = connection.execute(
+                "SELECT COUNT(DISTINCT source_id) AS count "
+                "FROM source_quality WHERE profile_id = ? "
+                "AND source_id IS NOT NULL AND source_id != ''",
+                (profile_id,),
+            ).fetchone()
+            rows = connection.execute(
+                "SELECT source_id, alpha, beta FROM source_quality "
+                "WHERE profile_id = ? "
+                "ORDER BY updated_at DESC LIMIT ?",
+                (profile_id, SOURCE_SCORE_LIMIT),
+            ).fetchall()
+        finally:
+            connection.close()
+    except sqlite3.Error as exc:
+        status = sqlite_status(exc)
+        logger.warning("source quality telemetry %s: %s", status, exc)
+        return {"scores": {}, "tracked_sources": 0, "status": status}
+
+    scores: dict[str, float] = {}
+    for row in rows:
+        alpha, beta = float(row["alpha"] or 0), float(row["beta"] or 0)
+        source_id = str(row["source_id"] or "")
+        if alpha + beta > 0 and source_id and source_id not in scores:
+            scores[source_id] = round(alpha / (alpha + beta), 4)
+    return {
+        "scores": scores,
+        "tracked_sources": int(count_row["count"] if count_row else 0),
+        "status": "available",
+    }
+
+
+def load_model_state(profile_id: str, db_path: Path) -> dict:
+    """Count persisted model artifacts without inferring them from signals."""
+    if not db_path.exists():
+        return {"models_trained": 0, "status": "missing_database"}
+    try:
+        connection = _readonly_connection(db_path)
+        try:
+            if not _table_exists(connection, "learning_model_state"):
+                return {"models_trained": 0, "status": "missing_table"}
+            row = connection.execute(
+                "SELECT COUNT(*) AS count FROM learning_model_state "
+                "WHERE profile_id = ?",
+                (profile_id,),
+            ).fetchone()
+        finally:
+            connection.close()
+    except sqlite3.Error as exc:
+        status = sqlite_status(exc)
+        logger.warning("model-state telemetry %s: %s", status, exc)
+        return {"models_trained": 0, "status": status}
+    return {
+        "models_trained": int(row["count"] if row else 0),
+        "status": "available",
+    }

package/src/superlocalmemory/server/routes/memories.py

@@ -507,9 +507,14 @@ async def search_memories(request: Request, body: SearchRequest):
         # a stalled connection and aborts with "signal is aborted without reason"
         # before the response arrives. Fix: run in a thread-pool executor so the
         # event loop stays alive to send keepalive frames.
-        # v3.4.64 (regression fix): fast=True skips spreading_activation + agentic
-        # LLM rounds (saves ~7s on cold graph traversal). fast=False is the SLOW
-        # path that enables both; the earlier comment had this inverted.
+        # v3.8.2: fast=True — the dashboard search BOX is a snappy retrieval
+        # list (all six local channels + reranker), never the internal agentic
+        # LLM round, which would reintroduce the multi-second hang this endpoint
+        # is regression-tested against (test_search_fast_param_and_profile_isolation).
+        # The human-facing LLM synthesis lives on separate paths that are NOT the
+        # search list: the "ask" memory-chat (/api/v3/chat/stream, Ollama Mode B)
+        # and the precomputed knowledge-cluster summaries (core.community_summary,
+        # Mode B/C). So search stays fast; synthesis is where the LLM adds value.
         import asyncio
         import time as _time
         engine = _get_engine(request)

package/src/superlocalmemory/server/routes/mesh.py

@@ -11,15 +11,18 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 
 from __future__ import annotations
 
-import asyncio
 import re
-from typing import Optional
+from datetime import datetime, timedelta, timezone
 
 from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel
 
 router = APIRouter(prefix="/mesh", tags=["mesh"])
 
+_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
+_STALE_AFTER = timedelta(minutes=5)
+_EXPIRE_AFTER = timedelta(minutes=30)
+
 
 # -- Request models --
 
@@ -89,6 +92,7 @@ def _get_broker(request: Request):
         client_host = request.client.host if request.client else ""
         if client_host not in ("127.0.0.1", "::1", "localhost"):
             import hmac
+
             from superlocalmemory.core.security_primitives import verify_install_token
 
             # Path 1: install token — dashboard/browser callers hold this and
@@ -154,7 +158,7 @@ def _reject_secret_state(key: str, value: str) -> None:
 # -- Routes --
 
 @router.post("/register")
-async def register(req: RegisterRequest, request: Request):
+def register(req: RegisterRequest, request: Request):
     broker = _get_broker(request)
     if not req.session_id:
         raise HTTPException(400, detail="session_id required")
@@ -165,7 +169,7 @@ async def register(req: RegisterRequest, request: Request):
 
 
 @router.post("/deregister")
-async def deregister(req: DeregisterRequest, request: Request):
+def deregister(req: DeregisterRequest, request: Request):
     broker = _get_broker(request)
     result = broker.deregister_peer(req.peer_id, profile_id=_active_profile())
     if not result.get("ok"):
@@ -185,7 +189,8 @@ def _peer_activity_counts(request: Request, session_ids: list[str]) -> dict:
         return {}
     try:
         from superlocalmemory.server.routes.helpers import (
-            get_engine_lazy, get_active_profile,
+            get_active_profile,
+            get_engine_lazy,
         )
         engine = get_engine_lazy(request.app.state)
         if engine is None:
@@ -216,21 +221,102 @@ def _peer_activity_counts(request: Request, session_ids: list[str]) -> dict:
         return {}
 
 
+def _parse_heartbeat(value: object) -> datetime | None:
+    """Parse a stored heartbeat as UTC without trusting malformed values."""
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+def _mesh_read_model(records: list[dict]) -> tuple[list[dict], list[dict]]:
+    """Return bounded remote peers and local sessions with read-time liveness.
+
+    The broker persists local agent sessions in ``mesh_peers`` for messaging.
+    That storage detail must not make them look like remote mesh neighbours in
+    the dashboard. Classification is read-only so an ordinary dashboard GET
+    neither mutates a live database nor waits for the five-minute cleanup loop.
+    """
+    now = datetime.now(timezone.utc)
+    remote: list[dict] = []
+    local: list[dict] = []
+    for record in records:
+        heartbeat = _parse_heartbeat(record.get("last_heartbeat"))
+        if heartbeat is None:
+            continue
+        age = now - heartbeat
+        if age >= _EXPIRE_AFTER:
+            continue
+        stale_at = heartbeat + _STALE_AFTER
+        expires_at = heartbeat + _EXPIRE_AFTER
+        status = "active" if age < _STALE_AFTER else "stale"
+        normalized = {
+            **record,
+            "status": status,
+            "stale_at": stale_at.isoformat(),
+            "expires_at": expires_at.isoformat(),
+        }
+        if str(record.get("host") or "").lower() in _LOOPBACK_HOSTS:
+            local.append(normalized)
+        else:
+            remote.append(normalized)
+    return remote, local
+
+
+def _mesh_counts(remote: list[dict], local: list[dict]) -> dict:
+    """Expose active counts separately while retaining the legacy total key."""
+    active_remote = sum(peer["status"] == "active" for peer in remote)
+    active_local = sum(session["status"] == "active" for session in local)
+    return {
+        "peer_count": active_remote + active_local,
+        "active_peer_count": active_remote + active_local,
+        "remote_peer_count": active_remote,
+        "local_session_count": active_local,
+        "stale_peer_count": sum(peer["status"] == "stale" for peer in remote),
+        "stale_local_session_count": sum(session["status"] == "stale" for session in local),
+    }
+
+
 @router.get("/peers")
-async def peers(request: Request):
+def peers(request: Request, view: str = "all"):
     broker = _get_broker(request)
-    peer_list = broker.list_all_peers(_active_profile())
+    if view not in {"all", "remote", "local"}:
+        raise HTTPException(422, detail="view must be all, remote, or local")
+    remote_peers, local_sessions = _mesh_read_model(
+        broker.list_all_peers(_active_profile()),
+    )
+    peer_list = (
+        remote_peers if view == "remote" else
+        local_sessions if view == "local" else
+        [*remote_peers, *local_sessions]
+    )
     session_ids = [p.get("session_id") for p in peer_list if p.get("session_id")]
     counts = _peer_activity_counts(request, session_ids)
+    enriched = []
     for p in peer_list:
         c = counts.get(p.get("session_id"), {})
-        p["tool_count"] = c.get("tool_count", 0)
-        p["memory_count"] = c.get("memory_count", 0)
-    return {"peers": peer_list}
+        enriched.append({
+            **p,
+            "tool_count": c.get("tool_count", 0),
+            "memory_count": c.get("memory_count", 0),
+        })
+    return {
+        "peers": enriched,
+        "remote_peers": remote_peers,
+        "local_sessions": local_sessions,
+        "view": view,
+        **_mesh_counts(remote_peers, local_sessions),
+    }
 
 
 @router.post("/heartbeat")
-async def heartbeat(req: HeartbeatRequest, request: Request):
+def heartbeat(req: HeartbeatRequest, request: Request):
+    """Update peer liveness without blocking the daemon's async event loop."""
     broker = _get_broker(request)
     result = broker.heartbeat(req.peer_id, profile_id=_active_profile())
     if not result.get("ok"):
@@ -239,7 +325,7 @@ async def heartbeat(req: HeartbeatRequest, request: Request):
 
 
 @router.post("/summary")
-async def summary(req: SummaryRequest, request: Request):
+def summary(req: SummaryRequest, request: Request):
     broker = _get_broker(request)
     result = broker.update_summary(req.peer_id, req.summary,
                                    profile_id=_active_profile())
@@ -249,21 +335,17 @@ async def summary(req: SummaryRequest, request: Request):
 
 
 @router.post("/send")
-async def send(req: SendRequest, request: Request):
+def send(req: SendRequest, request: Request):
     broker = _get_broker(request)
     to_target = req.to_peer or req.to  # v3.4.6: accept both field names
     if not to_target:
         raise HTTPException(400, detail="'to' or 'to_peer' required")
-    # Resolve the tenant on the event loop (the request ContextVar is set here);
-    # a worker thread would not inherit it.
     profile = _active_profile()
-    # send_message may make a blocking httpx call (up to 10s) when delivering
-    # to a remote peer. Offload to a worker thread so a slow/dead peer network
-    # never stalls the daemon event loop for all other users. The broker opens
-    # a fresh SQLite connection per call, so this is thread-safe.
-    result = await asyncio.to_thread(
-        broker.send_message, req.from_peer, to_target, req.content,
-        req.type, "", profile,
+    # This sync FastAPI route already runs in the worker thread pool, so the
+    # broker's SQLite retries and optional remote HTTP delivery cannot block
+    # the daemon event loop.
+    result = broker.send_message(
+        req.from_peer, to_target, req.content, req.type, "", profile,
     )
     if not result.get("ok"):
         status = 413 if "too large" in result.get("error", "") else 404
@@ -272,21 +354,21 @@ async def send(req: SendRequest, request: Request):
 
 
 @router.get("/inbox/{peer_id}")
-async def inbox(peer_id: str, request: Request, project_path: str = ""):
+def inbox(peer_id: str, request: Request, project_path: str = ""):
     broker = _get_broker(request)
     return {"messages": broker.get_inbox(peer_id, project_path,
                                          profile_id=_active_profile())}
 
 
 @router.post("/inbox/{peer_id}/read")
-async def mark_read(peer_id: str, req: ReadRequest, request: Request):
+def mark_read(peer_id: str, req: ReadRequest, request: Request):
     broker = _get_broker(request)
     return broker.mark_read(peer_id, req.message_ids,
                             profile_id=_active_profile())
 
 
 @router.get("/pending/{peer_id}")
-async def pending(peer_id: str, request: Request, project_path: str = ""):
+def pending(peer_id: str, request: Request, project_path: str = ""):
     """Get pending broadcast/project messages for this peer."""
     broker = _get_broker(request)
     messages = broker.get_pending(peer_id, project_path,
@@ -295,13 +377,13 @@ async def pending(peer_id: str, request: Request, project_path: str = ""):
 
 
 @router.get("/state")
-async def state_all(request: Request):
+def state_all(request: Request):
     broker = _get_broker(request)
     return {"state": broker.get_state(profile_id=_active_profile())}
 
 
 @router.post("/state")
-async def state_set(req: StateSetRequest, request: Request):
+def state_set(req: StateSetRequest, request: Request):
     broker = _get_broker(request)
     if not req.key:
         raise HTTPException(400, detail="key required")
@@ -311,7 +393,7 @@ async def state_set(req: StateSetRequest, request: Request):
 
 
 @router.get("/state/{key}")
-async def state_get(key: str, request: Request):
+def state_get(key: str, request: Request):
     broker = _get_broker(request)
     result = broker.get_state_key(key, profile_id=_active_profile())
     if result is None:
@@ -320,7 +402,7 @@ async def state_get(key: str, request: Request):
 
 
 @router.post("/lock")
-async def lock(req: LockRequest, request: Request):
+def lock(req: LockRequest, request: Request):
     broker = _get_broker(request)
     if not req.file_path or not req.locked_by:
         raise HTTPException(400, detail="file_path and locked_by required")
@@ -331,12 +413,19 @@ async def lock(req: LockRequest, request: Request):
 
 
 @router.get("/events")
-async def events(request: Request):
+def events(request: Request):
     broker = _get_broker(request)
     return {"events": broker.get_events(profile_id=_active_profile())}
 
 
 @router.get("/status")
-async def status(request: Request):
+def status(request: Request):
     broker = _get_broker(request)
-    return broker.get_status(profile_id=_active_profile())
+    broker_status = broker.get_status(profile_id=_active_profile())
+    remote_peers, local_sessions = _mesh_read_model(
+        broker.list_all_peers(_active_profile()),
+    )
+    return {
+        **broker_status,
+        **_mesh_counts(remote_peers, local_sessions),
+    }

package/src/superlocalmemory/server/routes/ratelimit.py

@@ -33,9 +33,9 @@ from superlocalmemory.infra.rate_limiter import (
     set_limits,
 )
 from superlocalmemory.server.routes.config_api import (
-    _atomic_write,
     _read_config,
     _require_admin,
+    _update_config,
 )
 
 logger = logging.getLogger(__name__)
@@ -81,22 +81,26 @@ def load_persisted_limits() -> None:
         logger.debug("load_persisted_limits skipped: %s", exc)
 
 
+def _require_read(request: Request) -> None:
+    from superlocalmemory.access.rbac import Permission
+    from superlocalmemory.server.rbac_enforce import require_permission
+    from superlocalmemory.server.routes.helpers import get_active_profile
+
+    require_permission(request, Permission.READ, profile=get_active_profile())
+
+
 @router.get("/ratelimit")
-def get_ratelimit() -> JSONResponse:
+def get_ratelimit(request: Request) -> JSONResponse:
+    _require_read(request)
     return JSONResponse(_effective())
 
 
 @router.put("/ratelimit")
-async def put_ratelimit(request: Request) -> JSONResponse:
+def put_ratelimit(
+    request: Request,
+    payload: RateLimitUpdate,
+) -> JSONResponse:
     _require_admin(request)
-    try:
-        raw = await request.json()
-    except Exception:
-        raw = {}
-    try:
-        payload = RateLimitUpdate(**(raw or {}))
-    except Exception as exc:
-        return JSONResponse(status_code=422, content={"error": str(exc)})
 
     if payload.write is None and payload.read is None and payload.window is None:
         return JSONResponse(
@@ -104,21 +108,25 @@ async def put_ratelimit(request: Request) -> JSONResponse:
             content={"error": "Provide at least one of write, read, window."},
         )
 
-    # Runtime apply (reconfigures every live limiter — no restart).
-    effective = set_limits(
-        write=payload.write, read=payload.read, window=payload.window,
-    )
-
-    # Persist so the override survives restart.
+    current = get_limits()
+    requested = {
+        "write": payload.write if payload.write is not None else current["write"],
+        "read": payload.read if payload.read is not None else current["read"],
+        "window": payload.window if payload.window is not None else current["window"],
+    }
+    # Persist first. A response must never say success for a runtime-only
+    # setting that silently disappears after restart.
     try:
-        cfg = _read_config()
-        cfg[_CONFIG_KEY] = {
-            "write": effective["write"],
-            "read": effective["read"],
-            "window": effective["window"],
-        }
-        _atomic_write(cfg)
+        _update_config(
+            lambda cfg: cfg.update({_CONFIG_KEY: dict(requested)}),
+        )
     except Exception as exc:
-        logger.warning("rate-limit persist failed (applied at runtime): %s", exc)
+        logger.warning("rate-limit persist failed: %s", exc)
+        return JSONResponse(
+            status_code=500,
+            content={"error": "Rate-limit configuration was not persisted."},
+        )
 
+    # Runtime apply reconfigures every live limiter; no restart is required.
+    set_limits(**requested)
     return JSONResponse({"success": True, **_effective()})