superlocalmemory 3.8.2 → 3.8.5

package/src/superlocalmemory/server/routes/memories.py

@@ -15,10 +15,30 @@ from .helpers import (
     get_db_connection, dict_factory, get_active_profile, get_engine_lazy,
     SearchRequest, DB_PATH, MEMORY_DIR,
 )
+from superlocalmemory.storage.memory_write import memory_write
 
 logger = logging.getLogger("superlocalmemory.routes.memories")
 router = APIRouter()
 
+# v3.8.3: GENEROUS latency budget for recall. SLM's value is quality recall
+# under heavy multi-agent load, so semantic recall is given ample time to
+# finish — the keyword fallback is a LAST-RESORT safety net for a genuine hang
+# (e.g. a wedged embedder), NOT an aggressive speed cutoff. Only if recall
+# exceeds this budget do we serve the fast keyword search so the caller ALWAYS
+# gets a result instead of hanging forever. Tune with SLM_SEARCH_RECALL_TIMEOUT_S.
+# The dashboard's fetch timeout is set ABOVE this so the browser waits for the
+# quality result rather than aborting early.
+_DEFAULT_RECALL_BUDGET_S = 25.0
+
+
+def _search_recall_timeout_s() -> float:
+    import os
+    try:
+        v = float(os.environ.get("SLM_SEARCH_RECALL_TIMEOUT_S", ""))
+        return v if v > 0 else _DEFAULT_RECALL_BUDGET_S
+    except (TypeError, ValueError):
+        return _DEFAULT_RECALL_BUDGET_S
+
 
 def _internal_error(detail: str = "Internal server error") -> HTTPException:
     """SEC-H-02: log the full traceback server-side; return a generic message.
@@ -526,35 +546,65 @@ async def search_memories(request: Request, body: SearchRequest):
             _window = getattr(body, "window", None) or ""
             if not _window and getattr(body, "date_from", None) and getattr(body, "date_to", None):
                 _window = f"{body.date_from}..{body.date_to}"
-            response = await loop.run_in_executor(
+            # v3.8.3: bound the synchronous recall. Under a concurrent
+            # maintenance pass or a busy embedder it can run tens of seconds
+            # and the browser aborts the fetch. If it exceeds the budget we
+            # fall through to the fast keyword search below, so the dashboard
+            # ALWAYS returns instead of failing with an abort.
+            _recall_future = loop.run_in_executor(
                 None,
                 lambda: engine.recall(
                     body.query, limit=body.limit, fast=True,
                     window=_window or None,
                 ),
             )
-            elapsed_ms = round((_time.monotonic() - t0) * 1000, 1)
-            from superlocalmemory.server.recall_serializer import (
-                recall_response_metadata,
-                serialize_recall_response,
-            )
-            results, no_confident_match = serialize_recall_response(
-                response,
-                limit=body.limit,
-                per_fact_max=300,
-                total_max=max(300, body.limit * 300),
-            )
-            return {
-                "query": body.query,
-                "results": results,
-                "total": len(results),
-                "query_type": getattr(response, "query_type", "semantic"),
-                "retrieval_time_ms": elapsed_ms,
-                "no_confident_match": no_confident_match,
-                **recall_response_metadata(response),
-            }
-
-        # Fallback: direct DB text search (engine not yet initialised)
+            # A run_in_executor thread cannot be cancelled, and wait_for() on it
+            # blocks until the thread finishes (defeating the timeout). So poll
+            # the future without blocking the event loop and give up at the
+            # deadline — the orphaned recall completes in the background and its
+            # result is discarded. This is what bounds dashboard-search latency.
+            _budget = _search_recall_timeout_s()
+            _deadline = loop.time() + _budget
+            while not _recall_future.done() and loop.time() < _deadline:
+                await asyncio.sleep(0.05)
+            if _recall_future.done():
+                response = _recall_future.result()
+            else:
+                # Ensure the orphaned future's eventual result/exception is
+                # retrieved so asyncio doesn't log "never retrieved".
+                _recall_future.add_done_callback(
+                    lambda f: (f.cancelled() or f.exception())
+                )
+                logger.warning(
+                    "search_memories: semantic recall exceeded %.0fs budget for "
+                    "%r — serving keyword fallback",
+                    _budget, (body.query or "")[:80],
+                )
+                response = None
+            if response is not None:
+                elapsed_ms = round((_time.monotonic() - t0) * 1000, 1)
+                from superlocalmemory.server.recall_serializer import (
+                    recall_response_metadata,
+                    serialize_recall_response,
+                )
+                results, no_confident_match = serialize_recall_response(
+                    response,
+                    limit=body.limit,
+                    per_fact_max=300,
+                    total_max=max(300, body.limit * 300),
+                )
+                return {
+                    "query": body.query,
+                    "results": results,
+                    "total": len(results),
+                    "query_type": getattr(response, "query_type", "semantic"),
+                    "retrieval_time_ms": elapsed_ms,
+                    "no_confident_match": no_confident_match,
+                    **recall_response_metadata(response),
+                }
+            # recall timed out — fall through to the fast keyword search.
+
+        # Fallback: direct DB text search (engine not ready OR recall over budget)
         conn = get_db_connection()
         conn.row_factory = dict_factory
         cursor = conn.cursor()
@@ -948,45 +998,43 @@ async def forget_memory(request: Request, fact_id: str):
         request, "delete", fact_id,
     )
     try:
-        conn = get_db_connection()
-        conn.row_factory = dict_factory
-        cursor = conn.cursor()
-        cursor.execute(
-            "SELECT fact_id, content, importance, confidence, "
-            "       canonical_entities_json, embedding, created_at "
-            "FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
-            (fact_id, active_profile),
-        )
-        row = cursor.fetchone()
-        if not row:
-            conn.close()
-            raise HTTPException(status_code=404, detail="Memory not found")
-        # Archive copy — payload_json small enough for the canonical row.
-        payload = {
-            "fact_id": row["fact_id"],
-            "content": row["content"],
-            "canonical_entities_json": row.get("canonical_entities_json"),
-            "importance": row.get("importance"),
-            "confidence": row.get("confidence"),
-            "created_at": row.get("created_at"),
-        }
         from datetime import datetime, timezone
-        archived_at = datetime.now(timezone.utc).isoformat()
         import uuid as _uuid
-        cursor.execute(
-            "INSERT INTO memory_archive "
-            "(archive_id, fact_id, profile_id, payload_json, archived_at, reason) "
-            "VALUES (?, ?, ?, ?, ?, ?)",
-            (str(_uuid.uuid4()), fact_id, active_profile,
-             _json.dumps(payload), archived_at, "user_forget_dashboard"),
-        )
-        cursor.execute(
-            "UPDATE atomic_facts SET archive_status = 'archived' "
-            "WHERE fact_id = ?",
-            (fact_id,),
-        )
-        conn.commit()
-        conn.close()
+        archived_at = datetime.now(timezone.utc).isoformat()
+        archive_id = str(_uuid.uuid4())
+        # memory_write: write lock (in-process) + busy_timeout (cross-process).
+        # SELECT + INSERT + UPDATE are atomic inside the same connection.
+        with memory_write(DB_PATH) as conn:
+            conn.row_factory = dict_factory
+            row = conn.execute(
+                "SELECT fact_id, content, importance, confidence, "
+                "       canonical_entities_json, embedding, created_at "
+                "FROM atomic_facts WHERE fact_id = ? AND profile_id = ?",
+                (fact_id, active_profile),
+            ).fetchone()
+            if not row:
+                raise HTTPException(status_code=404, detail="Memory not found")
+            # Archive copy — payload_json small enough for the canonical row.
+            payload = {
+                "fact_id": row["fact_id"],
+                "content": row["content"],
+                "canonical_entities_json": row.get("canonical_entities_json"),
+                "importance": row.get("importance"),
+                "confidence": row.get("confidence"),
+                "created_at": row.get("created_at"),
+            }
+            conn.execute(
+                "INSERT INTO memory_archive "
+                "(archive_id, fact_id, profile_id, payload_json, archived_at, reason) "
+                "VALUES (?, ?, ?, ?, ?, ?)",
+                (archive_id, fact_id, active_profile,
+                 _json.dumps(payload), archived_at, "user_forget_dashboard"),
+            )
+            conn.execute(
+                "UPDATE atomic_facts SET archive_status = 'archived' "
+                "WHERE fact_id = ?",
+                (fact_id,),
+            )
         engine._hooks.run_post("delete", hook_context)
         return {"success": True, "fact_id": fact_id, "archived_at": archived_at}
     except HTTPException:
@@ -1018,39 +1066,39 @@ async def merge_memory(request: Request, fact_id: str):
             raise HTTPException(400, "'into' exceeds 200-char limit")
         if kept == fact_id:
             raise HTTPException(400, "Cannot merge a fact into itself")
-        conn = get_db_connection()
-        conn.row_factory = dict_factory
-        cursor = conn.cursor()
-        # Both must belong to the active profile.
-        cursor.execute(
-            "SELECT fact_id FROM atomic_facts "
-            "WHERE fact_id IN (?, ?) AND profile_id = ?",
-            (fact_id, kept, active_profile),
-        )
-        found = {r["fact_id"] for r in cursor.fetchall()}
-        if fact_id not in found or kept not in found:
-            conn.close()
-            raise HTTPException(
-                404,
-                "Both fact_ids must exist in the active profile",
-            )
         from datetime import datetime, timezone
         merged_at = datetime.now(timezone.utc).isoformat()
-        cursor.execute(
-            "INSERT INTO memory_merge_log "
-            "(kept_fact_id, merged_fact_id, profile_id, reason, merged_at) "
-            "VALUES (?, ?, ?, ?, ?)",
-            (kept, fact_id, active_profile,
-             "user_merge_dashboard", merged_at),
-        )
-        cursor.execute(
-            "UPDATE atomic_facts "
-            "SET merged_into = ?, archive_status = 'archived' "
-            "WHERE fact_id = ?",
-            (kept, fact_id),
-        )
-        conn.commit()
-        conn.close()
+        # memory_write: write lock (in-process) + busy_timeout (cross-process).
+        # SELECT + INSERT + UPDATE are atomic inside the same connection.
+        with memory_write(DB_PATH) as conn:
+            conn.row_factory = dict_factory
+            # Both must belong to the active profile.
+            found = {
+                r["fact_id"]
+                for r in conn.execute(
+                    "SELECT fact_id FROM atomic_facts "
+                    "WHERE fact_id IN (?, ?) AND profile_id = ?",
+                    (fact_id, kept, active_profile),
+                ).fetchall()
+            }
+            if fact_id not in found or kept not in found:
+                raise HTTPException(
+                    404,
+                    "Both fact_ids must exist in the active profile",
+                )
+            conn.execute(
+                "INSERT INTO memory_merge_log "
+                "(kept_fact_id, merged_fact_id, profile_id, reason, merged_at) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (kept, fact_id, active_profile,
+                 "user_merge_dashboard", merged_at),
+            )
+            conn.execute(
+                "UPDATE atomic_facts "
+                "SET merged_into = ?, archive_status = 'archived' "
+                "WHERE fact_id = ?",
+                (kept, fact_id),
+            )
         engine._hooks.run_post("delete", hook_context)
         return {
             "success": True,

package/src/superlocalmemory/server/routes/mesh.py

@@ -90,7 +90,9 @@ def _get_broker(request: Request):
     secret = getattr(broker, "_shared_secret", None)
     if secret:
         client_host = request.client.host if request.client else ""
-        if client_host not in ("127.0.0.1", "::1", "localhost"):
+        from superlocalmemory.server.loopback import is_loopback as _is_loopback_host
+
+        if not _is_loopback_host(client_host):
             import hmac
 
             from superlocalmemory.core.security_primitives import verify_install_token
@@ -261,7 +263,10 @@ def _mesh_read_model(records: list[dict]) -> tuple[list[dict], list[dict]]:
             "stale_at": stale_at.isoformat(),
             "expires_at": expires_at.isoformat(),
         }
-        if str(record.get("host") or "").lower() in _LOOPBACK_HOSTS:
+        # display-only — not an auth decision; is_loopback() handles IPv4-mapped forms
+        # such as "::ffff:127.0.0.1" that _LOOPBACK_HOSTS misses.
+        from superlocalmemory.server.loopback import is_loopback as _is_loopback
+        if _is_loopback(str(record.get("host") or "").lower()):
             local.append(normalized)
         else:
             remote.append(normalized)

package/src/superlocalmemory/server/routes/profiles.py

@@ -25,6 +25,7 @@ from .helpers import (
     delete_profile_from_db,
     _load_profiles_json, _save_profiles_json,
 )
+from superlocalmemory.storage.memory_write import memory_write
 from superlocalmemory.server.profile_runtime import (
     commit_daemon_profile_switch,
     get_profile_runtime,
@@ -250,28 +251,26 @@ async def delete_profile(name: str, request: Request):
         # profile being deleted (not the active one).
         from superlocalmemory.server.rbac_enforce import require_manage as _rbac_manage
         _rbac_manage(request, profile=name)
-        # Move data to default before deleting (bypasses CASCADE)
-        conn = get_db_connection()
-        cursor = conn.cursor()
+        # Move data to default before deleting (bypasses CASCADE).
+        # memory_write: write lock + busy_timeout — two UPDATEs are atomic.
         moved = 0
-        try:
-            cursor.execute(
-                "UPDATE atomic_facts SET profile_id = 'default' WHERE profile_id = ?",
-                (name,),
-            )
-            moved = cursor.rowcount
-        except Exception:
-            pass
-        try:
-            cursor.execute(
-                "UPDATE memories SET profile_id = 'default' WHERE profile_id = ?",
-                (name,),
-            )
-            moved += cursor.rowcount
-        except Exception:
-            pass
-        conn.commit()
-        conn.close()
+        with memory_write(DB_PATH) as conn:
+            try:
+                cur = conn.execute(
+                    "UPDATE atomic_facts SET profile_id = 'default' WHERE profile_id = ?",
+                    (name,),
+                )
+                moved = cur.rowcount
+            except Exception:
+                pass
+            try:
+                cur2 = conn.execute(
+                    "UPDATE memories SET profile_id = 'default' WHERE profile_id = ?",
+                    (name,),
+                )
+                moved += cur2.rowcount
+            except Exception:
+                pass
 
         # Delete from BOTH stores
         delete_profile_from_db(name)

package/src/superlocalmemory/server/routes/rbac.py

@@ -158,7 +158,6 @@ def _require_authority_over_user(request: Request, target_user_id: str) -> None:
     shares at least one workspace on which the admin holds MANAGE.
     """
     from superlocalmemory.access.rbac import Permission
-    from superlocalmemory.server.rbac_enforce import resolve_principal
 
     principal = resolve_principal(request)
     if principal["kind"] == "owner":

package/src/superlocalmemory/server/routes/tiers.py

@@ -10,6 +10,7 @@ All connections use WAL mode + busy_timeout for concurrency safety.
 """
 
 import logging
+import os
 import re
 import sqlite3
 from contextlib import contextmanager
@@ -19,6 +20,7 @@ from fastapi import APIRouter, HTTPException, Request
 from pydantic import BaseModel, Field
 
 from superlocalmemory.server.route_mutations import authorize_route_mutation
+from superlocalmemory.storage.memory_write import memory_write
 
 from .helpers import DB_PATH, get_active_profile
 
@@ -34,12 +36,24 @@ class PinRequest(BaseModel):
     reason: str = Field(default="", max_length=_MAX_REASON_LENGTH)
 
 
+def _busy_ms() -> int:
+    try:
+        return max(0, int(os.environ.get("SLM_DB_BUSY_TIMEOUT_MS", "10000")))
+    except (TypeError, ValueError):
+        return 10000
+
+
 @contextmanager
 def _db():
-    """Context-managed DB connection with WAL + busy_timeout."""
-    conn = sqlite3.connect(str(DB_PATH))
+    """Context-managed DB connection with WAL + busy_timeout (READ paths only).
+
+    Write paths (pin, unpin) use ``memory_write()`` directly to also acquire
+    the process write lock and prevent in-process SQLITE_BUSY races.
+    """
+    ms = _busy_ms()
+    conn = sqlite3.connect(str(DB_PATH), timeout=ms / 1000.0)
     conn.execute("PRAGMA journal_mode=WAL")
-    conn.execute("PRAGMA busy_timeout=5000")
+    conn.execute(f"PRAGMA busy_timeout={ms}")
     conn.row_factory = sqlite3.Row
     try:
         yield conn
@@ -154,21 +168,20 @@ async def pin_fact_route(
         fact_id=body.fact_id,
     )
 
-    with _db() as conn:
-        try:
+    # memory_write: process write lock + busy_timeout.
+    # SELECT + INSERT + lifecycle update are atomic inside the same connection.
+    try:
+        with memory_write(DB_PATH) as conn:
             # Verify fact exists in this profile
-            c = conn.cursor()
-            c.execute(
+            if conn.execute(
                 "SELECT fact_id FROM atomic_facts "
                 "WHERE fact_id = ? AND profile_id = ?",
                 (body.fact_id, profile_id),
-            )
-            if c.fetchone() is None:
+            ).fetchone() is None:
                 raise HTTPException(
                     status_code=404,
                     detail=f"Fact {body.fact_id[:8]}... not found",
                 )
-
             now = datetime.now(UTC).isoformat()
             conn.execute(
                 "INSERT OR REPLACE INTO pinned_facts "
@@ -180,16 +193,15 @@ async def pin_fact_route(
             set_fact_lifecycle_zone(
                 conn, [body.fact_id], "active", profile_id=profile_id,
             )
-            conn.commit()
-            authorization.complete()
-            return {"success": True, "message": f"Fact {body.fact_id[:8]}... pinned"}
-        except HTTPException:
-            raise
-        except Exception as exc:
-            logger.error("pin_fact failed: %s", exc, exc_info=True)
-            raise HTTPException(
-                status_code=500, detail="Failed to pin fact",
-            ) from None
+        authorization.complete()
+        return {"success": True, "message": f"Fact {body.fact_id[:8]}... pinned"}
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.error("pin_fact failed: %s", exc, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="Failed to pin fact",
+        ) from None
 
 
 @router.post("/api/tiers/unpin")
@@ -213,17 +225,17 @@ async def unpin_fact_route(
         fact_id=body.fact_id,
     )
 
-    with _db() as conn:
-        try:
+    # memory_write: process write lock + busy_timeout.
+    try:
+        with memory_write(DB_PATH) as conn:
             conn.execute(
                 "DELETE FROM pinned_facts WHERE fact_id = ? AND profile_id = ?",
                 (body.fact_id, profile_id),
             )
-            conn.commit()
-            authorization.complete()
-            return {"success": True, "unpinned": True}
-        except Exception as exc:
-            logger.error("unpin_fact failed: %s", exc, exc_info=True)
-            raise HTTPException(
-                status_code=500, detail="Failed to unpin fact",
-            ) from None
+        authorization.complete()
+        return {"success": True, "unpinned": True}
+    except Exception as exc:
+        logger.error("unpin_fact failed: %s", exc, exc_info=True)
+        raise HTTPException(
+            status_code=500, detail="Failed to unpin fact",
+        ) from None