superlocalmemory 3.8.3 → 3.8.6

package/src/superlocalmemory/mcp/tools_core.py

@@ -20,7 +20,7 @@ from typing import Callable
 from mcp.types import ToolAnnotations
 
 from superlocalmemory.core.config import CANONICAL_RECALL_LIMIT
-from superlocalmemory.infra.data_root import canonical_data_root, state_path
+from superlocalmemory.infra.data_root import state_path
 from superlocalmemory.mcp.shared import authorize_mcp_mutation
 
 logger = logging.getLogger(__name__)
@@ -58,67 +58,6 @@ def _emit_event(event_type: str, payload: dict | None = None,
         pass
 
 
-def _record_recall_hits(
-    get_engine: Callable,
-    query: str,
-    results: list[dict],
-    *,
-    profile_id: str = "",
-    query_id: str = "",
-    fact_ids_candidates: list[str] | None = None,
-) -> None:
-    """Record honest shown-state signals (LLD-02 §4.9).
-
-    v3.4.22: No more fake positives. For every candidate we enqueue a
-    ``shown`` / ``not_shown`` flip based on whether it was returned in the
-    top-K presented to the user. Outcome/reward arrives in v3.4.22 via the
-    action-outcomes pipeline.
-
-    Non-blocking: all work funnels through ``signals.enqueue_shown_flip``
-    (module-level queue + background drain). Failures are swallowed —
-    signal quality is never load-bearing on recall correctness.
-    """
-    try:
-        from superlocalmemory.learning.signals import (
-            LearningSignals,
-            enqueue_shown_flip,
-        )
-
-        pid = profile_id
-        if not pid:
-            pid = get_engine().profile_id
-        slm_dir = canonical_data_root()
-
-        shown_ids = [r.get("fact_id", "") for r in results[:10]
-                     if r.get("fact_id")]
-        candidates = (fact_ids_candidates
-                      if fact_ids_candidates is not None
-                      else shown_ids)
-        if not candidates:
-            return
-
-        # Shown-flip enqueue per §4.9. No synthetic positives.
-        shown_set = set(shown_ids)
-        if query_id:
-            for fid in candidates:
-                enqueue_shown_flip(query_id, fid, shown=(fid in shown_set))
-
-        # Legacy zero-cost signals — unchanged (co-retrieval + confidence).
-        try:
-            signals = LearningSignals(slm_dir / "learning.db")
-            signals.record_co_retrieval(pid, shown_ids)
-        except Exception:
-            pass
-        try:
-            mem_db = str(slm_dir / "memory.db")
-            for fid in shown_ids[:5]:
-                LearningSignals.boost_confidence(mem_db, fid)
-        except Exception:
-            pass
-    except Exception:
-        pass
-
-
 def register_core_tools(server, get_engine: Callable) -> None:
     """Register the 13 core MCP tools on *server*."""
 
@@ -212,10 +151,10 @@ def register_core_tools(server, get_engine: Callable) -> None:
                         await _asyncio.sleep(0.05 * (attempt + 1))
                 return {
                     "success": False,
+                    "code": "DAEMON_UNAVAILABLE",
                     "retryable": True,
                     "error": (
-                        "Canonical daemon is temporarily unavailable; retry the "
-                        "same remember operation without starting a second writer."
+                        "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later."
                     ),
                 }
         except Exception as dexc:
@@ -241,11 +180,22 @@ def register_core_tools(server, get_engine: Callable) -> None:
                 worker_meta,
             )
             if not isinstance(stored, dict) or not stored.get("ok"):
-                raise RuntimeError(
-                    (stored or {}).get("error", "canonical worker store failed")
-                    if isinstance(stored, dict)
-                    else "canonical worker returned an invalid response"
-                )
+                if isinstance(stored, dict) and stored.get("code") == "DAEMON_UNAVAILABLE":
+                    return {
+                        "success": False,
+                        "code": "DAEMON_UNAVAILABLE",
+                        "retryable": True,
+                        "error": stored.get(
+                            "error",
+                            "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later.",
+                        ),
+                    }
+                return {
+                    "success": False,
+                    "code": "DAEMON_UNAVAILABLE",
+                    "retryable": True,
+                    "error": "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later.",
+                }
             fact_ids = list(stored.get("fact_ids") or [])
             materialization_state = str(
                 stored.get("materialization_state") or "complete"
@@ -275,9 +225,14 @@ def register_core_tools(server, get_engine: Callable) -> None:
                     else "Queryable now; canonical enrichment is still running."
                 ),
             }
-        except Exception as exc:
+        except Exception:
             logger.exception("remember failed")
-            return {"success": False, "error": str(exc)}
+            return {
+                "success": False,
+                "code": "DAEMON_UNAVAILABLE",
+                "retryable": True,
+                "error": "DAEMON_UNAVAILABLE: owned daemon is unavailable; retry later.",
+            }
 
     @server.tool(annotations=ToolAnnotations(readOnlyHint=True))
     async def recall(
@@ -378,22 +333,6 @@ def register_core_tools(server, get_engine: Callable) -> None:
                 window=window or None,
             )
             if result.get("ok"):
-                # Record implicit feedback: every returned result is a recall_hit
-                try:
-                    _record_recall_hits(
-                        get_engine,
-                        query,
-                        result.get("results", []),
-                        profile_id=str(result.get("profile", "")),
-                    )
-                except Exception:
-                    pass  # Feedback is non-critical, never block recall
-                _emit_event("memory.recalled", {
-                    "query": query[:80],
-                    "result_count": result.get("result_count", 0),
-                    "query_type": result.get("query_type", "unknown"),
-                    "agent_id": agent_id,
-                }, source_agent=agent_id)
                 return {
                     "success": True,
                     "results": result.get("results", []),

package/src/superlocalmemory/mcp/tools_evolution.py

@@ -16,13 +16,11 @@ from __future__ import annotations
 
 import json
 import logging
-import sqlite3
-from datetime import datetime, timezone
-from pathlib import Path
 from typing import Callable
 
 from mcp.types import ToolAnnotations
 from superlocalmemory.infra.data_root import state_path
+from superlocalmemory.storage.read_connection import ReadConnectionFactory
 
 logger = logging.getLogger(__name__)
 
@@ -146,10 +144,8 @@ def register_evolution_tools(server, get_engine: Callable) -> None:
         try:
             engine = get_engine()
             profile_id = engine.profile_id if engine else "default"
-            db_path = str(state_path("memory.db"))
-
-            conn = sqlite3.connect(db_path, timeout=10)
-            conn.row_factory = sqlite3.Row
+            db_path = state_path("memory.db")
+            conn = ReadConnectionFactory(db_path).open()
 
             # Gather per-skill invocation stats from tool_events
             # Skills are logged as tool_name='Skill' with actual skill name in input_summary
@@ -274,9 +270,8 @@ def register_evolution_tools(server, get_engine: Callable) -> None:
             skill_name: Specific skill name (empty = all skills)
         """
         try:
-            db_path = str(state_path("memory.db"))
-            conn = sqlite3.connect(db_path, timeout=10)
-            conn.row_factory = sqlite3.Row
+            db_path = state_path("memory.db")
+            conn = ReadConnectionFactory(db_path).open()
 
             if skill_name:
                 rows = conn.execute(

package/src/superlocalmemory/optimize/proxy/capture.py

@@ -9,7 +9,8 @@ Activation: set ``SLM_OPTIMIZE_CAPTURE=1`` in the daemon's environment. When on:
     at load time (see server._load_hooks), so capture never observes a mutated
     request or a cache hit; every line is a genuine upstream exchange.
   * each completed exchange is appended as one JSON line to
-    ``~/.superlocalmemory/optimize_capture.jsonl`` (0600, gitignored).
+    ``~/.superlocalmemory/optimize_capture.jsonl`` (owner-only: POSIX 0600 or
+    Windows owner DACL, gitignored).
 
 ISOLATION GUARANTEE: this module writes ONLY to optimize_capture.jsonl. It never
 opens memory.db, llmcache.db, or any SLM memory store. (Plan §9 hard rule.)
@@ -21,9 +22,11 @@ swallowed — it MUST NOT break the proxied request the user is waiting on.
 from __future__ import annotations
 
 import asyncio
+import errno
 import json
 import logging
 import os
+import stat
 import threading
 from pathlib import Path
 from typing import Any
@@ -55,6 +58,180 @@ def _capture_path() -> Path:
     return state_path(_CAPTURE_FILENAME)
 
 
+def _is_windows() -> bool:
+    """Return whether the running platform uses Windows DACLs."""
+    return os.name == "nt"
+
+
+def _is_link_or_reparse(info: os.stat_result) -> bool:
+    """Reject POSIX symlinks and Windows reparse points before capture."""
+    reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0)
+    attributes = getattr(info, "st_file_attributes", 0)
+    return stat.S_ISLNK(info.st_mode) or bool(reparse and attributes & reparse)
+
+
+def _windows_owner_dacl(
+    win32api: Any,
+    win32con: Any,
+    win32security: Any,
+) -> Any:
+    """Build one protected owner-only DACL for a Windows capture file."""
+    import ntsecuritycon
+
+    token = win32security.OpenProcessToken(
+        win32api.GetCurrentProcess(),
+        win32con.TOKEN_QUERY,
+    )
+    try:
+        owner_sid = win32security.GetTokenInformation(
+            token,
+            win32security.TokenUser,
+        )[0]
+    finally:
+        token.Close()
+    dacl = win32security.ACL()
+    dacl.AddAccessAllowedAce(
+        win32security.ACL_REVISION,
+        ntsecuritycon.FILE_ALL_ACCESS,
+        owner_sid,
+    )
+    return dacl
+
+
+def _open_windows_capture_append(path: Path) -> int:
+    """Create/open a Windows file with append + WRITE_DAC on one handle."""
+    try:
+        import msvcrt
+
+        import ntsecuritycon
+        import win32api
+        import win32con
+        import win32file
+        import win32security
+    except ImportError as exc:
+        raise OSError("Windows capture ACL support is unavailable") from exc
+
+    handle = None
+    try:
+        # A creation-time descriptor prevents a newly created capture file from
+        # ever inheriting a broader parent DACL. For an existing file Windows
+        # ignores this descriptor; _enforce_owner_only_permissions replaces
+        # that DACL through the same WRITE_DAC-capable handle before writing.
+        dacl = _windows_owner_dacl(win32api, win32con, win32security)
+        security_attributes = win32security.SECURITY_ATTRIBUTES()
+        security_attributes.bInheritHandle = False
+        security_attributes.SECURITY_DESCRIPTOR.SetSecurityDescriptorDacl(
+            1,
+            dacl,
+            0,
+        )
+        security_attributes.SECURITY_DESCRIPTOR.SetSecurityDescriptorControl(
+            win32security.SE_DACL_PROTECTED,
+            win32security.SE_DACL_PROTECTED,
+        )
+        handle = win32file.CreateFile(
+            os.fspath(path),
+            ntsecuritycon.FILE_APPEND_DATA | ntsecuritycon.WRITE_DAC,
+            win32con.FILE_SHARE_READ
+            | win32con.FILE_SHARE_WRITE
+            | win32con.FILE_SHARE_DELETE,
+            security_attributes,
+            win32con.OPEN_ALWAYS,
+            win32con.FILE_ATTRIBUTE_NORMAL
+            # pywin32 does not export this SDK constant from win32con on
+            # every supported Python build. Keep the Microsoft-defined value
+            # as a named fallback rather than silently following a reparse.
+            | getattr(win32con, "FILE_FLAG_OPEN_REPARSE_POINT", 0x00200000),
+            None,
+        )
+        file_info = win32file.GetFileInformationByHandle(handle)
+        if file_info[0] & win32con.FILE_ATTRIBUTE_REPARSE_POINT:
+            raise OSError(
+                errno.ELOOP,
+                "capture path is a Windows reparse point",
+                path,
+            )
+        try:
+            # Enforce the DACL while this is still the original CreateFile
+            # handle carrying WRITE_DAC. Transferring it into Python's CRT
+            # first can lose the authority SetSecurityInfo needs on Windows.
+            win32security.SetSecurityInfo(
+                handle,
+                win32security.SE_FILE_OBJECT,
+                win32security.DACL_SECURITY_INFORMATION
+                | win32security.PROTECTED_DACL_SECURITY_INFORMATION,
+                None,
+                None,
+                dacl,
+                None,
+            )
+        except Exception as exc:
+            raise OSError(
+                "Windows capture ACL could not be enforced "
+                f"({type(exc).__name__}: {exc})"
+            ) from exc
+
+        # Transfer the native handle to Python's CRT descriptor exactly once.
+        raw_handle = handle.Detach()
+        handle = None
+        try:
+            return msvcrt.open_osfhandle(
+                raw_handle,
+                os.O_WRONLY | os.O_APPEND | getattr(os, "O_BINARY", 0),
+            )
+        except BaseException:
+            win32api.CloseHandle(raw_handle)
+            raise
+    except OSError:
+        raise
+    except Exception as exc:
+        raise OSError(f"Windows secure capture open failed: {exc}") from exc
+    finally:
+        if handle is not None:
+            handle.Close()
+
+
+def _open_capture_append(path: Path) -> int:
+    """Open one append descriptor without following or racing a link target."""
+    try:
+        before = path.lstat()
+    except FileNotFoundError:
+        before = None
+    if before is not None and _is_link_or_reparse(before):
+        raise OSError(errno.ELOOP, "capture path is a link or reparse point", path)
+
+    if _is_windows():
+        fd = _open_windows_capture_append(path)
+    else:
+        flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0)
+        fd = os.open(path, flags, 0o600)
+    try:
+        opened = os.fstat(fd)
+        current = path.lstat()
+        if (
+            _is_link_or_reparse(current)
+            or (opened.st_dev, opened.st_ino) != (current.st_dev, current.st_ino)
+        ):
+            raise OSError(
+                errno.ELOOP,
+                "capture path changed during secure open",
+                path,
+            )
+    except BaseException:
+        os.close(fd)
+        raise
+    return fd
+
+
+def _enforce_owner_only_permissions(fd: int) -> None:
+    """Apply owner-only access control through the already-verified handle."""
+    if not _is_windows():
+        os.fchmod(fd, 0o600)
+        return
+    # _open_windows_capture_append applies the protected DACL on the original
+    # WRITE_DAC-capable CreateFile handle before CRT descriptor transfer.
+
+
 class ShadowCapture:
     """Thread-safe append-only JSONL writer for proxy exchanges (singleton)."""
 
@@ -95,10 +272,10 @@ class ShadowCapture:
         Fail-open: any error is logged and False is returned; never raised.
 
         Security: opens with a single ``os.open`` carrying ``O_CREAT |
-        O_APPEND | O_NOFOLLOW`` and mode ``0o600`` on EVERY write. O_NOFOLLOW
-        refuses a symlink pre-placed at the path (symlink-append attack), and
-        the unconditional 0600-on-create removes the stat/exists TOCTOU that
-        could otherwise drop the file to the process umask.
+        O_APPEND`` and mode ``0o600`` on every write. POSIX adds O_NOFOLLOW;
+        all platforms reject link/reparse metadata and verify that the opened
+        descriptor still identifies the current path. Permissions are then
+        enforced through that verified descriptor before any data is written.
         """
         try:
             line = json.dumps(entry, ensure_ascii=False, separators=(",", ":"))
@@ -109,9 +286,20 @@ class ShadowCapture:
         try:
             with self._write_lock:
                 self._path.parent.mkdir(parents=True, exist_ok=True)
-                flags = os.O_CREAT | os.O_WRONLY | os.O_APPEND | getattr(os, "O_NOFOLLOW", 0)
-                fd = os.open(self._path, flags, 0o600)
-                with os.fdopen(fd, "a", encoding="utf-8") as fh:
+                fd = _open_capture_append(self._path)
+                try:
+                    # Enforce the ACL through the verified descriptor so a
+                    # pathname swap cannot redirect chmod/icacls elsewhere.
+                    _enforce_owner_only_permissions(fd)
+                    # fdopen transfers descriptor ownership only after it
+                    # returns successfully. Keep this conversion inside the
+                    # explicit-close boundary so an allocation/codec failure
+                    # cannot leak one descriptor per captured request.
+                    fh = os.fdopen(fd, "a", encoding="utf-8")
+                except BaseException:
+                    os.close(fd)
+                    raise
+                with fh:
                     fh.write(line + "\n")
                 self._count += 1
             return True

package/src/superlocalmemory/retrieval/engine.py

@@ -904,13 +904,18 @@ class RetrievalEngine:
 
         return out
 
-    def close(self) -> None:
-        """Release the channel workers owned by this retrieval engine."""
+    def close(self, *, wait: bool = False) -> None:
+        """Release owned channel workers without blocking daemon shutdown.
+
+        Active channel calls have their own response deadline.  Waiting here
+        can still deadlock shutdown when an extension ignores that deadline,
+        so the daemon uses the executor's non-blocking cancellation path.
+        """
         with self._close_lock:
             if self._closed:
                 return
             self._closed = True
-        self._channel_executor.shutdown(wait=True, cancel_futures=True)
+        self._channel_executor.shutdown(wait=wait, cancel_futures=True)
 
     # -- Fact loading -------------------------------------------------------
 
@@ -955,7 +960,13 @@ class RetrievalEngine:
         Blended: alpha * sigmoid(CE_score) + (1 - alpha) * rrf_score.
         Speaker tags stripped before scoring (Bug 3 fix).
         """
-        # Bug 2 fix: score ALL candidates, not just top_k
+        # Bug 2 fix: score ALL candidates, not just top_k. v3.8.5: verified on
+        # the real DB that bounding the CE to the top-N fusion candidates both
+        # (a) gave NO latency win (the cross-encoder batches all pairs in one
+        # forward pass, so 60 vs 184 pairs is within noise) and (b) CHANGED the
+        # top-5 on 4/8 queries — the CE legitimately promotes items ranked below
+        # the fusion top-N into the answer. So exhaustive reranking stays: it is
+        # a quality feature, not the latency bottleneck.
         candidates = [
             (fact_map[fr.fact_id], fr.fused_score)
             for fr in fused if fr.fact_id in fact_map

package/src/superlocalmemory/retrieval/entity_channel.py

@@ -14,6 +14,7 @@ from __future__ import annotations
 
 import json
 import logging
+import os
 import re
 import threading
 from collections import defaultdict
@@ -31,6 +32,24 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
+def _adj_ttl_seconds() -> float:
+    """In-memory adjacency-cache TTL (seconds), env-overridable.
+
+    v3.8.5: raised from a hard-coded 300s to 3600s. The TTL only exists to
+    catch edge-WEIGHT mutations (pruning / MAX-merge) that leave the edge COUNT
+    unchanged — new memories already force a reload via the count check. At 300s
+    a 208K-edge graph rebuilt on the recall hot path every 5 idle minutes,
+    causing a recurring multi-second latency spike. Weight drift is a minor
+    ranking refinement, so a longer TTL trades negligible staleness for a big
+    latency win. Set SLM_ENTITY_ADJ_TTL_S to tune (0 disables time-based reload;
+    the count-based correctness reload always remains).
+    """
+    try:
+        return max(0.0, float(os.environ.get("SLM_ENTITY_ADJ_TTL_S", "3600")))
+    except (TypeError, ValueError):
+        return 3600.0
+
 _PROPER_NOUN_RE = re.compile(r"\b[A-Z][a-z]{1,}\b")
 
 _ENTITY_STOP: frozenset[str] = frozenset({
@@ -156,7 +175,12 @@ class EntityGraphChannel:
         # count-stable window would otherwise serve a stale adjacency map.
         import time as _t_ec
         _now_ec = _t_ec.monotonic()
-        _fresh = (_now_ec - getattr(self, "_adj_loaded_at", 0.0)) < 300.0
+        _ttl = _adj_ttl_seconds()
+        # TTL=0 disables the time-based reload entirely (count-based correctness
+        # reload still applies); otherwise the cache is fresh within the TTL.
+        _fresh = _ttl <= 0.0 or (
+            (_now_ec - getattr(self, "_adj_loaded_at", 0.0)) < _ttl
+        )
         if (self._adj_scope_key == scope_key
                 and (self._adj or self._visible_fact_ids)
                 and self._adj_edge_count == current_count