superlocalmemory 3.6.22 → 3.7.0

package/src/superlocalmemory/retrieval/hopfield_channel.py

@@ -19,22 +19,20 @@ Key features:
   - Returns [] on any error (HR-06)
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
 
 import logging
+import threading
 import time
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 import numpy as np
 
 from superlocalmemory.math.hopfield import HopfieldConfig, ModernHopfieldNetwork
-
-if TYPE_CHECKING:
-    from superlocalmemory.retrieval.vector_store import VectorStore
-    from superlocalmemory.storage.database import DatabaseManager
+from superlocalmemory.retrieval.scope_policy import filter_authorized_results
 
 logger = logging.getLogger(__name__)
 
@@ -73,11 +71,13 @@ class HopfieldChannel:
         self._vector_store = vector_store
         self._config = config or HopfieldConfig()
         self._hopfield = ModernHopfieldNetwork(self._config)
+        self._cache_lock = threading.RLock()
 
         # Memory matrix cache (per LLD Section 2.2, HR-09)
         self._cached_matrix: np.ndarray | None = None
         self._cached_fact_ids: list[str] = []
         self._cached_profile: str = ""
+        self._cached_scope_key: tuple[str, bool, bool] | None = None
         self._cached_count: int = 0
         self._cache_timestamp: float = 0.0
 
@@ -104,8 +104,17 @@ class HopfieldChannel:
         if not self._config.enabled:
             return []
 
+        include_global = bool(getattr(self, "include_global", False))
+        include_shared = bool(getattr(self, "include_shared", False))
         try:
-            return self._search_inner(query, profile_id, top_k)
+            with self._cache_lock:
+                return self._search_inner(
+                    query,
+                    profile_id,
+                    top_k,
+                    include_global=include_global,
+                    include_shared=include_shared,
+                )
         except Exception as exc:
             # HR-06: Return [] on any error
             logger.warning("Hopfield channel error: %s", exc)
@@ -118,6 +127,9 @@ class HopfieldChannel:
         query: Any,
         profile_id: str,
         top_k: int,
+        *,
+        include_global: bool = False,
+        include_shared: bool = False,
     ) -> list[tuple[str, float]]:
         """Core search logic, separated for clean error handling."""
         # Step 2: Convert query to numpy
@@ -132,11 +144,18 @@ class HopfieldChannel:
             return []
 
         # Step 3b (AUDIT FIX G-MEDIUM-02): Check skip_threshold BEFORE loading matrix
-        total_count = (
-            self._vector_store.count(profile_id)
-            if self._vector_store and getattr(self._vector_store, "available", False)
-            else 0
-        )
+        try:
+            total_count = self._db.get_fact_count(
+                profile_id,
+                include_global=include_global,
+                include_shared=include_shared,
+            )
+        except (AttributeError, TypeError):
+            total_count = (
+                self._vector_store.count(profile_id)
+                if self._vector_store and getattr(self._vector_store, "available", False)
+                else 0
+            )
         # Step 3c: Skip for very large stores
         if total_count > self._config.skip_threshold:
             logger.debug(
@@ -162,17 +181,42 @@ class HopfieldChannel:
         # VS exists. Routing on prefilter_candidates (not prefilter_threshold)
         # ensures the matrix is always bounded to ~prefilter_candidates rows.
         if vs_ok and total_count > self._config.prefilter_candidates:
-            return self._search_with_prefilter(q_vec, profile_id, [], top_k)
+            return self._search_with_prefilter(
+                q_vec,
+                profile_id,
+                [],
+                top_k,
+                include_global=include_global,
+                include_shared=include_shared,
+            )
 
         # Tiny store (or no VS): build (cached) full matrix.
-        memory_matrix, fact_ids = self._get_memory_matrix(profile_id)
+        memory_matrix, fact_ids = self._get_memory_matrix(
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+        )
         if memory_matrix is None or len(fact_ids) == 0:
             return []
         if vs_ok and len(fact_ids) > self._config.prefilter_candidates:
-            return self._search_with_prefilter(q_vec, profile_id, fact_ids, top_k)
-        return self._search_full_matrix(
+            return self._search_with_prefilter(
+                q_vec,
+                profile_id,
+                fact_ids,
+                top_k,
+                include_global=include_global,
+                include_shared=include_shared,
+            )
+        results = self._search_full_matrix(
             q_vec, memory_matrix, fact_ids, top_k,
         )
+        return filter_authorized_results(
+            self._db,
+            results,
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+        )
 
     def _search_full_matrix(
         self,
@@ -220,6 +264,9 @@ class HopfieldChannel:
         profile_id: str,
         all_fact_ids: list[str],
         top_k: int,
+        *,
+        include_global: bool = False,
+        include_shared: bool = False,
     ) -> list[tuple[str, float]]:
         """Two-stage retrieval for large stores (>prefilter_threshold facts).
 
@@ -243,15 +290,35 @@ class HopfieldChannel:
             top_k=self._config.prefilter_candidates,
             profile_id=profile_id,
         )
-        if not knn_results:
+        # The ANN index is owner-profile partitioned.  Supplement it with
+        # opted-in cross-profile facts, then authorize the combined candidates
+        # through the canonical DB predicate below.
+        external_facts = self._db.get_external_visible_facts(
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+        )
+        combined = {fact_id: score for fact_id, score in knn_results}
+        query_norm = float(np.linalg.norm(query))
+        for fact in external_facts:
+            embedding = getattr(fact, "embedding", None)
+            if embedding is None or len(embedding) != self._config.dimension:
+                continue
+            vector = np.array(embedding, dtype=np.float32)
+            denominator = query_norm * float(np.linalg.norm(vector))
+            if denominator <= 1e-8:
+                continue
+            score = (float(np.dot(query, vector) / denominator) + 1.0) / 2.0
+            combined[fact.fact_id] = max(combined.get(fact.fact_id, 0.0), score)
+        if not combined:
             return []
 
         # Stage 2: Load candidate facts
-        candidate_ids = [fid for fid, _ in knn_results]
+        candidate_ids = list(combined)
         candidates = self._db.get_facts_by_ids(
             candidate_ids, profile_id,
-            include_global=getattr(self, 'include_global', False),
-            include_shared=getattr(self, 'include_shared', False),
+            include_global=include_global,
+            include_shared=include_shared,
         )
         if not candidates:
             return []
@@ -276,10 +343,21 @@ class HopfieldChannel:
         sub_matrix = sub_matrix / norms
 
         # Stage 4: Hopfield on subset
-        return self._search_full_matrix(query, sub_matrix, sub_ids, top_k)
+        results = self._search_full_matrix(query, sub_matrix, sub_ids, top_k)
+        return filter_authorized_results(
+            self._db,
+            results,
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+        )
 
     def _get_memory_matrix(
-        self, profile_id: str,
+        self,
+        profile_id: str,
+        *,
+        include_global: bool = False,
+        include_shared: bool = False,
     ) -> tuple[np.ndarray | None, list[str]]:
         """Build or retrieve cached memory matrix X (n x d).
 
@@ -290,14 +368,22 @@ class HopfieldChannel:
             (memory_matrix, fact_ids) or (None, []) if no valid facts.
         """
         # Step 1: Check cache validity
-        current_count = (
-            self._vector_store.count(profile_id)
-            if self._vector_store and getattr(self._vector_store, "available", False)
-            else 0
-        )
+        scope_key = (profile_id, bool(include_global), bool(include_shared))
+        try:
+            current_count = self._db.get_fact_count(
+                profile_id,
+                include_global=include_global,
+                include_shared=include_shared,
+            )
+        except (AttributeError, TypeError):
+            current_count = (
+                self._vector_store.count(profile_id)
+                if self._vector_store and getattr(self._vector_store, "available", False)
+                else 0
+            )
 
         if (
-            self._cached_profile == profile_id
+            self._cached_scope_key == scope_key
             and self._cached_count == current_count
             and self._cached_matrix is not None
             and (time.monotonic() - self._cache_timestamp)
@@ -310,8 +396,8 @@ class HopfieldChannel:
         # deserialize the whole table just to slice it.
         facts = self._db.get_all_facts(
             profile_id, limit=5000,
-            include_global=getattr(self, 'include_global', False),
-            include_shared=getattr(self, 'include_shared', False),
+            include_global=include_global,
+            include_shared=include_shared,
         )
         if not facts:
             return (None, [])
@@ -341,6 +427,7 @@ class HopfieldChannel:
         self._cached_matrix = matrix
         self._cached_fact_ids = fact_ids
         self._cached_profile = profile_id
+        self._cached_scope_key = scope_key
         self._cached_count = current_count
         self._cache_timestamp = time.monotonic()
 
@@ -354,5 +441,6 @@ class HopfieldChannel:
         """
         self._cached_matrix = None
         self._cached_fact_ids = []
+        self._cached_scope_key = None
         self._cached_count = 0
         self._cache_timestamp = 0.0

package/src/superlocalmemory/retrieval/profile_channel.py

@@ -14,7 +14,7 @@ retrieval pool with high scores.
 Competitor reference: EverMemOS profile synthesis (~+15-20% SH).
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 from __future__ import annotations
 

package/src/superlocalmemory/retrieval/quantization_aware_search.py

@@ -6,8 +6,8 @@
 
 Merges results from:
   Tier 1: float32 (VectorStore.search -- exact cosine)
-  Tier 2: int8    (VectorStore.search_int8 -- sqlite-vec native)
-  Tier 3: polar   (QuantizedEmbeddingStore.search -- PolarQuant)
+  Tier 2: int8    (QuantizedEmbeddingStore -- 8-bit PolarQuant rows)
+  Tier 3: polar   (QuantizedEmbeddingStore -- 2/4-bit PolarQuant rows)
 
 Deduplicates by keeping the highest score per fact_id.
 Applies precision-dependent score penalties:
@@ -16,7 +16,7 @@ Applies precision-dependent score penalties:
   - polar:   config.polar_search_penalty (default 0.95x)
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -110,16 +110,17 @@ class QuantizationAwareSearch:
     def _search_int8(
         self, query: NDArray, profile_id: str, top_k: int,
     ) -> list[tuple[str, float]]:
-        """Tier 2: int8 approximate via VectorStore.search_int8.
+        """Tier 2: persisted 8-bit quantized embeddings.
 
         Applies 0.98x penalty to account for int8 quantization error.
-        Gracefully returns [] if VectorStore lacks search_int8 method.
         """
-        fn = getattr(self._vector_store, "search_int8", None)
-        if fn is None:
-            return []
         try:
-            raw = fn(query, profile_id=profile_id, top_k=top_k)
+            raw = self._quantized_store.search(
+                query,
+                profile_id,
+                top_k,
+                bit_widths=(8,),
+            )
             return [(fid, score * _INT8_PENALTY) for fid, score in raw]
         except Exception as exc:
             logger.debug("int8 search failed: %s", exc)
@@ -133,7 +134,12 @@ class QuantizationAwareSearch:
         Applies polar_search_penalty from config.
         """
         try:
-            raw = self._quantized_store.search(query, profile_id, top_k)
+            raw = self._quantized_store.search(
+                query,
+                profile_id,
+                top_k,
+                bit_widths=(2, 4),
+            )
             penalty = self._config.polar_search_penalty
             return [(fid, score * penalty) for fid, score in raw]
         except Exception as exc:

package/src/superlocalmemory/retrieval/reranker.py

@@ -11,7 +11,7 @@ at ~60 MB. Same isolation pattern as EmbeddingService.
 The worker subprocess auto-kills after 2 minutes idle.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -29,21 +29,27 @@ from typing import Any
 
 from pathlib import Path
 
+from superlocalmemory.infra.data_root import state_path
 from superlocalmemory.storage.models import AtomicFact
 
-_RERANKER_PID_FILE = Path.home() / ".superlocalmemory" / ".reranker-worker.pid"
+_RERANKER_PID_FILE = None  # test-only override
+
+
+def _reranker_pid_file() -> Path:
+    return _RERANKER_PID_FILE or state_path(".reranker-worker.pid")
 
 
 def _is_reranker_worker_alive() -> bool:
     """Check if a reranker worker PID is already alive (machine-wide singleton)."""
     try:
-        if not _RERANKER_PID_FILE.exists():
+        pid_file = _reranker_pid_file()
+        if not pid_file.exists():
             return False
-        pid = int(_RERANKER_PID_FILE.read_text().strip())
+        pid = int(pid_file.read_text().strip())
         os.kill(pid, 0)
         return True
     except (ValueError, OSError, ProcessLookupError):
-        _RERANKER_PID_FILE.unlink(missing_ok=True)
+        _reranker_pid_file().unlink(missing_ok=True)
         return False
 
 # Track all live reranker instances for atexit cleanup
@@ -215,8 +221,9 @@ class CrossEncoderReranker:
                 **popen_platform_kwargs(),
             )
             # v3.4.13: Register PID for machine-wide singleton
-            _RERANKER_PID_FILE.parent.mkdir(parents=True, exist_ok=True)
-            _RERANKER_PID_FILE.write_text(str(self._worker_proc.pid))
+            pid_file = _reranker_pid_file()
+            pid_file.parent.mkdir(parents=True, exist_ok=True)
+            pid_file.write_text(str(self._worker_proc.pid))
             logger.info(
                 "Reranker worker spawned (PID %d)", self._worker_proc.pid,
             )
@@ -321,22 +328,41 @@ class CrossEncoderReranker:
         return result_container[0] if result_container else ""
 
     def _kill_worker(self) -> None:
-        """Terminate worker subprocess."""
+        """Terminate the worker and close every owned pipe exactly once."""
         if self._idle_timer is not None:
             self._idle_timer.cancel()
             self._idle_timer = None
-        if self._worker_proc is not None:
+
+        proc = self._worker_proc
+        if proc is not None:
+            # Detach first so re-entrant/finalizer cleanup is idempotent.
+            self._worker_proc = None
+            self._worker_ready = False
             try:
-                self._worker_proc.stdin.write('{"cmd":"quit"}\n')
-                self._worker_proc.stdin.flush()
-                self._worker_proc.wait(timeout=3)
+                proc.stdin.write('{"cmd":"quit"}\n')
+                proc.stdin.flush()
+                proc.wait(timeout=3)
             except Exception:
                 try:
-                    self._worker_proc.kill()
+                    returncode = proc.poll()
                 except Exception:
-                    pass
-            self._worker_proc = None
-            self._worker_ready = False
+                    returncode = None
+                if returncode is None or not isinstance(returncode, int):
+                    try:
+                        proc.kill()
+                        proc.wait(timeout=3)
+                    except Exception:
+                        pass
+            finally:
+                # Explicit close prevents TextIOWrapper from flushing a dead
+                # child's stdin later from an unraisable object finalizer.
+                for stream_name in ("stdin", "stdout", "stderr"):
+                    stream = getattr(proc, stream_name, None)
+                    if stream is not None:
+                        try:
+                            stream.close()
+                        except (BrokenPipeError, OSError, ValueError):
+                            pass
 
     def _reset_idle_timer(self) -> None:
         """Reset idle timer — kills worker after 2 min inactivity."""
@@ -373,13 +399,23 @@ class CrossEncoderReranker:
         results (without reranking), and MCP gets reranked results
         (worker stays warm between calls).
         """
+        results, _, _ = self.rerank_with_status(query, candidates, top_k=top_k)
+        return results
+
+    def rerank_with_status(
+        self,
+        query: str,
+        candidates: list[tuple[AtomicFact, float]],
+        top_k: int = 10,
+    ) -> tuple[list[tuple[AtomicFact, float]], bool, str]:
+        """Return results plus whether cross-encoder inference actually ran."""
         if not candidates:
-            return []
+            return [], False, "no_candidates"
 
         # Non-blocking: if model isn't loaded yet, return fallback
         if not self._model_loaded:
             sorted_cands = sorted(candidates, key=lambda x: x[1], reverse=True)
-            return sorted_cands[:top_k]
+            return sorted_cands[:top_k], False, "fallback_not_ready"
 
         documents = [fact.content for fact, _ in candidates]
 
@@ -397,7 +433,7 @@ class CrossEncoderReranker:
         if resp is None or not resp.get("ok"):
             # Fallback: return by existing score
             sorted_cands = sorted(candidates, key=lambda x: x[1], reverse=True)
-            return sorted_cands[:top_k]
+            return sorted_cands[:top_k], False, "fallback_busy_or_unavailable"
 
         scores = resp["scores"]
         scored: list[tuple[AtomicFact, float]] = [
@@ -405,7 +441,7 @@ class CrossEncoderReranker:
             for (fact, _), score in zip(candidates, scores)
         ]
         scored.sort(key=lambda x: x[1], reverse=True)
-        return scored[:top_k]
+        return scored[:top_k], True, "applied"
 
     def score_pair(self, query: str, document: str) -> float:
         """Score a single (query, document) pair."""

package/src/superlocalmemory/retrieval/scope_policy.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Fail-closed authorization helpers for retrieval candidate paths.
+
+Candidate generators may use caches, approximate indexes, or graph stores that
+are not the authorization source of truth.  Every such path must therefore
+re-authorize fact IDs through ``DatabaseManager.get_facts_by_ids()``, whose SQL
+is built by the canonical ``_scope_where`` predicate.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Iterable
+
+from superlocalmemory.storage.database import _scope_where
+
+
+def authorized_fact_ids(
+    db: Any,
+    fact_ids: Iterable[str],
+    profile_id: str,
+    *,
+    include_global: bool = False,
+    include_shared: bool = False,
+) -> set[str]:
+    """Return only IDs visible under the canonical scope predicate.
+
+    Authorization errors fail closed.  The stable de-duplication avoids SQLite
+    parameter waste without changing candidate order at the caller boundary.
+    """
+    unique_ids = list(dict.fromkeys(fact_ids))
+    if not unique_ids:
+        return set()
+    try:
+        facts = db.get_facts_by_ids(
+            unique_ids,
+            profile_id,
+            include_global=bool(include_global),
+            include_shared=bool(include_shared),
+        )
+        if isinstance(facts, list):
+            return {fact.fact_id for fact in facts}
+    except Exception:
+        pass
+
+    # Lightweight DB wrappers used by maintenance paths may expose execute()
+    # without the higher-level method.  Keep the same canonical predicate.
+    try:
+        where, params = _scope_where(
+            profile_id,
+            include_global=include_global,
+            include_shared=include_shared,
+        )
+        placeholders = ",".join("?" for _ in unique_ids)
+        rows = db.execute(
+            f"SELECT fact_id FROM atomic_facts WHERE fact_id IN ({placeholders}) "
+            f"AND {where}",
+            (*unique_ids, *params),
+        )
+        if not isinstance(rows, list):
+            rows = list(rows)
+        return {dict(row)["fact_id"] for row in rows}
+    except Exception:
+        return set()
+
+
+def filter_authorized_results(
+    db: Any,
+    results: Iterable[tuple[str, float]],
+    profile_id: str,
+    *,
+    include_global: bool = False,
+    include_shared: bool = False,
+) -> list[tuple[str, float]]:
+    """Preserve result order/scores while removing unauthorized fact IDs."""
+    materialized = list(results)
+    allowed = authorized_fact_ids(
+        db,
+        (fact_id for fact_id, _score in materialized),
+        profile_id,
+        include_global=include_global,
+        include_shared=include_shared,
+    )
+    return [item for item in materialized if item[0] in allowed]