superlocalmemory 3.8.13 → 4.0.0

package/src/superlocalmemory/retrieval/hopfield_channel.py

@@ -2,11 +2,14 @@
 # Licensed under AGPL-3.0-or-later - see LICENSE file
 # Part of SuperLocalMemory V3
 
-"""SuperLocalMemory V3.3 -- Hopfield Associative Memory (6th Retrieval Channel).
+"""SuperLocalMemory V3.3 -- Hopfield Associative Memory (6th of 6 Retrieval Channels).
 
 Modern Continuous Hopfield Network retrieval channel based on
 Ramsauer et al. (2020): "Hopfield Networks is All You Need".
 
+Channel lineup (6 total): semantic, BM25, entity_graph, temporal,
+spreading_activation, hopfield (this module).
+
 The Hopfield channel excels at pattern completion for vague/noisy queries.
 It operates on the same embedding space as the semantic channel but uses
 an energy-based attention mechanism instead of cosine similarity.
@@ -14,7 +17,7 @@ an energy-based attention mechanism instead of cosine similarity.
 Key features:
   - Full memory matrix path for stores < 10K facts
   - ANN pre-filter path for stores 10K-100K (VectorStore KNN -> Hopfield refinement)
-  - Skip path for stores > 100K (other 5 channels are sufficient)
+  - Skip path for stores > 100K (other 5 non-Hopfield channels are sufficient)
   - TTL-based matrix cache to avoid rebuilding every query
   - Returns [] on any error (HR-06)
 
@@ -38,7 +41,7 @@ logger = logging.getLogger(__name__)
 
 
 class HopfieldChannel:
-    """6th retrieval channel: Modern Hopfield associative memory.
+    """6th of 6 retrieval channels: Modern Hopfield associative memory.
 
     Implements the RetrievalChannel protocol::
 
@@ -48,6 +51,9 @@ class HopfieldChannel:
     computes Hopfield attention scores (softmax of scaled dot products),
     then ranks facts by similarity to the completed pattern.
 
+    Six-channel retrieval model: semantic, BM25, entity_graph, temporal,
+    spreading_activation, hopfield (this channel).
+
     Routing logic (per LLD Section 2.2):
       - n > skip_threshold (100K): return [] immediately
       - n > prefilter_threshold (10K): ANN pre-filter + Hopfield on subset

package/src/superlocalmemory/retrieval/remote_reranker.py

@@ -48,6 +48,7 @@ Part of Qualixar | Author: Varun Pratap Bhardwaj
 from __future__ import annotations
 
 import json
+import ipaddress
 import logging
 import math
 import os
@@ -92,9 +93,6 @@ _MAX_ATTEMPTS = 2
 # logs; the operator must never have to guess whether reranking is running.
 _FAILURE_RELOG_INTERVAL_S = 60.0
 
-_ERROR_BODY_SNIPPET_CHARS = 200
-
-
 class RemoteRerankerError(RuntimeError):
     """A remote rerank request failed (transport, status, or schema)."""
 
@@ -161,6 +159,12 @@ def _validate_endpoint_url(endpoint: str) -> str | None:
             "retrieval.cross_encoder_endpoint has no host; expected something "
             "like \"http://127.0.0.1:8041/v1/rerank\"."
         )
+    if parsed.query or parsed.fragment:
+        return (
+            "retrieval.cross_encoder_endpoint must not include a query string "
+            "or fragment. Put bearer credentials in "
+            "SLM_CROSS_ENCODER_API_KEY and configure a clean endpoint URL."
+        )
     if parsed.username or parsed.password:
         # httpx logs "HTTP Request: POST <url>" at INFO using str(url), which
         # renders an embedded password in full. This module never logs the raw
@@ -173,9 +177,26 @@ def _validate_endpoint_url(endpoint: str) -> str | None:
             "retrieval.cross_encoder_api_key; it is sent as a Bearer header "
             "and never logged."
         )
+    if parsed.scheme == "http" and not _is_loopback_host(parsed.hostname):
+        return (
+            "retrieval.cross_encoder_endpoint must use HTTPS for non-loopback "
+            "hosts because recall queries and candidate memory text cross this "
+            "connection. Plain HTTP is allowed only for localhost/loopback."
+        )
     return None
 
 
+def _is_loopback_host(hostname: str) -> bool:
+    """Return True only for literal loopback names/addresses (no DNS trust)."""
+    host = (hostname or "").rstrip(".").lower()
+    if host == "localhost" or host.endswith(".localhost"):
+        return True
+    try:
+        return ipaddress.ip_address(host).is_loopback
+    except ValueError:
+        return False
+
+
 def normalize_rerank_endpoint(endpoint: str) -> str:
     """Append ``/rerank`` when the URL stops at the API root.
 
@@ -208,7 +229,15 @@ def redact_endpoint(endpoint: str) -> str:
         netloc = f"{netloc}:{parsed.port}"
     if parsed.username or parsed.password:
         netloc = f"***@{netloc}"
-    return urlunparse(parsed._replace(netloc=netloc))
+    return urlunparse(parsed._replace(netloc=netloc, query="", fragment=""))
+
+
+def _redact_remote_text(text: str) -> str:
+    """Remove recognized secrets and PII before a remote trust boundary."""
+    from superlocalmemory.core.pii import redact_pii_text
+    from superlocalmemory.core.security_primitives import redact_secrets
+
+    return redact_pii_text(redact_secrets(str(text), aggression="high"))
 
 
 # ---------------------------------------------------------------------------
@@ -240,15 +269,14 @@ def parse_rerank_response(payload: Any, expected: int) -> list[float]:
         index = _coerce_index(item, position, expected)
         if scores[index] is not None:
             raise RemoteRerankerError(
-                f"rerank endpoint returned index {index} more than once"
+                "rerank endpoint returned a duplicate document index"
             )
         scores[index] = _coerce_score(item, index)
 
     missing = [i for i, s in enumerate(scores) if s is None]
     if missing:
         raise RemoteRerankerError(
-            f"rerank endpoint returned no score for document index(es) "
-            f"{missing[:5]}{'…' if len(missing) > 5 else ''}"
+            "rerank endpoint omitted one or more document scores"
         )
     return [float(s) for s in scores]  # type: ignore[arg-type]
 
@@ -264,8 +292,8 @@ def _extract_results_array(payload: Any) -> list[Any]:
     results = payload.get("results")
     if results is None:
         raise RemoteRerankerError(
-            f"rerank response has no 'results' array (keys: "
-            f"{sorted(payload)[:8]}). Is cross_encoder_endpoint pointing at a "
+            "rerank response has no 'results' array. Is "
+            "cross_encoder_endpoint pointing at a "
             f"rerank route and not, say, /v1/embeddings?"
         )
     if not isinstance(results, list):
@@ -280,12 +308,12 @@ def _coerce_index(item: dict, position: int, expected: int) -> int:
     raw = item.get("index", position)
     if isinstance(raw, bool) or not isinstance(raw, int):
         raise RemoteRerankerError(
-            f"rerank result #{position} has non-integer index {raw!r}"
+            f"rerank result #{position} has a non-integer index"
         )
     if not 0 <= raw < expected:
         raise RemoteRerankerError(
-            f"rerank result #{position} has out-of-range index {raw} "
-            f"(sent {expected} documents)"
+            f"rerank result #{position} has an out-of-range index "
+            f"for a {expected}-document request"
         )
     return raw
 
@@ -297,18 +325,18 @@ def _coerce_score(item: dict, index: int) -> float:
             if isinstance(raw, bool) or not isinstance(raw, (int, float)):
                 raise RemoteRerankerError(
                     f"rerank result for index {index} has non-numeric "
-                    f"{key}={raw!r}"
+                    f"{key}"
                 )
             value = float(raw)
             if not math.isfinite(value):
                 raise RemoteRerankerError(
                     f"rerank result for index {index} has non-finite "
-                    f"{key}={raw!r}"
+                    f"{key}"
                 )
             return value
     raise RemoteRerankerError(
         f"rerank result for index {index} has neither 'relevance_score' nor "
-        f"'score' (keys: {sorted(item)[:8]})"
+        "'score'"
     )
 
 
@@ -494,8 +522,8 @@ class RemoteReranker:
             headers["Authorization"] = f"Bearer {self._api_key}"
         body = {
             "model": self._model_name,
-            "query": query,
-            "documents": documents,
+            "query": _redact_remote_text(query),
+            "documents": [_redact_remote_text(document) for document in documents],
         }
 
         last_error: RemoteRerankerError | None = None
@@ -534,12 +562,9 @@ class RemoteReranker:
                         f"followed — configure the final URL directly."
                     )
                 if resp.status_code >= 400:
-                    snippet = raw[:_ERROR_BODY_SNIPPET_CHARS].decode(
-                        "utf-8", "replace",
-                    )
                     message = (
-                        f"HTTP {resp.status_code} from {self.safe_endpoint}: "
-                        f"{snippet}"
+                        f"HTTP {resp.status_code} from {self.safe_endpoint}; "
+                        "response body suppressed"
                     )
                     if resp.status_code >= 500:
                         raise _RetryableRemoteError(message)

package/src/superlocalmemory/retrieval/reranker.py

@@ -433,7 +433,38 @@ class CrossEncoderReranker:
 
     @staticmethod
     def _readline_with_timeout(stream: Any, timeout_seconds: float) -> str:
-        """Read a line from stream with timeout. Returns '' on timeout."""
+        """Read a line from stream with timeout. Returns '' on timeout.
+
+        Prefer a deadline-driven selector poll of the stream's file descriptor
+        (POSIX pipes). That path never spawns a helper thread, so a hung
+        worker cannot leak reader threads or pin the pipe FD across timeouts.
+        A thread fallback remains only for streams without a usable fileno
+        (unit-test mocks) and for Windows, where selectors cannot wait on
+        pipes.
+        """
+        import selectors
+
+        timeout_seconds = max(0.0, float(timeout_seconds))
+        fd: int | None
+        try:
+            raw_fd = stream.fileno()
+            fd = raw_fd if isinstance(raw_fd, int) else None
+        except (AttributeError, OSError, ValueError, TypeError):
+            fd = None
+
+        # Windows select()/selectors only accept sockets, not subprocess pipes.
+        if fd is not None and sys.platform != "win32":
+            try:
+                with selectors.DefaultSelector() as sel:
+                    sel.register(fd, selectors.EVENT_READ)
+                    events = sel.select(timeout=timeout_seconds)
+                if not events:
+                    return ""
+                line = stream.readline()
+                return line if line else ""
+            except (OSError, ValueError):
+                return ""
+
         result_container: list[str] = []
         error_container: list[Exception] = []
 

package/src/superlocalmemory/retrieval/temporal_channel.py

@@ -14,7 +14,7 @@ from __future__ import annotations
 
 import logging
 import math
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING
 
 from dateutil.parser import ParserError, parse as dateutil_parse
@@ -30,6 +30,18 @@ logger = logging.getLogger(__name__)
 _MAX_PROXIMITY_DAYS: float = 365.0
 
 
+def _as_utc(dt: datetime | None) -> datetime | None:
+    """Coerce a datetime to timezone-aware UTC (naive values are assumed UTC).
+
+    Keeps proximity and interval comparisons from mixing naive and aware values.
+    """
+    if dt is None:
+        return None
+    if dt.tzinfo is None:
+        return dt.replace(tzinfo=timezone.utc)
+    return dt.astimezone(timezone.utc)
+
+
 def _parse_iso(s: str | None) -> datetime | None:
     if not s:
         return None
@@ -39,11 +51,11 @@ def _parse_iso(s: str | None) -> datetime | None:
     # across thousands of events — that was ~2.6s of the recall. dateutil is
     # now only the fallback for non-ISO strings.
     try:
-        return datetime.fromisoformat(s.replace("Z", "+00:00"))
+        return _as_utc(datetime.fromisoformat(s.replace("Z", "+00:00")))
     except (ValueError, TypeError):
         pass
     try:
-        return dateutil_parse(s)
+        return _as_utc(dateutil_parse(s))
     except (ParserError, ValueError, OverflowError, TypeError):
         return None
 
@@ -95,6 +107,7 @@ class TemporalChannel:
         query_dt = _parse_iso(dates.get("referenced_date"))
         if query_dt is None:
             query_dt = self._try_parse(query)
+        query_dt = _as_utc(query_dt)
 
         # Strategy 1: Entity-temporal metadata search
         # "When did Alice...?" → find all temporal events for Alice

package/src/superlocalmemory/retrieval/temporal_utils.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V4 | https://qualixar.com | https://varunpratap.com
+
+"""Temporal utility helpers for bi-temporal retrieval (Phase 4b).
+
+STORED TIMESTAMP FORMAT (empirically verified, Phase 4b):
+    The fact_temporal_validity table stores system_expired_at and valid_until
+    via datetime.now(timezone.utc).isoformat() which produces:
+        "YYYY-MM-DDTHH:MM:SS.microseconds+00:00"  (Python isoformat, +00:00 suffix)
+
+    normalize_as_of() outputs canonical datetime.isoformat() (preserving any
+    sub-second precision, same +00:00 suffix as the stored values) so that SQL
+    string comparisons ("system_expired_at <= as_of") are lexicographically
+    correct AND the inclusive boundary round-trips (audit P0/CRIT-1):
+
+        "2026-03-01T00:00:00.123456+00:00" <= "2026-06-01T00:00:00+00:00"
+        → first 10 chars "2026-03-01" < "2026-06-01" → TRUE  ✓
+
+    At sub-second boundary (same second, stored has micros):
+        "2024-01-01T12:00:00.123456+00:00" <= "2024-01-01T12:00:00+00:00"
+        → at pos 19: '.' (46) > '+' (43) → FALSE
+        Interpretation: stored supersession is 0.123456s AFTER as_of → fact
+        was still valid at that exact second. Semantically correct.
+
+    Contrast with SQLite strftime('%Y-%m-%dT%H:%M:%SZ', 'now') → "Z" suffix.
+    If as_of used "Z" suffix and stored uses "+00:00":
+        "2024-01-01T12:00:00.123456+00:00" <= "2024-01-01T12:00:00Z"
+        → at pos 19: '.' (46) < 'Z' (90) → TRUE
+        This would WRONGLY demote a fact whose supersession happened 0.123456s
+        AFTER as_of. Using "+00:00" avoids this error.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Optional
+
+
+def normalize_as_of(s: object) -> Optional[str]:
+    """Parse and normalize an as_of timestamp string to UTC ISO 8601.
+
+    Accepts:
+      - "2024-01-01T12:00:00Z"           → "2024-01-01T12:00:00+00:00"
+      - "2024-01-01T12:00:00+05:30"      → "2024-01-01T06:30:00+00:00"
+      - "2024-01-01"                      → "2024-01-01T00:00:00+00:00"
+      - "2024-01-01T12:00:00"            → "2024-01-01T12:00:00+00:00" (naive → UTC)
+      - "2024-01-01T12:00:00+00:00"      → "2024-01-01T12:00:00+00:00"
+
+    Returns None (not empty string) on invalid input so callers can
+    distinguish "no as_of" from "bad as_of" and reject or ignore accordingly.
+
+    Output format: canonical UTC ISO-8601 via datetime.isoformat():
+        - "YYYY-MM-DDTHH:MM:SS+00:00", OR
+          "YYYY-MM-DDTHH:MM:SS.ffffff+00:00" when the input carries sub-second
+          precision.
+        - Audit P0 (CRIT-1): sub-second precision is PRESERVED (not stripped) so
+          a caller can round-trip a real stored system_expired_at / valid_until
+          value — always written with microseconds by datetime.now(UTC).isoformat()
+          — and hit the inclusive boundary (`system_expired_at <= as_of`) EXACTLY.
+          Stripping microseconds made that boundary unreachable and silently
+          wrong on both temporal axes.
+        - +00:00 suffix matches the stored format so SQL string comparisons are
+          lexicographically correct (see module docstring for analysis).
+
+    Requirements:
+        Python >=3.11 (fromisoformat handles "Z" suffix natively). The
+        fallback .replace("Z", "+00:00") is kept for defensive compatibility.
+
+    Args:
+        s: Raw as_of string from user/HTTP/MCP/CLI input. May be None or
+           non-string; both return None (treated as "no as_of" by callers).
+
+    Returns:
+        Normalized UTC string "YYYY-MM-DDTHH:MM:SS+00:00", or None on error.
+    """
+    if not s or not isinstance(s, str):
+        return None
+    s = s.strip()
+    if not s:
+        return None
+
+    # Try ISO 8601 parse. Python 3.11 handles "Z" natively; the replace()
+    # makes this safe on any 3.x that might be used in tests.
+    for candidate in (s, s.replace("Z", "+00:00")):
+        try:
+            dt = datetime.fromisoformat(candidate)
+            # Naive datetime → assume UTC (documented assumption).
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=timezone.utc)
+            else:
+                dt = dt.astimezone(timezone.utc)
+            # Audit P0: isoformat() preserves microseconds when present so the
+            # value round-trips against microsecond-bearing stored timestamps.
+            return dt.isoformat()
+        except ValueError:
+            continue
+
+    # Date-only "YYYY-MM-DD" — treat as midnight UTC.
+    try:
+        dt = datetime.strptime(s[:10], "%Y-%m-%d").replace(tzinfo=timezone.utc)
+        return dt.isoformat()
+    except ValueError:
+        return None
+
+
+__all__ = ("normalize_as_of",)