superlocalmemory 4.0.4 → 4.0.6

package/src/superlocalmemory/retrieval/remote_reranker.py

@@ -110,12 +110,30 @@ def is_remote_cross_encoder_backend(backend: str) -> bool:
     return (backend or "").strip().lower() in REMOTE_CROSS_ENCODER_BACKENDS
 
 
-def validate_remote_reranker_config(backend: str, endpoint: str) -> str | None:
+def validate_remote_reranker_config(
+    backend: str,
+    endpoint: str,
+    trust_plain_http_lan: bool = True,
+) -> str | None:
     """Return an actionable error string, or None when the pair is coherent.
 
     Covers the issue-#103 leftover directly: an endpoint configured against a
     LOCAL backend used to be dropped on the floor by ``SLMConfig.load``. It now
     produces a named error naming both keys and the exact edit to make.
+
+    Args:
+        backend:            Value of ``retrieval.cross_encoder_backend``.
+        endpoint:           Value of ``retrieval.cross_encoder_endpoint``.
+        trust_plain_http_lan: When True (the default), numeric RFC1918/ULA/
+            link-local addresses may use plain HTTP — the same security posture
+            as the local reranker, where memory text only crosses loopback.
+            Set to False in hardened deployments (zero-trust networks, shared
+            colocation) to require HTTPS for all non-loopback hosts.
+
+    Threat model note: trusting a private-LAN address does NOT prevent a
+    MITM attack by an adversary on the same physical LAN (e.g. via ARP
+    spoofing). This flag means "the LAN is under my control and I accept that
+    risk." It is not a claim that RFC1918 traffic is cryptographically secure.
     """
     backend = (backend or "").strip()
     endpoint = (endpoint or "").strip()
@@ -139,11 +157,23 @@ def validate_remote_reranker_config(backend: str, endpoint: str) -> str | None:
         )
     if not remote:
         return None
-    return _validate_endpoint_url(endpoint)
-
-
-def _validate_endpoint_url(endpoint: str) -> str | None:
-    """Scheme/host allow-listing for the operator-supplied rerank URL."""
+    return _validate_endpoint_url(endpoint, trust_plain_http_lan=trust_plain_http_lan)
+
+
+def _validate_endpoint_url(
+    endpoint: str,
+    trust_plain_http_lan: bool = True,
+) -> str | None:
+    """Scheme/host allow-listing for the operator-supplied rerank URL.
+
+    Plain-HTTP allowances (most-to-least trusted):
+      1. Loopback (127.x, ::1, localhost) — always allowed.
+      2. Numeric RFC1918/ULA/link-local addresses — allowed when
+         ``trust_plain_http_lan`` is True (the default).  Only numeric
+         addresses qualify; bare hostnames are never trusted because DNS is
+         mutable and not a trust boundary.
+      3. Everything else (public IPs, bare hostnames) — always requires HTTPS.
+    """
     try:
         parsed = urlparse(endpoint)
     except ValueError as exc:
@@ -177,11 +207,39 @@ def _validate_endpoint_url(endpoint: str) -> str | None:
             "retrieval.cross_encoder_api_key; it is sent as a Bearer header "
             "and never logged."
         )
-    if parsed.scheme == "http" and not _is_loopback_host(parsed.hostname):
+    if parsed.scheme == "http":
+        hostname = parsed.hostname
+        if _is_loopback_host(hostname):
+            return None  # loopback always allowed regardless of trust flag
+        if trust_plain_http_lan and _is_private_lan_host(hostname):
+            # Numeric private address on an operator-trusted LAN. Threat model:
+            # an attacker on the same physical LAN can still MITM plain HTTP
+            # (ARP spoofing). This is allowed because the LAN is assumed to be
+            # under the operator's control. Set trust_plain_http_lan=False in
+            # hardened/zero-trust environments.
+            return None
+        if not _is_private_lan_host(hostname):
+            # Public IP, CGNAT, or a bare hostname (DNS not trusted as a
+            # proof of locality). Bare hostnames that happen to resolve to
+            # private IPs are NOT trusted: DNS can be poisoned or changed,
+            # so only provably-private numeric addresses are accepted.
+            return (
+                "retrieval.cross_encoder_endpoint must use HTTPS for this "
+                "host. Plain HTTP is allowed only for loopback "
+                "(127.x/::1/localhost) and numeric private-LAN addresses "
+                "(RFC1918: 10.x, 172.16-31.x, 192.168.x; IPv6 ULA fc00::/7; "
+                "link-local 169.254.x/fe80::). "
+                "Bare hostnames are not trusted even if they resolve to a "
+                "private IP — use a numeric address or configure HTTPS."
+            )
+        # Private-LAN address but trust_plain_http_lan is False (hardened mode)
         return (
-            "retrieval.cross_encoder_endpoint must use HTTPS for non-loopback "
-            "hosts because recall queries and candidate memory text cross this "
-            "connection. Plain HTTP is allowed only for localhost/loopback."
+            "retrieval.cross_encoder_endpoint uses plain HTTP to a "
+            "private-LAN address. HTTPS is required because "
+            "retrieval.trust_plain_http_lan is set to false. "
+            "Either configure a TLS-terminating proxy on the reranker, or "
+            "set retrieval.trust_plain_http_lan=true to permit plain HTTP "
+            "within your private network (default for new installs)."
         )
     return None
 
@@ -197,6 +255,42 @@ def _is_loopback_host(hostname: str) -> bool:
         return False
 
 
+def _is_private_lan_host(hostname: str) -> bool:
+    """True only for numeric private-range addresses (RFC1918, ULA, link-local).
+
+    Deliberate non-DNS: bare hostnames (e.g. ``my-reranker.lan``) return False
+    even if they currently resolve to a private IP. DNS is mutable and not a
+    trust boundary — an adversary who can influence DNS resolution can redirect
+    the endpoint to a public host, defeating the locality check. Only numeric
+    addresses are provably bound to a private range at configuration time.
+
+    Accepted ranges (Python 3.11+ ``ipaddress.is_private``):
+      IPv4  RFC1918: 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
+      IPv4  link-local: 169.254.0.0/16
+      IPv6  ULA: fc00::/7 (includes fd00::/8)
+      IPv6  link-local: fe80::/10
+
+    Excluded ranges (not accepted for plain HTTP):
+      CGNAT 100.64.0.0/10 — ISP-shared address space, not operator-controlled
+      172.15.0.0/8 and 172.32.0.0/8 — outside the 172.16.0.0/12 boundary
+      Public unicast addresses
+
+    IPv4-mapped IPv6 addresses (``::ffff:192.168.1.1``) are unwrapped to their
+    IPv4 equivalent before the range check, so they are handled consistently.
+    """
+    host = (hostname or "").rstrip(".").lower()
+    try:
+        addr = ipaddress.ip_address(host)
+    except ValueError:
+        # Not a numeric address — bare hostname, not provably private
+        return False
+    # Unwrap IPv4-mapped IPv6 (::ffff:192.168.1.1 → 192.168.1.1) so the
+    # RFC1918 check applies to the IPv4 portion.
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
+        addr = addr.ipv4_mapped
+    return addr.is_private
+
+
 def normalize_rerank_endpoint(endpoint: str) -> str:
     """Append ``/rerank`` when the URL stops at the API root.
 
@@ -370,8 +464,11 @@ class RemoteReranker:
         api_key: str = "",
         backend: str = "openai",
         timeout_seconds: float = _DEFAULT_READ_TIMEOUT_S,
+        trust_plain_http_lan: bool = True,
     ) -> None:
-        error = validate_remote_reranker_config(backend, endpoint)
+        error = validate_remote_reranker_config(
+            backend, endpoint, trust_plain_http_lan=trust_plain_http_lan,
+        )
         if error:
             raise RemoteRerankerConfigError(error)
 

package/src/superlocalmemory/retrieval/temporal_validity_filter.py

@@ -49,8 +49,12 @@ All demotions are non-destructive (P5-INT-01): facts stay in the candidate
 list but rank below valid facts. A factor of 0.0 restores the legacy hide
 behaviour (a score of zero is gated out by the evidence floor).
 
-Both lookups are bounded (candidate ids only), chunked, indexed, and
-fail-open: a DB error returns results unchanged.
+All correction-admission lookups are bounded (candidate ids only), chunked,
+and indexed.  Admission is deliberately **fail-closed**: if SLM cannot prove
+which candidates are invalidated, it returns no candidates rather than let an
+approved stale fact re-enter recall.  The legacy score-demotion filter follows
+the same rule for its system-invalidated lookup.  Event-time demotion remains a
+best-effort ranking signal; it is not the correction authority.
 
 Integrates with ChannelRegistry.register_filter() using the FilterFn signature:
     (all_channel_results, profile_id, context) -> filtered_results
@@ -62,6 +66,7 @@ License: AGPL-3.0-or-later
 from __future__ import annotations
 
 import logging
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
@@ -79,6 +84,21 @@ logger = logging.getLogger(__name__)
 _EVENT_TIME_DEMOTION_FACTOR: float = 0.5
 
 
+@dataclass
+class CorrectionAdmissionCache:
+    """Per-recall lifecycle admission cache.
+
+    The retrieval engine performs a mandatory second admission after bridge or
+    scene expansion.  Facts already checked before fusion do not need a second
+    database read in the *same* recall, while any newly expanded id is checked
+    immediately.  The cache never crosses requests, profiles, or DB writes.
+    """
+
+    checked_fact_ids: set[str] = field(default_factory=set)
+    inadmissible_fact_ids: set[str] = field(default_factory=set)
+    unavailable: bool = False
+
+
 def _normalized_as_of(as_of: str | None) -> str | None:
     """Normalize an optional transaction-time boundary at one boundary.
 
@@ -100,26 +120,69 @@ def _invalidated_candidate_ids(
     as_of: str | None = None,
     include_global: bool = False,
     include_shared: bool = False,
+    lifecycle_cache: CorrectionAdmissionCache | None = None,
 ) -> set[str] | None:
-    """Return admissibility failures, or ``None`` when the lookup is unhealthy.
+    """Return correction failures, or ``None`` when admission is unprovable.
 
-    ``None`` deliberately preserves the established fail-open availability
-    contract.  It is distinct from an empty set: a bad/mocked return value must
-    not accidentally exclude arbitrary candidates.
+    ``None`` is intentionally distinct from an empty set.  Callers MUST turn
+    it into an abstention (an empty candidate path), never treat it as "nothing
+    invalidated".  Treating an unavailable lifecycle read as an empty set is a
+    fail-open path through which an approved stale fact can be re-admitted.
     """
     if not fact_ids:
         return set()
+    if lifecycle_cache is not None and lifecycle_cache.unavailable:
+        return None
+    unchecked = (
+        fact_ids - lifecycle_cache.checked_fact_ids
+        if lifecycle_cache is not None
+        else fact_ids
+    )
+    if not unchecked:
+        return (
+            lifecycle_cache.inadmissible_fact_ids & fact_ids
+            if lifecycle_cache is not None
+            else set()
+        )
     try:
         kwargs: dict[str, Any] = {"as_of": _normalized_as_of(as_of)}
         if include_global:
             kwargs["include_global"] = True
         if include_shared:
             kwargs["include_shared"] = True
-        invalid = db.get_invalidated_fact_ids(list(fact_ids), profile_id, **kwargs)
+        # Do not infer support from a permissive mock's dynamic attributes.
+        # The concrete storage manager owns this optimized contract; older
+        # adapters continue through the two focused public queries below.
+        combined = getattr(type(db), "get_correction_inadmissible_fact_ids", None)
+        if callable(combined):
+            invalid = db.get_correction_inadmissible_fact_ids(
+                list(unchecked), profile_id, **kwargs,
+            )
+        else:
+            invalid = db.get_invalidated_fact_ids(list(unchecked), profile_id, **kwargs)
+            pending_successors = db.get_nonapplied_correction_successor_ids(
+                list(unchecked),
+                profile_id,
+                include_global=include_global,
+                include_shared=include_shared,
+            )
+            if not isinstance(pending_successors, set):
+                return None
+            invalid |= pending_successors
     except Exception as exc:
         logger.warning("Correction admission lookup failed: %s", exc)
+        if lifecycle_cache is not None:
+            lifecycle_cache.unavailable = True
         return None
-    return invalid if isinstance(invalid, set) else None
+    if not isinstance(invalid, set):
+        if lifecycle_cache is not None:
+            lifecycle_cache.unavailable = True
+        return None
+    if lifecycle_cache is not None:
+        lifecycle_cache.checked_fact_ids.update(unchecked)
+        lifecycle_cache.inadmissible_fact_ids.update(invalid)
+        return lifecycle_cache.inadmissible_fact_ids & fact_ids
+    return invalid
 
 
 def _strict_temporal_candidate_ids(
@@ -133,7 +196,7 @@ def _strict_temporal_candidate_ids(
     include_global: bool = False,
     include_shared: bool = False,
 ) -> set[str] | None:
-    """Return strict two-clock admission failures, fail-open on lookup error."""
+    """Return strict two-clock failures, or ``None`` when admission is unprovable."""
     if not fact_ids or (known_as_of is None and valid_at is None):
         return set()
     try:
@@ -151,6 +214,26 @@ def _strict_temporal_candidate_ids(
     return invalid if isinstance(invalid, set) else None
 
 
+def _abstain_candidates(
+    all_results: dict[str, list[tuple[str, float]]],
+    *,
+    stage: str,
+) -> dict[str, list[tuple[str, float]]]:
+    """Fail closed without changing the retrieval filter public contract.
+
+    ``ChannelRegistry`` and the retrieval engine both expect the original
+    channel-result shape.  Returning the same channel names with empty lists
+    is an explicit candidate-level abstention: downstream fusion naturally
+    produces no materializable facts, while callers keep their stable response
+    schema and can report ``no_confident_match``.
+    """
+    logger.error(
+        "Temporal correction admission unavailable at %s; abstaining from recall candidates",
+        stage,
+    )
+    return {channel_name: [] for channel_name in all_results}
+
+
 def admit_correction_candidates(
     all_results: dict[str, list[tuple[str, float]]],
     profile_id: str,
@@ -162,6 +245,7 @@ def admit_correction_candidates(
     include_unknown: bool = False,
     include_global: bool = False,
     include_shared: bool = False,
+    lifecycle_cache: CorrectionAdmissionCache | None = None,
 ) -> dict[str, list[tuple[str, float]]]:
     """Hard-exclude system-superseded facts before candidate fusion.
 
@@ -178,15 +262,20 @@ def admit_correction_candidates(
     invalid = _invalidated_candidate_ids(
         db, fact_ids, profile_id, as_of=as_of,
         include_global=include_global, include_shared=include_shared,
+        lifecycle_cache=lifecycle_cache,
     )
+    if invalid is None:
+        return _abstain_candidates(all_results, stage="pre_fusion.lifecycle")
     strict = _strict_temporal_candidate_ids(
         db, fact_ids, profile_id,
         known_as_of=known_as_of, valid_at=valid_at,
         include_unknown=include_unknown,
         include_global=include_global, include_shared=include_shared,
     )
+    if strict is None:
+        return _abstain_candidates(all_results, stage="pre_fusion.strict_temporal")
     if strict:
-        invalid = (invalid or set()) | strict
+        invalid |= strict
     if not invalid:
         return all_results
     return {
@@ -210,21 +299,35 @@ def admit_correction_fusion_results(
     include_unknown: bool = False,
     include_global: bool = False,
     include_shared: bool = False,
+    lifecycle_cache: CorrectionAdmissionCache | None = None,
 ) -> list[Any]:
     """Re-apply correction admission after graph/scene candidate expansion."""
     fact_ids = {result.fact_id for result in fused_results}
     invalid = _invalidated_candidate_ids(
         db, fact_ids, profile_id, as_of=as_of,
         include_global=include_global, include_shared=include_shared,
+        lifecycle_cache=lifecycle_cache,
     )
+    if invalid is None:
+        logger.error(
+            "Temporal correction admission unavailable at post_fusion.lifecycle; "
+            "abstaining from recall candidates",
+        )
+        return []
     strict = _strict_temporal_candidate_ids(
         db, fact_ids, profile_id,
         known_as_of=known_as_of, valid_at=valid_at,
         include_unknown=include_unknown,
         include_global=include_global, include_shared=include_shared,
     )
+    if strict is None:
+        logger.error(
+            "Temporal correction admission unavailable at post_fusion.strict_temporal; "
+            "abstaining from recall candidates",
+        )
+        return []
     if strict:
-        invalid = (invalid or set()) | strict
+        invalid |= strict
     if not invalid:
         return fused_results
     return [result for result in fused_results if result.fact_id not in invalid]
@@ -311,14 +414,11 @@ class TemporalValidityFilter:
         # When as_of is set: only supersessions that occurred AT OR BEFORE
         # as_of contribute (Phase 4b bi-temporal fix). Supersessions after
         # as_of are invisible — the fact was still valid at the query point.
-        try:
-            invalid = self._db.get_invalidated_fact_ids(
-                list(all_fact_ids), profile_id, as_of=as_of,
-            )
-        except Exception as exc:
-            # Fail-open: a validity-lookup error must never break retrieval.
-            logger.warning("Temporal validity lookup failed: %s", exc)
-            return all_results
+        invalid = _invalidated_candidate_ids(
+            self._db, all_fact_ids, profile_id, as_of=as_of,
+        )
+        if invalid is None:
+            return _abstain_candidates(all_results, stage="legacy_filter.lifecycle")
 
         # --- Axis 2: Event-time expiry (Phase 4 T1b) ---
         # Guard: skip event-time demotion when the caller signals it wants