superlocalmemory 3.8.10 → 3.8.12

package/src/superlocalmemory/core/reranker_worker.py

@@ -119,7 +119,9 @@ def _worker_main() -> None:
         if cmd == "load":
             name = req.get("model_name", "cross-encoder/ms-marco-MiniLM-L-12-v2")
             backend = req.get("backend", "onnx")
-            model, active_backend, model_name = _load_model(name, backend)
+            model, active_backend, model_name, load_error = _load_model(
+                name, backend,
+            )
             # V3.3.16: Run real inference to trigger ONNX CoreML JIT compilation.
             # Without this, first real rerank call triggers 30-60s compilation
             # that exceeds the caller's timeout, killing the worker.
@@ -147,6 +149,9 @@ def _worker_main() -> None:
                 "backend": active_backend,
                 "model": model_name,
                 "warmup_inference": warmup_ok,
+                # Carries the real reason to the parent so the warmup log can
+                # print it instead of a generic timeout message (issue #103).
+                "error": load_error,
             })
             continue
 
@@ -160,9 +165,14 @@ def _worker_main() -> None:
                 # Auto-load with defaults
                 name = req.get("model_name", "cross-encoder/ms-marco-MiniLM-L-12-v2")
                 backend = req.get("backend", "onnx")
-                model, active_backend, model_name = _load_model(name, backend)
+                model, active_backend, model_name, load_error = _load_model(
+                    name, backend,
+                )
             if model is None:
-                _respond({"ok": False, "error": "Model load failed"})
+                _respond({
+                    "ok": False,
+                    "error": load_error or "Model load failed",
+                })
                 continue
             try:
                 pairs = [(query, doc) for doc in documents]
@@ -195,9 +205,14 @@ def _worker_main() -> None:
             if model is None:
                 name = req.get("model_name", "cross-encoder/ms-marco-MiniLM-L-12-v2")
                 backend = req.get("backend", "onnx")
-                model, active_backend, model_name = _load_model(name, backend)
+                model, active_backend, model_name, load_error = _load_model(
+                    name, backend,
+                )
             if model is None:
-                _respond({"ok": False, "error": "Model load failed"})
+                _respond({
+                    "ok": False,
+                    "error": load_error or "Model load failed",
+                })
                 continue
             try:
                 try:
@@ -214,10 +229,18 @@ def _worker_main() -> None:
         _respond({"ok": False, "error": f"Unknown command: {cmd}"})
 
 
+_KNOWN_BACKENDS = ("onnx", "", "pytorch", "torch")
+# Backends this worker can never serve — they are handled over HTTP by
+# superlocalmemory.retrieval.remote_reranker in the parent process (#105).
+# Duplicated as a literal on purpose: this module runs as a bare subprocess
+# and must not import the retrieval package (or, transitively, httpx).
+_REMOTE_BACKENDS = ("openai", "remote")
+
+
 def _load_model(
     name: str, backend: str,
 ) -> tuple:
-    """Load cross-encoder model. Returns (model, backend_name, model_name).
+    """Load cross-encoder model. Returns (model, backend_name, model_name, error).
 
     V3.3.13: sentence-transformers 5.x+ supports backend='onnx' for
     CrossEncoder. We use a 3-tier fallback chain:
@@ -231,6 +254,33 @@ def _load_model(
       x86_64    → model_quint8_avx2.onnx
       Fallback  → model.onnx (generic)
     """
+    # v3.8.11 (issue #103): an unrecognised backend used to fall through to
+    # the PyTorch tier and fail there with a confusing model-load error. A
+    # user who set backend="openai" expecting a remote reranker got five
+    # silent failures and no hint that the value meant nothing. Name it.
+    #
+    # v3.8.12 (issue #105): remote reranking now EXISTS, but it is served in
+    # the parent process — this worker holds torch/ONNX and cannot forward an
+    # HTTP request. Reaching here with a remote backend means the parent
+    # routed wrong (or a caller drove the worker directly), so the message
+    # points at the config keys that select the remote path.
+    if backend in _REMOTE_BACKENDS:
+        return None, "", "", (
+            f"unknown backend {backend!r} for the LOCAL reranker worker. "
+            f"{backend!r} selects the remote reranker, which runs in the "
+            f"parent process — set retrieval.cross_encoder_endpoint (e.g. "
+            f"\"http://127.0.0.1:8041/v1/rerank\") so SuperLocalMemory routes "
+            f"reranking over HTTP instead of spawning this worker."
+        )
+    if backend not in _KNOWN_BACKENDS:
+        return None, "", "", (
+            f"unknown backend {backend!r}; supported values are 'onnx' or ''"
+            f" (PyTorch) for local reranking, or 'openai'/'remote' with "
+            f"retrieval.cross_encoder_endpoint set for a remote "
+            f"OpenAI-compatible /v1/rerank endpoint."
+        )
+
+    tier_errors: list[str] = []
     try:
         from sentence_transformers import CrossEncoder
 
@@ -242,24 +292,35 @@ def _load_model(
                     name, backend="onnx",
                     model_kwargs={"file_name": onnx_file},
                 )
-                return m, f"onnx-quantized({onnx_file})", name
-            except Exception:
-                pass
+                return m, f"onnx-quantized({onnx_file})", name, ""
+            except Exception as exc:
+                tier_errors.append(f"onnx-quantized: {exc}")
 
             # Tier 2: Generic ONNX (auto-exported by optimum)
             try:
                 m = CrossEncoder(name, backend="onnx")
-                return m, "onnx", name
-            except Exception:
-                pass
+                return m, "onnx", name, ""
+            except Exception as exc:
+                tier_errors.append(f"onnx: {exc}")
 
         # Tier 3: PyTorch (always works, no ONNX dependency needed)
         m = CrossEncoder(name)
-        return m, "pytorch", name
-    except ImportError:
-        return None, "", ""
-    except Exception:
-        return None, "", ""
+        return m, "pytorch", name, ""
+    except ImportError as exc:
+        # Previously indistinguishable from a bad model name.
+        return None, "", "", (
+            f"sentence-transformers is not installed ({exc}); "
+            f"install it or set retrieval.use_cross_encoder=false"
+        )
+    except Exception as exc:
+        tier_errors.append(f"pytorch: {exc}")
+        # Every tier's real error, propagated instead of discarded. Before
+        # 3.8.11 this returned (None, "", "") and the operator saw only a
+        # generic "did not confirm ready" line from the parent process.
+        return None, "", "", (
+            f"could not load cross-encoder model {name!r} "
+            f"(backend={backend or 'pytorch'}): " + "; ".join(tier_errors)
+        )
 
 
 def _respond(data: dict) -> None:

package/src/superlocalmemory/infra/daemon_identity.py

@@ -26,6 +26,7 @@ from pathlib import Path
 from typing import Any, Mapping
 
 from superlocalmemory.infra.data_root import canonical_data_root
+from superlocalmemory.infra.process_identity import process_start_token_for
 
 DAEMON_DESCRIPTOR_SCHEMA = 1
 DAEMON_PROTOCOL = 1
@@ -88,6 +89,11 @@ class DaemonDescriptor:
     state: str
     version: str
     started_at: float
+    # Clock-independent process identity. Optional and defaulted so a
+    # descriptor written by an older release still parses; platforms without a
+    # boot-relative start time (Windows) legitimately store None and fall back
+    # to the creation-time comparison. See infra/process_identity.py.
+    process_start_token: str | None = None
 
     def public_health_fields(self) -> dict[str, Any]:
         """Identity fields safe to expose on the loopback health endpoint."""
@@ -112,6 +118,7 @@ def build_descriptor(
     version: str,
     pid: int | None = None,
     process_create_time: float | None = None,
+    process_start_token: str | None = None,
     instance_id: str | None = None,
     capability: str | None = None,
     state: str = "starting",
@@ -141,6 +148,11 @@ def build_descriptor(
         state=state,
         version=version,
         started_at=float(started_at if started_at is not None else time.time()),
+        process_start_token=(
+            process_start_token
+            if process_start_token is not None
+            else process_start_token_for(actual_pid)
+        ),
     )
 
 
@@ -238,6 +250,10 @@ def read_descriptor(
         return None
     if not (1 <= descriptor.port <= 65535) or descriptor.pid <= 0:
         return None
+    if descriptor.process_start_token is not None and not isinstance(
+        descriptor.process_start_token, str
+    ):
+        return None
     return descriptor
 
 

package/src/superlocalmemory/infra/process_identity.py

@@ -0,0 +1,180 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Clock-independent identity for one running local process.
+
+Why this module exists
+----------------------
+A process's *wall-clock* creation time is not a stable identifier on every
+platform.  On Linux -- and therefore inside WSL2 -- psutil derives it as::
+
+    create_time = /proc/<pid>/stat:starttime / CLOCK_TICKS + /proc/stat:btime
+
+``starttime`` is boot-relative and never changes for the life of the process.
+``btime`` is the kernel's *estimate* of the boot instant, recomputed from the
+current wall clock, so any clock step moves ``btime`` and moves every process's
+computed ``create_time`` with it -- retroactively.  WSL2 periodically
+resynchronises its VM clock against the Windows host, so a ``create_time``
+recorded when the daemon started stops matching the ``create_time`` computed
+for that very same process minutes later.  Issue #104 measured a ~35 second
+divergence after roughly four minutes of uptime.
+
+Any *constant* tolerance on that comparison is a delay, not a fix: the
+divergence is unbounded and keeps growing.  The correct identifier is one the
+wall clock cannot move at all, which is what this module produces.
+
+What a start token is
+---------------------
+``process_start_token_for(pid)`` returns an opaque string identifying one
+process *instance*, derived without reference to wall-clock time, or ``None``
+when the platform cannot supply one.  Tokens are only comparable when they use
+the same scheme, so :func:`compare_start_tokens` is deliberately tri-state --
+callers fall back to a weaker signal instead of guessing.
+
+Schemes
+-------
+``lx1``
+    Linux/WSL2: ``lx1:<boot_id>:<starttime_ticks>``.  Both halves come straight
+    from procfs and are an opaque UUID and an integer tick count rather than
+    timestamps, so a clock adjustment cannot rewrite either.  ``boot_id``
+    differs across reboots, so a post-reboot PID collision can never look like
+    a match.
+``mn1``
+    Platforms where psutil exposes a monotonic creation time (macOS, NetBSD,
+    and Linux if procfs is unreadable): ``mn1:<value>``.  psutil builds its own
+    PID-reuse identity from exactly this value for exactly this reason.
+
+Windows has no monotonic variant, and needs none: its creation time comes from
+``GetProcessTimes``, a kernel timestamp that a clock adjustment does not
+rewrite.  ``process_start_token_for`` returns ``None`` there and the caller's
+creation-time comparison stays correct.
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+LINUX_SCHEME = "lx1"
+MONOTONIC_SCHEME = "mn1"
+
+_PROCFS = Path("/proc")
+# "man proc" numbers /proc/<pid>/stat fields from 1 and starttime is field 22.
+# The comm field can contain spaces and parentheses, so parsing starts after
+# the last ')', which drops fields 1 and 2 -- hence 22 - 3 == 19.
+_STARTTIME_INDEX = 19
+
+
+def _boot_id() -> str | None:
+    """Return this boot's opaque kernel identifier, or None when unavailable.
+
+    Deliberately strict: without a boot id, two processes from different boots
+    could share a PID *and* a tick count, so the token would be unsound. A
+    missing boot id therefore means "no token" rather than a weaker token.
+    """
+    try:
+        value = (_PROCFS / "sys" / "kernel" / "random" / "boot_id").read_text(
+            encoding="utf-8",
+        ).strip()
+    except OSError:
+        return None
+    return value or None
+
+
+def _linux_start_ticks(pid: int) -> int | None:
+    """Return boot-relative start ticks for a PID from procfs, or None."""
+    try:
+        data = (_PROCFS / str(pid) / "stat").read_bytes()
+    except OSError:
+        return None
+    closing = data.rfind(b")")
+    if closing < 0:
+        return None
+    fields = data[closing + 2:].split()
+    if len(fields) <= _STARTTIME_INDEX:
+        return None
+    try:
+        return int(fields[_STARTTIME_INDEX])
+    except ValueError:
+        return None
+
+
+def _linux_start_token(pid: int) -> str | None:
+    boot_id = _boot_id()
+    if boot_id is None:
+        return None
+    ticks = _linux_start_ticks(pid)
+    if ticks is None:
+        return None
+    return f"{LINUX_SCHEME}:{boot_id}:{ticks}"
+
+
+def _monotonic_start_token(pid: int) -> str | None:
+    """Return psutil's monotonic creation time as a token, or None.
+
+    ``Process._proc.create_time(monotonic=True)`` is the same private accessor
+    psutil uses internally to build ``Process._ident``.  It is guarded on every
+    axis -- missing psutil, missing attribute, platforms whose implementation
+    takes no ``monotonic`` keyword (Windows) -- and degrades to ``None``.
+    """
+    try:
+        import psutil
+
+        platform_process = getattr(psutil.Process(pid), "_proc", None)
+    except Exception:  # noqa: BLE001 - identity probing must never raise
+        return None
+    create_time = getattr(platform_process, "create_time", None)
+    if create_time is None:
+        return None
+    try:
+        raw = create_time(monotonic=True)
+    except TypeError:
+        # No monotonic variant on this platform (Windows).
+        return None
+    except Exception:  # noqa: BLE001
+        return None
+    try:
+        return f"{MONOTONIC_SCHEME}:{float(raw)!r}"
+    except (TypeError, ValueError):
+        return None
+
+
+def process_start_token_for(pid: int) -> str | None:
+    """Return a clock-independent identity token for ``pid``, or None.
+
+    ``None`` is a normal answer, not an error: it means "this platform cannot
+    prove process identity without the wall clock", and the caller should fall
+    back to comparing creation times.
+    """
+    try:
+        pid = int(pid)
+    except (TypeError, ValueError):
+        return None
+    if pid <= 0:
+        return None
+    if sys.platform.startswith("linux"):
+        token = _linux_start_token(pid)
+        if token is not None:
+            return token
+    return _monotonic_start_token(pid)
+
+
+def compare_start_tokens(recorded: str | None, observed: str | None) -> bool | None:
+    """Tri-state comparison of two start tokens.
+
+    ``True``  -- same scheme, identical value: proven the same process instance.
+    ``False`` -- same scheme, different value: proven a different instance.
+    ``None``  -- not comparable (either side missing, or different schemes);
+                 the caller must fall back rather than assume either way.
+    """
+    if not recorded or not observed:
+        return None
+    if not isinstance(recorded, str) or not isinstance(observed, str):
+        return None
+    if recorded.split(":", 1)[0] != observed.split(":", 1)[0]:
+        return None
+    return recorded == observed