superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/core/embeddings.py

@@ -1,1052 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SuperLocalMemory V3 — Embedding Service (Subprocess-Isolated).
-
-All PyTorch/model work runs in a SEPARATE subprocess. The main process
-(dashboard, MCP, CLI) never imports torch and stays at ~60 MB.
-
-The worker subprocess has a configurable idle timeout and respawns on the
-next embed call when it has been unloaded.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-"""
-
-from __future__ import annotations
-
-import atexit
-import json
-import logging
-import os
-import subprocess
-import sys
-import threading
-import time
-import weakref
-from contextlib import contextmanager
-from pathlib import Path
-from typing import TYPE_CHECKING, Iterator
-
-import numpy as np
-
-from superlocalmemory.core.config import EmbeddingConfig
-
-# Track all live embedding services for atexit cleanup
-_live_embedding_services: set[weakref.ref] = set()
-
-if TYPE_CHECKING:
-    from numpy.typing import NDArray
-
-logger = logging.getLogger(__name__)
-
-# Fisher variance constants
-_FISHER_VAR_MIN = 0.05
-_FISHER_VAR_MAX = 2.0
-_FISHER_VAR_RANGE = _FISHER_VAR_MAX - _FISHER_VAR_MIN
-
-
-class DimensionMismatchError(RuntimeError):
-    """Raised when the actual embedding dimension differs from config."""
-
-
-# ---------------------------------------------------------------------------
-# V3.3.28: System-wide concurrency guard for embedding workers.
-#
-# The memory blast incident (April 7, 2026) was caused by 20+ concurrent
-# `slm observe` CLI processes each spawning their own embedding_worker
-# subprocess (1.4 GB each). This file lock ensures only MAX_CONCURRENT
-# embedding workers can exist across ALL processes on the machine.
-#
-# Primary defense: daemon routing (cmd_observe → daemon → singleton engine).
-# This lock is the secondary safety net for when the daemon isn't available.
-# ---------------------------------------------------------------------------
-
-_MAX_CONCURRENT_WORKERS = int(os.environ.get("SLM_MAX_EMBEDDING_WORKERS", 1))
-_embedding_lock_fd: int | None = None
-_embedding_lock_state_guard = threading.Lock()
-
-
-def _embedding_lock_file() -> Path:
-    from superlocalmemory.infra.data_root import state_path
-
-    return state_path(".embedding.lock")
-
-
-def _embedding_pid_file() -> Path:
-    from superlocalmemory.infra.data_root import state_path
-
-    return state_path(".embedding-worker.pid")
-
-
-def _is_embedding_worker_alive() -> bool:
-    """Check if an embedding worker PID file exists and that PID is alive.
-
-    v3.4.13: Machine-wide singleton guard. Before spawning a new worker,
-    check if one is already running. Prevents duplicate 1.6GB workers.
-    """
-    try:
-        pid_file = _embedding_pid_file()
-        if not pid_file.exists():
-            return False
-        pid = int(pid_file.read_text().strip())
-        os.kill(pid, 0)  # Signal 0 = check if alive
-        return True
-    except (ValueError, OSError, ProcessLookupError):
-        # PID file invalid or process dead — clean up stale file
-        _embedding_pid_file().unlink(missing_ok=True)
-        return False
-
-
-def register_embedding_worker_pid(pid: int) -> None:
-    """Write the embedding worker PID to the machine-wide PID file."""
-    pid_file = _embedding_pid_file()
-    pid_file.parent.mkdir(parents=True, exist_ok=True)
-    pid_file.write_text(str(pid))
-
-
-def acquire_embedding_lock(timeout: float = 5.0) -> bool:
-    """Acquire system-wide embedding worker lock.
-
-    The caller must re-check the PID file after acquisition before spawning.
-    POSIX uses flock; Windows uses a one-byte msvcrt lock.
-    Returns True if lock acquired (safe to spawn), False if another worker active.
-    """
-    global _embedding_lock_fd
-
-    # Serialize local contenders as well as cross-process contenders. The file
-    # descriptor stays local until its OS lock succeeds, so a failed acquire
-    # can never overwrite and leak the descriptor that owns the live worker.
-    with _embedding_lock_state_guard:
-        if _embedding_lock_fd is not None:
-            return False
-        if _is_embedding_worker_alive():
-            return False
-
-        lock_file = _embedding_lock_file()
-        lock_file.parent.mkdir(parents=True, exist_ok=True)
-        candidate_fd: int | None = None
-        try:
-            candidate_fd = os.open(str(lock_file), os.O_CREAT | os.O_RDWR)
-            deadline = time.time() + timeout
-            while time.time() < deadline:
-                try:
-                    if sys.platform == "win32":
-                        import msvcrt
-
-                        if os.fstat(candidate_fd).st_size == 0:
-                            os.write(candidate_fd, b"\0")
-                        os.lseek(candidate_fd, 0, os.SEEK_SET)
-                        msvcrt.locking(candidate_fd, msvcrt.LK_NBLCK, 1)
-                    else:
-                        import fcntl
-
-                        fcntl.flock(
-                            candidate_fd,
-                            fcntl.LOCK_EX | fcntl.LOCK_NB,
-                        )
-                    _embedding_lock_fd = candidate_fd
-                    return True
-                except (BlockingIOError, OSError):
-                    time.sleep(0.2)
-            os.close(candidate_fd)
-            return False
-        except Exception:
-            if candidate_fd is not None:
-                try:
-                    os.close(candidate_fd)
-                except OSError:
-                    pass
-            return False
-
-
-def release_embedding_lock() -> None:
-    """Release system-wide embedding worker lock."""
-    global _embedding_lock_fd
-    with _embedding_lock_state_guard:
-        if _embedding_lock_fd is None:
-            return
-        try:
-            if sys.platform == "win32":
-                import msvcrt
-
-                os.lseek(_embedding_lock_fd, 0, os.SEEK_SET)
-                msvcrt.locking(_embedding_lock_fd, msvcrt.LK_UNLCK, 1)
-            else:
-                import fcntl
-
-                fcntl.flock(_embedding_lock_fd, fcntl.LOCK_UN)
-            os.close(_embedding_lock_fd)
-        except Exception:
-            try:
-                os.close(_embedding_lock_fd)
-            except OSError:
-                pass
-        _embedding_lock_fd = None
-
-
-_IDLE_TIMEOUT_SECONDS = 1800  # 30 minutes — keep interactive sessions warm.
-# V3.8.1: existing-user soaks showed that the five-minute policy repeatedly
-# recycled a ~1.1 GB local model and imposed 20-30 second cold starts. The
-# explicit environment override remains available for low-RAM installations.
-_IDLE_TIMEOUT_SECONDS = int(os.environ.get("SLM_EMBED_IDLE_TIMEOUT", _IDLE_TIMEOUT_SECONDS))
-# V3.3.21: Configurable response timeout — 180s default, but batch ingestion
-# (2-turn chunks across 10 conversations) needs 600s+ to survive cold-start
-# model downloads and ARM64 ONNX compilation pauses.
-_SUBPROCESS_RESPONSE_TIMEOUT = int(os.environ.get("SLM_EMBED_RESPONSE_TIMEOUT", 180))
-# V3.3.21: Increase recycle threshold to 5000 (was 1000). With 2-turn chunks,
-# a single conversation produces ~50-80 store calls. 10 conversations = 500-800.
-# Recycling at 1000 caused mid-ingestion worker death → timeout cascade.
-_WORKER_RECYCLE_AFTER = int(os.environ.get("SLM_EMBED_RECYCLE_AFTER", 5000))
-
-
-class EmbeddingService:
-    """Subprocess-isolated embedding service.
-
-    All model inference runs in a child process. The main process never
-    imports torch/sentence-transformers, keeping its memory at ~60 MB.
-
-    The worker auto-kills after 2 min idle. First embed after idle takes
-    ~3 sec (model reload). Subsequent embeds are instant (<100ms).
-    """
-
-    def __init__(self, config: EmbeddingConfig) -> None:
-        self._config = config
-        self._lock = threading.Lock()
-        self._worker_proc: subprocess.Popen | None = None
-        self._available = True
-        self._last_used: float = 0.0
-        self._idle_timer: threading.Timer | None = None
-        self._worker_ready = False
-        self._owns_worker_lock = False
-        self._request_count: int = 0
-        self._http_client: object | None = None
-        self._remote_ready = False
-
-        # Register for atexit cleanup (prevent orphaned workers)
-        ref = weakref.ref(self, _live_embedding_services.discard)
-        _live_embedding_services.add(ref)
-
-    def __del__(self) -> None:
-        """Kill worker subprocess when service is garbage-collected."""
-        try:
-            self._kill_worker()
-        except Exception:
-            pass
-        try:
-            if self._http_client is not None:
-                self._http_client.close()
-        except Exception:
-            pass
-
-    @property
-    def is_available(self) -> bool:
-        """Check if embedding service can produce embeddings."""
-        if self._config.is_openai_compatible:
-            return bool(self._config.api_endpoint)
-        if self._config.is_cloud:
-            return bool(self._config.api_endpoint and self._config.api_key)
-        return self._available
-
-    @property
-    def is_warm(self) -> bool:
-        """Return whether the configured backend has served a request.
-
-        ``is_available`` only means that the backend may be started.  A local
-        sentence-transformers cold start can take minutes on Apple Silicon, so
-        background enrichment must not mistake availability for readiness and
-        occupy the only worker ahead of an interactive recall.
-        """
-        config = getattr(self, "_config", None)
-        if config is not None and (
-            config.is_openai_compatible or config.is_cloud
-        ):
-            return bool(
-                getattr(self, "_remote_ready", False)
-                and self.is_available
-            )
-        proc = getattr(self, "_worker_proc", None)
-        if proc is None or getattr(self, "_request_count", 0) <= 0:
-            return False
-        try:
-            return proc.poll() is None
-        except Exception:
-            return False
-
-    @property
-    def dimension(self) -> int:
-        return self._config.dimension
-
-    def unload(self, timeout: float = 1.0) -> bool:
-        """Release the worker without blocking daemon shutdown on an embed call.
-
-        An in-flight request owns ``_lock`` while it waits for the worker's
-        response.  Shutdown must not wait behind a wedged response: callers
-        can continue teardown and the worker process will be handled by the
-        process supervisor if necessary.
-        """
-        if not self._lock.acquire(timeout=max(0.0, timeout)):
-            logger.warning("EmbeddingService: unload skipped; embed worker is busy")
-            return False
-        try:
-            self._kill_worker()
-            logger.info("EmbeddingService: worker killed (idle timeout)")
-            return True
-        finally:
-            self._lock.release()
-
-    def shutdown(self, timeout: float = 1.0) -> None:
-        """Force bounded process teardown even when an embed call owns the lock.
-
-        Shutdown is stronger than the idle-time ``unload`` operation.  Once
-        the engine is closing, no new request may use this service, so it is
-        safe to detach and terminate a wedged child without waiting behind the
-        request lock.
-        """
-        acquired = self._lock.acquire(timeout=max(0.0, timeout))
-        try:
-            self._kill_worker(timeout=min(max(0.0, timeout), 1.0))
-        finally:
-            if acquired:
-                self._lock.release()
-
-    # ------------------------------------------------------------------
-    # Public API
-    # ------------------------------------------------------------------
-
-    @contextmanager
-    def _request_lock(self) -> Iterator[None]:
-        """Admit one model request without starving an interactive recall.
-
-        A background caller may pass the recall gate and then queue behind an
-        in-flight model request. If a recall arrives while it is queued, a
-        plain mutex can let that background caller retake the worker first.
-        Re-check the gate after acquiring the mutex so at most the already
-        running background request can delay a newly arrived recall.
-        """
-        from superlocalmemory.core.recall_gate import (
-            in_flight,
-            is_background_work,
-            wait_for_foreground_idle,
-        )
-
-        if not is_background_work():
-            with self._lock:
-                yield
-            return
-
-        while True:
-            wait_for_foreground_idle()
-            self._lock.acquire()
-            if in_flight() == 0:
-                break
-            self._lock.release()
-        try:
-            yield
-        finally:
-            self._lock.release()
-
-    def embed(self, text: str) -> list[float] | None:
-        """Embed a single text string. Returns list of floats or None."""
-        if not text or not text.strip():
-            raise ValueError("Cannot embed empty text")
-        from superlocalmemory.core.recall_gate import wait_for_foreground_idle
-        wait_for_foreground_idle()
-        if self._config.is_openai_compatible:
-            try:
-                vecs = self._openai_compatible_embed_batch([text])
-                vec = vecs[0]
-                self._validate_dimension(np.asarray(vec))
-                self._remote_ready = True
-                return vec
-            except Exception:
-                self._remote_ready = False
-                raise
-        if self._config.is_cloud:
-            try:
-                vec = self._cloud_embed_single(text)
-                self._validate_dimension(np.asarray(vec))
-                self._remote_ready = True
-                return vec
-            except Exception:
-                self._remote_ready = False
-                raise
-        result = self._subprocess_embed([text])
-        if result is None:
-            return None
-        vec = result[0]
-        self._validate_dimension(np.asarray(vec))
-        return vec
-
-    def embed_batch(self, texts: list[str]) -> list[list[float] | None]:
-        """Embed a batch of texts."""
-        if not texts:
-            raise ValueError("Cannot embed empty batch")
-        from superlocalmemory.core.recall_gate import is_background_work
-        if is_background_work():
-            # A single large background batch can own the only local inference
-            # worker for tens of seconds. Slice it so a recall arriving after
-            # this call started gets priority before the next text.
-            return [self.embed(text) for text in texts]
-        if self._config.is_openai_compatible:
-            try:
-                results = self._openai_compatible_embed_batch(texts)
-                for vec in results:
-                    if vec is not None:
-                        self._validate_dimension(np.asarray(vec))
-                self._remote_ready = any(vec is not None for vec in results)
-                return results
-            except Exception:
-                self._remote_ready = False
-                raise
-        if self._config.is_cloud:
-            try:
-                results = self._cloud_embed_batch(texts)
-                for vec in results:
-                    if vec is not None:
-                        self._validate_dimension(np.asarray(vec))
-                self._remote_ready = any(vec is not None for vec in results)
-                return results
-            except Exception:
-                self._remote_ready = False
-                raise
-        result = self._subprocess_embed(texts)
-        if result is None:
-            return [None] * len(texts)
-        for vec in result:
-            if vec is not None:
-                self._validate_dimension(np.asarray(vec))
-        return result
-
-    def compute_fisher_params(
-        self, embedding: list[float],
-    ) -> tuple[list[float], list[float]]:
-        """Compute Fisher-Rao parameters from a raw embedding."""
-        arr = np.asarray(embedding, dtype=np.float64)
-        norm = float(np.linalg.norm(arr))
-        if norm < 1e-10:
-            mean = np.zeros(len(arr), dtype=np.float64)
-            variance = np.full(len(arr), _FISHER_VAR_MAX, dtype=np.float64)
-            return mean.tolist(), variance.tolist()
-        mean = arr / norm
-        abs_mean = np.abs(mean)
-        max_val = float(np.max(abs_mean)) + 1e-10
-        signal_strength = abs_mean / max_val
-        variance = _FISHER_VAR_MAX - _FISHER_VAR_RANGE * signal_strength
-        variance = np.clip(variance, _FISHER_VAR_MIN, _FISHER_VAR_MAX)
-        return mean.tolist(), variance.tolist()
-
-    # ------------------------------------------------------------------
-    # Subprocess worker management
-    # ------------------------------------------------------------------
-
-    def _subprocess_embed(self, texts: list[str]) -> list[list[float]] | None:
-        """Send texts to worker subprocess, get embeddings back.
-
-        Includes a timeout (_SUBPROCESS_RESPONSE_TIMEOUT seconds) so the CLI
-        never hangs indefinitely on cold model loads or network issues.
-        """
-        with self._request_lock():
-            # Only an explicit terminal disable (``False``) short-circuits. A
-            # ``None`` availability is the recall-health self-heal's "re-probe"
-            # signal (recall_health._heal_embedder) — it must fall through and
-            # respawn the worker, matching OllamaEmbedder's tri-state
-            # convention. Using ``not self._available`` here bricked the local
-            # worker on the first heal tick, because ``None`` is falsy.
-            if self._available is False:
-                return None
-            # Worker recycling: restart after N requests to prevent
-            # C++ allocator fragmentation over long-running sessions.
-            if self._request_count >= _WORKER_RECYCLE_AFTER and self._worker_proc is not None:
-                logger.info("Recycling embedding worker after %d requests", self._request_count)
-                self._kill_worker()
-                self._request_count = 0
-
-            self._ensure_worker()
-            if self._worker_proc is None:
-                return None
-
-            req = json.dumps({
-                "cmd": "embed",
-                "texts": texts,
-                "model_name": self._config.model_name,
-                "dimension": self._config.dimension,
-            }) + "\n"
-
-            try:
-                self._worker_proc.stdin.write(req)
-                self._worker_proc.stdin.flush()
-                resp_line = self._readline_with_timeout(
-                    self._worker_proc.stdout,
-                    _SUBPROCESS_RESPONSE_TIMEOUT,
-                )
-                if not resp_line:
-                    logger.warning(
-                        "Embedding worker timed out after %ds. "
-                        "Run 'slm setup' to download models and verify installation.",
-                        _SUBPROCESS_RESPONSE_TIMEOUT,
-                    )
-                    # Print to stderr so CLI users see this even without logging
-                    print(
-                        f"\n⚠ Embedding worker did not respond within "
-                        f"{_SUBPROCESS_RESPONSE_TIMEOUT}s.\n"
-                        f"  Run: slm setup   (download models + verify)\n"
-                        f"  Run: slm doctor  (diagnose issues)\n",
-                        file=sys.stderr,
-                    )
-                    self._kill_worker()
-                    return None
-                resp = json.loads(resp_line)
-                if not resp.get("ok"):
-                    logger.warning("Worker error: %s", resp.get("error"))
-                    # A well-formed worker error is a terminal local
-                    # dependency/model failure, not a transient pipe race.
-                    # Disable this service and terminate the child so every
-                    # recall does not respawn a heavyweight failing process.
-                    self._available = False
-                    self._kill_worker()
-                    return None
-                # A successful embed proves the worker is healthy, so clear any
-                # transient/``None`` availability left by a self-heal re-probe
-                # back to a definite ``True``. Without this the flag lingers at
-                # ``None`` and the next ``not``-style check elsewhere re-blocks.
-                self._available = True
-                self._reset_idle_timer()
-                self._request_count += 1
-                return resp["vectors"]
-            except (BrokenPipeError, OSError, json.JSONDecodeError) as exc:
-                logger.warning(
-                    "Embedding worker communication failed: %s — respawning.",
-                    exc,
-                )
-                self._kill_worker()
-                # V3.3.16: Auto-retry once after worker death (RSS watchdog
-                # or crash). Respawn + re-send instead of returning None.
-                try:
-                    self._ensure_worker()
-                    if self._worker_proc is not None:
-                        self._worker_proc.stdin.write(req)
-                        self._worker_proc.stdin.flush()
-                        resp_line = self._readline_with_timeout(
-                            self._worker_proc.stdout,
-                            _SUBPROCESS_RESPONSE_TIMEOUT,
-                        )
-                        if resp_line:
-                            resp = json.loads(resp_line)
-                            if resp.get("ok"):
-                                self._reset_idle_timer()
-                                self._request_count = 1
-                                return resp["vectors"]
-                except Exception:
-                    self._kill_worker()
-                return None
-
-    @staticmethod
-    def _readline_with_timeout(stream, timeout_seconds: float) -> str:
-        """Read a line from stream with a timeout. Returns '' on timeout.
-
-        Prefer a deadline-driven selector poll of the stream's file descriptor
-        (POSIX pipes). That path never spawns a helper thread, so a hung
-        embedding worker cannot leak reader threads or pin the pipe FD across
-        timeouts. A thread fallback remains only for streams without a usable
-        fileno (unit-test mocks) and for Windows, where selectors cannot wait
-        on pipes.
-        """
-        import selectors
-
-        timeout_seconds = max(0.0, float(timeout_seconds))
-        fd: int | None
-        try:
-            raw_fd = stream.fileno()
-            fd = raw_fd if isinstance(raw_fd, int) else None
-        except (AttributeError, OSError, ValueError, TypeError):
-            fd = None
-
-        # Windows select()/selectors only accept sockets, not subprocess pipes.
-        if fd is not None and sys.platform != "win32":
-            try:
-                with selectors.DefaultSelector() as sel:
-                    sel.register(fd, selectors.EVENT_READ)
-                    events = sel.select(timeout=timeout_seconds)
-                if not events:
-                    logger.warning(
-                        "Embedding worker did not respond within %ds",
-                        timeout_seconds,
-                    )
-                    return ""
-                # Readable or EOF. Protocol is one JSON line per response;
-                # the worker writes a complete line before we are woken.
-                line = stream.readline()
-                return line if line else ""
-            except (OSError, ValueError) as exc:
-                # Closed/invalid FD mid-wait — same as empty to the caller
-                # (which kills and may respawn the worker).
-                logger.debug("Embedding readline selector failed: %s", exc)
-                return ""
-
-        result_container: list[str] = []
-        error_container: list[Exception] = []
-
-        def _read() -> None:
-            try:
-                result_container.append(stream.readline())
-            except Exception as exc:
-                error_container.append(exc)
-
-        # Name contains ``_read`` so leak detectors can find abandoned readers.
-        reader = threading.Thread(
-            target=_read, daemon=True, name="slm_embed_readline_read",
-        )
-        reader.start()
-        reader.join(timeout=timeout_seconds)
-
-        if reader.is_alive():
-            logger.warning(
-                "Embedding worker did not respond within %ds", timeout_seconds,
-            )
-            # Close/shutdown the stream so the blocked readline() returns and
-            # the reader thread can exit. Raising alone would leak the thread
-            # (and its FD) on Windows pipes and fileno-less mocks.
-            for closer_name in ("close", "shutdown"):
-                closer = getattr(stream, closer_name, None)
-                if not callable(closer):
-                    continue
-                try:
-                    if closer_name == "shutdown":
-                        try:
-                            closer(True)  # type: ignore[misc]
-                        except TypeError:
-                            closer()
-                    else:
-                        closer()
-                except Exception:
-                    pass
-            # Bound the join so a stuck closer cannot hang the caller forever.
-            reader.join(timeout=min(1.0, max(0.05, timeout_seconds)))
-            return ""
-        if error_container:
-            raise error_container[0]
-        return result_container[0] if result_container else ""
-
-    @staticmethod
-    def _check_memory_pressure() -> bool:
-        """Check if system has enough memory to spawn a worker.
-
-        V3.3.28: Prevents spawning embedding workers (1.4 GB each) when
-        the system is already under memory pressure. Returns True if safe.
-        """
-        min_available_gb = float(os.environ.get("SLM_MIN_AVAILABLE_MEMORY_GB", "2.0"))
-        try:
-            if sys.platform == "darwin":
-                # macOS: use vm_stat to get free + inactive pages
-                import subprocess as _sp
-                result = _sp.run(["vm_stat"], capture_output=True, text=True, timeout=5)
-                if result.returncode == 0:
-                    lines = result.stdout.split("\n")
-                    page_size = 16384  # default on Apple Silicon
-                    free_pages = 0
-                    for line in lines:
-                        if "page size of" in line:
-                            try:
-                                page_size = int(line.split()[-2])
-                            except (ValueError, IndexError):
-                                pass
-                        if "Pages free" in line or "Pages inactive" in line:
-                            try:
-                                free_pages += int(line.split()[-1].rstrip("."))
-                            except (ValueError, IndexError):
-                                pass
-                    available_gb = (free_pages * page_size) / (1024 ** 3)
-                    if available_gb < min_available_gb:
-                        logger.warning(
-                            "Low memory (%.1f GB available, need %.1f GB) — "
-                            "deferring embedding worker spawn",
-                            available_gb, min_available_gb,
-                        )
-                        return False
-            else:
-                # Linux/other: use /proc/meminfo or psutil
-                try:
-                    with open("/proc/meminfo") as f:
-                        for line in f:
-                            if line.startswith("MemAvailable:"):
-                                available_kb = int(line.split()[1])
-                                available_gb = available_kb / (1024 * 1024)
-                                if available_gb < min_available_gb:
-                                    logger.warning(
-                                        "Low memory (%.1f GB available) — "
-                                        "deferring embedding worker spawn",
-                                        available_gb,
-                                    )
-                                    return False
-                                break
-                except FileNotFoundError:
-                    pass  # Not Linux, allow through
-        except Exception:
-            pass  # On error, allow through (don't block functionality)
-        return True
-
-    def _ensure_worker(self) -> None:
-        """Spawn worker subprocess if not running.
-
-        v3.4.13: Machine-wide singleton — checks PID file before spawning.
-        Only ONE embedding_worker can exist at a time on the machine.
-        """
-        if self._worker_proc is not None:
-            if self._worker_proc.poll() is None:
-                return
-            # An unexpectedly exited child still leaves this service holding
-            # its lifetime flock. Fully close the dead process and release that
-            # lock before attempting the normal acquire/spawn sequence. Merely
-            # dropping the Popen reference makes the process deadlock against
-            # its own old flock until the acquire timeout expires.
-            self._kill_worker()
-
-        # Serialize the check/spawn/register sequence across processes. Checking
-        # the PID file without this lock allows two cold callers to both see no
-        # worker and launch memory-heavy children.
-        if not acquire_embedding_lock():
-            logger.debug("Embedding worker owned by another process")
-            self._available = False
-            return
-        if _is_embedding_worker_alive():
-            release_embedding_lock()
-            logger.debug("Embedding worker already alive after lock acquisition")
-            self._available = False
-            return
-
-        # V3.3.28: Check memory pressure before spawning
-        if not self._check_memory_pressure():
-            release_embedding_lock()
-            logger.warning("Skipping embedding worker spawn due to memory pressure")
-            self._available = False
-            return
-
-        worker_module = "superlocalmemory.core.embedding_worker"
-        try:
-            env = {
-                **os.environ,
-                "CUDA_VISIBLE_DEVICES": "",
-                "PYTORCH_MPS_HIGH_WATERMARK_RATIO": "0.0",
-                "PYTORCH_MPS_MEM_LIMIT": "0",
-                "PYTORCH_ENABLE_MPS_FALLBACK": "1",
-                "TOKENIZERS_PARALLELISM": "false",
-                "TORCH_DEVICE": "cpu",
-                "ORT_DISABLE_COREML": "1",
-                # Restore parallel OpenMP. The package caps OMP_NUM_THREADS
-                # globally to avoid a torch+lightgbm libomp SIGSEGV in the
-                # main process. This worker loads torch but never lightgbm,
-                # so there is no collision risk and full parallelism is safe.
-                "OMP_NUM_THREADS": str(os.cpu_count() or 4),
-            }
-            from superlocalmemory.core.platform_utils import popen_platform_kwargs
-            self._worker_proc = subprocess.Popen(
-                [sys.executable, "-m", worker_module],
-                stdin=subprocess.PIPE,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.DEVNULL,
-                text=True,
-                bufsize=1,
-                env=env,
-                **popen_platform_kwargs(),
-            )
-            # v3.4.13: Register PID for machine-wide singleton guard
-            register_embedding_worker_pid(self._worker_proc.pid)
-            self._owns_worker_lock = True
-            logger.info("Embedding worker spawned (PID %d)", self._worker_proc.pid)
-            self._worker_ready = True
-        except Exception as exc:
-            failed_proc = self._worker_proc
-            if failed_proc is not None:
-                try:
-                    failed_proc.terminate()
-                    failed_proc.wait(timeout=3)
-                except Exception:
-                    try:
-                        failed_proc.kill()
-                        failed_proc.wait(timeout=3)
-                    except Exception:
-                        pass
-            release_embedding_lock()
-            self._owns_worker_lock = False
-            logger.warning(
-                "Failed to spawn embedding worker: %s. "
-                "Run 'slm doctor' to verify your Python environment. "
-                "Using Python: %s",
-                exc, sys.executable,
-            )
-            self._available = False
-            self._worker_proc = None
-
-    def _kill_worker(self, timeout: float = 3.0) -> None:
-        """Terminate the worker and close every owned pipe exactly once."""
-        if self._idle_timer is not None:
-            self._idle_timer.cancel()
-            self._idle_timer = None
-
-        proc = self._worker_proc
-        if proc is not None:
-            # Detach first so re-entrant/finalizer cleanup is idempotent.
-            self._worker_proc = None
-            self._worker_ready = False
-            try:
-                proc.stdin.write('{"cmd":"quit"}\n')
-                proc.stdin.flush()
-                proc.wait(timeout=max(0.0, timeout))
-            except Exception:
-                try:
-                    returncode = proc.poll()
-                except Exception:
-                    returncode = None
-                # MagicMock/unknown poll results are treated conservatively as
-                # live; a real exited child always reports an integer code.
-                if returncode is None or not isinstance(returncode, int):
-                    try:
-                        proc.kill()
-                        proc.wait(timeout=max(0.0, timeout))
-                    except Exception:
-                        pass
-            finally:
-                # TextIOWrapper.close() can itself raise BrokenPipeError while
-                # flushing buffered stdin. Suppress it here, while the stream
-                # is still strongly referenced, so it cannot surface later as
-                # an unraisable finalizer warning.
-                for stream_name in ("stdin", "stdout", "stderr"):
-                    stream = getattr(proc, stream_name, None)
-                    if stream is not None:
-                        try:
-                            stream.close()
-                        except (BrokenPipeError, OSError, ValueError):
-                            pass
-        if getattr(self, "_owns_worker_lock", False):
-            try:
-                pid_file = _embedding_pid_file()
-                if (
-                    proc is not None
-                    and pid_file.exists()
-                    and pid_file.read_text().strip() == str(proc.pid)
-                ):
-                    pid_file.unlink(missing_ok=True)
-            except (OSError, ValueError):
-                pass
-            finally:
-                self._owns_worker_lock = False
-                release_embedding_lock()
-
-    def _reset_idle_timer(self) -> None:
-        """Reset the configurable worker-idle timer.
-
-        F5 fix: opportunistically evict the worker mid-window when memory
-        pressure is detected (``_check_memory_pressure`` returns False).
-        This prevents the idle-timeout window from holding a worker alive
-        while system memory is exhausted.
-        """
-        if self._idle_timer is not None:
-            self._idle_timer.cancel()
-        if not self._check_memory_pressure():
-            # Pressure detected — kill worker immediately; do not schedule a
-            # new idle timer so no further embedding work is attempted until
-            # the next explicit request reloads the worker.
-            self._kill_worker()
-            self._idle_timer = None
-            return
-        self._idle_timer = threading.Timer(
-            _IDLE_TIMEOUT_SECONDS, self.unload,
-        )
-        self._idle_timer.daemon = True
-        self._idle_timer.start()
-        self._last_used = time.time()
-
-    # ------------------------------------------------------------------
-    # OpenAI-compatible embedding (V3.4.24 — any /v1/embeddings endpoint)
-    # ------------------------------------------------------------------
-
-    def _get_http_client(self):
-        """Reusable httpx client for OpenAI-compatible endpoints."""
-        if self._http_client is None:
-            import httpx
-            self._http_client = httpx.Client(
-                timeout=httpx.Timeout(connect=5.0, read=30.0, write=10.0, pool=5.0),
-            )
-        return self._http_client
-
-    def _openai_compatible_embed_batch(
-        self, texts: list[str], *, max_retries: int = 3,
-    ) -> list[list[float]]:
-        """Encode via any OpenAI-compatible embedding API.
-
-        V3.4.24: Standard ``/v1/embeddings`` format. Works with Ollama,
-        vLLM, LiteLLM, text-embeddings-inference, and any endpoint that
-        implements the OpenAI embeddings spec.
-        """
-        endpoint = self._config.api_endpoint.rstrip("/")
-        if not endpoint.endswith("/embeddings"):
-            endpoint = f"{endpoint}/embeddings"
-        headers = {"Content-Type": "application/json"}
-        if self._config.api_key:
-            headers["Authorization"] = f"Bearer {self._config.api_key}"
-        body = {
-            "input": texts,
-            "model": self._config.model_name,
-        }
-
-        client = self._get_http_client()
-        last_error: Exception | None = None
-        for attempt in range(max_retries):
-            from superlocalmemory.core.materialization_control import (
-                MaterializationDeferred,
-            )
-            from superlocalmemory.core.recall_gate import (
-                background_preempt_requested,
-                is_background_work,
-            )
-            if background_preempt_requested():
-                raise MaterializationDeferred(
-                    "background embedding yielded to runtime transition"
-                )
-            try:
-                request_kwargs = {"headers": headers, "json": body}
-                if is_background_work():
-                    # Runtime reconfigure drains admitted operations in five
-                    # seconds.  A background remote read must leave enough
-                    # scheduling margin to observe that transition and release
-                    # its lease, while interactive recall keeps the provider's
-                    # normal timeout budget.
-                    request_kwargs["timeout"] = 3.5
-                resp = client.post(endpoint, **request_kwargs)
-                resp.raise_for_status()
-                if background_preempt_requested():
-                    raise MaterializationDeferred(
-                        "background embedding yielded to runtime transition"
-                    )
-                data = resp.json()
-                if "data" not in data or not isinstance(data["data"], list):
-                    raise ValueError(
-                        f"Unexpected response: missing 'data' array. Keys: {list(data.keys())}"
-                    )
-                results: list[list[float]] = []
-                for item in sorted(data["data"], key=lambda d: d["index"]):
-                    results.append(item["embedding"])
-                if len(results) != len(texts):
-                    logger.warning(
-                        "Embedding count mismatch: sent %d texts, got %d vectors",
-                        len(texts), len(results),
-                    )
-                return results
-            except Exception as exc:
-                if isinstance(exc, MaterializationDeferred):
-                    raise
-                if background_preempt_requested():
-                    raise MaterializationDeferred(
-                        "background embedding yielded to runtime transition"
-                    ) from exc
-                last_error = exc
-                if attempt < max_retries - 1:
-                    time.sleep(2 ** attempt)
-        raise RuntimeError(
-            f"OpenAI-compatible embedding failed after {max_retries} retries: "
-            f"{last_error}"
-        )
-
-    # ------------------------------------------------------------------
-    # Cloud embedding (no subprocess needed — just HTTP)
-    # ------------------------------------------------------------------
-
-    def _cloud_embed_single(self, text: str) -> list[float]:
-        vecs = self._cloud_embed_batch([text])
-        return vecs[0]
-
-    def _cloud_embed_batch(
-        self, texts: list[str], *, max_retries: int = 3,
-    ) -> list[list[float]]:
-        """Encode via Azure OpenAI embedding API with retry.
-
-        V3.6.14: Reuses self._get_http_client() (shared persistent connection)
-        instead of creating a fresh httpx.Client per call/retry.  resp.json()
-        is now called inside the try block while the response object is still
-        in scope, eliminating reliance on httpx body-buffering after close.
-        """
-        url = (
-            f"{self._config.api_endpoint.rstrip('/')}/openai/deployments/"
-            f"{self._config.deployment_name}/embeddings"
-            f"?api-version={self._config.api_version}"
-        )
-        headers = {
-            "Content-Type": "application/json",
-            "api-key": self._config.api_key,
-        }
-        body = {"input": texts, "model": self._config.deployment_name}
-        client = self._get_http_client()
-        last_error: Exception | None = None
-        for attempt in range(max_retries):
-            from superlocalmemory.core.materialization_control import (
-                MaterializationDeferred,
-            )
-            from superlocalmemory.core.recall_gate import (
-                background_preempt_requested,
-                is_background_work,
-            )
-            if background_preempt_requested():
-                raise MaterializationDeferred(
-                    "background embedding yielded to runtime transition"
-                )
-            try:
-                request_kwargs = {"headers": headers, "json": body}
-                if is_background_work():
-                    request_kwargs["timeout"] = 3.5
-                resp = client.post(url, **request_kwargs)
-                resp.raise_for_status()
-                if background_preempt_requested():
-                    raise MaterializationDeferred(
-                        "background embedding yielded to runtime transition"
-                    )
-                data = resp.json()
-                results = []
-                for item in sorted(data["data"], key=lambda d: d["index"]):
-                    results.append(item["embedding"])
-                return results
-            except Exception as exc:
-                if isinstance(exc, MaterializationDeferred):
-                    raise
-                if background_preempt_requested():
-                    raise MaterializationDeferred(
-                        "background embedding yielded to runtime transition"
-                    ) from exc
-                last_error = exc
-                if attempt < max_retries - 1:
-                    time.sleep(2 ** attempt)
-        raise RuntimeError(f"Cloud embedding failed: {last_error}")
-
-    # ------------------------------------------------------------------
-    # Validation
-    # ------------------------------------------------------------------
-
-    def _validate_dimension(self, vec: NDArray) -> None:
-        actual = len(vec)
-        if actual != self._config.dimension:
-            raise DimensionMismatchError(
-                f"Embedding dimension {actual} != expected {self._config.dimension}"
-            )
-
-
-# ---------------------------------------------------------------------------
-# Module-level atexit: kill ALL embedding workers on process exit
-# ---------------------------------------------------------------------------
-
-def _cleanup_all_embedding_services() -> None:
-    """Kill all embedding worker subprocesses on interpreter exit.
-
-    Prevents orphaned 500-800 MB sentence-transformer workers surviving
-    after parent exits (especially during test runs with parallel agents).
-    """
-    for ref in list(_live_embedding_services):
-        svc = ref()
-        if svc is not None:
-            try:
-                svc._kill_worker()
-            except Exception:
-                pass
-    _live_embedding_services.clear()
-
-
-atexit.register(_cleanup_all_embedding_services)