superlocalmemory 3.4.0 → 3.4.3

package/src/superlocalmemory/core/health_monitor.py

@@ -0,0 +1,313 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Enterprise-grade health monitoring for the SLM Unified Daemon.
+
+Monitors:
+  - Global RSS budget (kill heaviest worker if over limit)
+  - Worker heartbeat (kill unresponsive workers after 60s)
+  - Structured JSON logging (daemon.json.log alongside text logs)
+  - Extensible health check registry (Phase C/D/E add checks)
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import threading
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Callable
+
+logger = logging.getLogger("superlocalmemory.health_monitor")
+
+# Try psutil — graceful fallback if not available
+try:
+    import psutil
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+    logger.info("psutil not available — health monitoring limited")
+
+
+# ---------------------------------------------------------------------------
+# Health Check Registry (extensible by other phases)
+# ---------------------------------------------------------------------------
+
+_HEALTH_CHECKS: list[Callable[[], dict]] = []
+
+
+def register_health_check(check_fn: Callable[[], dict]) -> None:
+    """Register a health check function. Returns dict with name, status, detail."""
+    _HEALTH_CHECKS.append(check_fn)
+
+
+def run_all_health_checks() -> list[dict]:
+    """Run all registered health checks. Returns list of results."""
+    results = []
+    for check_fn in _HEALTH_CHECKS:
+        try:
+            results.append(check_fn())
+        except Exception as e:
+            results.append({
+                "name": getattr(check_fn, '__name__', 'unknown'),
+                "status": "error",
+                "detail": str(e),
+            })
+    return results
+
+
+# ---------------------------------------------------------------------------
+# Structured JSON Logger (additive — does NOT replace text logs)
+# ---------------------------------------------------------------------------
+
+_json_logger: logging.Logger | None = None
+
+
+def setup_structured_logging(log_dir: Path | None = None) -> None:
+    """Set up JSON structured logging alongside existing text logs.
+
+    Creates a separate daemon.json.log file with RotatingFileHandler.
+    Text logs continue working unchanged.
+    """
+    global _json_logger
+
+    log_dir = log_dir or (Path.home() / ".superlocalmemory" / "logs")
+    log_dir.mkdir(parents=True, exist_ok=True)
+    json_log_path = log_dir / "daemon.json.log"
+
+    _json_logger = logging.getLogger("superlocalmemory.structured")
+    _json_logger.setLevel(logging.INFO)
+    _json_logger.propagate = False  # Don't send to text handler
+
+    from logging.handlers import RotatingFileHandler
+    handler = RotatingFileHandler(
+        str(json_log_path), maxBytes=10 * 1024 * 1024, backupCount=5,
+    )
+    handler.setFormatter(logging.Formatter("%(message)s"))
+    _json_logger.addHandler(handler)
+
+
+def log_structured(**fields) -> None:
+    """Emit a structured JSON log entry.
+
+    Always includes: timestamp, level. Caller provides the rest.
+    Example fields: worker_pid, memory_rss_mb, operation, latency_ms, message.
+    """
+    if _json_logger is None:
+        return
+    entry = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "level": fields.pop("level", "info"),
+        **fields,
+    }
+    try:
+        _json_logger.info(json.dumps(entry, default=str))
+    except Exception:
+        pass
+
+
+# ---------------------------------------------------------------------------
+# Health Monitor Thread
+# ---------------------------------------------------------------------------
+
+class HealthMonitor:
+    """Background thread monitoring worker health, RSS budget, heartbeats.
+
+    Self-healing: if the monitor itself crashes, it logs and retries
+    with exponential backoff (max 5 min sleep).
+    """
+
+    # SLM worker command-line identifiers for child process filtering
+    _WORKER_IDENTIFIERS = (
+        "superlocalmemory.core.embedding_worker",
+        "superlocalmemory.core.reranker_worker",
+        "superlocalmemory.core.recall_worker",
+    )
+
+    def __init__(
+        self,
+        global_rss_budget_mb: int = 4096,
+        heartbeat_timeout_sec: int = 60,
+        check_interval_sec: int = 30,
+        enable_structured_logging: bool = True,
+    ):
+        self._budget_mb = global_rss_budget_mb
+        self._heartbeat_timeout = heartbeat_timeout_sec
+        self._interval = check_interval_sec
+        self._enable_logging = enable_structured_logging
+        self._thread: threading.Thread | None = None
+        self._stop_event = threading.Event()
+        self._consecutive_failures = 0
+
+    def start(self) -> None:
+        """Start the health monitor in a daemon thread."""
+        if not PSUTIL_AVAILABLE:
+            logger.warning("Health monitor disabled: psutil not installed")
+            return
+
+        if self._enable_logging:
+            setup_structured_logging()
+
+        self._thread = threading.Thread(
+            target=self._run_loop, daemon=True, name="health-monitor",
+        )
+        self._thread.start()
+        logger.info("Health monitor started (budget=%dMB, heartbeat=%ds)",
+                     self._budget_mb, self._heartbeat_timeout)
+
+        # Register built-in health checks
+        register_health_check(self._check_daemon_health)
+        register_health_check(self._check_worker_health)
+        register_health_check(self._check_memory_budget)
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+    def _run_loop(self) -> None:
+        """Main monitoring loop with self-healing."""
+        while not self._stop_event.is_set():
+            try:
+                self._check_once()
+                self._consecutive_failures = 0
+            except Exception as exc:
+                self._consecutive_failures += 1
+                backoff = min(300, 30 * self._consecutive_failures)
+                logger.error("Health check failed (%d consecutive): %s. Backoff %ds.",
+                             self._consecutive_failures, exc, backoff)
+                log_structured(
+                    level="error", operation="health_check",
+                    message=f"Health check failed: {exc}",
+                    consecutive_failures=self._consecutive_failures,
+                )
+                self._stop_event.wait(backoff)
+                continue
+
+            self._stop_event.wait(self._interval)
+
+    def _check_once(self) -> None:
+        """Single health check cycle."""
+        proc = psutil.Process(os.getpid())
+        daemon_rss_mb = proc.memory_info().rss / (1024 * 1024)
+
+        # Find SLM worker children only (not adapters or other children)
+        children = proc.children(recursive=True)
+        slm_workers = []
+        for child in children:
+            try:
+                cmdline = " ".join(child.cmdline()).lower()
+                if any(ident in cmdline for ident in self._WORKER_IDENTIFIERS):
+                    rss_mb = child.memory_info().rss / (1024 * 1024)
+                    slm_workers.append({
+                        "pid": child.pid,
+                        "rss_mb": round(rss_mb, 1),
+                        "cmdline": cmdline[:80],
+                    })
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+
+        total_rss_mb = daemon_rss_mb + sum(w["rss_mb"] for w in slm_workers)
+
+        # Structured log entry
+        log_structured(
+            level="info",
+            operation="health_check",
+            memory_rss_mb=round(daemon_rss_mb, 1),
+            total_rss_mb=round(total_rss_mb, 1),
+            worker_count=len(slm_workers),
+            workers=slm_workers,
+            budget_mb=self._budget_mb,
+        )
+
+        # RSS budget enforcement
+        if total_rss_mb > self._budget_mb and slm_workers:
+            heaviest = max(slm_workers, key=lambda w: w["rss_mb"])
+            logger.warning(
+                "RSS budget exceeded (%.0fMB > %dMB). Killing heaviest worker PID %d (%.0fMB)",
+                total_rss_mb, self._budget_mb, heaviest["pid"], heaviest["rss_mb"],
+            )
+            log_structured(
+                level="warning",
+                operation="rss_budget_kill",
+                killed_pid=heaviest["pid"],
+                killed_rss_mb=heaviest["rss_mb"],
+                total_rss_mb=round(total_rss_mb, 1),
+            )
+            try:
+                psutil.Process(heaviest["pid"]).terminate()
+            except psutil.NoSuchProcess:
+                pass
+
+        # Heartbeat checks delegated to WorkerPool (Phase B wiring)
+        # WorkerPool tracks last_heartbeat per worker. HealthMonitor
+        # reads it here. Actual heartbeat protocol is in worker_pool.py.
+        try:
+            from superlocalmemory.core.worker_pool import WorkerPool
+            pool = WorkerPool.shared()
+            last_hb = getattr(pool, '_last_heartbeat', {})
+            now = time.monotonic()
+            for wpid, last_time in list(last_hb.items()):
+                if now - last_time > self._heartbeat_timeout:
+                    logger.warning("Worker PID %d unresponsive (no heartbeat for %ds). Killing.",
+                                   wpid, int(now - last_time))
+                    log_structured(
+                        level="warning",
+                        operation="heartbeat_kill",
+                        worker_pid=wpid,
+                        seconds_since_heartbeat=round(now - last_time),
+                    )
+                    try:
+                        psutil.Process(wpid).terminate()
+                    except psutil.NoSuchProcess:
+                        pass
+                    del last_hb[wpid]
+        except Exception:
+            pass  # WorkerPool not initialized yet — fine
+
+    # -- Built-in health checks for registry --
+
+    def _check_daemon_health(self) -> dict:
+        if not PSUTIL_AVAILABLE:
+            return {"name": "daemon", "status": "unknown", "detail": "psutil unavailable"}
+        proc = psutil.Process(os.getpid())
+        rss_mb = proc.memory_info().rss / (1024 * 1024)
+        return {
+            "name": "daemon",
+            "status": "ok" if rss_mb < 500 else "warning",
+            "detail": f"PID {os.getpid()}, RSS {rss_mb:.0f}MB",
+        }
+
+    def _check_worker_health(self) -> dict:
+        try:
+            from superlocalmemory.core.worker_pool import WorkerPool
+            pool = WorkerPool.shared()
+            wpid = pool.worker_pid
+            if wpid:
+                return {"name": "workers", "status": "ok", "detail": f"Worker PID {wpid}"}
+            return {"name": "workers", "status": "warning", "detail": "No active worker"}
+        except Exception as e:
+            return {"name": "workers", "status": "error", "detail": str(e)}
+
+    def _check_memory_budget(self) -> dict:
+        if not PSUTIL_AVAILABLE:
+            return {"name": "memory", "status": "unknown", "detail": "psutil unavailable"}
+        proc = psutil.Process(os.getpid())
+        total_rss = proc.memory_info().rss
+        for child in proc.children(recursive=True):
+            try:
+                total_rss += child.memory_info().rss
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                pass
+        total_mb = total_rss / (1024 * 1024)
+        status = "ok" if total_mb < self._budget_mb else "critical"
+        return {
+            "name": "memory",
+            "status": status,
+            "detail": f"{total_mb:.0f}MB / {self._budget_mb}MB budget",
+        }

package/src/superlocalmemory/core/reranker_worker.py

@@ -73,15 +73,29 @@ def _start_parent_watchdog() -> None:
     t.start()
 
 
-def _detect_onnx_variant() -> str:
-    """Auto-detect the best ONNX model variant for the current platform."""
+def _detect_onnx_variant(model_name: str = "") -> str:
+    """Auto-detect the best ONNX model variant for the current platform.
+
+    V3.4.2: Supports both legacy ms-marco-MiniLM (platform-specific quantized)
+    and new gte-modernbert-base (int8/uint8 quantized). Falls back to generic
+    model.onnx if platform-specific variant unavailable.
+    """
     arch = platform.machine().lower()
     is_64bit = struct.calcsize("P") * 8 == 64
 
+    # Legacy ms-marco-MiniLM models have platform-specific quantized variants
+    if "ms-marco" in model_name or "MiniLM" in model_name:
+        if sys.platform == "darwin" and arch in ("arm64", "aarch64"):
+            return "onnx/model_qint8_arm64.onnx"
+        if arch in ("x86_64", "amd64") and is_64bit:
+            return "onnx/model_quint8_avx2.onnx"
+        return "onnx/model.onnx"
+
+    # gte-modernbert-base and other modern models: int8 for ARM64, uint8 for x86
     if sys.platform == "darwin" and arch in ("arm64", "aarch64"):
-        return "onnx/model_qint8_arm64.onnx"
+        return "onnx/model_int8.onnx"
     if arch in ("x86_64", "amd64") and is_64bit:
-        return "onnx/model_quint8_avx2.onnx"
+        return "onnx/model_uint8.onnx"
     return "onnx/model.onnx"
 
 
@@ -239,7 +253,7 @@ def _load_model(
         if backend == "onnx":
             # Tier 1: Platform-specific quantized ONNX (fastest)
             try:
-                onnx_file = _detect_onnx_variant()
+                onnx_file = _detect_onnx_variant(name)
                 m = CrossEncoder(
                     name, backend="onnx",
                     model_kwargs={"file_name": onnx_file},

package/src/superlocalmemory/ingestion/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""SLM Ingestion — external source adapters for Gmail, Calendar, Transcripts.
+
+ALL adapters are OPT-IN. Nothing runs by default. User enables via:
+  slm adapters enable gmail
+  slm adapters enable calendar
+  slm adapters enable transcript
+
+Adapters are stateless external processes that POST to the daemon's /ingest endpoint.
+"""

package/src/superlocalmemory/ingestion/adapter_manager.py

@@ -0,0 +1,234 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Adapter lifecycle manager — start, stop, enable, disable ingestion adapters.
+
+All adapters run as separate subprocesses managed via PID files.
+Config stored in ~/.superlocalmemory/adapters.json.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+logger = logging.getLogger("superlocalmemory.ingestion.manager")
+
+_SLM_HOME = Path.home() / ".superlocalmemory"
+_ADAPTERS_CONFIG = _SLM_HOME / "adapters.json"
+_VALID_ADAPTERS = ("gmail", "calendar", "transcript")
+
+# Module paths for each adapter
+_ADAPTER_MODULES = {
+    "gmail": "superlocalmemory.ingestion.gmail_adapter",
+    "calendar": "superlocalmemory.ingestion.calendar_adapter",
+    "transcript": "superlocalmemory.ingestion.transcript_adapter",
+}
+
+
+def _load_config() -> dict:
+    if _ADAPTERS_CONFIG.exists():
+        return json.loads(_ADAPTERS_CONFIG.read_text())
+    return {name: {"enabled": False} for name in _VALID_ADAPTERS}
+
+
+def _save_config(config: dict) -> None:
+    _ADAPTERS_CONFIG.parent.mkdir(parents=True, exist_ok=True)
+    _ADAPTERS_CONFIG.write_text(json.dumps(config, indent=2))
+
+
+def _pid_file(name: str) -> Path:
+    return _SLM_HOME / f"adapter-{name}.pid"
+
+
+def _is_running(name: str) -> tuple[bool, int | None]:
+    """Check if adapter is running. Returns (running, pid)."""
+    pf = _pid_file(name)
+    if not pf.exists():
+        return False, None
+    try:
+        pid = int(pf.read_text().strip())
+        try:
+            import psutil
+            return psutil.pid_exists(pid), pid
+        except ImportError:
+            os.kill(pid, 0)
+            return True, pid
+    except (ValueError, ProcessLookupError, PermissionError):
+        pf.unlink(missing_ok=True)
+        return False, None
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def list_adapters() -> list[dict]:
+    """List all adapters with their status."""
+    config = _load_config()
+    result = []
+    for name in _VALID_ADAPTERS:
+        ac = config.get(name, {})
+        running, pid = _is_running(name)
+        result.append({
+            "name": name,
+            "enabled": ac.get("enabled", False),
+            "running": running,
+            "pid": pid,
+            "tier": ac.get("tier", ""),
+            "watch_dir": ac.get("watch_dir", ""),
+        })
+    return result
+
+
+def enable_adapter(name: str) -> dict:
+    """Enable an adapter in config."""
+    if name not in _VALID_ADAPTERS:
+        return {"ok": False, "error": f"Unknown adapter: {name}. Valid: {_VALID_ADAPTERS}"}
+    config = _load_config()
+    config.setdefault(name, {})["enabled"] = True
+    _save_config(config)
+    return {"ok": True, "message": f"{name} adapter enabled. Run `slm adapters start {name}` to start."}
+
+
+def disable_adapter(name: str) -> dict:
+    """Disable an adapter. Stops it if running."""
+    if name not in _VALID_ADAPTERS:
+        return {"ok": False, "error": f"Unknown adapter: {name}"}
+    stop_adapter(name)
+    config = _load_config()
+    config.setdefault(name, {})["enabled"] = False
+    _save_config(config)
+    return {"ok": True, "message": f"{name} adapter disabled"}
+
+
+def start_adapter(name: str) -> dict:
+    """Start an adapter subprocess."""
+    if name not in _VALID_ADAPTERS:
+        return {"ok": False, "error": f"Unknown adapter: {name}"}
+
+    config = _load_config()
+    if not config.get(name, {}).get("enabled"):
+        return {"ok": False, "error": f"{name} not enabled. Run `slm adapters enable {name}` first."}
+
+    running, pid = _is_running(name)
+    if running:
+        return {"ok": True, "message": f"{name} already running (PID {pid})"}
+
+    module = _ADAPTER_MODULES.get(name)
+    if not module:
+        return {"ok": False, "error": f"No module for {name}"}
+
+    cmd = [sys.executable, "-m", module]
+    log_dir = _SLM_HOME / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = log_dir / f"adapter-{name}.log"
+
+    kwargs: dict = {}
+    if sys.platform == "win32":
+        kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+    else:
+        kwargs["start_new_session"] = True
+
+    with open(log_path, "a") as lf:
+        proc = subprocess.Popen(cmd, stdout=lf, stderr=lf, **kwargs)
+
+    _pid_file(name).write_text(str(proc.pid))
+    return {"ok": True, "message": f"{name} started (PID {proc.pid})", "pid": proc.pid}
+
+
+def stop_adapter(name: str) -> dict:
+    """Stop a running adapter."""
+    running, pid = _is_running(name)
+    if not running:
+        return {"ok": True, "message": f"{name} not running"}
+
+    try:
+        import psutil
+        proc = psutil.Process(pid)
+        proc.terminate()
+        proc.wait(timeout=10)
+    except ImportError:
+        os.kill(pid, 15)  # SIGTERM
+    except Exception:
+        pass
+
+    _pid_file(name).unlink(missing_ok=True)
+    return {"ok": True, "message": f"{name} stopped"}
+
+
+def status_adapters() -> list[dict]:
+    """Get detailed status of all adapters."""
+    return list_adapters()
+
+
+# ---------------------------------------------------------------------------
+# CLI handler (called from commands.py)
+# ---------------------------------------------------------------------------
+
+def handle_adapters_cli(args: list[str]) -> None:
+    """Handle `slm adapters <action> [name]` commands."""
+    if not args:
+        args = ["list"]
+
+    action = args[0]
+    name = args[1] if len(args) > 1 else ""
+
+    if action == "list":
+        adapters = list_adapters()
+        print("  Ingestion Adapters:")
+        print("  " + "-" * 50)
+        for a in adapters:
+            status = "running" if a["running"] else ("enabled" if a["enabled"] else "disabled")
+            pid_str = f" (PID {a['pid']})" if a["pid"] else ""
+            print(f"  {a['name']:12s} {status:10s}{pid_str}")
+        print()
+
+    elif action == "enable":
+        if not name:
+            print("  Usage: slm adapters enable <gmail|calendar|transcript>")
+            return
+        result = enable_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "disable":
+        if not name:
+            print("  Usage: slm adapters disable <name>")
+            return
+        result = disable_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "start":
+        if not name:
+            print("  Usage: slm adapters start <name>")
+            return
+        result = start_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "stop":
+        if not name:
+            print("  Usage: slm adapters stop <name>")
+            return
+        result = stop_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "status":
+        adapters = status_adapters()
+        for a in adapters:
+            status = "RUNNING" if a["running"] else ("enabled" if a["enabled"] else "off")
+            print(f"  {a['name']:12s} [{status}]", end="")
+            if a["pid"]:
+                print(f"  PID={a['pid']}", end="")
+            print()
+
+    else:
+        print(f"  Unknown action: {action}")
+        print("  Usage: slm adapters <list|enable|disable|start|stop|status> [name]")