superlocalmemory 3.6.23 → 3.7.0

package/src/superlocalmemory/cli/context_commands.py

@@ -13,7 +13,6 @@ import json as _json
 import os
 from argparse import Namespace
 from pathlib import Path
-from typing import Callable
 
 from superlocalmemory.hooks.antigravity_adapter import AntigravityAdapter
 from superlocalmemory.hooks.copilot_adapter import CopilotAdapter
@@ -53,9 +52,11 @@ def _get_recall_fn() -> RecallFn:
 
 
 def _default_sync_log_db() -> Path:
-    return Path(os.environ.get("SLM_MEMORY_DB",
-                               str(Path.home() / ".superlocalmemory"
-                                   / "memory.db")))
+    explicit = os.environ.get("SLM_MEMORY_DB", "").strip()
+    if explicit:
+        return Path(explicit)
+    from superlocalmemory.infra.data_root import state_path
+    return state_path("memory.db")
 
 
 def build_default_adapters(

package/src/superlocalmemory/cli/daemon.py

@@ -24,13 +24,23 @@ from __future__ import annotations
 import json
 import logging
 import os
-import signal
+import socket
 import sys
 import time
-from http.server import HTTPServer, BaseHTTPRequestHandler
-from pathlib import Path
-import threading
-from threading import Thread
+from dataclasses import replace
+
+from superlocalmemory.infra.daemon_identity import (
+    build_descriptor,
+    descriptor_matches_health,
+    descriptor_path,
+    process_create_time_for,
+    read_descriptor,
+    write_descriptor,
+)
+from superlocalmemory.infra.data_root import (
+    assert_no_durable_root_conflict,
+    state_path,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -40,8 +50,8 @@ except ValueError:
     _DEFAULT_PORT = 8765
 _LEGACY_PORT = 8767   # backward-compat redirect
 _DEFAULT_IDLE_TIMEOUT = 0  # v3.4.3: 24/7 default (was 1800)
-_PID_FILE = Path.home() / ".superlocalmemory" / "daemon.pid"
-_PORT_FILE = Path.home() / ".superlocalmemory" / "daemon.port"
+_PID_FILE = None  # test-only override; runtime resolution stays dynamic
+_PORT_FILE = None  # test-only override; runtime resolution stays dynamic
 
 
 # ---------------------------------------------------------------------------
@@ -61,83 +71,222 @@ def _is_pid_alive(pid: int) -> bool:
             return False
 
 
-def is_daemon_running() -> bool:
-    """Check if daemon is alive via PID file + HTTP health check.
+def _descriptor_process_is_alive(descriptor) -> bool:
+    """Reject stale descriptors when a PID has been reused by another process."""
+    if not _is_pid_alive(descriptor.pid):
+        return False
+    try:
+        import psutil
+
+        process = psutil.Process(descriptor.pid)
+        # A terminated daemon can remain in the process table briefly as a
+        # zombie while its parent reaps it.  PID existence is therefore not
+        # liveness and must not block a namespace-owned restart.
+        if not process.is_running() or process.status() == psutil.STATUS_ZOMBIE:
+            return False
+        actual = float(process.create_time())
+    except ImportError:
+        return True
+    except Exception:
+        return False
+    return abs(actual - float(descriptor.process_create_time)) <= 1.0
+
+
+def _is_port_available(port: int) -> bool:
+    """Return whether the daemon port can be exclusively bound right now."""
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as candidate:
+            # The server binds with address reuse. Mirror that contract here:
+            # a recently closed listener may leave TCP connections in
+            # TIME_WAIT, which must not be mistaken for an active owner and
+            # block a safe restart for the full TCP timeout.
+            candidate.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            candidate.bind(("127.0.0.1", port))
+        return True
+    except OSError:
+        return False
+
+
+def _has_tcp_listener(port: int) -> bool:
+    """Return whether a process is actively accepting on the daemon port."""
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as candidate:
+            candidate.settimeout(0.5)
+            return candidate.connect_ex(("127.0.0.1", port)) == 0
+    except OSError:
+        return False
+
 
-    v3.4.4 FIX: If PID is alive, returns True EVEN IF health check fails.
-    This prevents starting duplicate daemons when the existing one is
-    warming up (Ollama processing, model download, embedding init).
+def wait_for_owned_daemon_shutdown(descriptor, timeout: float = 25.0) -> bool:
+    """Wait for the stopped instance *and* its TCP listener to be gone.
 
-    Priority:
-      1. PID file exists AND process alive → True (daemon warming up or ready)
-      2. No PID file → try health check on known ports (MCP/hook started daemon)
-      3. PID file stale (process dead) → clean up, return False
+    Restart must never spawn a replacement just because the descriptor was
+    removed: a graceful shutdown can remove it before Uvicorn releases the
+    port.  The 25-second budget covers Uvicorn's 10-second graceful drain plus
+    SLM worker cleanup. A descriptor carries process creation time, so PID
+    reuse cannot make this wait target an unrelated process.
     """
-    if _PID_FILE.exists():
-        try:
-            pid = int(_PID_FILE.read_text().strip())
-            if _is_pid_alive(pid):
-                # PID alive = daemon exists. Don't check health — it might be warming up.
-                # This is the critical fix: NEVER start a second daemon if PID is alive.
-                return True
-            else:
-                # Process died — clean up stale PID file
-                _PID_FILE.unlink(missing_ok=True)
-                _PORT_FILE.unlink(missing_ok=True)
-        except (ValueError, OSError):
-            _PID_FILE.unlink(missing_ok=True)
-
-    # No PID file — maybe daemon was started by MCP/hook without PID file.
-    # Try health check on known ports as last resort.
-    for try_port in (_DEFAULT_PORT, _LEGACY_PORT):
-        try:
-            import urllib.request
-            resp = urllib.request.urlopen(
-                f"http://127.0.0.1:{try_port}/health", timeout=2,
-            )
-            if resp.status == 200:
-                # Daemon running without PID file — write one for future checks
-                try:
-                    import json as _json
-                    data = _json.loads(resp.read().decode())
-                    pid = data.get("pid")
-                    if pid:
-                        _PID_FILE.parent.mkdir(parents=True, exist_ok=True)
-                        _PID_FILE.write_text(str(pid))
-                        _PORT_FILE.write_text(str(try_port))
-                except Exception:
-                    pass
-                return True
-        except Exception:
-            continue
+    port = descriptor.port if descriptor is not None else _DEFAULT_PORT
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        process_alive = (
+            descriptor is not None and _descriptor_process_is_alive(descriptor)
+        )
+        if not process_alive and _is_port_available(port):
+            return True
+        time.sleep(0.1)
     return False
 
 
+def is_daemon_running() -> bool:
+    """Return True only for a daemon owned by this canonical data namespace.
+
+    A PID or an HTTP 200 proves liveness, not ownership. V3.7 requires the
+    private local descriptor and the health endpoint to agree on namespace,
+    process instance, capability fingerprint, owner, PID, protocol, and port.
+    """
+    local_descriptor_path = descriptor_path()
+    descriptor = read_descriptor()
+    if descriptor is not None:
+        if not _descriptor_process_is_alive(descriptor):
+            return False
+        if descriptor.state == "starting":
+            return True
+        health = _fetch_health(descriptor.port)
+        return health is not None and descriptor_matches_health(descriptor, health)
+
+    # A malformed or foreign descriptor must fail closed; never fall through
+    # to legacy PID/port adoption in the same namespace.
+    if local_descriptor_path.exists():
+        return False
+
+    legacy = _verified_legacy_health()
+    return legacy is not None
+
+
+def _fetch_health(port: int) -> dict | None:
+    """Fetch loopback health without following cross-namespace discovery."""
+    try:
+        import urllib.request
+
+        expected_url = f"http://127.0.0.1:{port}/health"
+        response = urllib.request.urlopen(
+            expected_url, timeout=2,
+        )
+        if response.status != 200:
+            return None
+        geturl = getattr(response, "geturl", None)
+        final_url = geturl() if callable(geturl) else None
+        if final_url is not None and final_url != expected_url:
+            return None
+        payload = json.loads(response.read().decode())
+        return payload if isinstance(payload, dict) else None
+    except Exception:
+        return None
+
+
+def _is_verified_legacy_process(pid: int) -> bool:
+    """One-release bridge for a same-root V3.6 unified-daemon process."""
+    if not _is_pid_alive(pid):
+        return False
+    try:
+        import psutil
+
+        process = psutil.Process(pid)
+        command = " ".join(process.cmdline())
+        return "superlocalmemory.server.unified_daemon" in command
+    except Exception:
+        return False
+
+
+def _verified_legacy_health() -> dict | None:
+    """Accept legacy health only with a verified same-root daemon PID file."""
+    pid_file = descriptor_path().with_name("daemon.pid")
+    port_file = descriptor_path().with_name("daemon.port")
+    try:
+        pid = int(pid_file.read_text().strip())
+        port = int(port_file.read_text().strip()) if port_file.exists() else _DEFAULT_PORT
+    except (OSError, ValueError):
+        return None
+    if not _is_verified_legacy_process(pid):
+        return None
+    health = _fetch_health(port)
+    if health is None or int(health.get("pid", -1)) != pid:
+        return None
+    # Identity-bearing health without a descriptor is not legacy and cannot
+    # be adopted. It belongs to another namespace or stale state.
+    if health.get("daemon_protocol") is not None:
+        return None
+    return {**health, "_legacy_port": port}
+
+
 def _get_port() -> int:
-    if _PORT_FILE.exists():
-        try:
-            return int(_PORT_FILE.read_text().strip())
-        except ValueError:
-            pass
+    descriptor = read_descriptor()
+    if descriptor is not None:
+        return descriptor.port
+    if descriptor_path().exists():
+        return _DEFAULT_PORT
+    legacy = _verified_legacy_health()
+    if legacy is not None:
+        return int(legacy["_legacy_port"])
     return _DEFAULT_PORT
 
 
-def daemon_request(method: str, path: str, body: dict | None = None) -> dict | None:
-    """Send request to daemon. Returns parsed JSON or None on failure."""
-    port = _get_port()
+def daemon_request(
+    method: str,
+    path: str,
+    body: dict | None = None,
+    *,
+    timeout_seconds: float = 30.0,
+) -> dict | None:
+    """Send a request only after validating the owned daemon identity."""
+    descriptor = read_descriptor()
+    capability: str | None = None
+    if descriptor is not None:
+        health = _fetch_health(descriptor.port)
+        if health is None or not descriptor_matches_health(descriptor, health):
+            return None
+        if method.upper() == "GET" and path == "/health":
+            return health
+        port = descriptor.port
+        capability = descriptor.capability
+    elif descriptor_path().exists():
+        return None
+    else:
+        legacy = _verified_legacy_health()
+        if legacy is None:
+            return None
+        if method.upper() == "GET" and path == "/health":
+            return {key: value for key, value in legacy.items() if key != "_legacy_port"}
+        port = int(legacy["_legacy_port"])
     try:
         import urllib.request
         url = f"http://127.0.0.1:{port}{path}"
         data = json.dumps(body).encode() if body else None
         headers = {"Content-Type": "application/json"} if data else {}
+        if capability is not None:
+            headers["X-SLM-Daemon-Capability"] = capability
+            headers["X-SLM-Target-Instance"] = descriptor.instance_id
         req = urllib.request.Request(url, data=data, headers=headers, method=method)
-        resp = urllib.request.urlopen(req, timeout=30)
+        resp = urllib.request.urlopen(req, timeout=timeout_seconds)
         return json.loads(resp.read().decode())
     except Exception:
         return None
 
 
-_LOCK_FILE = Path.home() / ".superlocalmemory" / "daemon.lock"
+_LOCK_FILE = None  # test-only override; runtime resolution stays dynamic
+
+
+def _pid_file_path():
+    return _PID_FILE or state_path("daemon.pid")
+
+
+def _port_file_path():
+    return _PORT_FILE or state_path("daemon.port")
+
+
+def _lock_file_path():
+    return _LOCK_FILE or state_path("daemon.lock")
 
 
 def _start_daemon_subprocess() -> bool:
@@ -161,8 +310,17 @@ def _start_daemon_subprocess() -> bool:
     """
     if is_daemon_running():
         return True
+    # Never create a descriptor for a child that cannot own the listener.
+    # A closed connection in TIME_WAIT is not a listener and is safe: the
+    # server reserves its socket with SO_REUSEADDR during bootstrap.
+    if _has_tcp_listener(_DEFAULT_PORT):
+        logger.warning("daemon port %d is already owned; refusing a second start", _DEFAULT_PORT)
+        return False
+    assert_no_durable_root_conflict()
 
     import subprocess
+
+    from superlocalmemory import __version__ as _slm_version
     # v3.6.9 (#33): pass SLM_DAEMON_PORT as explicit --port= so the daemon
     # binds the right port even when the env var reaches the subprocess.
     _target_port = _DEFAULT_PORT
@@ -170,7 +328,7 @@ def _start_daemon_subprocess() -> bool:
         sys.executable, "-m", "superlocalmemory.server.unified_daemon",
         "--start", f"--port={_target_port}",
     ]
-    log_dir = Path.home() / ".superlocalmemory" / "logs"
+    log_dir = state_path("logs")
     log_dir.mkdir(parents=True, exist_ok=True)
     log_file = log_dir / "daemon.log"
 
@@ -189,14 +347,39 @@ def _start_daemon_subprocess() -> bool:
     daemon_env = os.environ.copy()
     daemon_env["OMP_NUM_THREADS"] = "1"
     daemon_env["KMP_DUPLICATE_LIB_OK"] = "TRUE"
+    bootstrap_descriptor = build_descriptor(
+        port=_target_port,
+        version=_slm_version,
+        pid=os.getpid(),
+        state="starting",
+    )
+    daemon_env["SLM_DAEMON_INSTANCE_ID"] = bootstrap_descriptor.instance_id
+    daemon_env["SLM_DAEMON_CAPABILITY"] = bootstrap_descriptor.capability
     kwargs["env"] = daemon_env
 
     with open(log_file, "a") as lf:
         proc = subprocess.Popen(cmd, stdout=lf, stderr=lf, **kwargs)
 
-    # Write PID immediately so other callers see it during warmup
-    _PID_FILE.write_text(str(proc.pid))
-    _PORT_FILE.write_text(str(_target_port))
+    # Publish the exact child identity immediately so concurrent callers know
+    # this namespace is warming up. If the child won the race and already
+    # published the same instance as ready, never overwrite it with starting.
+    child_descriptor = replace(
+        bootstrap_descriptor,
+        pid=proc.pid,
+        process_create_time=process_create_time_for(proc.pid),
+    )
+    current = read_descriptor()
+    if not (
+        current is not None
+        and current.instance_id == child_descriptor.instance_id
+        and current.pid == child_descriptor.pid
+        and current.state == "ready"
+    ):
+        write_descriptor(child_descriptor)
+
+    # One-release compatibility mirrors; never sufficient for ownership.
+    _pid_file_path().write_text(str(proc.pid))
+    _port_file_path().write_text(str(_target_port))
 
     return _wait_for_daemon(timeout=60)
 
@@ -218,12 +401,21 @@ def ensure_daemon() -> bool:
     """
     if is_daemon_running():
         return True
+    if (
+        os.environ.get("SLM_TEST_ISOLATION") == "1"
+        and os.environ.get("SLM_TEST_ALLOW_DAEMON_SPAWN") != "1"
+    ):
+        logger.debug(
+            "pytest isolation blocked daemon spawn; use an owned daemon fixture",
+        )
+        return False
 
     # File lock — prevent concurrent starts from multiple CLI/MCP calls
     lock_fd = None
     try:
-        _LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
-        lock_fd = open(_LOCK_FILE, "w")
+        lock_file = _lock_file_path()
+        lock_file.parent.mkdir(parents=True, exist_ok=True)
+        lock_fd = open(lock_file, "w")
 
         # Cross-platform file locking
         if sys.platform == "win32":
@@ -276,135 +468,38 @@ def ensure_daemon() -> bool:
             except Exception:
                 pass
             try:
-                _LOCK_FILE.unlink(missing_ok=True)
+                _lock_file_path().unlink(missing_ok=True)
             except Exception:
                 pass
 
 
 def _wait_for_daemon(timeout: int = 60) -> bool:
-    """Wait for daemon to become reachable. Checks PID alive first (fast),
-    then health endpoint (confirms HTTP server is bound)."""
+    """Wait for matching owned health; liveness alone is never readiness."""
     for _ in range(timeout * 2):  # check every 0.5s
         time.sleep(0.5)
-        if is_daemon_running():
-            # PID is alive — now optionally check if HTTP is ready
-            port = _get_port()
-            try:
-                import urllib.request
-                urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=2)
-                return True  # HTTP is ready
-            except Exception:
-                # PID alive but HTTP not ready — daemon is warming up, that's OK
+        descriptor = read_descriptor()
+        if descriptor is not None:
+            if not _descriptor_process_is_alive(descriptor):
+                continue
+            health = _fetch_health(descriptor.port)
+            if health is not None and descriptor_matches_health(descriptor, health):
                 return True
+            continue
+        if descriptor_path().exists():
+            continue
+        if _verified_legacy_health() is not None:
+            return True
     return False
 
 
 def stop_daemon() -> bool:
-    """Stop ALL SLM daemon processes and their workers.
+    """Stop only the daemon proven to belong to this data namespace.
 
-    v3.4.7: Nuclear cleanup — finds and kills ALL processes matching
-    superlocalmemory.server.unified_daemon, embedding_worker, recall_worker,
-    reranker_worker. Not just the PID file daemon. Multiple daemons can
-    accumulate from rapid restarts, MCP warmups, and concurrent sessions.
+    Machine-wide process-name scans are forbidden: they can kill another SLM
+    installation or a user's live workers during tests. V3.7 uses the owned
+    HTTP capability; the daemon itself terminates its child process tree.
     """
-    killed = 0
-
-    try:
-        import psutil
-        my_pid = os.getpid()
-        targets = [
-            "superlocalmemory.server.unified_daemon",
-            "superlocalmemory.core.embedding_worker",
-            "superlocalmemory.core.recall_worker",
-            "superlocalmemory.core.reranker_worker",
-        ]
-
-        for proc in psutil.process_iter(["pid", "cmdline"]):
-            try:
-                if proc.pid == my_pid:
-                    continue
-                cmdline = " ".join(proc.info.get("cmdline") or [])
-                if any(t in cmdline for t in targets):
-                    # Kill children first, then process
-                    for child in proc.children(recursive=True):
-                        try:
-                            child.kill()
-                            killed += 1
-                        except (psutil.NoSuchProcess, psutil.AccessDenied):
-                            pass
-                    proc.kill()
-                    killed += 1
-            except (psutil.NoSuchProcess, psutil.AccessDenied):
-                pass
-
-    except ImportError:
-        # Fallback: pkill by pattern
-        try:
-            import subprocess as _sp
-            for pattern in [
-                "superlocalmemory.server.unified_daemon",
-                "superlocalmemory.core.embedding_worker",
-                "superlocalmemory.core.recall_worker",
-                "superlocalmemory.core.reranker_worker",
-            ]:
-                result = _sp.run(
-                    ["pkill", "-9", "-f", pattern],
-                    capture_output=True, timeout=5,
-                )
-                if result.returncode == 0:
-                    killed += 1
-        except Exception:
-            pass
-
-    # Clean up PID/port files + worker PID files
-    _PID_FILE.unlink(missing_ok=True)
-    _PORT_FILE.unlink(missing_ok=True)
-    # v3.4.13: Clean worker PID files (singleton guards)
-    for pidfile in (".embedding-worker.pid", ".reranker-worker.pid"):
-        (Path.home() / ".superlocalmemory" / pidfile).unlink(missing_ok=True)
-
-    # v3.4.13: Wait for ALL workers to actually die before returning.
-    # Without this, `slm restart` starts a new daemon before old workers exit,
-    # causing duplicate embedding_workers (1.6GB each).
-    if killed:
-        logger.info("Stopped %d SLM processes, waiting for exit...", killed)
-        _wait_for_workers_dead(timeout=10)
-
-    return True
-
-
-def _wait_for_workers_dead(timeout: int = 10) -> None:
-    """Wait until no SLM worker processes remain alive."""
-    targets = [
-        "superlocalmemory.server.unified_daemon",
-        "superlocalmemory.core.embedding_worker",
-        "superlocalmemory.core.recall_worker",
-        "superlocalmemory.core.reranker_worker",
-    ]
-    my_pid = os.getpid()
-    deadline = time.time() + timeout
-
-    while time.time() < deadline:
-        alive = False
-        try:
-            import psutil
-            for proc in psutil.process_iter(["pid", "cmdline"]):
-                try:
-                    if proc.pid == my_pid:
-                        continue
-                    cmdline = " ".join(proc.info.get("cmdline") or [])
-                    if any(t in cmdline for t in targets):
-                        alive = True
-                        break
-                except (psutil.NoSuchProcess, psutil.AccessDenied):
-                    pass
-        except ImportError:
-            # No psutil — just wait a fixed time
-            time.sleep(3)
-            return
-
-        if not alive:
-            return
-        time.sleep(0.5)
-
-    logger.warning("Some SLM workers still alive after %ds timeout", timeout)
+    if read_descriptor() is None and _verified_legacy_health() is None:
+        return False
+    response = daemon_request("POST", "/stop")
+    return bool(response and response.get("status") == "stopping")

package/src/superlocalmemory/cli/db_migrate.py

@@ -18,11 +18,13 @@ from __future__ import annotations
 from argparse import Namespace
 from pathlib import Path
 
+from superlocalmemory.infra.data_root import DynamicStatePath
+
 
 # Canonical paths — match LLD-01 / LLD-07 layout. Callers can override
 # via the ``learning_db_path`` / ``memory_db_path`` attributes on args
 # (tests rely on this to point at fixture DBs).
-DEFAULT_HOME = Path.home() / ".superlocalmemory"
+DEFAULT_HOME = DynamicStatePath()
 
 
 def _resolve_paths(args: Namespace) -> tuple[Path, Path]:

package/src/superlocalmemory/cli/diagnostics_cmd.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Explicit manual export for local aggregate diagnostics."""
+
+from __future__ import annotations
+
+from argparse import Namespace
+
+
+def cmd_diagnostics(args: Namespace) -> None:
+    action = getattr(args, "diagnostics_command", None)
+    if action != "export":
+        raise SystemExit("choose a diagnostics subcommand: export")
+
+    from superlocalmemory.cli.json_output import json_print
+    from superlocalmemory.infra.local_diagnostics import default_diagnostics
+
+    default_diagnostics().export_json(args.destination)
+    data = {"exported": True, "reporting": "manual_export_only"}
+    if bool(getattr(args, "json", False)):
+        json_print("diagnostics", data=data)
+        return
+    print("Local aggregate diagnostics exported by explicit request.")
+    print("No automatic reporting was enabled.")
+
+
+__all__ = ["cmd_diagnostics"]