superlocalmemory 3.6.23 → 3.7.0

package/src/superlocalmemory/infra/data_root.py

@@ -0,0 +1,199 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Single authority for SuperLocalMemory runtime-state paths.
+
+The resolver is intentionally stdlib-only so CLI, daemon, MCP, and hook entry
+points can use it before importing the engine. Environment selection is an
+explicit process contract and always wins over persisted configuration.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+DATA_ROOT_ALIASES = ("SLM_DATA_DIR", "SL_MEMORY_PATH", "SLM_HOME")
+_DURABLE_IDENTITY_NAMES = frozenset(
+    {
+        ".install_token",
+        ".signer_key",
+        "api_key",
+        "credentials.json",
+    },
+)
+
+
+class DataRootConflictError(RuntimeError):
+    """Raised when two state-bearing roots make startup ambiguous."""
+
+
+def _canonical_path(value: str | Path) -> Path:
+    expanded = Path(value).expanduser().resolve(strict=False)
+    return Path(os.path.normcase(str(expanded)))
+
+
+def environment_data_root() -> Path | None:
+    """Return the winning environment root, or ``None`` when none is set."""
+    for name in DATA_ROOT_ALIASES:
+        value = os.environ.get(name, "").strip()
+        if value:
+            return _canonical_path(value)
+    return None
+
+
+def _legacy_configured_root(default_root: Path) -> Path | None:
+    """Read the bounded legacy ``config.json:base_dir`` relocation hint.
+
+    Only an absolute path is accepted. Relative paths historically depended on
+    the caller's working directory and cannot safely identify a process
+    namespace. Corrupt or unreadable configuration never changes the root.
+    """
+    config_path = default_root / "config.json"
+    try:
+        payload = json.loads(config_path.read_text(encoding="utf-8"))
+        value = payload.get("base_dir") if isinstance(payload, dict) else None
+        if not isinstance(value, str) or not value.strip():
+            return None
+        candidate = Path(value).expanduser()
+        if not candidate.is_absolute():
+            return None
+        return _canonical_path(candidate)
+    except (OSError, ValueError, TypeError):
+        return None
+
+
+def canonical_data_root(
+    configured_base_dir: str | Path | None = None,
+    *,
+    home: str | Path | None = None,
+) -> Path:
+    """Resolve the one runtime-state namespace for the current process.
+
+    Precedence:
+      1. ``SLM_DATA_DIR``
+      2. ``SL_MEMORY_PATH`` (legacy input shim)
+      3. ``SLM_HOME`` (legacy input shim)
+      4. explicit configured base directory
+      5. legacy default-root ``config.json:base_dir``
+      6. ``~/.superlocalmemory``
+
+    The function performs no writes and never moves data.
+    """
+    selected = environment_data_root()
+    if selected is not None:
+        return selected
+    if configured_base_dir is not None:
+        return _canonical_path(configured_base_dir)
+    home_path = _canonical_path(home if home is not None else Path.home())
+    default_root = _canonical_path(home_path / ".superlocalmemory")
+    return _legacy_configured_root(default_root) or default_root
+
+
+def _durable_markers(root: Path) -> tuple[Path, ...]:
+    """Return bounded evidence that ``root`` contains durable SLM state.
+
+    The startup guard deliberately ignores ``config.json`` on its own because
+    the default-root file is the compatibility locator for legacy custom
+    roots. Database sidecars count even if the base file was lost.
+    """
+    try:
+        children = tuple(root.iterdir())
+    except FileNotFoundError:
+        return ()
+    except OSError as exc:
+        raise DataRootConflictError(
+            f"SuperLocalMemory cannot inspect possible state root {root}: {exc}. "
+            "Startup is blocked until the root is readable or explicitly resolved."
+        ) from exc
+    markers = []
+    for path in children:
+        name = path.name
+        if (
+            name in _DURABLE_IDENTITY_NAMES
+            or name.endswith((".db", ".db-wal", ".db-shm", ".sqlite", ".sqlite3"))
+        ):
+            markers.append(path)
+    return tuple(sorted(markers))
+
+
+def assert_no_durable_root_conflict(
+    *,
+    home: str | Path | None = None,
+) -> None:
+    """Refuse ambiguous startup when selected and default roots hold state.
+
+    This check never writes, copies, or deletes data. A default root that only
+    contains the legacy relocation config is safe and does not trigger it.
+    """
+    home_path = _canonical_path(home if home is not None else Path.home())
+    default_root = _canonical_path(home_path / ".superlocalmemory")
+    selected_root = canonical_data_root(home=home_path)
+    if selected_root == default_root:
+        return
+    selected_markers = _durable_markers(selected_root)
+    default_markers = _durable_markers(default_root)
+    if not selected_markers or not default_markers:
+        return
+    raise DataRootConflictError(
+        "Conflicting SuperLocalMemory state roots detected: "
+        f"selected={selected_root} and default={default_root}. "
+        "SuperLocalMemory will not merge or move either root automatically. "
+        "Stop both namespaces, back them up, and choose an explicit migration "
+        "or one SLM_DATA_DIR before starting the daemon."
+    )
+
+
+def state_path(
+    *parts: str | Path,
+    configured_base_dir: str | Path | None = None,
+    home: str | Path | None = None,
+) -> Path:
+    """Return a canonical child path without permitting root escape."""
+    relative = Path()
+    for raw_part in parts:
+        part = Path(raw_part)
+        if part.is_absolute() or ".." in part.parts:
+            raise ValueError(f"state path must remain relative: {raw_part!s}")
+        relative /= part
+    root = canonical_data_root(configured_base_dir, home=home)
+    destination = _canonical_path(root / relative)
+    try:
+        destination.relative_to(root)
+    except ValueError as exc:
+        raise ValueError("state path escapes the canonical data root") from exc
+    return destination
+
+
+class DynamicStatePath(os.PathLike[str]):
+    """Backward-compatible path proxy that never freezes the active root."""
+
+    __slots__ = ("_parts",)
+
+    def __init__(self, *parts: str | Path) -> None:
+        self._parts = parts
+
+    def _resolve(self) -> Path:
+        return state_path(*self._parts)
+
+    def __fspath__(self) -> str:
+        return str(self._resolve())
+
+    def __truediv__(self, other: str | Path) -> Path:
+        return self._resolve() / other
+
+    def __str__(self) -> str:
+        return str(self._resolve())
+
+    def __repr__(self) -> str:
+        return f"DynamicStatePath({self._parts!r})"
+
+    def __eq__(self, other: object) -> bool:
+        try:
+            return self._resolve() == Path(other)  # type: ignore[arg-type]
+        except TypeError:
+            return False
+
+    def __getattr__(self, name: str):
+        return getattr(self._resolve(), name)

package/src/superlocalmemory/infra/event_bus.py

@@ -16,6 +16,8 @@ from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional
 
+from superlocalmemory.infra.data_root import state_path
+
 logger = logging.getLogger("superlocalmemory.events")
 
 # Default retention windows (hours)
@@ -66,7 +68,7 @@ class EventBus:
     def get_instance(cls, db_path: Optional[Path] = None) -> "EventBus":
         """Get or create the singleton EventBus for a database path."""
         if db_path is None:
-            db_path = Path.home() / ".superlocalmemory" / "memory.db"
+            db_path = state_path("memory.db")
 
         key = str(db_path)
         with cls._instances_lock:

package/src/superlocalmemory/infra/local_diagnostics.py

@@ -0,0 +1,327 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Bounded, content-free operational diagnostics stored on this machine.
+
+The store accepts only enumerated operations and normalized client families.
+It persists daily counters, latency buckets, coarse error classes, and client
+family transitions. Export is a deliberate local file operation; this module
+has no automatic reporting mechanism.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+import sqlite3
+import tempfile
+from collections.abc import Callable, Iterable
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from superlocalmemory.infra.data_root import state_path
+
+SCHEMA_VERSION = 1
+RETENTION_DAYS = 31
+MAX_COUNTER = 2**63 - 1
+
+_OPERATIONS = frozenset(
+    {
+        "activation",
+        "remember",
+        "recall",
+        "observe",
+        "session_open",
+        "session_close",
+    }
+)
+_CLIENT_FAMILIES = (
+    "claude",
+    "codex",
+    "cursor",
+    "windsurf",
+    "copilot",
+    "vscode",
+    "jetbrains",
+    "dashboard",
+    "hook",
+    "mcp",
+    "api",
+    "cli",
+)
+_LATENCY_BOUNDS_MS = (5, 25, 100, 500, 2_000, 10_000, 60_000)
+_ERROR_CLASSES = frozenset(
+    {"timeout", "authorization", "validation", "unavailable", "internal"}
+)
+
+
+def normalize_client(value: str | None) -> str:
+    """Collapse an arbitrary actor label to a fixed non-identifying family."""
+    tokens = frozenset(re.findall(r"[a-z0-9]+", str(value or "").lower()))
+    for family in _CLIENT_FAMILIES:
+        if family in tokens:
+            return family
+    return "other"
+
+
+def classify_error(error: BaseException | None) -> str | None:
+    """Map an exception type to a fixed class without inspecting its message."""
+    if error is None:
+        return None
+    if isinstance(error, TimeoutError):
+        return "timeout"
+    if isinstance(error, PermissionError):
+        return "authorization"
+    if isinstance(error, (TypeError, ValueError)):
+        return "validation"
+    if isinstance(error, (ConnectionError, FileNotFoundError)):
+        return "unavailable"
+    return "internal"
+
+
+def _latency_bucket(duration_ms: float | int | None) -> str | None:
+    if duration_ms is None:
+        return None
+    try:
+        value = float(duration_ms)
+    except (TypeError, ValueError, OverflowError):
+        return "invalid"
+    if not math.isfinite(value) or value < 0:
+        return "invalid"
+    for bound in _LATENCY_BOUNDS_MS:
+        if value <= bound:
+            return f"le_{bound}"
+    return "gt_60000"
+
+
+class LocalDiagnostics:
+    """SQLite-backed bounded aggregate diagnostics.
+
+    The schema intentionally has no free-text value column. Every persisted
+    dimension comes from a finite allow-list, so caller payloads cannot enter
+    the database or an export by accident.
+    """
+
+    def __init__(
+        self,
+        db_file: str | Path,
+        *,
+        clock: Callable[[], datetime] | None = None,
+        retention_days: int = RETENTION_DAYS,
+    ) -> None:
+        if retention_days < 1 or retention_days > RETENTION_DAYS:
+            raise ValueError(f"retention_days must be between 1 and {RETENTION_DAYS}")
+        self._db_file = Path(db_file)
+        self._clock = clock or (lambda: datetime.now(UTC))
+        self._retention_days = retention_days
+
+    def _connect(self) -> sqlite3.Connection:
+        self._db_file.parent.mkdir(parents=True, exist_ok=True)
+        if os.name == "posix":
+            os.chmod(self._db_file.parent, 0o700)
+        conn = sqlite3.connect(str(self._db_file), timeout=5.0)
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS diagnostic_aggregates ("
+            "day TEXT NOT NULL, group_name TEXT NOT NULL, "
+            "dimension TEXT NOT NULL, bucket TEXT NOT NULL, "
+            "value INTEGER NOT NULL CHECK(value >= 0), "
+            "PRIMARY KEY(day, group_name, dimension, bucket))"
+        )
+        conn.commit()
+        if os.name == "posix":
+            os.chmod(self._db_file, 0o600)
+        return conn
+
+    def _today(self) -> datetime:
+        current = self._clock()
+        if current.tzinfo is None:
+            current = current.replace(tzinfo=UTC)
+        return current.astimezone(UTC)
+
+    @staticmethod
+    def _increment(
+        conn: sqlite3.Connection,
+        day: str,
+        group_name: str,
+        dimension: str,
+        bucket: str,
+    ) -> None:
+        conn.execute(
+            "INSERT INTO diagnostic_aggregates "
+            "(day, group_name, dimension, bucket, value) VALUES (?,?,?,?,1) "
+            "ON CONFLICT(day, group_name, dimension, bucket) DO UPDATE SET "
+            "value=MIN(diagnostic_aggregates.value + 1, ?)",
+            (day, group_name, dimension, bucket, MAX_COUNTER),
+        )
+
+    def record(
+        self,
+        operation: str,
+        *,
+        client: str | None = None,
+        duration_ms: float | int | None = None,
+        error: BaseException | None = None,
+        source_clients: Iterable[str] = (),
+    ) -> None:
+        """Record one operation using only fixed aggregate dimensions."""
+        if operation not in _OPERATIONS:
+            raise ValueError(f"unsupported diagnostic operation: {operation}")
+        current = self._today()
+        day = current.date().isoformat()
+        cutoff = (current.date() - timedelta(days=self._retention_days - 1)).isoformat()
+        client_family = normalize_client(client)
+        status = "error" if error is not None else "success"
+        duration_bucket = _latency_bucket(duration_ms)
+        error_class = classify_error(error)
+        transitions = {
+            f"{normalize_client(source)}_to_{client_family}"
+            for source in source_clients
+            if normalize_client(source) != client_family
+        }
+
+        conn = self._connect()
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            conn.execute("DELETE FROM diagnostic_aggregates WHERE day < ?", (cutoff,))
+            self._increment(conn, day, "operations", operation, status)
+            self._increment(conn, day, "clients", client_family, operation)
+            if duration_bucket is not None:
+                self._increment(conn, day, "latency_ms", operation, duration_bucket)
+            if error_class is not None:
+                self._increment(conn, day, "errors", operation, error_class)
+            if operation == "activation":
+                self._increment(conn, day, "activations", client_family, "count")
+            for transition in sorted(transitions):
+                self._increment(conn, day, "handoffs", transition, "count")
+            conn.commit()
+        except BaseException:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    def export_payload(self) -> dict[str, Any]:
+        """Return a deterministic, sanitized aggregate document."""
+        days: dict[str, dict[str, Any]] = {}
+        if self._db_file.is_file():
+            conn = self._connect()
+            try:
+                rows = conn.execute(
+                    "SELECT day, group_name, dimension, bucket, value "
+                    "FROM diagnostic_aggregates "
+                    "ORDER BY day, group_name, dimension, bucket"
+                ).fetchall()
+            finally:
+                conn.close()
+            for day, group_name, dimension, bucket, value in rows:
+                entry = days.setdefault(str(day), {"day": str(day)})
+                group = entry.setdefault(str(group_name), {})
+                dimension_values = group.setdefault(str(dimension), {})
+                dimension_values[str(bucket)] = int(value)
+        return {
+            "schema_version": SCHEMA_VERSION,
+            "retention_days": self._retention_days,
+            "privacy": {
+                "local_only": True,
+                "reporting": "manual_export_only",
+                "content_values": False,
+                "query_values": False,
+                "fact_identifiers": False,
+                "filesystem_locations": False,
+                "identity_values": False,
+            },
+            "days": [days[key] for key in sorted(days)],
+        }
+
+    def export_json(self, destination: str | Path) -> dict[str, Any]:
+        """Atomically write a deterministic manual export with mode 0600."""
+        payload = self.export_payload()
+        output = Path(destination)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        data = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
+        temporary_name = ""
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="wb", dir=output.parent, prefix=f".{output.name}.", delete=False,
+            ) as temporary:
+                temporary_name = temporary.name
+                os.chmod(temporary_name, 0o600)
+                temporary.write(data)
+                temporary.flush()
+                os.fsync(temporary.fileno())
+            os.replace(temporary_name, output)
+            if os.name == "posix":
+                os.chmod(output, 0o600)
+        finally:
+            if temporary_name:
+                Path(temporary_name).unlink(missing_ok=True)
+        return payload
+
+
+def default_diagnostics() -> LocalDiagnostics:
+    """Return the diagnostics store for the active local state root."""
+    return LocalDiagnostics(state_path("diagnostics.db"))
+
+
+def record_operation(
+    operation: str,
+    *,
+    client: str | None = None,
+    duration_ms: float | int | None = None,
+    error: BaseException | None = None,
+    source_clients: Iterable[str] = (),
+) -> None:
+    """Best-effort hot-path adapter; diagnostics never affect correctness."""
+    try:
+        default_diagnostics().record(
+            operation,
+            client=client,
+            duration_ms=duration_ms,
+            error=error,
+            source_clients=source_clients,
+        )
+    except Exception:
+        return
+
+
+def record_recall(db: Any, response: Any, *, client: str | None) -> None:
+    """Record recall and aggregate cross-client provenance transitions."""
+    source_clients: list[str] = []
+    fact_ids = [
+        str(result.fact.fact_id)
+        for result in getattr(response, "results", ())
+        if getattr(getattr(result, "fact", None), "fact_id", None)
+    ]
+    profile_id = ""
+    if getattr(response, "results", None):
+        profile_id = str(getattr(response.results[0].fact, "profile_id", ""))
+    if fact_ids and profile_id:
+        try:
+            marks = ",".join("?" for _ in fact_ids)
+            rows = db.execute(
+                f"SELECT DISTINCT created_by FROM provenance "
+                f"WHERE profile_id=? AND fact_id IN ({marks})",
+                (profile_id, *fact_ids),
+            )
+            source_clients = [str(dict(row).get("created_by", "")) for row in rows]
+        except Exception:
+            source_clients = []
+    record_operation(
+        "recall",
+        client=client,
+        duration_ms=getattr(response, "retrieval_time_ms", None),
+        source_clients=source_clients,
+    )
+
+
+__all__ = [
+    "LocalDiagnostics",
+    "classify_error",
+    "default_diagnostics",
+    "normalize_client",
+    "record_operation",
+    "record_recall",
+]

package/src/superlocalmemory/infra/process_reaper.py

@@ -79,6 +79,29 @@ class SlmProcessInfo:
     age_hours: float
 
 
+def is_mcp_server_process(process: SlmProcessInfo) -> bool:
+    """Return whether *process* is an MCP stdio server, not an SLM daemon.
+
+    The startup reaper is deliberately aggressive (zero age threshold), but
+    only an orphaned MCP process is safe to reap from ``slm mcp``.  A unified
+    daemon is intentionally detached from the shell that launched it, so on
+    some hosts it has PPID 1 and looks like an orphan.  Reaping that daemon at
+    MCP startup drops in-flight CLI and dashboard requests.
+
+    Keep this predicate narrow and based on the command line recorded by the
+    OS: every supported MCP entry point includes the standalone ``mcp``
+    argument or imports the MCP server module.  The daemon's command does
+    neither.
+    """
+    command = process.command.lower()
+    return (
+        "/slm mcp" in command
+        or " slm mcp" in command
+        or "superlocalmemory.cli.main mcp" in command
+        or "superlocalmemory.mcp.server" in command
+    )
+
+
 # ---------------------------------------------------------------------------
 # Windows no-op stubs (AUDIT FIX H0-HIGH-02)
 # ---------------------------------------------------------------------------

package/src/superlocalmemory/ingestion/adapter_manager.py

@@ -8,7 +8,7 @@ All adapters run as separate subprocesses managed via PID files.
 Config stored in ~/.superlocalmemory/adapters.json.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -20,10 +20,12 @@ import subprocess
 import sys
 from pathlib import Path
 
+from superlocalmemory.infra.data_root import canonical_data_root, state_path
+
 logger = logging.getLogger("superlocalmemory.ingestion.manager")
 
-_SLM_HOME = Path.home() / ".superlocalmemory"
-_ADAPTERS_CONFIG = _SLM_HOME / "adapters.json"
+_SLM_HOME = None  # test-only compatibility override
+_ADAPTERS_CONFIG = None  # test-only compatibility override
 _VALID_ADAPTERS = ("gmail", "calendar", "transcript")
 
 # Module paths for each adapter
@@ -34,19 +36,35 @@ _ADAPTER_MODULES = {
 }
 
 
+def _slm_home() -> Path:
+    return Path(_SLM_HOME) if _SLM_HOME is not None else canonical_data_root()
+
+
+def _adapters_config_path() -> Path:
+    if _ADAPTERS_CONFIG is not None:
+        return Path(_ADAPTERS_CONFIG)
+    return state_path("adapters.json")
+
+
+def _adapter_log_dir() -> Path:
+    return _slm_home() / "logs"
+
+
 def _load_config() -> dict:
-    if _ADAPTERS_CONFIG.exists():
-        return json.loads(_ADAPTERS_CONFIG.read_text())
+    config_path = _adapters_config_path()
+    if config_path.exists():
+        return json.loads(config_path.read_text())
     return {name: {"enabled": False} for name in _VALID_ADAPTERS}
 
 
 def _save_config(config: dict) -> None:
-    _ADAPTERS_CONFIG.parent.mkdir(parents=True, exist_ok=True)
-    _ADAPTERS_CONFIG.write_text(json.dumps(config, indent=2))
+    config_path = _adapters_config_path()
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    config_path.write_text(json.dumps(config, indent=2))
 
 
 def _pid_file(name: str) -> Path:
-    return _SLM_HOME / f"adapter-{name}.pid"
+    return _slm_home() / f"adapter-{name}.pid"
 
 
 def _is_running(name: str) -> tuple[bool, int | None]:
@@ -128,7 +146,7 @@ def start_adapter(name: str) -> dict:
         return {"ok": False, "error": f"No module for {name}"}
 
     cmd = [sys.executable, "-m", module]
-    log_dir = _SLM_HOME / "logs"
+    log_dir = _adapter_log_dir()
     log_dir.mkdir(parents=True, exist_ok=True)
     log_path = log_dir / f"adapter-{name}.log"