superlocalmemory 3.6.22 → 3.7.0

package/src/superlocalmemory/infra/local_diagnostics.py

@@ -0,0 +1,327 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under AGPL-3.0-or-later - see LICENSE file
+
+"""Bounded, content-free operational diagnostics stored on this machine.
+
+The store accepts only enumerated operations and normalized client families.
+It persists daily counters, latency buckets, coarse error classes, and client
+family transitions. Export is a deliberate local file operation; this module
+has no automatic reporting mechanism.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+import sqlite3
+import tempfile
+from collections.abc import Callable, Iterable
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+
+from superlocalmemory.infra.data_root import state_path
+
+SCHEMA_VERSION = 1
+RETENTION_DAYS = 31
+MAX_COUNTER = 2**63 - 1
+
+_OPERATIONS = frozenset(
+    {
+        "activation",
+        "remember",
+        "recall",
+        "observe",
+        "session_open",
+        "session_close",
+    }
+)
+_CLIENT_FAMILIES = (
+    "claude",
+    "codex",
+    "cursor",
+    "windsurf",
+    "copilot",
+    "vscode",
+    "jetbrains",
+    "dashboard",
+    "hook",
+    "mcp",
+    "api",
+    "cli",
+)
+_LATENCY_BOUNDS_MS = (5, 25, 100, 500, 2_000, 10_000, 60_000)
+_ERROR_CLASSES = frozenset(
+    {"timeout", "authorization", "validation", "unavailable", "internal"}
+)
+
+
+def normalize_client(value: str | None) -> str:
+    """Collapse an arbitrary actor label to a fixed non-identifying family."""
+    tokens = frozenset(re.findall(r"[a-z0-9]+", str(value or "").lower()))
+    for family in _CLIENT_FAMILIES:
+        if family in tokens:
+            return family
+    return "other"
+
+
+def classify_error(error: BaseException | None) -> str | None:
+    """Map an exception type to a fixed class without inspecting its message."""
+    if error is None:
+        return None
+    if isinstance(error, TimeoutError):
+        return "timeout"
+    if isinstance(error, PermissionError):
+        return "authorization"
+    if isinstance(error, (TypeError, ValueError)):
+        return "validation"
+    if isinstance(error, (ConnectionError, FileNotFoundError)):
+        return "unavailable"
+    return "internal"
+
+
+def _latency_bucket(duration_ms: float | int | None) -> str | None:
+    if duration_ms is None:
+        return None
+    try:
+        value = float(duration_ms)
+    except (TypeError, ValueError, OverflowError):
+        return "invalid"
+    if not math.isfinite(value) or value < 0:
+        return "invalid"
+    for bound in _LATENCY_BOUNDS_MS:
+        if value <= bound:
+            return f"le_{bound}"
+    return "gt_60000"
+
+
+class LocalDiagnostics:
+    """SQLite-backed bounded aggregate diagnostics.
+
+    The schema intentionally has no free-text value column. Every persisted
+    dimension comes from a finite allow-list, so caller payloads cannot enter
+    the database or an export by accident.
+    """
+
+    def __init__(
+        self,
+        db_file: str | Path,
+        *,
+        clock: Callable[[], datetime] | None = None,
+        retention_days: int = RETENTION_DAYS,
+    ) -> None:
+        if retention_days < 1 or retention_days > RETENTION_DAYS:
+            raise ValueError(f"retention_days must be between 1 and {RETENTION_DAYS}")
+        self._db_file = Path(db_file)
+        self._clock = clock or (lambda: datetime.now(UTC))
+        self._retention_days = retention_days
+
+    def _connect(self) -> sqlite3.Connection:
+        self._db_file.parent.mkdir(parents=True, exist_ok=True)
+        if os.name == "posix":
+            os.chmod(self._db_file.parent, 0o700)
+        conn = sqlite3.connect(str(self._db_file), timeout=5.0)
+        conn.execute(
+            "CREATE TABLE IF NOT EXISTS diagnostic_aggregates ("
+            "day TEXT NOT NULL, group_name TEXT NOT NULL, "
+            "dimension TEXT NOT NULL, bucket TEXT NOT NULL, "
+            "value INTEGER NOT NULL CHECK(value >= 0), "
+            "PRIMARY KEY(day, group_name, dimension, bucket))"
+        )
+        conn.commit()
+        if os.name == "posix":
+            os.chmod(self._db_file, 0o600)
+        return conn
+
+    def _today(self) -> datetime:
+        current = self._clock()
+        if current.tzinfo is None:
+            current = current.replace(tzinfo=UTC)
+        return current.astimezone(UTC)
+
+    @staticmethod
+    def _increment(
+        conn: sqlite3.Connection,
+        day: str,
+        group_name: str,
+        dimension: str,
+        bucket: str,
+    ) -> None:
+        conn.execute(
+            "INSERT INTO diagnostic_aggregates "
+            "(day, group_name, dimension, bucket, value) VALUES (?,?,?,?,1) "
+            "ON CONFLICT(day, group_name, dimension, bucket) DO UPDATE SET "
+            "value=MIN(diagnostic_aggregates.value + 1, ?)",
+            (day, group_name, dimension, bucket, MAX_COUNTER),
+        )
+
+    def record(
+        self,
+        operation: str,
+        *,
+        client: str | None = None,
+        duration_ms: float | int | None = None,
+        error: BaseException | None = None,
+        source_clients: Iterable[str] = (),
+    ) -> None:
+        """Record one operation using only fixed aggregate dimensions."""
+        if operation not in _OPERATIONS:
+            raise ValueError(f"unsupported diagnostic operation: {operation}")
+        current = self._today()
+        day = current.date().isoformat()
+        cutoff = (current.date() - timedelta(days=self._retention_days - 1)).isoformat()
+        client_family = normalize_client(client)
+        status = "error" if error is not None else "success"
+        duration_bucket = _latency_bucket(duration_ms)
+        error_class = classify_error(error)
+        transitions = {
+            f"{normalize_client(source)}_to_{client_family}"
+            for source in source_clients
+            if normalize_client(source) != client_family
+        }
+
+        conn = self._connect()
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            conn.execute("DELETE FROM diagnostic_aggregates WHERE day < ?", (cutoff,))
+            self._increment(conn, day, "operations", operation, status)
+            self._increment(conn, day, "clients", client_family, operation)
+            if duration_bucket is not None:
+                self._increment(conn, day, "latency_ms", operation, duration_bucket)
+            if error_class is not None:
+                self._increment(conn, day, "errors", operation, error_class)
+            if operation == "activation":
+                self._increment(conn, day, "activations", client_family, "count")
+            for transition in sorted(transitions):
+                self._increment(conn, day, "handoffs", transition, "count")
+            conn.commit()
+        except BaseException:
+            conn.rollback()
+            raise
+        finally:
+            conn.close()
+
+    def export_payload(self) -> dict[str, Any]:
+        """Return a deterministic, sanitized aggregate document."""
+        days: dict[str, dict[str, Any]] = {}
+        if self._db_file.is_file():
+            conn = self._connect()
+            try:
+                rows = conn.execute(
+                    "SELECT day, group_name, dimension, bucket, value "
+                    "FROM diagnostic_aggregates "
+                    "ORDER BY day, group_name, dimension, bucket"
+                ).fetchall()
+            finally:
+                conn.close()
+            for day, group_name, dimension, bucket, value in rows:
+                entry = days.setdefault(str(day), {"day": str(day)})
+                group = entry.setdefault(str(group_name), {})
+                dimension_values = group.setdefault(str(dimension), {})
+                dimension_values[str(bucket)] = int(value)
+        return {
+            "schema_version": SCHEMA_VERSION,
+            "retention_days": self._retention_days,
+            "privacy": {
+                "local_only": True,
+                "reporting": "manual_export_only",
+                "content_values": False,
+                "query_values": False,
+                "fact_identifiers": False,
+                "filesystem_locations": False,
+                "identity_values": False,
+            },
+            "days": [days[key] for key in sorted(days)],
+        }
+
+    def export_json(self, destination: str | Path) -> dict[str, Any]:
+        """Atomically write a deterministic manual export with mode 0600."""
+        payload = self.export_payload()
+        output = Path(destination)
+        output.parent.mkdir(parents=True, exist_ok=True)
+        data = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
+        temporary_name = ""
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="wb", dir=output.parent, prefix=f".{output.name}.", delete=False,
+            ) as temporary:
+                temporary_name = temporary.name
+                os.chmod(temporary_name, 0o600)
+                temporary.write(data)
+                temporary.flush()
+                os.fsync(temporary.fileno())
+            os.replace(temporary_name, output)
+            if os.name == "posix":
+                os.chmod(output, 0o600)
+        finally:
+            if temporary_name:
+                Path(temporary_name).unlink(missing_ok=True)
+        return payload
+
+
+def default_diagnostics() -> LocalDiagnostics:
+    """Return the diagnostics store for the active local state root."""
+    return LocalDiagnostics(state_path("diagnostics.db"))
+
+
+def record_operation(
+    operation: str,
+    *,
+    client: str | None = None,
+    duration_ms: float | int | None = None,
+    error: BaseException | None = None,
+    source_clients: Iterable[str] = (),
+) -> None:
+    """Best-effort hot-path adapter; diagnostics never affect correctness."""
+    try:
+        default_diagnostics().record(
+            operation,
+            client=client,
+            duration_ms=duration_ms,
+            error=error,
+            source_clients=source_clients,
+        )
+    except Exception:
+        return
+
+
+def record_recall(db: Any, response: Any, *, client: str | None) -> None:
+    """Record recall and aggregate cross-client provenance transitions."""
+    source_clients: list[str] = []
+    fact_ids = [
+        str(result.fact.fact_id)
+        for result in getattr(response, "results", ())
+        if getattr(getattr(result, "fact", None), "fact_id", None)
+    ]
+    profile_id = ""
+    if getattr(response, "results", None):
+        profile_id = str(getattr(response.results[0].fact, "profile_id", ""))
+    if fact_ids and profile_id:
+        try:
+            marks = ",".join("?" for _ in fact_ids)
+            rows = db.execute(
+                f"SELECT DISTINCT created_by FROM provenance "
+                f"WHERE profile_id=? AND fact_id IN ({marks})",
+                (profile_id, *fact_ids),
+            )
+            source_clients = [str(dict(row).get("created_by", "")) for row in rows]
+        except Exception:
+            source_clients = []
+    record_operation(
+        "recall",
+        client=client,
+        duration_ms=getattr(response, "retrieval_time_ms", None),
+        source_clients=source_clients,
+    )
+
+
+__all__ = [
+    "LocalDiagnostics",
+    "classify_error",
+    "default_diagnostics",
+    "normalize_client",
+    "record_operation",
+    "record_recall",
+]

package/src/superlocalmemory/infra/process_reaper.py

@@ -79,6 +79,29 @@ class SlmProcessInfo:
     age_hours: float
 
 
+def is_mcp_server_process(process: SlmProcessInfo) -> bool:
+    """Return whether *process* is an MCP stdio server, not an SLM daemon.
+
+    The startup reaper is deliberately aggressive (zero age threshold), but
+    only an orphaned MCP process is safe to reap from ``slm mcp``.  A unified
+    daemon is intentionally detached from the shell that launched it, so on
+    some hosts it has PPID 1 and looks like an orphan.  Reaping that daemon at
+    MCP startup drops in-flight CLI and dashboard requests.
+
+    Keep this predicate narrow and based on the command line recorded by the
+    OS: every supported MCP entry point includes the standalone ``mcp``
+    argument or imports the MCP server module.  The daemon's command does
+    neither.
+    """
+    command = process.command.lower()
+    return (
+        "/slm mcp" in command
+        or " slm mcp" in command
+        or "superlocalmemory.cli.main mcp" in command
+        or "superlocalmemory.mcp.server" in command
+    )
+
+
 # ---------------------------------------------------------------------------
 # Windows no-op stubs (AUDIT FIX H0-HIGH-02)
 # ---------------------------------------------------------------------------

package/src/superlocalmemory/ingestion/adapter_manager.py

@@ -8,7 +8,7 @@ All adapters run as separate subprocesses managed via PID files.
 Config stored in ~/.superlocalmemory/adapters.json.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -20,10 +20,12 @@ import subprocess
 import sys
 from pathlib import Path
 
+from superlocalmemory.infra.data_root import canonical_data_root, state_path
+
 logger = logging.getLogger("superlocalmemory.ingestion.manager")
 
-_SLM_HOME = Path.home() / ".superlocalmemory"
-_ADAPTERS_CONFIG = _SLM_HOME / "adapters.json"
+_SLM_HOME = None  # test-only compatibility override
+_ADAPTERS_CONFIG = None  # test-only compatibility override
 _VALID_ADAPTERS = ("gmail", "calendar", "transcript")
 
 # Module paths for each adapter
@@ -34,19 +36,35 @@ _ADAPTER_MODULES = {
 }
 
 
+def _slm_home() -> Path:
+    return Path(_SLM_HOME) if _SLM_HOME is not None else canonical_data_root()
+
+
+def _adapters_config_path() -> Path:
+    if _ADAPTERS_CONFIG is not None:
+        return Path(_ADAPTERS_CONFIG)
+    return state_path("adapters.json")
+
+
+def _adapter_log_dir() -> Path:
+    return _slm_home() / "logs"
+
+
 def _load_config() -> dict:
-    if _ADAPTERS_CONFIG.exists():
-        return json.loads(_ADAPTERS_CONFIG.read_text())
+    config_path = _adapters_config_path()
+    if config_path.exists():
+        return json.loads(config_path.read_text())
     return {name: {"enabled": False} for name in _VALID_ADAPTERS}
 
 
 def _save_config(config: dict) -> None:
-    _ADAPTERS_CONFIG.parent.mkdir(parents=True, exist_ok=True)
-    _ADAPTERS_CONFIG.write_text(json.dumps(config, indent=2))
+    config_path = _adapters_config_path()
+    config_path.parent.mkdir(parents=True, exist_ok=True)
+    config_path.write_text(json.dumps(config, indent=2))
 
 
 def _pid_file(name: str) -> Path:
-    return _SLM_HOME / f"adapter-{name}.pid"
+    return _slm_home() / f"adapter-{name}.pid"
 
 
 def _is_running(name: str) -> tuple[bool, int | None]:
@@ -128,7 +146,7 @@ def start_adapter(name: str) -> dict:
         return {"ok": False, "error": f"No module for {name}"}
 
     cmd = [sys.executable, "-m", module]
-    log_dir = _SLM_HOME / "logs"
+    log_dir = _adapter_log_dir()
     log_dir.mkdir(parents=True, exist_ok=True)
     log_path = log_dir / f"adapter-{name}.log"
 

package/src/superlocalmemory/ingestion/base_adapter.py

@@ -12,12 +12,11 @@ All adapters inherit this. Enforces stateless, safe, cross-platform operation:
   - Structured logging
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
 
-import json
 import logging
 import os
 import signal
@@ -132,35 +131,30 @@ class BaseAdapter:
 
     def _ingest(self, item: IngestItem) -> bool:
         """POST to daemon /ingest endpoint. Returns True on success."""
-        try:
-            import urllib.request
-            payload = json.dumps({
-                "content": item.content,
-                "source_type": self.source_type,
-                "dedup_key": item.dedup_key,
-                "metadata": item.metadata if item.metadata else {},
-            }).encode()
-            req = urllib.request.Request(
-                f"{self.daemon_url}/ingest",
-                data=payload,
-                headers={"Content-Type": "application/json"},
-                method="POST",
-            )
-            resp = urllib.request.urlopen(req, timeout=30)
-            data = json.loads(resp.read().decode())
-            if data.get("ingested"):
-                self._items_this_hour += 1
-                self._total_ingested += 1
-                return True
-            return False  # Already ingested (dedup)
-        except Exception as exc:
-            error_str = str(exc)
-            if "429" in error_str:
-                logger.info("Daemon returned 429, backing off 5s")
-                self._stop_event.wait(5)
-                return self._ingest(item)  # Retry once
-            logger.debug("Ingest failed: %s", exc)
-            return False
+        payload = {
+            "content": item.content,
+            "source_type": self.source_type,
+            "dedup_key": item.dedup_key,
+            "metadata": item.metadata if item.metadata else {},
+        }
+        for attempt in range(2):
+            try:
+                from superlocalmemory.cli.daemon import daemon_request
+
+                data = daemon_request("POST", "/ingest", payload)
+                if data and data.get("ingested"):
+                    self._items_this_hour += 1
+                    self._total_ingested += 1
+                    return True
+                return False  # Already ingested (dedup)
+            except Exception as exc:
+                if "429" in str(exc) and attempt == 0:
+                    logger.info("Daemon returned 429, backing off 5s")
+                    self._stop_event.wait(5)
+                    continue
+                logger.debug("Ingest failed: %s", exc)
+                return False
+        return False
 
     def _rate_limited(self) -> bool:
         if time.time() - self._hour_start > 3600:

package/src/superlocalmemory/ingestion/calendar_adapter.py

@@ -10,7 +10,7 @@ Tier 2: Google Calendar API with OAuth polling — shares Gmail OAuth credential
 OPT-IN only. Enabled via: slm adapters enable calendar
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -22,10 +22,19 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 from superlocalmemory.ingestion.base_adapter import BaseAdapter, AdapterConfig, IngestItem
+from superlocalmemory.infra.data_root import state_path
 
 logger = logging.getLogger("superlocalmemory.ingestion.calendar")
 
 
+def _adapters_config_path() -> Path:
+    return state_path("adapters.json")
+
+
+def _import_dir() -> Path:
+    return state_path("import")
+
+
 class CalendarAdapter(BaseAdapter):
     """Google Calendar ingestion with automatic tier detection."""
 
@@ -62,7 +71,7 @@ class CalendarAdapter(BaseAdapter):
         if self._tier != "auto":
             return
 
-        adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+        adapters_path = _adapters_config_path()
         cfg = {}
         if adapters_path.exists():
             cfg = json.loads(adapters_path.read_text()).get("calendar", {})
@@ -79,7 +88,7 @@ class CalendarAdapter(BaseAdapter):
             return
 
         # Look for ICS files
-        import_dir = Path.home() / ".superlocalmemory" / "import"
+        import_dir = _import_dir()
         ics_files = list(import_dir.glob("*.ics")) if import_dir.exists() else []
         if ics_files:
             self._tier = "ics"
@@ -330,7 +339,7 @@ if __name__ == "__main__":
     import logging as _logging
     _logging.basicConfig(level=_logging.INFO, format="%(asctime)s %(message)s")
 
-    adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+    adapters_path = _adapters_config_path()
     tier = "auto"
     if adapters_path.exists():
         cfg = json.loads(adapters_path.read_text()).get("calendar", {})

package/src/superlocalmemory/ingestion/credentials.py

@@ -8,7 +8,7 @@ Uses OS keychain via keyring library (macOS Keychain, Windows Credential Locker,
 Linux SecretService). Falls back to file-based storage with restricted permissions.
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -19,12 +19,18 @@ import os
 import sys
 from pathlib import Path
 
+from superlocalmemory.infra.data_root import state_path
+
 logger = logging.getLogger("superlocalmemory.ingestion.credentials")
 
-_CRED_DIR = Path.home() / ".superlocalmemory" / "credentials"
+_CRED_DIR = None  # test-only compatibility override
 _SERVICE_PREFIX = "slm"
 
 
+def _credential_dir() -> Path:
+    return Path(_CRED_DIR) if _CRED_DIR is not None else state_path("credentials")
+
+
 def store_credential(service: str, key: str, value: str) -> bool:
     """Store a credential securely. Returns True on success."""
     # Try OS keychain first
@@ -38,8 +44,9 @@ def store_credential(service: str, key: str, value: str) -> bool:
 
     # Fallback: encrypted file with restricted permissions
     try:
-        _CRED_DIR.mkdir(parents=True, exist_ok=True)
-        cred_file = _CRED_DIR / f"{service}.json"
+        cred_dir = _credential_dir()
+        cred_dir.mkdir(parents=True, exist_ok=True)
+        cred_file = cred_dir / f"{service}.json"
 
         existing = {}
         if cred_file.exists():
@@ -54,7 +61,7 @@ def store_credential(service: str, key: str, value: str) -> bool:
         # Restrict permissions (Unix only — Windows skipped)
         if sys.platform != "win32":
             os.chmod(cred_file, 0o600)
-            os.chmod(_CRED_DIR, 0o700)
+            os.chmod(cred_dir, 0o700)
 
         logger.debug("Stored %s/%s in file (keychain unavailable)", service, key)
         return True
@@ -76,7 +83,7 @@ def load_credential(service: str, key: str) -> str | None:
 
     # Fallback: file
     try:
-        cred_file = _CRED_DIR / f"{service}.json"
+        cred_file = _credential_dir() / f"{service}.json"
         if cred_file.exists():
             data = json.loads(cred_file.read_text())
             return data.get(key)
@@ -100,7 +107,7 @@ def delete_credential(service: str, key: str) -> bool:
 
     # Also remove from file
     try:
-        cred_file = _CRED_DIR / f"{service}.json"
+        cred_file = _credential_dir() / f"{service}.json"
         if cred_file.exists():
             data = json.loads(cred_file.read_text())
             if key in data:

package/src/superlocalmemory/ingestion/gmail_adapter.py

@@ -12,7 +12,7 @@ Tier 3: Gmail API with Pub/Sub push — full GCP (future)
 OPT-IN only. Enabled via: slm adapters enable gmail
 
 Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: Elastic-2.0
+License: AGPL-3.0-or-later
 """
 
 from __future__ import annotations
@@ -23,10 +23,19 @@ import sys
 from pathlib import Path
 
 from superlocalmemory.ingestion.base_adapter import BaseAdapter, AdapterConfig, IngestItem
+from superlocalmemory.infra.data_root import state_path
 
 logger = logging.getLogger("superlocalmemory.ingestion.gmail")
 
 
+def _adapters_config_path() -> Path:
+    return state_path("adapters.json")
+
+
+def _import_dir() -> Path:
+    return state_path("import")
+
+
 class GmailAdapter(BaseAdapter):
     """Gmail ingestion with automatic tier detection."""
 
@@ -71,7 +80,7 @@ class GmailAdapter(BaseAdapter):
         if self._tier != "auto":
             return
 
-        adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+        adapters_path = _adapters_config_path()
         cfg = {}
         if adapters_path.exists():
             cfg = json.loads(adapters_path.read_text()).get("gmail", {})
@@ -93,7 +102,7 @@ class GmailAdapter(BaseAdapter):
             return
 
         # Default: look for mbox file
-        mbox_dir = Path.home() / ".superlocalmemory" / "import"
+        mbox_dir = _import_dir()
         mbox_files = list(mbox_dir.glob("*.mbox")) if mbox_dir.exists() else []
         if mbox_files:
             self._tier = "mbox"
@@ -359,7 +368,7 @@ if __name__ == "__main__":
     import logging as _logging
     _logging.basicConfig(level=_logging.INFO, format="%(asctime)s %(message)s")
 
-    adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+    adapters_path = _adapters_config_path()
     tier = "auto"
     if adapters_path.exists():
         cfg = json.loads(adapters_path.read_text()).get("gmail", {})