superlocalmemory 4.1.13 → 4.1.14

package/src/superlocalmemory/infra/version_integrity.py

@@ -1,229 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Does this process still match what is installed on disk?  (issue #107)
-
-Why this module exists
-----------------------
-Python imports a module once.  ``superlocalmemory.__version__`` is therefore
-frozen at the instant a process started, and upgrading the package underneath a
-long-lived ``slm mcp`` server changes nothing for that server -- it keeps
-serving the code it read at startup, forever.
-
-Nothing detected that.  The stale process did not error; it returned confident,
-plausible, *wrong* answers, and reported a ``serverInfo.version`` matching the
-code it had loaded, which is self-consistent and therefore useless as a
-staleness signal.  During the v3.8.12 work this machine had eighteen ``slm mcp``
-processes alive at once, spanning four days and two releases.  One of them made
-issue #106 look unfixed across two debugging sessions and contributed to v3.8.11
-shipping a wrong fix, because the "evidence" that the fix had failed was really
-a four-day-old process.
-
-The asymmetry that makes this dangerous
----------------------------------------
-A *loud* failure costs a user one confused minute.  A *silent* one costs
-whoever debugs it their entire session, because it actively argues that correct
-code is broken.  Everything here is therefore built so that no failure mode can
-produce a false :data:`STATE_CURRENT`.  Unreadable metadata, a hostile reader,
-a non-string return -- all resolve to :data:`STATE_UNKNOWN`, which reports
-"I could not tell" rather than "all is well".
-
-Why ``importlib.metadata`` is the right source
-----------------------------------------------
-It reads the ``*.dist-info`` directory from disk on each call rather than
-returning a value captured at import.  Verified empirically: rewriting a
-distribution's metadata underneath a live process and re-reading returns the
-*new* version, with no ``importlib.invalidate_caches()`` needed.  That is the
-one property this whole module rests on, so it is pinned by a test.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Callable, Optional
-
-from superlocalmemory import __version__
-
-__all__ = (
-    "STATE_AHEAD",
-    "STATE_CURRENT",
-    "STATE_MISMATCH",
-    "STATE_STALE",
-    "STATE_UNKNOWN",
-    "VersionIntegrity",
-    "check_version_integrity",
-    "installed_distribution_version",
-)
-
-#: Imported code matches the installed distribution.
-STATE_CURRENT = "current"
-#: Imported code is *older* than what is installed -- the #107 failure.
-STATE_STALE = "stale"
-#: Imported code is *newer* than the installed distribution.  Normal for an
-#: editable checkout; deliberately not reported as a problem, because a warning
-#: that fires on every maintainer's machine is a warning everyone learns to
-#: ignore, and then it will not be read on the day it matters.
-STATE_AHEAD = "ahead"
-#: The two differ but cannot be ordered (local labels, unexpected formats).
-#: Still surfaced -- a difference we cannot rank is not a difference we hide.
-STATE_MISMATCH = "mismatch"
-#: The installed version could not be determined at all.
-STATE_UNKNOWN = "unknown"
-
-_DISTRIBUTION_NAME = "superlocalmemory"
-
-_RESTART_HINT = (
-    "Restart this process to load the installed code "
-    "(`slm restart` for the daemon; restart your MCP client for `slm mcp`)."
-)
-
-
-@dataclass(frozen=True)
-class VersionIntegrity:
-    """The outcome of comparing imported code against the installed dist."""
-
-    running: str
-    installed: Optional[str]
-    state: str
-    detail: str
-    hint: str = ""
-
-    @property
-    def is_stale(self) -> bool:
-        """True only for the #107 failure: running behind what is installed.
-
-        Deliberately narrow.  Callers gate warnings on this, and widening it to
-        mean "anything unusual" would make an editable checkout look broken.
-        """
-        return self.state == STATE_STALE
-
-    @property
-    def differs(self) -> bool:
-        """True whenever imported and installed are known to be different."""
-        return self.state in (STATE_STALE, STATE_AHEAD, STATE_MISMATCH)
-
-    def as_dict(self) -> dict:
-        """JSON-safe payload for ``/health``, ``slm status --json``, doctor."""
-        return {
-            "running": self.running,
-            "installed": self.installed,
-            "state": self.state,
-            "detail": self.detail,
-            "hint": self.hint,
-            "is_stale": self.is_stale,
-        }
-
-
-def installed_distribution_version() -> str:
-    """Return the on-disk version of the installed distribution.
-
-    Raises whatever ``importlib.metadata`` raises; :func:`check_version_integrity`
-    is the layer that turns failure into :data:`STATE_UNKNOWN`.  Keeping the
-    raise here means a caller that genuinely wants the error can have it.
-    """
-    from importlib.metadata import version as _version
-
-    return _version(_DISTRIBUTION_NAME)
-
-
-def _version_tuple(raw: str) -> Optional[tuple[int, ...]]:
-    """Parse a plain dotted release into ints, or ``None`` if it is not one.
-
-    Intentionally strict and dependency-free: anything carrying a local label,
-    pre-release marker, or non-numeric field returns ``None`` and is reported as
-    :data:`STATE_MISMATCH`.  Guessing an order for such versions could mask a
-    real drift behind a confident-looking "current".
-    """
-    parts = raw.strip().split(".")
-    if not parts or any(not p.isdigit() for p in parts):
-        return None
-    return tuple(int(p) for p in parts)
-
-
-def check_version_integrity(
-    *,
-    running: Optional[str] = None,
-    installed_reader: Optional[Callable[[], str]] = None,
-) -> VersionIntegrity:
-    """Compare imported code against the installed distribution.
-
-    Never raises.  Both sides are injectable so tests can drive every branch
-    without touching the real environment.
-
-    Args:
-        running: Version of the *imported* code.  Defaults to
-            ``superlocalmemory.__version__``, which is frozen at import.
-        installed_reader: Callable returning the on-disk version.  Defaults to
-            reading the installed distribution metadata.
-    """
-    running_version = running if running is not None else __version__
-
-    reader = installed_reader or installed_distribution_version
-    installed: Optional[str] = None
-    try:
-        candidate = reader()
-    except BaseException:  # noqa: BLE001 - staleness reporting must never raise
-        # BaseException, not Exception: this runs on daemon and MCP startup
-        # paths, and a diagnostic must never be the reason a process dies.
-        candidate = None
-
-    if isinstance(candidate, str) and candidate.strip():
-        installed = candidate.strip()
-
-    if installed is None:
-        return VersionIntegrity(
-            running=running_version,
-            installed=None,
-            state=STATE_UNKNOWN,
-            detail=(
-                f"running {running_version}; could not read the installed "
-                f"distribution version, so staleness is undetermined"
-            ),
-        )
-
-    if running_version == installed:
-        return VersionIntegrity(
-            running=running_version,
-            installed=installed,
-            state=STATE_CURRENT,
-            detail=f"running {running_version}, matching the installed distribution",
-        )
-
-    running_parts = _version_tuple(running_version)
-    installed_parts = _version_tuple(installed)
-
-    if running_parts is None or installed_parts is None:
-        return VersionIntegrity(
-            running=running_version,
-            installed=installed,
-            state=STATE_MISMATCH,
-            detail=(
-                f"running {running_version} but {installed} is installed; "
-                f"the two cannot be ordered"
-            ),
-            hint=_RESTART_HINT,
-        )
-
-    if running_parts < installed_parts:
-        return VersionIntegrity(
-            running=running_version,
-            installed=installed,
-            state=STATE_STALE,
-            detail=(
-                f"running {running_version} but {installed} is installed — this "
-                f"process loaded its code before the upgrade and will keep "
-                f"serving {running_version} until it restarts"
-            ),
-            hint=_RESTART_HINT,
-        )
-
-    return VersionIntegrity(
-        running=running_version,
-        installed=installed,
-        state=STATE_AHEAD,
-        detail=(
-            f"running {running_version}, ahead of the installed {installed} "
-            f"(normal for an editable or source checkout)"
-        ),
-    )

package/src/superlocalmemory/ingestion/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""SLM Ingestion — external source adapters for Gmail, Calendar, Transcripts.
-
-ALL adapters are OPT-IN. Nothing runs by default. User enables via:
-  slm adapters enable gmail
-  slm adapters enable calendar
-  slm adapters enable transcript
-
-Adapters are stateless external processes that POST to the daemon's /ingest endpoint.
-"""

package/src/superlocalmemory/ingestion/adapter_manager.py

@@ -1,255 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Adapter lifecycle manager — start, stop, enable, disable ingestion adapters.
-
-All adapters run as separate subprocesses managed via PID files.
-Config stored in ~/.superlocalmemory/adapters.json.
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-from superlocalmemory.infra.data_root import canonical_data_root, state_path
-
-logger = logging.getLogger("superlocalmemory.ingestion.manager")
-
-_SLM_HOME = None  # test-only compatibility override
-_ADAPTERS_CONFIG = None  # test-only compatibility override
-_VALID_ADAPTERS = ("gmail", "calendar", "transcript")
-
-# Module paths for each adapter
-_ADAPTER_MODULES = {
-    "gmail": "superlocalmemory.ingestion.gmail_adapter",
-    "calendar": "superlocalmemory.ingestion.calendar_adapter",
-    "transcript": "superlocalmemory.ingestion.transcript_adapter",
-}
-
-
-def _slm_home() -> Path:
-    return Path(_SLM_HOME) if _SLM_HOME is not None else canonical_data_root()
-
-
-def _adapters_config_path() -> Path:
-    if _ADAPTERS_CONFIG is not None:
-        return Path(_ADAPTERS_CONFIG)
-    return state_path("adapters.json")
-
-
-def _adapter_log_dir() -> Path:
-    return _slm_home() / "logs"
-
-
-def _load_config() -> dict:
-    config_path = _adapters_config_path()
-    if config_path.exists():
-        return json.loads(config_path.read_text())
-    return {name: {"enabled": False} for name in _VALID_ADAPTERS}
-
-
-def _save_config(config: dict) -> None:
-    config_path = _adapters_config_path()
-    config_path.parent.mkdir(parents=True, exist_ok=True)
-    config_path.write_text(json.dumps(config, indent=2))
-
-
-def _pid_file(name: str) -> Path:
-    return _slm_home() / f"adapter-{name}.pid"
-
-
-def _is_running(name: str) -> tuple[bool, int | None]:
-    """Check if adapter is running. Returns (running, pid)."""
-    pf = _pid_file(name)
-    if not pf.exists():
-        return False, None
-    try:
-        pid = int(pf.read_text().strip())
-        try:
-            import psutil
-            return psutil.pid_exists(pid), pid
-        except ImportError:
-            os.kill(pid, 0)
-            return True, pid
-    except (ValueError, ProcessLookupError, PermissionError):
-        pf.unlink(missing_ok=True)
-        return False, None
-
-
-# ---------------------------------------------------------------------------
-# Public API
-# ---------------------------------------------------------------------------
-
-def list_adapters() -> list[dict]:
-    """List all adapters with their status."""
-    config = _load_config()
-    result = []
-    for name in _VALID_ADAPTERS:
-        ac = config.get(name, {})
-        running, pid = _is_running(name)
-        result.append({
-            "name": name,
-            "enabled": ac.get("enabled", False),
-            "running": running,
-            "pid": pid,
-            "tier": ac.get("tier", ""),
-            "watch_dir": ac.get("watch_dir", ""),
-        })
-    return result
-
-
-def enable_adapter(name: str) -> dict:
-    """Enable an adapter in config."""
-    if name not in _VALID_ADAPTERS:
-        return {"ok": False, "error": f"Unknown adapter: {name}. Valid: {_VALID_ADAPTERS}"}
-    config = _load_config()
-    config.setdefault(name, {})["enabled"] = True
-    _save_config(config)
-    return {"ok": True, "message": f"{name} adapter enabled. Run `slm adapters start {name}` to start."}
-
-
-def disable_adapter(name: str) -> dict:
-    """Disable an adapter. Stops it if running."""
-    if name not in _VALID_ADAPTERS:
-        return {"ok": False, "error": f"Unknown adapter: {name}"}
-    stop_adapter(name)
-    config = _load_config()
-    config.setdefault(name, {})["enabled"] = False
-    _save_config(config)
-    return {"ok": True, "message": f"{name} adapter disabled"}
-
-
-def start_adapter(name: str) -> dict:
-    """Start an adapter subprocess."""
-    if name not in _VALID_ADAPTERS:
-        return {"ok": False, "error": f"Unknown adapter: {name}"}
-
-    config = _load_config()
-    if not config.get(name, {}).get("enabled"):
-        return {"ok": False, "error": f"{name} not enabled. Run `slm adapters enable {name}` first."}
-
-    running, pid = _is_running(name)
-    if running:
-        return {"ok": True, "message": f"{name} already running (PID {pid})"}
-
-    module = _ADAPTER_MODULES.get(name)
-    if not module:
-        return {"ok": False, "error": f"No module for {name}"}
-
-    cmd = [sys.executable, "-m", module]
-    log_dir = _adapter_log_dir()
-    log_dir.mkdir(parents=True, exist_ok=True)
-    log_path = log_dir / f"adapter-{name}.log"
-
-    kwargs: dict = {}
-    if sys.platform == "win32":
-        kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
-    else:
-        kwargs["start_new_session"] = True
-
-    with open(log_path, "a") as lf:
-        proc = subprocess.Popen(cmd, stdout=lf, stderr=lf, **kwargs)
-
-    _pid_file(name).write_text(str(proc.pid))
-    return {"ok": True, "message": f"{name} started (PID {proc.pid})", "pid": proc.pid}
-
-
-def stop_adapter(name: str) -> dict:
-    """Stop a running adapter."""
-    running, pid = _is_running(name)
-    if not running:
-        return {"ok": True, "message": f"{name} not running"}
-
-    try:
-        import psutil
-        proc = psutil.Process(pid)
-        proc.terminate()
-        proc.wait(timeout=10)
-    except ImportError:
-        # No psutil: best-effort termination. On Windows os.kill(pid, 15) maps to
-        # TerminateProcess (a hard kill, not a graceful SIGTERM) — psutil's
-        # terminate() does the same there, so behavior is equivalent either way.
-        os.kill(pid, 15)
-    except Exception:
-        pass
-
-    _pid_file(name).unlink(missing_ok=True)
-    return {"ok": True, "message": f"{name} stopped"}
-
-
-def status_adapters() -> list[dict]:
-    """Get detailed status of all adapters."""
-    return list_adapters()
-
-
-# ---------------------------------------------------------------------------
-# CLI handler (called from commands.py)
-# ---------------------------------------------------------------------------
-
-def handle_adapters_cli(args: list[str]) -> None:
-    """Handle `slm adapters <action> [name]` commands."""
-    if not args:
-        args = ["list"]
-
-    action = args[0]
-    name = args[1] if len(args) > 1 else ""
-
-    if action == "list":
-        adapters = list_adapters()
-        print("  Ingestion Adapters:")
-        print("  " + "-" * 50)
-        for a in adapters:
-            status = "running" if a["running"] else ("enabled" if a["enabled"] else "disabled")
-            pid_str = f" (PID {a['pid']})" if a["pid"] else ""
-            print(f"  {a['name']:12s} {status:10s}{pid_str}")
-        print()
-
-    elif action == "enable":
-        if not name:
-            print("  Usage: slm adapters enable <gmail|calendar|transcript>")
-            return
-        result = enable_adapter(name)
-        print(f"  {result.get('message', result.get('error', ''))}")
-
-    elif action == "disable":
-        if not name:
-            print("  Usage: slm adapters disable <name>")
-            return
-        result = disable_adapter(name)
-        print(f"  {result.get('message', result.get('error', ''))}")
-
-    elif action == "start":
-        if not name:
-            print("  Usage: slm adapters start <name>")
-            return
-        result = start_adapter(name)
-        print(f"  {result.get('message', result.get('error', ''))}")
-
-    elif action == "stop":
-        if not name:
-            print("  Usage: slm adapters stop <name>")
-            return
-        result = stop_adapter(name)
-        print(f"  {result.get('message', result.get('error', ''))}")
-
-    elif action == "status":
-        adapters = status_adapters()
-        for a in adapters:
-            status = "RUNNING" if a["running"] else ("enabled" if a["enabled"] else "off")
-            print(f"  {a['name']:12s} [{status}]", end="")
-            if a["pid"]:
-                print(f"  PID={a['pid']}", end="")
-            print()
-
-    else:
-        print(f"  Unknown action: {action}")
-        print("  Usage: slm adapters <list|enable|disable|start|stop|status> [name]")

package/src/superlocalmemory/ingestion/base_adapter.py

@@ -1,171 +0,0 @@
-# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
-# Licensed under AGPL-3.0-or-later - see LICENSE file
-# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
-
-"""Base adapter class for all ingestion adapters.
-
-All adapters inherit this. Enforces stateless, safe, cross-platform operation:
-  - Clean shutdown via stop event + parent PID watchdog
-  - Rate limiting per hour
-  - Batch throttling with interruptible delays
-  - Retry on 429 responses
-  - Structured logging
-
-Part of Qualixar | Author: Varun Pratap Bhardwaj
-License: AGPL-3.0-or-later
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-import signal
-import sys
-import threading
-import time
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import NamedTuple
-
-logger = logging.getLogger("superlocalmemory.ingestion")
-
-
-@dataclass
-class AdapterConfig:
-    daemon_port: int = 8765
-    batch_size: int = 50
-    batch_delay_sec: float = 5.0
-    rate_limit_per_hour: int = 100
-
-
-class IngestItem(NamedTuple):
-    content: str
-    dedup_key: str
-    metadata: dict = {}
-
-
-class BaseAdapter:
-    """All ingestion adapters inherit this class.
-
-    Provides: run loop, rate limiting, retry, shutdown, parent watchdog.
-    Subclasses implement: fetch_items(), wait_for_next_cycle(), source_type.
-    """
-
-    source_type: str = "unknown"
-
-    def __init__(self, config: AdapterConfig | None = None):
-        self.config = config or AdapterConfig()
-        self.daemon_url = f"http://127.0.0.1:{self.config.daemon_port}"
-        self._items_this_hour = 0
-        self._hour_start = time.time()
-        self._stop_event = threading.Event()
-        self._parent_pid = os.getppid()
-        self._total_ingested = 0
-
-    def run(self) -> None:
-        """Main adapter loop. Subclasses don't override this."""
-        self._setup_signals()
-        logger.info("%s adapter started (PID %d)", self.source_type, os.getpid())
-
-        while not self._stop_event.is_set():
-            # Parent watchdog: exit if daemon died
-            try:
-                import psutil
-                if not psutil.pid_exists(self._parent_pid):
-                    logger.info("Parent daemon died, adapter exiting")
-                    break
-            except ImportError:
-                try:
-                    os.kill(self._parent_pid, 0)
-                except (ProcessLookupError, PermissionError):
-                    logger.info("Parent daemon died, adapter exiting")
-                    break
-
-            try:
-                items = self.fetch_items()
-            except Exception as exc:
-                logger.warning("fetch_items failed: %s", exc)
-                self._stop_event.wait(30)
-                continue
-
-            if not items:
-                self.wait_for_next_cycle()
-                continue
-
-            # Process in batches
-            for i in range(0, len(items), self.config.batch_size):
-                if self._stop_event.is_set():
-                    break
-                batch = items[i:i + self.config.batch_size]
-                for item in batch:
-                    if self._stop_event.is_set():
-                        break
-                    if self._rate_limited():
-                        logger.info("Rate limit reached (%d/hr), waiting",
-                                    self.config.rate_limit_per_hour)
-                        self._stop_event.wait(60)
-                        continue
-                    self._ingest(item)
-                # Interruptible batch delay
-                self._stop_event.wait(self.config.batch_delay_sec)
-
-            self.wait_for_next_cycle()
-
-        logger.info("%s adapter stopped (total ingested: %d)",
-                    self.source_type, self._total_ingested)
-
-    def stop(self) -> None:
-        self._stop_event.set()
-
-    # -- Subclass interface --
-
-    def fetch_items(self) -> list[IngestItem]:
-        """Fetch items from the source. Subclass MUST implement."""
-        raise NotImplementedError
-
-    def wait_for_next_cycle(self) -> None:
-        """Wait before next fetch cycle. Default: 5 min interruptible."""
-        self._stop_event.wait(300)
-
-    # -- Internal --
-
-    def _ingest(self, item: IngestItem) -> bool:
-        """POST to daemon /ingest endpoint. Returns True on success."""
-        payload = {
-            "content": item.content,
-            "source_type": self.source_type,
-            "dedup_key": item.dedup_key,
-            "metadata": item.metadata if item.metadata else {},
-        }
-        for attempt in range(2):
-            try:
-                from superlocalmemory.cli.daemon import daemon_request
-
-                data = daemon_request("POST", "/ingest", payload)
-                if data and data.get("ingested"):
-                    self._items_this_hour += 1
-                    self._total_ingested += 1
-                    return True
-                return False  # Already ingested (dedup)
-            except Exception as exc:
-                if "429" in str(exc) and attempt == 0:
-                    logger.info("Daemon returned 429, backing off 5s")
-                    self._stop_event.wait(5)
-                    continue
-                logger.debug("Ingest failed: %s", exc)
-                return False
-        return False
-
-    def _rate_limited(self) -> bool:
-        if time.time() - self._hour_start > 3600:
-            self._items_this_hour = 0
-            self._hour_start = time.time()
-        return self._items_this_hour >= self.config.rate_limit_per_hour
-
-    def _setup_signals(self) -> None:
-        """Set up clean shutdown on SIGTERM."""
-        def _handler(sig, frame):
-            self.stop()
-        signal.signal(signal.SIGTERM, _handler)
-        if sys.platform != "win32":
-            signal.signal(signal.SIGINT, _handler)