superlocalmemory 3.4.1 → 3.4.3

package/src/superlocalmemory/ingestion/adapter_manager.py

@@ -0,0 +1,234 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Adapter lifecycle manager — start, stop, enable, disable ingestion adapters.
+
+All adapters run as separate subprocesses managed via PID files.
+Config stored in ~/.superlocalmemory/adapters.json.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+logger = logging.getLogger("superlocalmemory.ingestion.manager")
+
+_SLM_HOME = Path.home() / ".superlocalmemory"
+_ADAPTERS_CONFIG = _SLM_HOME / "adapters.json"
+_VALID_ADAPTERS = ("gmail", "calendar", "transcript")
+
+# Module paths for each adapter
+_ADAPTER_MODULES = {
+    "gmail": "superlocalmemory.ingestion.gmail_adapter",
+    "calendar": "superlocalmemory.ingestion.calendar_adapter",
+    "transcript": "superlocalmemory.ingestion.transcript_adapter",
+}
+
+
+def _load_config() -> dict:
+    if _ADAPTERS_CONFIG.exists():
+        return json.loads(_ADAPTERS_CONFIG.read_text())
+    return {name: {"enabled": False} for name in _VALID_ADAPTERS}
+
+
+def _save_config(config: dict) -> None:
+    _ADAPTERS_CONFIG.parent.mkdir(parents=True, exist_ok=True)
+    _ADAPTERS_CONFIG.write_text(json.dumps(config, indent=2))
+
+
+def _pid_file(name: str) -> Path:
+    return _SLM_HOME / f"adapter-{name}.pid"
+
+
+def _is_running(name: str) -> tuple[bool, int | None]:
+    """Check if adapter is running. Returns (running, pid)."""
+    pf = _pid_file(name)
+    if not pf.exists():
+        return False, None
+    try:
+        pid = int(pf.read_text().strip())
+        try:
+            import psutil
+            return psutil.pid_exists(pid), pid
+        except ImportError:
+            os.kill(pid, 0)
+            return True, pid
+    except (ValueError, ProcessLookupError, PermissionError):
+        pf.unlink(missing_ok=True)
+        return False, None
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def list_adapters() -> list[dict]:
+    """List all adapters with their status."""
+    config = _load_config()
+    result = []
+    for name in _VALID_ADAPTERS:
+        ac = config.get(name, {})
+        running, pid = _is_running(name)
+        result.append({
+            "name": name,
+            "enabled": ac.get("enabled", False),
+            "running": running,
+            "pid": pid,
+            "tier": ac.get("tier", ""),
+            "watch_dir": ac.get("watch_dir", ""),
+        })
+    return result
+
+
+def enable_adapter(name: str) -> dict:
+    """Enable an adapter in config."""
+    if name not in _VALID_ADAPTERS:
+        return {"ok": False, "error": f"Unknown adapter: {name}. Valid: {_VALID_ADAPTERS}"}
+    config = _load_config()
+    config.setdefault(name, {})["enabled"] = True
+    _save_config(config)
+    return {"ok": True, "message": f"{name} adapter enabled. Run `slm adapters start {name}` to start."}
+
+
+def disable_adapter(name: str) -> dict:
+    """Disable an adapter. Stops it if running."""
+    if name not in _VALID_ADAPTERS:
+        return {"ok": False, "error": f"Unknown adapter: {name}"}
+    stop_adapter(name)
+    config = _load_config()
+    config.setdefault(name, {})["enabled"] = False
+    _save_config(config)
+    return {"ok": True, "message": f"{name} adapter disabled"}
+
+
+def start_adapter(name: str) -> dict:
+    """Start an adapter subprocess."""
+    if name not in _VALID_ADAPTERS:
+        return {"ok": False, "error": f"Unknown adapter: {name}"}
+
+    config = _load_config()
+    if not config.get(name, {}).get("enabled"):
+        return {"ok": False, "error": f"{name} not enabled. Run `slm adapters enable {name}` first."}
+
+    running, pid = _is_running(name)
+    if running:
+        return {"ok": True, "message": f"{name} already running (PID {pid})"}
+
+    module = _ADAPTER_MODULES.get(name)
+    if not module:
+        return {"ok": False, "error": f"No module for {name}"}
+
+    cmd = [sys.executable, "-m", module]
+    log_dir = _SLM_HOME / "logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_path = log_dir / f"adapter-{name}.log"
+
+    kwargs: dict = {}
+    if sys.platform == "win32":
+        kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+    else:
+        kwargs["start_new_session"] = True
+
+    with open(log_path, "a") as lf:
+        proc = subprocess.Popen(cmd, stdout=lf, stderr=lf, **kwargs)
+
+    _pid_file(name).write_text(str(proc.pid))
+    return {"ok": True, "message": f"{name} started (PID {proc.pid})", "pid": proc.pid}
+
+
+def stop_adapter(name: str) -> dict:
+    """Stop a running adapter."""
+    running, pid = _is_running(name)
+    if not running:
+        return {"ok": True, "message": f"{name} not running"}
+
+    try:
+        import psutil
+        proc = psutil.Process(pid)
+        proc.terminate()
+        proc.wait(timeout=10)
+    except ImportError:
+        os.kill(pid, 15)  # SIGTERM
+    except Exception:
+        pass
+
+    _pid_file(name).unlink(missing_ok=True)
+    return {"ok": True, "message": f"{name} stopped"}
+
+
+def status_adapters() -> list[dict]:
+    """Get detailed status of all adapters."""
+    return list_adapters()
+
+
+# ---------------------------------------------------------------------------
+# CLI handler (called from commands.py)
+# ---------------------------------------------------------------------------
+
+def handle_adapters_cli(args: list[str]) -> None:
+    """Handle `slm adapters <action> [name]` commands."""
+    if not args:
+        args = ["list"]
+
+    action = args[0]
+    name = args[1] if len(args) > 1 else ""
+
+    if action == "list":
+        adapters = list_adapters()
+        print("  Ingestion Adapters:")
+        print("  " + "-" * 50)
+        for a in adapters:
+            status = "running" if a["running"] else ("enabled" if a["enabled"] else "disabled")
+            pid_str = f" (PID {a['pid']})" if a["pid"] else ""
+            print(f"  {a['name']:12s} {status:10s}{pid_str}")
+        print()
+
+    elif action == "enable":
+        if not name:
+            print("  Usage: slm adapters enable <gmail|calendar|transcript>")
+            return
+        result = enable_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "disable":
+        if not name:
+            print("  Usage: slm adapters disable <name>")
+            return
+        result = disable_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "start":
+        if not name:
+            print("  Usage: slm adapters start <name>")
+            return
+        result = start_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "stop":
+        if not name:
+            print("  Usage: slm adapters stop <name>")
+            return
+        result = stop_adapter(name)
+        print(f"  {result.get('message', result.get('error', ''))}")
+
+    elif action == "status":
+        adapters = status_adapters()
+        for a in adapters:
+            status = "RUNNING" if a["running"] else ("enabled" if a["enabled"] else "off")
+            print(f"  {a['name']:12s} [{status}]", end="")
+            if a["pid"]:
+                print(f"  PID={a['pid']}", end="")
+            print()
+
+    else:
+        print(f"  Unknown action: {action}")
+        print("  Usage: slm adapters <list|enable|disable|start|stop|status> [name]")

package/src/superlocalmemory/ingestion/base_adapter.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Base adapter class for all ingestion adapters.
+
+All adapters inherit this. Enforces stateless, safe, cross-platform operation:
+  - Clean shutdown via stop event + parent PID watchdog
+  - Rate limiting per hour
+  - Batch throttling with interruptible delays
+  - Retry on 429 responses
+  - Structured logging
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import signal
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import NamedTuple
+
+logger = logging.getLogger("superlocalmemory.ingestion")
+
+
+@dataclass
+class AdapterConfig:
+    daemon_port: int = 8765
+    batch_size: int = 50
+    batch_delay_sec: float = 5.0
+    rate_limit_per_hour: int = 100
+
+
+class IngestItem(NamedTuple):
+    content: str
+    dedup_key: str
+    metadata: dict = {}
+
+
+class BaseAdapter:
+    """All ingestion adapters inherit this class.
+
+    Provides: run loop, rate limiting, retry, shutdown, parent watchdog.
+    Subclasses implement: fetch_items(), wait_for_next_cycle(), source_type.
+    """
+
+    source_type: str = "unknown"
+
+    def __init__(self, config: AdapterConfig | None = None):
+        self.config = config or AdapterConfig()
+        self.daemon_url = f"http://127.0.0.1:{self.config.daemon_port}"
+        self._items_this_hour = 0
+        self._hour_start = time.time()
+        self._stop_event = threading.Event()
+        self._parent_pid = os.getppid()
+        self._total_ingested = 0
+
+    def run(self) -> None:
+        """Main adapter loop. Subclasses don't override this."""
+        self._setup_signals()
+        logger.info("%s adapter started (PID %d)", self.source_type, os.getpid())
+
+        while not self._stop_event.is_set():
+            # Parent watchdog: exit if daemon died
+            try:
+                import psutil
+                if not psutil.pid_exists(self._parent_pid):
+                    logger.info("Parent daemon died, adapter exiting")
+                    break
+            except ImportError:
+                try:
+                    os.kill(self._parent_pid, 0)
+                except (ProcessLookupError, PermissionError):
+                    logger.info("Parent daemon died, adapter exiting")
+                    break
+
+            try:
+                items = self.fetch_items()
+            except Exception as exc:
+                logger.warning("fetch_items failed: %s", exc)
+                self._stop_event.wait(30)
+                continue
+
+            if not items:
+                self.wait_for_next_cycle()
+                continue
+
+            # Process in batches
+            for i in range(0, len(items), self.config.batch_size):
+                if self._stop_event.is_set():
+                    break
+                batch = items[i:i + self.config.batch_size]
+                for item in batch:
+                    if self._stop_event.is_set():
+                        break
+                    if self._rate_limited():
+                        logger.info("Rate limit reached (%d/hr), waiting",
+                                    self.config.rate_limit_per_hour)
+                        self._stop_event.wait(60)
+                        continue
+                    self._ingest(item)
+                # Interruptible batch delay
+                self._stop_event.wait(self.config.batch_delay_sec)
+
+            self.wait_for_next_cycle()
+
+        logger.info("%s adapter stopped (total ingested: %d)",
+                    self.source_type, self._total_ingested)
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+    # -- Subclass interface --
+
+    def fetch_items(self) -> list[IngestItem]:
+        """Fetch items from the source. Subclass MUST implement."""
+        raise NotImplementedError
+
+    def wait_for_next_cycle(self) -> None:
+        """Wait before next fetch cycle. Default: 5 min interruptible."""
+        self._stop_event.wait(300)
+
+    # -- Internal --
+
+    def _ingest(self, item: IngestItem) -> bool:
+        """POST to daemon /ingest endpoint. Returns True on success."""
+        try:
+            import urllib.request
+            payload = json.dumps({
+                "content": item.content,
+                "source_type": self.source_type,
+                "dedup_key": item.dedup_key,
+                "metadata": item.metadata if item.metadata else {},
+            }).encode()
+            req = urllib.request.Request(
+                f"{self.daemon_url}/ingest",
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            resp = urllib.request.urlopen(req, timeout=30)
+            data = json.loads(resp.read().decode())
+            if data.get("ingested"):
+                self._items_this_hour += 1
+                self._total_ingested += 1
+                return True
+            return False  # Already ingested (dedup)
+        except Exception as exc:
+            error_str = str(exc)
+            if "429" in error_str:
+                logger.info("Daemon returned 429, backing off 5s")
+                self._stop_event.wait(5)
+                return self._ingest(item)  # Retry once
+            logger.debug("Ingest failed: %s", exc)
+            return False
+
+    def _rate_limited(self) -> bool:
+        if time.time() - self._hour_start > 3600:
+            self._items_this_hour = 0
+            self._hour_start = time.time()
+        return self._items_this_hour >= self.config.rate_limit_per_hour
+
+    def _setup_signals(self) -> None:
+        """Set up clean shutdown on SIGTERM."""
+        def _handler(sig, frame):
+            self.stop()
+        signal.signal(signal.SIGTERM, _handler)
+        if sys.platform != "win32":
+            signal.signal(signal.SIGINT, _handler)