superlocalmemory 3.4.0 → 3.4.3

package/src/superlocalmemory/ingestion/base_adapter.py

@@ -0,0 +1,177 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Base adapter class for all ingestion adapters.
+
+All adapters inherit this. Enforces stateless, safe, cross-platform operation:
+  - Clean shutdown via stop event + parent PID watchdog
+  - Rate limiting per hour
+  - Batch throttling with interruptible delays
+  - Retry on 429 responses
+  - Structured logging
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import signal
+import sys
+import threading
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import NamedTuple
+
+logger = logging.getLogger("superlocalmemory.ingestion")
+
+
+@dataclass
+class AdapterConfig:
+    daemon_port: int = 8765
+    batch_size: int = 50
+    batch_delay_sec: float = 5.0
+    rate_limit_per_hour: int = 100
+
+
+class IngestItem(NamedTuple):
+    content: str
+    dedup_key: str
+    metadata: dict = {}
+
+
+class BaseAdapter:
+    """All ingestion adapters inherit this class.
+
+    Provides: run loop, rate limiting, retry, shutdown, parent watchdog.
+    Subclasses implement: fetch_items(), wait_for_next_cycle(), source_type.
+    """
+
+    source_type: str = "unknown"
+
+    def __init__(self, config: AdapterConfig | None = None):
+        self.config = config or AdapterConfig()
+        self.daemon_url = f"http://127.0.0.1:{self.config.daemon_port}"
+        self._items_this_hour = 0
+        self._hour_start = time.time()
+        self._stop_event = threading.Event()
+        self._parent_pid = os.getppid()
+        self._total_ingested = 0
+
+    def run(self) -> None:
+        """Main adapter loop. Subclasses don't override this."""
+        self._setup_signals()
+        logger.info("%s adapter started (PID %d)", self.source_type, os.getpid())
+
+        while not self._stop_event.is_set():
+            # Parent watchdog: exit if daemon died
+            try:
+                import psutil
+                if not psutil.pid_exists(self._parent_pid):
+                    logger.info("Parent daemon died, adapter exiting")
+                    break
+            except ImportError:
+                try:
+                    os.kill(self._parent_pid, 0)
+                except (ProcessLookupError, PermissionError):
+                    logger.info("Parent daemon died, adapter exiting")
+                    break
+
+            try:
+                items = self.fetch_items()
+            except Exception as exc:
+                logger.warning("fetch_items failed: %s", exc)
+                self._stop_event.wait(30)
+                continue
+
+            if not items:
+                self.wait_for_next_cycle()
+                continue
+
+            # Process in batches
+            for i in range(0, len(items), self.config.batch_size):
+                if self._stop_event.is_set():
+                    break
+                batch = items[i:i + self.config.batch_size]
+                for item in batch:
+                    if self._stop_event.is_set():
+                        break
+                    if self._rate_limited():
+                        logger.info("Rate limit reached (%d/hr), waiting",
+                                    self.config.rate_limit_per_hour)
+                        self._stop_event.wait(60)
+                        continue
+                    self._ingest(item)
+                # Interruptible batch delay
+                self._stop_event.wait(self.config.batch_delay_sec)
+
+            self.wait_for_next_cycle()
+
+        logger.info("%s adapter stopped (total ingested: %d)",
+                    self.source_type, self._total_ingested)
+
+    def stop(self) -> None:
+        self._stop_event.set()
+
+    # -- Subclass interface --
+
+    def fetch_items(self) -> list[IngestItem]:
+        """Fetch items from the source. Subclass MUST implement."""
+        raise NotImplementedError
+
+    def wait_for_next_cycle(self) -> None:
+        """Wait before next fetch cycle. Default: 5 min interruptible."""
+        self._stop_event.wait(300)
+
+    # -- Internal --
+
+    def _ingest(self, item: IngestItem) -> bool:
+        """POST to daemon /ingest endpoint. Returns True on success."""
+        try:
+            import urllib.request
+            payload = json.dumps({
+                "content": item.content,
+                "source_type": self.source_type,
+                "dedup_key": item.dedup_key,
+                "metadata": item.metadata if item.metadata else {},
+            }).encode()
+            req = urllib.request.Request(
+                f"{self.daemon_url}/ingest",
+                data=payload,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            resp = urllib.request.urlopen(req, timeout=30)
+            data = json.loads(resp.read().decode())
+            if data.get("ingested"):
+                self._items_this_hour += 1
+                self._total_ingested += 1
+                return True
+            return False  # Already ingested (dedup)
+        except Exception as exc:
+            error_str = str(exc)
+            if "429" in error_str:
+                logger.info("Daemon returned 429, backing off 5s")
+                self._stop_event.wait(5)
+                return self._ingest(item)  # Retry once
+            logger.debug("Ingest failed: %s", exc)
+            return False
+
+    def _rate_limited(self) -> bool:
+        if time.time() - self._hour_start > 3600:
+            self._items_this_hour = 0
+            self._hour_start = time.time()
+        return self._items_this_hour >= self.config.rate_limit_per_hour
+
+    def _setup_signals(self) -> None:
+        """Set up clean shutdown on SIGTERM."""
+        def _handler(sig, frame):
+            self.stop()
+        signal.signal(signal.SIGTERM, _handler)
+        if sys.platform != "win32":
+            signal.signal(signal.SIGINT, _handler)

package/src/superlocalmemory/ingestion/calendar_adapter.py

@@ -0,0 +1,340 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Google Calendar ingestion adapter — 2 tiers.
+
+Tier 1: ICS file import — zero setup
+Tier 2: Google Calendar API with OAuth polling — shares Gmail OAuth credentials
+
+OPT-IN only. Enabled via: slm adapters enable calendar
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from superlocalmemory.ingestion.base_adapter import BaseAdapter, AdapterConfig, IngestItem
+
+logger = logging.getLogger("superlocalmemory.ingestion.calendar")
+
+
+class CalendarAdapter(BaseAdapter):
+    """Google Calendar ingestion with automatic tier detection."""
+
+    source_type = "calendar"
+
+    def __init__(self, config: AdapterConfig | None = None, tier: str = "auto"):
+        super().__init__(config)
+        self._tier = tier
+        self._ics_path: str | None = None
+        self._ics_processed = False
+        self._sync_token: str | None = None
+        self._poll_interval = 900  # 15 min
+
+    def run(self) -> None:
+        self._detect_tier()
+        logger.info("Calendar adapter starting (tier=%s)", self._tier)
+        super().run()
+
+    def fetch_items(self) -> list[IngestItem]:
+        if self._tier == "ics":
+            return self._fetch_ics()
+        elif self._tier == "oauth":
+            return self._fetch_oauth()
+        return []
+
+    def wait_for_next_cycle(self) -> None:
+        if self._tier == "ics" and self._ics_processed:
+            logger.info("ICS import complete, adapter stopping")
+            self.stop()
+            return
+        self._stop_event.wait(self._poll_interval)
+
+    def _detect_tier(self) -> None:
+        if self._tier != "auto":
+            return
+
+        adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+        cfg = {}
+        if adapters_path.exists():
+            cfg = json.loads(adapters_path.read_text()).get("calendar", {})
+
+        if cfg.get("tier") == "ics" or cfg.get("ics_path"):
+            self._tier = "ics"
+            self._ics_path = cfg.get("ics_path", "")
+            return
+
+        # Check for OAuth (shares Gmail credentials)
+        from superlocalmemory.ingestion.credentials import has_credential
+        if has_credential("gmail", "refresh_token"):
+            self._tier = "oauth"
+            return
+
+        # Look for ICS files
+        import_dir = Path.home() / ".superlocalmemory" / "import"
+        ics_files = list(import_dir.glob("*.ics")) if import_dir.exists() else []
+        if ics_files:
+            self._tier = "ics"
+            self._ics_path = str(ics_files[0])
+            return
+
+        self._tier = "ics"
+
+    # -- Tier 1: ICS file import --
+
+    def _fetch_ics(self) -> list[IngestItem]:
+        if self._ics_processed or not self._ics_path:
+            return []
+
+        path = Path(self._ics_path)
+        if not path.exists():
+            logger.warning("ICS file not found: %s", path)
+            self._ics_processed = True
+            return []
+
+        items = []
+
+        try:
+            # Try icalendar library first
+            from icalendar import Calendar
+            cal = Calendar.from_ical(path.read_bytes())
+            events = [c for c in cal.walk() if c.name == "VEVENT"]
+            logger.info("Parsing ICS: %d events", len(events))
+
+            for event in events:
+                if self._stop_event.is_set():
+                    break
+                try:
+                    summary = str(event.get("SUMMARY", "(no title)"))
+                    dtstart = event.get("DTSTART")
+                    dtend = event.get("DTEND")
+                    description = str(event.get("DESCRIPTION", ""))
+                    location = str(event.get("LOCATION", ""))
+                    uid = str(event.get("UID", ""))
+
+                    start_str = dtstart.dt.isoformat() if dtstart else ""
+                    end_str = dtend.dt.isoformat() if dtend else ""
+
+                    # Extract attendees
+                    attendees = []
+                    att_list = event.get("ATTENDEE", [])
+                    if not isinstance(att_list, list):
+                        att_list = [att_list]
+                    for att in att_list:
+                        email = str(att).replace("mailto:", "").strip()
+                        if email and "@" in email:
+                            attendees.append(email.split("@")[0])  # Use name part
+
+                    content = (
+                        f"Calendar Event: {summary}\n"
+                        f"When: {start_str} to {end_str}\n"
+                    )
+                    if location:
+                        content += f"Where: {location}\n"
+                    if attendees:
+                        content += f"Attendees: {', '.join(attendees)}\n"
+                    if description:
+                        content += f"\n{description[:2000]}"
+
+                    dedup_key = f"{uid}-{start_str}" if uid else f"ics-{summary}-{start_str}"
+
+                    items.append(IngestItem(
+                        content=content,
+                        dedup_key=dedup_key,
+                        metadata={
+                            "summary": summary,
+                            "start": start_str,
+                            "end": end_str,
+                            "attendees": attendees,
+                            "source": "ics_import",
+                        },
+                    ))
+
+                    # Entity propagation for attendees
+                    for attendee in attendees:
+                        items.append(IngestItem(
+                            content=f"{attendee} attended meeting: {summary} on {start_str}",
+                            dedup_key=f"attendee-{attendee}-{dedup_key}",
+                            metadata={
+                                "entity_name": attendee,
+                                "event": summary,
+                                "source": "entity_propagation",
+                            },
+                        ))
+
+                except Exception as exc:
+                    logger.debug("Failed to parse event: %s", exc)
+
+        except ImportError:
+            # Fallback: basic ICS parsing without icalendar library
+            logger.info("icalendar not installed, using basic parser")
+            items = self._parse_ics_basic(path)
+
+        self._ics_processed = True
+        logger.info("ICS import: %d items", len(items))
+        return items
+
+    def _parse_ics_basic(self, path: Path) -> list[IngestItem]:
+        """Basic ICS parser without icalendar library."""
+        content = path.read_text(encoding="utf-8", errors="replace")
+        items = []
+        events = content.split("BEGIN:VEVENT")
+
+        for i, block in enumerate(events[1:]):  # Skip first (before any VEVENT)
+            try:
+                lines = block.split("\n")
+                props = {}
+                for line in lines:
+                    if ":" in line and not line.startswith(" "):
+                        key, _, val = line.partition(":")
+                        key = key.split(";")[0].strip()
+                        props[key] = val.strip()
+
+                summary = props.get("SUMMARY", "(no title)")
+                dtstart = props.get("DTSTART", "")
+                uid = props.get("UID", f"basic-{i}")
+
+                content_text = f"Calendar Event: {summary}\nWhen: {dtstart}"
+                items.append(IngestItem(
+                    content=content_text,
+                    dedup_key=f"{uid}-{dtstart}",
+                    metadata={"summary": summary, "start": dtstart, "source": "ics_basic"},
+                ))
+            except Exception:
+                pass
+
+        return items
+
+    # -- Tier 2: OAuth API polling --
+
+    def _fetch_oauth(self) -> list[IngestItem]:
+        """Poll Google Calendar API with OAuth."""
+        try:
+            from superlocalmemory.ingestion.credentials import load_credential
+            from google.oauth2.credentials import Credentials
+            from googleapiclient.discovery import build
+
+            refresh_token = load_credential("gmail", "refresh_token")
+            client_id = load_credential("gmail", "client_id")
+            client_secret = load_credential("gmail", "client_secret")
+
+            if not all([refresh_token, client_id, client_secret]):
+                logger.warning("Calendar OAuth credentials incomplete")
+                return []
+
+            creds = Credentials(
+                token=None,
+                refresh_token=refresh_token,
+                client_id=client_id,
+                client_secret=client_secret,
+                token_uri="https://oauth2.googleapis.com/token",
+            )
+
+            service = build("calendar", "v3", credentials=creds)
+
+            # Incremental sync
+            kwargs = {"calendarId": "primary", "singleEvents": True, "maxResults": 50}
+            if self._sync_token:
+                kwargs["syncToken"] = self._sync_token
+            else:
+                # Initial sync: last 30 days
+                from datetime import timedelta
+                time_min = (datetime.now(timezone.utc) - timedelta(days=30)).isoformat()
+                kwargs["timeMin"] = time_min
+
+            try:
+                results = service.events().list(**kwargs).execute()
+            except Exception as api_err:
+                if "410" in str(api_err):
+                    # Sync token expired — full re-sync
+                    logger.info("Calendar sync token expired, doing full re-sync")
+                    self._sync_token = None
+                    return self._fetch_oauth()
+                raise
+
+            self._sync_token = results.get("nextSyncToken")
+            events = results.get("items", [])
+
+            items = []
+            for event in events:
+                if self._stop_event.is_set():
+                    break
+                try:
+                    summary = event.get("summary", "(no title)")
+                    start = event.get("start", {}).get("dateTime", event.get("start", {}).get("date", ""))
+                    end = event.get("end", {}).get("dateTime", event.get("end", {}).get("date", ""))
+                    description = event.get("description", "")
+                    location = event.get("location", "")
+                    event_id = event.get("id", "")
+                    updated = event.get("updated", "")
+
+                    attendees = []
+                    for att in event.get("attendees", []):
+                        name = att.get("displayName") or att.get("email", "").split("@")[0]
+                        if name:
+                            attendees.append(name)
+
+                    content = f"Calendar Event: {summary}\nWhen: {start} to {end}\n"
+                    if location:
+                        content += f"Where: {location}\n"
+                    if attendees:
+                        content += f"Attendees: {', '.join(attendees)}\n"
+                    if description:
+                        content += f"\n{description[:2000]}"
+
+                    dedup_key = f"{event_id}-{updated}"
+
+                    items.append(IngestItem(
+                        content=content,
+                        dedup_key=dedup_key,
+                        metadata={
+                            "summary": summary, "start": start,
+                            "attendees": attendees, "source": "oauth",
+                        },
+                    ))
+
+                    # Entity propagation
+                    for attendee in attendees:
+                        items.append(IngestItem(
+                            content=f"{attendee} attended: {summary} on {start}",
+                            dedup_key=f"cal-attendee-{attendee}-{event_id}",
+                            metadata={"entity_name": attendee, "event": summary, "source": "entity_propagation"},
+                        ))
+
+                except Exception as exc:
+                    logger.debug("Calendar event parse error: %s", exc)
+
+            return items
+
+        except ImportError:
+            logger.warning("Calendar OAuth requires: pip install 'superlocalmemory[ingestion]'")
+            return []
+        except Exception as exc:
+            logger.warning("Calendar OAuth failed: %s", exc)
+            return []
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import logging as _logging
+    _logging.basicConfig(level=_logging.INFO, format="%(asctime)s %(message)s")
+
+    adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+    tier = "auto"
+    if adapters_path.exists():
+        cfg = json.loads(adapters_path.read_text()).get("calendar", {})
+        tier = cfg.get("tier", "auto")
+
+    adapter = CalendarAdapter(tier=tier)
+    adapter.run()

package/src/superlocalmemory/ingestion/credentials.py

@@ -0,0 +1,118 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Cross-platform credential storage for ingestion adapters.
+
+Uses OS keychain via keyring library (macOS Keychain, Windows Credential Locker,
+Linux SecretService). Falls back to file-based storage with restricted permissions.
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+
+logger = logging.getLogger("superlocalmemory.ingestion.credentials")
+
+_CRED_DIR = Path.home() / ".superlocalmemory" / "credentials"
+_SERVICE_PREFIX = "slm"
+
+
+def store_credential(service: str, key: str, value: str) -> bool:
+    """Store a credential securely. Returns True on success."""
+    # Try OS keychain first
+    try:
+        import keyring
+        keyring.set_password(f"{_SERVICE_PREFIX}-{service}", key, value)
+        logger.debug("Stored %s/%s in OS keychain", service, key)
+        return True
+    except Exception:
+        pass
+
+    # Fallback: encrypted file with restricted permissions
+    try:
+        _CRED_DIR.mkdir(parents=True, exist_ok=True)
+        cred_file = _CRED_DIR / f"{service}.json"
+
+        existing = {}
+        if cred_file.exists():
+            try:
+                existing = json.loads(cred_file.read_text())
+            except (json.JSONDecodeError, OSError):
+                pass
+
+        existing[key] = value
+        cred_file.write_text(json.dumps(existing, indent=2))
+
+        # Restrict permissions (Unix only — Windows skipped)
+        if sys.platform != "win32":
+            os.chmod(cred_file, 0o600)
+            os.chmod(_CRED_DIR, 0o700)
+
+        logger.debug("Stored %s/%s in file (keychain unavailable)", service, key)
+        return True
+    except Exception as exc:
+        logger.error("Failed to store credential %s/%s: %s", service, key, exc)
+        return False
+
+
+def load_credential(service: str, key: str) -> str | None:
+    """Load a credential. Tries keychain first, then file."""
+    # Try OS keychain
+    try:
+        import keyring
+        value = keyring.get_password(f"{_SERVICE_PREFIX}-{service}", key)
+        if value:
+            return value
+    except Exception:
+        pass
+
+    # Fallback: file
+    try:
+        cred_file = _CRED_DIR / f"{service}.json"
+        if cred_file.exists():
+            data = json.loads(cred_file.read_text())
+            return data.get(key)
+    except Exception:
+        pass
+
+    return None
+
+
+def delete_credential(service: str, key: str) -> bool:
+    """Delete a credential from both keychain and file."""
+    deleted = False
+
+    # Try keychain
+    try:
+        import keyring
+        keyring.delete_password(f"{_SERVICE_PREFIX}-{service}", key)
+        deleted = True
+    except Exception:
+        pass
+
+    # Also remove from file
+    try:
+        cred_file = _CRED_DIR / f"{service}.json"
+        if cred_file.exists():
+            data = json.loads(cred_file.read_text())
+            if key in data:
+                del data[key]
+                cred_file.write_text(json.dumps(data, indent=2))
+                deleted = True
+    except Exception:
+        pass
+
+    return deleted
+
+
+def has_credential(service: str, key: str) -> bool:
+    """Check if a credential exists."""
+    return load_credential(service, key) is not None