superlocalmemory 3.4.1 → 3.4.3

package/src/superlocalmemory/ingestion/gmail_adapter.py

@@ -0,0 +1,369 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Gmail ingestion adapter — 3 tiers of Gmail access.
+
+Tier 1: File import (.mbox from Google Takeout) — zero setup
+Tier 1.5: IMAP polling — no GCP, just email/password
+Tier 2: Gmail API with OAuth polling — needs GCP OAuth client, no Pub/Sub
+Tier 3: Gmail API with Pub/Sub push — full GCP (future)
+
+OPT-IN only. Enabled via: slm adapters enable gmail
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from pathlib import Path
+
+from superlocalmemory.ingestion.base_adapter import BaseAdapter, AdapterConfig, IngestItem
+
+logger = logging.getLogger("superlocalmemory.ingestion.gmail")
+
+
+class GmailAdapter(BaseAdapter):
+    """Gmail ingestion with automatic tier detection."""
+
+    source_type = "gmail"
+
+    def __init__(self, config: AdapterConfig | None = None, tier: str = "auto"):
+        super().__init__(config)
+        self._tier = tier
+        self._mbox_path: str | None = None
+        self._mbox_processed = False
+        self._history_id: str | None = None
+        self._poll_interval = 300  # 5 min for API polling
+
+    def run(self) -> None:
+        """Detect tier and run."""
+        self._detect_tier()
+        logger.info("Gmail adapter starting (tier=%s)", self._tier)
+        super().run()
+
+    def fetch_items(self) -> list[IngestItem]:
+        """Fetch items based on active tier."""
+        if self._tier == "mbox":
+            return self._fetch_mbox()
+        elif self._tier == "imap":
+            return self._fetch_imap()
+        elif self._tier == "oauth":
+            return self._fetch_oauth()
+        return []
+
+    def wait_for_next_cycle(self) -> None:
+        """Tier 1 (mbox): run once then stop. Others: poll interval."""
+        if self._tier == "mbox" and self._mbox_processed:
+            logger.info("MBOX import complete, adapter stopping")
+            self.stop()
+            return
+        self._stop_event.wait(self._poll_interval)
+
+    # -- Tier detection --
+
+    def _detect_tier(self) -> None:
+        """Auto-detect the best available tier."""
+        if self._tier != "auto":
+            return
+
+        adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+        cfg = {}
+        if adapters_path.exists():
+            cfg = json.loads(adapters_path.read_text()).get("gmail", {})
+
+        # Check for explicit tier
+        if cfg.get("tier") == "mbox" or cfg.get("mbox_path"):
+            self._tier = "mbox"
+            self._mbox_path = cfg.get("mbox_path", "")
+            return
+
+        if cfg.get("tier") == "imap":
+            self._tier = "imap"
+            return
+
+        # Check for OAuth credentials
+        from superlocalmemory.ingestion.credentials import has_credential
+        if has_credential("gmail", "refresh_token"):
+            self._tier = "oauth"
+            return
+
+        # Default: look for mbox file
+        mbox_dir = Path.home() / ".superlocalmemory" / "import"
+        mbox_files = list(mbox_dir.glob("*.mbox")) if mbox_dir.exists() else []
+        if mbox_files:
+            self._tier = "mbox"
+            self._mbox_path = str(mbox_files[0])
+            return
+
+        logger.warning("No Gmail credentials or MBOX file found. "
+                       "Place .mbox in ~/.superlocalmemory/import/ or run setup.")
+        self._tier = "mbox"  # Will return empty if no file
+
+    # -- Tier 1: MBOX file import --
+
+    def _fetch_mbox(self) -> list[IngestItem]:
+        """Parse .mbox file from Google Takeout."""
+        if self._mbox_processed or not self._mbox_path:
+            return []
+
+        path = Path(self._mbox_path)
+        if not path.exists():
+            logger.warning("MBOX file not found: %s", path)
+            self._mbox_processed = True
+            return []
+
+        import mailbox
+        items = []
+        mbox = mailbox.mbox(str(path))
+        total = len(mbox)
+        logger.info("Parsing MBOX: %d messages", total)
+
+        for i, message in enumerate(mbox):
+            if self._stop_event.is_set():
+                break
+
+            try:
+                msg_id = message.get("Message-ID", f"mbox-{i}")
+                subject = message.get("Subject", "(no subject)")
+                from_addr = message.get("From", "unknown")
+                date = message.get("Date", "")
+
+                # Extract plain text body
+                body = ""
+                if message.is_multipart():
+                    for part in message.walk():
+                        if part.get_content_type() == "text/plain":
+                            payload = part.get_payload(decode=True)
+                            if payload:
+                                body = payload.decode("utf-8", errors="replace")
+                                break
+                else:
+                    payload = message.get_payload(decode=True)
+                    if payload:
+                        body = payload.decode("utf-8", errors="replace")
+
+                # Truncate body
+                body = body[:3000] if body else ""
+
+                content = f"Email: {subject}\nFrom: {from_addr}\nDate: {date}\n\n{body}"
+
+                items.append(IngestItem(
+                    content=content,
+                    dedup_key=str(msg_id).strip("<>"),
+                    metadata={
+                        "subject": subject,
+                        "from": from_addr,
+                        "date": date,
+                        "source": "mbox_import",
+                    },
+                ))
+
+                # Progress logging
+                if (i + 1) % 100 == 0:
+                    logger.info("MBOX progress: %d/%d messages", i + 1, total)
+
+            except Exception as exc:
+                logger.debug("Failed to parse message %d: %s", i, exc)
+
+        self._mbox_processed = True
+        logger.info("MBOX import: %d messages extracted", len(items))
+        return items
+
+    # -- Tier 1.5: IMAP polling --
+
+    def _fetch_imap(self) -> list[IngestItem]:
+        """Poll via IMAP. Requires email + password credentials."""
+        try:
+            import imaplib
+            from superlocalmemory.ingestion.credentials import load_credential
+
+            host = load_credential("gmail", "imap_host") or "imap.gmail.com"
+            email = load_credential("gmail", "email")
+            password = load_credential("gmail", "password")
+
+            if not email or not password:
+                logger.warning("IMAP credentials not found. Run: slm adapters enable gmail --setup")
+                return []
+
+            conn = imaplib.IMAP4_SSL(host)
+            conn.login(email, password)
+            conn.select("INBOX")
+
+            # Fetch last 20 unseen messages
+            _, msg_nums = conn.search(None, "UNSEEN")
+            items = []
+
+            for num in msg_nums[0].split()[-20:]:
+                if self._stop_event.is_set():
+                    break
+                try:
+                    _, data = conn.fetch(num, "(RFC822)")
+                    import email as email_lib
+                    msg = email_lib.message_from_bytes(data[0][1])
+                    msg_id = msg.get("Message-ID", f"imap-{num.decode()}")
+                    subject = msg.get("Subject", "(no subject)")
+                    from_addr = msg.get("From", "unknown")
+
+                    body = ""
+                    if msg.is_multipart():
+                        for part in msg.walk():
+                            if part.get_content_type() == "text/plain":
+                                payload = part.get_payload(decode=True)
+                                if payload:
+                                    body = payload.decode("utf-8", errors="replace")
+                                    break
+                    else:
+                        payload = msg.get_payload(decode=True)
+                        if payload:
+                            body = payload.decode("utf-8", errors="replace")
+
+                    body = body[:3000] if body else ""
+                    content = f"Email: {subject}\nFrom: {from_addr}\n\n{body}"
+
+                    items.append(IngestItem(
+                        content=content,
+                        dedup_key=str(msg_id).strip("<>"),
+                        metadata={"subject": subject, "from": from_addr, "source": "imap"},
+                    ))
+                except Exception as exc:
+                    logger.debug("IMAP fetch error: %s", exc)
+
+            conn.logout()
+            return items
+
+        except Exception as exc:
+            logger.warning("IMAP polling failed: %s", exc)
+            return []
+
+    # -- Tier 2: OAuth API polling --
+
+    def _fetch_oauth(self) -> list[IngestItem]:
+        """Poll Gmail API with OAuth. Requires google-api-python-client."""
+        try:
+            from superlocalmemory.ingestion.credentials import load_credential
+
+            refresh_token = load_credential("gmail", "refresh_token")
+            client_id = load_credential("gmail", "client_id")
+            client_secret = load_credential("gmail", "client_secret")
+
+            if not all([refresh_token, client_id, client_secret]):
+                logger.warning("Gmail OAuth credentials incomplete. Run setup.")
+                return []
+
+            # Build credentials
+            from google.oauth2.credentials import Credentials
+            from googleapiclient.discovery import build
+
+            creds = Credentials(
+                token=None,
+                refresh_token=refresh_token,
+                client_id=client_id,
+                client_secret=client_secret,
+                token_uri="https://oauth2.googleapis.com/token",
+            )
+
+            service = build("gmail", "v1", credentials=creds)
+
+            # Get history since last sync
+            if self._history_id:
+                results = service.users().history().list(
+                    userId="me",
+                    startHistoryId=self._history_id,
+                    historyTypes=["messageAdded"],
+                ).execute()
+                history = results.get("history", [])
+                msg_ids = []
+                for h in history:
+                    for added in h.get("messagesAdded", []):
+                        msg_ids.append(added["message"]["id"])
+            else:
+                # Initial: get last 20 messages
+                results = service.users().messages().list(
+                    userId="me", maxResults=20,
+                ).execute()
+                msg_ids = [m["id"] for m in results.get("messages", [])]
+
+            # Update history ID for next cycle
+            profile = service.users().getProfile(userId="me").execute()
+            self._history_id = profile.get("historyId")
+
+            items = []
+            for msg_id in msg_ids:
+                if self._stop_event.is_set():
+                    break
+                try:
+                    msg = service.users().messages().get(
+                        userId="me", id=msg_id, format="full",
+                    ).execute()
+                    headers = {h["name"]: h["value"] for h in msg.get("payload", {}).get("headers", [])}
+                    subject = headers.get("Subject", "(no subject)")
+                    from_addr = headers.get("From", "unknown")
+                    date = headers.get("Date", "")
+
+                    # Extract body from payload
+                    body = self._extract_gmail_body(msg.get("payload", {}))
+                    body = body[:3000] if body else ""
+                    content = f"Email: {subject}\nFrom: {from_addr}\nDate: {date}\n\n{body}"
+
+                    items.append(IngestItem(
+                        content=content,
+                        dedup_key=msg_id,
+                        metadata={"subject": subject, "from": from_addr, "date": date, "source": "oauth"},
+                    ))
+                except Exception as exc:
+                    logger.debug("Gmail API fetch error for %s: %s", msg_id, exc)
+
+            return items
+
+        except ImportError:
+            logger.warning("Gmail OAuth requires: pip install 'superlocalmemory[ingestion]'")
+            return []
+        except Exception as exc:
+            logger.warning("Gmail OAuth polling failed: %s", exc)
+            return []
+
+    @staticmethod
+    def _extract_gmail_body(payload: dict) -> str:
+        """Extract plain text body from Gmail API payload."""
+        import base64
+
+        if payload.get("mimeType") == "text/plain":
+            data = payload.get("body", {}).get("data", "")
+            if data:
+                return base64.urlsafe_b64decode(data).decode("utf-8", errors="replace")
+
+        for part in payload.get("parts", []):
+            if part.get("mimeType") == "text/plain":
+                data = part.get("body", {}).get("data", "")
+                if data:
+                    return base64.urlsafe_b64decode(data).decode("utf-8", errors="replace")
+            # Recurse into nested parts
+            if "parts" in part:
+                result = GmailAdapter._extract_gmail_body(part)
+                if result:
+                    return result
+
+        return ""
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import logging as _logging
+    _logging.basicConfig(level=_logging.INFO, format="%(asctime)s %(message)s")
+
+    adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+    tier = "auto"
+    if adapters_path.exists():
+        cfg = json.loads(adapters_path.read_text()).get("gmail", {})
+        tier = cfg.get("tier", "auto")
+
+    adapter = GmailAdapter(tier=tier)
+    adapter.run()

package/src/superlocalmemory/ingestion/parsers.py

@@ -0,0 +1,100 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Parsers for ingestion file formats: SRT, VTT, MBOX, ICS."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from pathlib import Path
+from typing import NamedTuple
+
+
+class Utterance(NamedTuple):
+    speaker: str
+    text: str
+    timestamp: str
+
+
+def parse_srt(filepath: str | Path) -> list[Utterance]:
+    """Parse SubRip (.srt) file into utterances."""
+    content = Path(filepath).read_text(encoding="utf-8", errors="replace")
+    blocks = re.split(r"\n\n+", content.strip())
+    utterances = []
+    for block in blocks:
+        lines = block.strip().split("\n")
+        if len(lines) < 3:
+            continue
+        # Line 1: sequence number, Line 2: timestamps, Line 3+: text
+        timestamp = lines[1].strip() if len(lines) > 1 else ""
+        text = " ".join(lines[2:]).strip()
+        if text:
+            # Try to extract speaker from "Speaker: text" pattern
+            speaker, content_text = _extract_speaker(text)
+            utterances.append(Utterance(speaker=speaker, text=content_text, timestamp=timestamp))
+    return utterances
+
+
+def parse_vtt(filepath: str | Path) -> list[Utterance]:
+    """Parse WebVTT (.vtt) file into utterances."""
+    content = Path(filepath).read_text(encoding="utf-8", errors="replace")
+    # Remove WEBVTT header
+    content = re.sub(r"^WEBVTT.*?\n\n", "", content, flags=re.DOTALL)
+    blocks = re.split(r"\n\n+", content.strip())
+    utterances = []
+    for block in blocks:
+        lines = block.strip().split("\n")
+        timestamp = ""
+        text_lines = []
+        for line in lines:
+            if "-->" in line:
+                timestamp = line.strip()
+            elif line.strip() and not line.strip().isdigit():
+                # Remove VTT tags like <v Speaker>
+                clean = re.sub(r"<v\s+([^>]+)>", r"\1: ", line)
+                clean = re.sub(r"<[^>]+>", "", clean).strip()
+                if clean:
+                    text_lines.append(clean)
+        text = " ".join(text_lines)
+        if text:
+            speaker, content_text = _extract_speaker(text)
+            utterances.append(Utterance(speaker=speaker, text=content_text, timestamp=timestamp))
+    return utterances
+
+
+def parse_transcript_file(filepath: str | Path) -> tuple[str, list[str]]:
+    """Parse any transcript file (.srt, .vtt, .txt).
+
+    Returns (combined_text, list_of_speakers).
+    """
+    path = Path(filepath)
+    suffix = path.suffix.lower()
+
+    if suffix == ".srt":
+        utterances = parse_srt(path)
+    elif suffix == ".vtt":
+        utterances = parse_vtt(path)
+    else:
+        # Plain text — treat entire file as one utterance
+        text = path.read_text(encoding="utf-8", errors="replace")
+        return text[:5000], []
+
+    speakers = list({u.speaker for u in utterances if u.speaker != "unknown"})
+    combined = "\n".join(f"[{u.speaker}] {u.text}" for u in utterances)
+    return combined[:5000], speakers
+
+
+def content_hash(filepath: str | Path) -> str:
+    """SHA256 of file content (first 32 chars). Path-independent for dedup."""
+    content = Path(filepath).read_bytes()
+    return hashlib.sha256(content).hexdigest()[:32]
+
+
+def _extract_speaker(text: str) -> tuple[str, str]:
+    """Extract speaker from 'Speaker: text' or 'Speaker Name: text' pattern."""
+    match = re.match(r"^([A-Z][a-zA-Z\s]{0,30}):\s*(.+)", text)
+    if match:
+        return match.group(1).strip(), match.group(2).strip()
+    return "unknown", text

package/src/superlocalmemory/ingestion/transcript_adapter.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2026 Varun Pratap Bhardwaj / Qualixar
+# Licensed under the Elastic License 2.0 - see LICENSE file
+# Part of SuperLocalMemory V3 | https://qualixar.com | https://varunpratap.com
+
+"""Transcript ingestion adapter — watches for .srt/.vtt/.txt files.
+
+Uses watchdog (cross-platform file watcher) to detect new transcript files.
+Parses them, extracts speaker diarization, propagates entities, and POSTs
+to the daemon's /ingest endpoint.
+
+OPT-IN only. Enabled via: slm adapters enable transcript
+
+Part of Qualixar | Author: Varun Pratap Bhardwaj
+License: Elastic-2.0
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+import time
+from pathlib import Path
+
+from superlocalmemory.ingestion.base_adapter import BaseAdapter, AdapterConfig, IngestItem
+
+logger = logging.getLogger("superlocalmemory.ingestion.transcript")
+
+_WATCH_EXTENSIONS = {".srt", ".vtt", ".txt"}
+
+
+class TranscriptAdapter(BaseAdapter):
+    """Watches a directory for transcript files and ingests them."""
+
+    source_type = "transcript"
+
+    def __init__(self, watch_dir: str | Path, config: AdapterConfig | None = None):
+        super().__init__(config)
+        self._watch_dir = Path(watch_dir)
+        self._pending_files: list[Path] = []
+        self._observer = None
+
+    def run(self) -> None:
+        """Start file watcher then enter the base adapter loop."""
+        if not self._watch_dir.exists():
+            logger.error("Watch directory does not exist: %s", self._watch_dir)
+            return
+
+        # Start watchdog observer
+        try:
+            from watchdog.observers import Observer
+            from watchdog.events import FileSystemEventHandler
+
+            class _Handler(FileSystemEventHandler):
+                def __init__(self, adapter: TranscriptAdapter):
+                    self._adapter = adapter
+
+                def on_created(self, event):
+                    if event.is_directory:
+                        return
+                    path = Path(event.src_path)
+                    if path.suffix.lower() in _WATCH_EXTENSIONS:
+                        self._adapter._pending_files.append(path)
+
+            self._observer = Observer()
+            self._observer.schedule(_Handler(self), str(self._watch_dir), recursive=False)
+            self._observer.start()
+            logger.info("Watching for transcripts in: %s", self._watch_dir)
+        except ImportError:
+            logger.warning("watchdog not installed — polling mode only")
+
+        # Also scan for existing files on first run
+        for path in self._watch_dir.iterdir():
+            if path.suffix.lower() in _WATCH_EXTENSIONS and path.is_file():
+                self._pending_files.append(path)
+
+        super().run()
+
+        # Cleanup
+        if self._observer:
+            self._observer.stop()
+            self._observer.join()
+
+    def fetch_items(self) -> list[IngestItem]:
+        """Return pending transcript files as IngestItems."""
+        if not self._pending_files:
+            return []
+
+        items = []
+        batch = list(self._pending_files)
+        self._pending_files.clear()
+
+        for filepath in batch:
+            try:
+                from superlocalmemory.ingestion.parsers import (
+                    parse_transcript_file, content_hash,
+                )
+                combined_text, speakers = parse_transcript_file(filepath)
+                dedup = content_hash(filepath)
+
+                # Main transcript ingestion
+                items.append(IngestItem(
+                    content=f"Meeting transcript ({filepath.name}):\n{combined_text}",
+                    dedup_key=dedup,
+                    metadata={
+                        "filename": filepath.name,
+                        "speakers": speakers,
+                        "source": "file_watcher",
+                    },
+                ))
+
+                # Entity propagation: each speaker gets a timeline entry
+                for speaker in speakers:
+                    items.append(IngestItem(
+                        content=f"{speaker} participated in meeting: {filepath.stem}. "
+                                f"Transcript file: {filepath.name}",
+                        dedup_key=f"speaker-{speaker}-{dedup}",
+                        metadata={
+                            "entity_name": speaker,
+                            "meeting_file": filepath.name,
+                            "source": "entity_propagation",
+                        },
+                    ))
+
+            except Exception as exc:
+                logger.warning("Failed to parse %s: %s", filepath, exc)
+
+        return items
+
+    def wait_for_next_cycle(self) -> None:
+        """Wait 30s for new files (watchdog handles detection)."""
+        self._stop_event.wait(30)
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point: python -m superlocalmemory.ingestion.transcript_adapter
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import logging as _logging
+    _logging.basicConfig(level=_logging.INFO, format="%(asctime)s %(message)s")
+
+    # Load config
+    watch_dir = ""
+    adapters_path = Path.home() / ".superlocalmemory" / "adapters.json"
+    if adapters_path.exists():
+        cfg = json.loads(adapters_path.read_text())
+        watch_dir = cfg.get("transcript", {}).get("watch_dir", "")
+
+    if not watch_dir:
+        print("No watch_dir configured. Set it in ~/.superlocalmemory/adapters.json")
+        print('  {"transcript": {"enabled": true, "watch_dir": "/path/to/transcripts"}}')
+        sys.exit(1)
+
+    adapter = TranscriptAdapter(watch_dir=watch_dir)
+    adapter.run()

package/src/superlocalmemory/learning/consolidation_worker.py

@@ -73,7 +73,38 @@ class ConsolidationWorker:
         except Exception as exc:
             logger.debug("Pattern generation failed: %s", exc)
 
-        # 4. Check if ranker should retrain
+        # 4. Recompute graph intelligence (v3.4.2: wired into learning pipeline)
+        try:
+            from superlocalmemory.core.graph_analyzer import GraphAnalyzer
+            conn_ga = sqlite3.connect(self._memory_db, timeout=10)
+            conn_ga.execute("PRAGMA busy_timeout=5000")
+            conn_ga.row_factory = sqlite3.Row
+
+            class _DBProxy:
+                """Minimal DB proxy for GraphAnalyzer compatibility."""
+                def __init__(self, connection: sqlite3.Connection) -> None:
+                    self._conn = connection
+                def execute(self, sql: str, params: tuple = ()) -> list:
+                    cursor = self._conn.execute(sql, params)
+                    if sql.strip().upper().startswith(("INSERT", "UPDATE", "DELETE", "ALTER", "CREATE")):
+                        self._conn.commit()
+                        return []
+                    return cursor.fetchall()
+
+            ga = GraphAnalyzer(_DBProxy(conn_ga))
+            if not dry_run:
+                ga_result = ga.compute_and_store(profile_id)
+                stats["graph_nodes"] = ga_result.get("node_count", 0)
+                stats["graph_communities"] = ga_result.get("community_count", 0)
+                logger.info(
+                    "Graph analysis: %d nodes, %d communities",
+                    stats["graph_nodes"], stats["graph_communities"],
+                )
+            conn_ga.close()
+        except Exception as exc:
+            logger.debug("Graph analysis failed: %s", exc)
+
+        # 5. Check if ranker should retrain
         try:
             from superlocalmemory.learning.feedback import FeedbackCollector
             collector = FeedbackCollector(Path(self._learning_db))
@@ -88,6 +119,21 @@ class ConsolidationWorker:
         except Exception as exc:
             logger.debug("Retrain check failed: %s", exc)
 
+        # 6. Entity compilation (v3.4.3: compiled truth per entity)
+        if not dry_run:
+            try:
+                from superlocalmemory.learning.entity_compiler import EntityCompiler
+                from superlocalmemory.core.config import SLMConfig
+                config = SLMConfig.load()
+                compiler = EntityCompiler(self._memory_db, config)
+                ec_result = compiler.compile_all(profile_id)
+                stats["entities_compiled"] = ec_result.get("compiled", 0)
+                if ec_result["compiled"] > 0:
+                    logger.info("Entity compilation: %d entities compiled",
+                                ec_result["compiled"])
+            except Exception as exc:
+                logger.debug("Entity compilation failed: %s", exc)
+
         return stats
 
     def _deduplicate(self, profile_id: str, dry_run: bool) -> int: