jxa-mail-mcp 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

jxa_mail_mcp/index/sync.py

@@ -0,0 +1,305 @@
+"""Incremental sync via JXA for new emails.
+
+This module syncs new emails that arrived since the last index build.
+Uses JXA (slower) because it works without Full Disk Access.
+
+The sync process:
+1. Get set of already-indexed message IDs
+2. Fetch message IDs from Mail.app via JXA
+3. Find IDs that aren't in the index yet
+4. Fetch content for new emails via JXA
+5. Insert into index
+
+SECURITY NOTE: All strings passed to JXA are serialized via json.dumps()
+to prevent injection attacks.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import sqlite3
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from ..config import get_index_max_emails
+from ..executor import build_mailbox_setup_js
+from .schema import INSERT_EMAIL_SQL, email_to_row
+from .search import add_account_mailbox_filter
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+
+def get_indexed_message_ids(
+    conn: sqlite3.Connection,
+    account: str | None = None,
+    mailbox: str | None = None,
+) -> set[int]:
+    """
+    Get all message IDs currently in the index.
+
+    Args:
+        conn: Database connection
+        account: Optional account filter
+        mailbox: Optional mailbox filter
+
+    Returns:
+        Set of message IDs (note: only unique within account/mailbox)
+    """
+    # Use "WHERE 1=1" pattern to allow consistent filter appending
+    sql = "SELECT message_id FROM emails e WHERE 1=1"
+    params: list = []
+    sql = add_account_mailbox_filter(sql, params, account, mailbox)
+
+    cursor = conn.execute(sql, params)
+    return {row[0] for row in cursor}
+
+
+def fetch_mailbox_ids_jxa(account: str | None, mailbox: str) -> list[int]:
+    """
+    Fetch all message IDs from a mailbox via JXA.
+
+    Args:
+        account: Account name (None for first account)
+        mailbox: Mailbox name
+
+    Returns:
+        List of message IDs
+    """
+    from ..executor import execute_with_core
+
+    # Use shared helper for safe serialization
+    mailbox_setup = build_mailbox_setup_js(account, mailbox)
+
+    script = f"""
+{mailbox_setup}
+if (!mailbox) {{
+    JSON.stringify([]);
+}} else {{
+    const ids = mailbox.messages.id();
+    JSON.stringify(ids || []);
+}}
+"""
+
+    try:
+        result = execute_with_core(script)
+        return result if isinstance(result, list) else []
+    except Exception as e:
+        logger.warning("Failed to fetch IDs for %s/%s: %s", account, mailbox, e)
+        return []
+
+
+def fetch_emails_by_ids_jxa(
+    account: str | None, mailbox: str, message_ids: list[int]
+) -> list[dict]:
+    """
+    Fetch email content for specific IDs via JXA.
+
+    Args:
+        account: Account name
+        mailbox: Mailbox name
+        message_ids: List of message IDs to fetch
+
+    Returns:
+        List of email dicts with id, subject, sender, content, date_received
+    """
+    if not message_ids:
+        return []
+
+    from ..executor import execute_with_core
+
+    # Use shared helper for safe serialization
+    mailbox_setup = build_mailbox_setup_js(account, mailbox)
+    ids_json = json.dumps(message_ids)
+
+    script = f"""
+{mailbox_setup}
+if (!mailbox) {{
+    JSON.stringify([]);
+}} else {{
+    const targetIds = new Set({ids_json});
+
+    // Batch fetch IDs to find indices
+    const allIds = mailbox.messages.id() || [];
+    const indices = [];
+    for (let i = 0; i < allIds.length; i++) {{
+        if (targetIds.has(allIds[i])) {{
+            indices.push(i);
+        }}
+    }}
+
+    // Fetch properties for matching messages only
+    const results = [];
+    for (const idx of indices) {{
+        try {{
+            const msg = mailbox.messages[idx];
+            results.push({{
+                id: msg.id(),
+                subject: msg.subject() || '',
+                sender: msg.sender() || '',
+                content: msg.content() || '',
+                date_received: MailCore.formatDate(msg.dateReceived())
+            }});
+        }} catch (e) {{
+            // Skip messages that can't be read
+        }}
+    }}
+
+    JSON.stringify(results);
+}}
+"""
+
+    try:
+        result = execute_with_core(script)
+        return result if isinstance(result, list) else []
+    except Exception as e:
+        logger.warning(
+            "Failed to fetch emails for %s/%s: %s", account, mailbox, e
+        )
+        return []
+
+
+def get_all_mailboxes_jxa() -> list[tuple[str, str]]:
+    """
+    Get all account/mailbox pairs via JXA.
+
+    Returns:
+        List of (account_name, mailbox_name) tuples
+    """
+    from ..executor import execute_with_core
+
+    script = """
+const results = [];
+const accounts = Mail.accounts();
+const accountNames = Mail.accounts.name();
+
+for (let i = 0; i < accounts.length; i++) {
+    const account = accounts[i];
+    const accountName = accountNames[i];
+    const mailboxNames = account.mailboxes.name() || [];
+
+    for (const mbName of mailboxNames) {
+        results.push([accountName, mbName]);
+    }
+}
+
+JSON.stringify(results);
+"""
+
+    try:
+        result = execute_with_core(script)
+        return result if isinstance(result, list) else []
+    except Exception as e:
+        logger.warning("Failed to get mailboxes: %s", e)
+        return []
+
+
+def sync_incremental(
+    conn: sqlite3.Connection,
+    progress_callback: Callable[[int, int | None, str], None] | None = None,
+) -> int:
+    """
+    Sync new emails via JXA.
+
+    Compares indexed IDs with Mail.app and fetches only new emails.
+    Much faster than rebuild for startup sync.
+
+    Args:
+        conn: Database connection
+        progress_callback: Optional callback(current, total, message)
+
+    Returns:
+        Number of new emails synced
+    """
+    max_per_mailbox = get_index_max_emails()
+    total_synced = 0
+
+    # Get all mailboxes
+    if progress_callback:
+        progress_callback(0, None, "Discovering mailboxes...")
+
+    mailboxes = get_all_mailboxes_jxa()
+    if not mailboxes:
+        logger.info("No mailboxes found to sync")
+        return 0
+
+    logger.info("Syncing %d mailboxes", len(mailboxes))
+
+    for i, (account, mailbox) in enumerate(mailboxes):
+        if progress_callback:
+            progress_callback(
+                i, len(mailboxes), f"Syncing {account}/{mailbox}..."
+            )
+
+        # Get already indexed IDs for this mailbox
+        indexed_ids = get_indexed_message_ids(conn, account, mailbox)
+
+        # Check if we're at the limit
+        if len(indexed_ids) >= max_per_mailbox:
+            logger.debug("Mailbox %s/%s at limit, skipping", account, mailbox)
+            continue
+
+        # Get current IDs from Mail.app
+        current_ids = fetch_mailbox_ids_jxa(account, mailbox)
+        if not current_ids:
+            continue
+
+        # Find new IDs
+        new_ids = [mid for mid in current_ids if mid not in indexed_ids]
+
+        if not new_ids:
+            continue
+
+        # Limit to stay under max
+        remaining_capacity = max_per_mailbox - len(indexed_ids)
+        new_ids = new_ids[:remaining_capacity]
+
+        logger.debug(
+            "Fetching %d new emails from %s/%s", len(new_ids), account, mailbox
+        )
+
+        # Fetch content for new emails (in batches)
+        batch_size = 50
+        for batch_start in range(0, len(new_ids), batch_size):
+            batch_ids = new_ids[batch_start : batch_start + batch_size]
+            emails = fetch_emails_by_ids_jxa(account, mailbox, batch_ids)
+
+            if not emails:
+                continue
+
+            # Insert into database using centralized SQL and tuple converter
+            for email in emails:
+                try:
+                    row = email_to_row(email, account, mailbox)
+                    conn.execute(INSERT_EMAIL_SQL, row)
+                except sqlite3.IntegrityError:
+                    logger.debug(
+                        "Duplicate email ID %s in %s/%s",
+                        email["id"],
+                        account,
+                        mailbox,
+                    )
+                except sqlite3.Error as e:
+                    logger.error("Database error: %s", e)
+
+            total_synced += len(emails)
+
+        # Update sync state
+        now = datetime.now().isoformat()
+        conn.execute(
+            """INSERT OR REPLACE INTO sync_state
+               (account, mailbox, last_sync, message_count)
+               VALUES (?, ?, ?, ?)""",
+            (account, mailbox, now, len(indexed_ids) + len(new_ids)),
+        )
+
+        conn.commit()
+
+    if progress_callback:
+        progress_callback(len(mailboxes), len(mailboxes), "Sync complete")
+
+    logger.info("Synced %d new emails", total_synced)
+    return total_synced

jxa_mail_mcp/index/watcher.py

@@ -0,0 +1,341 @@
+"""File watcher for real-time index updates.
+
+Watches ~/Library/Mail/V10/ for .emlx file changes and updates the index.
+
+Uses watchfiles (Rust-based, efficient) to monitor:
+- New emails → parse and add to index
+- Deleted emails → remove from index
+
+The watcher runs in a background thread and batches updates to avoid
+overwhelming the database with rapid changes.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import sqlite3
+import threading
+import time
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from .disk import find_mail_directory, parse_emlx
+from .schema import INSERT_EMAIL_SQL, create_connection
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+# Regex to extract account/mailbox from path
+# ~/Library/Mail/V10/[AccountUUID]/[Mailbox].mbox/.../*.emlx
+PATH_PATTERN = re.compile(r"/V\d+/([^/]+)/([^/]+)\.mbox/.*?/(\d+)\.emlx$")
+
+# Constants for safety limits
+MAX_PENDING_CHANGES = 10000  # Prevent unbounded memory growth
+DELETE_BATCH_SIZE = 500  # SQLite variable limit safety
+FILE_RETRY_DELAY_MS = 200  # Wait for Mail.app to finish writing
+MAX_FILE_RETRIES = 3
+
+
+class IndexWatcher:
+    """
+    Watches Mail directory for changes and updates the index.
+
+    Usage:
+        watcher = IndexWatcher(db_path, on_update=callback)
+        watcher.start()
+        # ... later ...
+        watcher.stop()
+    """
+
+    def __init__(
+        self,
+        db_path: Path,
+        on_update: Callable[[int, int], None] | None = None,
+        debounce_ms: int = 500,
+    ):
+        """
+        Initialize the watcher.
+
+        Args:
+            db_path: Path to the index database
+            on_update: Optional callback(added, removed) after processing
+            debounce_ms: Milliseconds to wait before processing changes
+        """
+        self.db_path = db_path
+        self.on_update = on_update
+        self.debounce_ms = debounce_ms
+
+        self._mail_dir: Path | None = None
+        self._stop_event = threading.Event()
+        self._thread: threading.Thread | None = None
+
+        # Pending changes (debounced)
+        self._pending_adds: dict[
+            tuple[str, str, int], Path
+        ] = {}  # (acc, mb, id) -> path
+        self._pending_deletes: set[tuple[str, str, int]] = (
+            set()
+        )  # (acc, mb, id)
+        self._pending_lock = threading.Lock()
+
+        # Persistent connection for the watcher thread
+        self._conn: sqlite3.Connection | None = None
+
+    def start(self) -> bool:
+        """
+        Start watching for changes.
+
+        Returns:
+            True if started successfully, False if mail dir not found
+        """
+        try:
+            self._mail_dir = find_mail_directory()
+        except FileNotFoundError:
+            logger.warning("Mail directory not found, watcher not started")
+            return False
+
+        self._stop_event.clear()
+        self._thread = threading.Thread(
+            target=self._watch_loop,
+            name="IndexWatcher",
+            daemon=True,
+        )
+        self._thread.start()
+        logger.info("File watcher started for %s", self._mail_dir)
+        return True
+
+    def stop(self, timeout: float = 5.0) -> None:
+        """Stop watching and wait for thread to finish."""
+        self._stop_event.set()
+        if self._thread and self._thread.is_alive():
+            self._thread.join(timeout=timeout)
+        self._thread = None
+
+        # Close persistent connection
+        if self._conn:
+            try:
+                self._conn.close()
+            except Exception:
+                pass
+            self._conn = None
+
+        logger.info("File watcher stopped")
+
+    @property
+    def is_running(self) -> bool:
+        """Check if watcher is running."""
+        return self._thread is not None and self._thread.is_alive()
+
+    def _get_conn(self) -> sqlite3.Connection:
+        """Get or create a persistent connection for this thread."""
+        if self._conn is None:
+            self._conn = create_connection(self.db_path)
+        return self._conn
+
+    def _watch_loop(self) -> None:
+        """Main watch loop (runs in background thread)."""
+        try:
+            from watchfiles import Change, watch
+        except ImportError:
+            logger.error("watchfiles not installed, file watcher unavailable")
+            return
+
+        if not self._mail_dir:
+            return
+
+        logger.debug("Starting watch loop on %s", self._mail_dir)
+
+        for changes in watch(
+            self._mail_dir,
+            stop_event=self._stop_event,
+            debounce=self.debounce_ms,
+            recursive=True,
+        ):
+            if self._stop_event.is_set():
+                break
+
+            # Collect changes
+            for change_type, path_str in changes:
+                if not path_str.endswith(".emlx"):
+                    continue
+
+                # Security: Validate path is within mail directory
+                try:
+                    path = Path(path_str).resolve()
+                    if not str(path).startswith(str(self._mail_dir.resolve())):
+                        logger.warning(
+                            "Ignoring path outside mail dir: %s", path
+                        )
+                        continue
+                except (OSError, ValueError) as e:
+                    logger.warning("Invalid path %s: %s", path_str, e)
+                    continue
+
+                parsed = self._parse_path(path)
+                if not parsed:
+                    continue
+
+                account, mailbox, message_id = parsed
+                key = (account, mailbox, message_id)
+
+                with self._pending_lock:
+                    # Prevent unbounded memory growth
+                    total_pending = len(self._pending_adds) + len(
+                        self._pending_deletes
+                    )
+                    if total_pending >= MAX_PENDING_CHANGES:
+                        logger.warning(
+                            "Pending limit (%d) reached, clearing",
+                            MAX_PENDING_CHANGES,
+                        )
+                        # Clear half to make room
+                        self._pending_adds.clear()
+
+                    if change_type == Change.added:
+                        self._pending_adds[key] = path
+                        self._pending_deletes.discard(key)
+                    elif change_type == Change.deleted:
+                        self._pending_deletes.add(key)
+                        self._pending_adds.pop(key, None)
+                    elif change_type == Change.modified:
+                        # Treat as add (re-index)
+                        self._pending_adds[key] = path
+
+            # Process after debounce period
+            self._process_pending()
+
+    def _parse_path(self, path: Path) -> tuple[str, str, int] | None:
+        """
+        Extract account, mailbox, and message ID from path.
+
+        Returns:
+            (account_name, mailbox_name, message_id) or None if invalid
+        """
+        match = PATH_PATTERN.search(str(path))
+        if not match:
+            return None
+
+        account_uuid, mailbox_dir, message_id_str = match.groups()
+
+        try:
+            message_id = int(message_id_str)
+        except ValueError:
+            return None
+
+        # Use UUID as account name (more reliable than trying to map)
+        account_name = account_uuid
+        mailbox_name = mailbox_dir
+
+        return account_name, mailbox_name, message_id
+
+    def _process_pending(self) -> None:
+        """Process pending adds and deletes."""
+        with self._pending_lock:
+            adds = dict(self._pending_adds)
+            deletes = set(self._pending_deletes)
+            self._pending_adds.clear()
+            self._pending_deletes.clear()
+
+        if not adds and not deletes:
+            return
+
+        added_count = 0
+        deleted_count = 0
+
+        try:
+            conn = self._get_conn()
+
+            # Process deletes in batches to avoid SQLite variable limit
+            if deletes:
+                delete_list = list(deletes)
+                for i in range(0, len(delete_list), DELETE_BATCH_SIZE):
+                    batch = delete_list[i : i + DELETE_BATCH_SIZE]
+                    # Use composite key for deletion
+                    for account, mailbox, msg_id in batch:
+                        sql = """DELETE FROM emails WHERE account = ?
+                                 AND mailbox = ? AND message_id = ?"""
+                        conn.execute(sql, (account, mailbox, msg_id))
+                    deleted_count += len(batch)
+
+            # Process adds with retry for files still being written
+            for key, path in adds.items():
+                account, mailbox, message_id = key
+                email = None
+
+                # Retry logic for race condition with Mail.app writing
+                for attempt in range(MAX_FILE_RETRIES):
+                    try:
+                        email = parse_emlx(path)
+                        if email:
+                            break
+                    except OSError as e:
+                        if attempt < MAX_FILE_RETRIES - 1:
+                            logger.debug(
+                                "Retry %d for %s: %s", attempt + 1, path, e
+                            )
+                            time.sleep(FILE_RETRY_DELAY_MS / 1000)
+                        else:
+                            logger.warning(
+                                "Failed to parse %s after retries: %s", path, e
+                            )
+                    except Exception as e:
+                        logger.warning("Error parsing %s: %s", path, e)
+                        break
+
+                if email:
+                    try:
+                        # Use centralized SQL; indexed_at uses DEFAULT
+                        conn.execute(
+                            INSERT_EMAIL_SQL,
+                            (
+                                message_id,
+                                account,
+                                mailbox,
+                                email.get("subject", ""),
+                                email.get("sender", ""),
+                                email.get("content", ""),
+                                email.get("date_received", ""),
+                            ),
+                        )
+                        added_count += 1
+                    except sqlite3.IntegrityError as e:
+                        logger.debug("Duplicate email %s: %s", key, e)
+                    except sqlite3.Error as e:
+                        logger.error("Database error for %s: %s", key, e)
+
+            conn.commit()
+
+        except sqlite3.Error as e:
+            logger.error("Database error in watcher: %s", e)
+
+        # Notify callback
+        if self.on_update and (added_count or deleted_count):
+            try:
+                self.on_update(added_count, deleted_count)
+            except Exception as e:
+                logger.warning("Error in on_update callback: %s", e)
+
+        if added_count or deleted_count:
+            logger.debug(
+                "Processed: +%d -%d emails", added_count, deleted_count
+            )
+
+
+def create_watcher(
+    db_path: Path,
+    on_update: Callable[[int, int], None] | None = None,
+) -> IndexWatcher:
+    """
+    Create and return a new IndexWatcher.
+
+    Args:
+        db_path: Path to the index database
+        on_update: Optional callback(added, removed) after changes
+
+    Returns:
+        Configured IndexWatcher (call .start() to begin watching)
+    """
+    return IndexWatcher(db_path, on_update=on_update)