nia-sync 0.1.0__py3-none-any.whl

sync.py

@@ -0,0 +1,192 @@
+"""
+Sync engine for Nia Local Sync CLI.
+
+Handles:
+- Extracting data from local sources (databases, folders)
+- Uploading to cloud API
+- Cursor management for incremental sync
+"""
+import os
+import logging
+from pathlib import Path
+from typing import Any
+import httpx
+
+from config import API_BASE_URL, get_api_key, enable_source_sync
+from extractor import extract_incremental, detect_source_type
+
+logger = logging.getLogger(__name__)
+
+SYNC_TIMEOUT = 120  # 2 minutes per sync request
+
+
+def sync_all_sources(sources: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """
+    Sync all configured sources.
+
+    Args:
+        sources: List of source configs from cloud API
+
+    Returns:
+        List of results for each source
+    """
+    results = []
+
+    for source in sources:
+        result = sync_source(source)
+        results.append(result)
+
+    return results
+
+
+def sync_source(source: dict[str, Any]) -> dict[str, Any]:
+    """
+    Sync a single source.
+
+    Args:
+        source: Source config from cloud API with:
+            - local_folder_id: UUID of the local folder
+            - path: Local path to sync
+            - detected_type: Type of source
+            - cursor: Current sync cursor
+
+    Returns:
+        Result dict with status, path, and stats
+    """
+    local_folder_id = source.get("local_folder_id")
+    path = source.get("path", "")
+    detected_type = source.get("detected_type")
+    cursor = source.get("cursor", {})
+
+    # Expand ~ in path
+    path = os.path.expanduser(path)
+
+    # Validate path exists
+    if not os.path.exists(path):
+        return {
+            "path": path,
+            "status": "error",
+            "error": f"Path does not exist: {path}",
+        }
+
+    # Auto-enable sync if source exists locally but sync not enabled
+    if not source.get("sync_enabled", False):
+        logger.info(f"Auto-enabling sync for {path}")
+        enable_source_sync(local_folder_id, path)
+
+    # Auto-detect type if not specified
+    if not detected_type:
+        detected_type = detect_source_type(path)
+
+    logger.info(f"Syncing {path} (type={detected_type})")
+
+    try:
+        # Extract data incrementally
+        extraction_result = extract_incremental(
+            path=path,
+            source_type=detected_type,
+            cursor=cursor,
+        )
+
+        files = extraction_result.get("files", [])
+        new_cursor = extraction_result.get("cursor", {})
+        stats = extraction_result.get("stats", {})
+
+        if not files:
+            logger.info(f"No new data to sync for {path}")
+            return {
+                "path": path,
+                "status": "success",
+                "added": 0,
+                "message": "No new data",
+            }
+
+        # Upload to backend
+        upload_result = upload_sync_data(
+            local_folder_id=local_folder_id,
+            files=files,
+            cursor=new_cursor,
+            stats=stats,
+        )
+
+        if upload_result.get("status") == "ok":
+            # Update source cursor in-place so subsequent syncs use it
+            source["cursor"] = new_cursor
+            return {
+                "path": path,
+                "status": "success",
+                "added": len(files),
+                "chunks_indexed": upload_result.get("chunks_indexed", 0),
+                "new_cursor": new_cursor,
+            }
+        else:
+            return {
+                "path": path,
+                "status": "error",
+                "error": upload_result.get("message", "Upload failed"),
+            }
+
+    except PermissionError:
+        return {
+            "path": path,
+            "status": "error",
+            "error": "Permission denied. Grant Full Disk Access in System Settings > Privacy & Security.",
+        }
+    except Exception as e:
+        logger.error(f"Error syncing {path}: {e}", exc_info=True)
+        return {
+            "path": path,
+            "status": "error",
+            "error": str(e),
+        }
+
+
+def upload_sync_data(
+    local_folder_id: str,
+    files: list[dict[str, Any]],
+    cursor: dict[str, Any],
+    stats: dict[str, Any],
+) -> dict[str, Any]:
+    """
+    Upload extracted data to the cloud API.
+
+    Args:
+        local_folder_id: UUID of the local folder
+        files: List of extracted files with path, content, metadata
+        cursor: New cursor after extraction
+        stats: Extraction stats
+
+    Returns:
+        API response dict
+    """
+    api_key = get_api_key()
+    if not api_key:
+        return {"status": "error", "message": "Not authenticated"}
+
+    try:
+        with httpx.Client(timeout=SYNC_TIMEOUT) as client:
+            response = client.post(
+                f"{API_BASE_URL}/v2/daemon/sync",
+                headers={"Authorization": f"Bearer {api_key}"},
+                json={
+                    "local_folder_id": local_folder_id,
+                    "files": files,
+                    "cursor": cursor,
+                    "stats": stats,
+                },
+            )
+
+            if response.status_code == 200:
+                return response.json()
+            elif response.status_code == 401:
+                return {"status": "error", "message": "Authentication failed"}
+            elif response.status_code == 404:
+                return {"status": "error", "message": "Local folder not found"}
+            else:
+                detail = response.json().get("detail", response.text)
+                return {"status": "error", "message": f"API error: {detail}"}
+
+    except httpx.TimeoutException:
+        return {"status": "error", "message": "Request timeout"}
+    except httpx.RequestError as e:
+        return {"status": "error", "message": f"Network error: {e}"}

watcher.py

@@ -0,0 +1,304 @@
+"""
+File system watcher for real-time sync.
+
+Uses watchdog library to monitor file changes and trigger syncs
+with debouncing to prevent rapid-fire updates.
+"""
+import os
+import threading
+import logging
+from typing import Callable
+from pathlib import Path
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileSystemEvent
+
+logger = logging.getLogger(__name__)
+
+# File extensions to watch for changes
+WATCHED_EXTENSIONS = {
+    # Database files
+    '.db', '.db-wal', '.db-shm', '.sqlite', '.sqlite3',
+    # Document files
+    '.txt', '.md', '.json', '.yaml', '.yml',
+    # Code files (for folder sync)
+    '.py', '.js', '.ts', '.tsx', '.jsx', '.html', '.css',
+}
+
+
+class SyncEventHandler(FileSystemEventHandler):
+    """
+    Handles file system events with debouncing.
+
+    When a file changes, starts a timer. If more changes come in
+    before the timer expires, the timer resets. When the timer
+    finally expires, triggers the sync callback.
+    """
+
+    def __init__(
+        self,
+        source_id: str,
+        source_path: str,
+        on_change: Callable[[str], None],
+        debounce_sec: float = 2.0,
+    ):
+        super().__init__()
+        self.source_id = source_id
+        self.source_path = os.path.abspath(os.path.expanduser(source_path))
+        self.on_change = on_change
+        self.debounce_sec = debounce_sec
+        self._timer: threading.Timer | None = None
+        self._lock = threading.Lock()
+        self._pending_changes = 0
+
+        # For database files, also watch the WAL/SHM files
+        if self.source_path.endswith('.db'):
+            self._watched_files = {
+                self.source_path,
+                self.source_path + '-wal',
+                self.source_path + '-shm',
+            }
+        else:
+            self._watched_files = None  # Watch all files in directory
+
+    def _should_handle(self, event: FileSystemEvent) -> bool:
+        """Check if this event should trigger a sync for THIS source."""
+        if event.is_directory:
+            return False
+
+        event_path = os.path.abspath(event.src_path)
+
+        # If we're watching specific files (database), only trigger for those
+        if self._watched_files is not None:
+            return event_path in self._watched_files
+
+        # For directories, watch all relevant extensions
+        ext = Path(event_path).suffix.lower()
+        if ext in WATCHED_EXTENSIONS:
+            return True
+
+        return False
+
+    def on_modified(self, event: FileSystemEvent):
+        """Called when a file is modified."""
+        if self._should_handle(event):
+            logger.debug(f"Modified: {event.src_path}")
+            self._debounce()
+
+    def on_created(self, event: FileSystemEvent):
+        """Called when a file is created."""
+        if self._should_handle(event):
+            logger.debug(f"Created: {event.src_path}")
+            self._debounce()
+
+    def on_deleted(self, event: FileSystemEvent):
+        """Called when a file is deleted."""
+        if self._should_handle(event):
+            logger.debug(f"Deleted: {event.src_path}")
+            self._debounce()
+
+    def _debounce(self):
+        """Reset the debounce timer."""
+        with self._lock:
+            self._pending_changes += 1
+
+            # Cancel existing timer
+            if self._timer is not None:
+                self._timer.cancel()
+
+            # Start new timer
+            self._timer = threading.Timer(self.debounce_sec, self._trigger_sync)
+            self._timer.start()
+
+    def _trigger_sync(self):
+        """Called when debounce timer expires - triggers actual sync."""
+        with self._lock:
+            changes = self._pending_changes
+            self._pending_changes = 0
+            self._timer = None
+
+        logger.info(f"Triggering sync for {self.source_id} ({changes} changes detected)")
+
+        try:
+            self.on_change(self.source_id)
+        except Exception as e:
+            logger.error(f"Error in sync callback: {e}")
+
+    def cancel(self):
+        """Cancel any pending timer."""
+        with self._lock:
+            if self._timer is not None:
+                self._timer.cancel()
+                self._timer = None
+
+
+class FileWatcher:
+    """
+    Watches multiple source paths for changes.
+
+    Usage:
+        watcher = FileWatcher()
+        watcher.watch("source_id", "/path/to/file.db", on_change_callback)
+        watcher.start()
+        # ... later
+        watcher.stop()
+    """
+
+    def __init__(self, debounce_sec: float = 2.0):
+        self.debounce_sec = debounce_sec
+        self.observer = Observer()
+        self.handlers: dict[str, SyncEventHandler] = {}
+        self._watches: dict[str, any] = {}
+        self._lock = threading.Lock()
+        self._started = False
+
+    def watch(
+        self,
+        source_id: str,
+        path: str,
+        on_change: Callable[[str], None],
+    ) -> bool:
+        """
+        Add a path to watch.
+
+        Args:
+            source_id: Unique identifier for this source
+            path: File or directory path to watch
+            on_change: Callback when changes detected (receives source_id)
+
+        Returns:
+            True if successfully added, False otherwise
+        """
+        with self._lock:
+            # Skip if already watching this source
+            if source_id in self.handlers:
+                logger.debug(f"Already watching {source_id}")
+                return True
+
+            # Expand path
+            expanded = os.path.expanduser(path)
+
+            # For database files, watch the parent directory
+            # to catch .db-wal changes
+            if expanded.endswith('.db'):
+                watch_path = os.path.dirname(expanded)
+            else:
+                watch_path = expanded
+
+            # Verify path exists
+            if not os.path.exists(watch_path):
+                logger.warning(f"Path does not exist: {watch_path}")
+                return False
+
+            # Create handler
+            handler = SyncEventHandler(
+                source_id=source_id,
+                source_path=expanded,
+                on_change=on_change,
+                debounce_sec=self.debounce_sec,
+            )
+
+            # Schedule watch
+            try:
+                watch = self.observer.schedule(
+                    handler,
+                    watch_path,
+                    recursive=os.path.isdir(watch_path),
+                )
+                self.handlers[source_id] = handler
+                self._watches[source_id] = watch
+                logger.info(f"Watching {source_id}: {watch_path}")
+                return True
+            except Exception as e:
+                logger.error(f"Failed to watch {watch_path}: {e}")
+                return False
+
+    def unwatch(self, source_id: str):
+        """Stop watching a source."""
+        with self._lock:
+            if source_id not in self.handlers:
+                return
+
+            handler = self.handlers.pop(source_id)
+            handler.cancel()
+
+            watch = self._watches.pop(source_id, None)
+            if watch:
+                self.observer.unschedule(watch)
+
+            logger.info(f"Stopped watching {source_id}")
+
+    def start(self):
+        """Start the file watcher."""
+        if not self._started:
+            self.observer.start()
+            self._started = True
+            logger.info("File watcher started")
+
+    def stop(self):
+        """Stop the file watcher."""
+        if self._started:
+            # Cancel all pending timers
+            for handler in self.handlers.values():
+                handler.cancel()
+
+            self.observer.stop()
+            self.observer.join(timeout=5.0)
+            self._started = False
+            logger.info("File watcher stopped")
+
+    @property
+    def watching(self) -> list[str]:
+        """Get list of source IDs being watched."""
+        with self._lock:
+            return list(self.handlers.keys())
+
+
+class NewFolderHandler(FileSystemEventHandler):
+    """Detects new folders in watched directories."""
+
+    def __init__(self, on_folder_created: Callable[[str, str], None]):
+        super().__init__()
+        self.on_folder_created = on_folder_created
+
+    def on_created(self, event: FileSystemEvent):
+        if event.is_directory:
+            folder_name = os.path.basename(event.src_path)
+            self.on_folder_created(folder_name, event.src_path)
+
+
+class DirectoryWatcher:
+    """
+    Watches common directories for new folder creation.
+
+    Used to instantly detect when user creates/clones a folder that
+    matches an indexed source name.
+    """
+
+    def __init__(self):
+        self.observer = Observer()
+        self._started = False
+
+    def watch(self, directories: list[str], on_folder_created: Callable[[str, str], None]):
+        """Watch directories for new folders (non-recursive, top-level only)."""
+        handler = NewFolderHandler(on_folder_created)
+
+        for dir_path in directories:
+            expanded = os.path.expanduser(dir_path)
+            if os.path.isdir(expanded):
+                try:
+                    self.observer.schedule(handler, expanded, recursive=False)
+                    logger.debug(f"Watching directory for new folders: {expanded}")
+                except Exception as e:
+                    logger.warning(f"Can't watch {expanded}: {e}")
+
+    def start(self):
+        if not self._started:
+            self.observer.start()
+            self._started = True
+
+    def stop(self):
+        if self._started:
+            self.observer.stop()
+            self.observer.join(timeout=5.0)
+            self._started = False