nia-sync 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

extractor.py

@@ -136,6 +136,11 @@ MAX_ROWS = 100_000
 MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024  # 10MB per file
 
 
+def _connect_sqlite_readonly(db_path: str) -> sqlite3.Connection:
+    """Open SQLite database in read-only mode to avoid lock issues."""
+    return sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=1)
+
+
 def detect_source_type(path: str) -> str:
     """
     Auto-detect the type of source based on path and file structure.
@@ -177,7 +182,7 @@ def detect_source_type(path: str) -> str:
         return TYPE_FOLDER
 
     try:
-        conn = sqlite3.connect(path)
+        conn = _connect_sqlite_readonly(path)
         cursor = conn.cursor()
         cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
         tables = {row[0].lower() for row in cursor.fetchall()}
@@ -252,7 +257,7 @@ def _extract_imessage(
     max_timestamp = cursor.get("last_timestamp", 0)
     since_rowid = cursor.get("last_rowid")
 
-    conn = sqlite3.connect(db_path)
+    conn = _connect_sqlite_readonly(db_path)
     conn.row_factory = sqlite3.Row
     cur = conn.cursor()
 
@@ -348,7 +353,7 @@ def _extract_safari_history(
     max_visit_time = cursor.get("last_visit_time", 0)
     since_visit_time = cursor.get("last_visit_time")
 
-    conn = sqlite3.connect(db_path)
+    conn = _connect_sqlite_readonly(db_path)
     conn.row_factory = sqlite3.Row
     cur = conn.cursor()
 
@@ -439,7 +444,7 @@ def _extract_chrome_history(
     max_visit_time = cursor.get("last_visit_time", 0)
     since_visit_time = cursor.get("last_visit_time")
 
-    conn = sqlite3.connect(db_path)
+    conn = _connect_sqlite_readonly(db_path)
     conn.row_factory = sqlite3.Row
     cur = conn.cursor()
 
@@ -534,7 +539,7 @@ def _extract_firefox_history(
     max_visit_date = cursor.get("last_visit_date", 0)
     since_visit_date = cursor.get("last_visit_date")
 
-    conn = sqlite3.connect(db_path)
+    conn = _connect_sqlite_readonly(db_path)
     conn.row_factory = sqlite3.Row
     cur = conn.cursor()
 
@@ -760,7 +765,9 @@ def _extract_folder(
     """Extract text files from a regular folder with proper exclusion patterns."""
     files = []
     last_mtime = cursor.get("last_mtime", 0)
+    last_path = cursor.get("last_path", "")
     max_mtime = last_mtime
+    max_path = last_path
     extracted_count = 0
 
     # Allowed text file extensions
@@ -783,6 +790,8 @@ def _extract_folder(
             and not d.startswith(".")
             and not d.endswith(".egg-info")
         ]
+        dirs.sort()
+        filenames.sort()
 
         for filename in filenames:
             if extracted_count >= limit:
@@ -816,9 +825,12 @@ def _extract_folder(
             try:
                 stat = os.stat(file_path)
                 mtime = stat.st_mtime
+                rel_path = os.path.relpath(file_path, folder_path)
 
-                # Skip if not modified since last sync
-                if mtime <= last_mtime:
+                # Skip if not modified since last sync (tie-break by path)
+                if mtime < last_mtime:
+                    continue
+                if mtime == last_mtime and rel_path <= last_path:
                     continue
 
                 # Skip large files
@@ -831,11 +843,6 @@ def _extract_folder(
                 if not content.strip():
                     continue
 
-                max_mtime = max(max_mtime, mtime)
-
-                # Relative path from folder root
-                rel_path = os.path.relpath(file_path, folder_path)
-
                 files.append({
                     "path": rel_path,
                     "content": content,
@@ -846,6 +853,9 @@ def _extract_folder(
                     },
                 })
                 extracted_count += 1
+                if mtime > max_mtime or (mtime == max_mtime and rel_path > max_path):
+                    max_mtime = mtime
+                    max_path = rel_path
 
             except (PermissionError, IOError, OSError, UnicodeDecodeError) as e:
                 logger.warning(f"Could not read {file_path}: {e}")
@@ -855,7 +865,7 @@ def _extract_folder(
 
     return {
         "files": files,
-        "cursor": {"last_mtime": max_mtime},
+        "cursor": {"last_mtime": max_mtime, "last_path": max_path},
         "stats": {"extracted": len(files), "db_type": TYPE_FOLDER},
     }
 
@@ -871,7 +881,7 @@ def _extract_generic_db(
 
     skip_tables = {"sqlite_sequence", "sqlite_stat1", "sqlite_stat4"}
 
-    conn = sqlite3.connect(db_path)
+    conn = _connect_sqlite_readonly(db_path)
     cur = conn.cursor()
 
     cur.execute("SELECT name FROM sqlite_master WHERE type='table'")

main.py

@@ -13,14 +13,25 @@ Usage:
 """
 import os
 import typer
+import httpx
+import logging
 from rich.console import Console
 from rich.panel import Panel
 from rich.table import Table
 
 from auth import login as do_login, logout as do_logout, is_authenticated, get_api_key
-from config import get_sources, add_source, remove_source, enable_source_sync, NIA_SYNC_DIR, find_folder_path
+from config import get_sources, add_source, remove_source, enable_source_sync, NIA_SYNC_DIR, find_folder_path, API_BASE_URL, get_api_key
 from sync import sync_all_sources
-from extractor import detect_source_type
+from extractor import (
+    detect_source_type,
+    TYPE_FOLDER,
+    TYPE_TELEGRAM,
+    TYPE_GENERIC_DB,
+    TYPE_IMESSAGE,
+    TYPE_SAFARI_HISTORY,
+    TYPE_CHROME_HISTORY,
+    TYPE_FIREFOX_HISTORY,
+)
 
 app = typer.Typer(
     name="nia",
@@ -30,6 +41,7 @@ app = typer.Typer(
     epilog="[dim]Quick start: [cyan]nia login[/cyan] → [cyan]nia status[/cyan] → [cyan]nia[/cyan][/dim]",
 )
 console = Console()
+logger = logging.getLogger(__name__)
 
 
 @app.callback(invoke_without_command=True)
@@ -71,6 +83,14 @@ KNOWN_PATHS = {
     "firefox_history": "~/Library/Application Support/Firefox/Profiles/*/places.sqlite",
 }
 
+DB_SOURCE_TYPES = {
+    TYPE_IMESSAGE,
+    TYPE_SAFARI_HISTORY,
+    TYPE_CHROME_HISTORY,
+    TYPE_FIREFOX_HISTORY,
+    TYPE_GENERIC_DB,
+}
+
 
 def _check_local_sources():
     """Check for indexed sources that exist locally and can be synced."""
@@ -381,6 +401,18 @@ def _resolve_sources(sources: list[dict], log_discoveries: bool = False) -> list
     return resolved
 
 
+def _get_watched_files(source: dict) -> set[str] | None:
+    path = source.get("path")
+    detected_type = source.get("detected_type")
+    if not path or not detected_type:
+        return None
+    if detected_type in DB_SOURCE_TYPES:
+        expanded = os.path.abspath(os.path.expanduser(path))
+        watched = {expanded, f"{expanded}-wal", f"{expanded}-shm"}
+        return watched
+    return None
+
+
 @app.command(name="start", hidden=True)
 def daemon(
     watch: bool = typer.Option(True, "--watch/--poll", help="File watching (default) or polling"),
@@ -401,6 +433,9 @@ def daemon(
     pending_syncs: set[str] = set()  # source_ids pending sync
     sync_lock = threading.Lock()
     sources_by_id: dict[str, dict] = {}
+    last_sync_times: dict[str, float] = {}
+    last_heartbeat_time = 0.0
+    heartbeat_interval = 30
 
     def handle_signal(signum, frame):
         nonlocal running
@@ -442,6 +477,7 @@ def daemon(
                 if added > 0:
                     total_added += added
                     console.print(f"[green]✓ {src.get('display_name', 'Unknown')}[/green] - {added} items synced")
+                last_sync_times[source_id] = time.time()
             else:
                 error = result.get("error", "unknown error")
                 errors.append(f"{src.get('display_name', 'Unknown')}: {error}")
@@ -473,7 +509,8 @@ def daemon(
             # Add new watchers
             for source_id in new_source_ids - current_watching:
                 src = new_sources_by_id[source_id]
-                if watcher.watch(source_id, src["path"], on_source_changed):
+                watched_files = _get_watched_files(src)
+                if watcher.watch(source_id, src["path"], on_source_changed, watched_files=watched_files):
                     console.print(f"  [dim]+ Watching {src.get('display_name', 'Unknown')}[/dim]")
                     newly_added.append(source_id)
 
@@ -559,6 +596,19 @@ def daemon(
             # Process any pending syncs from file watcher
             sync_pending_sources()
 
+            # Heartbeat to backend to mark daemon online
+            now = time.time()
+            if now - last_heartbeat_time >= heartbeat_interval:
+                _send_heartbeat(list(sources_by_id.keys()))
+                last_heartbeat_time = now
+
+            # Sanity sync to catch missed events
+            if fallback_interval > 0:
+                for source_id in list(sources_by_id.keys()):
+                    last_sync = last_sync_times.get(source_id, 0)
+                    if now - last_sync >= fallback_interval:
+                        pending_syncs.add(source_id)
+
             # Instant refresh if new folder detected matching an unlinked source
             if refresh_triggered.is_set():
                 refresh_triggered.clear()
@@ -604,6 +654,7 @@ def daemon(
         sync_count = 0
         while running:
             resolved, _ = refresh_sources()
+            _send_heartbeat([src["local_folder_id"] for src in resolved])
 
             sync_count += 1
             console.print(f"\n[bold]Sync #{sync_count}[/bold] - {len(resolved)} source(s)")
@@ -633,5 +684,22 @@ def daemon(
     console.print("[green]✓ Stopped[/green]")
 
 
+def _send_heartbeat(source_ids: list[str]) -> None:
+    if not source_ids:
+        return
+    api_key = get_api_key()
+    if not api_key:
+        return
+    try:
+        with httpx.Client(timeout=10) as client:
+            client.post(
+                f"{API_BASE_URL}/v2/daemon/heartbeat",
+                headers={"Authorization": f"Bearer {api_key}"},
+                json={"source_ids": source_ids},
+            )
+    except Exception:
+        logger.debug("Heartbeat failed", exc_info=True)
+
+
 if __name__ == "__main__":
     app()

{nia_sync-0.1.1.dist-info → nia_sync-0.1.3.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: nia-sync
-Version: 0.1.1
-Summary: Keep your local files in sync with Nia
+Version: 0.1.3
+Summary: Keep your local files in sync with Nia Cloud
 Requires-Python: >=3.10
 Requires-Dist: typer>=0.9.0
 Requires-Dist: rich>=13.0.0

nia_sync-0.1.3.dist-info/RECORD

@@ -0,0 +1,11 @@
+auth.py,sha256=n0ezRqIbz3kZYcyIjH47_MDKgwNgAaVr0Y2NEfRxa50,5704
+config.py,sha256=JWxdL8INKo23lam4F49ZbllxbW3yorqczm7aaqepQCc,7507
+extractor.py,sha256=Y5Gc0AThTH9qMey4fYEqv4RosAqZl7GleKNMYs5ogxY,29181
+main.py,sha256=NtAkI083bZIXZp9WVGaCunveV8UrR0ZBctBkBhrVf24,25168
+sync.py,sha256=OrbjptK223zbzeelSTdZIF3O4gNQd5DLK8XXv-rnMXU,9583
+watcher.py,sha256=JmsN9uR7Ss1mDC-kApXL6Hg_wQZWTsO7rRIFkQu8GbM,9978
+nia_sync-0.1.3.dist-info/METADATA,sha256=_lnywxKDjuheFtFUGC008XUMS-5yD4bXsWVHkq-i9mw,246
+nia_sync-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nia_sync-0.1.3.dist-info/entry_points.txt,sha256=Fx8TIOgXqWdZzZEkEateDtcNfgnwuPW4jZTqlEUrHVs,33
+nia_sync-0.1.3.dist-info/top_level.txt,sha256=_ZWBugSHWwSpLXYJAcF6TlWmzECu18k0y1-EX27jtBw,40
+nia_sync-0.1.3.dist-info/RECORD,,

sync.py

@@ -8,6 +8,8 @@ Handles:
 """
 import os
 import logging
+import random
+import time
 from pathlib import Path
 from typing import Any
 import httpx
@@ -17,7 +19,25 @@ from extractor import extract_incremental, detect_source_type
 
 logger = logging.getLogger(__name__)
 
-SYNC_TIMEOUT = 120  # 2 minutes per sync request
+SYNC_TIMEOUT = 60  # 1 minute per sync request (reduced from 2 min)
+CONNECT_TIMEOUT = 10  # 10 second connection timeout
+MAX_FILES_PER_BATCH = 500  # Keep below backend limit (1000)
+MAX_RETRIES = 4
+RETRY_BASE_DELAY = 1.5
+RETRY_MAX_DELAY = 15.0
+
+# Reusable client for connection pooling
+_http_client: httpx.Client | None = None
+
+def get_http_client() -> httpx.Client:
+    """Get or create HTTP client with connection pooling."""
+    global _http_client
+    if _http_client is None:
+        _http_client = httpx.Client(
+            timeout=httpx.Timeout(SYNC_TIMEOUT, connect=CONNECT_TIMEOUT),
+            limits=httpx.Limits(max_keepalive_connections=5, max_connections=10),
+        )
+    return _http_client
 
 
 def sync_all_sources(sources: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -63,10 +83,12 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
 
     # Validate path exists
     if not os.path.exists(path):
+        error_message = f"Path does not exist: {path}"
+        report_sync_error(local_folder_id, error_message, path)
         return {
             "path": path,
             "status": "error",
-            "error": f"Path does not exist: {path}",
+            "error": error_message,
         }
 
     # Auto-detect type if not specified
@@ -96,8 +118,8 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
                 "message": "No new data",
             }
 
-        # Upload to backend
-        upload_result = upload_sync_data(
+        # Upload to backend in batches
+        upload_result = upload_sync_batches(
             local_folder_id=local_folder_id,
             files=files,
             cursor=new_cursor,
@@ -115,6 +137,7 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
                 "new_cursor": new_cursor,
             }
         else:
+            report_sync_error(local_folder_id, upload_result.get("message", "Upload failed"), path)
             return {
                 "path": path,
                 "status": "error",
@@ -122,13 +145,16 @@ def sync_source(source: dict[str, Any]) -> dict[str, Any]:
             }
 
     except PermissionError:
+        error_message = "Permission denied. Grant Full Disk Access in System Settings > Privacy & Security."
+        report_sync_error(local_folder_id, error_message, path)
         return {
             "path": path,
             "status": "error",
-            "error": "Permission denied. Grant Full Disk Access in System Settings > Privacy & Security.",
+            "error": error_message,
         }
     except Exception as e:
         logger.error(f"Error syncing {path}: {e}", exc_info=True)
+        report_sync_error(local_folder_id, str(e), path)
         return {
             "path": path,
             "status": "error",
@@ -141,6 +167,7 @@ def upload_sync_data(
     files: list[dict[str, Any]],
     cursor: dict[str, Any],
     stats: dict[str, Any],
+    is_final_batch: bool = True,
 ) -> dict[str, Any]:
     """
     Upload extracted data to the cloud API.
@@ -159,29 +186,120 @@ def upload_sync_data(
         return {"status": "error", "message": "Not authenticated"}
 
     try:
-        with httpx.Client(timeout=SYNC_TIMEOUT) as client:
-            response = client.post(
-                f"{API_BASE_URL}/v2/daemon/sync",
-                headers={"Authorization": f"Bearer {api_key}"},
-                json={
-                    "local_folder_id": local_folder_id,
-                    "files": files,
-                    "cursor": cursor,
-                    "stats": stats,
-                },
-            )
-
-            if response.status_code == 200:
-                return response.json()
-            elif response.status_code == 401:
-                return {"status": "error", "message": "Authentication failed"}
-            elif response.status_code == 404:
-                return {"status": "error", "message": "Local folder not found"}
-            else:
+        client = get_http_client()
+        response = _post_with_retries(
+            client=client,
+            url=f"{API_BASE_URL}/v2/daemon/sync",
+            headers={"Authorization": f"Bearer {api_key}"},
+            payload={
+                "local_folder_id": local_folder_id,
+                "files": files,
+                "cursor": cursor,
+                "stats": stats,
+                "is_final_batch": is_final_batch,
+            },
+        )
+
+        if response is None:
+            return {"status": "error", "message": "Request failed after retries"}
+
+        if response.status_code == 200:
+            return response.json()
+        elif response.status_code == 401:
+            return {"status": "error", "message": "Authentication failed"}
+        elif response.status_code == 404:
+            return {"status": "error", "message": "Local folder not found"}
+        else:
+            try:
                 detail = response.json().get("detail", response.text)
-                return {"status": "error", "message": f"API error: {detail}"}
+            except ValueError:
+                detail = response.text or f"HTTP {response.status_code}"
+            return {"status": "error", "message": f"API error: {detail}"}
 
     except httpx.TimeoutException:
         return {"status": "error", "message": "Request timeout"}
     except httpx.RequestError as e:
         return {"status": "error", "message": f"Network error: {e}"}
+
+
+def upload_sync_batches(
+    local_folder_id: str,
+    files: list[dict[str, Any]],
+    cursor: dict[str, Any],
+    stats: dict[str, Any],
+) -> dict[str, Any]:
+    """Upload files in batches and only advance cursor after all succeed."""
+    if not files:
+        return {"status": "ok", "chunks_indexed": 0}
+
+    total_batches = max(1, (len(files) + MAX_FILES_PER_BATCH - 1) // MAX_FILES_PER_BATCH)
+    chunks_indexed = 0
+
+    for batch_index, batch in enumerate(_iter_batches(files, MAX_FILES_PER_BATCH), start=1):
+        is_last_batch = batch_index == total_batches
+        result = upload_sync_data(
+            local_folder_id=local_folder_id,
+            files=batch,
+            cursor=cursor if is_last_batch else {},
+            stats=stats if is_last_batch else {},
+            is_final_batch=is_last_batch,
+        )
+
+        if result.get("status") != "ok":
+            return result
+
+        chunks_indexed += result.get("chunks_indexed", 0)
+
+    return {"status": "ok", "chunks_indexed": chunks_indexed}
+
+
+def report_sync_error(local_folder_id: str | None, error: str, path: str | None = None) -> None:
+    """Report local sync errors to backend for UI visibility."""
+    if not local_folder_id:
+        return
+    api_key = get_api_key()
+    if not api_key:
+        return
+
+    try:
+        client = get_http_client()
+        _post_with_retries(
+            client=client,
+            url=f"{API_BASE_URL}/v2/daemon/sources/{local_folder_id}/error",
+            headers={"Authorization": f"Bearer {api_key}"},
+            payload={"error": error, "path": path},
+        )
+    except Exception:
+        logger.debug("Failed to report sync error", exc_info=True)
+
+
+def _iter_batches(items: list[dict[str, Any]], size: int):
+    for i in range(0, len(items), size):
+        yield items[i:i + size]
+
+
+def _post_with_retries(
+    client: httpx.Client,
+    url: str,
+    headers: dict[str, str],
+    payload: dict[str, Any],
+) -> httpx.Response | None:
+    delay = RETRY_BASE_DELAY
+    for attempt in range(MAX_RETRIES):
+        try:
+            response = client.post(url, headers=headers, json=payload)
+            if response.status_code in {429} or response.status_code >= 500:
+                raise httpx.HTTPStatusError(
+                    f"Retryable status {response.status_code}",
+                    request=response.request,
+                    response=response,
+                )
+            return response
+        except (httpx.TimeoutException, httpx.RequestError, httpx.HTTPStatusError) as e:
+            is_last_attempt = attempt >= MAX_RETRIES - 1
+            if is_last_attempt:
+                logger.warning(f"POST failed after retries: {e}")
+                return None
+            jitter = random.uniform(0.8, 1.2)
+            time.sleep(min(RETRY_MAX_DELAY, delay) * jitter)
+            delay *= 2

watcher.py

@@ -41,6 +41,7 @@ class SyncEventHandler(FileSystemEventHandler):
         source_path: str,
         on_change: Callable[[str], None],
         debounce_sec: float = 2.0,
+        watched_files: set[str] | None = None,
     ):
         super().__init__()
         self.source_id = source_id
@@ -51,8 +52,13 @@ class SyncEventHandler(FileSystemEventHandler):
         self._lock = threading.Lock()
         self._pending_changes = 0
 
+        # Watch specific files if provided (e.g., DB files without extensions)
+        if watched_files:
+            self._watched_files = {
+                os.path.abspath(os.path.expanduser(p)) for p in watched_files
+            }
         # For database files, also watch the WAL/SHM files
-        if self.source_path.endswith('.db'):
+        elif self.source_path.endswith('.db'):
             self._watched_files = {
                 self.source_path,
                 self.source_path + '-wal',
@@ -157,6 +163,7 @@ class FileWatcher:
         source_id: str,
         path: str,
         on_change: Callable[[str], None],
+        watched_files: set[str] | None = None,
     ) -> bool:
         """
         Add a path to watch.
@@ -178,9 +185,8 @@ class FileWatcher:
             # Expand path
             expanded = os.path.expanduser(path)
 
-            # For database files, watch the parent directory
-            # to catch .db-wal changes
-            if expanded.endswith('.db'):
+            # For database files (or explicit watched files), watch the parent directory
+            if watched_files or expanded.endswith('.db'):
                 watch_path = os.path.dirname(expanded)
             else:
                 watch_path = expanded
@@ -196,6 +202,7 @@ class FileWatcher:
                 source_path=expanded,
                 on_change=on_change,
                 debounce_sec=self.debounce_sec,
+                watched_files=watched_files,
             )
 
             # Schedule watch

nia_sync-0.1.1.dist-info/RECORD

@@ -1,11 +0,0 @@
-auth.py,sha256=n0ezRqIbz3kZYcyIjH47_MDKgwNgAaVr0Y2NEfRxa50,5704
-config.py,sha256=JWxdL8INKo23lam4F49ZbllxbW3yorqczm7aaqepQCc,7507
-extractor.py,sha256=ViqOZQBtwrKUK_W0bOFTNU8Ar726mcZ6TJ6dS_PfVyk,28599
-main.py,sha256=k_4KXmQb8sRI-Wf8r2MMKUZbIa1SLuGUVtpU30ej63E,22975
-sync.py,sha256=W31mWjvo2qEszUaH_C9m7bQu0FMUsBFUaiw8QdcYDxs,5427
-watcher.py,sha256=9RvHVpn1ozTBgxPeOe20kkW6x5gb8A1b7e33D_7JBc4,9598
-nia_sync-0.1.1.dist-info/METADATA,sha256=3hhEQsrIu8LLcOm3uvUo_lyGSzQH5GmlSly8BXXRQ_o,240
-nia_sync-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nia_sync-0.1.1.dist-info/entry_points.txt,sha256=Fx8TIOgXqWdZzZEkEateDtcNfgnwuPW4jZTqlEUrHVs,33
-nia_sync-0.1.1.dist-info/top_level.txt,sha256=_ZWBugSHWwSpLXYJAcF6TlWmzECu18k0y1-EX27jtBw,40
-nia_sync-0.1.1.dist-info/RECORD,,