nogic 0.0.1__py3-none-any.whl

nogic/watcher/sync.py

@@ -0,0 +1,879 @@
+"""Sync service for file changes - sends to backend API."""
+
+import asyncio
+import hashlib
+import json as _json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+from typing import Callable, Optional
+
+import httpx
+from rich.progress import (
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    BarColumn,
+    TaskProgressColumn,
+    TimeElapsedColumn,
+    MofNCompleteColumn,
+)
+from rich.console import Console
+
+from nogic.config import Config, get_language
+from nogic.api import NogicClient, IndexProgress
+
+logger = logging.getLogger(__name__)
+
+def _int_env(name: str, default: int) -> int:
+    """Read an integer from an environment variable, falling back to default on bad values."""
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        return default
+
+# Batch size for uploading files (configurable via NOGIC_BATCH_SIZE)
+BATCH_SIZE = _int_env("NOGIC_BATCH_SIZE", 150)
+
+# Max payload size per batch in bytes (~10MB default, well under GCP LB 32MB limit)
+MAX_BATCH_BYTES = _int_env("NOGIC_MAX_BATCH_BYTES", 10 * 1024 * 1024)
+
+# Hard cap per file in bytes (~30MB) — must fit in a single request under GCP LB 32MB limit
+MAX_SINGLE_FILE_BYTES = _int_env("NOGIC_MAX_FILE_SIZE", 30 * 1024 * 1024)
+
+# Max concurrent uploads (configurable via NOGIC_MAX_CONCURRENT)
+MAX_CONCURRENT = _int_env("NOGIC_MAX_CONCURRENT", 8)
+
+STAGE_LABELS = {
+    "hashing": "Computing file hashes",
+    "parsing": "Parsing files",
+    "imports": "Resolving imports",
+    "nodes": "Creating symbol nodes",
+    "call_graph": "Building call graph",
+    "edges": "Linking relationships",
+    "embedding": "Generating embeddings",
+    "saving": "Saving to database",
+    "complete": "Complete",
+}
+
+# Check if legacy mode is forced via environment variable
+LEGACY_UPLOAD = os.environ.get("NOGIC_LEGACY_UPLOAD", "").lower() in ("1", "true", "yes")
+
+
+class SyncService:
+    """Sync service that indexes files via the backend API.
+
+    Uses backend as source of truth - no local sync state tracking.
+    """
+
+    def __init__(
+        self,
+        config: Config,
+        root_path: Path,
+        log: Optional[Callable[[str], None]] = None,
+        json_mode: bool = False,
+    ):
+        self.config = config
+        self.root_path = root_path
+        self.log = log or print
+        self.json_mode = json_mode
+        self._client: Optional[NogicClient] = None
+        # In-memory cache of file info for current session
+        self._file_cache: dict[str, dict] = {}  # path -> {hash, content, language}
+
+    def _emit_json(self, event: str, **kwargs):
+        """Emit a single NDJSON progress line to stdout (json_mode only)."""
+        payload = {"event": event, "timestamp": int(time.time()), **kwargs}
+        sys.stdout.write(_json.dumps(payload) + "\n")
+        sys.stdout.flush()
+
+    @property
+    def client(self) -> NogicClient:
+        if self._client is None:
+            self._client = NogicClient(self.config)
+        return self._client
+
+    def close(self):
+        if self._client:
+            self._client.close()
+            self._client = None
+
+    @staticmethod
+    def compute_hash(content: str) -> str:
+        """Compute SHA256 hash of content."""
+        return hashlib.sha256(content.encode()).hexdigest()
+
+    @staticmethod
+    def _split_into_batches(files: list[dict]) -> list[list[dict]]:
+        """Split files into batches respecting both count and payload size limits.
+
+        Files larger than MAX_BATCH_BYTES get their own solo batch so they
+        don't inflate a shared batch beyond the limit.
+        """
+        batches = []
+        current_batch = []
+        current_size = 0
+
+        for f in files:
+            file_size = len(f.get("content", "").encode("utf-8"))
+
+            # If this single file exceeds the batch byte limit, send it solo
+            if file_size > MAX_BATCH_BYTES:
+                if current_batch:
+                    batches.append(current_batch)
+                    current_batch = []
+                    current_size = 0
+                batches.append([f])
+                continue
+
+            # Start new batch if adding this file exceeds limits
+            if current_batch and (
+                len(current_batch) >= BATCH_SIZE
+                or current_size + file_size > MAX_BATCH_BYTES
+            ):
+                batches.append(current_batch)
+                current_batch = []
+                current_size = 0
+
+            current_batch.append(f)
+            current_size += file_size
+
+        if current_batch:
+            batches.append(current_batch)
+
+        return batches
+
+    def scan_files(
+        self,
+        root_path: Path,
+        ignore_check: Callable[[Path], bool]
+    ) -> list[dict]:
+        """
+        Scan directory and collect file info.
+
+        Returns list of {path, hash, content, language} for supported files.
+        """
+        # Collect files using os.walk for directory pruning (skips ignored dirs entirely)
+        if not self.json_mode:
+            console = Console()
+            ctx = console.status("[bold blue]Finding files...", spinner="dots")
+            ctx.__enter__()
+
+        all_files = []
+        for dirpath, dirnames, filenames in os.walk(root_path):
+            dp = Path(dirpath)
+            dirnames[:] = [
+                d for d in dirnames
+                if not ignore_check(dp / d)
+            ]
+            for fname in filenames:
+                fpath = dp / fname
+                if not ignore_check(fpath):
+                    all_files.append(fpath)
+
+        if not self.json_mode:
+            ctx.__exit__(None, None, None)
+
+        files_info = []
+        supported_count = 0
+        skipped_large = 0
+        total_files = len(all_files)
+
+        if self.json_mode:
+            # Emit progress via NDJSON
+            for i, path in enumerate(all_files):
+                if i % 50 == 0 or i == total_files - 1:
+                    self._emit_json("progress", phase="scanning", current=i + 1, total=total_files)
+
+                language = get_language(path)
+                if not language:
+                    continue
+
+                try:
+                    file_size = path.stat().st_size
+                except OSError:
+                    continue
+
+                if file_size > MAX_SINGLE_FILE_BYTES:
+                    skipped_large += 1
+                    continue
+
+                try:
+                    content = path.read_text(encoding="utf-8")
+                except (OSError, UnicodeDecodeError):
+                    continue
+
+                rel_path = str(path.relative_to(root_path))
+                content_hash = self.compute_hash(content)
+
+                files_info.append({
+                    "path": rel_path,
+                    "hash": content_hash,
+                    "content": content,
+                    "language": language,
+                })
+                supported_count += 1
+        else:
+            # Rich progress bar
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                transient=False,
+            ) as progress:
+                task = progress.add_task("Scanning files", total=total_files)
+
+                for path in all_files:
+                    progress.update(task, advance=1)
+
+                    language = get_language(path)
+                    if not language:
+                        continue
+
+                    try:
+                        file_size = path.stat().st_size
+                    except OSError:
+                        continue
+
+                    if file_size > MAX_SINGLE_FILE_BYTES:
+                        skipped_large += 1
+                        continue
+
+                    try:
+                        content = path.read_text(encoding="utf-8")
+                    except (OSError, UnicodeDecodeError):
+                        continue
+
+                    rel_path = str(path.relative_to(root_path))
+                    content_hash = self.compute_hash(content)
+
+                    files_info.append({
+                        "path": rel_path,
+                        "hash": content_hash,
+                        "content": content,
+                        "language": language,
+                    })
+                    supported_count += 1
+
+        msg = f"Found {supported_count} supported files ({len(all_files)} total)"
+        if skipped_large > 0:
+            msg += f", {skipped_large} skipped (>{MAX_SINGLE_FILE_BYTES // (1024 * 1024)}MB)"
+        self.log(msg)
+        return files_info
+
+    def sync_files(self, files_info: list[dict]) -> bool:
+        """
+        Sync files to backend.
+
+        Uses two-phase parallel indexing if server supports it:
+        1. Upload all batches in parallel to /v1/index/upload
+        2. Call /v1/index/finalize/stream once for global resolution
+
+        Falls back to legacy sequential flow if server doesn't support it.
+        """
+        if not files_info:
+            self.log("No supported files to sync")
+            return True
+
+        if not self.config.project_id:
+            self.log("Error: No project configured.")
+            return False
+
+        if not self.config.api_key:
+            self.log("Error: No API key configured.")
+            return False
+
+        # Check if we should use legacy mode
+        if LEGACY_UPLOAD:
+            self.log("Using legacy upload mode (NOGIC_LEGACY_UPLOAD=1)")
+            return self._sync_files_legacy(files_info)
+
+        # Try two-phase parallel flow
+        try:
+            # Check if server supports new endpoints
+            stats = self.client.get_staging_stats(self.config.project_id)
+            if stats is None:
+                # Server returned 404 - doesn't support two-phase indexing
+                self.log("Server doesn't support two-phase indexing, using legacy mode")
+                return self._sync_files_legacy(files_info)
+
+            # Use new parallel flow
+            return self._sync_files_parallel(files_info)
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 404:
+                self.log("Server doesn't support two-phase indexing, using legacy mode")
+                return self._sync_files_legacy(files_info)
+            raise
+        except Exception as e:
+            logger.warning(f"Two-phase check failed ({e}), falling back to legacy mode")
+            return self._sync_files_legacy(files_info)
+
+    def _sync_files_legacy(self, files_info: list[dict]) -> bool:
+        """Legacy sequential upload (old flow)."""
+        return self._upload_files_batched(files_info)
+
+    def _sync_files_parallel(self, files_info: list[dict]) -> bool:
+        """
+        Two-phase parallel upload flow.
+
+        Phase 1: Upload all batches in parallel
+        Phase 2: Finalize with streaming progress
+        """
+        # Prepare files for upload
+        upload_files = [
+            {
+                "path": f["path"],
+                "content": f["content"],
+                "language": f["language"],
+                "hash": f["hash"],
+            }
+            for f in files_info
+        ]
+
+        # Split into batches respecting both count and size limits
+        batches = self._split_into_batches(upload_files)
+
+        total_files = len(upload_files)
+        num_batches = len(batches)
+
+        # Clear any stale staging data from previous sessions
+        try:
+            self.client.clear_staging(self.config.project_id)
+        except Exception as e:
+            logger.debug(f"Could not clear staging (may not exist): {e}")
+
+        # Phase 1: Parallel upload
+        self.log(f"Phase 1: Uploading {total_files} files in {num_batches} batches...")
+
+        if self.json_mode:
+            self._emit_json("progress", phase="uploading", current=0, total=num_batches)
+
+            try:
+                result = asyncio.run(
+                    self._upload_batches_parallel_json(batches, num_batches)
+                )
+            except Exception as e:
+                self._emit_json("error", message=f"Parallel upload failed: {e}")
+                try:
+                    self.client.clear_staging(self.config.project_id)
+                except Exception:
+                    pass
+                self.log("Falling back to legacy mode...")
+                return self._sync_files_legacy(files_info)
+        else:
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                transient=False,
+            ) as progress:
+                upload_task = progress.add_task(
+                    "Uploading batches",
+                    total=num_batches
+                )
+
+                try:
+                    result = asyncio.run(
+                        self._upload_batches_parallel(batches, progress, upload_task)
+                    )
+                except Exception as e:
+                    self.log(f"Parallel upload failed: {e}")
+                    try:
+                        self.client.clear_staging(self.config.project_id)
+                    except Exception:
+                        pass
+                    self.log("Falling back to legacy mode...")
+                    return self._sync_files_legacy(files_info)
+
+        total_uploaded = result['total_parsed']
+        total_staged = result['total_staged']
+        total_skipped = result.get('total_skipped', 0)
+
+        if total_skipped == 0 and total_staged < total_uploaded:
+            total_skipped = total_uploaded - total_staged
+
+        self.log(f"Uploaded {total_uploaded} files ({total_staged} staged, {total_skipped} unchanged)")
+
+        if total_staged == 0:
+            self.log("All files unchanged, nothing to index")
+            if self.json_mode:
+                self._emit_json("complete", files_indexed=0, nodes_created=0, edges_created=0)
+            return True
+
+        # Phase 2: Finalize with streaming progress
+        self.log("Phase 2: Processing...")
+
+        total_indexed = 0
+        total_skipped = 0
+        total_nodes = 0
+        total_edges = 0
+        all_errors = []
+
+        if self.json_mode:
+            try:
+                for event in self.client.finalize_stream(self.config.project_id):
+                    if event.stage == "complete":
+                        total_indexed = event.files_indexed or 0
+                        total_skipped = event.files_skipped or 0
+                        total_nodes = event.nodes_created or 0
+                        total_edges = event.edges_created or 0
+                        if event.errors:
+                            all_errors.extend(event.errors)
+                        self._emit_json(
+                            "complete",
+                            files_indexed=total_indexed,
+                            files_skipped=total_skipped,
+                            nodes_created=total_nodes,
+                            edges_created=total_edges,
+                            errors=len(all_errors),
+                        )
+                        break
+
+                    if event.stage == "error":
+                        self._emit_json("error", message=event.message or "Processing error")
+                        break
+
+                    stage = event.stage
+                    current = event.current or 0
+                    total = event.total or 0
+                    label = STAGE_LABELS.get(stage, stage)
+                    self._emit_json("progress", phase="processing", stage=stage, label=label, current=current, total=total)
+
+            except httpx.HTTPStatusError as e:
+                self._emit_json("error", message=f"Finalize error: {e.response.status_code}")
+                raise
+            except httpx.RequestError as e:
+                self._emit_json("error", message=f"Request error during finalize: {e}")
+                raise
+        else:
+            # Rich progress for terminal
+            stage_tasks: dict = {}
+
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description:<12}"),
+                BarColumn(),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                transient=False,
+            ) as progress:
+                try:
+                    for event in self.client.finalize_stream(self.config.project_id):
+                        if event.stage == "complete":
+                            total_indexed = event.files_indexed or 0
+                            total_skipped = event.files_skipped or 0
+                            total_nodes = event.nodes_created or 0
+                            total_edges = event.edges_created or 0
+                            if event.errors:
+                                all_errors.extend(event.errors)
+                            for task_id in stage_tasks.values():
+                                progress.update(task_id, completed=progress.tasks[task_id].total)
+                            break
+
+                        if event.stage == "error":
+                            all_errors.append({"path": "finalize", "error": event.message})
+                            break
+
+                        stage = event.stage
+                        current = event.current or 0
+                        total = event.total or 0
+
+                        if stage not in stage_tasks:
+                            task_total = total if total > 0 else 100
+                            label = STAGE_LABELS.get(stage, stage)
+                            stage_tasks[stage] = progress.add_task(
+                                label,
+                                total=task_total,
+                            )
+
+                        task_id = stage_tasks[stage]
+
+                        if total > 0 and progress.tasks[task_id].total != total:
+                            progress.update(task_id, total=total)
+
+                        progress.update(task_id, completed=current)
+
+                except httpx.HTTPStatusError as e:
+                    self.log(f"Finalize error: {e.response.status_code}")
+                    raise
+                except httpx.RequestError as e:
+                    self.log(f"Request error during finalize: {e}")
+                    raise
+
+        # Summary
+        if total_skipped > 0:
+            self.log(f"Indexed {total_indexed} files ({total_skipped} unchanged), {total_nodes} nodes, {total_edges} edges")
+        else:
+            self.log(f"Indexed {total_indexed} files, {total_nodes} nodes, {total_edges} edges")
+
+        if all_errors:
+            self.log(f"Errors ({len(all_errors)}):")
+            for err in all_errors[:10]:
+                self.log(f"  {err.get('path', 'unknown')}: {err.get('error', 'unknown error')}")
+            if len(all_errors) > 10:
+                self.log(f"  ... and {len(all_errors) - 10} more errors")
+
+        return True
+
+    async def _upload_batches_parallel_json(
+        self,
+        batches: list[list[dict]],
+        num_batches: int,
+        max_concurrent: int = MAX_CONCURRENT,
+    ) -> dict:
+        """Upload all batches in parallel, emitting NDJSON progress."""
+        semaphore = asyncio.Semaphore(max_concurrent)
+        results = {"total_parsed": 0, "total_staged": 0, "total_skipped": 0, "errors": []}
+        completed = 0
+        lock = asyncio.Lock()
+
+        async with httpx.AsyncClient(
+            timeout=60.0,
+            limits=httpx.Limits(max_connections=max_concurrent + 2, max_keepalive_connections=max_concurrent),
+        ) as client:
+            async def upload_one(batch: list[dict], batch_num: int) -> dict:
+                nonlocal completed
+                async with semaphore:
+                    resp = await client.post(
+                        f"{self.config.api_url}/v1/index/upload",
+                        json={
+                            "project_id": self.config.project_id,
+                            "files": batch,
+                        },
+                        headers={
+                            "Authorization": f"Bearer {self.config.api_key}",
+                            "Content-Type": "application/json",
+                        },
+                    )
+                    resp.raise_for_status()
+                    data = resp.json()
+
+                    async with lock:
+                        completed += 1
+                        self._emit_json("progress", phase="uploading", current=completed, total=num_batches)
+
+                    return data
+
+            tasks = [upload_one(batch, i) for i, batch in enumerate(batches)]
+            responses = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for resp in responses:
+            if isinstance(resp, Exception):
+                results["errors"].append(str(resp))
+                logger.error(f"Batch upload failed: {resp}")
+            elif isinstance(resp, dict):
+                results["total_parsed"] += resp.get("files_parsed", 0)
+                results["total_staged"] = max(results["total_staged"], resp.get("total_staged", 0))
+                results["total_skipped"] += resp.get("files_skipped", 0)
+
+        if results["errors"]:
+            raise Exception(f"{len(results['errors'])} batches failed: {results['errors'][0]}")
+
+        return results
+
+    async def _upload_batches_parallel(
+        self,
+        batches: list[list[dict]],
+        progress: Progress,
+        task_id,
+        max_concurrent: int = MAX_CONCURRENT,
+    ) -> dict:
+        """Upload all batches in parallel using asyncio."""
+        semaphore = asyncio.Semaphore(max_concurrent)
+        results = {"total_parsed": 0, "total_staged": 0, "total_skipped": 0, "errors": []}
+        completed = 0
+        lock = asyncio.Lock()
+
+        # Create ONE shared client with connection pooling for all requests
+        # This enables HTTP/2 multiplexing and connection reuse
+        async with httpx.AsyncClient(
+            timeout=60.0,
+            limits=httpx.Limits(max_connections=max_concurrent + 2, max_keepalive_connections=max_concurrent),
+        ) as client:
+
+            async def upload_one(batch: list[dict], batch_num: int) -> dict:
+                nonlocal completed
+                async with semaphore:
+                    resp = await client.post(
+                        f"{self.config.api_url}/v1/index/upload",
+                        json={
+                            "project_id": self.config.project_id,
+                            "files": batch,
+                        },
+                        headers={
+                            "Authorization": f"Bearer {self.config.api_key}",
+                            "Content-Type": "application/json",
+                        },
+                    )
+                    resp.raise_for_status()
+                    data = resp.json()
+
+                    # Update progress (use lock to ensure thread-safe counter)
+                    async with lock:
+                        completed += 1
+                        progress.update(task_id, completed=completed)
+
+                    return data
+
+            # Create tasks for all batches
+            tasks = [
+                upload_one(batch, i)
+                for i, batch in enumerate(batches)
+            ]
+
+            # Run all tasks in parallel and collect results
+            responses = await asyncio.gather(*tasks, return_exceptions=True)
+
+        for resp in responses:
+            if isinstance(resp, Exception):
+                results["errors"].append(str(resp))
+                logger.error(f"Batch upload failed: {resp}")
+            elif isinstance(resp, dict):
+                results["total_parsed"] += resp.get("files_parsed", 0)
+                results["total_staged"] = max(results["total_staged"], resp.get("total_staged", 0))  # Use max (cumulative, order not guaranteed)
+                results["total_skipped"] += resp.get("files_skipped", 0)  # Sum skipped across batches
+
+        if results["errors"]:
+            # If any batch failed, raise an exception
+            raise Exception(f"{len(results['errors'])} batches failed: {results['errors'][0]}")
+
+        return results
+
+    def _upload_files_batched(self, files: list[dict]) -> bool:
+        """Upload files in batches with progress."""
+        if not files:
+            if self.json_mode:
+                self._emit_json("complete", files_indexed=0, nodes_created=0, edges_created=0)
+            return True
+
+        upload_files = [
+            {
+                "path": f["path"],
+                "content": f["content"],
+                "language": f["language"],
+                "hash": f["hash"],
+            }
+            for f in files
+        ]
+
+        total_files = len(upload_files)
+        batches = self._split_into_batches(upload_files)
+        num_batches = len(batches)
+
+        total_indexed = 0
+        total_skipped = 0
+        total_nodes = 0
+        total_edges = 0
+        all_errors = []
+        files_done = 0
+
+        if self.json_mode:
+            for batch_num, batch in enumerate(batches):
+                self._emit_json("progress", phase="uploading", current=batch_num, total=num_batches,
+                                description=f"Uploading batch {batch_num + 1}/{num_batches}")
+                try:
+                    result = self._send_batch_stream(batch)
+                    if result:
+                        total_indexed += result.files_indexed or 0
+                        total_skipped += result.files_skipped or 0
+                        total_nodes += result.nodes_created or 0
+                        total_edges += result.edges_created or 0
+                        if result.errors:
+                            all_errors.extend(result.errors)
+                except Exception as e:
+                    self._emit_json("error", message=f"Batch {batch_num + 1} failed: {e}")
+
+                files_done += len(batch)
+
+            self._emit_json(
+                "complete",
+                files_indexed=total_indexed,
+                files_skipped=total_skipped,
+                nodes_created=total_nodes,
+                edges_created=total_edges,
+                errors=len(all_errors),
+            )
+        else:
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                BarColumn(),
+                MofNCompleteColumn(),
+                TimeElapsedColumn(),
+                transient=False,
+            ) as progress:
+                upload_task = progress.add_task(
+                    "Uploading files",
+                    total=total_files
+                )
+
+                for batch_num, batch in enumerate(batches):
+                    progress.update(
+                        upload_task,
+                        description=f"Uploading batch {batch_num + 1}/{num_batches}"
+                    )
+
+                    try:
+                        result = self._send_batch_stream(batch, progress)
+                        if result:
+                            total_indexed += result.files_indexed or 0
+                            total_skipped += result.files_skipped or 0
+                            total_nodes += result.nodes_created or 0
+                            total_edges += result.edges_created or 0
+                            if result.errors:
+                                all_errors.extend(result.errors)
+                    except Exception as e:
+                        self.log(f"  [ERROR] Batch {batch_num + 1} failed: {e}")
+
+                    files_done += len(batch)
+                    progress.update(upload_task, completed=files_done)
+
+        # Summary
+        if total_skipped > 0:
+            self.log(f"Indexed {total_indexed} files ({total_skipped} unchanged), {total_nodes} nodes, {total_edges} edges")
+        else:
+            self.log(f"Indexed {total_indexed} files, {total_nodes} nodes, {total_edges} edges")
+        if all_errors:
+            self.log(f"Errors ({len(all_errors)}):")
+            for err in all_errors[:10]:
+                self.log(f"  {err.get('path', 'unknown')}: {err.get('error', 'unknown error')}")
+            if len(all_errors) > 10:
+                self.log(f"  ... and {len(all_errors) - 10} more errors")
+
+        return True
+
+    def _send_batch_stream(
+        self,
+        files: list[dict],
+        parent_progress: Optional[Progress] = None
+    ) -> Optional[IndexProgress]:
+        """Send a batch using streaming endpoint, return final result."""
+        final_result = None
+
+        try:
+            for event in self.client.index_files_stream(
+                self.config.project_id, files
+            ):
+                if event.stage == "complete":
+                    final_result = event
+                    break
+        except httpx.HTTPStatusError as e:
+            logger.error(f"Stream error: {e.response.status_code}")
+            # Try sync fallback
+            try:
+                result = self.client.index_files(self.config.project_id, files)
+                final_result = IndexProgress(
+                    stage="complete",
+                    message="Done",
+                    current=len(files),
+                    total=len(files),
+                    files_indexed=result.files_indexed,
+                    files_skipped=result.files_skipped,
+                    nodes_created=result.nodes_created,
+                    edges_created=result.edges_created,
+                    errors=result.errors,
+                )
+            except Exception as e2:
+                logger.error(f"Sync fallback also failed: {e2}")
+                raise
+        except httpx.RequestError as e:
+            logger.error(f"Request error: {e}")
+            raise
+
+        return final_result
+
+    def initial_scan(
+        self,
+        root_path: Path,
+        ignore_check: Callable[[Path], bool]
+    ):
+        """
+        Full scan and sync with backend.
+
+        This is the main entry point for watch --force or sync command.
+        """
+        # Step 1: Scan files locally
+        files_info = self.scan_files(root_path, ignore_check)
+
+        # Populate file cache so watch mode can skip unchanged files
+        for f in files_info:
+            self._file_cache[f["path"]] = {
+                "hash": f["hash"],
+                "language": f["language"],
+            }
+
+        # Step 2: Send to backend (includes hashes for server-side dedup)
+        self.sync_files(files_info)
+
+    def delete_file_immediate(self, path: Path) -> bool:
+        """Delete a file's nodes from the graph (for watch mode)."""
+        language = get_language(path)
+        if not language:
+            return False
+
+        rel_path = str(path.relative_to(self.root_path))
+
+        try:
+            deleted = self.client.delete_files(self.config.project_id, [rel_path])
+            # Remove from cache
+            self._file_cache.pop(rel_path, None)
+            return deleted > 0
+        except Exception as e:
+            logger.error(f"Failed to delete {rel_path}: {e}")
+            return False
+
+    def sync_file_immediate(self, path: Path) -> bool:
+        """
+        Sync a single file immediately (for watch mode).
+
+        Returns True if file was synced.
+        """
+        language = get_language(path)
+        if not language:
+            return False
+
+        try:
+            if path.stat().st_size > MAX_SINGLE_FILE_BYTES:
+                return False
+        except OSError:
+            return False
+
+        try:
+            content = path.read_text(encoding="utf-8")
+        except (OSError, UnicodeDecodeError):
+            return False
+
+        rel_path = str(path.relative_to(self.root_path))
+
+        # Check if content changed from our cache
+        content_hash = self.compute_hash(content)
+        cached = self._file_cache.get(rel_path)
+
+        if cached and cached.get("hash") == content_hash:
+            return False  # No change
+
+        # Update cache
+        self._file_cache[rel_path] = {
+            "hash": content_hash,
+            "content": content,
+            "language": language,
+        }
+
+        # Send to backend
+        files = [{"path": rel_path, "content": content, "language": language, "hash": content_hash}]
+
+        try:
+            result = self.client.index_files(self.config.project_id, files)
+            return result.files_indexed > 0
+        except Exception as e:
+            logger.error(f"Failed to sync {rel_path}: {e}")
+            return False