basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/sync/watch_service.py

@@ -1,22 +1,25 @@
 """Watch service for Basic Memory."""
 
-import dataclasses
-
-from loguru import logger
-from pydantic import BaseModel
+import asyncio
+import os
+from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Set, Sequence, Callable, Awaitable, TYPE_CHECKING
 
-from rich.console import Console
-from rich.live import Live
-from rich.table import Table
-from watchfiles import awatch, Change
-import os
+if TYPE_CHECKING:
+    from basic_memory.sync.sync_service import SyncService
 
-from basic_memory.config import ProjectConfig
-from basic_memory.sync.sync_service import SyncService
-from basic_memory.services.file_service import FileService
+from basic_memory.config import BasicMemoryConfig, WATCH_STATUS_JSON
+from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
+from basic_memory.models import Project
+from basic_memory.repository import ProjectRepository
+from loguru import logger
+from pydantic import BaseModel, Field
+from rich.console import Console
+from watchfiles import awatch
+from watchfiles.main import FileChange, Change
+import time
 
 
 class WatchEvent(BaseModel):
@@ -31,8 +34,8 @@ class WatchEvent(BaseModel):
 class WatchServiceState(BaseModel):
     # Service status
     running: bool = False
-    start_time: datetime = dataclasses.field(default_factory=datetime.now)
-    pid: int = dataclasses.field(default_factory=os.getpid)
+    start_time: datetime = Field(default_factory=datetime.now)
+    pid: int = Field(default_factory=os.getpid)
 
     # Stats
     error_count: int = 0
@@ -43,7 +46,7 @@ class WatchServiceState(BaseModel):
     synced_files: int = 0
 
     # Recent activity
-    recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
+    recent_events: List[WatchEvent] = Field(default_factory=list)
 
     def add_event(
         self,
@@ -71,148 +74,442 @@ class WatchServiceState(BaseModel):
         self.last_error = datetime.now()
 
 
+# Type alias for sync service factory function
+SyncServiceFactory = Callable[[Project], Awaitable["SyncService"]]
+
+
 class WatchService:
-    def __init__(self, sync_service: SyncService, file_service: FileService, config: ProjectConfig):
-        self.sync_service = sync_service
-        self.file_service = file_service
-        self.config = config
+    def __init__(
+        self,
+        app_config: BasicMemoryConfig,
+        project_repository: ProjectRepository,
+        quiet: bool = False,
+        sync_service_factory: Optional[SyncServiceFactory] = None,
+    ):
+        self.app_config = app_config
+        self.project_repository = project_repository
         self.state = WatchServiceState()
-        self.status_path = config.home / ".basic-memory" / "watch-status.json"
+        self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
-        self.console = Console()
-
-    def generate_table(self) -> Table:
-        """Generate status display table"""
-        table = Table()
-
-        # Add status row
-        table.add_column("Status", style="cyan")
-        table.add_column("Last Scan", style="cyan")
-        table.add_column("Files", style="cyan")
-        table.add_column("Errors", style="red")
-
-        # Add main status row
-        table.add_row(
-            "✓ Running" if self.state.running else "✗ Stopped",
-            self.state.last_scan.strftime("%H:%M:%S") if self.state.last_scan else "-",
-            str(self.state.synced_files),
-            f"{self.state.error_count} ({self.state.last_error.strftime('%H:%M:%S') if self.state.last_error else 'none'})",
-        )
-
-        if self.state.recent_events:
-            # Add recent events
-            table.add_section()
-            table.add_row("Recent Events", "", "", "")
-
-            for event in self.state.recent_events[:5]:  # Show last 5 events
-                color = {
-                    "new": "green",
-                    "modified": "yellow",
-                    "moved": "blue",
-                    "deleted": "red",
-                    "error": "red",
-                }.get(event.action, "white")
-
-                icon = {
-                    "new": "✚",
-                    "modified": "✎",
-                    "moved": "→",
-                    "deleted": "✖",
-                    "error": "!",
-                }.get(event.action, "*")
-
-                table.add_row(
-                    f"[{color}]{icon} {event.action}[/{color}]",
-                    event.timestamp.strftime("%H:%M:%S"),
-                    f"[{color}]{event.path}[/{color}]",
-                    f"[dim]{event.checksum[:8] if event.checksum else ''}[/dim]",
-                )
-
-        return table
-
-    async def run(self, console_status: bool = False):  # pragma: no cover
+        self._ignore_patterns_cache: dict[Path, Set[str]] = {}
+        self._sync_service_factory = sync_service_factory
+
+        # quiet mode for mcp so it doesn't mess up stdout
+        self.console = Console(quiet=quiet)
+
+    async def _get_sync_service(self, project: Project) -> "SyncService":
+        """Get sync service for a project, using factory if provided."""
+        if self._sync_service_factory:
+            return await self._sync_service_factory(project)
+        # Fall back to default factory
+        from basic_memory.sync.sync_service import get_sync_service
+
+        return await get_sync_service(project)
+
+    async def _schedule_restart(self, stop_event: asyncio.Event):
+        """Schedule a restart of the watch service after the configured interval."""
+        await asyncio.sleep(self.app_config.watch_project_reload_interval)
+        stop_event.set()
+
+    def _get_ignore_patterns(self, project_path: Path) -> Set[str]:
+        """Get or load ignore patterns for a project path."""
+        if project_path not in self._ignore_patterns_cache:
+            self._ignore_patterns_cache[project_path] = load_gitignore_patterns(project_path)
+        return self._ignore_patterns_cache[project_path]
+
+    async def _watch_projects_cycle(self, projects: Sequence[Project], stop_event: asyncio.Event):
+        """Run one cycle of watching the given projects until stop_event is set."""
+        project_paths = [project.path for project in projects]
+
+        async for changes in awatch(
+            *project_paths,
+            debounce=self.app_config.sync_delay,
+            watch_filter=self.filter_changes,
+            recursive=True,
+            stop_event=stop_event,
+        ):
+            # group changes by project and filter using ignore patterns
+            project_changes = defaultdict(list)
+            for change, path in changes:
+                for project in projects:
+                    if self.is_project_path(project, path):
+                        # Check if the file should be ignored based on gitignore patterns
+                        project_path = Path(project.path)
+                        file_path = Path(path)
+                        ignore_patterns = self._get_ignore_patterns(project_path)
+
+                        if should_ignore_path(file_path, project_path, ignore_patterns):
+                            logger.trace(
+                                f"Ignoring watched file change: {file_path.relative_to(project_path)}"
+                            )
+                            continue
+
+                        project_changes[project].append((change, path))
+                        break
+
+            # create coroutines to handle changes
+            change_handlers = [
+                self.handle_changes(project, changes)  # pyright: ignore
+                for project, changes in project_changes.items()
+            ]
+
+            # process changes
+            await asyncio.gather(*change_handlers)
+
+    async def run(self):  # pragma: no cover
         """Watch for file changes and sync them"""
-        logger.info("Watching for sync changes")
+
         self.state.running = True
         self.state.start_time = datetime.now()
         await self.write_status()
 
-        if console_status:
-            with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
-                try:
-                    async for changes in awatch(
-                        self.config.home,
-                        watch_filter=self.filter_changes,
-                        debounce=self.config.sync_delay,
-                        recursive=True,
-                    ):
-                        # Process changes
-                        await self.handle_changes(self.config.home)
-                        # Update display
-                        live.update(self.generate_table())
+        logger.info(
+            "Watch service started",
+            f"debounce_ms={self.app_config.sync_delay}",
+            f"pid={os.getpid()}",
+        )
 
+        try:
+            while self.state.running:
+                # Clear ignore patterns cache to pick up any .gitignore changes
+                self._ignore_patterns_cache.clear()
+
+                # Reload projects to catch any new/removed projects
+                projects = await self.project_repository.get_active_projects()
+
+                project_paths = [project.path for project in projects]
+                logger.debug(f"Starting watch cycle for directories: {project_paths}")
+
+                # Create stop event for this watch cycle
+                stop_event = asyncio.Event()
+
+                # Schedule restart after configured interval to reload projects
+                timer_task = asyncio.create_task(self._schedule_restart(stop_event))
+
+                try:
+                    await self._watch_projects_cycle(projects, stop_event)
                 except Exception as e:
+                    logger.exception("Watch service error during cycle", error=str(e))
                     self.state.record_error(str(e))
                     await self.write_status()
-                    raise
+                    # Continue to next cycle instead of exiting
+                    await asyncio.sleep(5)  # Brief pause before retry
                 finally:
-                    self.state.running = False
-                    await self.write_status()
+                    # Cancel timer task if it's still running
+                    if not timer_task.done():
+                        timer_task.cancel()
+                        try:
+                            await timer_task
+                        except asyncio.CancelledError:
+                            pass
+
+        except Exception as e:
+            logger.exception("Watch service error", error=str(e))
+            self.state.record_error(str(e))
+            await self.write_status()
+            raise
+
+        finally:
+            logger.info(
+                "Watch service stopped",
+                f"runtime_seconds={int((datetime.now() - self.state.start_time).total_seconds())}",
+            )
+
+            self.state.running = False
+            await self.write_status()
+
+    def filter_changes(self, change: Change, path: str) -> bool:  # pragma: no cover
+        """Filter to only watch non-hidden files and directories.
+
+        Returns:
+            True if the file should be watched, False if it should be ignored
+        """
+
+        # Skip hidden directories and files
+        path_parts = Path(path).parts
+        for part in path_parts:
+            if part.startswith("."):
+                return False
 
-        else:
-            try:
-                async for changes in awatch(
-                    self.config.home,
-                    watch_filter=self.filter_changes,
-                    debounce=self.config.sync_delay,
-                    recursive=True,
-                ):
-                    # Process changes
-                    await self.handle_changes(self.config.home)
-                    # Update display
-
-            except Exception as e:
-                self.state.record_error(str(e))
-                await self.write_status()
-                raise
-            finally:
-                self.state.running = False
-                await self.write_status()
+        # Skip temp files used in atomic operations
+        if path.endswith(".tmp"):
+            return False
+
+        return True
 
     async def write_status(self):
         """Write current state to status file"""
         self.status_path.write_text(WatchServiceState.model_dump_json(self.state, indent=2))
 
-    def filter_changes(self, change: Change, path: str) -> bool:
-        """Filter to only watch markdown files"""
-        return path.endswith(".md") and not Path(path).name.startswith(".")
+    def is_project_path(self, project: Project, path):
+        """
+        Checks if path is a subdirectory or file within a project
+        """
+        project_path = Path(project.path).resolve()
+        sub_path = Path(path).resolve()
+        return project_path in sub_path.parents
 
-    async def handle_changes(self, directory: Path):
+    async def handle_changes(self, project: Project, changes: Set[FileChange]) -> None:
         """Process a batch of file changes"""
+        # Check if project still exists in configuration before processing
+        # This prevents deleted projects from being recreated by background sync
+        from basic_memory.config import ConfigManager
+
+        config_manager = ConfigManager()
+        if (
+            project.name not in config_manager.projects
+            and project.permalink not in config_manager.projects
+        ):
+            logger.info(
+                f"Skipping sync for deleted project: {project.name}, change_count={len(changes)}"
+            )
+            return
+
+        sync_service = await self._get_sync_service(project)
+        file_service = sync_service.file_service
+
+        start_time = time.time()
+        directory = Path(project.path).resolve()
+        logger.info(
+            f"Processing project: {project.name} changes, change_count={len(changes)}, directory={directory}"
+        )
+
+        # Group changes by type
+        adds: List[str] = []
+        deletes: List[str] = []
+        modifies: List[str] = []
 
-        logger.debug(f"handling change in directory: {directory} ...")
-        # Process changes with timeout
-        report = await self.sync_service.sync(directory)
+        for change, path in changes:
+            # convert to relative path
+            relative_path = Path(path).relative_to(directory).as_posix()
+
+            # Skip .tmp files - they're temporary and shouldn't be synced
+            if relative_path.endswith(".tmp"):
+                continue
+
+            if change == Change.added:
+                adds.append(relative_path)
+            elif change == Change.deleted:
+                deletes.append(relative_path)
+            elif change == Change.modified:
+                modifies.append(relative_path)
+
+        logger.debug(
+            f"Grouped file changes, added={len(adds)}, deleted={len(deletes)}, modified={len(modifies)}"
+        )
+
+        # because of our atomic writes on updates, an add may be an existing file
+        # Avoid mutating `adds` while iterating (can skip items).
+        reclassified_as_modified: List[str] = []
+        for added_path in list(adds):  # pragma: no cover TODO add test
+            entity = await sync_service.entity_repository.get_by_file_path(added_path)
+            if entity is not None:
+                logger.debug(f"Existing file will be processed as modified, path={added_path}")
+                reclassified_as_modified.append(added_path)
+
+        if reclassified_as_modified:
+            adds = [p for p in adds if p not in reclassified_as_modified]
+            modifies.extend(reclassified_as_modified)
+
+        # Track processed files to avoid duplicates
+        processed: Set[str] = set()
+
+        # First handle potential moves
+        for added_path in adds:
+            if added_path in processed:
+                continue  # pragma: no cover
+
+            # Skip directories for added paths
+            # We don't need to process directories, only the files inside them
+            # This prevents errors when trying to compute checksums or read directories as files
+            added_full_path = directory / added_path
+            if not added_full_path.exists() or added_full_path.is_dir():
+                logger.debug("Skipping non-existent or directory path", path=added_path)
+                processed.add(added_path)
+                continue
+
+            for deleted_path in deletes:
+                if deleted_path in processed:
+                    continue  # pragma: no cover
+
+                # Skip directories for deleted paths (based on entity type in db)
+                deleted_entity = await sync_service.entity_repository.get_by_file_path(deleted_path)
+                if deleted_entity is None:
+                    # If this was a directory, it wouldn't have an entity
+                    logger.debug("Skipping unknown path for move detection", path=deleted_path)
+                    continue
+
+                if added_path != deleted_path:
+                    # Compare checksums to detect moves
+                    try:
+                        added_checksum = await file_service.compute_checksum(added_path)
+
+                        if deleted_entity and deleted_entity.checksum == added_checksum:
+                            await sync_service.handle_move(deleted_path, added_path)
+                            self.state.add_event(
+                                path=f"{deleted_path} -> {added_path}",
+                                action="moved",
+                                status="success",
+                            )
+                            self.console.print(f"[blue]→[/blue] {deleted_path} → {added_path}")
+                            logger.info(f"move: {deleted_path} -> {added_path}")
+                            processed.add(added_path)
+                            processed.add(deleted_path)
+                            break
+                    except Exception as e:  # pragma: no cover
+                        logger.warning(
+                            "Error checking for move",
+                            f"old_path={deleted_path}",
+                            f"new_path={added_path}",
+                            f"error={str(e)}",
+                        )
+
+        # Handle remaining changes - group them by type for concise output
+        moved_count = len([p for p in processed if p in deletes or p in adds])
+        delete_count = 0
+        add_count = 0
+        modify_count = 0
+
+        # Process deletes
+        for path in deletes:
+            if path not in processed:
+                # Check if file still exists on disk (vim atomic write edge case)
+                full_path = directory / path
+                if full_path.exists() and full_path.is_file():
+                    # File still exists despite DELETE event - treat as modification
+                    logger.debug(
+                        "File exists despite DELETE event, treating as modification", path=path
+                    )
+                    entity, checksum = await sync_service.sync_file(path, new=False)
+                    self.state.add_event(
+                        path=path, action="modified", status="success", checksum=checksum
+                    )
+                    self.console.print(f"[yellow]✎[/yellow] {path} (atomic write)")
+                    logger.info(f"atomic write detected: {path}")
+                    processed.add(path)
+                    modify_count += 1
+                else:
+                    # Check if this was a directory - skip if so
+                    # (we can't tell if the deleted path was a directory since it no longer exists,
+                    # so we check if there's an entity in the database for it)
+                    entity = await sync_service.entity_repository.get_by_file_path(path)
+                    if entity is None:
+                        # No entity means this was likely a directory - skip it
+                        logger.debug(
+                            f"Skipping deleted path with no entity (likely directory), path={path}"
+                        )
+                        processed.add(path)
+                        continue
+
+                    # File truly deleted
+                    logger.debug("Processing deleted file", path=path)
+                    await sync_service.handle_delete(path)
+                    self.state.add_event(path=path, action="deleted", status="success")
+                    self.console.print(f"[red]✕[/red] {path}")
+                    logger.info(f"deleted: {path}")
+                    processed.add(path)
+                    delete_count += 1
+
+        # Process adds
+        for path in adds:
+            if path not in processed:
+                # Skip directories - only process files
+                full_path = directory / path
+                if not full_path.exists() or full_path.is_dir():
+                    logger.debug(
+                        f"Skipping non-existent or directory path, path={path}"
+                    )  # pragma: no cover
+                    processed.add(path)  # pragma: no cover
+                    continue  # pragma: no cover
+
+                logger.debug(f"Processing new file, path={path}")
+                entity, checksum = await sync_service.sync_file(path, new=True)
+                if checksum:
+                    self.state.add_event(
+                        path=path, action="new", status="success", checksum=checksum
+                    )
+                    self.console.print(f"[green]✓[/green] {path}")
+                    logger.info(
+                        "new file processed",
+                        f"path={path}",
+                        f"checksum={checksum}",
+                    )
+                    processed.add(path)
+                    add_count += 1
+                else:  # pragma: no cover
+                    logger.warning(f"Error syncing new file, path={path}")  # pragma: no cover
+                    self.console.print(
+                        f"[orange]?[/orange] Error syncing: {path}"
+                    )  # pragma: no cover
+
+        # Process modifies - detect repeats
+        last_modified_path = None
+        repeat_count = 0
+
+        for path in modifies:
+            if path not in processed:
+                # Skip directories - only process files
+                full_path = directory / path
+                if not full_path.exists() or full_path.is_dir():
+                    logger.debug("Skipping non-existent or directory path", path=path)
+                    processed.add(path)
+                    continue
+
+                logger.debug(f"Processing modified file: path={path}")
+                entity, checksum = await sync_service.sync_file(path, new=False)
+                self.state.add_event(
+                    path=path, action="modified", status="success", checksum=checksum
+                )
+
+                # Check if this is a repeat of the last modified file
+                if path == last_modified_path:  # pragma: no cover
+                    repeat_count += 1  # pragma: no cover
+                    # Only show a message for the first repeat
+                    if repeat_count == 1:  # pragma: no cover
+                        self.console.print(
+                            f"[yellow]...[/yellow] Repeated changes to {path}"
+                        )  # pragma: no cover
+                else:
+                    # haven't processed this file
+                    self.console.print(f"[yellow]✎[/yellow] {path}")
+                    logger.info(f"modified: {path}")
+                    last_modified_path = path
+                    repeat_count = 0
+                    modify_count += 1
+
+                logger.debug(  # pragma: no cover
+                    "Modified file processed, "
+                    f"path={path} "
+                    f"entity_id={entity.id if entity else None} "
+                    f"checksum={checksum}",
+                )
+                processed.add(path)
+
+        # Add a concise summary instead of a divider
+        if processed:
+            changes = []  # pyright: ignore
+            if add_count > 0:
+                changes.append(f"[green]{add_count} added[/green]")  # pyright: ignore
+            if modify_count > 0:
+                changes.append(f"[yellow]{modify_count} modified[/yellow]")  # pyright: ignore
+            if moved_count > 0:
+                changes.append(f"[blue]{moved_count} moved[/blue]")  # pyright: ignore
+            if delete_count > 0:
+                changes.append(f"[red]{delete_count} deleted[/red]")  # pyright: ignore
+
+            if changes:
+                self.console.print(f"{', '.join(changes)}", style="dim")  # pyright: ignore
+                logger.info(f"changes: {len(changes)}")
+
+        duration_ms = int((time.time() - start_time) * 1000)
         self.state.last_scan = datetime.now()
-        self.state.synced_files = report.total
+        self.state.synced_files += len(processed)
 
-        # Update stats
-        for path in report.new:
-            self.state.add_event(
-                path=path, action="new", status="success", checksum=report.checksums[path]
-            )
-        for path in report.modified:
-            self.state.add_event(
-                path=path, action="modified", status="success", checksum=report.checksums[path]
-            )
-        for old_path, new_path in report.moves.items():
-            self.state.add_event(
-                path=f"{old_path} -> {new_path}",
-                action="moved",
-                status="success",
-                checksum=report.checksums[new_path],
-            )
-        for path in report.deleted:
-            self.state.add_event(path=path, action="deleted", status="success")
+        logger.info(
+            "File change processing completed, "
+            f"processed_files={len(processed)}, "
+            f"total_synced_files={self.state.synced_files}, "
+            f"duration_ms={duration_ms}"
+        )
 
         await self.write_status()