basic-memory 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

basic_memory/sync/watch_service.py

@@ -1,22 +1,21 @@
 """Watch service for Basic Memory."""
 
-import dataclasses
-
-from loguru import logger
-from pydantic import BaseModel
+import os
 from datetime import datetime
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Optional, Set
 
+from loguru import logger
+from pydantic import BaseModel
 from rich.console import Console
-from rich.live import Live
-from rich.table import Table
-from watchfiles import awatch, Change
-import os
+from watchfiles import awatch
+from watchfiles.main import FileChange, Change
 
 from basic_memory.config import ProjectConfig
-from basic_memory.sync.sync_service import SyncService
 from basic_memory.services.file_service import FileService
+from basic_memory.sync.sync_service import SyncService
+
+WATCH_STATUS_JSON = "watch-status.json"
 
 
 class WatchEvent(BaseModel):
@@ -31,8 +30,8 @@ class WatchEvent(BaseModel):
 class WatchServiceState(BaseModel):
     # Service status
     running: bool = False
-    start_time: datetime = dataclasses.field(default_factory=datetime.now)
-    pid: int = dataclasses.field(default_factory=os.getpid)
+    start_time: datetime = datetime.now()  # Use directly with Pydantic model
+    pid: int = os.getpid()  # Use directly with Pydantic model
 
     # Stats
     error_count: int = 0
@@ -43,7 +42,7 @@ class WatchServiceState(BaseModel):
     synced_files: int = 0
 
     # Recent activity
-    recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
+    recent_events: List[WatchEvent] = []  # Use directly with Pydantic model
 
     def add_event(
         self,
@@ -77,142 +76,269 @@ class WatchService:
         self.file_service = file_service
         self.config = config
         self.state = WatchServiceState()
-        self.status_path = config.home / ".basic-memory" / "watch-status.json"
+        self.status_path = config.home / ".basic-memory" / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
         self.console = Console()
 
-    def generate_table(self) -> Table:
-        """Generate status display table"""
-        table = Table()
-
-        # Add status row
-        table.add_column("Status", style="cyan")
-        table.add_column("Last Scan", style="cyan")
-        table.add_column("Files", style="cyan")
-        table.add_column("Errors", style="red")
-
-        # Add main status row
-        table.add_row(
-            "✓ Running" if self.state.running else "✗ Stopped",
-            self.state.last_scan.strftime("%H:%M:%S") if self.state.last_scan else "-",
-            str(self.state.synced_files),
-            f"{self.state.error_count} ({self.state.last_error.strftime('%H:%M:%S') if self.state.last_error else 'none'})",
+    async def run(self):  # pragma: no cover
+        """Watch for file changes and sync them"""
+        logger.info(
+            "Watch service started",
+            directory=str(self.config.home),
+            debounce_ms=self.config.sync_delay,
+            pid=os.getpid(),
         )
 
-        if self.state.recent_events:
-            # Add recent events
-            table.add_section()
-            table.add_row("Recent Events", "", "", "")
-
-            for event in self.state.recent_events[:5]:  # Show last 5 events
-                color = {
-                    "new": "green",
-                    "modified": "yellow",
-                    "moved": "blue",
-                    "deleted": "red",
-                    "error": "red",
-                }.get(event.action, "white")
-
-                icon = {
-                    "new": "✚",
-                    "modified": "✎",
-                    "moved": "→",
-                    "deleted": "✖",
-                    "error": "!",
-                }.get(event.action, "*")
-
-                table.add_row(
-                    f"[{color}]{icon} {event.action}[/{color}]",
-                    event.timestamp.strftime("%H:%M:%S"),
-                    f"[{color}]{event.path}[/{color}]",
-                    f"[dim]{event.checksum[:8] if event.checksum else ''}[/dim]",
-                )
-
-        return table
-
-    async def run(self, console_status: bool = False):  # pragma: no cover
-        """Watch for file changes and sync them"""
-        logger.info("Watching for sync changes")
         self.state.running = True
         self.state.start_time = datetime.now()
         await self.write_status()
 
-        if console_status:
-            with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
-                try:
-                    async for changes in awatch(
-                        self.config.home,
-                        watch_filter=self.filter_changes,
-                        debounce=self.config.sync_delay,
-                        recursive=True,
-                    ):
-                        # Process changes
-                        await self.handle_changes(self.config.home)
-                        # Update display
-                        live.update(self.generate_table())
-
-                except Exception as e:
-                    self.state.record_error(str(e))
-                    await self.write_status()
-                    raise
-                finally:
-                    self.state.running = False
-                    await self.write_status()
-
-        else:
-            try:
-                async for changes in awatch(
-                    self.config.home,
-                    watch_filter=self.filter_changes,
-                    debounce=self.config.sync_delay,
-                    recursive=True,
-                ):
-                    # Process changes
-                    await self.handle_changes(self.config.home)
-                    # Update display
-
-            except Exception as e:
-                self.state.record_error(str(e))
-                await self.write_status()
-                raise
-            finally:
-                self.state.running = False
-                await self.write_status()
+        try:
+            async for changes in awatch(
+                self.config.home,
+                debounce=self.config.sync_delay,
+                watch_filter=self.filter_changes,
+                recursive=True,
+            ):
+                await self.handle_changes(self.config.home, changes)
+
+        except Exception as e:
+            logger.exception("Watch service error", error=str(e), directory=str(self.config.home))
+
+            self.state.record_error(str(e))
+            await self.write_status()
+            raise
+
+        finally:
+            logger.info(
+                "Watch service stopped",
+                directory=str(self.config.home),
+                runtime_seconds=int((datetime.now() - self.state.start_time).total_seconds()),
+            )
+
+            self.state.running = False
+            await self.write_status()
+
+    def filter_changes(self, change: Change, path: str) -> bool:  # pragma: no cover
+        """Filter to only watch non-hidden files and directories.
+
+        Returns:
+            True if the file should be watched, False if it should be ignored
+        """
+        # Skip if path is invalid
+        try:
+            relative_path = Path(path).relative_to(self.config.home)
+        except ValueError:
+            # This is a defensive check for paths outside our home directory
+            return False
+
+        # Skip hidden directories and files
+        path_parts = relative_path.parts
+        for part in path_parts:
+            if part.startswith("."):
+                return False
+
+        return True
 
     async def write_status(self):
         """Write current state to status file"""
         self.status_path.write_text(WatchServiceState.model_dump_json(self.state, indent=2))
 
-    def filter_changes(self, change: Change, path: str) -> bool:
-        """Filter to only watch markdown files"""
-        return path.endswith(".md") and not Path(path).name.startswith(".")
-
-    async def handle_changes(self, directory: Path):
+    async def handle_changes(self, directory: Path, changes: Set[FileChange]):
         """Process a batch of file changes"""
+        import time
+        from typing import List, Set
+
+        start_time = time.time()
+
+        logger.info("Processing file changes", change_count=len(changes), directory=str(directory))
+
+        # Group changes by type
+        adds: List[str] = []
+        deletes: List[str] = []
+        modifies: List[str] = []
+
+        for change, path in changes:
+            # convert to relative path
+            relative_path = str(Path(path).relative_to(directory))
+            if change == Change.added:
+                adds.append(relative_path)
+            elif change == Change.deleted:
+                deletes.append(relative_path)
+            elif change == Change.modified:
+                modifies.append(relative_path)
+
+        logger.debug(
+            "Grouped file changes", added=len(adds), deleted=len(deletes), modified=len(modifies)
+        )
 
-        logger.debug(f"handling change in directory: {directory} ...")
-        # Process changes with timeout
-        report = await self.sync_service.sync(directory)
+        # Track processed files to avoid duplicates
+        processed: Set[str] = set()
+
+        # First handle potential moves
+        for added_path in adds:
+            if added_path in processed:
+                continue  # pragma: no cover
+
+            # Skip directories for added paths
+            # We don't need to process directories, only the files inside them
+            # This prevents errors when trying to compute checksums or read directories as files
+            added_full_path = directory / added_path
+            if added_full_path.is_dir():
+                logger.debug("Skipping directory for move detection", path=added_path)
+                processed.add(added_path)
+                continue
+
+            for deleted_path in deletes:
+                if deleted_path in processed:
+                    continue  # pragma: no cover
+
+                # Skip directories for deleted paths (based on entity type in db)
+                deleted_entity = await self.sync_service.entity_repository.get_by_file_path(
+                    deleted_path
+                )
+                if deleted_entity is None:
+                    # If this was a directory, it wouldn't have an entity
+                    logger.debug("Skipping unknown path for move detection", path=deleted_path)
+                    continue
+
+                if added_path != deleted_path:
+                    # Compare checksums to detect moves
+                    try:
+                        added_checksum = await self.file_service.compute_checksum(added_path)
+
+                        if deleted_entity and deleted_entity.checksum == added_checksum:
+                            await self.sync_service.handle_move(deleted_path, added_path)
+                            self.state.add_event(
+                                path=f"{deleted_path} -> {added_path}",
+                                action="moved",
+                                status="success",
+                            )
+                            self.console.print(f"[blue]→[/blue] {deleted_path} → {added_path}")
+                            processed.add(added_path)
+                            processed.add(deleted_path)
+                            break
+                    except Exception as e:  # pragma: no cover
+                        logger.warning(
+                            "Error checking for move",
+                            old_path=deleted_path,
+                            new_path=added_path,
+                            error=str(e),
+                        )
+
+        # Handle remaining changes - group them by type for concise output
+        moved_count = len([p for p in processed if p in deletes or p in adds])
+        delete_count = 0
+        add_count = 0
+        modify_count = 0
+
+        # Process deletes
+        for path in deletes:
+            if path not in processed:
+                logger.debug("Processing deleted file", path=path)
+                await self.sync_service.handle_delete(path)
+                self.state.add_event(path=path, action="deleted", status="success")
+                self.console.print(f"[red]✕[/red] {path}")
+                processed.add(path)
+                delete_count += 1
+
+        # Process adds
+        for path in adds:
+            if path not in processed:
+                # Skip directories - only process files
+                full_path = directory / path
+                if full_path.is_dir():  # pragma: no cover
+                    logger.debug("Skipping directory", path=path)
+                    processed.add(path)
+                    continue
+
+                logger.debug("Processing new file", path=path)
+                entity, checksum = await self.sync_service.sync_file(path, new=True)
+                if checksum:
+                    self.state.add_event(
+                        path=path, action="new", status="success", checksum=checksum
+                    )
+                    self.console.print(f"[green]✓[/green] {path}")
+                    logger.debug(
+                        "Added file processed",
+                        path=path,
+                        entity_id=entity.id if entity else None,
+                        checksum=checksum,
+                    )
+                    processed.add(path)
+                    add_count += 1
+                else:  # pragma: no cover
+                    logger.warning("Error syncing new file", path=path)  # pragma: no cover
+                    self.console.print(
+                        f"[orange]?[/orange] Error syncing: {path}"
+                    )  # pragma: no cover
+
+        # Process modifies - detect repeats
+        last_modified_path = None
+        repeat_count = 0
+
+        for path in modifies:
+            if path not in processed:
+                # Skip directories - only process files
+                full_path = directory / path
+                if full_path.is_dir():
+                    logger.debug("Skipping directory", path=path)
+                    processed.add(path)
+                    continue
+
+                logger.debug("Processing modified file", path=path)
+                entity, checksum = await self.sync_service.sync_file(path, new=False)
+                self.state.add_event(
+                    path=path, action="modified", status="success", checksum=checksum
+                )
+
+                # Check if this is a repeat of the last modified file
+                if path == last_modified_path:  # pragma: no cover
+                    repeat_count += 1  # pragma: no cover
+                    # Only show a message for the first repeat
+                    if repeat_count == 1:  # pragma: no cover
+                        self.console.print(
+                            f"[yellow]...[/yellow] Repeated changes to {path}"
+                        )  # pragma: no cover
+                else:
+                    # New file being modified
+                    self.console.print(f"[yellow]✎[/yellow] {path}")
+                    last_modified_path = path
+                    repeat_count = 0
+                    modify_count += 1
+
+                logger.debug(
+                    "Modified file processed",
+                    path=path,
+                    entity_id=entity.id if entity else None,
+                    checksum=checksum,
+                )
+                processed.add(path)
+
+        # Add a concise summary instead of a divider
+        if processed:
+            changes = []  # pyright: ignore
+            if add_count > 0:
+                changes.append(f"[green]{add_count} added[/green]")  # pyright: ignore
+            if modify_count > 0:
+                changes.append(f"[yellow]{modify_count} modified[/yellow]")  # pyright: ignore
+            if moved_count > 0:
+                changes.append(f"[blue]{moved_count} moved[/blue]")  # pyright: ignore
+            if delete_count > 0:
+                changes.append(f"[red]{delete_count} deleted[/red]")  # pyright: ignore
+
+            if changes:
+                self.console.print(f"{', '.join(changes)}", style="dim")  # pyright: ignore
+
+        duration_ms = int((time.time() - start_time) * 1000)
         self.state.last_scan = datetime.now()
-        self.state.synced_files = report.total
+        self.state.synced_files += len(processed)
 
-        # Update stats
-        for path in report.new:
-            self.state.add_event(
-                path=path, action="new", status="success", checksum=report.checksums[path]
-            )
-        for path in report.modified:
-            self.state.add_event(
-                path=path, action="modified", status="success", checksum=report.checksums[path]
-            )
-        for old_path, new_path in report.moves.items():
-            self.state.add_event(
-                path=f"{old_path} -> {new_path}",
-                action="moved",
-                status="success",
-                checksum=report.checksums[new_path],
-            )
-        for path in report.deleted:
-            self.state.add_event(path=path, action="deleted", status="success")
+        logger.info(
+            "File change processing completed",
+            processed_files=len(processed),
+            total_synced_files=self.state.synced_files,
+            duration_ms=duration_ms,
+        )
 
         await self.write_status()

basic_memory/utils.py

@@ -1,25 +1,37 @@
 """Utility functions for basic-memory."""
 
 import os
+
+import logging
 import re
 import sys
 from pathlib import Path
-from typing import Optional, Union
+from typing import Optional, Protocol, Union, runtime_checkable
 
 from loguru import logger
 from unidecode import unidecode
 
-import basic_memory
-from basic_memory.config import config
 
-import logfire
+@runtime_checkable
+class PathLike(Protocol):
+    """Protocol for objects that can be used as paths."""
+
+    def __str__(self) -> str: ...
+
+
+# In type annotations, use Union[Path, str] instead of FilePath for now
+# This preserves compatibility with existing code while we migrate
+FilePath = Union[Path, str]
 
+# Disable the "Queue is full" warning
+logging.getLogger("opentelemetry.sdk.metrics._internal.instrument").setLevel(logging.ERROR)
 
-def generate_permalink(file_path: Union[Path, str]) -> str:
+
+def generate_permalink(file_path: Union[Path, str, PathLike]) -> str:
     """Generate a stable permalink from a file path.
 
     Args:
-        file_path: Original file path
+        file_path: Original file path (str, Path, or PathLike)
 
     Returns:
         Normalized permalink that matches validation rules. Converts spaces and underscores
@@ -65,43 +77,33 @@ def generate_permalink(file_path: Union[Path, str]) -> str:
 
 
 def setup_logging(
-    home_dir: Path = config.home, log_file: Optional[str] = None, console: bool = True
+    env: str,
+    home_dir: Path,
+    log_file: Optional[str] = None,
+    log_level: str = "INFO",
+    console: bool = True,
 ) -> None:  # pragma: no cover
     """
     Configure logging for the application.
-    :param home_dir: the root directory for the application
-    :param log_file: the name of the log file to write to
-    :param app: the fastapi application instance
-    :param console: whether to log to the console
-    """
 
+    Args:
+        env: The environment name (dev, test, prod)
+        home_dir: The root directory for the application
+        log_file: The name of the log file to write to
+        log_level: The logging level to use
+        console: Whether to log to the console
+    """
     # Remove default handler and any existing handlers
     logger.remove()
 
-    # Add file handler if we are not running tests
-    if log_file and config.env != "test":
-        # enable pydantic logfire
-        logfire.configure(
-            code_source=logfire.CodeSource(
-                repository="https://github.com/basicmachines-co/basic-memory",
-                revision=basic_memory.__version__,
-                root_path="/src/basic_memory",
-            ),
-            environment=config.env,
-            console=False,
-        )
-        logger.configure(handlers=[logfire.loguru_handler()])
-
-        # instrument code spans
-        logfire.instrument_sqlite3()
-        logfire.instrument_httpx()
-
-        # setup logger
+    # Add file handler if we are not running tests and a log file is specified
+    if log_file and env != "test":
+        # Setup file logger
         log_path = home_dir / log_file
         logger.add(
             str(log_path),
-            level=config.log_level,
-            rotation="100 MB",
+            level=log_level,
+            rotation="10 MB",
             retention="10 days",
             backtrace=True,
             diagnose=True,
@@ -109,7 +111,20 @@ def setup_logging(
             colorize=False,
         )
 
-    # Add stderr handler
-    logger.add(sys.stderr, level=config.log_level, backtrace=True, diagnose=True, colorize=True)
+    # Add console logger if requested or in test mode
+    if env == "test" or console:
+        logger.add(sys.stderr, level=log_level, backtrace=True, diagnose=True, colorize=True)
+
+    logger.info(f"ENV: '{env}' Log level: '{log_level}' Logging to {log_file}")
+
+    # Reduce noise from third-party libraries
+    noisy_loggers = {
+        # HTTP client logs
+        "httpx": logging.WARNING,
+        # File watching logs
+        "watchfiles.main": logging.WARNING,
+    }
 
-    logger.info(f"ENV: '{config.env}' Log level: '{config.log_level}' Logging to {log_file}")
+    # Set log levels for noisy loggers
+    for logger_name, level in noisy_loggers.items():
+        logging.getLogger(logger_name).setLevel(level)