basic-memory 0.8.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

basic_memory/sync/watch_service.py

@@ -1,20 +1,20 @@
 """Watch service for Basic Memory."""
 
-import dataclasses
 import os
 from datetime import datetime
 from pathlib import Path
 from typing import List, Optional, Set
 
+from basic_memory.config import ProjectConfig
+from basic_memory.services.file_service import FileService
+from basic_memory.sync.sync_service import SyncService
 from loguru import logger
 from pydantic import BaseModel
 from rich.console import Console
 from watchfiles import awatch
 from watchfiles.main import FileChange, Change
 
-from basic_memory.config import ProjectConfig
-from basic_memory.services.file_service import FileService
-from basic_memory.sync.sync_service import SyncService
+WATCH_STATUS_JSON = "watch-status.json"
 
 
 class WatchEvent(BaseModel):
@@ -29,8 +29,8 @@ class WatchEvent(BaseModel):
 class WatchServiceState(BaseModel):
     # Service status
     running: bool = False
-    start_time: datetime = dataclasses.field(default_factory=datetime.now)
-    pid: int = dataclasses.field(default_factory=os.getpid)
+    start_time: datetime = datetime.now()  # Use directly with Pydantic model
+    pid: int = os.getpid()  # Use directly with Pydantic model
 
     # Stats
     error_count: int = 0
@@ -41,7 +41,7 @@ class WatchServiceState(BaseModel):
     synced_files: int = 0
 
     # Recent activity
-    recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
+    recent_events: List[WatchEvent] = []  # Use directly with Pydantic model
 
     def add_event(
         self,
@@ -75,16 +75,23 @@ class WatchService:
         self.file_service = file_service
         self.config = config
         self.state = WatchServiceState()
-        self.status_path = config.home / ".basic-memory" / "watch-status.json"
+        self.status_path = config.home / ".basic-memory" / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
         self.console = Console()
 
     async def run(self):  # pragma: no cover
         """Watch for file changes and sync them"""
-        logger.info("Watching for sync changes")
+        logger.info(
+            "Watch service started",
+            directory=str(self.config.home),
+            debounce_ms=self.config.sync_delay,
+            pid=os.getpid(),
+        )
+
         self.state.running = True
         self.state.start_time = datetime.now()
         await self.write_status()
+
         try:
             async for changes in awatch(
                 self.config.home,
@@ -95,14 +102,23 @@ class WatchService:
                 await self.handle_changes(self.config.home, changes)
 
         except Exception as e:
+            logger.exception("Watch service error", error=str(e), directory=str(self.config.home))
+
             self.state.record_error(str(e))
             await self.write_status()
             raise
+
         finally:
+            logger.info(
+                "Watch service stopped",
+                directory=str(self.config.home),
+                runtime_seconds=int((datetime.now() - self.state.start_time).total_seconds()),
+            )
+
             self.state.running = False
             await self.write_status()
 
-    def filter_changes(self, change: Change, path: str) -> bool:
+    def filter_changes(self, change: Change, path: str) -> bool:  # pragma: no cover
         """Filter to only watch non-hidden files and directories.
 
         Returns:
@@ -112,6 +128,7 @@ class WatchService:
         try:
             relative_path = Path(path).relative_to(self.config.home)
         except ValueError:
+            # This is a defensive check for paths outside our home directory
             return False
 
         # Skip hidden directories and files
@@ -120,6 +137,10 @@ class WatchService:
             if part.startswith("."):
                 return False
 
+        # Skip temp files used in atomic operations
+        if path.endswith(".tmp"):
+            return False
+
         return True
 
     async def write_status(self):
@@ -128,16 +149,26 @@ class WatchService:
 
     async def handle_changes(self, directory: Path, changes: Set[FileChange]):
         """Process a batch of file changes"""
-        logger.debug(f"handling {len(changes)} changes in directory: {directory} ...")
+        import time
+        from typing import List, Set
+
+        start_time = time.time()
+
+        logger.info("Processing file changes", change_count=len(changes), directory=str(directory))
 
         # Group changes by type
-        adds = []
-        deletes = []
-        modifies = []
+        adds: List[str] = []
+        deletes: List[str] = []
+        modifies: List[str] = []
 
         for change, path in changes:
             # convert to relative path
             relative_path = str(Path(path).relative_to(directory))
+
+            # Skip .tmp files - they're temporary and shouldn't be synced
+            if relative_path.endswith(".tmp"):
+                continue
+
             if change == Change.added:
                 adds.append(relative_path)
             elif change == Change.deleted:
@@ -145,25 +176,44 @@ class WatchService:
             elif change == Change.modified:
                 modifies.append(relative_path)
 
+        logger.debug(
+            "Grouped file changes", added=len(adds), deleted=len(deletes), modified=len(modifies)
+        )
+
         # Track processed files to avoid duplicates
-        processed = set()
+        processed: Set[str] = set()
 
         # First handle potential moves
         for added_path in adds:
             if added_path in processed:
                 continue  # pragma: no cover
 
+            # Skip directories for added paths
+            # We don't need to process directories, only the files inside them
+            # This prevents errors when trying to compute checksums or read directories as files
+            added_full_path = directory / added_path
+            if not added_full_path.exists() or added_full_path.is_dir():
+                logger.debug("Skipping non-existent or directory path", path=added_path)
+                processed.add(added_path)
+                continue
+
             for deleted_path in deletes:
                 if deleted_path in processed:
                     continue  # pragma: no cover
 
+                # Skip directories for deleted paths (based on entity type in db)
+                deleted_entity = await self.sync_service.entity_repository.get_by_file_path(
+                    deleted_path
+                )
+                if deleted_entity is None:
+                    # If this was a directory, it wouldn't have an entity
+                    logger.debug("Skipping unknown path for move detection", path=deleted_path)
+                    continue
+
                 if added_path != deleted_path:
                     # Compare checksums to detect moves
                     try:
                         added_checksum = await self.file_service.compute_checksum(added_path)
-                        deleted_entity = await self.sync_service.entity_repository.get_by_file_path(
-                            deleted_path
-                        )
 
                         if deleted_entity and deleted_entity.checksum == added_checksum:
                             await self.sync_service.handle_move(deleted_path, added_path)
@@ -172,43 +222,133 @@ class WatchService:
                                 action="moved",
                                 status="success",
                             )
-                            self.console.print(
-                                f"[blue]→[/blue] Moved: {deleted_path} → {added_path}"
-                            )
+                            self.console.print(f"[blue]→[/blue] {deleted_path} → {added_path}")
                             processed.add(added_path)
                             processed.add(deleted_path)
                             break
                     except Exception as e:  # pragma: no cover
-                        logger.warning(f"Error checking for move: {e}")
+                        logger.warning(
+                            "Error checking for move",
+                            old_path=deleted_path,
+                            new_path=added_path,
+                            error=str(e),
+                        )
 
-        # Handle remaining changes
+        # Handle remaining changes - group them by type for concise output
+        moved_count = len([p for p in processed if p in deletes or p in adds])
+        delete_count = 0
+        add_count = 0
+        modify_count = 0
+
+        # Process deletes
         for path in deletes:
             if path not in processed:
+                logger.debug("Processing deleted file", path=path)
                 await self.sync_service.handle_delete(path)
                 self.state.add_event(path=path, action="deleted", status="success")
-                self.console.print(f"[red]✕[/red] Deleted: {path}")
+                self.console.print(f"[red]✕[/red] {path}")
                 processed.add(path)
+                delete_count += 1
 
+        # Process adds
         for path in adds:
             if path not in processed:
-                _, checksum = await self.sync_service.sync_file(path, new=True)
-                self.state.add_event(path=path, action="new", status="success", checksum=checksum)
-                self.console.print(f"[green]✓[/green] Added: {path}")
-                processed.add(path)
+                # Skip directories - only process files
+                full_path = directory / path
+                if not full_path.exists() or full_path.is_dir():
+                    logger.debug(
+                        "Skipping non-existent or directory path", path=path
+                    )  # pragma: no cover
+                    processed.add(path)  # pragma: no cover
+                    continue  # pragma: no cover
+
+                logger.debug("Processing new file", path=path)
+                entity, checksum = await self.sync_service.sync_file(path, new=True)
+                if checksum:
+                    self.state.add_event(
+                        path=path, action="new", status="success", checksum=checksum
+                    )
+                    self.console.print(f"[green]✓[/green] {path}")
+                    logger.debug(
+                        "Added file processed",
+                        path=path,
+                        entity_id=entity.id if entity else None,
+                        checksum=checksum,
+                    )
+                    processed.add(path)
+                    add_count += 1
+                else:  # pragma: no cover
+                    logger.warning("Error syncing new file", path=path)  # pragma: no cover
+                    self.console.print(
+                        f"[orange]?[/orange] Error syncing: {path}"
+                    )  # pragma: no cover
+
+        # Process modifies - detect repeats
+        last_modified_path = None
+        repeat_count = 0
 
         for path in modifies:
             if path not in processed:
-                _, checksum = await self.sync_service.sync_file(path, new=False)
+                # Skip directories - only process files
+                full_path = directory / path
+                if not full_path.exists() or full_path.is_dir():
+                    logger.debug("Skipping non-existent or directory path", path=path)
+                    processed.add(path)
+                    continue
+
+                logger.debug("Processing modified file", path=path)
+                entity, checksum = await self.sync_service.sync_file(path, new=False)
                 self.state.add_event(
                     path=path, action="modified", status="success", checksum=checksum
                 )
-                self.console.print(f"[yellow]✎[/yellow] Modified: {path}")
+
+                # Check if this is a repeat of the last modified file
+                if path == last_modified_path:  # pragma: no cover
+                    repeat_count += 1  # pragma: no cover
+                    # Only show a message for the first repeat
+                    if repeat_count == 1:  # pragma: no cover
+                        self.console.print(
+                            f"[yellow]...[/yellow] Repeated changes to {path}"
+                        )  # pragma: no cover
+                else:
+                    # New file being modified
+                    self.console.print(f"[yellow]✎[/yellow] {path}")
+                    last_modified_path = path
+                    repeat_count = 0
+                    modify_count += 1
+
+                logger.debug(
+                    "Modified file processed",
+                    path=path,
+                    entity_id=entity.id if entity else None,
+                    checksum=checksum,
+                )
                 processed.add(path)
 
-        # Add a divider if we processed any files
+        # Add a concise summary instead of a divider
         if processed:
-            self.console.print("─" * 50, style="dim")
-
+            changes = []  # pyright: ignore
+            if add_count > 0:
+                changes.append(f"[green]{add_count} added[/green]")  # pyright: ignore
+            if modify_count > 0:
+                changes.append(f"[yellow]{modify_count} modified[/yellow]")  # pyright: ignore
+            if moved_count > 0:
+                changes.append(f"[blue]{moved_count} moved[/blue]")  # pyright: ignore
+            if delete_count > 0:
+                changes.append(f"[red]{delete_count} deleted[/red]")  # pyright: ignore
+
+            if changes:
+                self.console.print(f"{', '.join(changes)}", style="dim")  # pyright: ignore
+
+        duration_ms = int((time.time() - start_time) * 1000)
         self.state.last_scan = datetime.now()
         self.state.synced_files += len(processed)
-        await self.write_status()
+
+        logger.info(
+            "File change processing completed",
+            processed_files=len(processed),
+            total_synced_files=self.state.synced_files,
+            duration_ms=duration_ms,
+        )
+
+        await self.write_status()

basic_memory/utils.py

@@ -1,25 +1,37 @@
 """Utility functions for basic-memory."""
 
-import logging
 import os
+
+import logging
 import re
 import sys
 from pathlib import Path
-from typing import Optional, Union
+from typing import Optional, Protocol, Union, runtime_checkable
 
 from loguru import logger
 from unidecode import unidecode
 
-import basic_memory
 
-import logfire
+@runtime_checkable
+class PathLike(Protocol):
+    """Protocol for objects that can be used as paths."""
+
+    def __str__(self) -> str: ...
 
 
-def generate_permalink(file_path: Union[Path, str]) -> str:
+# In type annotations, use Union[Path, str] instead of FilePath for now
+# This preserves compatibility with existing code while we migrate
+FilePath = Union[Path, str]
+
+# Disable the "Queue is full" warning
+logging.getLogger("opentelemetry.sdk.metrics._internal.instrument").setLevel(logging.ERROR)
+
+
+def generate_permalink(file_path: Union[Path, str, PathLike]) -> str:
     """Generate a stable permalink from a file path.
 
     Args:
-        file_path: Original file path
+        file_path: Original file path (str, Path, or PathLike)
 
     Returns:
         Normalized permalink that matches validation rules. Converts spaces and underscores
@@ -73,38 +85,25 @@ def setup_logging(
 ) -> None:  # pragma: no cover
     """
     Configure logging for the application.
-    :param home_dir: the root directory for the application
-    :param log_file: the name of the log file to write to
-    :param app: the fastapi application instance
-    :param console: whether to log to the console
-    """
 
+    Args:
+        env: The environment name (dev, test, prod)
+        home_dir: The root directory for the application
+        log_file: The name of the log file to write to
+        log_level: The logging level to use
+        console: Whether to log to the console
+    """
     # Remove default handler and any existing handlers
     logger.remove()
 
-    # Add file handler if we are not running tests
+    # Add file handler if we are not running tests and a log file is specified
     if log_file and env != "test":
-        # enable pydantic logfire
-        logfire.configure(
-            code_source=logfire.CodeSource(
-                repository="https://github.com/basicmachines-co/basic-memory",
-                revision=basic_memory.__version__,
-            ),
-            environment=env,
-            console=False,
-        )
-        logger.configure(handlers=[logfire.loguru_handler()])
-
-        # instrument code spans
-        logfire.instrument_sqlite3()
-        logfire.instrument_httpx()
-
-        # setup logger
+        # Setup file logger
         log_path = home_dir / log_file
         logger.add(
             str(log_path),
             level=log_level,
-            rotation="100 MB",
+            rotation="10 MB",
             retention="10 days",
             backtrace=True,
             diagnose=True,
@@ -112,19 +111,20 @@ def setup_logging(
             colorize=False,
         )
 
+    # Add console logger if requested or in test mode
     if env == "test" or console:
-        # Add stderr handler
         logger.add(sys.stderr, level=log_level, backtrace=True, diagnose=True, colorize=True)
 
     logger.info(f"ENV: '{env}' Log level: '{log_level}' Logging to {log_file}")
 
-    # Get the logger for 'httpx'
-    httpx_logger = logging.getLogger("httpx")
-    # Set the logging level to WARNING to ignore INFO and DEBUG logs
-    httpx_logger.setLevel(logging.WARNING)
-
-    # turn watchfiles to WARNING
-    logging.getLogger("watchfiles.main").setLevel(logging.WARNING)
-
-    # disable open telemetry warning
-    logging.getLogger("instrumentor").setLevel(logging.ERROR)
+    # Reduce noise from third-party libraries
+    noisy_loggers = {
+        # HTTP client logs
+        "httpx": logging.WARNING,
+        # File watching logs
+        "watchfiles.main": logging.WARNING,
+    }
+
+    # Set log levels for noisy loggers
+    for logger_name, level in noisy_loggers.items():
+        logging.getLogger(logger_name).setLevel(level)