basic-memory 0.16.1__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/services/file_service.py

@@ -3,15 +3,19 @@
 import asyncio
 import hashlib
 import mimetypes
-from os import stat_result
+from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union
 
 import aiofiles
+
 import yaml
 
 from basic_memory import file_utils
-from basic_memory.file_utils import FileError, ParseError
+
+if TYPE_CHECKING:  # pragma: no cover
+    from basic_memory.config import BasicMemoryConfig
+from basic_memory.file_utils import FileError, FileMetadata, ParseError
 from basic_memory.markdown.markdown_processor import MarkdownProcessor
 from basic_memory.models import Entity as EntityModel
 from basic_memory.schemas import Entity as EntitySchema
@@ -41,9 +45,11 @@ class FileService:
         base_path: Path,
         markdown_processor: MarkdownProcessor,
         max_concurrent_files: int = 10,
+        app_config: Optional["BasicMemoryConfig"] = None,
     ):
         self.base_path = base_path.resolve()  # Get absolute path
         self.markdown_processor = markdown_processor
+        self.app_config = app_config
         # Semaphore to limit concurrent file operations
         # Prevents OOM on large projects by processing files in batches
         self._file_semaphore = asyncio.Semaphore(max_concurrent_files)
@@ -148,12 +154,15 @@ class FileService:
         Handles both absolute and relative paths. Relative paths are resolved
         against base_path.
 
+        If format_on_save is enabled in config, runs the configured formatter
+        after writing and returns the checksum of the formatted content.
+
         Args:
             path: Where to write (Path or string)
             content: Content to write
 
         Returns:
-            Checksum of written content
+            Checksum of written content (or formatted content if formatting enabled)
 
         Raises:
             FileOperationError: If write fails
@@ -176,8 +185,17 @@ class FileService:
 
             await file_utils.write_file_atomic(full_path, content)
 
-            # Compute and return checksum
-            checksum = await file_utils.compute_checksum(content)
+            # Format file if configured
+            final_content = content
+            if self.app_config:
+                formatted_content = await file_utils.format_file(
+                    full_path, self.app_config, is_markdown=self.is_markdown(path)
+                )
+                if formatted_content is not None:
+                    final_content = formatted_content  # pragma: no cover
+
+            # Compute and return checksum of final content
+            checksum = await file_utils.compute_checksum(final_content)
             logger.debug(f"File write completed path={full_path}, {checksum=}")
             return checksum
 
@@ -216,6 +234,45 @@ class FileService:
             )
             return content
 
+        except FileNotFoundError:
+            # Preserve FileNotFoundError so callers (e.g. sync) can treat it as deletion.
+            logger.warning("File not found", operation="read_file_content", path=str(full_path))
+            raise
+        except Exception as e:
+            logger.exception("File read error", path=str(full_path), error=str(e))
+            raise FileOperationError(f"Failed to read file: {e}")
+
+    async def read_file_bytes(self, path: FilePath) -> bytes:
+        """Read file content as bytes using true async I/O with aiofiles.
+
+        This method reads files in binary mode, suitable for non-text files
+        like images, PDFs, etc. For cloud compatibility with S3FileService.
+
+        Args:
+            path: Path to read (Path or string)
+
+        Returns:
+            File content as bytes
+
+        Raises:
+            FileOperationError: If read fails
+        """
+        # Convert string to Path if needed
+        path_obj = self.base_path / path if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
+
+        try:
+            logger.debug("Reading file bytes", operation="read_file_bytes", path=str(full_path))
+            async with aiofiles.open(full_path, mode="rb") as f:
+                content = await f.read()
+
+            logger.debug(
+                "File read completed",
+                path=str(full_path),
+                content_length=len(content),
+            )
+            return content
+
         except Exception as e:
             logger.exception("File read error", path=str(full_path), error=str(e))
             raise FileOperationError(f"Failed to read file: {e}")
@@ -276,6 +333,43 @@ class FileService:
         full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
         full_path.unlink(missing_ok=True)
 
+    async def move_file(self, source: FilePath, destination: FilePath) -> None:
+        """Move/rename a file from source to destination.
+
+        This method abstracts the underlying storage (filesystem vs cloud).
+        Default implementation uses atomic filesystem rename, but cloud-backed
+        implementations (e.g., S3) can override to copy+delete.
+
+        Args:
+            source: Source path (relative to base_path or absolute)
+            destination: Destination path (relative to base_path or absolute)
+
+        Raises:
+            FileOperationError: If the move fails
+        """
+        # Convert strings to Paths and resolve relative paths against base_path
+        src_obj = self.base_path / source if isinstance(source, str) else source
+        dst_obj = self.base_path / destination if isinstance(destination, str) else destination
+        src_full = src_obj if src_obj.is_absolute() else self.base_path / src_obj
+        dst_full = dst_obj if dst_obj.is_absolute() else self.base_path / dst_obj
+
+        try:
+            # Ensure destination directory exists
+            await self.ensure_directory(dst_full.parent)
+
+            # Use semaphore for concurrency control and run blocking rename in executor
+            async with self._file_semaphore:
+                loop = asyncio.get_event_loop()
+                await loop.run_in_executor(None, lambda: src_full.rename(dst_full))
+        except Exception as e:  # pragma: no cover
+            logger.exception(
+                "File move error",
+                source=str(src_full),
+                destination=str(dst_full),
+                error=str(e),
+            )
+            raise FileOperationError(f"Failed to move file {source} -> {destination}: {e}")
+
     async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
         """Update frontmatter fields in a file while preserving all content.
 
@@ -311,14 +405,14 @@ class FileService:
                 try:
                     current_fm = file_utils.parse_frontmatter(content)
                     content = file_utils.remove_frontmatter(content)
-                except (ParseError, yaml.YAMLError) as e:
+                except (ParseError, yaml.YAMLError) as e:  # pragma: no cover
                     # Log warning and treat as plain markdown without frontmatter
-                    logger.warning(
+                    logger.warning(  # pragma: no cover
                         f"Failed to parse YAML frontmatter in {full_path}: {e}. "
                         "Treating file as plain markdown without frontmatter."
                     )
                     # Keep full content, treat as having no frontmatter
-                    current_fm = {}
+                    current_fm = {}  # pragma: no cover
 
             # Update frontmatter
             new_fm = {**current_fm, **updates}
@@ -332,9 +426,19 @@ class FileService:
             )
 
             await file_utils.write_file_atomic(full_path, final_content)
-            return await file_utils.compute_checksum(final_content)
 
-        except Exception as e:
+            # Format file if configured
+            content_for_checksum = final_content
+            if self.app_config:
+                formatted_content = await file_utils.format_file(
+                    full_path, self.app_config, is_markdown=self.is_markdown(path)
+                )
+                if formatted_content is not None:
+                    content_for_checksum = formatted_content  # pragma: no cover
+
+            return await file_utils.compute_checksum(content_for_checksum)
+
+        except Exception as e:  # pragma: no cover
             # Only log real errors (not YAML parsing, which is handled above)
             if not isinstance(e, (ParseError, yaml.YAMLError)):
                 logger.error(
@@ -381,20 +485,31 @@ class FileService:
                 logger.error("Failed to compute checksum", path=str(full_path), error=str(e))
                 raise FileError(f"Failed to compute checksum for {path}: {e}")
 
-    def file_stats(self, path: FilePath) -> stat_result:
-        """Return file stats for a given path.
+    async def get_file_metadata(self, path: FilePath) -> FileMetadata:
+        """Return file metadata for a given path.
+
+        This method is async to support cloud implementations (S3FileService)
+        where file metadata requires async operations (head_object).
 
         Args:
             path: Path to the file (Path or string)
 
         Returns:
-            File statistics
+            FileMetadata with size, created_at, and modified_at
         """
         # Convert string to Path if needed
         path_obj = self.base_path / path if isinstance(path, str) else path
         full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
-        # get file timestamps
-        return full_path.stat()
+
+        # Run blocking stat() in thread pool to maintain async compatibility
+        loop = asyncio.get_event_loop()
+        stat_result = await loop.run_in_executor(None, full_path.stat)
+
+        return FileMetadata(
+            size=stat_result.st_size,
+            created_at=datetime.fromtimestamp(stat_result.st_ctime).astimezone(),
+            modified_at=datetime.fromtimestamp(stat_result.st_mtime).astimezone(),
+        )
 
     def content_type(self, path: FilePath) -> str:
         """Return content_type for a given path.

basic_memory/services/initialization.py

@@ -5,8 +5,11 @@ to ensure consistent application startup across all entry points.
 """
 
 import asyncio
+import os
+import sys
 from pathlib import Path
 
+
 from loguru import logger
 
 from basic_memory import db
@@ -27,15 +30,12 @@ async def initialize_database(app_config: BasicMemoryConfig) -> None:
         Database migrations are now handled automatically when the database
         connection is first established via get_or_create_db().
     """
-    # Trigger database initialization and migrations by getting the database connection
     try:
         await db.get_or_create_db(app_config.database_path)
         logger.info("Database initialization completed")
     except Exception as e:
-        logger.error(f"Error initializing database: {e}")
-        # Allow application to continue - it might still work
-        # depending on what the error was, and will fail with a
-        # more specific error if the database is actually unusable
+        logger.error(f"Error during database initialization: {e}")
+        raise
 
 
 async def reconcile_projects_with_config(app_config: BasicMemoryConfig):
@@ -49,31 +49,29 @@ async def reconcile_projects_with_config(app_config: BasicMemoryConfig):
     """
     logger.info("Reconciling projects from config with database...")
 
-    # Get database session - migrations handled centrally
+    # Get database session (engine already created by initialize_database)
     _, session_maker = await db.get_or_create_db(
         db_path=app_config.database_path,
         db_type=db.DatabaseType.FILESYSTEM,
-        ensure_migrations=False,
     )
     project_repository = ProjectRepository(session_maker)
 
     # Import ProjectService here to avoid circular imports
     from basic_memory.services.project_service import ProjectService
 
+    # Create project service and synchronize projects
+    project_service = ProjectService(repository=project_repository)
     try:
-        # Create project service and synchronize projects
-        project_service = ProjectService(repository=project_repository)
         await project_service.synchronize_projects()
         logger.info("Projects successfully reconciled between config and database")
     except Exception as e:
-        # Log the error but continue with initialization
         logger.error(f"Error during project synchronization: {e}")
         logger.info("Continuing with initialization despite synchronization error")
 
 
 async def initialize_file_sync(
     app_config: BasicMemoryConfig,
-):
+) -> None:
     """Initialize file synchronization services. This function starts the watch service and does not return
 
     Args:
@@ -82,15 +80,20 @@ async def initialize_file_sync(
     Returns:
         The watch service task that's monitoring file changes
     """
+    # Never start file watching during tests. Even "background" watchers add tasks/threads
+    # and can interact badly with strict asyncio teardown (especially on Windows/aiosqlite).
+    # Skip file sync in test environments to avoid interference with tests
+    if app_config.is_test_env:
+        logger.info("Test environment detected - skipping file sync initialization")
+        return None
 
     # delay import
     from basic_memory.sync import WatchService
 
-    # Load app configuration - migrations handled centrally
+    # Get database session (migrations already run if needed)
     _, session_maker = await db.get_or_create_db(
         db_path=app_config.database_path,
         db_type=db.DatabaseType.FILESYSTEM,
-        ensure_migrations=False,
     )
     project_repository = ProjectRepository(session_maker)
 
@@ -104,6 +107,12 @@ async def initialize_file_sync(
     # Get active projects
     active_projects = await project_repository.get_active_projects()
 
+    # Filter to constrained project if MCP server was started with --project
+    constrained_project = os.environ.get("BASIC_MEMORY_MCP_PROJECT")
+    if constrained_project:
+        active_projects = [p for p in active_projects if p.name == constrained_project]
+        logger.info(f"Background sync constrained to project: {constrained_project}")
+
     # Start sync for all projects as background tasks (non-blocking)
     async def sync_project_background(project: Project):
         """Sync a single project in the background."""
@@ -131,12 +140,10 @@ async def initialize_file_sync(
 
     # Then start the watch service in the background
     logger.info("Starting watch service for all projects")
+
     # run the watch service
-    try:
-        await watch_service.run()
-        logger.info("Watch service started")
-    except Exception as e:  # pragma: no cover
-        logger.error(f"Error starting watch service: {e}")
+    await watch_service.run()
+    logger.info("Watch service started")
 
     return None
 
@@ -155,6 +162,11 @@ async def initialize_app(
     Args:
         app_config: The Basic Memory project configuration
     """
+    # Skip initialization in cloud mode - cloud manages its own projects
+    if app_config.cloud_mode_enabled:
+        logger.debug("Skipping initialization in cloud mode - projects managed by cloud")
+        return
+
     logger.info("Initializing app...")
     # Initialize database first
     await initialize_database(app_config)
@@ -181,11 +193,24 @@ def ensure_initialization(app_config: BasicMemoryConfig) -> None:
         logger.debug("Skipping initialization in cloud mode - projects managed by cloud")
         return
 
-    try:
-        result = asyncio.run(initialize_app(app_config))
-        logger.info(f"Initialization completed successfully: result={result}")
-    except Exception as e:  # pragma: no cover
-        logger.exception(f"Error during initialization: {e}")
-        # Continue execution even if initialization fails
-        # The command might still work, or will fail with a
-        # more specific error message
+    async def _init_and_cleanup():
+        """Initialize app and clean up database connections.
+
+        Database connections created during initialization must be cleaned up
+        before the event loop closes, otherwise the process will hang indefinitely.
+        """
+        try:
+            await initialize_app(app_config)
+        finally:
+            # Always cleanup database connections to prevent process hang
+            await db.shutdown_db()
+
+    # On Windows, use SelectorEventLoop to avoid ProactorEventLoop cleanup issues
+    # The ProactorEventLoop can raise "IndexError: pop from an empty deque" during
+    # event loop cleanup when there are pending handles. SelectorEventLoop is more
+    # stable for our use case (no subprocess pipes or named pipes needed).
+    if sys.platform == "win32":
+        asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+    asyncio.run(_init_and_cleanup())
+    logger.info("Initialization completed successfully")

basic_memory/services/link_resolver.py

@@ -2,6 +2,7 @@
 
 from typing import Optional, Tuple
 
+
 from loguru import logger
 
 from basic_memory.models import Entity

basic_memory/services/project_service.py

@@ -8,6 +8,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional, Sequence
 
+
 from loguru import logger
 from sqlalchemy import text
 
@@ -23,9 +24,6 @@ from basic_memory.config import WATCH_STATUS_JSON, ConfigManager, get_project_co
 from basic_memory.utils import generate_permalink
 
 
-config = ConfigManager().config
-
-
 class ProjectService:
     """Service for managing Basic Memory projects."""
 
@@ -46,7 +44,7 @@ class ProjectService:
         return ConfigManager()
 
     @property
-    def config(self) -> ProjectConfig:
+    def config(self) -> ProjectConfig:  # pragma: no cover
         """Get the current project configuration.
 
         Returns:
@@ -143,6 +141,7 @@ class ProjectService:
         """
         # If project_root is set, constrain all projects to that directory
         project_root = self.config_manager.config.project_root
+        sanitized_name = None
         if project_root:
             base_path = Path(project_root)
 
@@ -155,11 +154,11 @@ class ProjectService:
             resolved_path = (base_path / sanitized_name).resolve().as_posix()
 
             # Verify the resolved path is actually under project_root
-            if not resolved_path.startswith(base_path.resolve().as_posix()):
+            if not resolved_path.startswith(base_path.resolve().as_posix()):  # pragma: no cover
                 raise ValueError(
                     f"BASIC_MEMORY_PROJECT_ROOT is set to {project_root}. "
                     f"All projects must be created under this directory. Invalid path: {path}"
-                )
+                )  # pragma: no cover
 
             # Check for case-insensitive path collisions with existing projects
             existing_projects = await self.list_projects()
@@ -168,11 +167,11 @@ class ProjectService:
                     existing.path.lower() == resolved_path.lower()
                     and existing.path != resolved_path
                 ):
-                    raise ValueError(
+                    raise ValueError(  # pragma: no cover
                         f"Path collision detected: '{resolved_path}' conflicts with existing project "
                         f"'{existing.name}' at '{existing.path}'. "
                         f"In cloud mode, paths are normalized to lowercase to prevent case-sensitivity issues."
-                    )
+                    )  # pragma: no cover
         else:
             resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix()
 
@@ -199,14 +198,15 @@ class ProjectService:
                         f"Projects cannot share directory trees."
                     )
 
-        # First add to config file (this will validate the project doesn't exist)
-        project_config = self.config_manager.add_project(name, resolved_path)
+        if not self.config_manager.config.cloud_mode:
+            # First add to config file (this will validate the project doesn't exist)
+            self.config_manager.add_project(name, resolved_path)
 
         # Then add to database
         project_data = {
             "name": name,
             "path": resolved_path,
-            "permalink": generate_permalink(project_config.name),
+            "permalink": sanitized_name,
             "is_active": True,
             # Don't set is_default=False to avoid UNIQUE constraint issues
             # Let it default to NULL, only set to True when explicitly making default
@@ -237,20 +237,22 @@ class ProjectService:
         # Get project from database first
         project = await self.get_project(name)
         if not project:
-            raise ValueError(f"Project '{name}' not found")
+            raise ValueError(f"Project '{name}' not found")  # pragma: no cover
 
         project_path = project.path
 
         # Check if project is default (in cloud mode, check database; in local mode, check config)
         if project.is_default or name == self.config_manager.config.default_project:
-            raise ValueError(f"Cannot remove the default project '{name}'")
+            raise ValueError(f"Cannot remove the default project '{name}'")  # pragma: no cover
 
         # Remove from config if it exists there (may not exist in cloud mode)
         try:
             self.config_manager.remove_project(name)
-        except ValueError:
+        except ValueError:  # pragma: no cover
             # Project not in config - that's OK in cloud mode, continue with database deletion
-            logger.debug(f"Project '{name}' not found in config, removing from database only")
+            logger.debug(  # pragma: no cover
+                f"Project '{name}' not found in config, removing from database only"
+            )
 
         # Remove from database
         await self.repository.delete(project.id)
@@ -265,11 +267,13 @@ class ProjectService:
                     await asyncio.to_thread(shutil.rmtree, project_path)
                     logger.info(f"Deleted project directory: {project_path}")
                 else:
-                    logger.warning(
+                    logger.warning(  # pragma: no cover
                         f"Project directory not found or not a directory: {project_path}"
-                    )
-            except Exception as e:
-                logger.warning(f"Failed to delete project directory {project_path}: {e}")
+                    )  # pragma: no cover
+            except Exception as e:  # pragma: no cover
+                logger.warning(  # pragma: no cover
+                    f"Failed to delete project directory {project_path}: {e}"
+                )
 
     async def set_default_project(self, name: str) -> None:
         """Set the default project in configuration and database.
@@ -283,15 +287,17 @@ class ProjectService:
         if not self.repository:  # pragma: no cover
             raise ValueError("Repository is required for set_default_project")
 
-        # First update config file (this will validate the project exists)
-        self.config_manager.set_default_project(name)
-
-        # Then update database using the same lookup logic as get_project
+        # Look up project in database first to validate it exists
         project = await self.get_project(name)
-        if project:
-            await self.repository.set_as_default(project.id)
-        else:
-            logger.error(f"Project '{name}' exists in config but not in database")
+        if not project:
+            raise ValueError(f"Project '{name}' not found")
+
+        # Update database
+        await self.repository.set_as_default(project.id)
+
+        # Update config file only in local mode (cloud mode uses database only)
+        if not self.config_manager.config.cloud_mode:
+            self.config_manager.set_default_project(name)
 
         logger.info(f"Project '{name}' set as default in configuration and database")
 
@@ -430,8 +436,8 @@ class ProjectService:
         Raises:
             ValueError: If the project doesn't exist or repository isn't initialized
         """
-        if not self.repository:
-            raise ValueError("Repository is required for move_project")
+        if not self.repository:  # pragma: no cover
+            raise ValueError("Repository is required for move_project")  # pragma: no cover
 
         # Resolve to absolute path
         resolved_path = Path(os.path.abspath(os.path.expanduser(new_path))).as_posix()
@@ -766,25 +772,42 @@ class ProjectService:
         )
 
         # Query for monthly entity creation (project filtered)
+        # Use different date formatting for SQLite vs Postgres
+        from basic_memory.config import DatabaseBackend
+
+        is_postgres = self.config_manager.config.database_backend == DatabaseBackend.POSTGRES
+        date_format = (
+            "to_char(created_at, 'YYYY-MM')" if is_postgres else "strftime('%Y-%m', created_at)"
+        )
+
+        # Postgres needs datetime objects, SQLite needs ISO strings
+        six_months_param = six_months_ago if is_postgres else six_months_ago.isoformat()
+
         entity_growth_result = await self.repository.execute_query(
-            text("""
-            SELECT 
-                strftime('%Y-%m', created_at) AS month,
+            text(f"""
+            SELECT
+                {date_format} AS month,
                 COUNT(*) AS count
             FROM entity
             WHERE created_at >= :six_months_ago AND project_id = :project_id
             GROUP BY month
             ORDER BY month
         """),
-            {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
+            {"six_months_ago": six_months_param, "project_id": project_id},
         )
         entity_growth = {row[0]: row[1] for row in entity_growth_result.fetchall()}
 
         # Query for monthly observation creation (project filtered)
+        date_format_entity = (
+            "to_char(entity.created_at, 'YYYY-MM')"
+            if is_postgres
+            else "strftime('%Y-%m', entity.created_at)"
+        )
+
         observation_growth_result = await self.repository.execute_query(
-            text("""
-            SELECT 
-                strftime('%Y-%m', entity.created_at) AS month,
+            text(f"""
+            SELECT
+                {date_format_entity} AS month,
                 COUNT(*) AS count
             FROM observation
             INNER JOIN entity ON observation.entity_id = entity.id
@@ -792,15 +815,15 @@ class ProjectService:
             GROUP BY month
             ORDER BY month
         """),
-            {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
+            {"six_months_ago": six_months_param, "project_id": project_id},
         )
         observation_growth = {row[0]: row[1] for row in observation_growth_result.fetchall()}
 
         # Query for monthly relation creation (project filtered)
         relation_growth_result = await self.repository.execute_query(
-            text("""
-            SELECT 
-                strftime('%Y-%m', entity.created_at) AS month,
+            text(f"""
+            SELECT
+                {date_format_entity} AS month,
                 COUNT(*) AS count
             FROM relation
             INNER JOIN entity ON relation.from_id = entity.id
@@ -808,7 +831,7 @@ class ProjectService:
             GROUP BY month
             ORDER BY month
         """),
-            {"six_months_ago": six_months_ago.isoformat(), "project_id": project_id},
+            {"six_months_ago": six_months_param, "project_id": project_id},
         )
         relation_growth = {row[0]: row[1] for row in relation_growth_result.fetchall()}
 
@@ -849,8 +872,10 @@ class ProjectService:
         watch_status = None
         watch_status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
         if watch_status_path.exists():
-            try:
-                watch_status = json.loads(watch_status_path.read_text(encoding="utf-8"))
+            try:  # pragma: no cover
+                watch_status = json.loads(  # pragma: no cover
+                    watch_status_path.read_text(encoding="utf-8")
+                )
             except Exception:  # pragma: no cover
                 pass