basic-memory 0.14.3__py3-none-any.whl → 0.15.0__py3-none-any.whl

basic_memory/services/initialization.py

@@ -11,7 +11,10 @@ from loguru import logger
 
 from basic_memory import db
 from basic_memory.config import BasicMemoryConfig
-from basic_memory.repository import ProjectRepository
+from basic_memory.models import Project
+from basic_memory.repository import (
+    ProjectRepository,
+)
 
 
 async def initialize_database(app_config: BasicMemoryConfig) -> None:
@@ -101,18 +104,20 @@ async def initialize_file_sync(
     # Get active projects
     active_projects = await project_repository.get_active_projects()
 
-    # First, sync all projects sequentially
-    for project in active_projects:
+    # Start sync for all projects as background tasks (non-blocking)
+    async def sync_project_background(project: Project):
+        """Sync a single project in the background."""
         # avoid circular imports
-        from basic_memory.cli.commands.sync import get_sync_service
-
-        logger.info(f"Starting sync for project: {project.name}")
-        sync_service = await get_sync_service(project)
-        sync_dir = Path(project.path)
+        from basic_memory.sync.sync_service import get_sync_service
 
+        logger.info(f"Starting background sync for project: {project.name}")
         try:
+            # Create sync service
+            sync_service = await get_sync_service(project)
+
+            sync_dir = Path(project.path)
             await sync_service.sync(sync_dir, project_name=project.name)
-            logger.info(f"Sync completed successfully for project: {project.name}")
+            logger.info(f"Background sync completed successfully for project: {project.name}")
 
             # Mark project as watching for changes after successful sync
             from basic_memory.services.sync_status_service import sync_status_tracker
@@ -120,12 +125,19 @@ async def initialize_file_sync(
             sync_status_tracker.start_project_watch(project.name)
             logger.info(f"Project {project.name} is now watching for changes")
         except Exception as e:  # pragma: no cover
-            logger.error(f"Error syncing project {project.name}: {e}")
+            logger.error(f"Error in background sync for project {project.name}: {e}")
             # Mark sync as failed for this project
             from basic_memory.services.sync_status_service import sync_status_tracker
 
             sync_status_tracker.fail_project_sync(project.name, str(e))
-            # Continue with other projects even if one fails
+
+    # Create background tasks for all project syncs (non-blocking)
+    sync_tasks = [
+        asyncio.create_task(sync_project_background(project)) for project in active_projects
+    ]
+    logger.info(f"Created {len(sync_tasks)} background sync tasks")
+
+    # Don't await the tasks - let them run in background while we continue
 
     # Then start the watch service in the background
     logger.info("Starting watch service for all projects")
@@ -169,9 +181,16 @@ def ensure_initialization(app_config: BasicMemoryConfig) -> None:
     This is a wrapper for the async initialize_app function that can be
     called from synchronous code like CLI entry points.
 
+    No-op if app_config.cloud_mode == True. Cloud basic memory manages it's own projects
+
     Args:
         app_config: The Basic Memory project configuration
     """
+    # Skip initialization in cloud mode - cloud manages its own projects
+    if app_config.cloud_mode_enabled:
+        logger.debug("Skipping initialization in cloud mode - projects managed by cloud")
+        return
+
     try:
         result = asyncio.run(initialize_app(app_config))
         logger.info(f"Initialization completed successfully: result={result}")

basic_memory/services/project_service.py

@@ -21,6 +21,9 @@ from basic_memory.config import WATCH_STATUS_JSON, ConfigManager, get_project_co
 from basic_memory.utils import generate_permalink
 
 
+config = ConfigManager().config
+
+
 class ProjectService:
     """Service for managing Basic Memory projects."""
 
@@ -96,11 +99,16 @@ class ProjectService:
         Raises:
             ValueError: If the project already exists
         """
-        if not self.repository:  # pragma: no cover
-            raise ValueError("Repository is required for add_project")
-
-        # Resolve to absolute path
-        resolved_path = os.path.abspath(os.path.expanduser(path))
+        # in cloud mode, don't allow arbitrary paths.
+        if config.cloud_mode:
+            basic_memory_home = os.getenv("BASIC_MEMORY_HOME")
+            assert basic_memory_home is not None
+            base_path = Path(basic_memory_home)
+
+            # Resolve to absolute path
+            resolved_path = Path(os.path.abspath(os.path.expanduser(base_path / path))).as_posix()
+        else:
+            resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix()
 
         # First add to config file (this will validate the project doesn't exist)
         project_config = self.config_manager.add_project(name, resolved_path)
@@ -139,8 +147,8 @@ class ProjectService:
         # First remove from config (this will validate the project exists and is not default)
         self.config_manager.remove_project(name)
 
-        # Then remove from database
-        project = await self.repository.get_by_name(name)
+        # Then remove from database using robust lookup
+        project = await self.get_project(name)
         if project:
             await self.repository.delete(project.id)
 
@@ -161,8 +169,8 @@ class ProjectService:
         # First update config file (this will validate the project exists)
         self.config_manager.set_default_project(name)
 
-        # Then update database
-        project = await self.repository.get_by_name(name)
+        # Then update database using the same lookup logic as get_project
+        project = await self.get_project(name)
         if project:
             await self.repository.set_as_default(project.id)
         else:
@@ -170,15 +178,6 @@ class ProjectService:
 
         logger.info(f"Project '{name}' set as default in configuration and database")
 
-        # Refresh MCP session to pick up the new default project
-        try:
-            from basic_memory.mcp.project_session import session
-
-            session.refresh_from_config()
-        except ImportError:  # pragma: no cover
-            # MCP components might not be available in all contexts (e.g., CLI-only usage)
-            logger.debug("MCP session not available, skipping session refresh")
-
     async def _ensure_single_default_project(self) -> None:
         """Ensure only one project has is_default=True.
 
@@ -300,15 +299,6 @@ class ProjectService:
 
         logger.info("Project synchronization complete")
 
-        # Refresh MCP session to ensure it's in sync with current config
-        try:
-            from basic_memory.mcp.project_session import session
-
-            session.refresh_from_config()
-        except ImportError:
-            # MCP components might not be available in all contexts
-            logger.debug("MCP session not available, skipping session refresh")
-
     async def move_project(self, name: str, new_path: str) -> None:
         """Move a project to a new location.
 
@@ -323,7 +313,7 @@ class ProjectService:
             raise ValueError("Repository is required for move_project")
 
         # Resolve to absolute path
-        resolved_path = os.path.abspath(os.path.expanduser(new_path))
+        resolved_path = Path(os.path.abspath(os.path.expanduser(new_path))).as_posix()
 
         # Validate project exists in config
         if name not in self.config_manager.projects:
@@ -338,8 +328,8 @@ class ProjectService:
         config.projects[name] = resolved_path
         self.config_manager.save_config(config)
 
-        # Update in database
-        project = await self.repository.get_by_name(name)
+        # Update in database using robust lookup
+        project = await self.get_project(name)
         if project:
             await self.repository.update_path(project.id, resolved_path)
             logger.info(f"Moved project '{name}' from {old_path} to {resolved_path}")
@@ -370,15 +360,15 @@ class ProjectService:
         if name not in self.config_manager.projects:
             raise ValueError(f"Project '{name}' not found in configuration")
 
-        # Get project from database
-        project = await self.repository.get_by_name(name)
+        # Get project from database using robust lookup
+        project = await self.get_project(name)
         if not project:
             logger.error(f"Project '{name}' exists in config but not in database")
             return
 
         # Update path if provided
         if updated_path:
-            resolved_path = os.path.abspath(os.path.expanduser(updated_path))
+            resolved_path = Path(os.path.abspath(os.path.expanduser(updated_path))).as_posix()
 
             # Update in config
             config = self.config_manager.load_config()

basic_memory/sync/sync_service.py

@@ -1,7 +1,9 @@
 """Service for syncing files between filesystem and database."""
 
+import asyncio
 import os
 import time
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
@@ -10,12 +12,16 @@ from typing import Dict, Optional, Set, Tuple
 from loguru import logger
 from sqlalchemy.exc import IntegrityError
 
-from basic_memory.config import BasicMemoryConfig
+from basic_memory import db
+from basic_memory.config import BasicMemoryConfig, ConfigManager
 from basic_memory.file_utils import has_frontmatter
-from basic_memory.markdown import EntityParser
-from basic_memory.models import Entity
-from basic_memory.repository import EntityRepository, RelationRepository
+from basic_memory.ignore_utils import load_bmignore_patterns, should_ignore_path
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from basic_memory.models import Entity, Project
+from basic_memory.repository import EntityRepository, RelationRepository, ObservationRepository
+from basic_memory.repository.search_repository import SearchRepository
 from basic_memory.services import EntityService, FileService
+from basic_memory.services.link_resolver import LinkResolver
 from basic_memory.services.search_service import SearchService
 from basic_memory.services.sync_status_service import sync_status_tracker, SyncStatus
 
@@ -80,6 +86,43 @@ class SyncService:
         self.relation_repository = relation_repository
         self.search_service = search_service
         self.file_service = file_service
+        self._thread_pool = ThreadPoolExecutor(max_workers=app_config.sync_thread_pool_size)
+        # Load ignore patterns once at initialization for performance
+        self._ignore_patterns = load_bmignore_patterns()
+
+    async def _read_file_async(self, file_path: Path) -> str:
+        """Read file content in thread pool to avoid blocking the event loop."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(self._thread_pool, file_path.read_text, "utf-8")
+
+    async def _compute_checksum_async(self, path: str) -> str:
+        """Compute file checksum in thread pool to avoid blocking the event loop."""
+
+        def _sync_compute_checksum(path_str: str) -> str:
+            # Synchronous version for thread pool execution
+            path_obj = self.file_service.base_path / path_str
+
+            if self.file_service.is_markdown(path_str):
+                content = path_obj.read_text(encoding="utf-8")
+            else:
+                content = path_obj.read_bytes()
+
+            # Use the synchronous version of compute_checksum
+            import hashlib
+
+            if isinstance(content, str):
+                content_bytes = content.encode("utf-8")
+            else:
+                content_bytes = content
+            return hashlib.sha256(content_bytes).hexdigest()
+
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(self._thread_pool, _sync_compute_checksum, path)
+
+    def __del__(self):
+        """Cleanup thread pool when service is destroyed."""
+        if hasattr(self, "_thread_pool"):
+            self._thread_pool.shutdown(wait=False)
 
     async def sync(self, directory: Path, project_name: Optional[str] = None) -> SyncReport:
         """Sync all files with database."""
@@ -289,14 +332,14 @@ class SyncService:
         logger.debug(f"Parsing markdown file, path: {path}, new: {new}")
 
         file_path = self.entity_parser.base_path / path
-        file_content = file_path.read_text(encoding="utf-8")
+        file_content = await self._read_file_async(file_path)
         file_contains_frontmatter = has_frontmatter(file_content)
 
         # entity markdown will always contain front matter, so it can be used up create/update the entity
         entity_markdown = await self.entity_parser.parse_file(path)
 
-        # if the file contains frontmatter, resolve a permalink
-        if file_contains_frontmatter:
+        # if the file contains frontmatter, resolve a permalink (unless disabled)
+        if file_contains_frontmatter and not self.app_config.disable_permalinks:
             # Resolve permalink - this handles all the cases including conflicts
             permalink = await self.entity_service.resolve_permalink(path, markdown=entity_markdown)
 
@@ -326,7 +369,7 @@ class SyncService:
         # After updating relations, we need to compute the checksum again
         # This is necessary for files with wikilinks to ensure consistent checksums
         # after relation processing is complete
-        final_checksum = await self.file_service.compute_checksum(path)
+        final_checksum = await self._compute_checksum_async(path)
 
         # set checksum
         await self.entity_repository.update(entity.id, {"checksum": final_checksum})
@@ -350,15 +393,15 @@ class SyncService:
         Returns:
             Tuple of (entity, checksum)
         """
-        checksum = await self.file_service.compute_checksum(path)
+        checksum = await self._compute_checksum_async(path)
         if new:
             # Generate permalink from path
             await self.entity_service.resolve_permalink(path)
 
             # get file timestamps
             file_stats = self.file_service.file_stats(path)
-            created = datetime.fromtimestamp(file_stats.st_ctime)
-            modified = datetime.fromtimestamp(file_stats.st_mtime)
+            created = datetime.fromtimestamp(file_stats.st_ctime).astimezone()
+            modified = datetime.fromtimestamp(file_stats.st_mtime).astimezone()
 
             # get mime type
             content_type = self.file_service.content_type(path)
@@ -487,8 +530,10 @@ class SyncService:
             updates = {"file_path": new_path}
 
             # If configured, also update permalink to match new path
-            if self.app_config.update_permalinks_on_move and self.file_service.is_markdown(
-                new_path
+            if (
+                self.app_config.update_permalinks_on_move
+                and not self.app_config.disable_permalinks
+                and self.file_service.is_markdown(new_path)
             ):
                 # generate new permalink value
                 new_permalink = await self.entity_service.resolve_permalink(new_path)
@@ -548,12 +593,27 @@ class SyncService:
             # update search index
             await self.search_service.index_entity(updated)
 
-    async def resolve_relations(self):
-        """Try to resolve any unresolved relations"""
+    async def resolve_relations(self, entity_id: int | None = None):
+        """Try to resolve unresolved relations.
 
-        unresolved_relations = await self.relation_repository.find_unresolved_relations()
+        Args:
+            entity_id: If provided, only resolve relations for this specific entity.
+                      Otherwise, resolve all unresolved relations in the database.
+        """
 
-        logger.info("Resolving forward references", count=len(unresolved_relations))
+        if entity_id:
+            # Only get unresolved relations for the specific entity
+            unresolved_relations = (
+                await self.relation_repository.find_unresolved_relations_for_entity(entity_id)
+            )
+            logger.info(
+                f"Resolving forward references for entity {entity_id}",
+                count=len(unresolved_relations),
+            )
+        else:
+            # Get all unresolved relations (original behavior)
+            unresolved_relations = await self.relation_repository.find_unresolved_relations()
+            logger.info("Resolving all forward references", count=len(unresolved_relations))
 
         for relation in unresolved_relations:
             logger.trace(
@@ -608,19 +668,35 @@ class SyncService:
 
         logger.debug(f"Scanning directory {directory}")
         result = ScanResult()
+        ignored_count = 0
 
         for root, dirnames, filenames in os.walk(str(directory)):
-            # Skip dot directories in-place
-            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+            # Convert root to Path for easier manipulation
+            root_path = Path(root)
+
+            # Filter out ignored directories in-place
+            dirnames_to_remove = []
+            for dirname in dirnames:
+                dir_path = root_path / dirname
+                if should_ignore_path(dir_path, directory, self._ignore_patterns):
+                    dirnames_to_remove.append(dirname)
+                    ignored_count += 1
+
+            # Remove ignored directories from dirnames to prevent os.walk from descending
+            for dirname in dirnames_to_remove:
+                dirnames.remove(dirname)
 
             for filename in filenames:
-                # Skip dot files
-                if filename.startswith("."):
+                path = root_path / filename
+
+                # Check if file should be ignored
+                if should_ignore_path(path, directory, self._ignore_patterns):
+                    ignored_count += 1
+                    logger.trace(f"Ignoring file per .bmignore: {path.relative_to(directory)}")
                     continue
 
-                path = Path(root) / filename
-                rel_path = str(path.relative_to(directory))
-                checksum = await self.file_service.compute_checksum(rel_path)
+                rel_path = path.relative_to(directory).as_posix()
+                checksum = await self._compute_checksum_async(rel_path)
                 result.files[rel_path] = checksum
                 result.checksums[checksum] = rel_path
 
@@ -631,7 +707,55 @@ class SyncService:
             f"{directory} scan completed "
             f"directory={str(directory)} "
             f"files_found={len(result.files)} "
+            f"files_ignored={ignored_count} "
             f"duration_ms={duration_ms}"
         )
 
         return result
+
+
+async def get_sync_service(project: Project) -> SyncService:  # pragma: no cover
+    """Get sync service instance with all dependencies."""
+
+    app_config = ConfigManager().config
+    _, session_maker = await db.get_or_create_db(
+        db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM
+    )
+
+    project_path = Path(project.path)
+    entity_parser = EntityParser(project_path)
+    markdown_processor = MarkdownProcessor(entity_parser)
+    file_service = FileService(project_path, markdown_processor)
+
+    # Initialize repositories
+    entity_repository = EntityRepository(session_maker, project_id=project.id)
+    observation_repository = ObservationRepository(session_maker, project_id=project.id)
+    relation_repository = RelationRepository(session_maker, project_id=project.id)
+    search_repository = SearchRepository(session_maker, project_id=project.id)
+
+    # Initialize services
+    search_service = SearchService(search_repository, entity_repository, file_service)
+    link_resolver = LinkResolver(entity_repository, search_service)
+
+    # Initialize services
+    entity_service = EntityService(
+        entity_parser,
+        entity_repository,
+        observation_repository,
+        relation_repository,
+        file_service,
+        link_resolver,
+    )
+
+    # Create sync service
+    sync_service = SyncService(
+        app_config=app_config,
+        entity_service=entity_service,
+        entity_parser=entity_parser,
+        entity_repository=entity_repository,
+        relation_repository=relation_repository,
+        search_service=search_service,
+        file_service=file_service,
+    )
+
+    return sync_service