basic-memory 0.14.4__py3-none-any.whl → 0.15.1__py3-none-any.whl

basic_memory/services/directory_service.py

@@ -3,8 +3,9 @@
 import fnmatch
 import logging
 import os
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Sequence
 
+from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
 from basic_memory.schemas.directory import DirectoryNode
 
@@ -89,6 +90,49 @@ class DirectoryService:
         # Return the root node with its children
         return root_node
 
+    async def get_directory_structure(self) -> DirectoryNode:
+        """Build a hierarchical directory structure without file details.
+
+        Optimized method for folder navigation that only returns directory nodes,
+        no file metadata. Much faster than get_directory_tree() for large knowledge bases.
+
+        Returns:
+            DirectoryNode tree containing only folders (type="directory")
+        """
+        # Get unique directories without loading entities
+        directories = await self.entity_repository.get_distinct_directories()
+
+        # Create a root directory node
+        root_node = DirectoryNode(name="Root", directory_path="/", type="directory")
+
+        # Map to store directory nodes by path for easy lookup
+        dir_map: Dict[str, DirectoryNode] = {"/": root_node}
+
+        # Build tree with just folders
+        for dir_path in directories:
+            parts = [p for p in dir_path.split("/") if p]
+            current_path = "/"
+
+            for i, part in enumerate(parts):
+                parent_path = current_path
+                # Build the directory path
+                current_path = (
+                    f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
+                )
+
+                # Create directory node if it doesn't exist
+                if current_path not in dir_map:
+                    dir_node = DirectoryNode(
+                        name=part, directory_path=current_path, type="directory"
+                    )
+                    dir_map[current_path] = dir_node
+
+                    # Add to parent's children
+                    if parent_path in dir_map:
+                        dir_map[parent_path].children.append(dir_node)
+
+        return root_node
+
     async def list_directory(
         self,
         dir_name: str = "/",
@@ -118,8 +162,13 @@ class DirectoryService:
         if dir_name != "/" and dir_name.endswith("/"):
             dir_name = dir_name.rstrip("/")
 
-        # Get the full directory tree
-        root_tree = await self.get_directory_tree()
+        # Optimize: Query only entities in the target directory
+        # instead of loading the entire tree
+        dir_prefix = dir_name.lstrip("/")
+        entity_rows = await self.entity_repository.find_by_directory_prefix(dir_prefix)
+
+        # Build a partial tree from only the relevant entities
+        root_tree = self._build_directory_tree_from_entities(entity_rows, dir_name)
 
         # Find the target directory node
         target_node = self._find_directory_node(root_tree, dir_name)
@@ -132,6 +181,78 @@ class DirectoryService:
 
         return result
 
+    def _build_directory_tree_from_entities(
+        self, entity_rows: Sequence[Entity], root_path: str
+    ) -> DirectoryNode:
+        """Build a directory tree from a subset of entities.
+
+        Args:
+            entity_rows: Sequence of entity objects to build tree from
+            root_path: Root directory path for the tree
+
+        Returns:
+            DirectoryNode representing the tree root
+        """
+        # Create a root directory node
+        root_node = DirectoryNode(name="Root", directory_path=root_path, type="directory")
+
+        # Map to store directory nodes by path for easy lookup
+        dir_map: Dict[str, DirectoryNode] = {root_path: root_node}
+
+        # First pass: create all directory nodes
+        for file in entity_rows:
+            # Process directory path components
+            parts = [p for p in file.file_path.split("/") if p]
+
+            # Create directory structure
+            current_path = "/"
+            for i, part in enumerate(parts[:-1]):  # Skip the filename
+                parent_path = current_path
+                # Build the directory path
+                current_path = (
+                    f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
+                )
+
+                # Create directory node if it doesn't exist
+                if current_path not in dir_map:
+                    dir_node = DirectoryNode(
+                        name=part, directory_path=current_path, type="directory"
+                    )
+                    dir_map[current_path] = dir_node
+
+                    # Add to parent's children
+                    if parent_path in dir_map:
+                        dir_map[parent_path].children.append(dir_node)
+
+        # Second pass: add file nodes to their parent directories
+        for file in entity_rows:
+            file_name = os.path.basename(file.file_path)
+            parent_dir = os.path.dirname(file.file_path)
+            directory_path = "/" if parent_dir == "" else f"/{parent_dir}"
+
+            # Create file node
+            file_node = DirectoryNode(
+                name=file_name,
+                file_path=file.file_path,
+                directory_path=f"/{file.file_path}",
+                type="file",
+                title=file.title,
+                permalink=file.permalink,
+                entity_id=file.id,
+                entity_type=file.entity_type,
+                content_type=file.content_type,
+                updated_at=file.updated_at,
+            )
+
+            # Add to parent directory's children
+            if directory_path in dir_map:
+                dir_map[directory_path].children.append(file_node)
+            elif root_path in dir_map:
+                # Fallback to root if parent not found
+                dir_map[root_path].children.append(file_node)
+
+        return root_node
+
     def _find_directory_node(
         self, root: DirectoryNode, target_path: str
     ) -> Optional[DirectoryNode]:

basic_memory/services/entity_service.py

@@ -9,7 +9,12 @@ from loguru import logger
 from sqlalchemy.exc import IntegrityError
 
 from basic_memory.config import ProjectConfig, BasicMemoryConfig
-from basic_memory.file_utils import has_frontmatter, parse_frontmatter, remove_frontmatter, dump_frontmatter
+from basic_memory.file_utils import (
+    has_frontmatter,
+    parse_frontmatter,
+    remove_frontmatter,
+    dump_frontmatter,
+)
 from basic_memory.markdown import EntityMarkdown
 from basic_memory.markdown.entity_parser import EntityParser
 from basic_memory.markdown.utils import entity_model_from_markdown, schema_to_markdown
@@ -37,6 +42,7 @@ class EntityService(BaseService[EntityModel]):
         relation_repository: RelationRepository,
         file_service: FileService,
         link_resolver: LinkResolver,
+        app_config: Optional[BasicMemoryConfig] = None,
     ):
         super().__init__(entity_repository)
         self.observation_repository = observation_repository
@@ -44,8 +50,11 @@ class EntityService(BaseService[EntityModel]):
         self.entity_parser = entity_parser
         self.file_service = file_service
         self.link_resolver = link_resolver
+        self.app_config = app_config
 
-    async def detect_file_path_conflicts(self, file_path: str) -> List[Entity]:
+    async def detect_file_path_conflicts(
+        self, file_path: str, skip_check: bool = False
+    ) -> List[Entity]:
         """Detect potential file path conflicts for a given file path.
 
         This checks for entities with similar file paths that might cause conflicts:
@@ -56,10 +65,14 @@ class EntityService(BaseService[EntityModel]):
 
         Args:
             file_path: The file path to check for conflicts
+            skip_check: If True, skip the check and return empty list (optimization for bulk operations)
 
         Returns:
             List of entities that might conflict with the given file path
         """
+        if skip_check:
+            return []
+
         from basic_memory.utils import detect_potential_file_conflicts
 
         conflicts = []
@@ -79,7 +92,10 @@ class EntityService(BaseService[EntityModel]):
         return conflicts
 
     async def resolve_permalink(
-        self, file_path: Permalink | Path, markdown: Optional[EntityMarkdown] = None
+        self,
+        file_path: Permalink | Path,
+        markdown: Optional[EntityMarkdown] = None,
+        skip_conflict_check: bool = False,
     ) -> str:
         """Get or generate unique permalink for an entity.
 
@@ -94,7 +110,9 @@ class EntityService(BaseService[EntityModel]):
         file_path_str = Path(file_path).as_posix()
 
         # Check for potential file path conflicts before resolving permalink
-        conflicts = await self.detect_file_path_conflicts(file_path_str)
+        conflicts = await self.detect_file_path_conflicts(
+            file_path_str, skip_check=skip_conflict_check
+        )
         if conflicts:
             logger.warning(
                 f"Detected potential file path conflicts for '{file_path_str}': "
@@ -139,10 +157,11 @@ class EntityService(BaseService[EntityModel]):
             f"Creating or updating entity: {schema.file_path}, permalink: {schema.permalink}"
         )
 
-        # Try to find existing entity using smart resolution
-        existing = await self.link_resolver.resolve_link(
-            schema.file_path
-        ) or await self.link_resolver.resolve_link(schema.permalink)
+        # Try to find existing entity using strict resolution (no fuzzy search)
+        # This prevents incorrectly matching similar file paths like "Node A.md" and "Node C.md"
+        existing = await self.link_resolver.resolve_link(schema.file_path, strict=True)
+        if not existing and schema.permalink:
+            existing = await self.link_resolver.resolve_link(schema.permalink, strict=True)
 
         if existing:
             logger.debug(f"Found existing entity: {existing.file_path}")
@@ -189,9 +208,15 @@ class EntityService(BaseService[EntityModel]):
                     relations=[],
                 )
 
-        # Get unique permalink (prioritizing content frontmatter)
-        permalink = await self.resolve_permalink(file_path, content_markdown)
-        schema._permalink = permalink
+        # Get unique permalink (prioritizing content frontmatter) unless disabled
+        if self.app_config and self.app_config.disable_permalinks:
+            # Use empty string as sentinel to indicate permalinks are disabled
+            # The permalink property will return None when it sees empty string
+            schema._permalink = ""
+        else:
+            # Generate and set permalink
+            permalink = await self.resolve_permalink(file_path, content_markdown)
+            schema._permalink = permalink
 
         post = await schema_to_markdown(schema)
 
@@ -249,15 +274,16 @@ class EntityService(BaseService[EntityModel]):
                     relations=[],
                 )
 
-        # Check if we need to update the permalink based on content frontmatter
+        # Check if we need to update the permalink based on content frontmatter (unless disabled)
         new_permalink = entity.permalink  # Default to existing
-        if content_markdown and content_markdown.frontmatter.permalink:
-            # Resolve permalink with the new content frontmatter
-            resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
-            if resolved_permalink != entity.permalink:
-                new_permalink = resolved_permalink
-                # Update the schema to use the new permalink
-                schema._permalink = new_permalink
+        if self.app_config and not self.app_config.disable_permalinks:
+            if content_markdown and content_markdown.frontmatter.permalink:
+                # Resolve permalink with the new content frontmatter
+                resolved_permalink = await self.resolve_permalink(file_path, content_markdown)
+                if resolved_permalink != entity.permalink:
+                    new_permalink = resolved_permalink
+                    # Update the schema to use the new permalink
+                    schema._permalink = new_permalink
 
         # Create post with new content from schema
         post = await schema_to_markdown(schema)
@@ -417,34 +443,58 @@ class EntityService(BaseService[EntityModel]):
         # Clear existing relations first
         await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
 
-        # Process each relation
-        for rel in markdown.relations:
-            # Resolve the target permalink
-            target_entity = await self.link_resolver.resolve_link(
-                rel.target,
-            )
-
-            # if the target is found, store the id
-            target_id = target_entity.id if target_entity else None
-            # if the target is found, store the title, otherwise add the target for a "forward link"
-            target_name = target_entity.title if target_entity else rel.target
-
-            # Create the relation
-            relation = Relation(
-                from_id=db_entity.id,
-                to_id=target_id,
-                to_name=target_name,
-                relation_type=rel.type,
-                context=rel.context,
-            )
-            try:
-                await self.relation_repository.add(relation)
-            except IntegrityError:
-                # Unique constraint violation - relation already exists
-                logger.debug(
-                    f"Skipping duplicate relation {rel.type} from {db_entity.permalink} target: {rel.target}"
+        # Batch resolve all relation targets in parallel
+        if markdown.relations:
+            import asyncio
+
+            # Create tasks for all relation lookups
+            lookup_tasks = [
+                self.link_resolver.resolve_link(rel.target) for rel in markdown.relations
+            ]
+
+            # Execute all lookups in parallel
+            resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
+
+            # Process results and create relation records
+            relations_to_add = []
+            for rel, resolved in zip(markdown.relations, resolved_entities):
+                # Handle exceptions from gather and None results
+                target_entity: Optional[Entity] = None
+                if not isinstance(resolved, Exception):
+                    # Type narrowing: resolved is Optional[Entity] here, not Exception
+                    target_entity = resolved  # type: ignore
+
+                # if the target is found, store the id
+                target_id = target_entity.id if target_entity else None
+                # if the target is found, store the title, otherwise add the target for a "forward link"
+                target_name = target_entity.title if target_entity else rel.target
+
+                # Create the relation
+                relation = Relation(
+                    from_id=db_entity.id,
+                    to_id=target_id,
+                    to_name=target_name,
+                    relation_type=rel.type,
+                    context=rel.context,
                 )
-                continue
+                relations_to_add.append(relation)
+
+            # Batch insert all relations
+            if relations_to_add:
+                try:
+                    await self.relation_repository.add_all(relations_to_add)
+                except IntegrityError:
+                    # Some relations might be duplicates - fall back to individual inserts
+                    logger.debug("Batch relation insert failed, trying individual inserts")
+                    for relation in relations_to_add:
+                        try:
+                            await self.relation_repository.add(relation)
+                        except IntegrityError:
+                            # Unique constraint violation - relation already exists
+                            logger.debug(
+                                f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
+                            )
+                            continue
 
         return await self.repository.get_by_file_path(path)
 
@@ -728,8 +778,10 @@ class EntityService(BaseService[EntityModel]):
             # 6. Prepare database updates
             updates = {"file_path": destination_path}
 
-            # 7. Update permalink if configured or if entity has null permalink
-            if app_config.update_permalinks_on_move or old_permalink is None:
+            # 7. Update permalink if configured or if entity has null permalink (unless disabled)
+            if not app_config.disable_permalinks and (
+                app_config.update_permalinks_on_move or old_permalink is None
+            ):
                 # Generate new permalink from destination path
                 new_permalink = await self.resolve_permalink(destination_path)
 

basic_memory/services/initialization.py

@@ -11,7 +11,10 @@ from loguru import logger
 
 from basic_memory import db
 from basic_memory.config import BasicMemoryConfig
-from basic_memory.repository import ProjectRepository
+from basic_memory.models import Project
+from basic_memory.repository import (
+    ProjectRepository,
+)
 
 
 async def initialize_database(app_config: BasicMemoryConfig) -> None:
@@ -101,18 +104,20 @@ async def initialize_file_sync(
     # Get active projects
     active_projects = await project_repository.get_active_projects()
 
-    # First, sync all projects sequentially
-    for project in active_projects:
+    # Start sync for all projects as background tasks (non-blocking)
+    async def sync_project_background(project: Project):
+        """Sync a single project in the background."""
         # avoid circular imports
-        from basic_memory.cli.commands.sync import get_sync_service
-
-        logger.info(f"Starting sync for project: {project.name}")
-        sync_service = await get_sync_service(project)
-        sync_dir = Path(project.path)
+        from basic_memory.sync.sync_service import get_sync_service
 
+        logger.info(f"Starting background sync for project: {project.name}")
         try:
+            # Create sync service
+            sync_service = await get_sync_service(project)
+
+            sync_dir = Path(project.path)
             await sync_service.sync(sync_dir, project_name=project.name)
-            logger.info(f"Sync completed successfully for project: {project.name}")
+            logger.info(f"Background sync completed successfully for project: {project.name}")
 
             # Mark project as watching for changes after successful sync
             from basic_memory.services.sync_status_service import sync_status_tracker
@@ -120,12 +125,19 @@ async def initialize_file_sync(
             sync_status_tracker.start_project_watch(project.name)
             logger.info(f"Project {project.name} is now watching for changes")
         except Exception as e:  # pragma: no cover
-            logger.error(f"Error syncing project {project.name}: {e}")
+            logger.error(f"Error in background sync for project {project.name}: {e}")
             # Mark sync as failed for this project
             from basic_memory.services.sync_status_service import sync_status_tracker
 
             sync_status_tracker.fail_project_sync(project.name, str(e))
-            # Continue with other projects even if one fails
+
+    # Create background tasks for all project syncs (non-blocking)
+    sync_tasks = [
+        asyncio.create_task(sync_project_background(project)) for project in active_projects
+    ]
+    logger.info(f"Created {len(sync_tasks)} background sync tasks")
+
+    # Don't await the tasks - let them run in background while we continue
 
     # Then start the watch service in the background
     logger.info("Starting watch service for all projects")
@@ -169,9 +181,16 @@ def ensure_initialization(app_config: BasicMemoryConfig) -> None:
     This is a wrapper for the async initialize_app function that can be
     called from synchronous code like CLI entry points.
 
+    No-op if app_config.cloud_mode == True. Cloud basic memory manages it's own projects
+
     Args:
         app_config: The Basic Memory project configuration
     """
+    # Skip initialization in cloud mode - cloud manages its own projects
+    if app_config.cloud_mode_enabled:
+        logger.debug("Skipping initialization in cloud mode - projects managed by cloud")
+        return
+
     try:
         result = asyncio.run(initialize_app(app_config))
         logger.info(f"Initialization completed successfully: result={result}")

basic_memory/services/project_service.py

@@ -21,6 +21,9 @@ from basic_memory.config import WATCH_STATUS_JSON, ConfigManager, get_project_co
 from basic_memory.utils import generate_permalink
 
 
+config = ConfigManager().config
+
+
 class ProjectService:
     """Service for managing Basic Memory projects."""
 
@@ -77,7 +80,13 @@ class ProjectService:
         return os.environ.get("BASIC_MEMORY_PROJECT", self.config_manager.default_project)
 
     async def list_projects(self) -> Sequence[Project]:
-        return await self.repository.find_all()
+        """List all projects without loading entity relationships.
+
+        Returns only basic project fields (name, path, etc.) without
+        eager loading the entities relationship which could load thousands
+        of entities for large knowledge bases.
+        """
+        return await self.repository.find_all(use_load_options=False)
 
     async def get_project(self, name: str) -> Optional[Project]:
         """Get the file path for a project by name or permalink."""
@@ -85,6 +94,40 @@ class ProjectService:
             name
         )
 
+    def _check_nested_paths(self, path1: str, path2: str) -> bool:
+        """Check if two paths are nested (one is a prefix of the other).
+
+        Args:
+            path1: First path to compare
+            path2: Second path to compare
+
+        Returns:
+            True if one path is nested within the other, False otherwise
+
+        Examples:
+            _check_nested_paths("/foo", "/foo/bar")     # True (child under parent)
+            _check_nested_paths("/foo/bar", "/foo")     # True (parent over child)
+            _check_nested_paths("/foo", "/bar")         # False (siblings)
+        """
+        # Normalize paths to ensure proper comparison
+        p1 = Path(path1).resolve()
+        p2 = Path(path2).resolve()
+
+        # Check if either path is a parent of the other
+        try:
+            # Check if p2 is under p1
+            p2.relative_to(p1)
+            return True
+        except ValueError:
+            # Not nested in this direction, check the other
+            try:
+                # Check if p1 is under p2
+                p1.relative_to(p2)
+                return True
+            except ValueError:
+                # Not nested in either direction
+                return False
+
     async def add_project(self, name: str, path: str, set_default: bool = False) -> None:
         """Add a new project to the configuration and database.
 
@@ -94,13 +137,65 @@ class ProjectService:
             set_default: Whether to set this project as the default
 
         Raises:
-            ValueError: If the project already exists
+            ValueError: If the project already exists or path collides with existing project
         """
-        if not self.repository:  # pragma: no cover
-            raise ValueError("Repository is required for add_project")
+        # If project_root is set, constrain all projects to that directory
+        project_root = self.config_manager.config.project_root
+        if project_root:
+            base_path = Path(project_root)
+
+            # In cloud mode (when project_root is set), ignore user's path completely
+            # and use sanitized project name as the directory name
+            # This ensures flat structure: /app/data/test-bisync instead of /app/data/documents/test bisync
+            sanitized_name = generate_permalink(name)
+
+            # Construct path using sanitized project name only
+            resolved_path = (base_path / sanitized_name).resolve().as_posix()
+
+            # Verify the resolved path is actually under project_root
+            if not resolved_path.startswith(base_path.resolve().as_posix()):
+                raise ValueError(
+                    f"BASIC_MEMORY_PROJECT_ROOT is set to {project_root}. "
+                    f"All projects must be created under this directory. Invalid path: {path}"
+                )
 
-        # Resolve to absolute path
-        resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix()
+            # Check for case-insensitive path collisions with existing projects
+            existing_projects = await self.list_projects()
+            for existing in existing_projects:
+                if (
+                    existing.path.lower() == resolved_path.lower()
+                    and existing.path != resolved_path
+                ):
+                    raise ValueError(
+                        f"Path collision detected: '{resolved_path}' conflicts with existing project "
+                        f"'{existing.name}' at '{existing.path}'. "
+                        f"In cloud mode, paths are normalized to lowercase to prevent case-sensitivity issues."
+                    )
+        else:
+            resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix()
+
+        # Check for nested paths with existing projects
+        existing_projects = await self.list_projects()
+        for existing in existing_projects:
+            if self._check_nested_paths(resolved_path, existing.path):
+                # Determine which path is nested within which for appropriate error message
+                p_new = Path(resolved_path).resolve()
+                p_existing = Path(existing.path).resolve()
+
+                # Check if new path is nested under existing project
+                if p_new.is_relative_to(p_existing):
+                    raise ValueError(
+                        f"Cannot create project at '{resolved_path}': "
+                        f"path is nested within existing project '{existing.name}' at '{existing.path}'. "
+                        f"Projects cannot share directory trees."
+                    )
+                else:
+                    # Existing project is nested under new path
+                    raise ValueError(
+                        f"Cannot create project at '{resolved_path}': "
+                        f"existing project '{existing.name}' at '{existing.path}' is nested within this path. "
+                        f"Projects cannot share directory trees."
+                    )
 
         # First add to config file (this will validate the project doesn't exist)
         project_config = self.config_manager.add_project(name, resolved_path)
@@ -170,15 +265,6 @@ class ProjectService:
 
         logger.info(f"Project '{name}' set as default in configuration and database")
 
-        # Refresh MCP session to pick up the new default project
-        try:
-            from basic_memory.mcp.project_session import session
-
-            session.refresh_from_config()
-        except ImportError:  # pragma: no cover
-            # MCP components might not be available in all contexts (e.g., CLI-only usage)
-            logger.debug("MCP session not available, skipping session refresh")
-
     async def _ensure_single_default_project(self) -> None:
         """Ensure only one project has is_default=True.
 
@@ -300,15 +386,6 @@ class ProjectService:
 
         logger.info("Project synchronization complete")
 
-        # Refresh MCP session to ensure it's in sync with current config
-        try:
-            from basic_memory.mcp.project_session import session
-
-            session.refresh_from_config()
-        except ImportError:
-            # MCP components might not be available in all contexts
-            logger.debug("MCP session not available, skipping session refresh")
-
     async def move_project(self, name: str, new_path: str) -> None:
         """Move a project to a new location.