basic-memory 0.15.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

basic_memory/repository/entity_repository.py

@@ -176,6 +176,66 @@ class EntityRepository(Repository[Entity]):
                     entity = await self._handle_permalink_conflict(entity, session)
                     return entity
 
+    async def get_distinct_directories(self) -> List[str]:
+        """Extract unique directory paths from file_path column.
+
+        Optimized method for getting directory structure without loading full entities
+        or relationships. Returns a sorted list of unique directory paths.
+
+        Returns:
+            List of unique directory paths (e.g., ["notes", "notes/meetings", "specs"])
+        """
+        # Query only file_path column, no entity objects or relationships
+        query = select(Entity.file_path).distinct()
+        query = self._add_project_filter(query)
+
+        # Execute with use_query_options=False to skip eager loading
+        result = await self.execute_query(query, use_query_options=False)
+        file_paths = [row for row in result.scalars().all()]
+
+        # Parse file paths to extract unique directories
+        directories = set()
+        for file_path in file_paths:
+            parts = [p for p in file_path.split("/") if p]
+            # Add all parent directories (exclude filename which is the last part)
+            for i in range(len(parts) - 1):
+                dir_path = "/".join(parts[: i + 1])
+                directories.add(dir_path)
+
+        return sorted(directories)
+
+    async def find_by_directory_prefix(self, directory_prefix: str) -> Sequence[Entity]:
+        """Find entities whose file_path starts with the given directory prefix.
+
+        Optimized method for listing directory contents without loading all entities.
+        Uses SQL LIKE pattern matching to filter entities by directory path.
+
+        Args:
+            directory_prefix: Directory path prefix (e.g., "docs", "docs/guides")
+                             Empty string returns all entities (root directory)
+
+        Returns:
+            Sequence of entities in the specified directory and subdirectories
+        """
+        # Build SQL LIKE pattern
+        if directory_prefix == "" or directory_prefix == "/":
+            # Root directory - return all entities
+            return await self.find_all()
+
+        # Remove leading/trailing slashes for consistency
+        directory_prefix = directory_prefix.strip("/")
+
+        # Query entities with file_path starting with prefix
+        # Pattern matches "prefix/" to ensure we get files IN the directory,
+        # not just files whose names start with the prefix
+        pattern = f"{directory_prefix}/%"
+
+        query = self.select().where(Entity.file_path.like(pattern))
+
+        # Skip eager loading - we only need basic entity fields for directory trees
+        result = await self.execute_query(query, use_query_options=False)
+        return list(result.scalars().all())
+
     async def _handle_permalink_conflict(self, entity: Entity, session: AsyncSession) -> Entity:
         """Handle permalink conflicts by generating a unique permalink."""
         base_permalink = entity.permalink

basic_memory/repository/repository.py

@@ -152,12 +152,25 @@ class Repository[T: Base]:
         # Add project filter if applicable
         return self._add_project_filter(query)
 
-    async def find_all(self, skip: int = 0, limit: Optional[int] = None) -> Sequence[T]:
-        """Fetch records from the database with pagination."""
+    async def find_all(
+        self, skip: int = 0, limit: Optional[int] = None, use_load_options: bool = True
+    ) -> Sequence[T]:
+        """Fetch records from the database with pagination.
+
+        Args:
+            skip: Number of records to skip
+            limit: Maximum number of records to return
+            use_load_options: Whether to apply eager loading options (default: True)
+        """
         logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})")
 
         async with db.scoped_session(self.session_maker) as session:
-            query = select(self.Model).offset(skip).options(*self.get_load_options())
+            query = select(self.Model).offset(skip)
+
+            # Only apply load options if requested
+            if use_load_options:
+                query = query.options(*self.get_load_options())
+
             # Add project filter if applicable
             query = self._add_project_filter(query)
 

basic_memory/repository/search_repository.py

@@ -559,6 +559,48 @@ class SearchRepository:
             logger.debug(f"indexed row {search_index_row}")
             await session.commit()
 
+    async def bulk_index_items(self, search_index_rows: List[SearchIndexRow]):
+        """Index multiple items in a single batch operation.
+
+        Note: This method assumes that any existing records for the entity_id
+        have already been deleted (typically via delete_by_entity_id).
+
+        Args:
+            search_index_rows: List of SearchIndexRow objects to index
+        """
+        if not search_index_rows:
+            return
+
+        async with db.scoped_session(self.session_maker) as session:
+            # Prepare all insert data with project_id
+            insert_data_list = []
+            for row in search_index_rows:
+                insert_data = row.to_insert()
+                insert_data["project_id"] = self.project_id
+                insert_data_list.append(insert_data)
+
+            # Batch insert all records using executemany
+            await session.execute(
+                text("""
+                    INSERT INTO search_index (
+                        id, title, content_stems, content_snippet, permalink, file_path, type, metadata,
+                        from_id, to_id, relation_type,
+                        entity_id, category,
+                        created_at, updated_at,
+                        project_id
+                    ) VALUES (
+                        :id, :title, :content_stems, :content_snippet, :permalink, :file_path, :type, :metadata,
+                        :from_id, :to_id, :relation_type,
+                        :entity_id, :category,
+                        :created_at, :updated_at,
+                        :project_id
+                    )
+                """),
+                insert_data_list,
+            )
+            logger.debug(f"Bulk indexed {len(search_index_rows)} rows")
+            await session.commit()
+
     async def delete_by_entity_id(self, entity_id: int):
         """Delete an item from the search index by entity_id."""
         async with db.scoped_session(self.session_maker) as session:

basic_memory/schemas/project_info.py

@@ -183,7 +183,7 @@ class ProjectItem(BaseModel):
 
     @property
     def home(self) -> Path:  # pragma: no cover
-        return Path(self.name)
+        return Path(self.path).expanduser()
 
     @property
     def project_url(self) -> str:  # pragma: no cover

basic_memory/services/directory_service.py

@@ -3,8 +3,9 @@
 import fnmatch
 import logging
 import os
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Sequence
 
+from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
 from basic_memory.schemas.directory import DirectoryNode
 
@@ -89,6 +90,49 @@ class DirectoryService:
         # Return the root node with its children
         return root_node
 
+    async def get_directory_structure(self) -> DirectoryNode:
+        """Build a hierarchical directory structure without file details.
+
+        Optimized method for folder navigation that only returns directory nodes,
+        no file metadata. Much faster than get_directory_tree() for large knowledge bases.
+
+        Returns:
+            DirectoryNode tree containing only folders (type="directory")
+        """
+        # Get unique directories without loading entities
+        directories = await self.entity_repository.get_distinct_directories()
+
+        # Create a root directory node
+        root_node = DirectoryNode(name="Root", directory_path="/", type="directory")
+
+        # Map to store directory nodes by path for easy lookup
+        dir_map: Dict[str, DirectoryNode] = {"/": root_node}
+
+        # Build tree with just folders
+        for dir_path in directories:
+            parts = [p for p in dir_path.split("/") if p]
+            current_path = "/"
+
+            for i, part in enumerate(parts):
+                parent_path = current_path
+                # Build the directory path
+                current_path = (
+                    f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
+                )
+
+                # Create directory node if it doesn't exist
+                if current_path not in dir_map:
+                    dir_node = DirectoryNode(
+                        name=part, directory_path=current_path, type="directory"
+                    )
+                    dir_map[current_path] = dir_node
+
+                    # Add to parent's children
+                    if parent_path in dir_map:
+                        dir_map[parent_path].children.append(dir_node)
+
+        return root_node
+
     async def list_directory(
         self,
         dir_name: str = "/",
@@ -118,8 +162,13 @@ class DirectoryService:
         if dir_name != "/" and dir_name.endswith("/"):
             dir_name = dir_name.rstrip("/")
 
-        # Get the full directory tree
-        root_tree = await self.get_directory_tree()
+        # Optimize: Query only entities in the target directory
+        # instead of loading the entire tree
+        dir_prefix = dir_name.lstrip("/")
+        entity_rows = await self.entity_repository.find_by_directory_prefix(dir_prefix)
+
+        # Build a partial tree from only the relevant entities
+        root_tree = self._build_directory_tree_from_entities(entity_rows, dir_name)
 
         # Find the target directory node
         target_node = self._find_directory_node(root_tree, dir_name)
@@ -132,6 +181,78 @@ class DirectoryService:
 
         return result
 
+    def _build_directory_tree_from_entities(
+        self, entity_rows: Sequence[Entity], root_path: str
+    ) -> DirectoryNode:
+        """Build a directory tree from a subset of entities.
+
+        Args:
+            entity_rows: Sequence of entity objects to build tree from
+            root_path: Root directory path for the tree
+
+        Returns:
+            DirectoryNode representing the tree root
+        """
+        # Create a root directory node
+        root_node = DirectoryNode(name="Root", directory_path=root_path, type="directory")
+
+        # Map to store directory nodes by path for easy lookup
+        dir_map: Dict[str, DirectoryNode] = {root_path: root_node}
+
+        # First pass: create all directory nodes
+        for file in entity_rows:
+            # Process directory path components
+            parts = [p for p in file.file_path.split("/") if p]
+
+            # Create directory structure
+            current_path = "/"
+            for i, part in enumerate(parts[:-1]):  # Skip the filename
+                parent_path = current_path
+                # Build the directory path
+                current_path = (
+                    f"{current_path}{part}" if current_path == "/" else f"{current_path}/{part}"
+                )
+
+                # Create directory node if it doesn't exist
+                if current_path not in dir_map:
+                    dir_node = DirectoryNode(
+                        name=part, directory_path=current_path, type="directory"
+                    )
+                    dir_map[current_path] = dir_node
+
+                    # Add to parent's children
+                    if parent_path in dir_map:
+                        dir_map[parent_path].children.append(dir_node)
+
+        # Second pass: add file nodes to their parent directories
+        for file in entity_rows:
+            file_name = os.path.basename(file.file_path)
+            parent_dir = os.path.dirname(file.file_path)
+            directory_path = "/" if parent_dir == "" else f"/{parent_dir}"
+
+            # Create file node
+            file_node = DirectoryNode(
+                name=file_name,
+                file_path=file.file_path,
+                directory_path=f"/{file.file_path}",
+                type="file",
+                title=file.title,
+                permalink=file.permalink,
+                entity_id=file.id,
+                entity_type=file.entity_type,
+                content_type=file.content_type,
+                updated_at=file.updated_at,
+            )
+
+            # Add to parent directory's children
+            if directory_path in dir_map:
+                dir_map[directory_path].children.append(file_node)
+            elif root_path in dir_map:
+                # Fallback to root if parent not found
+                dir_map[root_path].children.append(file_node)
+
+        return root_node
+
     def _find_directory_node(
         self, root: DirectoryNode, target_path: str
     ) -> Optional[DirectoryNode]:

basic_memory/services/entity_service.py

@@ -52,7 +52,9 @@ class EntityService(BaseService[EntityModel]):
         self.link_resolver = link_resolver
         self.app_config = app_config
 
-    async def detect_file_path_conflicts(self, file_path: str) -> List[Entity]:
+    async def detect_file_path_conflicts(
+        self, file_path: str, skip_check: bool = False
+    ) -> List[Entity]:
         """Detect potential file path conflicts for a given file path.
 
         This checks for entities with similar file paths that might cause conflicts:
@@ -63,10 +65,14 @@ class EntityService(BaseService[EntityModel]):
 
         Args:
             file_path: The file path to check for conflicts
+            skip_check: If True, skip the check and return empty list (optimization for bulk operations)
 
         Returns:
             List of entities that might conflict with the given file path
         """
+        if skip_check:
+            return []
+
         from basic_memory.utils import detect_potential_file_conflicts
 
         conflicts = []
@@ -86,7 +92,10 @@ class EntityService(BaseService[EntityModel]):
         return conflicts
 
     async def resolve_permalink(
-        self, file_path: Permalink | Path, markdown: Optional[EntityMarkdown] = None
+        self,
+        file_path: Permalink | Path,
+        markdown: Optional[EntityMarkdown] = None,
+        skip_conflict_check: bool = False,
     ) -> str:
         """Get or generate unique permalink for an entity.
 
@@ -101,7 +110,9 @@ class EntityService(BaseService[EntityModel]):
         file_path_str = Path(file_path).as_posix()
 
         # Check for potential file path conflicts before resolving permalink
-        conflicts = await self.detect_file_path_conflicts(file_path_str)
+        conflicts = await self.detect_file_path_conflicts(
+            file_path_str, skip_check=skip_conflict_check
+        )
         if conflicts:
             logger.warning(
                 f"Detected potential file path conflicts for '{file_path_str}': "
@@ -445,6 +456,7 @@ class EntityService(BaseService[EntityModel]):
             resolved_entities = await asyncio.gather(*lookup_tasks, return_exceptions=True)
 
             # Process results and create relation records
+            relations_to_add = []
             for rel, resolved in zip(markdown.relations, resolved_entities):
                 # Handle exceptions from gather and None results
                 target_entity: Optional[Entity] = None
@@ -465,14 +477,24 @@ class EntityService(BaseService[EntityModel]):
                     relation_type=rel.type,
                     context=rel.context,
                 )
+                relations_to_add.append(relation)
+
+            # Batch insert all relations
+            if relations_to_add:
                 try:
-                    await self.relation_repository.add(relation)
+                    await self.relation_repository.add_all(relations_to_add)
                 except IntegrityError:
-                    # Unique constraint violation - relation already exists
-                    logger.debug(
-                        f"Skipping duplicate relation {rel.type} from {db_entity.permalink} target: {rel.target}"
-                    )
-                    continue
+                    # Some relations might be duplicates - fall back to individual inserts
+                    logger.debug("Batch relation insert failed, trying individual inserts")
+                    for relation in relations_to_add:
+                        try:
+                            await self.relation_repository.add(relation)
+                        except IntegrityError:
+                            # Unique constraint violation - relation already exists
+                            logger.debug(
+                                f"Skipping duplicate relation {relation.relation_type} from {db_entity.permalink}"
+                            )
+                            continue
 
         return await self.repository.get_by_file_path(path)
 

basic_memory/services/project_service.py

@@ -80,7 +80,13 @@ class ProjectService:
         return os.environ.get("BASIC_MEMORY_PROJECT", self.config_manager.default_project)
 
     async def list_projects(self) -> Sequence[Project]:
-        return await self.repository.find_all()
+        """List all projects without loading entity relationships.
+
+        Returns only basic project fields (name, path, etc.) without
+        eager loading the entities relationship which could load thousands
+        of entities for large knowledge bases.
+        """
+        return await self.repository.find_all(use_load_options=False)
 
     async def get_project(self, name: str) -> Optional[Project]:
         """Get the file path for a project by name or permalink."""
@@ -88,6 +94,40 @@ class ProjectService:
             name
         )
 
+    def _check_nested_paths(self, path1: str, path2: str) -> bool:
+        """Check if two paths are nested (one is a prefix of the other).
+
+        Args:
+            path1: First path to compare
+            path2: Second path to compare
+
+        Returns:
+            True if one path is nested within the other, False otherwise
+
+        Examples:
+            _check_nested_paths("/foo", "/foo/bar")     # True (child under parent)
+            _check_nested_paths("/foo/bar", "/foo")     # True (parent over child)
+            _check_nested_paths("/foo", "/bar")         # False (siblings)
+        """
+        # Normalize paths to ensure proper comparison
+        p1 = Path(path1).resolve()
+        p2 = Path(path2).resolve()
+
+        # Check if either path is a parent of the other
+        try:
+            # Check if p2 is under p1
+            p2.relative_to(p1)
+            return True
+        except ValueError:
+            # Not nested in this direction, check the other
+            try:
+                # Check if p1 is under p2
+                p1.relative_to(p2)
+                return True
+            except ValueError:
+                # Not nested in either direction
+                return False
+
     async def add_project(self, name: str, path: str, set_default: bool = False) -> None:
         """Add a new project to the configuration and database.
 
@@ -97,19 +137,66 @@ class ProjectService:
             set_default: Whether to set this project as the default
 
         Raises:
-            ValueError: If the project already exists
+            ValueError: If the project already exists or path collides with existing project
         """
-        # in cloud mode, don't allow arbitrary paths.
-        if config.cloud_mode:
-            basic_memory_home = os.getenv("BASIC_MEMORY_HOME")
-            assert basic_memory_home is not None
-            base_path = Path(basic_memory_home)
-
-            # Resolve to absolute path
-            resolved_path = Path(os.path.abspath(os.path.expanduser(base_path / path))).as_posix()
+        # If project_root is set, constrain all projects to that directory
+        project_root = self.config_manager.config.project_root
+        if project_root:
+            base_path = Path(project_root)
+
+            # In cloud mode (when project_root is set), ignore user's path completely
+            # and use sanitized project name as the directory name
+            # This ensures flat structure: /app/data/test-bisync instead of /app/data/documents/test bisync
+            sanitized_name = generate_permalink(name)
+
+            # Construct path using sanitized project name only
+            resolved_path = (base_path / sanitized_name).resolve().as_posix()
+
+            # Verify the resolved path is actually under project_root
+            if not resolved_path.startswith(base_path.resolve().as_posix()):
+                raise ValueError(
+                    f"BASIC_MEMORY_PROJECT_ROOT is set to {project_root}. "
+                    f"All projects must be created under this directory. Invalid path: {path}"
+                )
+
+            # Check for case-insensitive path collisions with existing projects
+            existing_projects = await self.list_projects()
+            for existing in existing_projects:
+                if (
+                    existing.path.lower() == resolved_path.lower()
+                    and existing.path != resolved_path
+                ):
+                    raise ValueError(
+                        f"Path collision detected: '{resolved_path}' conflicts with existing project "
+                        f"'{existing.name}' at '{existing.path}'. "
+                        f"In cloud mode, paths are normalized to lowercase to prevent case-sensitivity issues."
+                    )
         else:
             resolved_path = Path(os.path.abspath(os.path.expanduser(path))).as_posix()
 
+        # Check for nested paths with existing projects
+        existing_projects = await self.list_projects()
+        for existing in existing_projects:
+            if self._check_nested_paths(resolved_path, existing.path):
+                # Determine which path is nested within which for appropriate error message
+                p_new = Path(resolved_path).resolve()
+                p_existing = Path(existing.path).resolve()
+
+                # Check if new path is nested under existing project
+                if p_new.is_relative_to(p_existing):
+                    raise ValueError(
+                        f"Cannot create project at '{resolved_path}': "
+                        f"path is nested within existing project '{existing.name}' at '{existing.path}'. "
+                        f"Projects cannot share directory trees."
+                    )
+                else:
+                    # Existing project is nested under new path
+                    raise ValueError(
+                        f"Cannot create project at '{resolved_path}': "
+                        f"existing project '{existing.name}' at '{existing.path}' is nested within this path. "
+                        f"Projects cannot share directory trees."
+                    )
+
         # First add to config file (this will validate the project doesn't exist)
         project_config = self.config_manager.add_project(name, resolved_path)
 

basic_memory/services/search_service.py

@@ -113,8 +113,10 @@ class SearchService:
         # Add word boundaries
         variants.update(w.strip() for w in text.lower().split() if w.strip())
 
-        # Add trigrams for fuzzy matching
-        variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
+        # Trigrams disabled: They create massive search index bloat, increasing DB size significantly
+        # and slowing down indexing performance. FTS5 search works well without them.
+        # See: https://github.com/basicmachines-co/basic-memory/issues/351
+        # variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
 
         return variants
 
@@ -219,6 +221,9 @@ class SearchService:
         The project_id is automatically added by the repository when indexing.
         """
 
+        # Collect all search index rows to batch insert at the end
+        rows_to_index = []
+
         content_stems = []
         content_snippet = ""
         title_variants = self._generate_variants(entity.title)
@@ -241,8 +246,8 @@ class SearchService:
 
         entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
 
-        # Index entity
-        await self.repository.index_item(
+        # Add entity row
+        rows_to_index.append(
             SearchIndexRow(
                 id=entity.id,
                 type=SearchItemType.ENTITY.value,
@@ -261,13 +266,13 @@ class SearchService:
             )
         )
 
-        # Index each observation with permalink
+        # Add observation rows
         for obs in entity.observations:
             # Index with parent entity's file path since that's where it's defined
             obs_content_stems = "\n".join(
                 p for p in self._generate_variants(obs.content) if p and p.strip()
             )
-            await self.repository.index_item(
+            rows_to_index.append(
                 SearchIndexRow(
                     id=obs.id,
                     type=SearchItemType.OBSERVATION.value,
@@ -287,7 +292,7 @@ class SearchService:
                 )
             )
 
-        # Only index outgoing relations (ones defined in this file)
+        # Add relation rows (only outgoing relations defined in this file)
         for rel in entity.outgoing_relations:
             # Create descriptive title showing the relationship
             relation_title = (
@@ -299,7 +304,7 @@ class SearchService:
             rel_content_stems = "\n".join(
                 p for p in self._generate_variants(relation_title) if p and p.strip()
             )
-            await self.repository.index_item(
+            rows_to_index.append(
                 SearchIndexRow(
                     id=rel.id,
                     title=relation_title,
@@ -317,6 +322,9 @@ class SearchService:
                 )
             )
 
+        # Batch insert all rows at once
+        await self.repository.bulk_index_items(rows_to_index)
+
     async def delete_by_permalink(self, permalink: str):
         """Delete an item from the search index."""
         await self.repository.delete_by_permalink(permalink)