basic-memory 0.7.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/repository/entity_repository.py

@@ -3,10 +3,14 @@
 from pathlib import Path
 from typing import List, Optional, Sequence, Union
 
+from loguru import logger
+from sqlalchemy import select
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
 from sqlalchemy.orm import selectinload
 from sqlalchemy.orm.interfaces import LoaderOption
 
+from basic_memory import db
 from basic_memory.models.knowledge import Entity, Observation, Relation
 from basic_memory.repository.repository import Repository
 
@@ -18,9 +22,14 @@ class EntityRepository(Repository[Entity]):
     to strings before passing to repository methods.
     """
 
-    def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
-        """Initialize with session maker."""
-        super().__init__(session_maker, Entity)
+    def __init__(self, session_maker: async_sessionmaker[AsyncSession], project_id: int):
+        """Initialize with session maker and project_id filter.
+
+        Args:
+            session_maker: SQLAlchemy session maker
+            project_id: Project ID to filter all operations by
+        """
+        super().__init__(session_maker, Entity, project_id=project_id)
 
     async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
         """Get entity by permalink.
@@ -31,14 +40,15 @@ class EntityRepository(Repository[Entity]):
         query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
         return await self.find_one(query)
 
-    async def get_by_title(self, title: str) -> Optional[Entity]:
+    async def get_by_title(self, title: str) -> Sequence[Entity]:
         """Get entity by title.
 
         Args:
             title: Title of the entity to find
         """
         query = self.select().where(Entity.title == title).options(*self.get_load_options())
-        return await self.find_one(query)
+        result = await self.execute_query(query)
+        return list(result.scalars().all())
 
     async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
         """Get entity by file_path.
@@ -48,18 +58,35 @@ class EntityRepository(Repository[Entity]):
         """
         query = (
             self.select()
-            .where(Entity.file_path == str(file_path))
+            .where(Entity.file_path == Path(file_path).as_posix())
             .options(*self.get_load_options())
         )
         return await self.find_one(query)
 
+    async def find_by_checksum(self, checksum: str) -> Sequence[Entity]:
+        """Find entities with the given checksum.
+
+        Used for move detection - finds entities that may have been moved to a new path.
+        Multiple entities may have the same checksum if files were copied.
+
+        Args:
+            checksum: File content checksum to search for
+
+        Returns:
+            Sequence of entities with matching checksum (may be empty)
+        """
+        query = self.select().where(Entity.checksum == checksum)
+        # Don't load relationships for move detection - we only need file_path and checksum
+        result = await self.execute_query(query, use_query_options=False)
+        return list(result.scalars().all())
+
     async def delete_by_file_path(self, file_path: Union[Path, str]) -> bool:
         """Delete entity with the provided file_path.
 
         Args:
             file_path: Path to the entity file (will be converted to string internally)
         """
-        return await self.delete_by_fields(file_path=str(file_path))
+        return await self.delete_by_fields(file_path=Path(file_path).as_posix())
 
     def get_load_options(self) -> List[LoaderOption]:
         """Get SQLAlchemy loader options for eager loading relationships."""
@@ -90,3 +117,198 @@ class EntityRepository(Repository[Entity]):
 
         result = await self.execute_query(query)
         return list(result.scalars().all())
+
+    async def upsert_entity(self, entity: Entity) -> Entity:
+        """Insert or update entity using simple try/catch with database-level conflict resolution.
+
+        Handles file_path race conditions by checking for existing entity on IntegrityError.
+        For permalink conflicts, generates a unique permalink with numeric suffix.
+
+        Args:
+            entity: The entity to insert or update
+
+        Returns:
+            The inserted or updated entity
+        """
+        async with db.scoped_session(self.session_maker) as session:
+            # Set project_id if applicable and not already set
+            self._set_project_id_if_needed(entity)
+
+            # Try simple insert first
+            try:
+                session.add(entity)
+                await session.flush()
+
+                # Return with relationships loaded
+                query = (
+                    self.select()
+                    .where(Entity.file_path == entity.file_path)
+                    .options(*self.get_load_options())
+                )
+                result = await session.execute(query)
+                found = result.scalar_one_or_none()
+                if not found:  # pragma: no cover
+                    raise RuntimeError(
+                        f"Failed to retrieve entity after insert: {entity.file_path}"
+                    )
+                return found
+
+            except IntegrityError as e:
+                # Check if this is a FOREIGN KEY constraint failure
+                error_str = str(e)
+                if "FOREIGN KEY constraint failed" in error_str:
+                    # Import locally to avoid circular dependency (repository -> services -> repository)
+                    from basic_memory.services.exceptions import SyncFatalError
+
+                    # Project doesn't exist in database - this is a fatal sync error
+                    raise SyncFatalError(
+                        f"Cannot sync file '{entity.file_path}': "
+                        f"project_id={entity.project_id} does not exist in database. "
+                        f"The project may have been deleted. This sync will be terminated."
+                    ) from e
+
+                await session.rollback()
+
+                # Re-query after rollback to get a fresh, attached entity
+                existing_result = await session.execute(
+                    select(Entity)
+                    .where(
+                        Entity.file_path == entity.file_path, Entity.project_id == entity.project_id
+                    )
+                    .options(*self.get_load_options())
+                )
+                existing_entity = existing_result.scalar_one_or_none()
+
+                if existing_entity:
+                    # File path conflict - update the existing entity
+                    logger.debug(
+                        f"Resolving file_path conflict for {entity.file_path}, "
+                        f"entity_id={existing_entity.id}, observations={len(entity.observations)}"
+                    )
+                    # Use merge to avoid session state conflicts
+                    # Set the ID to update existing entity
+                    entity.id = existing_entity.id
+
+                    # Ensure observations reference the correct entity_id
+                    for obs in entity.observations:
+                        obs.entity_id = existing_entity.id
+                        # Clear any existing ID to force INSERT as new observation
+                        obs.id = None
+
+                    # Merge the entity which will update the existing one
+                    merged_entity = await session.merge(entity)
+
+                    await session.commit()
+
+                    # Re-query to get proper relationships loaded
+                    final_result = await session.execute(
+                        select(Entity)
+                        .where(Entity.id == merged_entity.id)
+                        .options(*self.get_load_options())
+                    )
+                    return final_result.scalar_one()
+
+                else:
+                    # No file_path conflict - must be permalink conflict
+                    # Generate unique permalink and retry
+                    entity = await self._handle_permalink_conflict(entity, session)
+                    return entity
+
+    async def get_all_file_paths(self) -> List[str]:
+        """Get all file paths for this project - optimized for deletion detection.
+
+        Returns only file_path strings without loading entities or relationships.
+        Used by streaming sync to detect deleted files efficiently.
+
+        Returns:
+            List of file_path strings for all entities in the project
+        """
+        query = select(Entity.file_path)
+        query = self._add_project_filter(query)
+
+        result = await self.execute_query(query, use_query_options=False)
+        return list(result.scalars().all())
+
+    async def get_distinct_directories(self) -> List[str]:
+        """Extract unique directory paths from file_path column.
+
+        Optimized method for getting directory structure without loading full entities
+        or relationships. Returns a sorted list of unique directory paths.
+
+        Returns:
+            List of unique directory paths (e.g., ["notes", "notes/meetings", "specs"])
+        """
+        # Query only file_path column, no entity objects or relationships
+        query = select(Entity.file_path).distinct()
+        query = self._add_project_filter(query)
+
+        # Execute with use_query_options=False to skip eager loading
+        result = await self.execute_query(query, use_query_options=False)
+        file_paths = [row for row in result.scalars().all()]
+
+        # Parse file paths to extract unique directories
+        directories = set()
+        for file_path in file_paths:
+            parts = [p for p in file_path.split("/") if p]
+            # Add all parent directories (exclude filename which is the last part)
+            for i in range(len(parts) - 1):
+                dir_path = "/".join(parts[: i + 1])
+                directories.add(dir_path)
+
+        return sorted(directories)
+
+    async def find_by_directory_prefix(self, directory_prefix: str) -> Sequence[Entity]:
+        """Find entities whose file_path starts with the given directory prefix.
+
+        Optimized method for listing directory contents without loading all entities.
+        Uses SQL LIKE pattern matching to filter entities by directory path.
+
+        Args:
+            directory_prefix: Directory path prefix (e.g., "docs", "docs/guides")
+                             Empty string returns all entities (root directory)
+
+        Returns:
+            Sequence of entities in the specified directory and subdirectories
+        """
+        # Build SQL LIKE pattern
+        if directory_prefix == "" or directory_prefix == "/":
+            # Root directory - return all entities
+            return await self.find_all()
+
+        # Remove leading/trailing slashes for consistency
+        directory_prefix = directory_prefix.strip("/")
+
+        # Query entities with file_path starting with prefix
+        # Pattern matches "prefix/" to ensure we get files IN the directory,
+        # not just files whose names start with the prefix
+        pattern = f"{directory_prefix}/%"
+
+        query = self.select().where(Entity.file_path.like(pattern))
+
+        # Skip eager loading - we only need basic entity fields for directory trees
+        result = await self.execute_query(query, use_query_options=False)
+        return list(result.scalars().all())
+
+    async def _handle_permalink_conflict(self, entity: Entity, session: AsyncSession) -> Entity:
+        """Handle permalink conflicts by generating a unique permalink."""
+        base_permalink = entity.permalink
+        suffix = 1
+
+        # Find a unique permalink
+        while True:
+            test_permalink = f"{base_permalink}-{suffix}"
+            existing = await session.execute(
+                select(Entity).where(
+                    Entity.permalink == test_permalink, Entity.project_id == entity.project_id
+                )
+            )
+            if existing.scalar_one_or_none() is None:
+                # Found unique permalink
+                entity.permalink = test_permalink
+                break
+            suffix += 1
+
+        # Insert with unique permalink
+        session.add(entity)
+        await session.flush()
+        return entity

basic_memory/repository/observation_repository.py

@@ -1,6 +1,6 @@
 """Repository for managing Observation objects."""
 
-from typing import Sequence
+from typing import Dict, List, Sequence
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import async_sessionmaker
@@ -12,8 +12,14 @@ from basic_memory.repository.repository import Repository
 class ObservationRepository(Repository[Observation]):
     """Repository for Observation model with memory-specific operations."""
 
-    def __init__(self, session_maker: async_sessionmaker):
-        super().__init__(session_maker, Observation)
+    def __init__(self, session_maker: async_sessionmaker, project_id: int):
+        """Initialize with session maker and project_id filter.
+
+        Args:
+            session_maker: SQLAlchemy session maker
+            project_id: Project ID to filter all operations by
+        """
+        super().__init__(session_maker, Observation, project_id=project_id)
 
     async def find_by_entity(self, entity_id: int) -> Sequence[Observation]:
         """Find all observations for a specific entity."""
@@ -38,3 +44,29 @@ class ObservationRepository(Repository[Observation]):
         query = select(Observation.category).distinct()
         result = await self.execute_query(query, use_query_options=False)
         return result.scalars().all()
+
+    async def find_by_entities(self, entity_ids: List[int]) -> Dict[int, List[Observation]]:
+        """Find all observations for multiple entities in a single query.
+
+        Args:
+            entity_ids: List of entity IDs to fetch observations for
+
+        Returns:
+            Dictionary mapping entity_id to list of observations
+        """
+        if not entity_ids:  # pragma: no cover
+            return {}
+
+        # Query observations for all entities in the list
+        query = select(Observation).filter(Observation.entity_id.in_(entity_ids))
+        result = await self.execute_query(query)
+        observations = result.scalars().all()
+
+        # Group observations by entity_id
+        observations_by_entity = {}
+        for obs in observations:
+            if obs.entity_id not in observations_by_entity:
+                observations_by_entity[obs.entity_id] = []
+            observations_by_entity[obs.entity_id].append(obs)
+
+        return observations_by_entity

basic_memory/repository/project_info_repository.py

@@ -0,0 +1,10 @@
+from basic_memory.repository.repository import Repository
+from basic_memory.models.project import Project
+
+
+class ProjectInfoRepository(Repository):
+    """Repository for statistics queries."""
+
+    def __init__(self, session_maker):
+        # Initialize with Project model as a reference
+        super().__init__(session_maker, Project)

basic_memory/repository/project_repository.py

@@ -0,0 +1,103 @@
+"""Repository for managing projects in Basic Memory."""
+
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from basic_memory import db
+from basic_memory.models.project import Project
+from basic_memory.repository.repository import Repository
+
+
+class ProjectRepository(Repository[Project]):
+    """Repository for Project model.
+
+    Projects represent collections of knowledge entities grouped together.
+    Each entity, observation, and relation belongs to a specific project.
+    """
+
+    def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
+        """Initialize with session maker."""
+        super().__init__(session_maker, Project)
+
+    async def get_by_name(self, name: str) -> Optional[Project]:
+        """Get project by name.
+
+        Args:
+            name: Unique name of the project
+        """
+        query = self.select().where(Project.name == name)
+        return await self.find_one(query)
+
+    async def get_by_permalink(self, permalink: str) -> Optional[Project]:
+        """Get project by permalink.
+
+        Args:
+            permalink: URL-friendly identifier for the project
+        """
+        query = self.select().where(Project.permalink == permalink)
+        return await self.find_one(query)
+
+    async def get_by_path(self, path: Union[Path, str]) -> Optional[Project]:
+        """Get project by filesystem path.
+
+        Args:
+            path: Path to the project directory (will be converted to string internally)
+        """
+        query = self.select().where(Project.path == Path(path).as_posix())
+        return await self.find_one(query)
+
+    async def get_default_project(self) -> Optional[Project]:
+        """Get the default project (the one marked as is_default=True)."""
+        query = self.select().where(Project.is_default.is_not(None))
+        return await self.find_one(query)
+
+    async def get_active_projects(self) -> Sequence[Project]:
+        """Get all active projects."""
+        query = self.select().where(Project.is_active == True)  # noqa: E712
+        result = await self.execute_query(query)
+        return list(result.scalars().all())
+
+    async def set_as_default(self, project_id: int) -> Optional[Project]:
+        """Set a project as the default and unset previous default.
+
+        Args:
+            project_id: ID of the project to set as default
+
+        Returns:
+            The updated project if found, None otherwise
+        """
+        async with db.scoped_session(self.session_maker) as session:
+            # First, clear the default flag for all projects using direct SQL
+            await session.execute(
+                text("UPDATE project SET is_default = NULL WHERE is_default IS NOT NULL")
+            )
+            await session.flush()
+
+            # Set the new default project
+            target_project = await self.select_by_id(session, project_id)
+            if target_project:
+                target_project.is_default = True
+                await session.flush()
+                return target_project
+            return None  # pragma: no cover
+
+    async def update_path(self, project_id: int, new_path: str) -> Optional[Project]:
+        """Update project path.
+
+        Args:
+            project_id: ID of the project to update
+            new_path: New filesystem path for the project
+
+        Returns:
+            The updated project if found, None otherwise
+        """
+        async with db.scoped_session(self.session_maker) as session:
+            project = await self.select_by_id(session, project_id)
+            if project:
+                project.path = new_path
+                await session.flush()
+                return project
+            return None

basic_memory/repository/relation_repository.py

@@ -16,8 +16,14 @@ from basic_memory.repository.repository import Repository
 class RelationRepository(Repository[Relation]):
     """Repository for Relation model with memory-specific operations."""
 
-    def __init__(self, session_maker: async_sessionmaker):
-        super().__init__(session_maker, Relation)
+    def __init__(self, session_maker: async_sessionmaker, project_id: int):
+        """Initialize with session maker and project_id filter.
+
+        Args:
+            session_maker: SQLAlchemy session maker
+            project_id: Project ID to filter all operations by
+        """
+        super().__init__(session_maker, Relation, project_id=project_id)
 
     async def find_relation(
         self, from_permalink: str, to_permalink: str, relation_type: str
@@ -67,5 +73,18 @@ class RelationRepository(Repository[Relation]):
         result = await self.execute_query(query)
         return result.scalars().all()
 
+    async def find_unresolved_relations_for_entity(self, entity_id: int) -> Sequence[Relation]:
+        """Find unresolved relations for a specific entity.
+
+        Args:
+            entity_id: The entity whose unresolved outgoing relations to find.
+
+        Returns:
+            List of unresolved relations where this entity is the source.
+        """
+        query = select(Relation).filter(Relation.from_id == entity_id, Relation.to_id.is_(None))
+        result = await self.execute_query(query)
+        return result.scalars().all()
+
     def get_load_options(self) -> List[LoaderOption]:
         return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)]