basic-memory 0.12.2__py3-none-any.whl → 0.13.0__py3-none-any.whl

basic_memory/services/search_service.py

@@ -1,5 +1,6 @@
 """Service for search operations."""
 
+import ast
 from datetime import datetime
 from typing import List, Optional, Set
 
@@ -66,7 +67,7 @@ class SearchService:
             logger.debug("no criteria passed to query")
             return []
 
-        logger.debug(f"Searching with query: {query}")
+        logger.trace(f"Searching with query: {query}")
 
         after_date = (
             (
@@ -85,7 +86,7 @@ class SearchService:
             permalink_match=query.permalink_match,
             title=query.title,
             types=query.types,
-            entity_types=query.entity_types,
+            search_item_types=query.entity_types,
             after_date=after_date,
             limit=limit,
             offset=offset,
@@ -117,6 +118,38 @@ class SearchService:
 
         return variants
 
+    def _extract_entity_tags(self, entity: Entity) -> List[str]:
+        """Extract tags from entity metadata for search indexing.
+
+        Handles multiple tag formats:
+        - List format: ["tag1", "tag2"]
+        - String format: "['tag1', 'tag2']" or "[tag1, tag2]"
+        - Empty: [] or "[]"
+
+        Returns a list of tag strings for search indexing.
+        """
+        if not entity.entity_metadata or "tags" not in entity.entity_metadata:
+            return []
+
+        tags = entity.entity_metadata["tags"]
+
+        # Handle list format (preferred)
+        if isinstance(tags, list):
+            return [str(tag) for tag in tags if tag]
+
+        # Handle string format (legacy)
+        if isinstance(tags, str):
+            try:
+                # Parse string representation of list
+                parsed_tags = ast.literal_eval(tags)
+                if isinstance(parsed_tags, list):
+                    return [str(tag) for tag in parsed_tags if tag]
+            except (ValueError, SyntaxError):
+                # If parsing fails, treat as single tag
+                return [tags] if tags.strip() else []
+
+        return []  # pragma: no cover
+
     async def index_entity(
         self,
         entity: Entity,
@@ -156,6 +189,7 @@ class SearchService:
                 },
                 created_at=entity.created_at,
                 updated_at=entity.updated_at,
+                project_id=entity.project_id,
             )
         )
 
@@ -169,16 +203,20 @@ class SearchService:
         1. Entities
            - permalink: direct from entity (e.g., "specs/search")
            - file_path: physical file location
+           - project_id: project context for isolation
 
         2. Observations
            - permalink: entity permalink + /observations/id (e.g., "specs/search/observations/123")
            - file_path: parent entity's file (where observation is defined)
+           - project_id: inherited from parent entity
 
         3. Relations (only index outgoing relations defined in this file)
            - permalink: from_entity/relation_type/to_entity (e.g., "specs/search/implements/features/search-ui")
            - file_path: source entity's file (where relation is defined)
+           - project_id: inherited from source entity
 
         Each type gets its own row in the search index with appropriate metadata.
+        The project_id is automatically added by the repository when indexing.
         """
 
         content_stems = []
@@ -196,6 +234,11 @@ class SearchService:
 
         content_stems.extend(self._generate_variants(entity.file_path))
 
+        # Add entity tags from frontmatter to search content
+        entity_tags = self._extract_entity_tags(entity)
+        if entity_tags:
+            content_stems.extend(entity_tags)
+
         entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
 
         # Index entity
@@ -214,6 +257,7 @@ class SearchService:
                 },
                 created_at=entity.created_at,
                 updated_at=entity.updated_at,
+                project_id=entity.project_id,
             )
         )
 
@@ -239,6 +283,7 @@ class SearchService:
                     },
                     created_at=entity.created_at,
                     updated_at=entity.updated_at,
+                    project_id=entity.project_id,
                 )
             )
 
@@ -268,6 +313,7 @@ class SearchService:
                     relation_type=rel.relation_type,
                     created_at=entity.created_at,
                     updated_at=entity.updated_at,
+                    project_id=entity.project_id,
                 )
             )
 
@@ -278,3 +324,32 @@ class SearchService:
     async def delete_by_entity_id(self, entity_id: int):
         """Delete an item from the search index."""
         await self.repository.delete_by_entity_id(entity_id)
+
+    async def handle_delete(self, entity: Entity):
+        """Handle complete entity deletion from search index including observations and relations.
+
+        This replicates the logic from sync_service.handle_delete() to properly clean up
+        all search index entries for an entity and its related data.
+        """
+        logger.debug(
+            f"Cleaning up search index for entity_id={entity.id}, file_path={entity.file_path}, "
+            f"observations={len(entity.observations)}, relations={len(entity.outgoing_relations)}"
+        )
+
+        # Clean up search index - same logic as sync_service.handle_delete()
+        permalinks = (
+            [entity.permalink]
+            + [o.permalink for o in entity.observations]
+            + [r.permalink for r in entity.outgoing_relations]
+        )
+
+        logger.debug(
+            f"Deleting search index entries for entity_id={entity.id}, "
+            f"index_entries={len(permalinks)}"
+        )
+
+        for permalink in permalinks:
+            if permalink:
+                await self.delete_by_permalink(permalink)
+            else:
+                await self.delete_by_entity_id(entity.id)

basic_memory/services/sync_status_service.py

@@ -0,0 +1,181 @@
+"""Simple sync status tracking service."""
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Dict, Optional
+
+
+class SyncStatus(Enum):
+    """Status of sync operations."""
+
+    IDLE = "idle"
+    SCANNING = "scanning"
+    SYNCING = "syncing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    WATCHING = "watching"
+
+
+@dataclass
+class ProjectSyncStatus:
+    """Sync status for a single project."""
+
+    project_name: str
+    status: SyncStatus
+    message: str = ""
+    files_total: int = 0
+    files_processed: int = 0
+    error: Optional[str] = None
+
+
+class SyncStatusTracker:
+    """Global tracker for all sync operations."""
+
+    def __init__(self):
+        self._project_statuses: Dict[str, ProjectSyncStatus] = {}
+        self._global_status: SyncStatus = SyncStatus.IDLE
+
+    def start_project_sync(self, project_name: str, files_total: int = 0) -> None:
+        """Start tracking sync for a project."""
+        self._project_statuses[project_name] = ProjectSyncStatus(
+            project_name=project_name,
+            status=SyncStatus.SCANNING,
+            message="Scanning files",
+            files_total=files_total,
+            files_processed=0,
+        )
+        self._update_global_status()
+
+    def update_project_progress(  # pragma: no cover
+        self,
+        project_name: str,
+        status: SyncStatus,
+        message: str = "",
+        files_processed: int = 0,
+        files_total: Optional[int] = None,
+    ) -> None:
+        """Update progress for a project."""
+        if project_name not in self._project_statuses:  # pragma: no cover
+            return
+
+        project_status = self._project_statuses[project_name]
+        project_status.status = status
+        project_status.message = message
+        project_status.files_processed = files_processed
+
+        if files_total is not None:
+            project_status.files_total = files_total
+
+        self._update_global_status()
+
+    def complete_project_sync(self, project_name: str) -> None:
+        """Mark project sync as completed."""
+        if project_name in self._project_statuses:
+            self._project_statuses[project_name].status = SyncStatus.COMPLETED
+            self._project_statuses[project_name].message = "Sync completed"
+            self._update_global_status()
+
+    def fail_project_sync(self, project_name: str, error: str) -> None:
+        """Mark project sync as failed."""
+        if project_name in self._project_statuses:
+            self._project_statuses[project_name].status = SyncStatus.FAILED
+            self._project_statuses[project_name].error = error
+            self._update_global_status()
+
+    def start_project_watch(self, project_name: str) -> None:
+        """Mark project as watching for changes (steady state after sync)."""
+        if project_name in self._project_statuses:
+            self._project_statuses[project_name].status = SyncStatus.WATCHING
+            self._project_statuses[project_name].message = "Watching for changes"
+            self._update_global_status()
+        else:
+            # Create new status if project isn't tracked yet
+            self._project_statuses[project_name] = ProjectSyncStatus(
+                project_name=project_name,
+                status=SyncStatus.WATCHING,
+                message="Watching for changes",
+                files_total=0,
+                files_processed=0,
+            )
+            self._update_global_status()
+
+    def _update_global_status(self) -> None:
+        """Update global status based on project statuses."""
+        if not self._project_statuses:  # pragma: no cover
+            self._global_status = SyncStatus.IDLE
+            return
+
+        statuses = [p.status for p in self._project_statuses.values()]
+
+        if any(s == SyncStatus.FAILED for s in statuses):
+            self._global_status = SyncStatus.FAILED
+        elif any(s in (SyncStatus.SCANNING, SyncStatus.SYNCING) for s in statuses):
+            self._global_status = SyncStatus.SYNCING
+        elif all(s in (SyncStatus.COMPLETED, SyncStatus.WATCHING) for s in statuses):
+            self._global_status = SyncStatus.COMPLETED
+        else:
+            self._global_status = SyncStatus.SYNCING
+
+    @property
+    def global_status(self) -> SyncStatus:
+        """Get overall sync status."""
+        return self._global_status
+
+    @property
+    def is_syncing(self) -> bool:
+        """Check if any sync operation is in progress."""
+        return self._global_status in (SyncStatus.SCANNING, SyncStatus.SYNCING)
+
+    @property
+    def is_ready(self) -> bool:  # pragma: no cover
+        """Check if system is ready (no sync in progress)."""
+        return self._global_status in (SyncStatus.IDLE, SyncStatus.COMPLETED)
+
+    def get_project_status(self, project_name: str) -> Optional[ProjectSyncStatus]:
+        """Get status for a specific project."""
+        return self._project_statuses.get(project_name)
+
+    def get_all_projects(self) -> Dict[str, ProjectSyncStatus]:
+        """Get all project statuses."""
+        return self._project_statuses.copy()
+
+    def get_summary(self) -> str:  # pragma: no cover
+        """Get a user-friendly summary of sync status."""
+        if self._global_status == SyncStatus.IDLE:
+            return "✅ System ready"
+        elif self._global_status == SyncStatus.COMPLETED:
+            return "✅ All projects synced successfully"
+        elif self._global_status == SyncStatus.FAILED:
+            failed_projects = [
+                p.project_name
+                for p in self._project_statuses.values()
+                if p.status == SyncStatus.FAILED
+            ]
+            return f"❌ Sync failed for: {', '.join(failed_projects)}"
+        else:
+            active_projects = [
+                p.project_name
+                for p in self._project_statuses.values()
+                if p.status in (SyncStatus.SCANNING, SyncStatus.SYNCING)
+            ]
+            total_files = sum(p.files_total for p in self._project_statuses.values())
+            processed_files = sum(p.files_processed for p in self._project_statuses.values())
+
+            if total_files > 0:
+                progress_pct = (processed_files / total_files) * 100
+                return f"🔄 Syncing {len(active_projects)} projects ({processed_files}/{total_files} files, {progress_pct:.0f}%)"
+            else:
+                return f"🔄 Syncing {len(active_projects)} projects"
+
+    def clear_completed(self) -> None:
+        """Remove completed project statuses to clean up memory."""
+        self._project_statuses = {
+            name: status
+            for name, status in self._project_statuses.items()
+            if status.status != SyncStatus.COMPLETED
+        }
+        self._update_global_status()
+
+
+# Global sync status tracker instance
+sync_status_tracker = SyncStatusTracker()

basic_memory/sync/background_sync.py

@@ -0,0 +1,25 @@
+import asyncio
+
+from loguru import logger
+
+from basic_memory.config import config as project_config
+from basic_memory.sync import SyncService, WatchService
+
+
+async def sync_and_watch(
+    sync_service: SyncService, watch_service: WatchService
+):  # pragma: no cover
+    """Run sync and watch service."""
+
+    logger.info(f"Starting watch service to sync file changes in dir: {project_config.home}")
+    # full sync
+    await sync_service.sync(project_config.home)
+
+    # watch changes
+    await watch_service.run()
+
+
+async def create_background_sync_task(
+    sync_service: SyncService, watch_service: WatchService
+):  # pragma: no cover
+    return asyncio.create_task(sync_and_watch(sync_service, watch_service))

basic_memory/sync/sync_service.py

@@ -10,13 +10,14 @@ from typing import Dict, Optional, Set, Tuple
 from loguru import logger
 from sqlalchemy.exc import IntegrityError
 
-from basic_memory.config import ProjectConfig
+from basic_memory.config import BasicMemoryConfig
 from basic_memory.file_utils import has_frontmatter
 from basic_memory.markdown import EntityParser
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService, FileService
 from basic_memory.services.search_service import SearchService
+from basic_memory.services.sync_status_service import sync_status_tracker, SyncStatus
 
 
 @dataclass
@@ -64,7 +65,7 @@ class SyncService:
 
     def __init__(
         self,
-        config: ProjectConfig,
+        app_config: BasicMemoryConfig,
         entity_service: EntityService,
         entity_parser: EntityParser,
         entity_repository: EntityRepository,
@@ -72,7 +73,7 @@ class SyncService:
         search_service: SearchService,
         file_service: FileService,
     ):
-        self.config = config
+        self.app_config = app_config
         self.entity_service = entity_service
         self.entity_parser = entity_parser
         self.entity_repository = entity_repository
@@ -80,23 +81,38 @@ class SyncService:
         self.search_service = search_service
         self.file_service = file_service
 
-    async def sync(self, directory: Path) -> SyncReport:
+    async def sync(self, directory: Path, project_name: Optional[str] = None) -> SyncReport:
         """Sync all files with database."""
 
         start_time = time.time()
         logger.info(f"Sync operation started for directory: {directory}")
 
+        # Start tracking sync for this project if project name provided
+        if project_name:
+            sync_status_tracker.start_project_sync(project_name)
+
         # initial paths from db to sync
         # path -> checksum
         report = await self.scan(directory)
 
-        # Initialize progress tracking if requested
+        # Update progress with file counts
+        if project_name:
+            sync_status_tracker.update_project_progress(
+                project_name=project_name,
+                status=SyncStatus.SYNCING,
+                message="Processing file changes",
+                files_total=report.total,
+                files_processed=0,
+            )
+
         # order of sync matters to resolve relations effectively
         logger.info(
             f"Sync changes detected: new_files={len(report.new)}, modified_files={len(report.modified)}, "
             + f"deleted_files={len(report.deleted)}, moved_files={len(report.moves)}"
         )
 
+        files_processed = 0
+
         # sync moves first
         for old_path, new_path in report.moves.items():
             # in the case where a file has been deleted and replaced by another file
@@ -109,19 +125,56 @@ class SyncService:
             else:
                 await self.handle_move(old_path, new_path)
 
+            files_processed += 1
+            if project_name:
+                sync_status_tracker.update_project_progress(  # pragma: no cover
+                    project_name=project_name,
+                    status=SyncStatus.SYNCING,
+                    message="Processing moves",
+                    files_processed=files_processed,
+                )
+
         # deleted next
         for path in report.deleted:
             await self.handle_delete(path)
+            files_processed += 1
+            if project_name:
+                sync_status_tracker.update_project_progress(  # pragma: no cover
+                    project_name=project_name,
+                    status=SyncStatus.SYNCING,
+                    message="Processing deletions",
+                    files_processed=files_processed,
+                )
 
         # then new and modified
         for path in report.new:
             await self.sync_file(path, new=True)
+            files_processed += 1
+            if project_name:
+                sync_status_tracker.update_project_progress(
+                    project_name=project_name,
+                    status=SyncStatus.SYNCING,
+                    message="Processing new files",
+                    files_processed=files_processed,
+                )
 
         for path in report.modified:
             await self.sync_file(path, new=False)
+            files_processed += 1
+            if project_name:
+                sync_status_tracker.update_project_progress(  # pragma: no cover
+                    project_name=project_name,
+                    status=SyncStatus.SYNCING,
+                    message="Processing modified files",
+                    files_processed=files_processed,
+                )
 
         await self.resolve_relations()
 
+        # Mark sync as completed
+        if project_name:
+            sync_status_tracker.complete_project_sync(project_name)
+
         duration_ms = int((time.time() - start_time) * 1000)
         logger.info(
             f"Sync operation completed: directory={directory}, total_changes={report.total}, duration_ms={duration_ms}"
@@ -133,7 +186,7 @@ class SyncService:
         """Scan directory for changes compared to database state."""
 
         db_paths = await self.get_db_file_state()
-        logger.debug(f"Found {len(db_paths)} db paths")
+        logger.info(f"Scanning directory {directory}. Found {len(db_paths)} db paths")
 
         # Track potentially moved files by checksum
         scan_result = await self.scan_directory(directory)
@@ -173,6 +226,7 @@ class SyncService:
                 # deleted
                 else:
                     report.deleted.add(db_path)
+        logger.info(f"Completed scan for directory {directory}, found {report.total} changes.")
         return report
 
     async def get_db_file_state(self) -> Dict[str, str]:
@@ -218,7 +272,7 @@ class SyncService:
             return entity, checksum
 
         except Exception as e:  # pragma: no cover
-            logger.exception("Failed to sync file", path=path, error=str(e))
+            logger.error(f"Failed to sync file: path={path}, error={str(e)}")
             return None, None
 
     async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
@@ -310,18 +364,43 @@ class SyncService:
             content_type = self.file_service.content_type(path)
 
             file_path = Path(path)
-            entity = await self.entity_repository.add(
-                Entity(
-                    entity_type="file",
-                    file_path=path,
-                    checksum=checksum,
-                    title=file_path.name,
-                    created_at=created,
-                    updated_at=modified,
-                    content_type=content_type,
+            try:
+                entity = await self.entity_repository.add(
+                    Entity(
+                        entity_type="file",
+                        file_path=path,
+                        checksum=checksum,
+                        title=file_path.name,
+                        created_at=created,
+                        updated_at=modified,
+                        content_type=content_type,
+                    )
                 )
-            )
-            return entity, checksum
+                return entity, checksum
+            except IntegrityError as e:
+                # Handle race condition where entity was created by another process
+                if "UNIQUE constraint failed: entity.file_path" in str(e):
+                    logger.info(
+                        f"Entity already exists for file_path={path}, updating instead of creating"
+                    )
+                    # Treat as update instead of create
+                    entity = await self.entity_repository.get_by_file_path(path)
+                    if entity is None:  # pragma: no cover
+                        logger.error(f"Entity not found after constraint violation, path={path}")
+                        raise ValueError(f"Entity not found after constraint violation: {path}")
+
+                    updated = await self.entity_repository.update(
+                        entity.id, {"file_path": path, "checksum": checksum}
+                    )
+
+                    if updated is None:  # pragma: no cover
+                        logger.error(f"Failed to update entity, entity_id={entity.id}, path={path}")
+                        raise ValueError(f"Failed to update entity with ID {entity.id}")
+
+                    return updated, checksum
+                else:
+                    # Re-raise if it's a different integrity error
+                    raise
         else:
             entity = await self.entity_repository.get_by_file_path(path)
             if entity is None:  # pragma: no cover
@@ -378,7 +457,9 @@ class SyncService:
             updates = {"file_path": new_path}
 
             # If configured, also update permalink to match new path
-            if self.config.update_permalinks_on_move:
+            if self.app_config.update_permalinks_on_move and self.file_service.is_markdown(
+                new_path
+            ):
                 # generate new permalink value
                 new_permalink = await self.entity_service.resolve_permalink(new_path)
 
@@ -426,7 +507,7 @@ class SyncService:
         logger.info("Resolving forward references", count=len(unresolved_relations))
 
         for relation in unresolved_relations:
-            logger.debug(
+            logger.trace(
                 "Attempting to resolve relation "
                 f"relation_id={relation.id} "
                 f"from_id={relation.from_id} "
@@ -494,7 +575,7 @@ class SyncService:
                 result.files[rel_path] = checksum
                 result.checksums[checksum] = rel_path
 
-                logger.debug("Found file", path=rel_path, checksum=checksum)
+                logger.trace(f"Found file, path={rel_path}, checksum={checksum}")
 
         duration_ms = int((time.time() - start_time) * 1000)
         logger.debug(