basic-memory 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

basic_memory/services/search_service.py

@@ -6,6 +6,7 @@ from typing import List, Optional, Set
 from dateparser import parse
 from fastapi import BackgroundTasks
 from loguru import logger
+from sqlalchemy import text
 
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
@@ -39,9 +40,10 @@ class SearchService:
 
     async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
         """Reindex all content from database."""
-        logger.info("Starting full reindex")
 
+        logger.info("Starting full reindex")
         # Clear and recreate search index
+        await self.repository.execute_query(text("DROP TABLE IF EXISTS search_index"), params={})
         await self.init_search_index()
 
         # Reindex all entities
@@ -145,6 +147,7 @@ class SearchService:
         await self.repository.index_item(
             SearchIndexRow(
                 id=entity.id,
+                entity_id=entity.id,
                 type=SearchItemType.ENTITY.value,
                 title=entity.title,
                 file_path=entity.file_path,
@@ -178,33 +181,51 @@ class SearchService:
         Each type gets its own row in the search index with appropriate metadata.
         """
 
-        assert entity.permalink is not None, (
-            "entity.permalink should not be None for markdown entities"
-        )
+        if entity.permalink is None:  # pragma: no cover
+            logger.error(
+                "Missing permalink for markdown entity",
+                entity_id=entity.id,
+                title=entity.title,
+                file_path=entity.file_path,
+            )
+            raise ValueError(
+                f"Entity permalink should not be None for markdown entity: {entity.id} ({entity.title})"
+            )
 
-        content_parts = []
+        content_stems = []
+        content_snippet = ""
         title_variants = self._generate_variants(entity.title)
-        content_parts.extend(title_variants)
+        content_stems.extend(title_variants)
 
         content = await self.file_service.read_entity_content(entity)
         if content:
-            content_parts.append(content)
+            content_stems.append(content)
+            content_snippet = f"{content[:250]}"
 
-        content_parts.extend(self._generate_variants(entity.permalink))
-        content_parts.extend(self._generate_variants(entity.file_path))
+        content_stems.extend(self._generate_variants(entity.permalink))
+        content_stems.extend(self._generate_variants(entity.file_path))
 
-        entity_content = "\n".join(p for p in content_parts if p and p.strip())
+        entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
+
+        if entity.permalink is None:  # pragma: no cover
+            logger.error(
+                "Missing permalink for markdown entity",
+                entity_id=entity.id,
+                title=entity.title,
+                file_path=entity.file_path,
+            )
+            raise ValueError(
+                f"Entity permalink should not be None for markdown entity: {entity.id} ({entity.title})"
+            )
 
-        assert entity.permalink is not None, (
-            "entity.permalink should not be None for markdown entities"
-        )
         # Index entity
         await self.repository.index_item(
             SearchIndexRow(
                 id=entity.id,
                 type=SearchItemType.ENTITY.value,
                 title=entity.title,
-                content=entity_content,
+                content_stems=entity_content_stems,
+                content_snippet=content_snippet,
                 permalink=entity.permalink,
                 file_path=entity.file_path,
                 entity_id=entity.id,
@@ -219,12 +240,16 @@ class SearchService:
         # Index each observation with permalink
         for obs in entity.observations:
             # Index with parent entity's file path since that's where it's defined
+            obs_content_stems = "\n".join(
+                p for p in self._generate_variants(obs.content) if p and p.strip()
+            )
             await self.repository.index_item(
                 SearchIndexRow(
                     id=obs.id,
                     type=SearchItemType.OBSERVATION.value,
-                    title=f"{obs.category}: {obs.content[:50]}...",
-                    content=obs.content,
+                    title=f"{obs.category}: {obs.content[:100]}...",
+                    content_stems=obs_content_stems,
+                    content_snippet=obs.content,
                     permalink=obs.permalink,
                     file_path=entity.file_path,
                     category=obs.category,
@@ -246,11 +271,15 @@ class SearchService:
                 else f"{rel.from_entity.title}"
             )
 
+            rel_content_stems = "\n".join(
+                p for p in self._generate_variants(relation_title) if p and p.strip()
+            )
             await self.repository.index_item(
                 SearchIndexRow(
                     id=rel.id,
                     title=relation_title,
                     permalink=rel.permalink,
+                    content_stems=rel_content_stems,
                     file_path=entity.file_path,
                     type=SearchItemType.RELATION.value,
                     entity_id=entity.id,

basic_memory/sync/sync_service.py

@@ -1,21 +1,23 @@
 """Service for syncing files between filesystem and database."""
 
 import os
+
 from dataclasses import dataclass
 from dataclasses import field
 from datetime import datetime
 from pathlib import Path
-from typing import Set, Dict
-from typing import Tuple
+from typing import Dict, Optional, Set, Tuple
 
-import logfire
 from loguru import logger
+from sqlalchemy.exc import IntegrityError
 
 from basic_memory.markdown import EntityParser
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository, RelationRepository
 from basic_memory.services import EntityService, FileService
 from basic_memory.services.search_service import SearchService
+import time
+from rich.progress import Progress, TextColumn, BarColumn, TaskProgressColumn
 
 
 @dataclass
@@ -77,22 +79,123 @@ class SyncService:
         self.search_service = search_service
         self.file_service = file_service
 
-    async def sync(self, directory: Path) -> SyncReport:
+    async def sync(self, directory: Path, show_progress: bool = True) -> SyncReport:
         """Sync all files with database."""
 
-        with logfire.span(f"sync {directory}", directory=directory):  # pyright: ignore [reportGeneralTypeIssues]
-            # initial paths from db to sync
-            # path -> checksum
-            report = await self.scan(directory)
+        start_time = time.time()
+        console = None
+        progress = None  # Will be initialized if show_progress is True
+
+        logger.info("Sync operation started", directory=str(directory))
+
+        # initial paths from db to sync
+        # path -> checksum
+        if show_progress:
+            from rich.console import Console
+
+            console = Console()
+            console.print(f"Scanning directory: {directory}")
+
+        report = await self.scan(directory)
 
-            # order of sync matters to resolve relations effectively
+        # Initialize progress tracking if requested
+        if show_progress and report.total > 0:
+            progress = Progress(
+                TextColumn("[bold blue]{task.description}"),
+                BarColumn(),
+                TaskProgressColumn(),
+                console=console,
+                expand=True,
+            )
+
+        # order of sync matters to resolve relations effectively
+        logger.info(
+            "Sync changes detected",
+            new_files=len(report.new),
+            modified_files=len(report.modified),
+            deleted_files=len(report.deleted),
+            moved_files=len(report.moves),
+        )
+
+        if show_progress and report.total > 0:
+            with progress:  # pyright: ignore
+                # Track each category separately
+                move_task = None
+                if report.moves:  # pragma: no cover
+                    move_task = progress.add_task("[blue]Moving files...", total=len(report.moves))  # pyright: ignore
+
+            delete_task = None
+            if report.deleted:  # pragma: no cover
+                delete_task = progress.add_task(  # pyright: ignore
+                    "[red]Deleting files...", total=len(report.deleted)
+                )
+
+            new_task = None
+            if report.new:
+                new_task = progress.add_task(  # pyright: ignore
+                    "[green]Adding new files...", total=len(report.new)
+                )
+
+            modify_task = None
+            if report.modified:  # pragma: no cover
+                modify_task = progress.add_task(  # pyright: ignore
+                    "[yellow]Updating modified files...", total=len(report.modified)
+                )
 
+            # sync moves first
+            for i, (old_path, new_path) in enumerate(report.moves.items()):
+                # in the case where a file has been deleted and replaced by another file
+                # it will show up in the move and modified lists, so handle it in modified
+                if new_path in report.modified:  # pragma: no cover
+                    report.modified.remove(new_path)
+                    logger.debug(
+                        "File marked as moved and modified",
+                        old_path=old_path,
+                        new_path=new_path,
+                        action="processing as modified",
+                    )
+                else:  # pragma: no cover
+                    await self.handle_move(old_path, new_path)
+
+                if move_task is not None:  # pragma: no cover
+                    progress.update(move_task, advance=1)  # pyright: ignore
+
+            # deleted next
+            for i, path in enumerate(report.deleted):  # pragma: no cover
+                await self.handle_delete(path)
+                if delete_task is not None:  # pragma: no cover
+                    progress.update(delete_task, advance=1)  # pyright: ignore
+
+            # then new and modified
+            for i, path in enumerate(report.new):
+                await self.sync_file(path, new=True)
+                if new_task is not None:
+                    progress.update(new_task, advance=1)  # pyright: ignore
+
+            for i, path in enumerate(report.modified):  # pragma: no cover
+                await self.sync_file(path, new=False)
+                if modify_task is not None:  # pragma: no cover
+                    progress.update(modify_task, advance=1)  # pyright: ignore
+
+            # Final step - resolving relations
+            if report.total > 0:
+                relation_task = progress.add_task("[cyan]Resolving relations...", total=1)  # pyright: ignore
+                await self.resolve_relations()
+                progress.update(relation_task, advance=1)  # pyright: ignore
+        else:
+            # No progress display - proceed with normal sync
             # sync moves first
             for old_path, new_path in report.moves.items():
                 # in the case where a file has been deleted and replaced by another file
                 # it will show up in the move and modified lists, so handle it in modified
                 if new_path in report.modified:
                     report.modified.remove(new_path)
+                    logger.debug(
+                        "File marked as moved and modified",
+                        old_path=old_path,
+                        new_path=new_path,
+                        action="processing as modified",
+                    )
                 else:
                     await self.handle_move(old_path, new_path)
 
@@ -108,7 +211,16 @@ class SyncService:
                 await self.sync_file(path, new=False)
 
             await self.resolve_relations()
-            return report
+
+        duration_ms = int((time.time() - start_time) * 1000)
+        logger.info(
+            "Sync operation completed",
+            directory=str(directory),
+            total_changes=report.total,
+            duration_ms=duration_ms,
+        )
+
+        return report
 
     async def scan(self, directory):
         """Scan directory for changes compared to database state."""
@@ -166,25 +278,55 @@ class SyncService:
         db_records = await self.entity_repository.find_all()
         return {r.file_path: r.checksum or "" for r in db_records}
 
-    async def sync_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
-        """Sync a single file."""
+    async def sync_file(
+        self, path: str, new: bool = True
+    ) -> Tuple[Optional[Entity], Optional[str]]:
+        """Sync a single file.
 
+        Args:
+            path: Path to file to sync
+            new: Whether this is a new file
+
+        Returns:
+            Tuple of (entity, checksum) or (None, None) if sync fails
+        """
         try:
+            logger.debug(
+                "Syncing file",
+                path=path,
+                is_new=new,
+                is_markdown=self.file_service.is_markdown(path),
+            )
+
             if self.file_service.is_markdown(path):
                 entity, checksum = await self.sync_markdown_file(path, new)
             else:
                 entity, checksum = await self.sync_regular_file(path, new)
-            await self.search_service.index_entity(entity)
+
+            if entity is not None:
+                await self.search_service.index_entity(entity)
+
+                logger.debug(
+                    "File sync completed", path=path, entity_id=entity.id, checksum=checksum
+                )
             return entity, checksum
 
         except Exception as e:  # pragma: no cover
-            logger.error(f"Failed to sync {path}: {e}")
-            raise
+            logger.exception("Failed to sync file", path=path, error=str(e))
+            return None, None
 
-    async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
-        """Sync a markdown file with full processing."""
+    async def sync_markdown_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
+        """Sync a markdown file with full processing.
+
+        Args:
+            path: Path to markdown file
+            new: Whether this is a new file
 
+        Returns:
+            Tuple of (entity, checksum)
+        """
         # Parse markdown first to get any existing permalink
+        logger.debug("Parsing markdown file", path=path)
         entity_markdown = await self.entity_parser.parse_file(path)
 
         # Resolve permalink - this handles all the cases including conflicts
@@ -192,7 +334,13 @@ class SyncService:
 
         # If permalink changed, update the file
         if permalink != entity_markdown.frontmatter.permalink:
-            logger.info(f"Updating permalink in {path}: {permalink}")
+            logger.info(
+                "Updating permalink",
+                path=path,
+                old_permalink=entity_markdown.frontmatter.permalink,
+                new_permalink=permalink,
+            )
+
             entity_markdown.frontmatter.metadata["permalink"] = permalink
             checksum = await self.file_service.update_frontmatter(path, {"permalink": permalink})
         else:
@@ -201,12 +349,14 @@ class SyncService:
         # if the file is new, create an entity
         if new:
             # Create entity with final permalink
-            logger.debug(f"Creating new entity from markdown: {path}")
+            logger.debug("Creating new entity from markdown", path=path, permalink=permalink)
+
             await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown)
 
         # otherwise we need to update the entity and observations
         else:
-            logger.debug(f"Updating entity from markdown: {path}")
+            logger.debug("Updating entity from markdown", path=path, permalink=permalink)
+
             await self.entity_service.update_entity_and_observations(Path(path), entity_markdown)
 
         # Update relations and search index
@@ -214,11 +364,27 @@ class SyncService:
 
         # set checksum
         await self.entity_repository.update(entity.id, {"checksum": checksum})
+
+        logger.debug(
+            "Markdown sync completed",
+            path=path,
+            entity_id=entity.id,
+            observation_count=len(entity.observations),
+            relation_count=len(entity.relations),
+        )
+
         return entity, checksum
 
-    async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Entity, str]:
-        """Sync a non-markdown file with basic tracking."""
+    async def sync_regular_file(self, path: str, new: bool = True) -> Tuple[Optional[Entity], str]:
+        """Sync a non-markdown file with basic tracking.
 
+        Args:
+            path: Path to file
+            new: Whether this is a new file
+
+        Returns:
+            Tuple of (entity, checksum)
+        """
         checksum = await self.file_service.compute_checksum(path)
         if new:
             # Generate permalink from path
@@ -247,11 +413,18 @@ class SyncService:
             return entity, checksum
         else:
             entity = await self.entity_repository.get_by_file_path(path)
-            assert entity is not None, "entity should not be None for existing file"
+            if entity is None:  # pragma: no cover
+                logger.error("Entity not found for existing file", path=path)
+                raise ValueError(f"Entity not found for existing file: {path}")
+
             updated = await self.entity_repository.update(
                 entity.id, {"file_path": path, "checksum": checksum}
             )
-            assert updated is not None, "entity should be updated"
+
+            if updated is None:  # pragma: no cover
+                logger.error("Failed to update entity", entity_id=entity.id, path=path)
+                raise ValueError(f"Failed to update entity with ID {entity.id}")
+
             return updated, checksum
 
     async def handle_delete(self, file_path: str):
@@ -260,7 +433,12 @@ class SyncService:
         # First get entity to get permalink before deletion
         entity = await self.entity_repository.get_by_file_path(file_path)
         if entity:
-            logger.debug(f"Deleting entity and cleaning up search index: {file_path}")
+            logger.info(
+                "Deleting entity",
+                file_path=file_path,
+                entity_id=entity.id,
+                permalink=entity.permalink,
+            )
 
             # Delete from db (this cascades to observations/relations)
             await self.entity_service.delete_entity_by_file_path(file_path)
@@ -271,7 +449,14 @@ class SyncService:
                 + [o.permalink for o in entity.observations]
                 + [r.permalink for r in entity.relations]
             )
-            logger.debug(f"Deleting from search index: {permalinks}")
+
+            logger.debug(
+                "Cleaning up search index",
+                entity_id=entity.id,
+                file_path=file_path,
+                index_entries=len(permalinks),
+            )
+
             for permalink in permalinks:
                 if permalink:
                     await self.search_service.delete_by_permalink(permalink)
@@ -279,12 +464,30 @@ class SyncService:
                     await self.search_service.delete_by_entity_id(entity.id)
 
     async def handle_move(self, old_path, new_path):
-        logger.debug(f"Moving entity: {old_path} -> {new_path}")
+        logger.info("Moving entity", old_path=old_path, new_path=new_path)
+
         entity = await self.entity_repository.get_by_file_path(old_path)
         if entity:
             # Update file_path but keep the same permalink for link stability
             updated = await self.entity_repository.update(entity.id, {"file_path": new_path})
-            assert updated is not None, "entity should be updated"
+
+            if updated is None:  # pragma: no cover
+                logger.error(
+                    "Failed to update entity path",
+                    entity_id=entity.id,
+                    old_path=old_path,
+                    new_path=new_path,
+                )
+                raise ValueError(f"Failed to update entity path for ID {entity.id}")
+
+            logger.debug(
+                "Entity path updated",
+                entity_id=entity.id,
+                permalink=entity.permalink,
+                old_path=old_path,
+                new_path=new_path,
+            )
+
             # update search index
             await self.search_service.index_entity(updated)
 
@@ -292,22 +495,44 @@ class SyncService:
         """Try to resolve any unresolved relations"""
 
         unresolved_relations = await self.relation_repository.find_unresolved_relations()
-        logger.debug(f"Attempting to resolve {len(unresolved_relations)} forward references")
+
+        logger.info("Resolving forward references", count=len(unresolved_relations))
+
         for relation in unresolved_relations:
+            logger.debug(
+                "Attempting to resolve relation",
+                relation_id=relation.id,
+                from_id=relation.from_id,
+                to_name=relation.to_name,
+            )
+
             resolved_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
 
             # ignore reference to self
             if resolved_entity and resolved_entity.id != relation.from_id:
                 logger.debug(
-                    f"Resolved forward reference: {relation.to_name} -> {resolved_entity.title}"
-                )
-                await self.relation_repository.update(
-                    relation.id,
-                    {
-                        "to_id": resolved_entity.id,
-                        "to_name": resolved_entity.title,
-                    },
+                    "Resolved forward reference",
+                    relation_id=relation.id,
+                    from_id=relation.from_id,
+                    to_name=relation.to_name,
+                    resolved_id=resolved_entity.id,
+                    resolved_title=resolved_entity.title,
                 )
+                try:
+                    await self.relation_repository.update(
+                        relation.id,
+                        {
+                            "to_id": resolved_entity.id,
+                            "to_name": resolved_entity.title,
+                        },
+                    )
+                except IntegrityError:  # pragma: no cover
+                    logger.debug(
+                        "Ignoring duplicate relation",
+                        relation_id=relation.id,
+                        from_id=relation.from_id,
+                        to_name=relation.to_name,
+                    )
 
                 # update search index
                 await self.search_service.index_entity(resolved_entity)
@@ -322,8 +547,9 @@ class SyncService:
         Returns:
             ScanResult containing found files and any errors
         """
+        start_time = time.time()
 
-        logger.debug(f"Scanning directory: {directory}")
+        logger.debug("Scanning directory", directory=str(directory))
         result = ScanResult()
 
         for root, dirnames, filenames in os.walk(str(directory)):
@@ -340,6 +566,15 @@ class SyncService:
                 checksum = await self.file_service.compute_checksum(rel_path)
                 result.files[rel_path] = checksum
                 result.checksums[checksum] = rel_path
-                logger.debug(f"Found file: {rel_path} with checksum: {checksum}")
+
+                logger.debug("Found file", path=rel_path, checksum=checksum)
+
+        duration_ms = int((time.time() - start_time) * 1000)
+        logger.debug(
+            "Directory scan completed",
+            directory=str(directory),
+            files_found=len(result.files),
+            duration_ms=duration_ms,
+        )
 
         return result