basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/services/search_service.py

@@ -1,10 +1,14 @@
 """Service for search operations."""
 
+import ast
 from datetime import datetime
 from typing import List, Optional, Set
 
+
+from dateparser import parse
 from fastapi import BackgroundTasks
 from loguru import logger
+from sqlalchemy import text
 
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
@@ -12,6 +16,21 @@ from basic_memory.repository.search_repository import SearchRepository, SearchIn
 from basic_memory.schemas.search import SearchQuery, SearchItemType
 from basic_memory.services import FileService
 
+# Maximum size for content_stems field to stay under Postgres's 8KB index row limit.
+# We use 6000 characters to leave headroom for other indexed columns and overhead.
+MAX_CONTENT_STEMS_SIZE = 6000
+
+
+def _mtime_to_datetime(entity: Entity) -> datetime:
+    """Convert entity mtime (file modification time) to datetime.
+
+    Returns the file's actual modification time, falling back to updated_at
+    if mtime is not available.
+    """
+    if entity.mtime:
+        return datetime.fromtimestamp(entity.mtime).astimezone()
+    return entity.updated_at
+
 
 class SearchService:
     """Service for search operations.
@@ -38,9 +57,10 @@ class SearchService:
 
     async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
         """Reindex all content from database."""
-        logger.info("Starting full reindex")
 
+        logger.info("Starting full reindex")
         # Clear and recreate search index
+        await self.repository.execute_query(text("DROP TABLE IF EXISTS search_index"), params={})
         await self.init_search_index()
 
         # Reindex all entities
@@ -63,13 +83,13 @@ class SearchService:
             logger.debug("no criteria passed to query")
             return []
 
-        logger.debug(f"Searching with query: {query}")
+        logger.trace(f"Searching with query: {query}")
 
         after_date = (
             (
                 query.after_date
                 if isinstance(query.after_date, datetime)
-                else datetime.fromisoformat(query.after_date)
+                else parse(query.after_date)
             )
             if query.after_date
             else None
@@ -82,7 +102,7 @@ class SearchService:
             permalink_match=query.permalink_match,
             title=query.title,
             types=query.types,
-            entity_types=query.entity_types,
+            search_item_types=query.entity_types,
             after_date=after_date,
             limit=limit,
             offset=offset,
@@ -109,84 +129,216 @@ class SearchService:
         # Add word boundaries
         variants.update(w.strip() for w in text.lower().split() if w.strip())
 
-        # Add trigrams for fuzzy matching
-        variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
+        # Trigrams disabled: They create massive search index bloat, increasing DB size significantly
+        # and slowing down indexing performance. FTS5 search works well without them.
+        # See: https://github.com/basicmachines-co/basic-memory/issues/351
+        # variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
 
         return variants
 
+    def _extract_entity_tags(self, entity: Entity) -> List[str]:
+        """Extract tags from entity metadata for search indexing.
+
+        Handles multiple tag formats:
+        - List format: ["tag1", "tag2"]
+        - String format: "['tag1', 'tag2']" or "[tag1, tag2]"
+        - Empty: [] or "[]"
+
+        Returns a list of tag strings for search indexing.
+        """
+        if not entity.entity_metadata or "tags" not in entity.entity_metadata:
+            return []
+
+        tags = entity.entity_metadata["tags"]
+
+        # Handle list format (preferred)
+        if isinstance(tags, list):
+            return [str(tag) for tag in tags if tag]
+
+        # Handle string format (legacy)
+        if isinstance(tags, str):
+            try:
+                # Parse string representation of list
+                parsed_tags = ast.literal_eval(tags)
+                if isinstance(parsed_tags, list):
+                    return [str(tag) for tag in parsed_tags if tag]
+            except (ValueError, SyntaxError):
+                # If parsing fails, treat as single tag
+                return [tags] if tags.strip() else []
+
+        return []  # pragma: no cover
+
     async def index_entity(
         self,
         entity: Entity,
         background_tasks: Optional[BackgroundTasks] = None,
+        content: str | None = None,
+    ) -> None:
+        if background_tasks:
+            background_tasks.add_task(self.index_entity_data, entity, content)
+        else:
+            await self.index_entity_data(entity, content)
+
+    async def index_entity_data(
+        self,
+        entity: Entity,
+        content: str | None = None,
+    ) -> None:
+        logger.info(
+            f"[BackgroundTask] Starting search index for entity_id={entity.id} "
+            f"permalink={entity.permalink} project_id={entity.project_id}"
+        )
+        try:
+            # delete all search index data associated with entity
+            await self.repository.delete_by_entity_id(entity_id=entity.id)
+
+            # reindex
+            await self.index_entity_markdown(
+                entity, content
+            ) if entity.is_markdown else await self.index_entity_file(entity)
+
+            logger.info(
+                f"[BackgroundTask] Completed search index for entity_id={entity.id} "
+                f"permalink={entity.permalink}"
+            )
+        except Exception as e:  # pragma: no cover
+            # Background task failure logging; exceptions are re-raised.
+            # Avoid forcing synthetic failures just for line coverage.
+            logger.error(  # pragma: no cover
+                f"[BackgroundTask] Failed search index for entity_id={entity.id} "
+                f"permalink={entity.permalink} error={e}"
+            )
+            raise  # pragma: no cover
+
+    async def index_entity_file(
+        self,
+        entity: Entity,
+    ) -> None:
+        # Index entity file with no content
+        await self.repository.index_item(
+            SearchIndexRow(
+                id=entity.id,
+                entity_id=entity.id,
+                type=SearchItemType.ENTITY.value,
+                title=entity.title,
+                permalink=entity.permalink,  # Required for Postgres NOT NULL constraint
+                file_path=entity.file_path,
+                metadata={
+                    "entity_type": entity.entity_type,
+                },
+                created_at=entity.created_at,
+                updated_at=_mtime_to_datetime(entity),
+                project_id=entity.project_id,
+            )
+        )
+
+    async def index_entity_markdown(
+        self,
+        entity: Entity,
+        content: str | None = None,
     ) -> None:
         """Index an entity and all its observations and relations.
 
+        Args:
+            entity: The entity to index
+            content: Optional pre-loaded content (avoids file read). If None, will read from file.
+
         Indexing structure:
         1. Entities
            - permalink: direct from entity (e.g., "specs/search")
            - file_path: physical file location
+           - project_id: project context for isolation
 
         2. Observations
            - permalink: entity permalink + /observations/id (e.g., "specs/search/observations/123")
            - file_path: parent entity's file (where observation is defined)
+           - project_id: inherited from parent entity
 
         3. Relations (only index outgoing relations defined in this file)
            - permalink: from_entity/relation_type/to_entity (e.g., "specs/search/implements/features/search-ui")
            - file_path: source entity's file (where relation is defined)
+           - project_id: inherited from source entity
 
         Each type gets its own row in the search index with appropriate metadata.
+        The project_id is automatically added by the repository when indexing.
         """
-        if background_tasks:
-            background_tasks.add_task(self.index_entity_data, entity)
-        else:
-            await self.index_entity_data(entity)
 
-    async def index_entity_data(
-        self,
-        entity: Entity,
-    ) -> None:
-        """Actually perform the indexing."""
+        # Collect all search index rows to batch insert at the end
+        rows_to_index = []
 
-        content_parts = []
+        content_stems = []
+        content_snippet = ""
         title_variants = self._generate_variants(entity.title)
-        content_parts.extend(title_variants)
+        content_stems.extend(title_variants)
 
-        content = await self.file_service.read_entity_content(entity)
+        # Use provided content or read from file
+        if content is None:
+            content = await self.file_service.read_entity_content(entity)
         if content:
-            content_parts.append(content)
+            content_stems.append(content)
+            content_snippet = f"{content[:250]}"
 
-        content_parts.extend(self._generate_variants(entity.permalink))
-        content_parts.extend(self._generate_variants(entity.file_path))
+        if entity.permalink:
+            content_stems.extend(self._generate_variants(entity.permalink))
 
-        entity_content = "\n".join(p for p in content_parts if p and p.strip())
+        content_stems.extend(self._generate_variants(entity.file_path))
 
-        # Index entity
-        await self.repository.index_item(
+        # Add entity tags from frontmatter to search content
+        entity_tags = self._extract_entity_tags(entity)
+        if entity_tags:
+            content_stems.extend(entity_tags)
+
+        entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
+
+        # Truncate to stay under Postgres's 8KB index row limit
+        if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE:  # pragma: no cover
+            entity_content_stems = entity_content_stems[:MAX_CONTENT_STEMS_SIZE]  # pragma: no cover
+
+        # Add entity row
+        rows_to_index.append(
             SearchIndexRow(
                 id=entity.id,
                 type=SearchItemType.ENTITY.value,
                 title=entity.title,
-                content=entity_content,
+                content_stems=entity_content_stems,
+                content_snippet=content_snippet,
                 permalink=entity.permalink,
                 file_path=entity.file_path,
+                entity_id=entity.id,
                 metadata={
                     "entity_type": entity.entity_type,
                 },
                 created_at=entity.created_at,
-                updated_at=entity.updated_at,
+                updated_at=_mtime_to_datetime(entity),
+                project_id=entity.project_id,
             )
         )
 
-        # Index each observation with permalink
+        # Add observation rows - dedupe by permalink to avoid unique constraint violations
+        # Two observations with same entity/category/content generate identical permalinks
+        seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
         for obs in entity.observations:
+            obs_permalink = obs.permalink
+            if obs_permalink in seen_permalinks:
+                logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
+                continue
+            seen_permalinks.add(obs_permalink)
+
             # Index with parent entity's file path since that's where it's defined
-            await self.repository.index_item(
+            obs_content_stems = "\n".join(
+                p for p in self._generate_variants(obs.content) if p and p.strip()
+            )
+            # Truncate to stay under Postgres's 8KB index row limit
+            if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE:  # pragma: no cover
+                obs_content_stems = obs_content_stems[:MAX_CONTENT_STEMS_SIZE]  # pragma: no cover
+            rows_to_index.append(
                 SearchIndexRow(
                     id=obs.id,
                     type=SearchItemType.OBSERVATION.value,
-                    title=f"{obs.category}: {obs.content[:50]}...",
-                    content=obs.content,
-                    permalink=obs.permalink,
+                    title=f"{obs.category}: {obs.content[:100]}...",
+                    content_stems=obs_content_stems,
+                    content_snippet=obs.content,
+                    permalink=obs_permalink,
                     file_path=entity.file_path,
                     category=obs.category,
                     entity_id=entity.id,
@@ -194,11 +346,12 @@ class SearchService:
                         "tags": obs.tags,
                     },
                     created_at=entity.created_at,
-                    updated_at=entity.updated_at,
+                    updated_at=_mtime_to_datetime(entity),
+                    project_id=entity.project_id,
                 )
             )
 
-        # Only index outgoing relations (ones defined in this file)
+        # Add relation rows (only outgoing relations defined in this file)
         for rel in entity.outgoing_relations:
             # Create descriptive title showing the relationship
             relation_title = (
@@ -207,21 +360,63 @@ class SearchService:
                 else f"{rel.from_entity.title}"
             )
 
-            await self.repository.index_item(
+            rel_content_stems = "\n".join(
+                p for p in self._generate_variants(relation_title) if p and p.strip()
+            )
+            rows_to_index.append(
                 SearchIndexRow(
                     id=rel.id,
                     title=relation_title,
                     permalink=rel.permalink,
+                    content_stems=rel_content_stems,
                     file_path=entity.file_path,
                     type=SearchItemType.RELATION.value,
+                    entity_id=entity.id,
                     from_id=rel.from_id,
                     to_id=rel.to_id,
                     relation_type=rel.relation_type,
                     created_at=entity.created_at,
-                    updated_at=entity.updated_at,
+                    updated_at=_mtime_to_datetime(entity),
+                    project_id=entity.project_id,
                 )
             )
 
-    async def delete_by_permalink(self, path_id: str):
+        # Batch insert all rows at once
+        await self.repository.bulk_index_items(rows_to_index)
+
+    async def delete_by_permalink(self, permalink: str):
+        """Delete an item from the search index."""
+        await self.repository.delete_by_permalink(permalink)
+
+    async def delete_by_entity_id(self, entity_id: int):
         """Delete an item from the search index."""
-        await self.repository.delete_by_permalink(path_id)
+        await self.repository.delete_by_entity_id(entity_id)
+
+    async def handle_delete(self, entity: Entity):
+        """Handle complete entity deletion from search index including observations and relations.
+
+        This replicates the logic from sync_service.handle_delete() to properly clean up
+        all search index entries for an entity and its related data.
+        """
+        logger.debug(
+            f"Cleaning up search index for entity_id={entity.id}, file_path={entity.file_path}, "
+            f"observations={len(entity.observations)}, relations={len(entity.outgoing_relations)}"
+        )
+
+        # Clean up search index - same logic as sync_service.handle_delete()
+        permalinks = (
+            [entity.permalink]
+            + [o.permalink for o in entity.observations]
+            + [r.permalink for r in entity.outgoing_relations]
+        )
+
+        logger.debug(
+            f"Deleting search index entries for entity_id={entity.id}, "
+            f"index_entries={len(permalinks)}"
+        )
+
+        for permalink in permalinks:
+            if permalink:
+                await self.delete_by_permalink(permalink)
+            else:
+                await self.delete_by_entity_id(entity.id)

basic_memory/sync/__init__.py

@@ -1,5 +1,7 @@
-from .file_change_scanner import FileChangeScanner
+"""Basic Memory sync services."""
+
+from .coordinator import SyncCoordinator, SyncStatus
 from .sync_service import SyncService
 from .watch_service import WatchService
 
-__all__ = ["SyncService", "FileChangeScanner", "WatchService"]
+__all__ = ["SyncService", "WatchService", "SyncCoordinator", "SyncStatus"]

basic_memory/sync/background_sync.py

@@ -0,0 +1,26 @@
+import asyncio
+
+from loguru import logger
+
+from basic_memory.config import get_project_config
+from basic_memory.sync import SyncService, WatchService
+
+
+async def sync_and_watch(
+    sync_service: SyncService, watch_service: WatchService
+):  # pragma: no cover
+    """Run sync and watch service."""
+
+    config = get_project_config()
+    logger.info(f"Starting watch service to sync file changes in dir: {config.home}")
+    # full sync
+    await sync_service.sync(config.home)
+
+    # watch changes
+    await watch_service.run()
+
+
+async def create_background_sync_task(
+    sync_service: SyncService, watch_service: WatchService
+):  # pragma: no cover
+    return asyncio.create_task(sync_and_watch(sync_service, watch_service))

basic_memory/sync/coordinator.py

@@ -0,0 +1,160 @@
+"""SyncCoordinator - centralized sync/watch lifecycle management.
+
+This module provides a single coordinator that manages the lifecycle of
+file synchronization and watch services across all entry points (API, MCP, CLI).
+
+The coordinator handles:
+- Starting/stopping watch service
+- Scheduling background sync
+- Reporting status
+- Clean shutdown behavior
+"""
+
+import asyncio
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Optional
+
+from loguru import logger
+
+from basic_memory.config import BasicMemoryConfig
+
+
+class SyncStatus(Enum):
+    """Status of the sync coordinator."""
+
+    NOT_STARTED = auto()
+    STARTING = auto()
+    RUNNING = auto()
+    STOPPING = auto()
+    STOPPED = auto()
+    ERROR = auto()
+
+
+@dataclass
+class SyncCoordinator:
+    """Centralized coordinator for sync/watch lifecycle.
+
+    Manages the lifecycle of file synchronization services, providing:
+    - Unified start/stop interface
+    - Status tracking
+    - Clean shutdown with proper task cancellation
+
+    Args:
+        config: BasicMemoryConfig with sync settings
+        should_sync: Whether sync should be enabled (from container decision)
+        skip_reason: Human-readable reason if sync is skipped
+
+    Usage:
+        coordinator = SyncCoordinator(config=config, should_sync=True)
+        await coordinator.start()
+        # ... application runs ...
+        await coordinator.stop()
+    """
+
+    config: BasicMemoryConfig
+    should_sync: bool = True
+    skip_reason: Optional[str] = None
+
+    # Internal state (not constructor args)
+    _status: SyncStatus = field(default=SyncStatus.NOT_STARTED, init=False)
+    _sync_task: Optional[asyncio.Task] = field(default=None, init=False)
+
+    @property
+    def status(self) -> SyncStatus:
+        """Current status of the coordinator."""
+        return self._status
+
+    @property
+    def is_running(self) -> bool:
+        """Whether sync is currently running."""
+        return self._status == SyncStatus.RUNNING
+
+    async def start(self) -> None:
+        """Start the sync/watch service if enabled.
+
+        This is a non-blocking call that starts the sync task in the background.
+        Use stop() to cleanly shut down.
+        """
+        if not self.should_sync:
+            if self.skip_reason:
+                logger.info(f"{self.skip_reason} - skipping local file sync")
+            self._status = SyncStatus.STOPPED
+            return
+
+        if self._status in (SyncStatus.RUNNING, SyncStatus.STARTING):
+            logger.warning("Sync coordinator already running or starting")
+            return
+
+        self._status = SyncStatus.STARTING
+        logger.info("Starting file sync in background")
+
+        try:
+            # Deferred import to avoid circular dependency
+            from basic_memory.services.initialization import initialize_file_sync
+
+            async def _file_sync_runner() -> None:  # pragma: no cover
+                """Run the file sync service."""
+                try:
+                    await initialize_file_sync(self.config)
+                except asyncio.CancelledError:
+                    logger.debug("File sync cancelled")
+                    raise
+                except Exception as e:
+                    logger.error(f"Error in file sync: {e}")
+                    self._status = SyncStatus.ERROR
+                    raise
+
+            self._sync_task = asyncio.create_task(_file_sync_runner())
+            self._status = SyncStatus.RUNNING
+            logger.info("Sync coordinator started successfully")
+
+        except Exception as e:  # pragma: no cover
+            logger.error(f"Failed to start sync coordinator: {e}")
+            self._status = SyncStatus.ERROR
+            raise
+
+    async def stop(self) -> None:
+        """Stop the sync/watch service cleanly.
+
+        Cancels the background task and waits for it to complete.
+        Safe to call even if not running.
+        """
+        if self._status in (SyncStatus.NOT_STARTED, SyncStatus.STOPPED):
+            return
+
+        if self._sync_task is None:  # pragma: no cover
+            self._status = SyncStatus.STOPPED
+            return
+
+        self._status = SyncStatus.STOPPING
+        logger.info("Stopping sync coordinator...")
+
+        self._sync_task.cancel()
+        try:
+            await self._sync_task
+        except asyncio.CancelledError:
+            logger.info("File sync task cancelled successfully")
+
+        self._sync_task = None
+        self._status = SyncStatus.STOPPED
+        logger.info("Sync coordinator stopped")
+
+    def get_status_info(self) -> dict:
+        """Get status information for reporting.
+
+        Returns:
+            Dictionary with status details for diagnostics
+        """
+        return {
+            "status": self._status.name,
+            "should_sync": self.should_sync,
+            "skip_reason": self.skip_reason,
+            "has_task": self._sync_task is not None,
+        }
+
+
+__all__ = [
+    "SyncCoordinator",
+    "SyncStatus",
+]