basic-memory 0.16.1__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/services/search_service.py

@@ -4,6 +4,7 @@ import ast
 from datetime import datetime
 from typing import List, Optional, Set
 
+
 from dateparser import parse
 from fastapi import BackgroundTasks
 from loguru import logger
@@ -15,6 +16,21 @@ from basic_memory.repository.search_repository import SearchRepository, SearchIn
 from basic_memory.schemas.search import SearchQuery, SearchItemType
 from basic_memory.services import FileService
 
+# Maximum size for content_stems field to stay under Postgres's 8KB index row limit.
+# We use 6000 characters to leave headroom for other indexed columns and overhead.
+MAX_CONTENT_STEMS_SIZE = 6000
+
+
+def _mtime_to_datetime(entity: Entity) -> datetime:
+    """Convert entity mtime (file modification time) to datetime.
+
+    Returns the file's actual modification time, falling back to updated_at
+    if mtime is not available.
+    """
+    if entity.mtime:
+        return datetime.fromtimestamp(entity.mtime).astimezone()
+    return entity.updated_at
+
 
 class SearchService:
     """Service for search operations.
@@ -156,23 +172,43 @@ class SearchService:
         self,
         entity: Entity,
         background_tasks: Optional[BackgroundTasks] = None,
+        content: str | None = None,
     ) -> None:
         if background_tasks:
-            background_tasks.add_task(self.index_entity_data, entity)
+            background_tasks.add_task(self.index_entity_data, entity, content)
         else:
-            await self.index_entity_data(entity)
+            await self.index_entity_data(entity, content)
 
     async def index_entity_data(
         self,
         entity: Entity,
+        content: str | None = None,
     ) -> None:
-        # delete all search index data associated with entity
-        await self.repository.delete_by_entity_id(entity_id=entity.id)
-
-        # reindex
-        await self.index_entity_markdown(
-            entity
-        ) if entity.is_markdown else await self.index_entity_file(entity)
+        logger.info(
+            f"[BackgroundTask] Starting search index for entity_id={entity.id} "
+            f"permalink={entity.permalink} project_id={entity.project_id}"
+        )
+        try:
+            # delete all search index data associated with entity
+            await self.repository.delete_by_entity_id(entity_id=entity.id)
+
+            # reindex
+            await self.index_entity_markdown(
+                entity, content
+            ) if entity.is_markdown else await self.index_entity_file(entity)
+
+            logger.info(
+                f"[BackgroundTask] Completed search index for entity_id={entity.id} "
+                f"permalink={entity.permalink}"
+            )
+        except Exception as e:  # pragma: no cover
+            # Background task failure logging; exceptions are re-raised.
+            # Avoid forcing synthetic failures just for line coverage.
+            logger.error(  # pragma: no cover
+                f"[BackgroundTask] Failed search index for entity_id={entity.id} "
+                f"permalink={entity.permalink} error={e}"
+            )
+            raise  # pragma: no cover
 
     async def index_entity_file(
         self,
@@ -185,12 +221,13 @@ class SearchService:
                 entity_id=entity.id,
                 type=SearchItemType.ENTITY.value,
                 title=entity.title,
+                permalink=entity.permalink,  # Required for Postgres NOT NULL constraint
                 file_path=entity.file_path,
                 metadata={
                     "entity_type": entity.entity_type,
                 },
                 created_at=entity.created_at,
-                updated_at=entity.updated_at,
+                updated_at=_mtime_to_datetime(entity),
                 project_id=entity.project_id,
             )
         )
@@ -198,9 +235,14 @@ class SearchService:
     async def index_entity_markdown(
         self,
         entity: Entity,
+        content: str | None = None,
     ) -> None:
         """Index an entity and all its observations and relations.
 
+        Args:
+            entity: The entity to index
+            content: Optional pre-loaded content (avoids file read). If None, will read from file.
+
         Indexing structure:
         1. Entities
            - permalink: direct from entity (e.g., "specs/search")
@@ -229,7 +271,9 @@ class SearchService:
         title_variants = self._generate_variants(entity.title)
         content_stems.extend(title_variants)
 
-        content = await self.file_service.read_entity_content(entity)
+        # Use provided content or read from file
+        if content is None:
+            content = await self.file_service.read_entity_content(entity)
         if content:
             content_stems.append(content)
             content_snippet = f"{content[:250]}"
@@ -246,6 +290,10 @@ class SearchService:
 
         entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
 
+        # Truncate to stay under Postgres's 8KB index row limit
+        if len(entity_content_stems) > MAX_CONTENT_STEMS_SIZE:  # pragma: no cover
+            entity_content_stems = entity_content_stems[:MAX_CONTENT_STEMS_SIZE]  # pragma: no cover
+
         # Add entity row
         rows_to_index.append(
             SearchIndexRow(
@@ -261,17 +309,28 @@ class SearchService:
                     "entity_type": entity.entity_type,
                 },
                 created_at=entity.created_at,
-                updated_at=entity.updated_at,
+                updated_at=_mtime_to_datetime(entity),
                 project_id=entity.project_id,
             )
         )
 
-        # Add observation rows
+        # Add observation rows - dedupe by permalink to avoid unique constraint violations
+        # Two observations with same entity/category/content generate identical permalinks
+        seen_permalinks: set[str] = {entity.permalink} if entity.permalink else set()
         for obs in entity.observations:
+            obs_permalink = obs.permalink
+            if obs_permalink in seen_permalinks:
+                logger.debug(f"Skipping duplicate observation permalink: {obs_permalink}")
+                continue
+            seen_permalinks.add(obs_permalink)
+
             # Index with parent entity's file path since that's where it's defined
             obs_content_stems = "\n".join(
                 p for p in self._generate_variants(obs.content) if p and p.strip()
             )
+            # Truncate to stay under Postgres's 8KB index row limit
+            if len(obs_content_stems) > MAX_CONTENT_STEMS_SIZE:  # pragma: no cover
+                obs_content_stems = obs_content_stems[:MAX_CONTENT_STEMS_SIZE]  # pragma: no cover
             rows_to_index.append(
                 SearchIndexRow(
                     id=obs.id,
@@ -279,7 +338,7 @@ class SearchService:
                     title=f"{obs.category}: {obs.content[:100]}...",
                     content_stems=obs_content_stems,
                     content_snippet=obs.content,
-                    permalink=obs.permalink,
+                    permalink=obs_permalink,
                     file_path=entity.file_path,
                     category=obs.category,
                     entity_id=entity.id,
@@ -287,7 +346,7 @@ class SearchService:
                         "tags": obs.tags,
                     },
                     created_at=entity.created_at,
-                    updated_at=entity.updated_at,
+                    updated_at=_mtime_to_datetime(entity),
                     project_id=entity.project_id,
                 )
             )
@@ -317,7 +376,7 @@ class SearchService:
                     to_id=rel.to_id,
                     relation_type=rel.relation_type,
                     created_at=entity.created_at,
-                    updated_at=entity.updated_at,
+                    updated_at=_mtime_to_datetime(entity),
                     project_id=entity.project_id,
                 )
             )

basic_memory/sync/__init__.py

@@ -1,6 +1,7 @@
 """Basic Memory sync services."""
 
+from .coordinator import SyncCoordinator, SyncStatus
 from .sync_service import SyncService
 from .watch_service import WatchService
 
-__all__ = ["SyncService", "WatchService"]
+__all__ = ["SyncService", "WatchService", "SyncCoordinator", "SyncStatus"]

basic_memory/sync/coordinator.py

@@ -0,0 +1,160 @@
+"""SyncCoordinator - centralized sync/watch lifecycle management.
+
+This module provides a single coordinator that manages the lifecycle of
+file synchronization and watch services across all entry points (API, MCP, CLI).
+
+The coordinator handles:
+- Starting/stopping watch service
+- Scheduling background sync
+- Reporting status
+- Clean shutdown behavior
+"""
+
+import asyncio
+from dataclasses import dataclass, field
+from enum import Enum, auto
+from typing import Optional
+
+from loguru import logger
+
+from basic_memory.config import BasicMemoryConfig
+
+
+class SyncStatus(Enum):
+    """Status of the sync coordinator."""
+
+    NOT_STARTED = auto()
+    STARTING = auto()
+    RUNNING = auto()
+    STOPPING = auto()
+    STOPPED = auto()
+    ERROR = auto()
+
+
+@dataclass
+class SyncCoordinator:
+    """Centralized coordinator for sync/watch lifecycle.
+
+    Manages the lifecycle of file synchronization services, providing:
+    - Unified start/stop interface
+    - Status tracking
+    - Clean shutdown with proper task cancellation
+
+    Args:
+        config: BasicMemoryConfig with sync settings
+        should_sync: Whether sync should be enabled (from container decision)
+        skip_reason: Human-readable reason if sync is skipped
+
+    Usage:
+        coordinator = SyncCoordinator(config=config, should_sync=True)
+        await coordinator.start()
+        # ... application runs ...
+        await coordinator.stop()
+    """
+
+    config: BasicMemoryConfig
+    should_sync: bool = True
+    skip_reason: Optional[str] = None
+
+    # Internal state (not constructor args)
+    _status: SyncStatus = field(default=SyncStatus.NOT_STARTED, init=False)
+    _sync_task: Optional[asyncio.Task] = field(default=None, init=False)
+
+    @property
+    def status(self) -> SyncStatus:
+        """Current status of the coordinator."""
+        return self._status
+
+    @property
+    def is_running(self) -> bool:
+        """Whether sync is currently running."""
+        return self._status == SyncStatus.RUNNING
+
+    async def start(self) -> None:
+        """Start the sync/watch service if enabled.
+
+        This is a non-blocking call that starts the sync task in the background.
+        Use stop() to cleanly shut down.
+        """
+        if not self.should_sync:
+            if self.skip_reason:
+                logger.info(f"{self.skip_reason} - skipping local file sync")
+            self._status = SyncStatus.STOPPED
+            return
+
+        if self._status in (SyncStatus.RUNNING, SyncStatus.STARTING):
+            logger.warning("Sync coordinator already running or starting")
+            return
+
+        self._status = SyncStatus.STARTING
+        logger.info("Starting file sync in background")
+
+        try:
+            # Deferred import to avoid circular dependency
+            from basic_memory.services.initialization import initialize_file_sync
+
+            async def _file_sync_runner() -> None:  # pragma: no cover
+                """Run the file sync service."""
+                try:
+                    await initialize_file_sync(self.config)
+                except asyncio.CancelledError:
+                    logger.debug("File sync cancelled")
+                    raise
+                except Exception as e:
+                    logger.error(f"Error in file sync: {e}")
+                    self._status = SyncStatus.ERROR
+                    raise
+
+            self._sync_task = asyncio.create_task(_file_sync_runner())
+            self._status = SyncStatus.RUNNING
+            logger.info("Sync coordinator started successfully")
+
+        except Exception as e:  # pragma: no cover
+            logger.error(f"Failed to start sync coordinator: {e}")
+            self._status = SyncStatus.ERROR
+            raise
+
+    async def stop(self) -> None:
+        """Stop the sync/watch service cleanly.
+
+        Cancels the background task and waits for it to complete.
+        Safe to call even if not running.
+        """
+        if self._status in (SyncStatus.NOT_STARTED, SyncStatus.STOPPED):
+            return
+
+        if self._sync_task is None:  # pragma: no cover
+            self._status = SyncStatus.STOPPED
+            return
+
+        self._status = SyncStatus.STOPPING
+        logger.info("Stopping sync coordinator...")
+
+        self._sync_task.cancel()
+        try:
+            await self._sync_task
+        except asyncio.CancelledError:
+            logger.info("File sync task cancelled successfully")
+
+        self._sync_task = None
+        self._status = SyncStatus.STOPPED
+        logger.info("Sync coordinator stopped")
+
+    def get_status_info(self) -> dict:
+        """Get status information for reporting.
+
+        Returns:
+            Dictionary with status details for diagnostics
+        """
+        return {
+            "status": self._status.name,
+            "should_sync": self.should_sync,
+            "skip_reason": self.skip_reason,
+            "has_task": self._sync_task is not None,
+        }
+
+
+__all__ = [
+    "SyncCoordinator",
+    "SyncStatus",
+]