basic-memory 0.2.12__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/services/search_service.py

@@ -1,10 +1,13 @@
 """Service for search operations."""
 
+import ast
 from datetime import datetime
 from typing import List, Optional, Set
 
+from dateparser import parse
 from fastapi import BackgroundTasks
 from loguru import logger
+from sqlalchemy import text
 
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
@@ -38,9 +41,10 @@ class SearchService:
 
     async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
         """Reindex all content from database."""
-        logger.info("Starting full reindex")
 
+        logger.info("Starting full reindex")
         # Clear and recreate search index
+        await self.repository.execute_query(text("DROP TABLE IF EXISTS search_index"), params={})
         await self.init_search_index()
 
         # Reindex all entities
@@ -51,7 +55,7 @@ class SearchService:
 
         logger.info("Reindex complete")
 
-    async def search(self, query: SearchQuery) -> List[SearchIndexRow]:
+    async def search(self, query: SearchQuery, limit=10, offset=0) -> List[SearchIndexRow]:
         """Search across all indexed content.
 
         Supports three modes:
@@ -63,27 +67,29 @@ class SearchService:
             logger.debug("no criteria passed to query")
             return []
 
-        logger.debug(f"Searching with query: {query}")
+        logger.trace(f"Searching with query: {query}")
 
         after_date = (
             (
                 query.after_date
                 if isinstance(query.after_date, datetime)
-                else datetime.fromisoformat(query.after_date)
+                else parse(query.after_date)
             )
             if query.after_date
             else None
         )
 
-        # permalink search
+        # search
         results = await self.repository.search(
             search_text=query.text,
             permalink=query.permalink,
             permalink_match=query.permalink_match,
             title=query.title,
             types=query.types,
-            entity_types=query.entity_types,
+            search_item_types=query.entity_types,
             after_date=after_date,
+            limit=limit,
+            offset=offset,
         )
 
         return results
@@ -107,15 +113,91 @@ class SearchService:
         # Add word boundaries
         variants.update(w.strip() for w in text.lower().split() if w.strip())
 
-        # Add trigrams for fuzzy matching
-        variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
+        # Trigrams disabled: They create massive search index bloat, increasing DB size significantly
+        # and slowing down indexing performance. FTS5 search works well without them.
+        # See: https://github.com/basicmachines-co/basic-memory/issues/351
+        # variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
 
         return variants
 
+    def _extract_entity_tags(self, entity: Entity) -> List[str]:
+        """Extract tags from entity metadata for search indexing.
+
+        Handles multiple tag formats:
+        - List format: ["tag1", "tag2"]
+        - String format: "['tag1', 'tag2']" or "[tag1, tag2]"
+        - Empty: [] or "[]"
+
+        Returns a list of tag strings for search indexing.
+        """
+        if not entity.entity_metadata or "tags" not in entity.entity_metadata:
+            return []
+
+        tags = entity.entity_metadata["tags"]
+
+        # Handle list format (preferred)
+        if isinstance(tags, list):
+            return [str(tag) for tag in tags if tag]
+
+        # Handle string format (legacy)
+        if isinstance(tags, str):
+            try:
+                # Parse string representation of list
+                parsed_tags = ast.literal_eval(tags)
+                if isinstance(parsed_tags, list):
+                    return [str(tag) for tag in parsed_tags if tag]
+            except (ValueError, SyntaxError):
+                # If parsing fails, treat as single tag
+                return [tags] if tags.strip() else []
+
+        return []  # pragma: no cover
+
     async def index_entity(
         self,
         entity: Entity,
         background_tasks: Optional[BackgroundTasks] = None,
+    ) -> None:
+        if background_tasks:
+            background_tasks.add_task(self.index_entity_data, entity)
+        else:
+            await self.index_entity_data(entity)
+
+    async def index_entity_data(
+        self,
+        entity: Entity,
+    ) -> None:
+        # delete all search index data associated with entity
+        await self.repository.delete_by_entity_id(entity_id=entity.id)
+
+        # reindex
+        await self.index_entity_markdown(
+            entity
+        ) if entity.is_markdown else await self.index_entity_file(entity)
+
+    async def index_entity_file(
+        self,
+        entity: Entity,
+    ) -> None:
+        # Index entity file with no content
+        await self.repository.index_item(
+            SearchIndexRow(
+                id=entity.id,
+                entity_id=entity.id,
+                type=SearchItemType.ENTITY.value,
+                title=entity.title,
+                file_path=entity.file_path,
+                metadata={
+                    "entity_type": entity.entity_type,
+                },
+                created_at=entity.created_at,
+                updated_at=entity.updated_at,
+                project_id=entity.project_id,
+            )
+        )
+
+    async def index_entity_markdown(
+        self,
+        entity: Entity,
     ) -> None:
         """Index an entity and all its observations and relations.
 
@@ -123,67 +205,80 @@ class SearchService:
         1. Entities
            - permalink: direct from entity (e.g., "specs/search")
            - file_path: physical file location
+           - project_id: project context for isolation
 
         2. Observations
            - permalink: entity permalink + /observations/id (e.g., "specs/search/observations/123")
            - file_path: parent entity's file (where observation is defined)
+           - project_id: inherited from parent entity
 
         3. Relations (only index outgoing relations defined in this file)
            - permalink: from_entity/relation_type/to_entity (e.g., "specs/search/implements/features/search-ui")
            - file_path: source entity's file (where relation is defined)
+           - project_id: inherited from source entity
 
         Each type gets its own row in the search index with appropriate metadata.
+        The project_id is automatically added by the repository when indexing.
         """
-        if background_tasks:
-            background_tasks.add_task(self.index_entity_data, entity)
-        else:
-            await self.index_entity_data(entity)
 
-    async def index_entity_data(
-        self,
-        entity: Entity,
-    ) -> None:
-        """Actually perform the indexing."""
+        # Collect all search index rows to batch insert at the end
+        rows_to_index = []
 
-        content_parts = []
+        content_stems = []
+        content_snippet = ""
         title_variants = self._generate_variants(entity.title)
-        content_parts.extend(title_variants)
+        content_stems.extend(title_variants)
 
         content = await self.file_service.read_entity_content(entity)
         if content:
-            content_parts.append(content)
+            content_stems.append(content)
+            content_snippet = f"{content[:250]}"
 
-        content_parts.extend(self._generate_variants(entity.permalink))
-        content_parts.extend(self._generate_variants(entity.file_path))
+        if entity.permalink:
+            content_stems.extend(self._generate_variants(entity.permalink))
 
-        entity_content = "\n".join(p for p in content_parts if p and p.strip())
+        content_stems.extend(self._generate_variants(entity.file_path))
 
-        # Index entity
-        await self.repository.index_item(
+        # Add entity tags from frontmatter to search content
+        entity_tags = self._extract_entity_tags(entity)
+        if entity_tags:
+            content_stems.extend(entity_tags)
+
+        entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
+
+        # Add entity row
+        rows_to_index.append(
             SearchIndexRow(
                 id=entity.id,
                 type=SearchItemType.ENTITY.value,
                 title=entity.title,
-                content=entity_content,
+                content_stems=entity_content_stems,
+                content_snippet=content_snippet,
                 permalink=entity.permalink,
                 file_path=entity.file_path,
+                entity_id=entity.id,
                 metadata={
                     "entity_type": entity.entity_type,
                 },
                 created_at=entity.created_at,
                 updated_at=entity.updated_at,
+                project_id=entity.project_id,
             )
         )
 
-        # Index each observation with permalink
+        # Add observation rows
         for obs in entity.observations:
             # Index with parent entity's file path since that's where it's defined
-            await self.repository.index_item(
+            obs_content_stems = "\n".join(
+                p for p in self._generate_variants(obs.content) if p and p.strip()
+            )
+            rows_to_index.append(
                 SearchIndexRow(
                     id=obs.id,
                     type=SearchItemType.OBSERVATION.value,
-                    title=f"{obs.category}: {obs.content[:50]}...",
-                    content=obs.content,
+                    title=f"{obs.category}: {obs.content[:100]}...",
+                    content_stems=obs_content_stems,
+                    content_snippet=obs.content,
                     permalink=obs.permalink,
                     file_path=entity.file_path,
                     category=obs.category,
@@ -193,10 +288,11 @@ class SearchService:
                     },
                     created_at=entity.created_at,
                     updated_at=entity.updated_at,
+                    project_id=entity.project_id,
                 )
             )
 
-        # Only index outgoing relations (ones defined in this file)
+        # Add relation rows (only outgoing relations defined in this file)
         for rel in entity.outgoing_relations:
             # Create descriptive title showing the relationship
             relation_title = (
@@ -205,21 +301,63 @@ class SearchService:
                 else f"{rel.from_entity.title}"
             )
 
-            await self.repository.index_item(
+            rel_content_stems = "\n".join(
+                p for p in self._generate_variants(relation_title) if p and p.strip()
+            )
+            rows_to_index.append(
                 SearchIndexRow(
                     id=rel.id,
                     title=relation_title,
                     permalink=rel.permalink,
+                    content_stems=rel_content_stems,
                     file_path=entity.file_path,
                     type=SearchItemType.RELATION.value,
+                    entity_id=entity.id,
                     from_id=rel.from_id,
                     to_id=rel.to_id,
                     relation_type=rel.relation_type,
                     created_at=entity.created_at,
                     updated_at=entity.updated_at,
+                    project_id=entity.project_id,
                 )
             )
 
-    async def delete_by_permalink(self, path_id: str):
+        # Batch insert all rows at once
+        await self.repository.bulk_index_items(rows_to_index)
+
+    async def delete_by_permalink(self, permalink: str):
+        """Delete an item from the search index."""
+        await self.repository.delete_by_permalink(permalink)
+
+    async def delete_by_entity_id(self, entity_id: int):
         """Delete an item from the search index."""
-        await self.repository.delete_by_permalink(path_id)
+        await self.repository.delete_by_entity_id(entity_id)
+
+    async def handle_delete(self, entity: Entity):
+        """Handle complete entity deletion from search index including observations and relations.
+
+        This replicates the logic from sync_service.handle_delete() to properly clean up
+        all search index entries for an entity and its related data.
+        """
+        logger.debug(
+            f"Cleaning up search index for entity_id={entity.id}, file_path={entity.file_path}, "
+            f"observations={len(entity.observations)}, relations={len(entity.outgoing_relations)}"
+        )
+
+        # Clean up search index - same logic as sync_service.handle_delete()
+        permalinks = (
+            [entity.permalink]
+            + [o.permalink for o in entity.observations]
+            + [r.permalink for r in entity.outgoing_relations]
+        )
+
+        logger.debug(
+            f"Deleting search index entries for entity_id={entity.id}, "
+            f"index_entries={len(permalinks)}"
+        )
+
+        for permalink in permalinks:
+            if permalink:
+                await self.delete_by_permalink(permalink)
+            else:
+                await self.delete_by_entity_id(entity.id)

basic_memory/sync/__init__.py

@@ -1,5 +1,6 @@
-from .file_change_scanner import FileChangeScanner
+"""Basic Memory sync services."""
+
 from .sync_service import SyncService
 from .watch_service import WatchService
 
-__all__ = ["SyncService", "FileChangeScanner", "WatchService"]
+__all__ = ["SyncService", "WatchService"]

basic_memory/sync/background_sync.py

@@ -0,0 +1,26 @@
+import asyncio
+
+from loguru import logger
+
+from basic_memory.config import get_project_config
+from basic_memory.sync import SyncService, WatchService
+
+
+async def sync_and_watch(
+    sync_service: SyncService, watch_service: WatchService
+):  # pragma: no cover
+    """Run sync and watch service."""
+
+    config = get_project_config()
+    logger.info(f"Starting watch service to sync file changes in dir: {config.home}")
+    # full sync
+    await sync_service.sync(config.home)
+
+    # watch changes
+    await watch_service.run()
+
+
+async def create_background_sync_task(
+    sync_service: SyncService, watch_service: WatchService
+):  # pragma: no cover
+    return asyncio.create_task(sync_and_watch(sync_service, watch_service))