basic-memory 0.14.4__py3-none-any.whl → 0.15.1__py3-none-any.whl

basic_memory/services/search_service.py

@@ -113,8 +113,10 @@ class SearchService:
         # Add word boundaries
         variants.update(w.strip() for w in text.lower().split() if w.strip())
 
-        # Add trigrams for fuzzy matching
-        variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
+        # Trigrams disabled: They create massive search index bloat, increasing DB size significantly
+        # and slowing down indexing performance. FTS5 search works well without them.
+        # See: https://github.com/basicmachines-co/basic-memory/issues/351
+        # variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
 
         return variants
 
@@ -219,6 +221,9 @@ class SearchService:
         The project_id is automatically added by the repository when indexing.
         """
 
+        # Collect all search index rows to batch insert at the end
+        rows_to_index = []
+
         content_stems = []
         content_snippet = ""
         title_variants = self._generate_variants(entity.title)
@@ -241,8 +246,8 @@ class SearchService:
 
         entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
 
-        # Index entity
-        await self.repository.index_item(
+        # Add entity row
+        rows_to_index.append(
             SearchIndexRow(
                 id=entity.id,
                 type=SearchItemType.ENTITY.value,
@@ -261,13 +266,13 @@ class SearchService:
             )
         )
 
-        # Index each observation with permalink
+        # Add observation rows
         for obs in entity.observations:
             # Index with parent entity's file path since that's where it's defined
             obs_content_stems = "\n".join(
                 p for p in self._generate_variants(obs.content) if p and p.strip()
             )
-            await self.repository.index_item(
+            rows_to_index.append(
                 SearchIndexRow(
                     id=obs.id,
                     type=SearchItemType.OBSERVATION.value,
@@ -287,7 +292,7 @@ class SearchService:
                 )
             )
 
-        # Only index outgoing relations (ones defined in this file)
+        # Add relation rows (only outgoing relations defined in this file)
         for rel in entity.outgoing_relations:
             # Create descriptive title showing the relationship
             relation_title = (
@@ -299,7 +304,7 @@ class SearchService:
             rel_content_stems = "\n".join(
                 p for p in self._generate_variants(relation_title) if p and p.strip()
             )
-            await self.repository.index_item(
+            rows_to_index.append(
                 SearchIndexRow(
                     id=rel.id,
                     title=relation_title,
@@ -317,6 +322,9 @@ class SearchService:
                 )
             )
 
+        # Batch insert all rows at once
+        await self.repository.bulk_index_items(rows_to_index)
+
     async def delete_by_permalink(self, permalink: str):
         """Delete an item from the search index."""
         await self.repository.delete_by_permalink(permalink)

basic_memory/sync/sync_service.py

@@ -1,21 +1,28 @@
 """Service for syncing files between filesystem and database."""
 
+import asyncio
 import os
 import time
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, Optional, Set, Tuple
 
 from loguru import logger
+from sqlalchemy import select
 from sqlalchemy.exc import IntegrityError
 
-from basic_memory.config import BasicMemoryConfig
+from basic_memory import db
+from basic_memory.config import BasicMemoryConfig, ConfigManager
 from basic_memory.file_utils import has_frontmatter
-from basic_memory.markdown import EntityParser
-from basic_memory.models import Entity
-from basic_memory.repository import EntityRepository, RelationRepository
+from basic_memory.ignore_utils import load_bmignore_patterns, should_ignore_path
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from basic_memory.models import Entity, Project
+from basic_memory.repository import EntityRepository, RelationRepository, ObservationRepository
+from basic_memory.repository.search_repository import SearchRepository
 from basic_memory.services import EntityService, FileService
+from basic_memory.services.link_resolver import LinkResolver
 from basic_memory.services.search_service import SearchService
 from basic_memory.services.sync_status_service import sync_status_tracker, SyncStatus
 
@@ -80,6 +87,43 @@ class SyncService:
         self.relation_repository = relation_repository
         self.search_service = search_service
         self.file_service = file_service
+        self._thread_pool = ThreadPoolExecutor(max_workers=app_config.sync_thread_pool_size)
+        # Load ignore patterns once at initialization for performance
+        self._ignore_patterns = load_bmignore_patterns()
+
+    async def _read_file_async(self, file_path: Path) -> str:
+        """Read file content in thread pool to avoid blocking the event loop."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(self._thread_pool, file_path.read_text, "utf-8")
+
+    async def _compute_checksum_async(self, path: str) -> str:
+        """Compute file checksum in thread pool to avoid blocking the event loop."""
+
+        def _sync_compute_checksum(path_str: str) -> str:
+            # Synchronous version for thread pool execution
+            path_obj = self.file_service.base_path / path_str
+
+            if self.file_service.is_markdown(path_str):
+                content = path_obj.read_text(encoding="utf-8")
+            else:
+                content = path_obj.read_bytes()
+
+            # Use the synchronous version of compute_checksum
+            import hashlib
+
+            if isinstance(content, str):
+                content_bytes = content.encode("utf-8")
+            else:
+                content_bytes = content
+            return hashlib.sha256(content_bytes).hexdigest()
+
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(self._thread_pool, _sync_compute_checksum, path)
+
+    def __del__(self):
+        """Cleanup thread pool when service is destroyed."""
+        if hasattr(self, "_thread_pool"):
+            self._thread_pool.shutdown(wait=False)
 
     async def sync(self, directory: Path, project_name: Optional[str] = None) -> SyncReport:
         """Sync all files with database."""
@@ -231,15 +275,25 @@ class SyncService:
 
     async def get_db_file_state(self) -> Dict[str, str]:
         """Get file_path and checksums from database.
-        Args:
-            db_records: database records
+
+        Optimized to query only the columns we need (file_path, checksum) without
+        loading full entities or their relationships. This is 10-100x faster for
+        large projects compared to loading all entities with observations/relations.
+
         Returns:
-            Dict mapping file paths to FileState
-            :param db_records: the data from the db
+            Dict mapping file paths to checksums
         """
-        db_records = await self.entity_repository.find_all()
-        logger.info(f"Found {len(db_records)} db records")
-        return {r.file_path: r.checksum or "" for r in db_records}
+        # Query only the columns we need - no entity objects or relationships
+        query = select(Entity.file_path, Entity.checksum).where(
+            Entity.project_id == self.entity_repository.project_id
+        )
+
+        async with db.scoped_session(self.entity_repository.session_maker) as session:
+            result = await session.execute(query)
+            rows = result.all()
+
+        logger.info(f"Found {len(rows)} db file records")
+        return {row.file_path: row.checksum or "" for row in rows}
 
     async def sync_file(
         self, path: str, new: bool = True
@@ -289,16 +343,18 @@ class SyncService:
         logger.debug(f"Parsing markdown file, path: {path}, new: {new}")
 
         file_path = self.entity_parser.base_path / path
-        file_content = file_path.read_text(encoding="utf-8")
+        file_content = await self._read_file_async(file_path)
         file_contains_frontmatter = has_frontmatter(file_content)
 
         # entity markdown will always contain front matter, so it can be used up create/update the entity
         entity_markdown = await self.entity_parser.parse_file(path)
 
-        # if the file contains frontmatter, resolve a permalink
-        if file_contains_frontmatter:
-            # Resolve permalink - this handles all the cases including conflicts
-            permalink = await self.entity_service.resolve_permalink(path, markdown=entity_markdown)
+        # if the file contains frontmatter, resolve a permalink (unless disabled)
+        if file_contains_frontmatter and not self.app_config.disable_permalinks:
+            # Resolve permalink - skip conflict checks during bulk sync for performance
+            permalink = await self.entity_service.resolve_permalink(
+                path, markdown=entity_markdown, skip_conflict_check=True
+            )
 
             # If permalink changed, update the file
             if permalink != entity_markdown.frontmatter.permalink:
@@ -326,7 +382,7 @@ class SyncService:
         # After updating relations, we need to compute the checksum again
         # This is necessary for files with wikilinks to ensure consistent checksums
         # after relation processing is complete
-        final_checksum = await self.file_service.compute_checksum(path)
+        final_checksum = await self._compute_checksum_async(path)
 
         # set checksum
         await self.entity_repository.update(entity.id, {"checksum": final_checksum})
@@ -350,10 +406,10 @@ class SyncService:
         Returns:
             Tuple of (entity, checksum)
         """
-        checksum = await self.file_service.compute_checksum(path)
+        checksum = await self._compute_checksum_async(path)
         if new:
-            # Generate permalink from path
-            await self.entity_service.resolve_permalink(path)
+            # Generate permalink from path - skip conflict checks during bulk sync
+            await self.entity_service.resolve_permalink(path, skip_conflict_check=True)
 
             # get file timestamps
             file_stats = self.file_service.file_stats(path)
@@ -487,11 +543,15 @@ class SyncService:
             updates = {"file_path": new_path}
 
             # If configured, also update permalink to match new path
-            if self.app_config.update_permalinks_on_move and self.file_service.is_markdown(
-                new_path
+            if (
+                self.app_config.update_permalinks_on_move
+                and not self.app_config.disable_permalinks
+                and self.file_service.is_markdown(new_path)
             ):
-                # generate new permalink value
-                new_permalink = await self.entity_service.resolve_permalink(new_path)
+                # generate new permalink value - skip conflict checks during bulk sync
+                new_permalink = await self.entity_service.resolve_permalink(
+                    new_path, skip_conflict_check=True
+                )
 
                 # write to file and get new checksum
                 new_checksum = await self.file_service.update_frontmatter(
@@ -548,12 +608,27 @@ class SyncService:
             # update search index
             await self.search_service.index_entity(updated)
 
-    async def resolve_relations(self):
-        """Try to resolve any unresolved relations"""
+    async def resolve_relations(self, entity_id: int | None = None):
+        """Try to resolve unresolved relations.
 
-        unresolved_relations = await self.relation_repository.find_unresolved_relations()
+        Args:
+            entity_id: If provided, only resolve relations for this specific entity.
+                      Otherwise, resolve all unresolved relations in the database.
+        """
 
-        logger.info("Resolving forward references", count=len(unresolved_relations))
+        if entity_id:
+            # Only get unresolved relations for the specific entity
+            unresolved_relations = (
+                await self.relation_repository.find_unresolved_relations_for_entity(entity_id)
+            )
+            logger.info(
+                f"Resolving forward references for entity {entity_id}",
+                count=len(unresolved_relations),
+            )
+        else:
+            # Get all unresolved relations (original behavior)
+            unresolved_relations = await self.relation_repository.find_unresolved_relations()
+            logger.info("Resolving all forward references", count=len(unresolved_relations))
 
         for relation in unresolved_relations:
             logger.trace(
@@ -608,19 +683,35 @@ class SyncService:
 
         logger.debug(f"Scanning directory {directory}")
         result = ScanResult()
+        ignored_count = 0
 
         for root, dirnames, filenames in os.walk(str(directory)):
-            # Skip dot directories in-place
-            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+            # Convert root to Path for easier manipulation
+            root_path = Path(root)
+
+            # Filter out ignored directories in-place
+            dirnames_to_remove = []
+            for dirname in dirnames:
+                dir_path = root_path / dirname
+                if should_ignore_path(dir_path, directory, self._ignore_patterns):
+                    dirnames_to_remove.append(dirname)
+                    ignored_count += 1
+
+            # Remove ignored directories from dirnames to prevent os.walk from descending
+            for dirname in dirnames_to_remove:
+                dirnames.remove(dirname)
 
             for filename in filenames:
-                # Skip dot files
-                if filename.startswith("."):
+                path = root_path / filename
+
+                # Check if file should be ignored
+                if should_ignore_path(path, directory, self._ignore_patterns):
+                    ignored_count += 1
+                    logger.trace(f"Ignoring file per .bmignore: {path.relative_to(directory)}")
                     continue
 
-                path = Path(root) / filename
                 rel_path = path.relative_to(directory).as_posix()
-                checksum = await self.file_service.compute_checksum(rel_path)
+                checksum = await self._compute_checksum_async(rel_path)
                 result.files[rel_path] = checksum
                 result.checksums[checksum] = rel_path
 
@@ -631,7 +722,55 @@ class SyncService:
             f"{directory} scan completed "
             f"directory={str(directory)} "
             f"files_found={len(result.files)} "
+            f"files_ignored={ignored_count} "
             f"duration_ms={duration_ms}"
         )
 
         return result
+
+
+async def get_sync_service(project: Project) -> SyncService:  # pragma: no cover
+    """Get sync service instance with all dependencies."""
+
+    app_config = ConfigManager().config
+    _, session_maker = await db.get_or_create_db(
+        db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM
+    )
+
+    project_path = Path(project.path)
+    entity_parser = EntityParser(project_path)
+    markdown_processor = MarkdownProcessor(entity_parser)
+    file_service = FileService(project_path, markdown_processor)
+
+    # Initialize repositories
+    entity_repository = EntityRepository(session_maker, project_id=project.id)
+    observation_repository = ObservationRepository(session_maker, project_id=project.id)
+    relation_repository = RelationRepository(session_maker, project_id=project.id)
+    search_repository = SearchRepository(session_maker, project_id=project.id)
+
+    # Initialize services
+    search_service = SearchService(search_repository, entity_repository, file_service)
+    link_resolver = LinkResolver(entity_repository, search_service)
+
+    # Initialize services
+    entity_service = EntityService(
+        entity_parser,
+        entity_repository,
+        observation_repository,
+        relation_repository,
+        file_service,
+        link_resolver,
+    )
+
+    # Create sync service
+    sync_service = SyncService(
+        app_config=app_config,
+        entity_service=entity_service,
+        entity_parser=entity_parser,
+        entity_repository=entity_repository,
+        relation_repository=relation_repository,
+        search_service=search_service,
+        file_service=file_service,
+    )
+
+    return sync_service

basic_memory/sync/watch_service.py

@@ -5,9 +5,10 @@ import os
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Sequence
 
 from basic_memory.config import BasicMemoryConfig, WATCH_STATUS_JSON
+from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
 from basic_memory.models import Project
 from basic_memory.repository import ProjectRepository
 from loguru import logger
@@ -15,6 +16,7 @@ from pydantic import BaseModel
 from rich.console import Console
 from watchfiles import awatch
 from watchfiles.main import FileChange, Change
+import time
 
 
 class WatchEvent(BaseModel):
@@ -81,54 +83,110 @@ class WatchService:
         self.state = WatchServiceState()
         self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
+        self._ignore_patterns_cache: dict[Path, Set[str]] = {}
 
         # quiet mode for mcp so it doesn't mess up stdout
         self.console = Console(quiet=quiet)
 
+    async def _schedule_restart(self, stop_event: asyncio.Event):
+        """Schedule a restart of the watch service after the configured interval."""
+        await asyncio.sleep(self.app_config.watch_project_reload_interval)
+        stop_event.set()
+
+    def _get_ignore_patterns(self, project_path: Path) -> Set[str]:
+        """Get or load ignore patterns for a project path."""
+        if project_path not in self._ignore_patterns_cache:
+            self._ignore_patterns_cache[project_path] = load_gitignore_patterns(project_path)
+        return self._ignore_patterns_cache[project_path]
+
+    async def _watch_projects_cycle(self, projects: Sequence[Project], stop_event: asyncio.Event):
+        """Run one cycle of watching the given projects until stop_event is set."""
+        project_paths = [project.path for project in projects]
+
+        async for changes in awatch(
+            *project_paths,
+            debounce=self.app_config.sync_delay,
+            watch_filter=self.filter_changes,
+            recursive=True,
+            stop_event=stop_event,
+        ):
+            # group changes by project and filter using ignore patterns
+            project_changes = defaultdict(list)
+            for change, path in changes:
+                for project in projects:
+                    if self.is_project_path(project, path):
+                        # Check if the file should be ignored based on gitignore patterns
+                        project_path = Path(project.path)
+                        file_path = Path(path)
+                        ignore_patterns = self._get_ignore_patterns(project_path)
+
+                        if should_ignore_path(file_path, project_path, ignore_patterns):
+                            logger.trace(
+                                f"Ignoring watched file change: {file_path.relative_to(project_path)}"
+                            )
+                            continue
+
+                        project_changes[project].append((change, path))
+                        break
+
+            # create coroutines to handle changes
+            change_handlers = [
+                self.handle_changes(project, changes)  # pyright: ignore
+                for project, changes in project_changes.items()
+            ]
+
+            # process changes
+            await asyncio.gather(*change_handlers)
+
     async def run(self):  # pragma: no cover
         """Watch for file changes and sync them"""
 
-        projects = await self.project_repository.get_active_projects()
-        project_paths = [project.path for project in projects]
+        self.state.running = True
+        self.state.start_time = datetime.now()
+        await self.write_status()
 
         logger.info(
             "Watch service started",
-            f"directories={project_paths}",
             f"debounce_ms={self.app_config.sync_delay}",
             f"pid={os.getpid()}",
         )
 
-        self.state.running = True
-        self.state.start_time = datetime.now()
-        await self.write_status()
-
         try:
-            async for changes in awatch(
-                *project_paths,
-                debounce=self.app_config.sync_delay,
-                watch_filter=self.filter_changes,
-                recursive=True,
-            ):
-                # group changes by project
-                project_changes = defaultdict(list)
-                for change, path in changes:
-                    for project in projects:
-                        if self.is_project_path(project, path):
-                            project_changes[project].append((change, path))
-                            break
-
-                # create coroutines to handle changes
-                change_handlers = [
-                    self.handle_changes(project, changes)  # pyright: ignore
-                    for project, changes in project_changes.items()
-                ]
-
-                # process changes
-                await asyncio.gather(*change_handlers)
+            while self.state.running:
+                # Clear ignore patterns cache to pick up any .gitignore changes
+                self._ignore_patterns_cache.clear()
+
+                # Reload projects to catch any new/removed projects
+                projects = await self.project_repository.get_active_projects()
+
+                project_paths = [project.path for project in projects]
+                logger.debug(f"Starting watch cycle for directories: {project_paths}")
+
+                # Create stop event for this watch cycle
+                stop_event = asyncio.Event()
+
+                # Schedule restart after configured interval to reload projects
+                timer_task = asyncio.create_task(self._schedule_restart(stop_event))
+
+                try:
+                    await self._watch_projects_cycle(projects, stop_event)
+                except Exception as e:
+                    logger.exception("Watch service error during cycle", error=str(e))
+                    self.state.record_error(str(e))
+                    await self.write_status()
+                    # Continue to next cycle instead of exiting
+                    await asyncio.sleep(5)  # Brief pause before retry
+                finally:
+                    # Cancel timer task if it's still running
+                    if not timer_task.done():
+                        timer_task.cancel()
+                        try:
+                            await timer_task
+                        except asyncio.CancelledError:
+                            pass
 
         except Exception as e:
             logger.exception("Watch service error", error=str(e))
-
             self.state.record_error(str(e))
             await self.write_status()
             raise
@@ -175,11 +233,8 @@ class WatchService:
 
     async def handle_changes(self, project: Project, changes: Set[FileChange]) -> None:
         """Process a batch of file changes"""
-        import time
-        from typing import List, Set
-
-        # Lazily initialize sync service for project changes
-        from basic_memory.cli.commands.sync import get_sync_service
+        # avoid circular imports
+        from basic_memory.sync.sync_service import get_sync_service
 
         sync_service = await get_sync_service(project)
         file_service = sync_service.file_service
@@ -288,9 +343,13 @@ class WatchService:
                 full_path = directory / path
                 if full_path.exists() and full_path.is_file():
                     # File still exists despite DELETE event - treat as modification
-                    logger.debug("File exists despite DELETE event, treating as modification", path=path)
+                    logger.debug(
+                        "File exists despite DELETE event, treating as modification", path=path
+                    )
                     entity, checksum = await sync_service.sync_file(path, new=False)
-                    self.state.add_event(path=path, action="modified", status="success", checksum=checksum)
+                    self.state.add_event(
+                        path=path, action="modified", status="success", checksum=checksum
+                    )
                     self.console.print(f"[yellow]✎[/yellow] {path} (atomic write)")
                     logger.info(f"atomic write detected: {path}")
                     processed.add(path)
@@ -302,10 +361,12 @@ class WatchService:
                     entity = await sync_service.entity_repository.get_by_file_path(path)
                     if entity is None:
                         # No entity means this was likely a directory - skip it
-                        logger.debug(f"Skipping deleted path with no entity (likely directory), path={path}")
+                        logger.debug(
+                            f"Skipping deleted path with no entity (likely directory), path={path}"
+                        )
                         processed.add(path)
                         continue
-                    
+
                     # File truly deleted
                     logger.debug("Processing deleted file", path=path)
                     await sync_service.handle_delete(path)

basic_memory/utils.py

@@ -223,7 +223,8 @@ def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
     if isinstance(tags, str):
         # Check if it's a JSON array string (common issue from AI assistants)
         import json
-        if tags.strip().startswith('[') and tags.strip().endswith(']'):
+
+        if tags.strip().startswith("[") and tags.strip().endswith("]"):
             try:
                 # Try to parse as JSON array
                 parsed_json = json.loads(tags)
@@ -233,7 +234,7 @@ def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
             except json.JSONDecodeError:
                 # Not valid JSON, fall through to comma-separated parsing
                 pass
-        
+
         # Split by comma, strip whitespace, then strip leading '#' characters
         return [tag.strip().lstrip("#") for tag in tags.split(",") if tag and tag.strip()]
 
@@ -330,8 +331,8 @@ def detect_potential_file_conflicts(file_path: str, existing_paths: List[str]) -
     return conflicts
 
 
-def validate_project_path(path: str, project_path: Path) -> bool:
-    """Ensure path stays within project boundaries."""
+def valid_project_path_value(path: str):
+    """Ensure project path is valid."""
     # Allow empty strings as they resolve to the project root
     if not path:
         return True
@@ -352,6 +353,15 @@ def validate_project_path(path: str, project_path: Path) -> bool:
     if path.strip() and any(ord(c) < 32 and c not in [" ", "\t"] for c in path):
         return False
 
+    return True
+
+
+def validate_project_path(path: str, project_path: Path) -> bool:
+    """Ensure path is valid and stays within project boundaries."""
+
+    if not valid_project_path_value(path):
+        return False
+
     try:
         resolved = (project_path / path).resolve()
         return resolved.is_relative_to(project_path.resolve())