basic-memory 0.14.4__py3-none-any.whl → 0.15.0__py3-none-any.whl

basic_memory/sync/sync_service.py

@@ -1,7 +1,9 @@
 """Service for syncing files between filesystem and database."""
 
+import asyncio
 import os
 import time
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
@@ -10,12 +12,16 @@ from typing import Dict, Optional, Set, Tuple
 from loguru import logger
 from sqlalchemy.exc import IntegrityError
 
-from basic_memory.config import BasicMemoryConfig
+from basic_memory import db
+from basic_memory.config import BasicMemoryConfig, ConfigManager
 from basic_memory.file_utils import has_frontmatter
-from basic_memory.markdown import EntityParser
-from basic_memory.models import Entity
-from basic_memory.repository import EntityRepository, RelationRepository
+from basic_memory.ignore_utils import load_bmignore_patterns, should_ignore_path
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from basic_memory.models import Entity, Project
+from basic_memory.repository import EntityRepository, RelationRepository, ObservationRepository
+from basic_memory.repository.search_repository import SearchRepository
 from basic_memory.services import EntityService, FileService
+from basic_memory.services.link_resolver import LinkResolver
 from basic_memory.services.search_service import SearchService
 from basic_memory.services.sync_status_service import sync_status_tracker, SyncStatus
 
@@ -80,6 +86,43 @@ class SyncService:
         self.relation_repository = relation_repository
         self.search_service = search_service
         self.file_service = file_service
+        self._thread_pool = ThreadPoolExecutor(max_workers=app_config.sync_thread_pool_size)
+        # Load ignore patterns once at initialization for performance
+        self._ignore_patterns = load_bmignore_patterns()
+
+    async def _read_file_async(self, file_path: Path) -> str:
+        """Read file content in thread pool to avoid blocking the event loop."""
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(self._thread_pool, file_path.read_text, "utf-8")
+
+    async def _compute_checksum_async(self, path: str) -> str:
+        """Compute file checksum in thread pool to avoid blocking the event loop."""
+
+        def _sync_compute_checksum(path_str: str) -> str:
+            # Synchronous version for thread pool execution
+            path_obj = self.file_service.base_path / path_str
+
+            if self.file_service.is_markdown(path_str):
+                content = path_obj.read_text(encoding="utf-8")
+            else:
+                content = path_obj.read_bytes()
+
+            # Use the synchronous version of compute_checksum
+            import hashlib
+
+            if isinstance(content, str):
+                content_bytes = content.encode("utf-8")
+            else:
+                content_bytes = content
+            return hashlib.sha256(content_bytes).hexdigest()
+
+        loop = asyncio.get_event_loop()
+        return await loop.run_in_executor(self._thread_pool, _sync_compute_checksum, path)
+
+    def __del__(self):
+        """Cleanup thread pool when service is destroyed."""
+        if hasattr(self, "_thread_pool"):
+            self._thread_pool.shutdown(wait=False)
 
     async def sync(self, directory: Path, project_name: Optional[str] = None) -> SyncReport:
         """Sync all files with database."""
@@ -289,14 +332,14 @@ class SyncService:
         logger.debug(f"Parsing markdown file, path: {path}, new: {new}")
 
         file_path = self.entity_parser.base_path / path
-        file_content = file_path.read_text(encoding="utf-8")
+        file_content = await self._read_file_async(file_path)
         file_contains_frontmatter = has_frontmatter(file_content)
 
         # entity markdown will always contain front matter, so it can be used up create/update the entity
         entity_markdown = await self.entity_parser.parse_file(path)
 
-        # if the file contains frontmatter, resolve a permalink
-        if file_contains_frontmatter:
+        # if the file contains frontmatter, resolve a permalink (unless disabled)
+        if file_contains_frontmatter and not self.app_config.disable_permalinks:
             # Resolve permalink - this handles all the cases including conflicts
             permalink = await self.entity_service.resolve_permalink(path, markdown=entity_markdown)
 
@@ -326,7 +369,7 @@ class SyncService:
         # After updating relations, we need to compute the checksum again
         # This is necessary for files with wikilinks to ensure consistent checksums
         # after relation processing is complete
-        final_checksum = await self.file_service.compute_checksum(path)
+        final_checksum = await self._compute_checksum_async(path)
 
         # set checksum
         await self.entity_repository.update(entity.id, {"checksum": final_checksum})
@@ -350,7 +393,7 @@ class SyncService:
         Returns:
             Tuple of (entity, checksum)
         """
-        checksum = await self.file_service.compute_checksum(path)
+        checksum = await self._compute_checksum_async(path)
         if new:
             # Generate permalink from path
             await self.entity_service.resolve_permalink(path)
@@ -487,8 +530,10 @@ class SyncService:
             updates = {"file_path": new_path}
 
             # If configured, also update permalink to match new path
-            if self.app_config.update_permalinks_on_move and self.file_service.is_markdown(
-                new_path
+            if (
+                self.app_config.update_permalinks_on_move
+                and not self.app_config.disable_permalinks
+                and self.file_service.is_markdown(new_path)
             ):
                 # generate new permalink value
                 new_permalink = await self.entity_service.resolve_permalink(new_path)
@@ -548,12 +593,27 @@ class SyncService:
             # update search index
             await self.search_service.index_entity(updated)
 
-    async def resolve_relations(self):
-        """Try to resolve any unresolved relations"""
+    async def resolve_relations(self, entity_id: int | None = None):
+        """Try to resolve unresolved relations.
 
-        unresolved_relations = await self.relation_repository.find_unresolved_relations()
+        Args:
+            entity_id: If provided, only resolve relations for this specific entity.
+                      Otherwise, resolve all unresolved relations in the database.
+        """
 
-        logger.info("Resolving forward references", count=len(unresolved_relations))
+        if entity_id:
+            # Only get unresolved relations for the specific entity
+            unresolved_relations = (
+                await self.relation_repository.find_unresolved_relations_for_entity(entity_id)
+            )
+            logger.info(
+                f"Resolving forward references for entity {entity_id}",
+                count=len(unresolved_relations),
+            )
+        else:
+            # Get all unresolved relations (original behavior)
+            unresolved_relations = await self.relation_repository.find_unresolved_relations()
+            logger.info("Resolving all forward references", count=len(unresolved_relations))
 
         for relation in unresolved_relations:
             logger.trace(
@@ -608,19 +668,35 @@ class SyncService:
 
         logger.debug(f"Scanning directory {directory}")
         result = ScanResult()
+        ignored_count = 0
 
         for root, dirnames, filenames in os.walk(str(directory)):
-            # Skip dot directories in-place
-            dirnames[:] = [d for d in dirnames if not d.startswith(".")]
+            # Convert root to Path for easier manipulation
+            root_path = Path(root)
+
+            # Filter out ignored directories in-place
+            dirnames_to_remove = []
+            for dirname in dirnames:
+                dir_path = root_path / dirname
+                if should_ignore_path(dir_path, directory, self._ignore_patterns):
+                    dirnames_to_remove.append(dirname)
+                    ignored_count += 1
+
+            # Remove ignored directories from dirnames to prevent os.walk from descending
+            for dirname in dirnames_to_remove:
+                dirnames.remove(dirname)
 
             for filename in filenames:
-                # Skip dot files
-                if filename.startswith("."):
+                path = root_path / filename
+
+                # Check if file should be ignored
+                if should_ignore_path(path, directory, self._ignore_patterns):
+                    ignored_count += 1
+                    logger.trace(f"Ignoring file per .bmignore: {path.relative_to(directory)}")
                     continue
 
-                path = Path(root) / filename
                 rel_path = path.relative_to(directory).as_posix()
-                checksum = await self.file_service.compute_checksum(rel_path)
+                checksum = await self._compute_checksum_async(rel_path)
                 result.files[rel_path] = checksum
                 result.checksums[checksum] = rel_path
 
@@ -631,7 +707,55 @@ class SyncService:
             f"{directory} scan completed "
             f"directory={str(directory)} "
             f"files_found={len(result.files)} "
+            f"files_ignored={ignored_count} "
             f"duration_ms={duration_ms}"
         )
 
         return result
+
+
+async def get_sync_service(project: Project) -> SyncService:  # pragma: no cover
+    """Get sync service instance with all dependencies."""
+
+    app_config = ConfigManager().config
+    _, session_maker = await db.get_or_create_db(
+        db_path=app_config.database_path, db_type=db.DatabaseType.FILESYSTEM
+    )
+
+    project_path = Path(project.path)
+    entity_parser = EntityParser(project_path)
+    markdown_processor = MarkdownProcessor(entity_parser)
+    file_service = FileService(project_path, markdown_processor)
+
+    # Initialize repositories
+    entity_repository = EntityRepository(session_maker, project_id=project.id)
+    observation_repository = ObservationRepository(session_maker, project_id=project.id)
+    relation_repository = RelationRepository(session_maker, project_id=project.id)
+    search_repository = SearchRepository(session_maker, project_id=project.id)
+
+    # Initialize services
+    search_service = SearchService(search_repository, entity_repository, file_service)
+    link_resolver = LinkResolver(entity_repository, search_service)
+
+    # Initialize services
+    entity_service = EntityService(
+        entity_parser,
+        entity_repository,
+        observation_repository,
+        relation_repository,
+        file_service,
+        link_resolver,
+    )
+
+    # Create sync service
+    sync_service = SyncService(
+        app_config=app_config,
+        entity_service=entity_service,
+        entity_parser=entity_parser,
+        entity_repository=entity_repository,
+        relation_repository=relation_repository,
+        search_service=search_service,
+        file_service=file_service,
+    )
+
+    return sync_service

basic_memory/sync/watch_service.py

@@ -5,9 +5,10 @@ import os
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
-from typing import List, Optional, Set
+from typing import List, Optional, Set, Sequence
 
 from basic_memory.config import BasicMemoryConfig, WATCH_STATUS_JSON
+from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
 from basic_memory.models import Project
 from basic_memory.repository import ProjectRepository
 from loguru import logger
@@ -15,6 +16,7 @@ from pydantic import BaseModel
 from rich.console import Console
 from watchfiles import awatch
 from watchfiles.main import FileChange, Change
+import time
 
 
 class WatchEvent(BaseModel):
@@ -81,54 +83,110 @@ class WatchService:
         self.state = WatchServiceState()
         self.status_path = Path.home() / ".basic-memory" / WATCH_STATUS_JSON
         self.status_path.parent.mkdir(parents=True, exist_ok=True)
+        self._ignore_patterns_cache: dict[Path, Set[str]] = {}
 
         # quiet mode for mcp so it doesn't mess up stdout
         self.console = Console(quiet=quiet)
 
+    async def _schedule_restart(self, stop_event: asyncio.Event):
+        """Schedule a restart of the watch service after the configured interval."""
+        await asyncio.sleep(self.app_config.watch_project_reload_interval)
+        stop_event.set()
+
+    def _get_ignore_patterns(self, project_path: Path) -> Set[str]:
+        """Get or load ignore patterns for a project path."""
+        if project_path not in self._ignore_patterns_cache:
+            self._ignore_patterns_cache[project_path] = load_gitignore_patterns(project_path)
+        return self._ignore_patterns_cache[project_path]
+
+    async def _watch_projects_cycle(self, projects: Sequence[Project], stop_event: asyncio.Event):
+        """Run one cycle of watching the given projects until stop_event is set."""
+        project_paths = [project.path for project in projects]
+
+        async for changes in awatch(
+            *project_paths,
+            debounce=self.app_config.sync_delay,
+            watch_filter=self.filter_changes,
+            recursive=True,
+            stop_event=stop_event,
+        ):
+            # group changes by project and filter using ignore patterns
+            project_changes = defaultdict(list)
+            for change, path in changes:
+                for project in projects:
+                    if self.is_project_path(project, path):
+                        # Check if the file should be ignored based on gitignore patterns
+                        project_path = Path(project.path)
+                        file_path = Path(path)
+                        ignore_patterns = self._get_ignore_patterns(project_path)
+
+                        if should_ignore_path(file_path, project_path, ignore_patterns):
+                            logger.trace(
+                                f"Ignoring watched file change: {file_path.relative_to(project_path)}"
+                            )
+                            continue
+
+                        project_changes[project].append((change, path))
+                        break
+
+            # create coroutines to handle changes
+            change_handlers = [
+                self.handle_changes(project, changes)  # pyright: ignore
+                for project, changes in project_changes.items()
+            ]
+
+            # process changes
+            await asyncio.gather(*change_handlers)
+
     async def run(self):  # pragma: no cover
         """Watch for file changes and sync them"""
 
-        projects = await self.project_repository.get_active_projects()
-        project_paths = [project.path for project in projects]
+        self.state.running = True
+        self.state.start_time = datetime.now()
+        await self.write_status()
 
         logger.info(
             "Watch service started",
-            f"directories={project_paths}",
             f"debounce_ms={self.app_config.sync_delay}",
             f"pid={os.getpid()}",
         )
 
-        self.state.running = True
-        self.state.start_time = datetime.now()
-        await self.write_status()
-
         try:
-            async for changes in awatch(
-                *project_paths,
-                debounce=self.app_config.sync_delay,
-                watch_filter=self.filter_changes,
-                recursive=True,
-            ):
-                # group changes by project
-                project_changes = defaultdict(list)
-                for change, path in changes:
-                    for project in projects:
-                        if self.is_project_path(project, path):
-                            project_changes[project].append((change, path))
-                            break
-
-                # create coroutines to handle changes
-                change_handlers = [
-                    self.handle_changes(project, changes)  # pyright: ignore
-                    for project, changes in project_changes.items()
-                ]
-
-                # process changes
-                await asyncio.gather(*change_handlers)
+            while self.state.running:
+                # Clear ignore patterns cache to pick up any .gitignore changes
+                self._ignore_patterns_cache.clear()
+
+                # Reload projects to catch any new/removed projects
+                projects = await self.project_repository.get_active_projects()
+
+                project_paths = [project.path for project in projects]
+                logger.debug(f"Starting watch cycle for directories: {project_paths}")
+
+                # Create stop event for this watch cycle
+                stop_event = asyncio.Event()
+
+                # Schedule restart after configured interval to reload projects
+                timer_task = asyncio.create_task(self._schedule_restart(stop_event))
+
+                try:
+                    await self._watch_projects_cycle(projects, stop_event)
+                except Exception as e:
+                    logger.exception("Watch service error during cycle", error=str(e))
+                    self.state.record_error(str(e))
+                    await self.write_status()
+                    # Continue to next cycle instead of exiting
+                    await asyncio.sleep(5)  # Brief pause before retry
+                finally:
+                    # Cancel timer task if it's still running
+                    if not timer_task.done():
+                        timer_task.cancel()
+                        try:
+                            await timer_task
+                        except asyncio.CancelledError:
+                            pass
 
         except Exception as e:
             logger.exception("Watch service error", error=str(e))
-
             self.state.record_error(str(e))
             await self.write_status()
             raise
@@ -175,11 +233,8 @@ class WatchService:
 
     async def handle_changes(self, project: Project, changes: Set[FileChange]) -> None:
         """Process a batch of file changes"""
-        import time
-        from typing import List, Set
-
-        # Lazily initialize sync service for project changes
-        from basic_memory.cli.commands.sync import get_sync_service
+        # avoid circular imports
+        from basic_memory.sync.sync_service import get_sync_service
 
         sync_service = await get_sync_service(project)
         file_service = sync_service.file_service
@@ -288,9 +343,13 @@ class WatchService:
                 full_path = directory / path
                 if full_path.exists() and full_path.is_file():
                     # File still exists despite DELETE event - treat as modification
-                    logger.debug("File exists despite DELETE event, treating as modification", path=path)
+                    logger.debug(
+                        "File exists despite DELETE event, treating as modification", path=path
+                    )
                     entity, checksum = await sync_service.sync_file(path, new=False)
-                    self.state.add_event(path=path, action="modified", status="success", checksum=checksum)
+                    self.state.add_event(
+                        path=path, action="modified", status="success", checksum=checksum
+                    )
                     self.console.print(f"[yellow]✎[/yellow] {path} (atomic write)")
                     logger.info(f"atomic write detected: {path}")
                     processed.add(path)
@@ -302,10 +361,12 @@ class WatchService:
                     entity = await sync_service.entity_repository.get_by_file_path(path)
                     if entity is None:
                         # No entity means this was likely a directory - skip it
-                        logger.debug(f"Skipping deleted path with no entity (likely directory), path={path}")
+                        logger.debug(
+                            f"Skipping deleted path with no entity (likely directory), path={path}"
+                        )
                         processed.add(path)
                         continue
-                    
+
                     # File truly deleted
                     logger.debug("Processing deleted file", path=path)
                     await sync_service.handle_delete(path)

basic_memory/utils.py

@@ -223,7 +223,8 @@ def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
     if isinstance(tags, str):
         # Check if it's a JSON array string (common issue from AI assistants)
         import json
-        if tags.strip().startswith('[') and tags.strip().endswith(']'):
+
+        if tags.strip().startswith("[") and tags.strip().endswith("]"):
             try:
                 # Try to parse as JSON array
                 parsed_json = json.loads(tags)
@@ -233,7 +234,7 @@ def parse_tags(tags: Union[List[str], str, None]) -> List[str]:
             except json.JSONDecodeError:
                 # Not valid JSON, fall through to comma-separated parsing
                 pass
-        
+
         # Split by comma, strip whitespace, then strip leading '#' characters
         return [tag.strip().lstrip("#") for tag in tags.split(",") if tag and tag.strip()]
 
@@ -330,8 +331,8 @@ def detect_potential_file_conflicts(file_path: str, existing_paths: List[str]) -
     return conflicts
 
 
-def validate_project_path(path: str, project_path: Path) -> bool:
-    """Ensure path stays within project boundaries."""
+def valid_project_path_value(path: str):
+    """Ensure project path is valid."""
     # Allow empty strings as they resolve to the project root
     if not path:
         return True
@@ -352,6 +353,15 @@ def validate_project_path(path: str, project_path: Path) -> bool:
     if path.strip() and any(ord(c) < 32 and c not in [" ", "\t"] for c in path):
         return False
 
+    return True
+
+
+def validate_project_path(path: str, project_path: Path) -> bool:
+    """Ensure path is valid and stays within project boundaries."""
+
+    if not valid_project_path_value(path):
+        return False
+
     try:
         resolved = (project_path / path).resolve()
         return resolved.is_relative_to(project_path.resolve())

{basic_memory-0.14.4.dist-info → basic_memory-0.15.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: basic-memory
-Version: 0.14.4
+Version: 0.15.0
 Summary: Local-first knowledge management combining Zettelkasten with knowledge graphs
 Project-URL: Homepage, https://github.com/basicmachines-co/basic-memory
 Project-URL: Repository, https://github.com/basicmachines-co/basic-memory
@@ -9,11 +9,12 @@ Author-email: Basic Machines <hello@basic-machines.co>
 License: AGPL-3.0-or-later
 License-File: LICENSE
 Requires-Python: >=3.12.1
+Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: aiosqlite>=0.20.0
 Requires-Dist: alembic>=1.14.1
 Requires-Dist: dateparser>=1.2.0
 Requires-Dist: fastapi[standard]>=0.115.8
-Requires-Dist: fastmcp==2.10.2
+Requires-Dist: fastmcp>=2.10.2
 Requires-Dist: greenlet>=3.1.1
 Requires-Dist: icecream>=2.1.3
 Requires-Dist: loguru>=0.7.3
@@ -409,10 +410,10 @@ canvas(nodes, edges, title, folder) - Generate knowledge visualizations
 
 See the [Documentation](https://memory.basicmachines.co/) for more info, including:
 
-- [Complete User Guide](https://memory.basicmachines.co/docs/user-guide)
-- [CLI tools](https://memory.basicmachines.co/docs/cli-reference)
-- [Managing multiple Projects](https://memory.basicmachines.co/docs/cli-reference#project)
-- [Importing data from OpenAI/Claude Projects](https://memory.basicmachines.co/docs/cli-reference#import)
+- [Complete User Guide](https://docs.basicmemory.com/user-guide/)
+- [CLI tools](https://docs.basicmemory.com/guides/cli-reference/)
+- [Managing multiple Projects](https://docs.basicmemory.com/guides/cli-reference/#project)
+- [Importing data from OpenAI/Claude Projects](https://docs.basicmemory.com/guides/cli-reference/#import)
 
 ## License