basic-memory 0.0.0__py3-none-any.whl

basic_memory/sync/file_change_scanner.py

@@ -0,0 +1,162 @@
+"""Service for detecting changes between filesystem and database."""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, Sequence
+
+from loguru import logger
+
+from basic_memory.file_utils import compute_checksum
+from basic_memory.models import Entity
+from basic_memory.repository.entity_repository import EntityRepository
+from basic_memory.sync.utils import SyncReport
+
+
+@dataclass
+class FileState:
+    """State of a file including file path, permalink and checksum info."""
+
+    file_path: str
+    permalink: str
+    checksum: str
+
+
+@dataclass
+class ScanResult:
+    """Result of scanning a directory."""
+
+    # file_path -> checksum
+    files: Dict[str, str] = field(default_factory=dict)
+    # file_path -> error message
+    errors: Dict[str, str] = field(default_factory=dict)
+
+
+class FileChangeScanner:
+    """
+    Service for detecting changes between filesystem and database.
+    The filesystem is treated as the source of truth.
+    """
+
+    def __init__(self, entity_repository: EntityRepository):
+        self.entity_repository = entity_repository
+
+    async def scan_directory(self, directory: Path) -> ScanResult:
+        """
+        Scan directory for markdown files and their checksums.
+        Only processes .md files, logs and skips others.
+
+        Args:
+            directory: Directory to scan
+
+        Returns:
+            ScanResult containing found files and any errors
+        """
+        logger.debug(f"Scanning directory: {directory}")
+        result = ScanResult()
+
+        if not directory.exists():
+            logger.debug(f"Directory does not exist: {directory}")
+            return result
+
+        for path in directory.rglob("*"):
+            if not path.is_file() or not path.name.endswith(".md"):
+                if path.is_file():
+                    logger.debug(f"Skipping non-markdown file: {path}")
+                continue
+
+            try:
+                # Get relative path first - used in error reporting if needed
+                rel_path = str(path.relative_to(directory))
+                content = path.read_text()
+                checksum = await compute_checksum(content)
+
+                if checksum:  # Only store valid checksums
+                    result.files[rel_path] = checksum
+                else:
+                    result.errors[rel_path] = "Failed to compute checksum"
+
+            except Exception as e:
+                rel_path = str(path.relative_to(directory))
+                result.errors[rel_path] = str(e)
+                logger.error(f"Failed to read {rel_path}: {e}")
+
+        logger.debug(f"Found {len(result.files)} markdown files")
+        if result.errors:
+            logger.warning(f"Encountered {len(result.errors)} errors while scanning")
+
+        return result
+
+    async def find_changes(
+        self, directory: Path, db_file_state: Dict[str, FileState]
+    ) -> SyncReport:
+        """Find changes between filesystem and database."""
+        # Get current files and checksums
+        scan_result = await self.scan_directory(directory)
+        current_files = scan_result.files
+
+        # Build report
+        report = SyncReport(total=len(current_files))
+
+        # Track potentially moved files by checksum
+        files_by_checksum = {}  # checksum -> file_path
+
+        # First find potential new files and record checksums
+        for file_path, checksum in current_files.items():
+            logger.debug(f"{file_path} ({checksum[:8]})")
+
+            if file_path not in db_file_state:
+                # Could be new or could be the destination of a move
+                report.new.add(file_path)
+                files_by_checksum[checksum] = file_path
+            elif checksum != db_file_state[file_path].checksum:
+                report.modified.add(file_path)
+
+            report.checksums[file_path] = checksum
+
+        # Now detect moves and deletions
+        for db_file_path, db_state in db_file_state.items():
+            if db_file_path not in current_files:
+                if db_state.checksum in files_by_checksum:
+                    # Found a move - file exists at new path with same checksum
+                    new_path = files_by_checksum[db_state.checksum]
+                    report.moves[db_file_path] = new_path
+                    # Remove from new files since it's a move
+                    report.new.remove(new_path)
+                else:
+                    # Actually deleted
+                    report.deleted.add(db_file_path)
+
+        # Log summary
+        logger.debug(f"Total files: {report.total}")
+        logger.debug(f"Changes found: {report.total_changes}")
+        logger.debug(f"  New: {len(report.new)}")
+        logger.debug(f"  Modified: {len(report.modified)}")
+        logger.debug(f"  Moved: {len(report.moves)}")
+        logger.debug(f"  Deleted: {len(report.deleted)}")
+
+        if scan_result.errors:
+            logger.warning("Files skipped due to errors:")
+            for file_path, error in scan_result.errors.items():
+                logger.warning(f"  {file_path}: {error}")
+
+        return report
+
+    async def get_db_file_state(self, db_records: Sequence[Entity]) -> Dict[str, FileState]:
+        """Get file_path and checksums from database.
+        Args:
+            db_records: database records
+        Returns:
+            Dict mapping file paths to FileState
+            :param db_records: the data from the db
+        """
+        return {
+            r.file_path: FileState(
+                file_path=r.file_path, permalink=r.permalink, checksum=r.checksum
+            )
+            for r in db_records
+        }
+
+    async def find_knowledge_changes(self, directory: Path) -> SyncReport:
+        """Find changes in knowledge directory."""
+        db_file_state = await self.get_db_file_state(await self.entity_repository.find_all())
+        return await self.find_changes(directory=directory, db_file_state=db_file_state)

basic_memory/sync/sync_service.py

@@ -0,0 +1,140 @@
+"""Service for syncing files between filesystem and database."""
+
+from pathlib import Path
+from typing import Dict
+
+from loguru import logger
+
+from basic_memory.markdown import EntityParser, EntityMarkdown
+from basic_memory.repository import EntityRepository, RelationRepository
+from basic_memory.services import EntityService
+from basic_memory.services.search_service import SearchService
+from basic_memory.sync import FileChangeScanner
+from basic_memory.sync.utils import SyncReport
+
+
+class SyncService:
+    """Syncs documents and knowledge files with database.
+
+    Implements two-pass sync strategy for knowledge files to handle relations:
+    1. First pass creates/updates entities without relations
+    2. Second pass processes relations after all entities exist
+    """
+
+    def __init__(
+        self,
+        scanner: FileChangeScanner,
+        entity_service: EntityService,
+        entity_parser: EntityParser,
+        entity_repository: EntityRepository,
+        relation_repository: RelationRepository,
+        search_service: SearchService,
+    ):
+        self.scanner = scanner
+        self.entity_service = entity_service
+        self.entity_parser = entity_parser
+        self.entity_repository = entity_repository
+        self.relation_repository = relation_repository
+        self.search_service = search_service
+
+    async def handle_entity_deletion(self, file_path: str):
+        """Handle complete entity deletion including search index cleanup."""
+        # First get entity to get permalink before deletion
+        entity = await self.entity_repository.get_by_file_path(file_path)
+        if entity:
+            logger.debug(f"Deleting entity and cleaning up search index: {file_path}")
+
+            # Delete from db (this cascades to observations/relations)
+            await self.entity_service.delete_entity_by_file_path(file_path)
+
+            # Clean up search index
+            permalinks = (
+                [entity.permalink]
+                + [o.permalink for o in entity.observations]
+                + [r.permalink for r in entity.relations]
+            )
+            logger.debug(f"Deleting from search index: {permalinks}")
+            for permalink in permalinks:
+                await self.search_service.delete_by_permalink(permalink)
+
+        else:
+            logger.debug(f"No entity found to delete: {file_path}")
+
+    async def sync(self, directory: Path) -> SyncReport:
+        """Sync knowledge files with database."""
+        changes = await self.scanner.find_knowledge_changes(directory)
+        logger.info(f"Found {changes.total_changes} knowledge changes")
+
+        # Handle moves first
+        for old_path, new_path in changes.moves.items():
+            logger.debug(f"Moving entity: {old_path} -> {new_path}")
+            entity = await self.entity_repository.get_by_file_path(old_path)
+            if entity:
+                # Update file_path but keep the same permalink for link stability
+                updated = await self.entity_repository.update(
+                    entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
+                )
+                # update search index
+                await self.search_service.index_entity(updated)
+
+        # Handle deletions next
+        # remove rows from db for files no longer present
+        for file_path in changes.deleted:
+            await self.handle_entity_deletion(file_path)
+
+        # Parse files that need updating
+        parsed_entities: Dict[str, EntityMarkdown] = {}
+
+        for file_path in [*changes.new, *changes.modified]:
+            entity_markdown = await self.entity_parser.parse_file(directory / file_path)
+            parsed_entities[file_path] = entity_markdown
+
+        # First pass: Create/update entities
+        # entities will have a null checksum to indicate they are not complete
+        for file_path, entity_markdown in parsed_entities.items():
+            # if the file is new, create an entity
+            if file_path in changes.new:
+                logger.debug(f"Creating new entity_markdown: {file_path}")
+                await self.entity_service.create_entity_from_markdown(
+                    file_path, entity_markdown
+                )
+            # otherwise we need to update the entity and observations
+            else:
+                logger.debug(f"Updating entity_markdown: {file_path}")
+                await self.entity_service.update_entity_and_observations(
+                    file_path, entity_markdown
+                )
+
+        # Second pass
+        for file_path, entity_markdown in parsed_entities.items():
+            logger.debug(f"Updating relations for: {file_path}")
+
+            # Process relations
+            checksum = changes.checksums[file_path]
+            entity = await self.entity_service.update_entity_relations(
+                file_path, entity_markdown
+            )
+
+            # add to search index
+            await self.search_service.index_entity(entity)
+
+            # Set final checksum to mark sync complete
+            await self.entity_repository.update(entity.id, {"checksum": checksum})
+
+        # Third pass: Try to resolve any forward references
+        logger.debug("Attempting to resolve forward references")
+        for relation in await self.relation_repository.find_unresolved_relations():
+            target_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
+            # check we found a link that is not the source
+            if target_entity and target_entity.id != relation.from_id:
+                logger.debug(f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}")
+                await self.relation_repository.update(relation.id, {
+                    "to_id": target_entity.id,
+                    "to_name": target_entity.title  # Update to actual title
+                })
+
+                # update search index
+                await self.search_service.index_entity(target_entity)
+
+
+        return changes

basic_memory/sync/utils.py

@@ -0,0 +1,66 @@
+"""Types and utilities for file sync."""
+
+from dataclasses import dataclass, field
+from typing import Set, Dict, Optional
+from watchfiles import Change
+from basic_memory.services.file_service import FileService
+
+
+@dataclass
+class FileChange:
+    """A change to a file detected by the watch service.
+    
+    Attributes:
+        change_type: Type of change (added, modified, deleted)
+        path: Path to the file
+        checksum: File checksum (None for deleted files)
+    """
+    change_type: Change
+    path: str
+    checksum: Optional[str] = None
+
+    @classmethod
+    async def from_path(cls, path: str, change_type: Change, file_service: FileService) -> "FileChange":
+        """Create FileChange from a path, computing checksum if file exists.
+        
+        Args:
+            path: Path to the file
+            change_type: Type of change detected
+            file_service: Service to read file and compute checksum
+            
+        Returns:
+            FileChange with computed checksum for non-deleted files
+        """
+        file_path = file_service.path(path)
+        content, checksum = await file_service.read_file(file_path) if change_type != Change.deleted else (None, None)
+        return cls(path=file_path, change_type=change_type, checksum=checksum)
+
+
+@dataclass
+class SyncReport:
+    """Report of file changes found compared to database state.
+    
+    Attributes:
+        total: Total number of files in directory being synced
+        new: Files that exist on disk but not in database
+        modified: Files that exist in both but have different checksums
+        deleted: Files that exist in database but not on disk
+        moves: Files that have been moved from one location to another
+        checksums: Current checksums for files on disk
+    """
+    total: int = 0
+    new: Set[str] = field(default_factory=set)
+    modified: Set[str] = field(default_factory=set)
+    deleted: Set[str] = field(default_factory=set)
+    moves: Dict[str, str] = field(default_factory=dict)  # old_path -> new_path  
+    checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
+
+    @property
+    def total_changes(self) -> int:
+        """Total number of changes."""
+        return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
+
+    @property
+    def syned_files(self) -> int:
+        """Total number of files synced."""
+        return len(self.new) + len(self.modified) + len(self.moves)

basic_memory/sync/watch_service.py

@@ -0,0 +1,197 @@
+"""Watch service for Basic Memory."""
+
+import dataclasses
+
+from loguru import logger
+from pydantic import BaseModel
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional
+
+from rich import box
+from rich.console import Console
+from rich.live import Live
+from rich.table import Table
+from watchfiles import awatch, Change
+import os
+
+from basic_memory.config import ProjectConfig
+from basic_memory.sync.sync_service import SyncService
+from basic_memory.services.file_service import FileService
+
+
+class WatchEvent(BaseModel):
+    timestamp: datetime
+    path: str
+    action: str  # new, delete, etc
+    status: str  # success, error
+    checksum: Optional[str]
+    error: Optional[str] = None
+
+
+class WatchServiceState(BaseModel):
+    # Service status
+    running: bool = False
+    start_time: datetime = dataclasses.field(default_factory=datetime.now)
+    pid: int = dataclasses.field(default_factory=os.getpid)
+
+    # Stats
+    error_count: int = 0
+    last_error: Optional[datetime] = None
+    last_scan: Optional[datetime] = None
+
+    # File counts
+    synced_files: int = 0
+
+    # Recent activity
+    recent_events: List[WatchEvent] = dataclasses.field(default_factory=list)
+
+    def add_event(
+        self,
+        path: str,
+        action: str,
+        status: str,
+        checksum: Optional[str] = None,
+        error: Optional[str] = None,
+    ) -> WatchEvent:
+        event = WatchEvent(
+            timestamp=datetime.now(),
+            path=path,
+            action=action,
+            status=status,
+            checksum=checksum,
+            error=error,
+        )
+        self.recent_events.insert(0, event)
+        self.recent_events = self.recent_events[:100]  # Keep last 100
+        return event
+
+    def record_error(self, error: str):
+        self.error_count += 1
+        self.add_event(path="", action="sync", status="error", error=error)
+        self.last_error = datetime.now()
+
+
+class WatchService:
+    def __init__(self, sync_service: SyncService, file_service: FileService, config: ProjectConfig):
+        self.sync_service = sync_service
+        self.file_service = file_service
+        self.config = config
+        self.state = WatchServiceState()
+        self.status_path = config.home / ".basic-memory" / "watch-status.json"
+        self.status_path.parent.mkdir(parents=True, exist_ok=True)
+        self.console = Console()
+
+    def generate_table(self) -> Table:
+        """Generate status display table"""
+        table = Table(title="Basic Memory Sync Status")
+
+        # Add status row
+        table.add_column("Status", style="cyan")
+        table.add_column("Last Scan", style="cyan")
+        table.add_column("Files", style="cyan")
+        table.add_column("Errors", style="red")
+
+        # Add main status row
+        table.add_row(
+            "✓ Running" if self.state.running else "✗ Stopped",
+            self.state.last_scan.strftime("%H:%M:%S") if self.state.last_scan else "-",
+            str(self.state.synced_files),
+            f"{self.state.error_count} ({self.state.last_error.strftime('%H:%M:%S') if self.state.last_error else 'none'})",
+        )
+
+        if self.state.recent_events:
+            # Add recent events
+            table.add_section()
+            table.add_row("Recent Events", "", "", "")
+
+            for event in self.state.recent_events[:5]:  # Show last 5 events
+                color = {
+                    "new": "green",
+                    "modified": "yellow",
+                    "moved": "blue",
+                    "deleted": "red",
+                    "error": "red",
+                }.get(event.action, "white")
+
+                icon = {
+                    "new": "✚",
+                    "modified": "✎",
+                    "moved": "→",
+                    "deleted": "✖",
+                    "error": "!",
+                }.get(event.action, "*")
+
+                table.add_row(
+                    f"[{color}]{icon} {event.action}[/{color}]",
+                    event.timestamp.strftime("%H:%M:%S"),
+                    f"[{color}]{event.path}[/{color}]",
+                    f"[dim]{event.checksum[:8] if event.checksum else ''}[/dim]",
+                )
+
+        return table
+
+    async def run(self):
+        """Watch for file changes and sync them"""
+        self.state.running = True
+        self.state.start_time = datetime.now()
+        await self.write_status()
+
+        with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
+            try:
+                async for changes in awatch(
+                    self.config.home,
+                    watch_filter=self.filter_changes,
+                    debounce=self.config.sync_delay,
+                    recursive=True,
+                ):
+                    # Process changes
+                    await self.handle_changes(self.config.home)
+                    # Update display
+                    live.update(self.generate_table())
+
+            except Exception as e:
+                self.state.record_error(str(e))
+                await self.write_status()
+                raise
+            finally:
+                self.state.running = False
+                await self.write_status()
+
+    async def write_status(self):
+        """Write current state to status file"""
+        self.status_path.write_text(WatchServiceState.model_dump_json(self.state, indent=2))
+
+    def filter_changes(self, change: Change, path: str) -> bool:
+        """Filter to only watch markdown files"""
+        return path.endswith(".md") and not Path(path).name.startswith(".")
+
+    async def handle_changes(self, directory: Path):
+        """Process a batch of file changes"""
+
+        logger.debug(f"handling change in directory: {directory} ...")
+        # Process changes with timeout
+        report = await self.sync_service.sync(directory)
+        self.state.last_scan = datetime.now()
+        self.state.synced_files = report.total
+
+        # Update stats
+        for path in report.new:
+            self.state.add_event(
+                path=path, action="new", status="success", checksum=report.checksums[path]
+            )
+        for path in report.modified:
+            self.state.add_event(
+                path=path, action="modified", status="success", checksum=report.checksums[path]
+            )
+        for old_path, new_path in report.moves.items():
+            self.state.add_event(
+                path=f"{old_path} -> {new_path}",
+                action="moved",
+                status="success",
+                checksum=report.checksums[new_path],
+            )
+        for path in report.deleted:
+            self.state.add_event(path=path, action="deleted", status="success")
+
+        await self.write_status()

basic_memory/utils.py

@@ -0,0 +1,78 @@
+"""Utility functions for basic-memory."""
+import os
+import re
+import unicodedata
+from pathlib import Path
+
+from unidecode import unidecode
+
+
+def sanitize_name(name: str) -> str:
+    """
+    Sanitize a name for filesystem use:
+    - Convert to lowercase
+    - Replace spaces/punctuation with underscores
+    - Remove emojis and other special characters
+    - Collapse multiple underscores
+    - Trim leading/trailing underscores
+    """
+    # Normalize unicode to compose characters where possible
+    name = unicodedata.normalize("NFKD", name)
+    # Remove emojis and other special characters, keep only letters, numbers, spaces
+    name = "".join(c for c in name if c.isalnum() or c.isspace())
+    # Replace spaces with underscores
+    name = name.replace(" ", "_")
+    # Remove newline
+    name = name.replace("\n", "")
+    # Convert to lowercase
+    name = name.lower()
+    # Collapse multiple underscores and trim
+    name = re.sub(r"_+", "_", name).strip("_")
+
+    return name
+
+
+def generate_permalink(file_path: Path | str) -> str:
+    """Generate a stable permalink from a file path.
+
+    Args:
+        file_path: Original file path
+
+    Returns:
+        Normalized permalink that matches validation rules. Converts spaces and underscores
+        to hyphens for consistency.
+
+    Examples:
+        >>> generate_permalink("docs/My Feature.md")
+        'docs/my-feature'
+        >>> generate_permalink("specs/API (v2).md")
+        'specs/api-v2'
+        >>> generate_permalink("design/unified_model_refactor.md")
+        'design/unified-model-refactor'
+    """
+    # Remove extension
+    base = os.path.splitext(file_path)[0]
+
+    # Transliterate unicode to ascii
+    ascii_text = unidecode(base)
+
+    # Insert dash between camelCase
+    ascii_text = re.sub(r"([a-z0-9])([A-Z])", r"\1-\2", ascii_text)
+
+    # Convert to lowercase
+    lower_text = ascii_text.lower()
+
+    # replace underscores with hyphens
+    text_with_hyphens = lower_text.replace('_', '-')
+
+    # Replace remaining invalid chars with hyphens
+    clean_text = re.sub(r'[^a-z0-9/\-]', '-', text_with_hyphens)
+
+    # Collapse multiple hyphens
+    clean_text = re.sub(r'-+', '-', clean_text)
+
+    # Clean each path segment
+    segments = clean_text.split('/')
+    clean_segments = [s.strip('-') for s in segments]
+
+    return '/'.join(clean_segments)