basic-memory 0.0.0__py3-none-any.whl

basic_memory/services/file_service.py

@@ -0,0 +1,213 @@
+"""Service for file operations with checksum tracking."""
+
+from pathlib import Path
+from typing import Optional, Tuple
+
+from loguru import logger
+
+from basic_memory import file_utils
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
+from basic_memory.markdown.utils import entity_model_to_markdown
+from basic_memory.models import Entity as EntityModel
+from basic_memory.services.exceptions import FileOperationError
+from basic_memory.schemas import Entity as EntitySchema
+
+class FileService:
+    """
+    Service for handling file operations.
+
+    Features:
+    - Consistent file writing with checksums
+    - Frontmatter management
+    - Atomic operations
+    - Error handling
+    """
+
+    def __init__(
+        self,
+        base_path: Path,
+        markdown_processor: MarkdownProcessor,
+    ):
+        self.base_path = base_path
+        self.markdown_processor = markdown_processor
+
+    def get_entity_path(self, entity: EntityModel| EntitySchema) -> Path:
+        """Generate absolute filesystem path for entity."""
+        return self.base_path / f"{entity.file_path}"
+
+    # TODO move to tests
+    async def write_entity_file(
+        self,
+        entity: EntityModel,
+        content: Optional[str] = None,
+        expected_checksum: Optional[str] = None,
+    ) -> Tuple[Path, str]:
+        """Write entity to filesystem and return path and checksum.
+
+        Uses read->modify->write pattern:
+        1. Read existing file if it exists
+        2. Update with new content if provided
+        3. Write back atomically
+
+        Args:
+            entity: Entity model to write
+            content: Optional new content (preserves existing if None)
+            expected_checksum: Optional checksum to verify file hasn't changed
+
+        Returns:
+            Tuple of (file path, new checksum)
+
+        Raises:
+            FileOperationError: If write fails
+        """
+        try:
+            path = self.get_entity_path(entity)
+
+            # Read current state if file exists
+            if path.exists():
+                # read the existing file
+                existing_markdown = await self.markdown_processor.read_file(path)
+
+                # if content is supplied use it or existing content
+                content=content or existing_markdown.content
+            
+            # Create new file structure with provided content
+            markdown = entity_model_to_markdown(entity, content=content)
+
+            # Write back atomically
+            checksum = await self.markdown_processor.write_file(
+                path=path, markdown=markdown, expected_checksum=expected_checksum
+            )
+
+            return path, checksum
+
+        except Exception as e:
+            logger.exception(f"Failed to write entity file: {e}")
+            raise FileOperationError(f"Failed to write entity file: {e}")
+
+    async def read_entity_content(self, entity: EntityModel) -> str:
+        """Get entity's content without frontmatter or structured sections (used to index for search)
+
+        Args:
+            entity: Entity to read content for
+
+        Returns:
+            Raw content without frontmatter, observations, or relations
+
+        Raises:
+            FileOperationError: If entity file doesn't exist
+        """
+        logger.debug(f"Reading entity with permalink: {entity.permalink}")
+
+        try:
+            file_path = self.get_entity_path(entity)
+            markdown = await self.markdown_processor.read_file(file_path)
+            return markdown.content or ""
+
+        except Exception as e:
+            logger.error(f"Failed to read entity content: {e}")
+            raise FileOperationError(f"Failed to read entity content: {e}")
+
+    async def delete_entity_file(self, entity: EntityModel) -> None:
+        """Delete entity file from filesystem."""
+        try:
+            path = self.get_entity_path(entity)
+            await self.delete_file(path)
+        except Exception as e:
+            logger.error(f"Failed to delete entity file: {e}")
+            raise FileOperationError(f"Failed to delete entity file: {e}")
+
+    async def exists(self, path: Path) -> bool:
+        """
+        Check if file exists at the provided path. If path is relative, it is assumed to be relative to base_path.
+
+        Args:
+            path: Path to check
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        try:
+            if path.is_absolute():
+                return path.exists()
+            else:
+                return (self.base_path / path).exists()
+        except Exception as e:
+            logger.error(f"Failed to check file existence {path}: {e}")
+            raise FileOperationError(f"Failed to check file existence: {e}")
+
+    async def write_file(self, path: Path, content: str) -> str:
+        """
+        Write content to file and return checksum.
+
+        Args:
+            path: Path where to write
+            content: Content to write
+
+        Returns:
+            Checksum of written content
+
+        Raises:
+            FileOperationError: If write fails
+        """
+        
+        path = path if path.is_absolute() else self.base_path / path
+        try:
+            # Ensure parent directory exists
+            await file_utils.ensure_directory(path.parent)
+
+            # Write content atomically
+            await file_utils.write_file_atomic(path, content)
+
+            # Compute and return checksum
+            checksum = await file_utils.compute_checksum(content)
+            logger.debug(f"wrote file: {path}, checksum: {checksum} content: \n{content}")
+            return checksum
+
+        except Exception as e:
+            logger.error(f"Failed to write file {path}: {e}")
+            raise FileOperationError(f"Failed to write file: {e}")
+
+    async def read_file(self, path: Path) -> Tuple[str, str]:
+        """
+        Read file and compute checksum.
+
+        Args:
+            path: Path to read
+
+        Returns:
+            Tuple of (content, checksum)
+
+        Raises:
+            FileOperationError: If read fails
+        """
+        path = path if path.is_absolute() else self.base_path / path
+        try:
+            content = path.read_text()
+            checksum = await file_utils.compute_checksum(content)
+            logger.debug(f"read file: {path}, checksum: {checksum}")
+            return content, checksum
+
+        except Exception as e:
+            logger.error(f"Failed to read file {path}: {e}")
+            raise FileOperationError(f"Failed to read file: {e}")
+
+    async def delete_file(self, path: Path) -> None:
+        """
+        Delete file if it exists.
+
+        Args:
+            path: Path to delete
+
+        Raises:
+            FileOperationError: If deletion fails
+        """
+        path = path if path.is_absolute() else self.base_path / path
+        try:
+            path.unlink(missing_ok=True)
+        except Exception as e:
+            logger.error(f"Failed to delete file {path}: {e}")
+            raise FileOperationError(f"Failed to delete file: {e}")
+
+    def path(self, path_string: str, absolute: bool = False):
+        return Path( self.base_path / path_string ) if absolute else Path(path_string).relative_to(self.base_path)

basic_memory/services/link_resolver.py

@@ -0,0 +1,126 @@
+"""Service for resolving markdown links to permalinks."""
+
+from typing import Optional, Tuple, List
+
+from loguru import logger
+
+from basic_memory.repository.entity_repository import EntityRepository
+from basic_memory.services.search_service import SearchService
+from basic_memory.models import Entity
+from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
+
+
+class LinkResolver:
+    """Service for resolving markdown links to permalinks.
+
+    Uses a combination of exact matching and search-based resolution:
+    1. Try exact permalink match (fastest)
+    2. Try exact title match
+    3. Fall back to search for fuzzy matching
+    4. Generate new permalink if no match found
+    """
+
+    def __init__(self, entity_repository: EntityRepository, search_service: SearchService):
+        """Initialize with repositories."""
+        self.entity_repository = entity_repository
+        self.search_service = search_service
+
+    async def resolve_link(self, link_text: str, use_search: bool = True) -> Optional[Entity]:
+        """Resolve a markdown link to a permalink."""
+        logger.debug(f"Resolving link: {link_text}")
+
+        # Clean link text and extract any alias
+        clean_text, alias = self._normalize_link_text(link_text)
+
+        # 1. Try exact permalink match first (most efficient)
+        entity = await self.entity_repository.get_by_permalink(clean_text)
+        if entity:
+            logger.debug(f"Found exact permalink match: {entity.permalink}")
+            return entity
+
+        # 2. Try exact title match
+        entity = await self.entity_repository.get_by_title(clean_text)
+        if entity:
+            logger.debug(f"Found title match: {entity.title}")
+            return entity
+
+        if use_search:
+            
+            # 3. Fall back to search for fuzzy matching on title if specified
+            results = await self.search_service.search(
+                query=SearchQuery(title=clean_text, types=[SearchItemType.ENTITY]),
+            )
+    
+            if results:
+                # Look for best match
+                best_match = self._select_best_match(clean_text, results)
+                logger.debug(f"Selected best match from {len(results)} results: {best_match.permalink}")
+                return await self.entity_repository.get_by_permalink(best_match.permalink)
+    
+            # if we couldn't find anything then return None
+        return None
+
+    def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
+        """Normalize link text and extract alias if present.
+
+        Args:
+            link_text: Raw link text from markdown
+
+        Returns:
+            Tuple of (normalized_text, alias or None)
+        """
+        # Strip whitespace
+        text = link_text.strip()
+
+        # Remove enclosing brackets if present
+        if text.startswith("[[") and text.endswith("]]"):
+            text = text[2:-2]
+
+        # Handle Obsidian-style aliases (format: [[actual|alias]])
+        alias = None
+        if "|" in text:
+            text, alias = text.split("|", 1)
+            text = text.strip()
+            alias = alias.strip()
+
+        return text, alias
+
+    def _select_best_match(self, search_text: str, results: List[SearchResult]) -> Entity:
+        """Select best match from search results.
+
+        Uses multiple criteria:
+        1. Word matches in title field
+        2. Word matches in path
+        3. Overall search score
+        """
+        if not results:
+            raise ValueError("Cannot select from empty results")
+
+        # Get search terms for matching
+        terms = search_text.lower().split()
+
+        # Score each result
+        scored_results = []
+        for result in results:
+            # Start with base score (lower is better)
+            score = result.score
+
+            # Parse path components
+            path_parts = result.permalink.lower().split("/")
+            last_part = path_parts[-1] if path_parts else ""
+
+            # Title word match boosts
+            term_matches = [term for term in terms if term in last_part]
+            if term_matches:
+                score *= 0.5  # Boost for each matching term
+
+            # Exact title match is best
+            if last_part == search_text.lower():
+                score *= 0.2
+
+            scored_results.append((score, result))
+
+        # Sort by score (lowest first) and return best
+        scored_results.sort(key=lambda x: x[0], reverse=True)
+
+        return scored_results[0][1]

basic_memory/services/search_service.py

@@ -0,0 +1,218 @@
+"""Service for search operations."""
+
+from typing import List, Optional, Set
+
+from fastapi import BackgroundTasks
+from loguru import logger
+
+from basic_memory.models import Entity
+from basic_memory.repository import EntityRepository
+from basic_memory.repository.search_repository import SearchRepository, SearchIndexRow
+from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
+from basic_memory.services import FileService
+from basic_memory.services.exceptions import FileOperationError
+
+
+class SearchService:
+    """Service for search operations.
+
+    Supports three primary search modes:
+    1. Exact permalink lookup
+    2. Pattern matching with * (e.g., 'specs/*')
+    3. Full-text search across title/content
+    """
+
+    def __init__(
+        self,
+        search_repository: SearchRepository,
+        entity_repository: EntityRepository,
+        file_service: FileService,
+    ):
+        self.repository = search_repository
+        self.entity_repository = entity_repository
+        self.file_service = file_service
+
+    async def init_search_index(self):
+        """Create FTS5 virtual table if it doesn't exist."""
+        await self.repository.init_search_index()
+
+    async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
+        """Reindex all content from database."""
+        logger.info("Starting full reindex")
+
+        # Clear and recreate search index
+        await self.init_search_index()
+
+        # Reindex all entities
+        logger.debug("Indexing entities")
+        entities = await self.entity_repository.find_all()
+        for entity in entities:
+            await self.index_entity(entity, background_tasks)
+
+        logger.info("Reindex complete")
+
+    async def search(
+        self, query: SearchQuery, context: Optional[List[str]] = None
+    ) -> List[SearchResult]:
+        """Search across all indexed content.
+
+        Supports three modes:
+        1. Exact permalink: finds direct matches for a specific path
+        2. Pattern match: handles * wildcards in paths
+        3. Text search: full-text search across title/content
+        """
+        if query.no_criteria():
+            logger.debug("no criteria passed to query")
+            return []
+
+        logger.debug(f"Searching with query: {query}")
+
+        # permalink search
+        results = await self.repository.search(
+            search_text=query.text,
+            permalink=query.permalink,
+            permalink_match=query.permalink_match,
+            title=query.title,
+            types=query.types,
+            entity_types=query.entity_types,
+            after_date=query.after_date,
+        )
+
+        return results
+
+    def _generate_variants(self, text: str) -> Set[str]:
+        """Generate text variants for better fuzzy matching.
+
+        Creates variations of the text to improve match chances:
+        - Original form
+        - Lowercase form
+        - Path segments (for permalinks)
+        - Common word boundaries
+        """
+        variants = {text, text.lower()}
+
+        # Add path segments
+        if "/" in text:
+            variants.update(p.strip() for p in text.split("/") if p.strip())
+
+        # Add word boundaries
+        variants.update(w.strip() for w in text.lower().split() if w.strip())
+
+        # Add trigrams for fuzzy matching
+        variants.update(text[i : i + 3].lower() for i in range(len(text) - 2))
+
+        return variants
+
+    async def index_entity(
+        self,
+        entity: Entity,
+        background_tasks: Optional[BackgroundTasks] = None,
+    ) -> None:
+        """Index an entity and all its observations and relations.
+
+        Indexing structure:
+        1. Entities
+           - permalink: direct from entity (e.g., "specs/search")
+           - file_path: physical file location
+
+        2. Observations
+           - permalink: entity permalink + /observations/id (e.g., "specs/search/observations/123")
+           - file_path: parent entity's file (where observation is defined)
+
+        3. Relations (only index outgoing relations defined in this file)
+           - permalink: from_entity/relation_type/to_entity (e.g., "specs/search/implements/features/search-ui")
+           - file_path: source entity's file (where relation is defined)
+
+        Each type gets its own row in the search index with appropriate metadata.
+        """
+        if background_tasks:
+            background_tasks.add_task(self.index_entity_data, entity)
+        else:
+            await self.index_entity_data(entity)
+
+    async def index_entity_data(
+        self,
+        entity: Entity,
+    ) -> None:
+        """Actually perform the indexing."""
+
+        content_parts = []
+        title_variants = self._generate_variants(entity.title)
+        content_parts.extend(title_variants)
+
+        # TODO should we do something to content on indexing?
+        content = await self.file_service.read_entity_content(entity)
+        if content:
+            content_parts.append(content)
+
+        content_parts.extend(self._generate_variants(entity.permalink))
+        content_parts.extend(self._generate_variants(entity.file_path))
+
+        entity_content = "\n".join(p for p in content_parts if p and p.strip())
+
+        # Index entity
+        await self.repository.index_item(
+            SearchIndexRow(
+                id=entity.id,
+                type=SearchItemType.ENTITY.value,
+                title=entity.title,
+                content=entity_content,
+                permalink=entity.permalink,
+                file_path=entity.file_path,
+                metadata={
+                    "entity_type": entity.entity_type,
+                },
+                created_at=entity.created_at.isoformat(),
+                updated_at=entity.updated_at.isoformat(),
+            )
+        )
+
+        # Index each observation with permalink
+        for obs in entity.observations:
+            # Index with parent entity's file path since that's where it's defined
+            await self.repository.index_item(
+                SearchIndexRow(
+                    id=obs.id,
+                    type=SearchItemType.OBSERVATION.value,
+                    title=f"{obs.category}: {obs.content[:50]}...",
+                    content=obs.content,
+                    permalink=obs.permalink,
+                    file_path=entity.file_path,
+                    category=obs.category,
+                    entity_id=entity.id,
+                    metadata={
+                        "tags": obs.tags,
+                    },
+                    created_at=entity.created_at.isoformat(),
+                    updated_at=entity.updated_at.isoformat(),
+                )
+            )
+
+        # Only index outgoing relations (ones defined in this file)
+        for rel in entity.outgoing_relations:
+            # Create descriptive title showing the relationship
+            relation_title = (
+                f"{rel.from_entity.title} → {rel.to_entity.title}"
+                if rel.to_entity
+                else f"{rel.from_entity.title}"
+            )
+
+            await self.repository.index_item(
+                SearchIndexRow(
+                    id=rel.id,
+                    title=relation_title,
+                    content=rel.context or "",
+                    permalink=rel.permalink,
+                    file_path=entity.file_path,
+                    type=SearchItemType.RELATION.value,
+                    from_id=rel.from_id,
+                    to_id=rel.to_id,
+                    relation_type=rel.relation_type,
+                    created_at=entity.created_at.isoformat(),
+                    updated_at=entity.updated_at.isoformat(),
+                )
+            )
+
+    async def delete_by_permalink(self, path_id: str):
+        """Delete an item from the search index."""
+        await self.repository.delete_by_permalink(path_id)

basic_memory/services/service.py

@@ -0,0 +1,36 @@
+"""Base service class."""
+
+from datetime import datetime
+from typing import TypeVar, Generic, List, Sequence
+
+from basic_memory.models import Base
+from basic_memory.repository.repository import Repository
+
+T = TypeVar("T", bound=Base)
+R = TypeVar("R", bound=Repository)
+
+class BaseService(Generic[T]):
+    """Base service that takes a repository."""
+
+    def __init__(self, repository: R):
+        """Initialize service with repository."""
+        self.repository = repository
+
+    async def add(self, model: T) -> T:
+        """Add model to repository."""
+        return await self.repository.add(model)
+
+    async def add_all(self, models: List[T]) -> Sequence[T]:
+        """Add a List of models to repository."""
+        return await self.repository.add_all(models)
+
+    async def get_modified_since(self, since: datetime) -> Sequence[T]:
+        """Get all items modified since the given timestamp.
+        
+        Args:
+            since: Datetime to search from
+            
+        Returns:
+            Sequence of items modified since the timestamp
+        """
+        return await self.repository.find_modified_since(since)

basic_memory/sync/__init__.py

@@ -0,0 +1,5 @@
+from .file_change_scanner import FileChangeScanner
+from .sync_service import SyncService
+
+__all__ = ["SyncService", "FileChangeScanner"]
+