basic-memory 0.7.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

basic_memory/services/file_service.py

@@ -1,15 +1,19 @@
 """Service for file operations with checksum tracking."""
 
+import mimetypes
+from os import stat_result
 from pathlib import Path
-from typing import Tuple, Union
+from typing import Any, Dict, Tuple, Union
 
 from loguru import logger
 
 from basic_memory import file_utils
+from basic_memory.file_utils import FileError
 from basic_memory.markdown.markdown_processor import MarkdownProcessor
 from basic_memory.models import Entity as EntityModel
 from basic_memory.schemas import Entity as EntitySchema
 from basic_memory.services.exceptions import FileOperationError
+from basic_memory.utils import FilePath
 
 
 class FileService:
@@ -57,7 +61,7 @@ class FileService:
         Returns:
             Raw content string without metadata sections
         """
-        logger.debug(f"Reading entity with permalink: {entity.permalink}")
+        logger.debug("Reading entity content", entity_id=entity.id, permalink=entity.permalink)
 
         file_path = self.get_entity_path(entity)
         markdown = await self.markdown_processor.read_file(file_path)
@@ -75,13 +79,13 @@ class FileService:
         path = self.get_entity_path(entity)
         await self.delete_file(path)
 
-    async def exists(self, path: Union[Path, str]) -> bool:
+    async def exists(self, path: FilePath) -> bool:
         """Check if file exists at the provided path.
 
         If path is relative, it is assumed to be relative to base_path.
 
         Args:
-            path: Path to check (Path object or string)
+            path: Path to check (Path or string)
 
         Returns:
             True if file exists, False otherwise
@@ -90,23 +94,25 @@ class FileService:
             FileOperationError: If check fails
         """
         try:
-            path = Path(path)
-            if path.is_absolute():
-                return path.exists()
+            # Convert string to Path if needed
+            path_obj = Path(path) if isinstance(path, str) else path
+
+            if path_obj.is_absolute():
+                return path_obj.exists()
             else:
-                return (self.base_path / path).exists()
+                return (self.base_path / path_obj).exists()
         except Exception as e:
-            logger.error(f"Failed to check file existence {path}: {e}")
+            logger.error("Failed to check file existence", path=str(path), error=str(e))
             raise FileOperationError(f"Failed to check file existence: {e}")
 
-    async def write_file(self, path: Union[Path, str], content: str) -> str:
+    async def write_file(self, path: FilePath, content: str) -> str:
         """Write content to file and return checksum.
 
         Handles both absolute and relative paths. Relative paths are resolved
         against base_path.
 
         Args:
-            path: Where to write (Path object or string)
+            path: Where to write (Path or string)
             content: Content to write
 
         Returns:
@@ -115,33 +121,43 @@ class FileService:
         Raises:
             FileOperationError: If write fails
         """
-        path = Path(path)
-        full_path = path if path.is_absolute() else self.base_path / path
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
 
         try:
             # Ensure parent directory exists
             await file_utils.ensure_directory(full_path.parent)
 
             # Write content atomically
+            logger.info(
+                "Writing file",
+                operation="write_file",
+                path=str(full_path),
+                content_length=len(content),
+                is_markdown=full_path.suffix.lower() == ".md",
+            )
+
             await file_utils.write_file_atomic(full_path, content)
 
             # Compute and return checksum
             checksum = await file_utils.compute_checksum(content)
-            logger.debug(f"wrote file: {full_path}, checksum: {checksum}")
+            logger.debug("File write completed", path=str(full_path), checksum=checksum)
             return checksum
 
         except Exception as e:
-            logger.error(f"Failed to write file {full_path}: {e}")
+            logger.exception("File write error", path=str(full_path), error=str(e))
             raise FileOperationError(f"Failed to write file: {e}")
 
-    async def read_file(self, path: Union[Path, str]) -> Tuple[str, str]:
+    # TODO remove read_file
+    async def read_file(self, path: FilePath) -> Tuple[str, str]:
         """Read file and compute checksum.
 
         Handles both absolute and relative paths. Relative paths are resolved
         against base_path.
 
         Args:
-            path: Path to read (Path object or string)
+            path: Path to read (Path or string)
 
         Returns:
             Tuple of (content, checksum)
@@ -149,28 +165,131 @@ class FileService:
         Raises:
             FileOperationError: If read fails
         """
-        path = Path(path)
-        full_path = path if path.is_absolute() else self.base_path / path
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
 
         try:
-            content = path.read_text()
+            logger.debug("Reading file", operation="read_file", path=str(full_path))
+
+            content = full_path.read_text()
             checksum = await file_utils.compute_checksum(content)
-            logger.debug(f"read file: {full_path}, checksum: {checksum}")
+
+            logger.debug(
+                "File read completed",
+                path=str(full_path),
+                checksum=checksum,
+                content_length=len(content),
+            )
             return content, checksum
 
         except Exception as e:
-            logger.error(f"Failed to read file {full_path}: {e}")
+            logger.exception("File read error", path=str(full_path), error=str(e))
             raise FileOperationError(f"Failed to read file: {e}")
 
-    async def delete_file(self, path: Union[Path, str]) -> None:
+    async def delete_file(self, path: FilePath) -> None:
         """Delete file if it exists.
 
         Handles both absolute and relative paths. Relative paths are resolved
         against base_path.
 
         Args:
-            path: Path to delete (Path object or string)
+            path: Path to delete (Path or string)
         """
-        path = Path(path)
-        full_path = path if path.is_absolute() else self.base_path / path
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
         full_path.unlink(missing_ok=True)
+
+    async def update_frontmatter(self, path: FilePath, updates: Dict[str, Any]) -> str:
+        """
+        Update frontmatter fields in a file while preserving all content.
+
+        Args:
+            path: Path to the file (Path or string)
+            updates: Dictionary of frontmatter fields to update
+
+        Returns:
+            Checksum of updated file
+        """
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
+        return await file_utils.update_frontmatter(full_path, updates)
+
+    async def compute_checksum(self, path: FilePath) -> str:
+        """Compute checksum for a file.
+
+        Args:
+            path: Path to the file (Path or string)
+
+        Returns:
+            Checksum of the file content
+
+        Raises:
+            FileError: If checksum computation fails
+        """
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
+
+        try:
+            if self.is_markdown(path):
+                # read str
+                content = full_path.read_text()
+            else:
+                # read bytes
+                content = full_path.read_bytes()
+            return await file_utils.compute_checksum(content)
+
+        except Exception as e:  # pragma: no cover
+            logger.error("Failed to compute checksum", path=str(full_path), error=str(e))
+            raise FileError(f"Failed to compute checksum for {path}: {e}")
+
+    def file_stats(self, path: FilePath) -> stat_result:
+        """Return file stats for a given path.
+
+        Args:
+            path: Path to the file (Path or string)
+
+        Returns:
+            File statistics
+        """
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
+        # get file timestamps
+        return full_path.stat()
+
+    def content_type(self, path: FilePath) -> str:
+        """Return content_type for a given path.
+
+        Args:
+            path: Path to the file (Path or string)
+
+        Returns:
+            MIME type of the file
+        """
+        # Convert string to Path if needed
+        path_obj = Path(path) if isinstance(path, str) else path
+        full_path = path_obj if path_obj.is_absolute() else self.base_path / path_obj
+        # get file timestamps
+        mime_type, _ = mimetypes.guess_type(full_path.name)
+
+        # .canvas files are json
+        if full_path.suffix == ".canvas":
+            mime_type = "application/json"
+
+        content_type = mime_type or "text/plain"
+        return content_type
+
+    def is_markdown(self, path: FilePath) -> bool:
+        """Check if a file is a markdown file.
+
+        Args:
+            path: Path to the file (Path or string)
+
+        Returns:
+            True if the file is a markdown file, False otherwise
+        """
+        return self.content_type(path) == "text/markdown"

basic_memory/services/link_resolver.py

@@ -1,14 +1,13 @@
 """Service for resolving markdown links to permalinks."""
 
-from typing import Optional, Tuple, List
+from typing import Optional, Tuple
 
 from loguru import logger
 
-from basic_memory.repository.entity_repository import EntityRepository
-from basic_memory.repository.search_repository import SearchIndexRow
-from basic_memory.services.search_service import SearchService
 from basic_memory.models import Entity
+from basic_memory.repository.entity_repository import EntityRepository
 from basic_memory.schemas.search import SearchQuery, SearchItemType
+from basic_memory.services.search_service import SearchService
 
 
 class LinkResolver:
@@ -41,8 +40,9 @@ class LinkResolver:
             return entity
 
         # 2. Try exact title match
-        entity = await self.entity_repository.get_by_title(clean_text)
-        if entity:
+        found = await self.entity_repository.get_by_title(clean_text)
+        if found and len(found) == 1:
+            entity = found[0]
             logger.debug(f"Found title match: {entity.title}")
             return entity
 
@@ -54,11 +54,12 @@ class LinkResolver:
 
             if results:
                 # Look for best match
-                best_match = self._select_best_match(clean_text, results)
+                best_match = min(results, key=lambda x: x.score)  # pyright: ignore
                 logger.debug(
                     f"Selected best match from {len(results)} results: {best_match.permalink}"
                 )
-                return await self.entity_repository.get_by_permalink(best_match.permalink)
+                if best_match.permalink:
+                    return await self.entity_repository.get_by_permalink(best_match.permalink)
 
         # if we couldn't find anything then return None
         return None
@@ -87,41 +88,3 @@ class LinkResolver:
             alias = alias.strip()
 
         return text, alias
-
-    def _select_best_match(self, search_text: str, results: List[SearchIndexRow]) -> SearchIndexRow:
-        """Select best match from search results.
-
-        Uses multiple criteria:
-        1. Word matches in title field
-        2. Word matches in path
-        3. Overall search score
-        """
-        # Get search terms for matching
-        terms = search_text.lower().split()
-
-        # Score each result
-        scored_results = []
-        for result in results:
-            # Start with base score (lower is better)
-            score = result.score
-            assert score is not None
-
-            # Parse path components
-            path_parts = result.permalink.lower().split("/")
-            last_part = path_parts[-1] if path_parts else ""
-
-            # Title word match boosts
-            term_matches = [term for term in terms if term in last_part]
-            if term_matches:
-                score *= 0.5  # Boost for each matching term
-
-            # Exact title match is best
-            if last_part == search_text.lower():
-                score *= 0.2
-
-            scored_results.append((score, result))
-
-        # Sort by score (lowest first) and return best
-        scored_results.sort(key=lambda x: x[0], reverse=True)
-
-        return scored_results[0][1]

basic_memory/services/search_service.py

@@ -3,8 +3,10 @@
 from datetime import datetime
 from typing import List, Optional, Set
 
+from dateparser import parse
 from fastapi import BackgroundTasks
 from loguru import logger
+from sqlalchemy import text
 
 from basic_memory.models import Entity
 from basic_memory.repository import EntityRepository
@@ -38,9 +40,10 @@ class SearchService:
 
     async def reindex_all(self, background_tasks: Optional[BackgroundTasks] = None) -> None:
         """Reindex all content from database."""
-        logger.info("Starting full reindex")
 
+        logger.info("Starting full reindex")
         # Clear and recreate search index
+        await self.repository.execute_query(text("DROP TABLE IF EXISTS search_index"), params={})
         await self.init_search_index()
 
         # Reindex all entities
@@ -69,7 +72,7 @@ class SearchService:
             (
                 query.after_date
                 if isinstance(query.after_date, datetime)
-                else datetime.fromisoformat(query.after_date)
+                else parse(query.after_date)
             )
             if query.after_date
             else None
@@ -118,6 +121,47 @@ class SearchService:
         self,
         entity: Entity,
         background_tasks: Optional[BackgroundTasks] = None,
+    ) -> None:
+        if background_tasks:
+            background_tasks.add_task(self.index_entity_data, entity)
+        else:
+            await self.index_entity_data(entity)
+
+    async def index_entity_data(
+        self,
+        entity: Entity,
+    ) -> None:
+        # delete all search index data associated with entity
+        await self.repository.delete_by_entity_id(entity_id=entity.id)
+
+        # reindex
+        await self.index_entity_markdown(
+            entity
+        ) if entity.is_markdown else await self.index_entity_file(entity)
+
+    async def index_entity_file(
+        self,
+        entity: Entity,
+    ) -> None:
+        # Index entity file with no content
+        await self.repository.index_item(
+            SearchIndexRow(
+                id=entity.id,
+                entity_id=entity.id,
+                type=SearchItemType.ENTITY.value,
+                title=entity.title,
+                file_path=entity.file_path,
+                metadata={
+                    "entity_type": entity.entity_type,
+                },
+                created_at=entity.created_at,
+                updated_at=entity.updated_at,
+            )
+        )
+
+    async def index_entity_markdown(
+        self,
+        entity: Entity,
     ) -> None:
         """Index an entity and all its observations and relations.
 
@@ -136,29 +180,43 @@ class SearchService:
 
         Each type gets its own row in the search index with appropriate metadata.
         """
-        if background_tasks:
-            background_tasks.add_task(self.index_entity_data, entity)
-        else:
-            await self.index_entity_data(entity)
 
-    async def index_entity_data(
-        self,
-        entity: Entity,
-    ) -> None:
-        """Actually perform the indexing."""
+        if entity.permalink is None:  # pragma: no cover
+            logger.error(
+                "Missing permalink for markdown entity",
+                entity_id=entity.id,
+                title=entity.title,
+                file_path=entity.file_path,
+            )
+            raise ValueError(
+                f"Entity permalink should not be None for markdown entity: {entity.id} ({entity.title})"
+            )
 
-        content_parts = []
+        content_stems = []
+        content_snippet = ""
         title_variants = self._generate_variants(entity.title)
-        content_parts.extend(title_variants)
+        content_stems.extend(title_variants)
 
         content = await self.file_service.read_entity_content(entity)
         if content:
-            content_parts.append(content)
+            content_stems.append(content)
+            content_snippet = f"{content[:250]}"
 
-        content_parts.extend(self._generate_variants(entity.permalink))
-        content_parts.extend(self._generate_variants(entity.file_path))
+        content_stems.extend(self._generate_variants(entity.permalink))
+        content_stems.extend(self._generate_variants(entity.file_path))
 
-        entity_content = "\n".join(p for p in content_parts if p and p.strip())
+        entity_content_stems = "\n".join(p for p in content_stems if p and p.strip())
+
+        if entity.permalink is None:  # pragma: no cover
+            logger.error(
+                "Missing permalink for markdown entity",
+                entity_id=entity.id,
+                title=entity.title,
+                file_path=entity.file_path,
+            )
+            raise ValueError(
+                f"Entity permalink should not be None for markdown entity: {entity.id} ({entity.title})"
+            )
 
         # Index entity
         await self.repository.index_item(
@@ -166,9 +224,11 @@ class SearchService:
                 id=entity.id,
                 type=SearchItemType.ENTITY.value,
                 title=entity.title,
-                content=entity_content,
+                content_stems=entity_content_stems,
+                content_snippet=content_snippet,
                 permalink=entity.permalink,
                 file_path=entity.file_path,
+                entity_id=entity.id,
                 metadata={
                     "entity_type": entity.entity_type,
                 },
@@ -180,12 +240,16 @@ class SearchService:
         # Index each observation with permalink
         for obs in entity.observations:
             # Index with parent entity's file path since that's where it's defined
+            obs_content_stems = "\n".join(
+                p for p in self._generate_variants(obs.content) if p and p.strip()
+            )
             await self.repository.index_item(
                 SearchIndexRow(
                     id=obs.id,
                     type=SearchItemType.OBSERVATION.value,
-                    title=f"{obs.category}: {obs.content[:50]}...",
-                    content=obs.content,
+                    title=f"{obs.category}: {obs.content[:100]}...",
+                    content_stems=obs_content_stems,
+                    content_snippet=obs.content,
                     permalink=obs.permalink,
                     file_path=entity.file_path,
                     category=obs.category,
@@ -207,13 +271,18 @@ class SearchService:
                 else f"{rel.from_entity.title}"
             )
 
+            rel_content_stems = "\n".join(
+                p for p in self._generate_variants(relation_title) if p and p.strip()
+            )
             await self.repository.index_item(
                 SearchIndexRow(
                     id=rel.id,
                     title=relation_title,
                     permalink=rel.permalink,
+                    content_stems=rel_content_stems,
                     file_path=entity.file_path,
                     type=SearchItemType.RELATION.value,
+                    entity_id=entity.id,
                     from_id=rel.from_id,
                     to_id=rel.to_id,
                     relation_type=rel.relation_type,
@@ -222,6 +291,10 @@ class SearchService:
                 )
             )
 
-    async def delete_by_permalink(self, path_id: str):
+    async def delete_by_permalink(self, permalink: str):
+        """Delete an item from the search index."""
+        await self.repository.delete_by_permalink(permalink)
+
+    async def delete_by_entity_id(self, entity_id: int):
         """Delete an item from the search index."""
-        await self.repository.delete_by_permalink(path_id)
+        await self.repository.delete_by_entity_id(entity_id)

basic_memory/sync/__init__.py

@@ -1,5 +1,6 @@
-from .file_change_scanner import FileChangeScanner
+"""Basic Memory sync services."""
+
 from .sync_service import SyncService
 from .watch_service import WatchService
 
-__all__ = ["SyncService", "FileChangeScanner", "WatchService"]
+__all__ = ["SyncService", "WatchService"]