basic-memory 0.16.1__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/deps.py

@@ -1,411 +1,16 @@
-"""Dependency injection functions for basic-memory services."""
-
-from typing import Annotated
-from loguru import logger
-
-from fastapi import Depends, HTTPException, Path, status, Request
-from sqlalchemy.ext.asyncio import (
-    AsyncSession,
-    AsyncEngine,
-    async_sessionmaker,
-)
-import pathlib
-
-from basic_memory import db
-from basic_memory.config import ProjectConfig, BasicMemoryConfig, ConfigManager
-from basic_memory.importers import (
-    ChatGPTImporter,
-    ClaudeConversationsImporter,
-    ClaudeProjectsImporter,
-    MemoryJsonImporter,
-)
-from basic_memory.markdown import EntityParser
-from basic_memory.markdown.markdown_processor import MarkdownProcessor
-from basic_memory.repository.entity_repository import EntityRepository
-from basic_memory.repository.observation_repository import ObservationRepository
-from basic_memory.repository.project_repository import ProjectRepository
-from basic_memory.repository.relation_repository import RelationRepository
-from basic_memory.repository.search_repository import SearchRepository
-from basic_memory.services import EntityService, ProjectService
-from basic_memory.services.context_service import ContextService
-from basic_memory.services.directory_service import DirectoryService
-from basic_memory.services.file_service import FileService
-from basic_memory.services.link_resolver import LinkResolver
-from basic_memory.services.search_service import SearchService
-from basic_memory.sync import SyncService
-from basic_memory.utils import generate_permalink
-
-
-def get_app_config() -> BasicMemoryConfig:  # pragma: no cover
-    app_config = ConfigManager().config
-    return app_config
-
-
-AppConfigDep = Annotated[BasicMemoryConfig, Depends(get_app_config)]  # pragma: no cover
-
-
-## project
-
-
-async def get_project_config(
-    project: "ProjectPathDep", project_repository: "ProjectRepositoryDep"
-) -> ProjectConfig:  # pragma: no cover
-    """Get the current project referenced from request state.
-
-    Args:
-        request: The current request object
-        project_repository: Repository for project operations
-
-    Returns:
-        The resolved project config
-
-    Raises:
-        HTTPException: If project is not found
-    """
-    # Convert project name to permalink for lookup
-    project_permalink = generate_permalink(str(project))
-    project_obj = await project_repository.get_by_permalink(project_permalink)
-    if project_obj:
-        return ProjectConfig(name=project_obj.name, home=pathlib.Path(project_obj.path))
-
-    # Not found
-    raise HTTPException(  # pragma: no cover
-        status_code=status.HTTP_404_NOT_FOUND, detail=f"Project '{project}' not found."
-    )
-
-
-ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]  # pragma: no cover
-
-## sqlalchemy
-
-
-async def get_engine_factory(
-    request: Request,
-) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:  # pragma: no cover
-    """Get cached engine and session maker from app state.
-
-    For API requests, returns cached connections from app.state for optimal performance.
-    For non-API contexts (CLI), falls back to direct database connection.
-    """
-    # Try to get cached connections from app state (API context)
-    if (
-        hasattr(request, "app")
-        and hasattr(request.app.state, "engine")
-        and hasattr(request.app.state, "session_maker")
-    ):
-        return request.app.state.engine, request.app.state.session_maker
-
-    # Fallback for non-API contexts (CLI)
-    logger.debug("Using fallback database connection for non-API context")
-    app_config = get_app_config()
-    engine, session_maker = await db.get_or_create_db(app_config.database_path)
-    return engine, session_maker
-
-
-EngineFactoryDep = Annotated[
-    tuple[AsyncEngine, async_sessionmaker[AsyncSession]], Depends(get_engine_factory)
-]
-
-
-async def get_session_maker(engine_factory: EngineFactoryDep) -> async_sessionmaker[AsyncSession]:
-    """Get session maker."""
-    _, session_maker = engine_factory
-    return session_maker
-
-
-SessionMakerDep = Annotated[async_sessionmaker, Depends(get_session_maker)]
-
-
-## repositories
-
-
-async def get_project_repository(
-    session_maker: SessionMakerDep,
-) -> ProjectRepository:
-    """Get the project repository."""
-    return ProjectRepository(session_maker)
-
-
-ProjectRepositoryDep = Annotated[ProjectRepository, Depends(get_project_repository)]
-ProjectPathDep = Annotated[str, Path()]  # Use Path dependency to extract from URL
-
-
-async def get_project_id(
-    project_repository: ProjectRepositoryDep,
-    project: ProjectPathDep,
-) -> int:
-    """Get the current project ID from request state.
-
-    When using sub-applications with /{project} mounting, the project value
-    is stored in request.state by middleware.
-
-    Args:
-        request: The current request object
-        project_repository: Repository for project operations
-
-    Returns:
-        The resolved project ID
-
-    Raises:
-        HTTPException: If project is not found
-    """
-    # Convert project name to permalink for lookup
-    project_permalink = generate_permalink(str(project))
-    project_obj = await project_repository.get_by_permalink(project_permalink)
-    if project_obj:
-        return project_obj.id
-
-    # Try by name if permalink lookup fails
-    project_obj = await project_repository.get_by_name(str(project))  # pragma: no cover
-    if project_obj:  # pragma: no cover
-        return project_obj.id
-
-    # Not found
-    raise HTTPException(  # pragma: no cover
-        status_code=status.HTTP_404_NOT_FOUND, detail=f"Project '{project}' not found."
-    )
-
-
-"""
-The project_id dependency is used in the following:
-- EntityRepository
-- ObservationRepository
-- RelationRepository
-- SearchRepository
-- ProjectInfoRepository
+"""Dependency injection functions for basic-memory services.
+
+DEPRECATED: This module is a backwards-compatibility shim.
+Import from basic_memory.deps package submodules instead:
+- basic_memory.deps.config for configuration
+- basic_memory.deps.db for database/session
+- basic_memory.deps.projects for project resolution
+- basic_memory.deps.repositories for data access
+- basic_memory.deps.services for business logic
+- basic_memory.deps.importers for import functionality
+
+This file will be removed once all callers are migrated.
 """
-ProjectIdDep = Annotated[int, Depends(get_project_id)]
-
-
-async def get_entity_repository(
-    session_maker: SessionMakerDep,
-    project_id: ProjectIdDep,
-) -> EntityRepository:
-    """Create an EntityRepository instance for the current project."""
-    return EntityRepository(session_maker, project_id=project_id)
-
-
-EntityRepositoryDep = Annotated[EntityRepository, Depends(get_entity_repository)]
-
-
-async def get_observation_repository(
-    session_maker: SessionMakerDep,
-    project_id: ProjectIdDep,
-) -> ObservationRepository:
-    """Create an ObservationRepository instance for the current project."""
-    return ObservationRepository(session_maker, project_id=project_id)
-
-
-ObservationRepositoryDep = Annotated[ObservationRepository, Depends(get_observation_repository)]
-
-
-async def get_relation_repository(
-    session_maker: SessionMakerDep,
-    project_id: ProjectIdDep,
-) -> RelationRepository:
-    """Create a RelationRepository instance for the current project."""
-    return RelationRepository(session_maker, project_id=project_id)
-
-
-RelationRepositoryDep = Annotated[RelationRepository, Depends(get_relation_repository)]
-
-
-async def get_search_repository(
-    session_maker: SessionMakerDep,
-    project_id: ProjectIdDep,
-) -> SearchRepository:
-    """Create a SearchRepository instance for the current project."""
-    return SearchRepository(session_maker, project_id=project_id)
-
-
-SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)]
-
-
-# ProjectInfoRepository is deprecated and will be removed in a future version.
-# Use ProjectRepository instead, which has the same functionality plus more project-specific operations.
-
-## services
-
-
-async def get_entity_parser(project_config: ProjectConfigDep) -> EntityParser:
-    return EntityParser(project_config.home)
-
-
-EntityParserDep = Annotated["EntityParser", Depends(get_entity_parser)]
-
-
-async def get_markdown_processor(entity_parser: EntityParserDep) -> MarkdownProcessor:
-    return MarkdownProcessor(entity_parser)
-
-
-MarkdownProcessorDep = Annotated[MarkdownProcessor, Depends(get_markdown_processor)]
-
-
-async def get_file_service(
-    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
-) -> FileService:
-    logger.debug(
-        f"Creating FileService for project: {project_config.name}, base_path: {project_config.home}"
-    )
-    file_service = FileService(project_config.home, markdown_processor)
-    logger.debug(f"Created FileService for project: {file_service} ")
-    return file_service
-
-
-FileServiceDep = Annotated[FileService, Depends(get_file_service)]
-
-
-async def get_entity_service(
-    entity_repository: EntityRepositoryDep,
-    observation_repository: ObservationRepositoryDep,
-    relation_repository: RelationRepositoryDep,
-    entity_parser: EntityParserDep,
-    file_service: FileServiceDep,
-    link_resolver: "LinkResolverDep",
-    app_config: AppConfigDep,
-) -> EntityService:
-    """Create EntityService with repository."""
-    return EntityService(
-        entity_repository=entity_repository,
-        observation_repository=observation_repository,
-        relation_repository=relation_repository,
-        entity_parser=entity_parser,
-        file_service=file_service,
-        link_resolver=link_resolver,
-        app_config=app_config,
-    )
-
-
-EntityServiceDep = Annotated[EntityService, Depends(get_entity_service)]
-
-
-async def get_search_service(
-    search_repository: SearchRepositoryDep,
-    entity_repository: EntityRepositoryDep,
-    file_service: FileServiceDep,
-) -> SearchService:
-    """Create SearchService with dependencies."""
-    return SearchService(search_repository, entity_repository, file_service)
-
-
-SearchServiceDep = Annotated[SearchService, Depends(get_search_service)]
-
-
-async def get_link_resolver(
-    entity_repository: EntityRepositoryDep, search_service: SearchServiceDep
-) -> LinkResolver:
-    return LinkResolver(entity_repository=entity_repository, search_service=search_service)
-
-
-LinkResolverDep = Annotated[LinkResolver, Depends(get_link_resolver)]
-
-
-async def get_context_service(
-    search_repository: SearchRepositoryDep,
-    entity_repository: EntityRepositoryDep,
-    observation_repository: ObservationRepositoryDep,
-) -> ContextService:
-    return ContextService(
-        search_repository=search_repository,
-        entity_repository=entity_repository,
-        observation_repository=observation_repository,
-    )
-
-
-ContextServiceDep = Annotated[ContextService, Depends(get_context_service)]
-
-
-async def get_sync_service(
-    app_config: AppConfigDep,
-    entity_service: EntityServiceDep,
-    entity_parser: EntityParserDep,
-    entity_repository: EntityRepositoryDep,
-    relation_repository: RelationRepositoryDep,
-    project_repository: ProjectRepositoryDep,
-    search_service: SearchServiceDep,
-    file_service: FileServiceDep,
-) -> SyncService:  # pragma: no cover
-    """
-
-    :rtype: object
-    """
-    return SyncService(
-        app_config=app_config,
-        entity_service=entity_service,
-        entity_parser=entity_parser,
-        entity_repository=entity_repository,
-        relation_repository=relation_repository,
-        project_repository=project_repository,
-        search_service=search_service,
-        file_service=file_service,
-    )
-
-
-SyncServiceDep = Annotated[SyncService, Depends(get_sync_service)]
-
-
-async def get_project_service(
-    project_repository: ProjectRepositoryDep,
-) -> ProjectService:
-    """Create ProjectService with repository."""
-    return ProjectService(repository=project_repository)
-
-
-ProjectServiceDep = Annotated[ProjectService, Depends(get_project_service)]
-
-
-async def get_directory_service(
-    entity_repository: EntityRepositoryDep,
-) -> DirectoryService:
-    """Create DirectoryService with dependencies."""
-    return DirectoryService(
-        entity_repository=entity_repository,
-    )
-
-
-DirectoryServiceDep = Annotated[DirectoryService, Depends(get_directory_service)]
-
-
-# Import
-
-
-async def get_chatgpt_importer(
-    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
-) -> ChatGPTImporter:
-    """Create ChatGPTImporter with dependencies."""
-    return ChatGPTImporter(project_config.home, markdown_processor)
-
-
-ChatGPTImporterDep = Annotated[ChatGPTImporter, Depends(get_chatgpt_importer)]
-
-
-async def get_claude_conversations_importer(
-    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
-) -> ClaudeConversationsImporter:
-    """Create ChatGPTImporter with dependencies."""
-    return ClaudeConversationsImporter(project_config.home, markdown_processor)
-
-
-ClaudeConversationsImporterDep = Annotated[
-    ClaudeConversationsImporter, Depends(get_claude_conversations_importer)
-]
-
-
-async def get_claude_projects_importer(
-    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
-) -> ClaudeProjectsImporter:
-    """Create ChatGPTImporter with dependencies."""
-    return ClaudeProjectsImporter(project_config.home, markdown_processor)
-
-
-ClaudeProjectsImporterDep = Annotated[ClaudeProjectsImporter, Depends(get_claude_projects_importer)]
-
-
-async def get_memory_json_importer(
-    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
-) -> MemoryJsonImporter:
-    """Create ChatGPTImporter with dependencies."""
-    return MemoryJsonImporter(project_config.home, markdown_processor)
-
 
-MemoryJsonImporterDep = Annotated[MemoryJsonImporter, Depends(get_memory_json_importer)]
+# Re-export everything from the deps package for backwards compatibility
+from basic_memory.deps import *  # noqa: F401, F403  # pragma: no cover

basic_memory/file_utils.py

@@ -1,9 +1,13 @@
 """Utilities for file operations."""
 
+import asyncio
 import hashlib
+import shlex
+from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
 import re
-from typing import Any, Dict, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Union
 
 import aiofiles
 import yaml
@@ -12,6 +16,23 @@ from loguru import logger
 
 from basic_memory.utils import FilePath
 
+if TYPE_CHECKING:  # pragma: no cover
+    from basic_memory.config import BasicMemoryConfig
+
+
+@dataclass
+class FileMetadata:
+    """File metadata for cloud-compatible file operations.
+
+    This dataclass provides a cloud-agnostic way to represent file metadata,
+    enabling S3FileService to return metadata from head_object responses
+    instead of mock stat_result with zeros.
+    """
+
+    size: int
+    created_at: datetime
+    modified_at: datetime
+
 
 class FileError(Exception):
     """Base exception for file operations."""
@@ -53,6 +74,28 @@ async def compute_checksum(content: Union[str, bytes]) -> str:
         raise FileError(f"Failed to compute checksum: {e}")
 
 
+# UTF-8 BOM character that can appear at the start of files
+UTF8_BOM = "\ufeff"
+
+
+def strip_bom(content: str) -> str:
+    """Strip UTF-8 BOM from the start of content if present.
+
+    BOM (Byte Order Mark) characters can be present in files created on Windows
+    or copied from certain sources. They should be stripped before processing
+    frontmatter. See issue #452.
+
+    Args:
+        content: Content that may start with BOM
+
+    Returns:
+        Content with BOM removed if present
+    """
+    if content and content.startswith(UTF8_BOM):
+        return content[1:]
+    return content
+
+
 async def write_file_atomic(path: FilePath, content: str) -> None:
     """
     Write file with atomic operation using temporary file.
@@ -84,6 +127,168 @@ async def write_file_atomic(path: FilePath, content: str) -> None:
         raise FileWriteError(f"Failed to write file {path}: {e}")
 
 
+async def format_markdown_builtin(path: Path) -> Optional[str]:
+    """
+    Format a markdown file using the built-in mdformat formatter.
+
+    Uses mdformat with GFM (GitHub Flavored Markdown) support for consistent
+    formatting without requiring Node.js or external tools.
+
+    Args:
+        path: Path to the markdown file to format
+
+    Returns:
+        Formatted content if successful, None if formatting failed.
+    """
+    try:
+        import mdformat
+    except ImportError:  # pragma: no cover
+        logger.warning(
+            "mdformat not installed, skipping built-in formatting",
+            path=str(path),
+        )
+        return None
+
+    try:
+        # Read original content
+        async with aiofiles.open(path, mode="r", encoding="utf-8") as f:
+            content = await f.read()
+
+        # Format using mdformat with GFM and frontmatter extensions
+        # mdformat is synchronous, so we run it in a thread executor
+        loop = asyncio.get_event_loop()
+        formatted_content = await loop.run_in_executor(
+            None,
+            lambda: mdformat.text(
+                content,
+                extensions={"gfm", "frontmatter"},  # GFM + YAML frontmatter support
+                options={"wrap": "no"},  # Don't wrap lines
+            ),
+        )
+
+        # Only write if content changed
+        if formatted_content != content:
+            async with aiofiles.open(path, mode="w", encoding="utf-8") as f:
+                await f.write(formatted_content)
+
+        logger.debug(
+            "Formatted file with mdformat",
+            path=str(path),
+            changed=formatted_content != content,
+        )
+        return formatted_content
+
+    except Exception as e:  # pragma: no cover
+        logger.warning(
+            "mdformat formatting failed",
+            path=str(path),
+            error=str(e),
+        )
+        return None
+
+
+async def format_file(
+    path: Path,
+    config: "BasicMemoryConfig",
+    is_markdown: bool = False,
+) -> Optional[str]:
+    """
+    Format a file using configured formatter.
+
+    By default, uses the built-in mdformat formatter for markdown files (pure Python,
+    no Node.js required). External formatters like Prettier can be configured via
+    formatter_command or per-extension formatters.
+
+    Args:
+        path: File to format
+        config: Configuration with formatter settings
+        is_markdown: Whether this is a markdown file (caller should use FileService.is_markdown)
+
+    Returns:
+        Formatted content if successful, None if formatting was skipped or failed.
+        Failures are logged as warnings but don't raise exceptions.
+    """
+    if not config.format_on_save:
+        return None
+
+    extension = path.suffix.lstrip(".")
+    formatter = config.formatters.get(extension) or config.formatter_command
+
+    # Use built-in mdformat for markdown files when no external formatter configured
+    if not formatter:
+        if is_markdown:
+            return await format_markdown_builtin(path)
+        else:
+            logger.debug("No formatter configured for extension", extension=extension)
+            return None
+
+    # Use external formatter
+    # Replace {file} placeholder with the actual path
+    cmd = formatter.replace("{file}", str(path))
+
+    try:
+        # Parse command into args list for safer execution (no shell=True)
+        args = shlex.split(cmd)
+
+        proc = await asyncio.create_subprocess_exec(
+            *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+
+        try:
+            stdout, stderr = await asyncio.wait_for(
+                proc.communicate(),
+                timeout=config.formatter_timeout,
+            )
+        except asyncio.TimeoutError:
+            proc.kill()
+            await proc.wait()
+            logger.warning(
+                "Formatter timed out",
+                path=str(path),
+                timeout=config.formatter_timeout,
+            )
+            return None
+
+        if proc.returncode != 0:
+            logger.warning(
+                "Formatter exited with non-zero status",
+                path=str(path),
+                returncode=proc.returncode,
+                stderr=stderr.decode("utf-8", errors="replace") if stderr else "",
+            )
+            # Still try to read the file - formatter may have partially worked
+            # or the file may be unchanged
+
+        # Read formatted content
+        async with aiofiles.open(path, mode="r", encoding="utf-8") as f:
+            formatted_content = await f.read()
+
+        logger.debug(
+            "Formatted file successfully",
+            path=str(path),
+            formatter=args[0] if args else formatter,
+        )
+        return formatted_content
+
+    except FileNotFoundError:
+        # Formatter executable not found
+        logger.warning(
+            "Formatter executable not found",
+            command=cmd.split()[0] if cmd else "",
+            path=str(path),
+        )
+        return None
+    except Exception as e:  # pragma: no cover
+        logger.warning(
+            "Formatter failed",
+            path=str(path),
+            error=str(e),
+        )
+        return None
+
+
 def has_frontmatter(content: str) -> bool:
     """
     Check if content contains valid YAML frontmatter.
@@ -97,7 +302,8 @@ def has_frontmatter(content: str) -> bool:
     if not content:
         return False
 
-    content = content.strip()
+    # Strip BOM before checking for frontmatter markers
+    content = strip_bom(content).strip()
     if not content.startswith("---"):
         return False
 
@@ -118,6 +324,8 @@ def parse_frontmatter(content: str) -> Dict[str, Any]:
         ParseError: If frontmatter is invalid or parsing fails
     """
     try:
+        # Strip BOM before parsing frontmatter
+        content = strip_bom(content)
         if not content.strip().startswith("---"):
             raise ParseError("Content has no frontmatter")
 
@@ -159,7 +367,8 @@ def remove_frontmatter(content: str) -> str:
     Raises:
         ParseError: If content starts with frontmatter marker but is malformed
     """
-    content = content.strip()
+    # Strip BOM before processing
+    content = strip_bom(content).strip()
 
     # Return as-is if no frontmatter marker
     if not content.startswith("---"):