basic-memory 0.7.0__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/ignore_utils.py

@@ -0,0 +1,297 @@
+"""Utilities for handling .gitignore patterns and file filtering."""
+
+import fnmatch
+from pathlib import Path
+from typing import Set
+
+
+# Common directories and patterns to ignore by default
+# These are used as fallback if .bmignore doesn't exist
+DEFAULT_IGNORE_PATTERNS = {
+    # Hidden files (files starting with dot)
+    ".*",
+    # Basic Memory internal files
+    "*.db",
+    "*.db-shm",
+    "*.db-wal",
+    "config.json",
+    # Version control
+    ".git",
+    ".svn",
+    # Python
+    "__pycache__",
+    "*.pyc",
+    "*.pyo",
+    "*.pyd",
+    ".pytest_cache",
+    ".coverage",
+    "*.egg-info",
+    ".tox",
+    ".mypy_cache",
+    ".ruff_cache",
+    # Virtual environments
+    ".venv",
+    "venv",
+    "env",
+    ".env",
+    # Node.js
+    "node_modules",
+    # Build artifacts
+    "build",
+    "dist",
+    ".cache",
+    # IDE
+    ".idea",
+    ".vscode",
+    # OS files
+    ".DS_Store",
+    "Thumbs.db",
+    "desktop.ini",
+    # Obsidian
+    ".obsidian",
+    # Temporary files
+    "*.tmp",
+    "*.swp",
+    "*.swo",
+    "*~",
+}
+
+
+def get_bmignore_path() -> Path:
+    """Get path to .bmignore file.
+
+    Returns:
+        Path to ~/.basic-memory/.bmignore
+    """
+    return Path.home() / ".basic-memory" / ".bmignore"
+
+
+def create_default_bmignore() -> None:
+    """Create default .bmignore file if it doesn't exist.
+
+    This ensures users have a file they can customize for all Basic Memory operations.
+    """
+    bmignore_path = get_bmignore_path()
+
+    if bmignore_path.exists():
+        return
+
+    bmignore_path.parent.mkdir(parents=True, exist_ok=True)
+    bmignore_path.write_text("""# Basic Memory Ignore Patterns
+# This file is used by both 'bm cloud upload', 'bm cloud bisync', and file sync
+# Patterns use standard gitignore-style syntax
+
+# Hidden files (files starting with dot)
+.*
+
+# Basic Memory internal files (includes test databases)
+*.db
+*.db-shm
+*.db-wal
+config.json
+
+# Version control
+.git
+.svn
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache
+.coverage
+*.egg-info
+.tox
+.mypy_cache
+.ruff_cache
+
+# Virtual environments
+.venv
+venv
+env
+.env
+
+# Node.js
+node_modules
+
+# Build artifacts
+build
+dist
+.cache
+
+# IDE
+.idea
+.vscode
+
+# OS files
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Obsidian
+.obsidian
+
+# Temporary files
+*.tmp
+*.swp
+*.swo
+*~
+""")
+
+
+def load_bmignore_patterns() -> Set[str]:
+    """Load patterns from .bmignore file.
+
+    Returns:
+        Set of patterns from .bmignore, or DEFAULT_IGNORE_PATTERNS if file doesn't exist
+    """
+    bmignore_path = get_bmignore_path()
+
+    # Create default file if it doesn't exist
+    if not bmignore_path.exists():
+        create_default_bmignore()
+
+    patterns = set()
+
+    try:
+        with bmignore_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                # Skip empty lines and comments
+                if line and not line.startswith("#"):
+                    patterns.add(line)
+    except Exception:  # pragma: no cover
+        # If we can't read .bmignore, fall back to defaults
+        return set(DEFAULT_IGNORE_PATTERNS)  # pragma: no cover
+
+    # If no patterns were loaded, use defaults
+    if not patterns:  # pragma: no cover
+        return set(DEFAULT_IGNORE_PATTERNS)  # pragma: no cover
+
+    return patterns
+
+
+def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[str]:
+    """Load gitignore patterns from .gitignore file and .bmignore.
+
+    Combines patterns from:
+    1. ~/.basic-memory/.bmignore (user's global ignore patterns)
+    2. {base_path}/.gitignore (project-specific patterns, if use_gitignore=True)
+
+    Args:
+        base_path: The base directory to search for .gitignore file
+        use_gitignore: If False, only load patterns from .bmignore (default: True)
+
+    Returns:
+        Set of patterns to ignore
+    """
+    # Start with patterns from .bmignore
+    patterns = load_bmignore_patterns()
+
+    if use_gitignore:
+        gitignore_file = base_path / ".gitignore"
+        if gitignore_file.exists():
+            try:
+                with gitignore_file.open("r", encoding="utf-8") as f:
+                    for line in f:
+                        line = line.strip()
+                        # Skip empty lines and comments
+                        if line and not line.startswith("#"):
+                            patterns.add(line)
+            except Exception:
+                # If we can't read .gitignore, just use default patterns
+                pass
+
+    return patterns
+
+
+def should_ignore_path(file_path: Path, base_path: Path, ignore_patterns: Set[str]) -> bool:
+    """Check if a file path should be ignored based on gitignore patterns.
+
+    Args:
+        file_path: The file path to check
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to match against
+
+    Returns:
+        True if the path should be ignored, False otherwise
+    """
+    # Get the relative path from base
+    try:
+        relative_path = file_path.relative_to(base_path)
+        relative_str = str(relative_path)
+        relative_posix = relative_path.as_posix()  # Use forward slashes for matching
+
+        # Check each pattern
+        for pattern in ignore_patterns:
+            # Handle patterns starting with / (root relative)
+            if pattern.startswith("/"):
+                root_pattern = pattern[1:]  # Remove leading /
+
+                # For directory patterns ending with /
+                if root_pattern.endswith("/"):
+                    dir_name = root_pattern[:-1]  # Remove trailing /
+                    # Check if the first part of the path matches the directory name
+                    if len(relative_path.parts) > 0 and relative_path.parts[0] == dir_name:
+                        return True
+                else:
+                    # Regular root-relative pattern
+                    if fnmatch.fnmatch(relative_posix, root_pattern):
+                        return True
+                continue
+
+            # Handle directory patterns (ending with /)
+            if pattern.endswith("/"):
+                dir_name = pattern[:-1]  # Remove trailing /
+                # Check if any path part matches the directory name
+                if dir_name in relative_path.parts:
+                    return True
+                continue
+
+            # Direct name match (e.g., ".git", "node_modules")
+            if pattern in relative_path.parts:
+                return True
+
+            # Check if any individual path part matches the glob pattern
+            # This handles cases like ".*" matching ".hidden.md" in "concept/.hidden.md"
+            for part in relative_path.parts:
+                if fnmatch.fnmatch(part, pattern):
+                    return True
+
+            # Glob pattern match on full path
+            if fnmatch.fnmatch(relative_posix, pattern) or fnmatch.fnmatch(relative_str, pattern):
+                return True  # pragma: no cover
+
+        return False
+    except ValueError:
+        # If we can't get relative path, don't ignore
+        return False
+
+
+def filter_files(
+    files: list[Path], base_path: Path, ignore_patterns: Set[str] | None = None
+) -> tuple[list[Path], int]:
+    """Filter a list of files based on gitignore patterns.
+
+    Args:
+        files: List of file paths to filter
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to ignore. If None, loads from .gitignore
+
+    Returns:
+        Tuple of (filtered_files, ignored_count)
+    """
+    if ignore_patterns is None:
+        ignore_patterns = load_gitignore_patterns(base_path)
+
+    filtered_files = []
+    ignored_count = 0
+
+    for file_path in files:
+        if should_ignore_path(file_path, base_path, ignore_patterns):
+            ignored_count += 1
+        else:
+            filtered_files.append(file_path)
+
+    return filtered_files, ignored_count

basic_memory/importers/__init__.py

@@ -0,0 +1,27 @@
+"""Import services for Basic Memory."""
+
+from basic_memory.importers.base import Importer
+from basic_memory.importers.chatgpt_importer import ChatGPTImporter
+from basic_memory.importers.claude_conversations_importer import (
+    ClaudeConversationsImporter,
+)
+from basic_memory.importers.claude_projects_importer import ClaudeProjectsImporter
+from basic_memory.importers.memory_json_importer import MemoryJsonImporter
+from basic_memory.schemas.importer import (
+    ChatImportResult,
+    EntityImportResult,
+    ImportResult,
+    ProjectImportResult,
+)
+
+__all__ = [
+    "Importer",
+    "ChatGPTImporter",
+    "ClaudeConversationsImporter",
+    "ClaudeProjectsImporter",
+    "MemoryJsonImporter",
+    "ImportResult",
+    "ChatImportResult",
+    "EntityImportResult",
+    "ProjectImportResult",
+]

basic_memory/importers/base.py

@@ -0,0 +1,100 @@
+"""Base import service for Basic Memory."""
+
+import logging
+from abc import abstractmethod
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Optional, TypeVar
+
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
+from basic_memory.markdown.schemas import EntityMarkdown
+from basic_memory.schemas.importer import ImportResult
+
+if TYPE_CHECKING:  # pragma: no cover
+    from basic_memory.services.file_service import FileService
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=ImportResult)
+
+
+class Importer[T: ImportResult]:
+    """Base class for all import services.
+
+    All file operations are delegated to FileService, which can be overridden
+    in cloud environments to use S3 or other storage backends.
+    """
+
+    def __init__(
+        self,
+        base_path: Path,
+        markdown_processor: MarkdownProcessor,
+        file_service: "FileService",
+    ):
+        """Initialize the import service.
+
+        Args:
+            base_path: Base path for the project.
+            markdown_processor: MarkdownProcessor instance for markdown serialization.
+            file_service: FileService instance for all file operations.
+        """
+        self.base_path = base_path.resolve()  # Get absolute path
+        self.markdown_processor = markdown_processor
+        self.file_service = file_service
+
+    @abstractmethod
+    async def import_data(self, source_data, destination_folder: str, **kwargs: Any) -> T:
+        """Import data from source file to destination folder.
+
+        Args:
+            source_path: Path to the source file.
+            destination_folder: Destination folder within the project.
+            **kwargs: Additional keyword arguments for specific import types.
+
+        Returns:
+            ImportResult containing statistics and status of the import.
+        """
+        pass  # pragma: no cover
+
+    async def write_entity(self, entity: EntityMarkdown, file_path: str | Path) -> str:
+        """Write entity to file using FileService.
+
+        This method serializes the entity to markdown and writes it using
+        FileService, which handles directory creation and storage backend
+        abstraction (local filesystem vs cloud storage).
+
+        Args:
+            entity: EntityMarkdown instance to write.
+            file_path: Relative path to write the entity to. FileService handles base_path.
+
+        Returns:
+            Checksum of written file.
+        """
+        content = self.markdown_processor.to_markdown_string(entity)
+        # FileService.write_file handles directory creation and returns checksum
+        return await self.file_service.write_file(file_path, content)
+
+    async def ensure_folder_exists(self, folder: str) -> None:
+        """Ensure folder exists using FileService.
+
+        For cloud storage (S3), this is essentially a no-op since S3 doesn't
+        have actual folders - they're just key prefixes.
+
+        Args:
+            folder: Relative folder path within the project. FileService handles base_path.
+        """
+        await self.file_service.ensure_directory(folder)
+
+    @abstractmethod
+    def handle_error(
+        self, message: str, error: Optional[Exception] = None
+    ) -> T:  # pragma: no cover
+        """Handle errors during import.
+
+        Args:
+            message: Error message.
+            error: Optional exception that caused the error.
+
+        Returns:
+            ImportResult with error information.
+        """
+        pass

basic_memory/importers/chatgpt_importer.py

@@ -0,0 +1,245 @@
+"""ChatGPT import service for Basic Memory."""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Set
+
+from basic_memory.markdown.schemas import EntityFrontmatter, EntityMarkdown
+from basic_memory.importers.base import Importer
+from basic_memory.schemas.importer import ChatImportResult
+from basic_memory.importers.utils import clean_filename, format_timestamp
+
+logger = logging.getLogger(__name__)
+
+
+class ChatGPTImporter(Importer[ChatImportResult]):
+    """Service for importing ChatGPT conversations."""
+
+    def handle_error(  # pragma: no cover
+        self, message: str, error: Optional[Exception] = None
+    ) -> ChatImportResult:
+        """Return a failed ChatImportResult with an error message."""
+        error_msg = f"{message}: {error}" if error else message
+        return ChatImportResult(
+            import_count={},
+            success=False,
+            error_message=error_msg,
+            conversations=0,
+            messages=0,
+        )
+
+    async def import_data(
+        self, source_data, destination_folder: str, **kwargs: Any
+    ) -> ChatImportResult:
+        """Import conversations from ChatGPT JSON export.
+
+        Args:
+            source_path: Path to the ChatGPT conversations.json file.
+            destination_folder: Destination folder within the project.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            ChatImportResult containing statistics and status of the import.
+        """
+        try:  # pragma: no cover
+            # Ensure the destination folder exists
+            await self.ensure_folder_exists(destination_folder)
+            conversations = source_data
+
+            # Process each conversation
+            messages_imported = 0
+            chats_imported = 0
+
+            for chat in conversations:
+                # Convert to entity
+                entity = self._format_chat_content(destination_folder, chat)
+
+                # Write file using relative path - FileService handles base_path
+                file_path = f"{entity.frontmatter.metadata['permalink']}.md"
+                await self.write_entity(entity, file_path)
+
+                # Count messages
+                msg_count = sum(
+                    1
+                    for node in chat["mapping"].values()
+                    if node.get("message")
+                    and not node.get("message", {})
+                    .get("metadata", {})
+                    .get("is_visually_hidden_from_conversation")
+                )
+
+                chats_imported += 1
+                messages_imported += msg_count
+
+            return ChatImportResult(
+                import_count={"conversations": chats_imported, "messages": messages_imported},
+                success=True,
+                conversations=chats_imported,
+                messages=messages_imported,
+            )
+
+        except Exception as e:  # pragma: no cover
+            logger.exception("Failed to import ChatGPT conversations")
+            return self.handle_error("Failed to import ChatGPT conversations", e)
+
+    def _format_chat_content(
+        self, folder: str, conversation: Dict[str, Any]
+    ) -> EntityMarkdown:  # pragma: no cover
+        """Convert chat conversation to Basic Memory entity.
+
+        Args:
+            folder: Destination folder name.
+            conversation: ChatGPT conversation data.
+
+        Returns:
+            EntityMarkdown instance representing the conversation.
+        """
+        # Extract timestamps
+        created_at = conversation["create_time"]
+        modified_at = conversation["update_time"]
+
+        root_id = None
+        # Find root message
+        for node_id, node in conversation["mapping"].items():
+            if node.get("parent") is None:
+                root_id = node_id
+                break
+
+        # Generate permalink
+        date_prefix = datetime.fromtimestamp(created_at).astimezone().strftime("%Y%m%d")
+        clean_title = clean_filename(conversation["title"])
+
+        # Format content
+        content = self._format_chat_markdown(
+            title=conversation["title"],
+            mapping=conversation["mapping"],
+            root_id=root_id,
+            created_at=created_at,
+            modified_at=modified_at,
+        )
+
+        # Create entity
+        entity = EntityMarkdown(
+            frontmatter=EntityFrontmatter(
+                metadata={
+                    "type": "conversation",
+                    "title": conversation["title"],
+                    "created": format_timestamp(created_at),
+                    "modified": format_timestamp(modified_at),
+                    "permalink": f"{folder}/{date_prefix}-{clean_title}",
+                }
+            ),
+            content=content,
+        )
+
+        return entity
+
+    def _format_chat_markdown(
+        self,
+        title: str,
+        mapping: Dict[str, Any],
+        root_id: Optional[str],
+        created_at: float,
+        modified_at: float,
+    ) -> str:  # pragma: no cover
+        """Format chat as clean markdown.
+
+        Args:
+            title: Chat title.
+            mapping: Message mapping.
+            root_id: Root message ID.
+            created_at: Creation timestamp.
+            modified_at: Modification timestamp.
+
+        Returns:
+            Formatted markdown content.
+        """
+        # Start with title
+        lines = [f"# {title}\n"]
+
+        # Traverse message tree
+        seen_msgs: Set[str] = set()
+        messages = self._traverse_messages(mapping, root_id, seen_msgs)
+
+        # Format each message
+        for msg in messages:
+            # Skip hidden messages
+            if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
+                continue
+
+            # Get author and timestamp
+            author = msg["author"]["role"].title()
+            ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
+
+            # Add message header
+            lines.append(f"### {author} ({ts})")
+
+            # Add message content
+            content = self._get_message_content(msg)
+            if content:
+                lines.append(content)
+
+            # Add spacing
+            lines.append("")
+
+        return "\n".join(lines)
+
+    def _get_message_content(self, message: Dict[str, Any]) -> str:  # pragma: no cover
+        """Extract clean message content.
+
+        Args:
+            message: Message data.
+
+        Returns:
+            Cleaned message content.
+        """
+        if not message or "content" not in message:
+            return ""
+
+        content = message["content"]
+        if content.get("content_type") == "text":
+            return "\n".join(content.get("parts", []))
+        elif content.get("content_type") == "code":
+            return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
+        return ""
+
+    def _traverse_messages(
+        self, mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
+    ) -> List[Dict[str, Any]]:  # pragma: no cover
+        """Traverse message tree iteratively to handle deep conversations.
+
+        Args:
+            mapping: Message mapping.
+            root_id: Root message ID.
+            seen: Set of seen message IDs.
+
+        Returns:
+            List of message data.
+        """
+        messages = []
+        if not root_id:
+            return messages
+
+        # Use iterative approach with stack to avoid recursion depth issues
+        stack = [root_id]
+
+        while stack:
+            node_id = stack.pop()
+            if not node_id:
+                continue
+
+            node = mapping.get(node_id)
+            if not node:
+                continue
+
+            # Process current node if it has a message and hasn't been seen
+            if node["id"] not in seen and node.get("message"):
+                seen.add(node["id"])
+                messages.append(node["message"])
+
+            # Add children to stack in reverse order to maintain conversation flow
+            children = node.get("children", [])
+            for child_id in reversed(children):
+                stack.append(child_id)
+
+        return messages