basic-memory 0.2.12__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/file_utils.py

@@ -2,11 +2,16 @@
 
 import hashlib
 from pathlib import Path
-from typing import Dict, Any
+import re
+from typing import Any, Dict, Union
 
+import aiofiles
 import yaml
+import frontmatter
 from loguru import logger
 
+from basic_memory.utils import FilePath
+
 
 class FileError(Exception):
     """Base exception for file operations."""
@@ -26,12 +31,12 @@ class ParseError(FileError):
     pass
 
 
-async def compute_checksum(content: str) -> str:
+async def compute_checksum(content: Union[str, bytes]) -> str:
     """
     Compute SHA-256 checksum of content.
 
     Args:
-        content: Text content to hash
+        content: Content to hash (either text string or bytes)
 
     Returns:
         SHA-256 hex digest
@@ -40,48 +45,42 @@ async def compute_checksum(content: str) -> str:
         FileError: If checksum computation fails
     """
     try:
-        return hashlib.sha256(content.encode()).hexdigest()
+        if isinstance(content, str):
+            content = content.encode()
+        return hashlib.sha256(content).hexdigest()
     except Exception as e:  # pragma: no cover
         logger.error(f"Failed to compute checksum: {e}")
         raise FileError(f"Failed to compute checksum: {e}")
 
 
-async def ensure_directory(path: Path) -> None:
-    """
-    Ensure directory exists, creating if necessary.
-
-    Args:
-        path: Directory path to ensure
-
-    Raises:
-        FileWriteError: If directory creation fails
-    """
-    try:
-        path.mkdir(parents=True, exist_ok=True)
-    except Exception as e:  # pragma: no cover
-        logger.error(f"Failed to create directory: {path}: {e}")
-        raise FileWriteError(f"Failed to create directory {path}: {e}")
-
-
-async def write_file_atomic(path: Path, content: str) -> None:
+async def write_file_atomic(path: FilePath, content: str) -> None:
     """
     Write file with atomic operation using temporary file.
 
+    Uses aiofiles for true async I/O (non-blocking).
+
     Args:
-        path: Target file path
+        path: Target file path (Path or string)
         content: Content to write
 
     Raises:
         FileWriteError: If write operation fails
     """
-    temp_path = path.with_suffix(".tmp")
+    # Convert string to Path if needed
+    path_obj = Path(path) if isinstance(path, str) else path
+    temp_path = path_obj.with_suffix(".tmp")
+
     try:
-        temp_path.write_text(content)
-        temp_path.replace(path)
-        logger.debug(f"wrote file: {path}")
+        # Use aiofiles for non-blocking write
+        async with aiofiles.open(temp_path, mode="w", encoding="utf-8") as f:
+            await f.write(content)
+
+        # Atomic rename (this is fast, doesn't need async)
+        temp_path.replace(path_obj)
+        logger.debug("Wrote file atomically", path=str(path_obj), content_length=len(content))
     except Exception as e:  # pragma: no cover
         temp_path.unlink(missing_ok=True)
-        logger.error(f"Failed to write file: {path}: {e}")
+        logger.error("Failed to write file", path=str(path_obj), error=str(e))
         raise FileWriteError(f"Failed to write file {path}: {e}")
 
 
@@ -95,6 +94,9 @@ def has_frontmatter(content: str) -> bool:
     Returns:
         True if content has valid frontmatter markers (---), False otherwise
     """
+    if not content:
+        return False
+
     content = content.strip()
     if not content.startswith("---"):
         return False
@@ -171,44 +173,97 @@ def remove_frontmatter(content: str) -> str:
     return parts[2].strip()
 
 
-async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
-    """Update frontmatter fields in a file while preserving all content.
-
-    Only modifies the frontmatter section, leaving all content untouched.
-    Creates frontmatter section if none exists.
-    Returns checksum of updated file.
+def dump_frontmatter(post: frontmatter.Post) -> str:
+    """
+    Serialize frontmatter.Post to markdown with Obsidian-compatible YAML format.
+
+    This function ensures that:
+    1. Tags are formatted as YAML lists instead of JSON arrays
+    2. String values are properly quoted to handle special characters (colons, etc.)
+
+    Good (Obsidian compatible):
+    ---
+    title: "L2 Governance Core (Split: Core)"
+    tags:
+    - system
+    - overview
+    - reference
+    ---
+
+    Bad (causes parsing errors):
+    ---
+    title: L2 Governance Core (Split: Core)  # Unquoted colon breaks YAML
+    tags: ["system", "overview", "reference"]
+    ---
 
     Args:
-        path: Path to markdown file
-        updates: Dict of frontmatter fields to update
+        post: frontmatter.Post object to serialize
 
     Returns:
-        Checksum of updated file
+        String containing markdown with properly formatted YAML frontmatter
+    """
+    if not post.metadata:
+        # No frontmatter, just return content
+        return post.content
+
+    # Serialize YAML with block style for lists
+    # SafeDumper automatically quotes values with special characters (colons, etc.)
+    yaml_str = yaml.dump(
+        post.metadata,
+        sort_keys=False,
+        allow_unicode=True,
+        default_flow_style=False,
+        Dumper=yaml.SafeDumper,
+    )
+
+    # Construct the final markdown with frontmatter
+    if post.content:
+        return f"---\n{yaml_str}---\n\n{post.content}"
+    else:
+        return f"---\n{yaml_str}---\n"
+
+
+def sanitize_for_filename(text: str, replacement: str = "-") -> str:
+    """
+    Sanitize string to be safe for use as a note title
+    Replaces path separators and other problematic characters
+    with hyphens.
+    """
+    # replace both POSIX and Windows path separators
+    text = re.sub(r"[/\\]", replacement, text)
 
-    Raises:
-        FileError: If file operations fail
-        ParseError: If frontmatter parsing fails
+    # replace some other problematic chars
+    text = re.sub(r'[<>:"|?*]', replacement, text)
+
+    # compress multiple, repeated replacements
+    text = re.sub(f"{re.escape(replacement)}+", replacement, text)
+
+    return text.strip(replacement)
+
+
+def sanitize_for_folder(folder: str) -> str:
     """
-    try:
-        # Read current content
-        content = path.read_text()
+    Sanitize folder path to be safe for use in file system paths.
+    Removes leading/trailing whitespace, compresses multiple slashes,
+    and removes special characters except for /, -, and _.
+    """
+    if not folder:
+        return ""
 
-        # Parse current frontmatter
-        current_fm = {}
-        if has_frontmatter(content):
-            current_fm = parse_frontmatter(content)
-            content = remove_frontmatter(content)
+    sanitized = folder.strip()
 
-        # Update frontmatter
-        new_fm = {**current_fm, **updates}
+    if sanitized.startswith("./"):
+        sanitized = sanitized[2:]
 
-        # Write new file with updated frontmatter
-        yaml_fm = yaml.dump(new_fm, sort_keys=False)
-        final_content = f"---\n{yaml_fm}---\n\n{content.strip()}"
+    # ensure no special characters (except for a few that are allowed)
+    sanitized = "".join(
+        c for c in sanitized if c.isalnum() or c in (".", " ", "-", "_", "\\", "/")
+    ).rstrip()
 
-        await write_file_atomic(path, final_content)
-        return await compute_checksum(final_content)
+    # compress multiple, repeated instances of path separators
+    sanitized = re.sub(r"[\\/]+", "/", sanitized)
 
-    except Exception as e:  # pragma: no cover
-        logger.error(f"Failed to update frontmatter in {path}: {e}")
-        raise FileError(f"Failed to update frontmatter: {e}")
+    # trim any leading/trailing path separators
+    sanitized = sanitized.strip("\\/")
+
+    return sanitized

basic_memory/ignore_utils.py

@@ -0,0 +1,297 @@
+"""Utilities for handling .gitignore patterns and file filtering."""
+
+import fnmatch
+from pathlib import Path
+from typing import Set
+
+
+# Common directories and patterns to ignore by default
+# These are used as fallback if .bmignore doesn't exist
+DEFAULT_IGNORE_PATTERNS = {
+    # Hidden files (files starting with dot)
+    ".*",
+    # Basic Memory internal files
+    "*.db",
+    "*.db-shm",
+    "*.db-wal",
+    "config.json",
+    # Version control
+    ".git",
+    ".svn",
+    # Python
+    "__pycache__",
+    "*.pyc",
+    "*.pyo",
+    "*.pyd",
+    ".pytest_cache",
+    ".coverage",
+    "*.egg-info",
+    ".tox",
+    ".mypy_cache",
+    ".ruff_cache",
+    # Virtual environments
+    ".venv",
+    "venv",
+    "env",
+    ".env",
+    # Node.js
+    "node_modules",
+    # Build artifacts
+    "build",
+    "dist",
+    ".cache",
+    # IDE
+    ".idea",
+    ".vscode",
+    # OS files
+    ".DS_Store",
+    "Thumbs.db",
+    "desktop.ini",
+    # Obsidian
+    ".obsidian",
+    # Temporary files
+    "*.tmp",
+    "*.swp",
+    "*.swo",
+    "*~",
+}
+
+
+def get_bmignore_path() -> Path:
+    """Get path to .bmignore file.
+
+    Returns:
+        Path to ~/.basic-memory/.bmignore
+    """
+    return Path.home() / ".basic-memory" / ".bmignore"
+
+
+def create_default_bmignore() -> None:
+    """Create default .bmignore file if it doesn't exist.
+
+    This ensures users have a file they can customize for all Basic Memory operations.
+    """
+    bmignore_path = get_bmignore_path()
+
+    if bmignore_path.exists():
+        return
+
+    bmignore_path.parent.mkdir(parents=True, exist_ok=True)
+    bmignore_path.write_text("""# Basic Memory Ignore Patterns
+# This file is used by both 'bm cloud upload', 'bm cloud bisync', and file sync
+# Patterns use standard gitignore-style syntax
+
+# Hidden files (files starting with dot)
+.*
+
+# Basic Memory internal files (includes test databases)
+*.db
+*.db-shm
+*.db-wal
+config.json
+
+# Version control
+.git
+.svn
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache
+.coverage
+*.egg-info
+.tox
+.mypy_cache
+.ruff_cache
+
+# Virtual environments
+.venv
+venv
+env
+.env
+
+# Node.js
+node_modules
+
+# Build artifacts
+build
+dist
+.cache
+
+# IDE
+.idea
+.vscode
+
+# OS files
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Obsidian
+.obsidian
+
+# Temporary files
+*.tmp
+*.swp
+*.swo
+*~
+""")
+
+
+def load_bmignore_patterns() -> Set[str]:
+    """Load patterns from .bmignore file.
+
+    Returns:
+        Set of patterns from .bmignore, or DEFAULT_IGNORE_PATTERNS if file doesn't exist
+    """
+    bmignore_path = get_bmignore_path()
+
+    # Create default file if it doesn't exist
+    if not bmignore_path.exists():
+        create_default_bmignore()
+
+    patterns = set()
+
+    try:
+        with bmignore_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                # Skip empty lines and comments
+                if line and not line.startswith("#"):
+                    patterns.add(line)
+    except Exception:
+        # If we can't read .bmignore, fall back to defaults
+        return set(DEFAULT_IGNORE_PATTERNS)
+
+    # If no patterns were loaded, use defaults
+    if not patterns:
+        return set(DEFAULT_IGNORE_PATTERNS)
+
+    return patterns
+
+
+def load_gitignore_patterns(base_path: Path, use_gitignore: bool = True) -> Set[str]:
+    """Load gitignore patterns from .gitignore file and .bmignore.
+
+    Combines patterns from:
+    1. ~/.basic-memory/.bmignore (user's global ignore patterns)
+    2. {base_path}/.gitignore (project-specific patterns, if use_gitignore=True)
+
+    Args:
+        base_path: The base directory to search for .gitignore file
+        use_gitignore: If False, only load patterns from .bmignore (default: True)
+
+    Returns:
+        Set of patterns to ignore
+    """
+    # Start with patterns from .bmignore
+    patterns = load_bmignore_patterns()
+
+    if use_gitignore:
+        gitignore_file = base_path / ".gitignore"
+        if gitignore_file.exists():
+            try:
+                with gitignore_file.open("r", encoding="utf-8") as f:
+                    for line in f:
+                        line = line.strip()
+                        # Skip empty lines and comments
+                        if line and not line.startswith("#"):
+                            patterns.add(line)
+            except Exception:
+                # If we can't read .gitignore, just use default patterns
+                pass
+
+    return patterns
+
+
+def should_ignore_path(file_path: Path, base_path: Path, ignore_patterns: Set[str]) -> bool:
+    """Check if a file path should be ignored based on gitignore patterns.
+
+    Args:
+        file_path: The file path to check
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to match against
+
+    Returns:
+        True if the path should be ignored, False otherwise
+    """
+    # Get the relative path from base
+    try:
+        relative_path = file_path.relative_to(base_path)
+        relative_str = str(relative_path)
+        relative_posix = relative_path.as_posix()  # Use forward slashes for matching
+
+        # Check each pattern
+        for pattern in ignore_patterns:
+            # Handle patterns starting with / (root relative)
+            if pattern.startswith("/"):
+                root_pattern = pattern[1:]  # Remove leading /
+
+                # For directory patterns ending with /
+                if root_pattern.endswith("/"):
+                    dir_name = root_pattern[:-1]  # Remove trailing /
+                    # Check if the first part of the path matches the directory name
+                    if len(relative_path.parts) > 0 and relative_path.parts[0] == dir_name:
+                        return True
+                else:
+                    # Regular root-relative pattern
+                    if fnmatch.fnmatch(relative_posix, root_pattern):
+                        return True
+                continue
+
+            # Handle directory patterns (ending with /)
+            if pattern.endswith("/"):
+                dir_name = pattern[:-1]  # Remove trailing /
+                # Check if any path part matches the directory name
+                if dir_name in relative_path.parts:
+                    return True
+                continue
+
+            # Direct name match (e.g., ".git", "node_modules")
+            if pattern in relative_path.parts:
+                return True
+
+            # Check if any individual path part matches the glob pattern
+            # This handles cases like ".*" matching ".hidden.md" in "concept/.hidden.md"
+            for part in relative_path.parts:
+                if fnmatch.fnmatch(part, pattern):
+                    return True
+
+            # Glob pattern match on full path
+            if fnmatch.fnmatch(relative_posix, pattern) or fnmatch.fnmatch(relative_str, pattern):
+                return True
+
+        return False
+    except ValueError:
+        # If we can't get relative path, don't ignore
+        return False
+
+
+def filter_files(
+    files: list[Path], base_path: Path, ignore_patterns: Set[str] | None = None
+) -> tuple[list[Path], int]:
+    """Filter a list of files based on gitignore patterns.
+
+    Args:
+        files: List of file paths to filter
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to ignore. If None, loads from .gitignore
+
+    Returns:
+        Tuple of (filtered_files, ignored_count)
+    """
+    if ignore_patterns is None:
+        ignore_patterns = load_gitignore_patterns(base_path)
+
+    filtered_files = []
+    ignored_count = 0
+
+    for file_path in files:
+        if should_ignore_path(file_path, base_path, ignore_patterns):
+            ignored_count += 1
+        else:
+            filtered_files.append(file_path)
+
+    return filtered_files, ignored_count

basic_memory/importers/__init__.py

@@ -0,0 +1,27 @@
+"""Import services for Basic Memory."""
+
+from basic_memory.importers.base import Importer
+from basic_memory.importers.chatgpt_importer import ChatGPTImporter
+from basic_memory.importers.claude_conversations_importer import (
+    ClaudeConversationsImporter,
+)
+from basic_memory.importers.claude_projects_importer import ClaudeProjectsImporter
+from basic_memory.importers.memory_json_importer import MemoryJsonImporter
+from basic_memory.schemas.importer import (
+    ChatImportResult,
+    EntityImportResult,
+    ImportResult,
+    ProjectImportResult,
+)
+
+__all__ = [
+    "Importer",
+    "ChatGPTImporter",
+    "ClaudeConversationsImporter",
+    "ClaudeProjectsImporter",
+    "MemoryJsonImporter",
+    "ImportResult",
+    "ChatImportResult",
+    "EntityImportResult",
+    "ProjectImportResult",
+]

basic_memory/importers/base.py

@@ -0,0 +1,79 @@
+"""Base import service for Basic Memory."""
+
+import logging
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, Optional, TypeVar
+
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
+from basic_memory.markdown.schemas import EntityMarkdown
+from basic_memory.schemas.importer import ImportResult
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T", bound=ImportResult)
+
+
+class Importer[T: ImportResult]:
+    """Base class for all import services."""
+
+    def __init__(self, base_path: Path, markdown_processor: MarkdownProcessor):
+        """Initialize the import service.
+
+        Args:
+            markdown_processor: MarkdownProcessor instance for writing markdown files.
+        """
+        self.base_path = base_path.resolve()  # Get absolute path
+        self.markdown_processor = markdown_processor
+
+    @abstractmethod
+    async def import_data(self, source_data, destination_folder: str, **kwargs: Any) -> T:
+        """Import data from source file to destination folder.
+
+        Args:
+            source_path: Path to the source file.
+            destination_folder: Destination folder within the project.
+            **kwargs: Additional keyword arguments for specific import types.
+
+        Returns:
+            ImportResult containing statistics and status of the import.
+        """
+        pass  # pragma: no cover
+
+    async def write_entity(self, entity: EntityMarkdown, file_path: Path) -> None:
+        """Write entity to file using markdown processor.
+
+        Args:
+            entity: EntityMarkdown instance to write.
+            file_path: Path to write the entity to.
+        """
+        await self.markdown_processor.write_file(file_path, entity)
+
+    def ensure_folder_exists(self, folder: str) -> Path:
+        """Ensure folder exists, create if it doesn't.
+
+        Args:
+            base_path: Base path of the project.
+            folder: Folder name or path within the project.
+
+        Returns:
+            Path to the folder.
+        """
+        folder_path = self.base_path / folder
+        folder_path.mkdir(parents=True, exist_ok=True)
+        return folder_path
+
+    @abstractmethod
+    def handle_error(
+        self, message: str, error: Optional[Exception] = None
+    ) -> T:  # pragma: no cover
+        """Handle errors during import.
+
+        Args:
+            message: Error message.
+            error: Optional exception that caused the error.
+
+        Returns:
+            ImportResult with error information.
+        """
+        pass