basic-memory 0.14.3__py3-none-any.whl → 0.15.0__py3-none-any.whl

basic_memory/db.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 from contextlib import asynccontextmanager
 from enum import Enum, auto
 from pathlib import Path
@@ -9,7 +10,7 @@ from alembic import command
 from alembic.config import Config
 
 from loguru import logger
-from sqlalchemy import text
+from sqlalchemy import text, event
 from sqlalchemy.ext.asyncio import (
     create_async_engine,
     async_sessionmaker,
@@ -17,6 +18,7 @@ from sqlalchemy.ext.asyncio import (
     AsyncEngine,
     async_scoped_session,
 )
+from sqlalchemy.pool import NullPool
 
 from basic_memory.repository.search_repository import SearchRepository
 
@@ -73,13 +75,77 @@ async def scoped_session(
         await factory.remove()
 
 
+def _configure_sqlite_connection(dbapi_conn, enable_wal: bool = True) -> None:
+    """Configure SQLite connection with WAL mode and optimizations.
+
+    Args:
+        dbapi_conn: Database API connection object
+        enable_wal: Whether to enable WAL mode (should be False for in-memory databases)
+    """
+    cursor = dbapi_conn.cursor()
+    try:
+        # Enable WAL mode for better concurrency (not supported for in-memory databases)
+        if enable_wal:
+            cursor.execute("PRAGMA journal_mode=WAL")
+        # Set busy timeout to handle locked databases
+        cursor.execute("PRAGMA busy_timeout=10000")  # 10 seconds
+        # Optimize for performance
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.execute("PRAGMA cache_size=-64000")  # 64MB cache
+        cursor.execute("PRAGMA temp_store=MEMORY")
+        # Windows-specific optimizations
+        if os.name == "nt":
+            cursor.execute("PRAGMA locking_mode=NORMAL")  # Ensure normal locking on Windows
+    except Exception as e:
+        # Log but don't fail - some PRAGMAs may not be supported
+        logger.warning(f"Failed to configure SQLite connection: {e}")
+    finally:
+        cursor.close()
+
+
 def _create_engine_and_session(
     db_path: Path, db_type: DatabaseType = DatabaseType.FILESYSTEM
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
     """Internal helper to create engine and session maker."""
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
-    engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+
+    # Configure connection args with Windows-specific settings
+    connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
+
+    # Add Windows-specific parameters to improve reliability
+    if os.name == "nt":  # Windows
+        connect_args.update(
+            {
+                "timeout": 30.0,  # Increase timeout to 30 seconds for Windows
+                "isolation_level": None,  # Use autocommit mode
+            }
+        )
+        # Use NullPool for Windows filesystem databases to avoid connection pooling issues
+        # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
+        # between connections
+        if db_type == DatabaseType.FILESYSTEM:
+            engine = create_async_engine(
+                db_url,
+                connect_args=connect_args,
+                poolclass=NullPool,  # Disable connection pooling on Windows
+                echo=False,
+            )
+        else:
+            # In-memory databases need connection pooling to maintain state
+            engine = create_async_engine(db_url, connect_args=connect_args)
+    else:
+        engine = create_async_engine(db_url, connect_args=connect_args)
+
+    # Enable WAL mode for better concurrency and reliability
+    # Note: WAL mode is not supported for in-memory databases
+    enable_wal = db_type != DatabaseType.MEMORY
+
+    @event.listens_for(engine.sync_engine, "connect")
+    def enable_wal_mode(dbapi_conn, connection_record):
+        """Enable WAL mode on each connection."""
+        _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
+
     session_maker = async_sessionmaker(engine, expire_on_commit=False)
     return engine, session_maker
 
@@ -140,7 +206,42 @@ async def engine_session_factory(
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
 
-    _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+    # Configure connection args with Windows-specific settings
+    connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
+
+    # Add Windows-specific parameters to improve reliability
+    if os.name == "nt":  # Windows
+        connect_args.update(
+            {
+                "timeout": 30.0,  # Increase timeout to 30 seconds for Windows
+                "isolation_level": None,  # Use autocommit mode
+            }
+        )
+        # Use NullPool for Windows filesystem databases to avoid connection pooling issues
+        # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
+        # between connections
+        if db_type == DatabaseType.FILESYSTEM:
+            _engine = create_async_engine(
+                db_url,
+                connect_args=connect_args,
+                poolclass=NullPool,  # Disable connection pooling on Windows
+                echo=False,
+            )
+        else:
+            # In-memory databases need connection pooling to maintain state
+            _engine = create_async_engine(db_url, connect_args=connect_args)
+    else:
+        _engine = create_async_engine(db_url, connect_args=connect_args)
+
+    # Enable WAL mode for better concurrency and reliability
+    # Note: WAL mode is not supported for in-memory databases
+    enable_wal = db_type != DatabaseType.MEMORY
+
+    @event.listens_for(_engine.sync_engine, "connect")
+    def enable_wal_mode(dbapi_conn, connection_record):
+        """Enable WAL mode on each connection."""
+        _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
+
     try:
         _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 

basic_memory/deps.py

@@ -3,7 +3,7 @@
 from typing import Annotated
 from loguru import logger
 
-from fastapi import Depends, HTTPException, Path, status
+from fastapi import Depends, HTTPException, Path, status, Request
 from sqlalchemy.ext.asyncio import (
     AsyncSession,
     AsyncEngine,
@@ -78,9 +78,24 @@ ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]  # prag
 
 
 async def get_engine_factory(
-    app_config: AppConfigDep,
+    request: Request,
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:  # pragma: no cover
-    """Get engine and session maker."""
+    """Get cached engine and session maker from app state.
+
+    For API requests, returns cached connections from app.state for optimal performance.
+    For non-API contexts (CLI), falls back to direct database connection.
+    """
+    # Try to get cached connections from app state (API context)
+    if (
+        hasattr(request, "app")
+        and hasattr(request.app.state, "engine")
+        and hasattr(request.app.state, "session_maker")
+    ):
+        return request.app.state.engine, request.app.state.session_maker
+
+    # Fallback for non-API contexts (CLI)
+    logger.debug("Using fallback database connection for non-API context")
+    app_config = get_app_config()
     engine, session_maker = await db.get_or_create_db(app_config.database_path)
     return engine, session_maker
 
@@ -245,6 +260,7 @@ async def get_entity_service(
     entity_parser: EntityParserDep,
     file_service: FileServiceDep,
     link_resolver: "LinkResolverDep",
+    app_config: AppConfigDep,
 ) -> EntityService:
     """Create EntityService with repository."""
     return EntityService(
@@ -254,6 +270,7 @@ async def get_entity_service(
         entity_parser=entity_parser,
         file_service=file_service,
         link_resolver=link_resolver,
+        app_config=app_config,
     )
 
 

basic_memory/file_utils.py

@@ -2,9 +2,11 @@
 
 import hashlib
 from pathlib import Path
+import re
 from typing import Any, Dict, Union
 
 import yaml
+import frontmatter
 from loguru import logger
 
 from basic_memory.utils import FilePath
@@ -233,3 +235,90 @@ async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str:
             error=str(e),
         )
         raise FileError(f"Failed to update frontmatter: {e}")
+
+
+def dump_frontmatter(post: frontmatter.Post) -> str:
+    """
+    Serialize frontmatter.Post to markdown with Obsidian-compatible YAML format.
+
+    This function ensures that tags are formatted as YAML lists instead of JSON arrays:
+
+    Good (Obsidian compatible):
+    ---
+    tags:
+    - system
+    - overview
+    - reference
+    ---
+
+    Bad (current behavior):
+    ---
+    tags: ["system", "overview", "reference"]
+    ---
+
+    Args:
+        post: frontmatter.Post object to serialize
+
+    Returns:
+        String containing markdown with properly formatted YAML frontmatter
+    """
+    if not post.metadata:
+        # No frontmatter, just return content
+        return post.content
+
+    # Serialize YAML with block style for lists
+    yaml_str = yaml.dump(
+        post.metadata, sort_keys=False, allow_unicode=True, default_flow_style=False
+    )
+
+    # Construct the final markdown with frontmatter
+    if post.content:
+        return f"---\n{yaml_str}---\n\n{post.content}"
+    else:
+        return f"---\n{yaml_str}---\n"
+
+
+def sanitize_for_filename(text: str, replacement: str = "-") -> str:
+    """
+    Sanitize string to be safe for use as a note title
+    Replaces path separators and other problematic characters
+    with hyphens.
+    """
+    # replace both POSIX and Windows path separators
+    text = re.sub(r"[/\\]", replacement, text)
+
+    # replace some other problematic chars
+    text = re.sub(r'[<>:"|?*]', replacement, text)
+
+    # compress multiple, repeated replacements
+    text = re.sub(f"{re.escape(replacement)}+", replacement, text)
+
+    return text.strip(replacement)
+
+
+def sanitize_for_folder(folder: str) -> str:
+    """
+    Sanitize folder path to be safe for use in file system paths.
+    Removes leading/trailing whitespace, compresses multiple slashes,
+    and removes special characters except for /, -, and _.
+    """
+    if not folder:
+        return ""
+
+    sanitized = folder.strip()
+
+    if sanitized.startswith("./"):
+        sanitized = sanitized[2:]
+
+    # ensure no special characters (except for a few that are allowed)
+    sanitized = "".join(
+        c for c in sanitized if c.isalnum() or c in (".", " ", "-", "_", "\\", "/")
+    ).rstrip()
+
+    # compress multiple, repeated instances of path separators
+    sanitized = re.sub(r"[\\/]+", "/", sanitized)
+
+    # trim any leading/trailing path separators
+    sanitized = sanitized.strip("\\/")
+
+    return sanitized

basic_memory/ignore_utils.py

@@ -0,0 +1,295 @@
+"""Utilities for handling .gitignore patterns and file filtering."""
+
+import fnmatch
+from pathlib import Path
+from typing import Set
+
+
+# Common directories and patterns to ignore by default
+# These are used as fallback if .bmignore doesn't exist
+DEFAULT_IGNORE_PATTERNS = {
+    # Hidden files (files starting with dot)
+    ".*",
+    # Basic Memory internal files
+    "memory.db",
+    "memory.db-shm",
+    "memory.db-wal",
+    "config.json",
+    # Version control
+    ".git",
+    ".svn",
+    # Python
+    "__pycache__",
+    "*.pyc",
+    "*.pyo",
+    "*.pyd",
+    ".pytest_cache",
+    ".coverage",
+    "*.egg-info",
+    ".tox",
+    ".mypy_cache",
+    ".ruff_cache",
+    # Virtual environments
+    ".venv",
+    "venv",
+    "env",
+    ".env",
+    # Node.js
+    "node_modules",
+    # Build artifacts
+    "build",
+    "dist",
+    ".cache",
+    # IDE
+    ".idea",
+    ".vscode",
+    # OS files
+    ".DS_Store",
+    "Thumbs.db",
+    "desktop.ini",
+    # Obsidian
+    ".obsidian",
+    # Temporary files
+    "*.tmp",
+    "*.swp",
+    "*.swo",
+    "*~",
+}
+
+
+def get_bmignore_path() -> Path:
+    """Get path to .bmignore file.
+
+    Returns:
+        Path to ~/.basic-memory/.bmignore
+    """
+    return Path.home() / ".basic-memory" / ".bmignore"
+
+
+def create_default_bmignore() -> None:
+    """Create default .bmignore file if it doesn't exist.
+
+    This ensures users have a file they can customize for all Basic Memory operations.
+    """
+    bmignore_path = get_bmignore_path()
+
+    if bmignore_path.exists():
+        return
+
+    bmignore_path.parent.mkdir(parents=True, exist_ok=True)
+    bmignore_path.write_text("""# Basic Memory Ignore Patterns
+# This file is used by both 'bm cloud upload', 'bm cloud bisync', and file sync
+# Patterns use standard gitignore-style syntax
+
+# Hidden files (files starting with dot)
+.*
+
+# Basic Memory internal files
+memory.db
+memory.db-shm
+memory.db-wal
+config.json
+
+# Version control
+.git
+.svn
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache
+.coverage
+*.egg-info
+.tox
+.mypy_cache
+.ruff_cache
+
+# Virtual environments
+.venv
+venv
+env
+.env
+
+# Node.js
+node_modules
+
+# Build artifacts
+build
+dist
+.cache
+
+# IDE
+.idea
+.vscode
+
+# OS files
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Obsidian
+.obsidian
+
+# Temporary files
+*.tmp
+*.swp
+*.swo
+*~
+""")
+
+
+def load_bmignore_patterns() -> Set[str]:
+    """Load patterns from .bmignore file.
+
+    Returns:
+        Set of patterns from .bmignore, or DEFAULT_IGNORE_PATTERNS if file doesn't exist
+    """
+    bmignore_path = get_bmignore_path()
+
+    # Create default file if it doesn't exist
+    if not bmignore_path.exists():
+        create_default_bmignore()
+
+    patterns = set()
+
+    try:
+        with bmignore_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                # Skip empty lines and comments
+                if line and not line.startswith("#"):
+                    patterns.add(line)
+    except Exception:
+        # If we can't read .bmignore, fall back to defaults
+        return set(DEFAULT_IGNORE_PATTERNS)
+
+    # If no patterns were loaded, use defaults
+    if not patterns:
+        return set(DEFAULT_IGNORE_PATTERNS)
+
+    return patterns
+
+
+def load_gitignore_patterns(base_path: Path) -> Set[str]:
+    """Load gitignore patterns from .gitignore file and .bmignore.
+
+    Combines patterns from:
+    1. ~/.basic-memory/.bmignore (user's global ignore patterns)
+    2. {base_path}/.gitignore (project-specific patterns)
+
+    Args:
+        base_path: The base directory to search for .gitignore file
+
+    Returns:
+        Set of patterns to ignore
+    """
+    # Start with patterns from .bmignore
+    patterns = load_bmignore_patterns()
+
+    gitignore_file = base_path / ".gitignore"
+    if gitignore_file.exists():
+        try:
+            with gitignore_file.open("r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    # Skip empty lines and comments
+                    if line and not line.startswith("#"):
+                        patterns.add(line)
+        except Exception:
+            # If we can't read .gitignore, just use default patterns
+            pass
+
+    return patterns
+
+
+def should_ignore_path(file_path: Path, base_path: Path, ignore_patterns: Set[str]) -> bool:
+    """Check if a file path should be ignored based on gitignore patterns.
+
+    Args:
+        file_path: The file path to check
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to match against
+
+    Returns:
+        True if the path should be ignored, False otherwise
+    """
+    # Get the relative path from base
+    try:
+        relative_path = file_path.relative_to(base_path)
+        relative_str = str(relative_path)
+        relative_posix = relative_path.as_posix()  # Use forward slashes for matching
+
+        # Check each pattern
+        for pattern in ignore_patterns:
+            # Handle patterns starting with / (root relative)
+            if pattern.startswith("/"):
+                root_pattern = pattern[1:]  # Remove leading /
+
+                # For directory patterns ending with /
+                if root_pattern.endswith("/"):
+                    dir_name = root_pattern[:-1]  # Remove trailing /
+                    # Check if the first part of the path matches the directory name
+                    if len(relative_path.parts) > 0 and relative_path.parts[0] == dir_name:
+                        return True
+                else:
+                    # Regular root-relative pattern
+                    if fnmatch.fnmatch(relative_posix, root_pattern):
+                        return True
+                continue
+
+            # Handle directory patterns (ending with /)
+            if pattern.endswith("/"):
+                dir_name = pattern[:-1]  # Remove trailing /
+                # Check if any path part matches the directory name
+                if dir_name in relative_path.parts:
+                    return True
+                continue
+
+            # Direct name match (e.g., ".git", "node_modules")
+            if pattern in relative_path.parts:
+                return True
+
+            # Check if any individual path part matches the glob pattern
+            # This handles cases like ".*" matching ".hidden.md" in "concept/.hidden.md"
+            for part in relative_path.parts:
+                if fnmatch.fnmatch(part, pattern):
+                    return True
+
+            # Glob pattern match on full path
+            if fnmatch.fnmatch(relative_posix, pattern) or fnmatch.fnmatch(relative_str, pattern):
+                return True
+
+        return False
+    except ValueError:
+        # If we can't get relative path, don't ignore
+        return False
+
+
+def filter_files(
+    files: list[Path], base_path: Path, ignore_patterns: Set[str] | None = None
+) -> tuple[list[Path], int]:
+    """Filter a list of files based on gitignore patterns.
+
+    Args:
+        files: List of file paths to filter
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to ignore. If None, loads from .gitignore
+
+    Returns:
+        Tuple of (filtered_files, ignored_count)
+    """
+    if ignore_patterns is None:
+        ignore_patterns = load_gitignore_patterns(base_path)
+
+    filtered_files = []
+    ignored_count = 0
+
+    for file_path in files:
+        if should_ignore_path(file_path, base_path, ignore_patterns):
+            ignored_count += 1
+        else:
+            filtered_files.append(file_path)
+
+    return filtered_files, ignored_count

basic_memory/importers/chatgpt_importer.py

@@ -93,7 +93,7 @@ class ChatGPTImporter(Importer[ChatImportResult]):
                 break
 
         # Generate permalink
-        date_prefix = datetime.fromtimestamp(created_at).strftime("%Y%m%d")
+        date_prefix = datetime.fromtimestamp(created_at).astimezone().strftime("%Y%m%d")
         clean_title = clean_filename(conversation["title"])
 
         # Format content

basic_memory/importers/utils.py

@@ -43,13 +43,13 @@ def format_timestamp(timestamp: Any) -> str:  # pragma: no cover
         except ValueError:
             try:
                 # Try unix timestamp as string
-                timestamp = datetime.fromtimestamp(float(timestamp))
+                timestamp = datetime.fromtimestamp(float(timestamp)).astimezone()
             except ValueError:
                 # Return as is if we can't parse it
                 return timestamp
     elif isinstance(timestamp, (int, float)):
         # Unix timestamp
-        timestamp = datetime.fromtimestamp(timestamp)
+        timestamp = datetime.fromtimestamp(timestamp).astimezone()
 
     if isinstance(timestamp, datetime):
         return timestamp.strftime("%Y-%m-%d %H:%M:%S")

basic_memory/markdown/entity_parser.py

@@ -130,6 +130,6 @@ class EntityParser:
             content=post.content,
             observations=entity_content.observations,
             relations=entity_content.relations,
-            created=datetime.fromtimestamp(file_stats.st_ctime),
-            modified=datetime.fromtimestamp(file_stats.st_mtime),
+            created=datetime.fromtimestamp(file_stats.st_ctime).astimezone(),
+            modified=datetime.fromtimestamp(file_stats.st_mtime).astimezone(),
         )

basic_memory/markdown/markdown_processor.py

@@ -2,11 +2,11 @@ from pathlib import Path
 from typing import Optional
 from collections import OrderedDict
 
-import frontmatter
 from frontmatter import Post
 from loguru import logger
 
 from basic_memory import file_utils
+from basic_memory.file_utils import dump_frontmatter
 from basic_memory.markdown.entity_parser import EntityParser
 from basic_memory.markdown.schemas import EntityMarkdown, Observation, Relation
 
@@ -115,7 +115,7 @@ class MarkdownProcessor:
 
         # Create Post object for frontmatter
         post = Post(content, **frontmatter_dict)
-        final_content = frontmatter.dumps(post, sort_keys=False)
+        final_content = dump_frontmatter(post)
 
         logger.debug(f"writing file {path} with content:\n{final_content}")