basic-memory 0.14.4__py3-none-any.whl → 0.15.0__py3-none-any.whl

basic_memory/db.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 from contextlib import asynccontextmanager
 from enum import Enum, auto
 from pathlib import Path
@@ -9,7 +10,7 @@ from alembic import command
 from alembic.config import Config
 
 from loguru import logger
-from sqlalchemy import text
+from sqlalchemy import text, event
 from sqlalchemy.ext.asyncio import (
     create_async_engine,
     async_sessionmaker,
@@ -17,6 +18,7 @@ from sqlalchemy.ext.asyncio import (
     AsyncEngine,
     async_scoped_session,
 )
+from sqlalchemy.pool import NullPool
 
 from basic_memory.repository.search_repository import SearchRepository
 
@@ -73,13 +75,77 @@ async def scoped_session(
         await factory.remove()
 
 
+def _configure_sqlite_connection(dbapi_conn, enable_wal: bool = True) -> None:
+    """Configure SQLite connection with WAL mode and optimizations.
+
+    Args:
+        dbapi_conn: Database API connection object
+        enable_wal: Whether to enable WAL mode (should be False for in-memory databases)
+    """
+    cursor = dbapi_conn.cursor()
+    try:
+        # Enable WAL mode for better concurrency (not supported for in-memory databases)
+        if enable_wal:
+            cursor.execute("PRAGMA journal_mode=WAL")
+        # Set busy timeout to handle locked databases
+        cursor.execute("PRAGMA busy_timeout=10000")  # 10 seconds
+        # Optimize for performance
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.execute("PRAGMA cache_size=-64000")  # 64MB cache
+        cursor.execute("PRAGMA temp_store=MEMORY")
+        # Windows-specific optimizations
+        if os.name == "nt":
+            cursor.execute("PRAGMA locking_mode=NORMAL")  # Ensure normal locking on Windows
+    except Exception as e:
+        # Log but don't fail - some PRAGMAs may not be supported
+        logger.warning(f"Failed to configure SQLite connection: {e}")
+    finally:
+        cursor.close()
+
+
 def _create_engine_and_session(
     db_path: Path, db_type: DatabaseType = DatabaseType.FILESYSTEM
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
     """Internal helper to create engine and session maker."""
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
-    engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+
+    # Configure connection args with Windows-specific settings
+    connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
+
+    # Add Windows-specific parameters to improve reliability
+    if os.name == "nt":  # Windows
+        connect_args.update(
+            {
+                "timeout": 30.0,  # Increase timeout to 30 seconds for Windows
+                "isolation_level": None,  # Use autocommit mode
+            }
+        )
+        # Use NullPool for Windows filesystem databases to avoid connection pooling issues
+        # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
+        # between connections
+        if db_type == DatabaseType.FILESYSTEM:
+            engine = create_async_engine(
+                db_url,
+                connect_args=connect_args,
+                poolclass=NullPool,  # Disable connection pooling on Windows
+                echo=False,
+            )
+        else:
+            # In-memory databases need connection pooling to maintain state
+            engine = create_async_engine(db_url, connect_args=connect_args)
+    else:
+        engine = create_async_engine(db_url, connect_args=connect_args)
+
+    # Enable WAL mode for better concurrency and reliability
+    # Note: WAL mode is not supported for in-memory databases
+    enable_wal = db_type != DatabaseType.MEMORY
+
+    @event.listens_for(engine.sync_engine, "connect")
+    def enable_wal_mode(dbapi_conn, connection_record):
+        """Enable WAL mode on each connection."""
+        _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
+
     session_maker = async_sessionmaker(engine, expire_on_commit=False)
     return engine, session_maker
 
@@ -140,7 +206,42 @@ async def engine_session_factory(
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
 
-    _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+    # Configure connection args with Windows-specific settings
+    connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
+
+    # Add Windows-specific parameters to improve reliability
+    if os.name == "nt":  # Windows
+        connect_args.update(
+            {
+                "timeout": 30.0,  # Increase timeout to 30 seconds for Windows
+                "isolation_level": None,  # Use autocommit mode
+            }
+        )
+        # Use NullPool for Windows filesystem databases to avoid connection pooling issues
+        # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
+        # between connections
+        if db_type == DatabaseType.FILESYSTEM:
+            _engine = create_async_engine(
+                db_url,
+                connect_args=connect_args,
+                poolclass=NullPool,  # Disable connection pooling on Windows
+                echo=False,
+            )
+        else:
+            # In-memory databases need connection pooling to maintain state
+            _engine = create_async_engine(db_url, connect_args=connect_args)
+    else:
+        _engine = create_async_engine(db_url, connect_args=connect_args)
+
+    # Enable WAL mode for better concurrency and reliability
+    # Note: WAL mode is not supported for in-memory databases
+    enable_wal = db_type != DatabaseType.MEMORY
+
+    @event.listens_for(_engine.sync_engine, "connect")
+    def enable_wal_mode(dbapi_conn, connection_record):
+        """Enable WAL mode on each connection."""
+        _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
+
     try:
         _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 

basic_memory/deps.py

@@ -3,7 +3,7 @@
 from typing import Annotated
 from loguru import logger
 
-from fastapi import Depends, HTTPException, Path, status
+from fastapi import Depends, HTTPException, Path, status, Request
 from sqlalchemy.ext.asyncio import (
     AsyncSession,
     AsyncEngine,
@@ -78,9 +78,24 @@ ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]  # prag
 
 
 async def get_engine_factory(
-    app_config: AppConfigDep,
+    request: Request,
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:  # pragma: no cover
-    """Get engine and session maker."""
+    """Get cached engine and session maker from app state.
+
+    For API requests, returns cached connections from app.state for optimal performance.
+    For non-API contexts (CLI), falls back to direct database connection.
+    """
+    # Try to get cached connections from app state (API context)
+    if (
+        hasattr(request, "app")
+        and hasattr(request.app.state, "engine")
+        and hasattr(request.app.state, "session_maker")
+    ):
+        return request.app.state.engine, request.app.state.session_maker
+
+    # Fallback for non-API contexts (CLI)
+    logger.debug("Using fallback database connection for non-API context")
+    app_config = get_app_config()
     engine, session_maker = await db.get_or_create_db(app_config.database_path)
     return engine, session_maker
 
@@ -245,6 +260,7 @@ async def get_entity_service(
     entity_parser: EntityParserDep,
     file_service: FileServiceDep,
     link_resolver: "LinkResolverDep",
+    app_config: AppConfigDep,
 ) -> EntityService:
     """Create EntityService with repository."""
     return EntityService(
@@ -254,6 +270,7 @@ async def get_entity_service(
         entity_parser=entity_parser,
         file_service=file_service,
         link_resolver=link_resolver,
+        app_config=app_config,
     )
 
 

basic_memory/file_utils.py

@@ -240,40 +240,37 @@ async def update_frontmatter(path: FilePath, updates: Dict[str, Any]) -> str:
 def dump_frontmatter(post: frontmatter.Post) -> str:
     """
     Serialize frontmatter.Post to markdown with Obsidian-compatible YAML format.
-    
+
     This function ensures that tags are formatted as YAML lists instead of JSON arrays:
-    
+
     Good (Obsidian compatible):
     ---
     tags:
     - system
-    - overview  
+    - overview
     - reference
     ---
-    
+
     Bad (current behavior):
     ---
     tags: ["system", "overview", "reference"]
     ---
-    
+
     Args:
         post: frontmatter.Post object to serialize
-        
+
     Returns:
         String containing markdown with properly formatted YAML frontmatter
-    """    
+    """
     if not post.metadata:
         # No frontmatter, just return content
         return post.content
-        
+
     # Serialize YAML with block style for lists
     yaml_str = yaml.dump(
-        post.metadata,
-        sort_keys=False,
-        allow_unicode=True,
-        default_flow_style=False
+        post.metadata, sort_keys=False, allow_unicode=True, default_flow_style=False
     )
-    
+
     # Construct the final markdown with frontmatter
     if post.content:
         return f"---\n{yaml_str}---\n\n{post.content}"
@@ -298,3 +295,30 @@ def sanitize_for_filename(text: str, replacement: str = "-") -> str:
 
     return text.strip(replacement)
 
+
+def sanitize_for_folder(folder: str) -> str:
+    """
+    Sanitize folder path to be safe for use in file system paths.
+    Removes leading/trailing whitespace, compresses multiple slashes,
+    and removes special characters except for /, -, and _.
+    """
+    if not folder:
+        return ""
+
+    sanitized = folder.strip()
+
+    if sanitized.startswith("./"):
+        sanitized = sanitized[2:]
+
+    # ensure no special characters (except for a few that are allowed)
+    sanitized = "".join(
+        c for c in sanitized if c.isalnum() or c in (".", " ", "-", "_", "\\", "/")
+    ).rstrip()
+
+    # compress multiple, repeated instances of path separators
+    sanitized = re.sub(r"[\\/]+", "/", sanitized)
+
+    # trim any leading/trailing path separators
+    sanitized = sanitized.strip("\\/")
+
+    return sanitized

basic_memory/ignore_utils.py

@@ -0,0 +1,295 @@
+"""Utilities for handling .gitignore patterns and file filtering."""
+
+import fnmatch
+from pathlib import Path
+from typing import Set
+
+
+# Common directories and patterns to ignore by default
+# These are used as fallback if .bmignore doesn't exist
+DEFAULT_IGNORE_PATTERNS = {
+    # Hidden files (files starting with dot)
+    ".*",
+    # Basic Memory internal files
+    "memory.db",
+    "memory.db-shm",
+    "memory.db-wal",
+    "config.json",
+    # Version control
+    ".git",
+    ".svn",
+    # Python
+    "__pycache__",
+    "*.pyc",
+    "*.pyo",
+    "*.pyd",
+    ".pytest_cache",
+    ".coverage",
+    "*.egg-info",
+    ".tox",
+    ".mypy_cache",
+    ".ruff_cache",
+    # Virtual environments
+    ".venv",
+    "venv",
+    "env",
+    ".env",
+    # Node.js
+    "node_modules",
+    # Build artifacts
+    "build",
+    "dist",
+    ".cache",
+    # IDE
+    ".idea",
+    ".vscode",
+    # OS files
+    ".DS_Store",
+    "Thumbs.db",
+    "desktop.ini",
+    # Obsidian
+    ".obsidian",
+    # Temporary files
+    "*.tmp",
+    "*.swp",
+    "*.swo",
+    "*~",
+}
+
+
+def get_bmignore_path() -> Path:
+    """Get path to .bmignore file.
+
+    Returns:
+        Path to ~/.basic-memory/.bmignore
+    """
+    return Path.home() / ".basic-memory" / ".bmignore"
+
+
+def create_default_bmignore() -> None:
+    """Create default .bmignore file if it doesn't exist.
+
+    This ensures users have a file they can customize for all Basic Memory operations.
+    """
+    bmignore_path = get_bmignore_path()
+
+    if bmignore_path.exists():
+        return
+
+    bmignore_path.parent.mkdir(parents=True, exist_ok=True)
+    bmignore_path.write_text("""# Basic Memory Ignore Patterns
+# This file is used by both 'bm cloud upload', 'bm cloud bisync', and file sync
+# Patterns use standard gitignore-style syntax
+
+# Hidden files (files starting with dot)
+.*
+
+# Basic Memory internal files
+memory.db
+memory.db-shm
+memory.db-wal
+config.json
+
+# Version control
+.git
+.svn
+
+# Python
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.pytest_cache
+.coverage
+*.egg-info
+.tox
+.mypy_cache
+.ruff_cache
+
+# Virtual environments
+.venv
+venv
+env
+.env
+
+# Node.js
+node_modules
+
+# Build artifacts
+build
+dist
+.cache
+
+# IDE
+.idea
+.vscode
+
+# OS files
+.DS_Store
+Thumbs.db
+desktop.ini
+
+# Obsidian
+.obsidian
+
+# Temporary files
+*.tmp
+*.swp
+*.swo
+*~
+""")
+
+
+def load_bmignore_patterns() -> Set[str]:
+    """Load patterns from .bmignore file.
+
+    Returns:
+        Set of patterns from .bmignore, or DEFAULT_IGNORE_PATTERNS if file doesn't exist
+    """
+    bmignore_path = get_bmignore_path()
+
+    # Create default file if it doesn't exist
+    if not bmignore_path.exists():
+        create_default_bmignore()
+
+    patterns = set()
+
+    try:
+        with bmignore_path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                # Skip empty lines and comments
+                if line and not line.startswith("#"):
+                    patterns.add(line)
+    except Exception:
+        # If we can't read .bmignore, fall back to defaults
+        return set(DEFAULT_IGNORE_PATTERNS)
+
+    # If no patterns were loaded, use defaults
+    if not patterns:
+        return set(DEFAULT_IGNORE_PATTERNS)
+
+    return patterns
+
+
+def load_gitignore_patterns(base_path: Path) -> Set[str]:
+    """Load gitignore patterns from .gitignore file and .bmignore.
+
+    Combines patterns from:
+    1. ~/.basic-memory/.bmignore (user's global ignore patterns)
+    2. {base_path}/.gitignore (project-specific patterns)
+
+    Args:
+        base_path: The base directory to search for .gitignore file
+
+    Returns:
+        Set of patterns to ignore
+    """
+    # Start with patterns from .bmignore
+    patterns = load_bmignore_patterns()
+
+    gitignore_file = base_path / ".gitignore"
+    if gitignore_file.exists():
+        try:
+            with gitignore_file.open("r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    # Skip empty lines and comments
+                    if line and not line.startswith("#"):
+                        patterns.add(line)
+        except Exception:
+            # If we can't read .gitignore, just use default patterns
+            pass
+
+    return patterns
+
+
+def should_ignore_path(file_path: Path, base_path: Path, ignore_patterns: Set[str]) -> bool:
+    """Check if a file path should be ignored based on gitignore patterns.
+
+    Args:
+        file_path: The file path to check
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to match against
+
+    Returns:
+        True if the path should be ignored, False otherwise
+    """
+    # Get the relative path from base
+    try:
+        relative_path = file_path.relative_to(base_path)
+        relative_str = str(relative_path)
+        relative_posix = relative_path.as_posix()  # Use forward slashes for matching
+
+        # Check each pattern
+        for pattern in ignore_patterns:
+            # Handle patterns starting with / (root relative)
+            if pattern.startswith("/"):
+                root_pattern = pattern[1:]  # Remove leading /
+
+                # For directory patterns ending with /
+                if root_pattern.endswith("/"):
+                    dir_name = root_pattern[:-1]  # Remove trailing /
+                    # Check if the first part of the path matches the directory name
+                    if len(relative_path.parts) > 0 and relative_path.parts[0] == dir_name:
+                        return True
+                else:
+                    # Regular root-relative pattern
+                    if fnmatch.fnmatch(relative_posix, root_pattern):
+                        return True
+                continue
+
+            # Handle directory patterns (ending with /)
+            if pattern.endswith("/"):
+                dir_name = pattern[:-1]  # Remove trailing /
+                # Check if any path part matches the directory name
+                if dir_name in relative_path.parts:
+                    return True
+                continue
+
+            # Direct name match (e.g., ".git", "node_modules")
+            if pattern in relative_path.parts:
+                return True
+
+            # Check if any individual path part matches the glob pattern
+            # This handles cases like ".*" matching ".hidden.md" in "concept/.hidden.md"
+            for part in relative_path.parts:
+                if fnmatch.fnmatch(part, pattern):
+                    return True
+
+            # Glob pattern match on full path
+            if fnmatch.fnmatch(relative_posix, pattern) or fnmatch.fnmatch(relative_str, pattern):
+                return True
+
+        return False
+    except ValueError:
+        # If we can't get relative path, don't ignore
+        return False
+
+
+def filter_files(
+    files: list[Path], base_path: Path, ignore_patterns: Set[str] | None = None
+) -> tuple[list[Path], int]:
+    """Filter a list of files based on gitignore patterns.
+
+    Args:
+        files: List of file paths to filter
+        base_path: The base directory for relative path calculation
+        ignore_patterns: Set of patterns to ignore. If None, loads from .gitignore
+
+    Returns:
+        Tuple of (filtered_files, ignored_count)
+    """
+    if ignore_patterns is None:
+        ignore_patterns = load_gitignore_patterns(base_path)
+
+    filtered_files = []
+    ignored_count = 0
+
+    for file_path in files:
+        if should_ignore_path(file_path, base_path, ignore_patterns):
+            ignored_count += 1
+        else:
+            filtered_files.append(file_path)
+
+    return filtered_files, ignored_count

basic_memory/markdown/plugins.py

@@ -9,6 +9,7 @@ from markdown_it.token import Token
 def is_observation(token: Token) -> bool:
     """Check if token looks like our observation format."""
     import re
+
     if token.type != "inline":  # pragma: no cover
         return False
     # Use token.tag which contains the actual content for test tokens, fallback to content
@@ -18,15 +19,15 @@ def is_observation(token: Token) -> bool:
     # if it's a markdown_task, return false
     if content.startswith("[ ]") or content.startswith("[x]") or content.startswith("[-]"):
         return False
-    
+
     # Exclude markdown links: [text](url)
     if re.match(r"^\[.*?\]\(.*?\)$", content):
         return False
-    
+
     # Exclude wiki links: [[text]]
     if re.match(r"^\[\[.*?\]\]$", content):
         return False
-    
+
     # Check for proper observation format: [category] content
     match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
     has_tags = "#" in content
@@ -36,9 +37,10 @@ def is_observation(token: Token) -> bool:
 def parse_observation(token: Token) -> Dict[str, Any]:
     """Extract observation parts from token."""
     import re
+
     # Use token.tag which contains the actual content for test tokens, fallback to content
     content = (token.tag or token.content).strip()
-    
+
     # Parse [category] with regex
     match = re.match(r"^\[([^\[\]()]+)\]\s+(.+)", content)
     category = None
@@ -50,7 +52,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:
         empty_match = re.match(r"^\[\]\s+(.+)", content)
         if empty_match:
             content = empty_match.group(1).strip()
-    
+
     # Parse (context)
     context = None
     if content.endswith(")"):
@@ -58,7 +60,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:
         if start != -1:
             context = content[start + 1 : -1].strip()
             content = content[:start].strip()
-    
+
     # Extract tags and keep original content
     tags = []
     parts = content.split()
@@ -69,7 +71,7 @@ def parse_observation(token: Token) -> Dict[str, Any]:
                 tags.extend(subtags)
             else:
                 tags.append(part[1:])
-    
+
     return {
         "category": category,
         "content": content,

basic_memory/mcp/async_client.py

@@ -1,4 +1,4 @@
-from httpx import ASGITransport, AsyncClient
+from httpx import ASGITransport, AsyncClient, Timeout
 from loguru import logger
 
 from basic_memory.api.app import app as fastapi_app
@@ -9,19 +9,31 @@ def create_client() -> AsyncClient:
     """Create an HTTP client based on configuration.
 
     Returns:
-        AsyncClient configured for either local ASGI or remote HTTP transport
+        AsyncClient configured for either local ASGI or remote proxy
     """
     config_manager = ConfigManager()
-    config = config_manager.load_config()
+    config = config_manager.config
 
-    if config.api_url:
-        # Use HTTP transport for remote API
-        logger.info(f"Creating HTTP client for remote Basic Memory API: {config.api_url}")
-        return AsyncClient(base_url=config.api_url)
+    # Configure timeout for longer operations like write_note
+    # Default httpx timeout is 5 seconds which is too short for file operations
+    timeout = Timeout(
+        connect=10.0,  # 10 seconds for connection
+        read=30.0,  # 30 seconds for reading response
+        write=30.0,  # 30 seconds for writing request
+        pool=30.0,  # 30 seconds for connection pool
+    )
+
+    if config.cloud_mode_enabled:
+        # Use HTTP transport to proxy endpoint
+        proxy_base_url = f"{config.cloud_host}/proxy"
+        logger.info(f"Creating HTTP client for proxy at: {proxy_base_url}")
+        return AsyncClient(base_url=proxy_base_url, timeout=timeout)
     else:
-        # Use ASGI transport for local API
-        logger.debug("Creating ASGI client for local Basic Memory API")
-        return AsyncClient(transport=ASGITransport(app=fastapi_app), base_url="http://test")
+        # Default: use ASGI transport for local API (development mode)
+        logger.info("Creating ASGI client for local Basic Memory API")
+        return AsyncClient(
+            transport=ASGITransport(app=fastapi_app), base_url="http://test", timeout=timeout
+        )
 
 
 # Create shared async client