basic-memory 0.7.0__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/db.py

@@ -1,16 +1,16 @@
 import asyncio
+import os
 from contextlib import asynccontextmanager
 from enum import Enum, auto
 from pathlib import Path
 from typing import AsyncGenerator, Optional
 
-
-from basic_memory.config import ProjectConfig
+from basic_memory.config import BasicMemoryConfig, ConfigManager
 from alembic import command
 from alembic.config import Config
 
 from loguru import logger
-from sqlalchemy import text
+from sqlalchemy import text, event
 from sqlalchemy.ext.asyncio import (
     create_async_engine,
     async_sessionmaker,
@@ -18,12 +18,14 @@ from sqlalchemy.ext.asyncio import (
     AsyncEngine,
     async_scoped_session,
 )
+from sqlalchemy.pool import NullPool
 
 from basic_memory.repository.search_repository import SearchRepository
 
 # Module level state
 _engine: Optional[AsyncEngine] = None
 _session_maker: Optional[async_sessionmaker[AsyncSession]] = None
+_migrations_completed: bool = False
 
 
 class DatabaseType(Enum):
@@ -73,32 +75,119 @@ async def scoped_session(
         await factory.remove()
 
 
+def _configure_sqlite_connection(dbapi_conn, enable_wal: bool = True) -> None:
+    """Configure SQLite connection with WAL mode and optimizations.
+
+    Args:
+        dbapi_conn: Database API connection object
+        enable_wal: Whether to enable WAL mode (should be False for in-memory databases)
+    """
+    cursor = dbapi_conn.cursor()
+    try:
+        # Enable WAL mode for better concurrency (not supported for in-memory databases)
+        if enable_wal:
+            cursor.execute("PRAGMA journal_mode=WAL")
+        # Set busy timeout to handle locked databases
+        cursor.execute("PRAGMA busy_timeout=10000")  # 10 seconds
+        # Optimize for performance
+        cursor.execute("PRAGMA synchronous=NORMAL")
+        cursor.execute("PRAGMA cache_size=-64000")  # 64MB cache
+        cursor.execute("PRAGMA temp_store=MEMORY")
+        # Windows-specific optimizations
+        if os.name == "nt":
+            cursor.execute("PRAGMA locking_mode=NORMAL")  # Ensure normal locking on Windows
+    except Exception as e:
+        # Log but don't fail - some PRAGMAs may not be supported
+        logger.warning(f"Failed to configure SQLite connection: {e}")
+    finally:
+        cursor.close()
+
+
+def _create_engine_and_session(
+    db_path: Path, db_type: DatabaseType = DatabaseType.FILESYSTEM
+) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
+    """Internal helper to create engine and session maker."""
+    db_url = DatabaseType.get_db_url(db_path, db_type)
+    logger.debug(f"Creating engine for db_url: {db_url}")
+
+    # Configure connection args with Windows-specific settings
+    connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
+
+    # Add Windows-specific parameters to improve reliability
+    if os.name == "nt":  # Windows
+        connect_args.update(
+            {
+                "timeout": 30.0,  # Increase timeout to 30 seconds for Windows
+                "isolation_level": None,  # Use autocommit mode
+            }
+        )
+        # Use NullPool for Windows filesystem databases to avoid connection pooling issues
+        # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
+        # between connections
+        if db_type == DatabaseType.FILESYSTEM:
+            engine = create_async_engine(
+                db_url,
+                connect_args=connect_args,
+                poolclass=NullPool,  # Disable connection pooling on Windows
+                echo=False,
+            )
+        else:
+            # In-memory databases need connection pooling to maintain state
+            engine = create_async_engine(db_url, connect_args=connect_args)
+    else:
+        engine = create_async_engine(db_url, connect_args=connect_args)
+
+    # Enable WAL mode for better concurrency and reliability
+    # Note: WAL mode is not supported for in-memory databases
+    enable_wal = db_type != DatabaseType.MEMORY
+
+    @event.listens_for(engine.sync_engine, "connect")
+    def enable_wal_mode(dbapi_conn, connection_record):
+        """Enable WAL mode on each connection."""
+        _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
+
+    session_maker = async_sessionmaker(engine, expire_on_commit=False)
+    return engine, session_maker
+
+
 async def get_or_create_db(
     db_path: Path,
     db_type: DatabaseType = DatabaseType.FILESYSTEM,
+    ensure_migrations: bool = True,
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:  # pragma: no cover
     """Get or create database engine and session maker."""
     global _engine, _session_maker
 
     if _engine is None:
-        db_url = DatabaseType.get_db_url(db_path, db_type)
-        logger.debug(f"Creating engine for db_url: {db_url}")
-        _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
-        _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
+        _engine, _session_maker = _create_engine_and_session(db_path, db_type)
+
+        # Run migrations automatically unless explicitly disabled
+        if ensure_migrations:
+            app_config = ConfigManager().config
+            await run_migrations(app_config, db_type)
+
+    # These checks should never fail since we just created the engine and session maker
+    # if they were None, but we'll check anyway for the type checker
+    if _engine is None:
+        logger.error("Failed to create database engine", db_path=str(db_path))
+        raise RuntimeError("Database engine initialization failed")
+
+    if _session_maker is None:
+        logger.error("Failed to create session maker", db_path=str(db_path))
+        raise RuntimeError("Session maker initialization failed")
 
-    assert _engine is not None  # for type checker
-    assert _session_maker is not None  # for type checker
     return _engine, _session_maker
 
 
 async def shutdown_db() -> None:  # pragma: no cover
     """Clean up database connections."""
-    global _engine, _session_maker
+    global _engine, _session_maker, _migrations_completed
 
     if _engine:
         await _engine.dispose()
         _engine = None
         _session_maker = None
+        _migrations_completed = False
 
 
 @asynccontextmanager
@@ -112,27 +201,79 @@ async def engine_session_factory(
     for each test. For production use, use get_or_create_db() instead.
     """
 
-    global _engine, _session_maker
+    global _engine, _session_maker, _migrations_completed
 
     db_url = DatabaseType.get_db_url(db_path, db_type)
     logger.debug(f"Creating engine for db_url: {db_url}")
 
-    _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+    # Configure connection args with Windows-specific settings
+    connect_args: dict[str, bool | float | None] = {"check_same_thread": False}
+
+    # Add Windows-specific parameters to improve reliability
+    if os.name == "nt":  # Windows
+        connect_args.update(
+            {
+                "timeout": 30.0,  # Increase timeout to 30 seconds for Windows
+                "isolation_level": None,  # Use autocommit mode
+            }
+        )
+        # Use NullPool for Windows filesystem databases to avoid connection pooling issues
+        # Important: Do NOT use NullPool for in-memory databases as it will destroy the database
+        # between connections
+        if db_type == DatabaseType.FILESYSTEM:
+            _engine = create_async_engine(
+                db_url,
+                connect_args=connect_args,
+                poolclass=NullPool,  # Disable connection pooling on Windows
+                echo=False,
+            )
+        else:
+            # In-memory databases need connection pooling to maintain state
+            _engine = create_async_engine(db_url, connect_args=connect_args)
+    else:
+        _engine = create_async_engine(db_url, connect_args=connect_args)
+
+    # Enable WAL mode for better concurrency and reliability
+    # Note: WAL mode is not supported for in-memory databases
+    enable_wal = db_type != DatabaseType.MEMORY
+
+    @event.listens_for(_engine.sync_engine, "connect")
+    def enable_wal_mode(dbapi_conn, connection_record):
+        """Enable WAL mode on each connection."""
+        _configure_sqlite_connection(dbapi_conn, enable_wal=enable_wal)
+
     try:
         _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
 
-        assert _engine is not None  # for type checker
-        assert _session_maker is not None  # for type checker
+        # Verify that engine and session maker are initialized
+        if _engine is None:  # pragma: no cover
+            logger.error("Database engine is None in engine_session_factory")
+            raise RuntimeError("Database engine initialization failed")
+
+        if _session_maker is None:  # pragma: no cover
+            logger.error("Session maker is None in engine_session_factory")
+            raise RuntimeError("Session maker initialization failed")
+
         yield _engine, _session_maker
     finally:
         if _engine:
             await _engine.dispose()
             _engine = None
             _session_maker = None
+            _migrations_completed = False
 
 
-async def run_migrations(app_config: ProjectConfig, database_type=DatabaseType.FILESYSTEM):
+async def run_migrations(
+    app_config: BasicMemoryConfig, database_type=DatabaseType.FILESYSTEM, force: bool = False
+):  # pragma: no cover
     """Run any pending alembic migrations."""
+    global _migrations_completed
+
+    # Skip if migrations already completed unless forced
+    if _migrations_completed and not force:
+        logger.debug("Migrations already completed in this session, skipping")
+        return
+
     logger.info("Running database migrations...")
     try:
         # Get the absolute path to the alembic directory relative to this file
@@ -154,8 +295,18 @@ async def run_migrations(app_config: ProjectConfig, database_type=DatabaseType.F
         command.upgrade(config, "head")
         logger.info("Migrations completed successfully")
 
-        _, session_maker = await get_or_create_db(app_config.database_path, database_type)
-        await SearchRepository(session_maker).init_search_index()
+        # Get session maker - ensure we don't trigger recursive migration calls
+        if _session_maker is None:
+            _, session_maker = _create_engine_and_session(app_config.database_path, database_type)
+        else:
+            session_maker = _session_maker
+
+        # initialize the search Index schema
+        # the project_id is not used for init_search_index, so we pass a dummy value
+        await SearchRepository(session_maker, 1).init_search_index()
+
+        # Mark migrations as completed
+        _migrations_completed = True
     except Exception as e:  # pragma: no cover
         logger.error(f"Error running migrations: {e}")
         raise

basic_memory/deps.py

@@ -1,52 +1,104 @@
 """Dependency injection functions for basic-memory services."""
 
 from typing import Annotated
+from loguru import logger
 
-import logfire
-from fastapi import Depends
+from fastapi import Depends, HTTPException, Path, status, Request
 from sqlalchemy.ext.asyncio import (
     AsyncSession,
     AsyncEngine,
     async_sessionmaker,
 )
+import pathlib
 
 from basic_memory import db
-from basic_memory.config import ProjectConfig, config
+from basic_memory.config import ProjectConfig, BasicMemoryConfig, ConfigManager
+from basic_memory.importers import (
+    ChatGPTImporter,
+    ClaudeConversationsImporter,
+    ClaudeProjectsImporter,
+    MemoryJsonImporter,
+)
 from basic_memory.markdown import EntityParser
 from basic_memory.markdown.markdown_processor import MarkdownProcessor
 from basic_memory.repository.entity_repository import EntityRepository
 from basic_memory.repository.observation_repository import ObservationRepository
+from basic_memory.repository.project_repository import ProjectRepository
 from basic_memory.repository.relation_repository import RelationRepository
 from basic_memory.repository.search_repository import SearchRepository
-from basic_memory.services import (
-    EntityService,
-)
+from basic_memory.services import EntityService, ProjectService
 from basic_memory.services.context_service import ContextService
+from basic_memory.services.directory_service import DirectoryService
 from basic_memory.services.file_service import FileService
 from basic_memory.services.link_resolver import LinkResolver
 from basic_memory.services.search_service import SearchService
+from basic_memory.sync import SyncService
+from basic_memory.utils import generate_permalink
+
+
+def get_app_config() -> BasicMemoryConfig:  # pragma: no cover
+    app_config = ConfigManager().config
+    return app_config
+
+
+AppConfigDep = Annotated[BasicMemoryConfig, Depends(get_app_config)]  # pragma: no cover
 
 
 ## project
 
 
-def get_project_config() -> ProjectConfig:  # pragma: no cover
-    return config
+async def get_project_config(
+    project: "ProjectPathDep", project_repository: "ProjectRepositoryDep"
+) -> ProjectConfig:  # pragma: no cover
+    """Get the current project referenced from request state.
 
+    Args:
+        request: The current request object
+        project_repository: Repository for project operations
 
-ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]  # pragma: no cover
+    Returns:
+        The resolved project config
+
+    Raises:
+        HTTPException: If project is not found
+    """
+    # Convert project name to permalink for lookup
+    project_permalink = generate_permalink(str(project))
+    project_obj = await project_repository.get_by_permalink(project_permalink)
+    if project_obj:
+        return ProjectConfig(name=project_obj.name, home=pathlib.Path(project_obj.path))
 
+    # Not found
+    raise HTTPException(  # pragma: no cover
+        status_code=status.HTTP_404_NOT_FOUND, detail=f"Project '{project}' not found."
+    )
+
+
+ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]  # pragma: no cover
 
 ## sqlalchemy
 
 
 async def get_engine_factory(
-    project_config: ProjectConfigDep,
+    request: Request,
 ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:  # pragma: no cover
-    """Get engine and session maker."""
-    engine, session_maker = await db.get_or_create_db(project_config.database_path)
-    if project_config.env != "test":
-        logfire.instrument_sqlalchemy(engine=engine)
+    """Get cached engine and session maker from app state.
+
+    For API requests, returns cached connections from app.state for optimal performance.
+    For non-API contexts (CLI), falls back to direct database connection.
+    """
+    # Try to get cached connections from app state (API context)
+    if (
+        hasattr(request, "app")
+        and hasattr(request.app.state, "engine")
+        and hasattr(request.app.state, "session_maker")
+    ):
+        return request.app.state.engine, request.app.state.session_maker
+
+    # Fallback for non-API contexts (CLI)
+    logger.debug("Using fallback database connection for non-API context")
+    app_config = get_app_config()
+    engine, session_maker = await db.get_or_create_db(app_config.database_path)
     return engine, session_maker
 
 
@@ -67,11 +119,70 @@ SessionMakerDep = Annotated[async_sessionmaker, Depends(get_session_maker)]
 ## repositories
 
 
+async def get_project_repository(
+    session_maker: SessionMakerDep,
+) -> ProjectRepository:
+    """Get the project repository."""
+    return ProjectRepository(session_maker)
+
+
+ProjectRepositoryDep = Annotated[ProjectRepository, Depends(get_project_repository)]
+ProjectPathDep = Annotated[str, Path()]  # Use Path dependency to extract from URL
+
+
+async def get_project_id(
+    project_repository: ProjectRepositoryDep,
+    project: ProjectPathDep,
+) -> int:
+    """Get the current project ID from request state.
+
+    When using sub-applications with /{project} mounting, the project value
+    is stored in request.state by middleware.
+
+    Args:
+        request: The current request object
+        project_repository: Repository for project operations
+
+    Returns:
+        The resolved project ID
+
+    Raises:
+        HTTPException: If project is not found
+    """
+    # Convert project name to permalink for lookup
+    project_permalink = generate_permalink(str(project))
+    project_obj = await project_repository.get_by_permalink(project_permalink)
+    if project_obj:
+        return project_obj.id
+
+    # Try by name if permalink lookup fails
+    project_obj = await project_repository.get_by_name(str(project))  # pragma: no cover
+    if project_obj:  # pragma: no cover
+        return project_obj.id
+
+    # Not found
+    raise HTTPException(  # pragma: no cover
+        status_code=status.HTTP_404_NOT_FOUND, detail=f"Project '{project}' not found."
+    )
+
+
+"""
+The project_id dependency is used in the following:
+- EntityRepository
+- ObservationRepository
+- RelationRepository
+- SearchRepository
+- ProjectInfoRepository
+"""
+ProjectIdDep = Annotated[int, Depends(get_project_id)]
+
+
 async def get_entity_repository(
     session_maker: SessionMakerDep,
+    project_id: ProjectIdDep,
 ) -> EntityRepository:
-    """Create an EntityRepository instance."""
-    return EntityRepository(session_maker)
+    """Create an EntityRepository instance for the current project."""
+    return EntityRepository(session_maker, project_id=project_id)
 
 
 EntityRepositoryDep = Annotated[EntityRepository, Depends(get_entity_repository)]
@@ -79,9 +190,10 @@ EntityRepositoryDep = Annotated[EntityRepository, Depends(get_entity_repository)
 
 async def get_observation_repository(
     session_maker: SessionMakerDep,
+    project_id: ProjectIdDep,
 ) -> ObservationRepository:
-    """Create an ObservationRepository instance."""
-    return ObservationRepository(session_maker)
+    """Create an ObservationRepository instance for the current project."""
+    return ObservationRepository(session_maker, project_id=project_id)
 
 
 ObservationRepositoryDep = Annotated[ObservationRepository, Depends(get_observation_repository)]
@@ -89,9 +201,10 @@ ObservationRepositoryDep = Annotated[ObservationRepository, Depends(get_observat
 
 async def get_relation_repository(
     session_maker: SessionMakerDep,
+    project_id: ProjectIdDep,
 ) -> RelationRepository:
-    """Create a RelationRepository instance."""
-    return RelationRepository(session_maker)
+    """Create a RelationRepository instance for the current project."""
+    return RelationRepository(session_maker, project_id=project_id)
 
 
 RelationRepositoryDep = Annotated[RelationRepository, Depends(get_relation_repository)]
@@ -99,14 +212,18 @@ RelationRepositoryDep = Annotated[RelationRepository, Depends(get_relation_repos
 
 async def get_search_repository(
     session_maker: SessionMakerDep,
+    project_id: ProjectIdDep,
 ) -> SearchRepository:
-    """Create a SearchRepository instance."""
-    return SearchRepository(session_maker)
+    """Create a SearchRepository instance for the current project."""
+    return SearchRepository(session_maker, project_id=project_id)
 
 
 SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)]
 
 
+# ProjectInfoRepository is deprecated and will be removed in a future version.
+# Use ProjectRepository instead, which has the same functionality plus more project-specific operations.
+
 ## services
 
 
@@ -127,7 +244,12 @@ MarkdownProcessorDep = Annotated[MarkdownProcessor, Depends(get_markdown_process
 async def get_file_service(
     project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
 ) -> FileService:
-    return FileService(project_config.home, markdown_processor)
+    logger.debug(
+        f"Creating FileService for project: {project_config.name}, base_path: {project_config.home}"
+    )
+    file_service = FileService(project_config.home, markdown_processor)
+    logger.debug(f"Created FileService for project: {file_service} ")
+    return file_service
 
 
 FileServiceDep = Annotated[FileService, Depends(get_file_service)]
@@ -140,6 +262,7 @@ async def get_entity_service(
     entity_parser: EntityParserDep,
     file_service: FileServiceDep,
     link_resolver: "LinkResolverDep",
+    app_config: AppConfigDep,
 ) -> EntityService:
     """Create EntityService with repository."""
     return EntityService(
@@ -149,6 +272,7 @@ async def get_entity_service(
         entity_parser=entity_parser,
         file_service=file_service,
         link_resolver=link_resolver,
+        app_config=app_config,
     )
 
 
@@ -177,9 +301,111 @@ LinkResolverDep = Annotated[LinkResolver, Depends(get_link_resolver)]
 
 
 async def get_context_service(
-    search_repository: SearchRepositoryDep, entity_repository: EntityRepositoryDep
+    search_repository: SearchRepositoryDep,
+    entity_repository: EntityRepositoryDep,
+    observation_repository: ObservationRepositoryDep,
 ) -> ContextService:
-    return ContextService(search_repository, entity_repository)
+    return ContextService(
+        search_repository=search_repository,
+        entity_repository=entity_repository,
+        observation_repository=observation_repository,
+    )
 
 
 ContextServiceDep = Annotated[ContextService, Depends(get_context_service)]
+
+
+async def get_sync_service(
+    app_config: AppConfigDep,
+    entity_service: EntityServiceDep,
+    entity_parser: EntityParserDep,
+    entity_repository: EntityRepositoryDep,
+    relation_repository: RelationRepositoryDep,
+    project_repository: ProjectRepositoryDep,
+    search_service: SearchServiceDep,
+    file_service: FileServiceDep,
+) -> SyncService:  # pragma: no cover
+    """
+
+    :rtype: object
+    """
+    return SyncService(
+        app_config=app_config,
+        entity_service=entity_service,
+        entity_parser=entity_parser,
+        entity_repository=entity_repository,
+        relation_repository=relation_repository,
+        project_repository=project_repository,
+        search_service=search_service,
+        file_service=file_service,
+    )
+
+
+SyncServiceDep = Annotated[SyncService, Depends(get_sync_service)]
+
+
+async def get_project_service(
+    project_repository: ProjectRepositoryDep,
+) -> ProjectService:
+    """Create ProjectService with repository."""
+    return ProjectService(repository=project_repository)
+
+
+ProjectServiceDep = Annotated[ProjectService, Depends(get_project_service)]
+
+
+async def get_directory_service(
+    entity_repository: EntityRepositoryDep,
+) -> DirectoryService:
+    """Create DirectoryService with dependencies."""
+    return DirectoryService(
+        entity_repository=entity_repository,
+    )
+
+
+DirectoryServiceDep = Annotated[DirectoryService, Depends(get_directory_service)]
+
+
+# Import
+
+
+async def get_chatgpt_importer(
+    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
+) -> ChatGPTImporter:
+    """Create ChatGPTImporter with dependencies."""
+    return ChatGPTImporter(project_config.home, markdown_processor)
+
+
+ChatGPTImporterDep = Annotated[ChatGPTImporter, Depends(get_chatgpt_importer)]
+
+
+async def get_claude_conversations_importer(
+    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
+) -> ClaudeConversationsImporter:
+    """Create ChatGPTImporter with dependencies."""
+    return ClaudeConversationsImporter(project_config.home, markdown_processor)
+
+
+ClaudeConversationsImporterDep = Annotated[
+    ClaudeConversationsImporter, Depends(get_claude_conversations_importer)
+]
+
+
+async def get_claude_projects_importer(
+    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
+) -> ClaudeProjectsImporter:
+    """Create ChatGPTImporter with dependencies."""
+    return ClaudeProjectsImporter(project_config.home, markdown_processor)
+
+
+ClaudeProjectsImporterDep = Annotated[ClaudeProjectsImporter, Depends(get_claude_projects_importer)]
+
+
+async def get_memory_json_importer(
+    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
+) -> MemoryJsonImporter:
+    """Create ChatGPTImporter with dependencies."""
+    return MemoryJsonImporter(project_config.home, markdown_processor)
+
+
+MemoryJsonImporterDep = Annotated[MemoryJsonImporter, Depends(get_memory_json_importer)]