basic-memory 0.16.1__py3-none-any.whl → 0.17.4__py3-none-any.whl

basic_memory/alembic/versions/f8a9b2c3d4e5_add_pg_trgm_for_fuzzy_link_resolution.py

@@ -0,0 +1,239 @@
+"""Add project_id to relation/observation and pg_trgm for fuzzy link resolution
+
+Revision ID: f8a9b2c3d4e5
+Revises: 314f1ea54dc4
+Create Date: 2025-12-01 12:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import text
+
+
+def column_exists(connection, table: str, column: str) -> bool:
+    """Check if a column exists in a table (idempotent migration support)."""
+    if connection.dialect.name == "postgresql":
+        result = connection.execute(
+            text(
+                "SELECT 1 FROM information_schema.columns "
+                "WHERE table_name = :table AND column_name = :column"
+            ),
+            {"table": table, "column": column},
+        )
+        return result.fetchone() is not None
+    else:
+        # SQLite
+        result = connection.execute(text(f"PRAGMA table_info({table})"))
+        columns = [row[1] for row in result]
+        return column in columns
+
+
+def index_exists(connection, index_name: str) -> bool:
+    """Check if an index exists (idempotent migration support)."""
+    if connection.dialect.name == "postgresql":
+        result = connection.execute(
+            text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
+            {"index_name": index_name},
+        )
+        return result.fetchone() is not None
+    else:
+        # SQLite
+        result = connection.execute(
+            text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"),
+            {"index_name": index_name},
+        )
+        return result.fetchone() is not None
+
+
+# revision identifiers, used by Alembic.
+revision: str = "f8a9b2c3d4e5"
+down_revision: Union[str, None] = "314f1ea54dc4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add project_id to relation and observation tables, plus pg_trgm indexes.
+
+    This migration:
+    1. Adds project_id column to relation and observation tables (denormalization)
+    2. Backfills project_id from the associated entity
+    3. Enables pg_trgm extension for trigram-based fuzzy matching (Postgres only)
+    4. Creates GIN indexes on entity title and permalink for fast similarity searches
+    5. Creates partial index on unresolved relations for efficient bulk resolution
+    """
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    # -------------------------------------------------------------------------
+    # Add project_id to relation table
+    # -------------------------------------------------------------------------
+
+    # Step 1: Add project_id column as nullable first (idempotent)
+    if not column_exists(connection, "relation", "project_id"):
+        op.add_column("relation", sa.Column("project_id", sa.Integer(), nullable=True))
+
+        # Step 2: Backfill project_id from entity.project_id via from_id
+        if dialect == "postgresql":
+            op.execute("""
+                UPDATE relation
+                SET project_id = entity.project_id
+                FROM entity
+                WHERE relation.from_id = entity.id
+            """)
+        else:
+            # SQLite syntax
+            op.execute("""
+                UPDATE relation
+                SET project_id = (
+                    SELECT entity.project_id
+                    FROM entity
+                    WHERE entity.id = relation.from_id
+                )
+            """)
+
+        # Step 3: Make project_id NOT NULL and add foreign key
+        if dialect == "postgresql":
+            op.alter_column("relation", "project_id", nullable=False)
+            op.create_foreign_key(
+                "fk_relation_project_id",
+                "relation",
+                "project",
+                ["project_id"],
+                ["id"],
+            )
+        else:
+            # SQLite requires batch operations for ALTER COLUMN
+            with op.batch_alter_table("relation") as batch_op:
+                batch_op.alter_column("project_id", nullable=False)
+                batch_op.create_foreign_key(
+                    "fk_relation_project_id",
+                    "project",
+                    ["project_id"],
+                    ["id"],
+                )
+
+    # Step 4: Create index on relation.project_id (idempotent)
+    if not index_exists(connection, "ix_relation_project_id"):
+        op.create_index("ix_relation_project_id", "relation", ["project_id"])
+
+    # -------------------------------------------------------------------------
+    # Add project_id to observation table
+    # -------------------------------------------------------------------------
+
+    # Step 1: Add project_id column as nullable first (idempotent)
+    if not column_exists(connection, "observation", "project_id"):
+        op.add_column("observation", sa.Column("project_id", sa.Integer(), nullable=True))
+
+        # Step 2: Backfill project_id from entity.project_id via entity_id
+        if dialect == "postgresql":
+            op.execute("""
+                UPDATE observation
+                SET project_id = entity.project_id
+                FROM entity
+                WHERE observation.entity_id = entity.id
+            """)
+        else:
+            # SQLite syntax
+            op.execute("""
+                UPDATE observation
+                SET project_id = (
+                    SELECT entity.project_id
+                    FROM entity
+                    WHERE entity.id = observation.entity_id
+                )
+            """)
+
+        # Step 3: Make project_id NOT NULL and add foreign key
+        if dialect == "postgresql":
+            op.alter_column("observation", "project_id", nullable=False)
+            op.create_foreign_key(
+                "fk_observation_project_id",
+                "observation",
+                "project",
+                ["project_id"],
+                ["id"],
+            )
+        else:
+            # SQLite requires batch operations for ALTER COLUMN
+            with op.batch_alter_table("observation") as batch_op:
+                batch_op.alter_column("project_id", nullable=False)
+                batch_op.create_foreign_key(
+                    "fk_observation_project_id",
+                    "project",
+                    ["project_id"],
+                    ["id"],
+                )
+
+    # Step 4: Create index on observation.project_id (idempotent)
+    if not index_exists(connection, "ix_observation_project_id"):
+        op.create_index("ix_observation_project_id", "observation", ["project_id"])
+
+    # Postgres-specific: pg_trgm and GIN indexes
+    if dialect == "postgresql":
+        # Enable pg_trgm extension for fuzzy string matching
+        op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+        # Create trigram indexes on entity table for fuzzy matching
+        # GIN indexes with gin_trgm_ops support similarity searches
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_entity_title_trgm
+            ON entity USING gin (title gin_trgm_ops)
+        """)
+
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_entity_permalink_trgm
+            ON entity USING gin (permalink gin_trgm_ops)
+        """)
+
+        # Create partial index on unresolved relations for efficient bulk resolution
+        # This makes "WHERE to_id IS NULL AND project_id = X" queries very fast
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_relation_unresolved
+            ON relation (project_id, to_name)
+            WHERE to_id IS NULL
+        """)
+
+        # Create index on relation.to_name for join performance in bulk resolution
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_relation_to_name
+            ON relation (to_name)
+        """)
+
+
+def downgrade() -> None:
+    """Remove project_id from relation/observation and pg_trgm indexes."""
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    if dialect == "postgresql":
+        # Drop Postgres-specific indexes
+        op.execute("DROP INDEX IF EXISTS idx_relation_to_name")
+        op.execute("DROP INDEX IF EXISTS idx_relation_unresolved")
+        op.execute("DROP INDEX IF EXISTS idx_entity_permalink_trgm")
+        op.execute("DROP INDEX IF EXISTS idx_entity_title_trgm")
+        # Note: We don't drop the pg_trgm extension as other code may depend on it
+
+        # Drop project_id from observation
+        op.drop_index("ix_observation_project_id", table_name="observation")
+        op.drop_constraint("fk_observation_project_id", "observation", type_="foreignkey")
+        op.drop_column("observation", "project_id")
+
+        # Drop project_id from relation
+        op.drop_index("ix_relation_project_id", table_name="relation")
+        op.drop_constraint("fk_relation_project_id", "relation", type_="foreignkey")
+        op.drop_column("relation", "project_id")
+    else:
+        # SQLite requires batch operations
+        op.drop_index("ix_observation_project_id", table_name="observation")
+        with op.batch_alter_table("observation") as batch_op:
+            batch_op.drop_constraint("fk_observation_project_id", type_="foreignkey")
+            batch_op.drop_column("project_id")
+
+        op.drop_index("ix_relation_project_id", table_name="relation")
+        with op.batch_alter_table("relation") as batch_op:
+            batch_op.drop_constraint("fk_relation_project_id", type_="foreignkey")
+            batch_op.drop_column("project_id")

basic_memory/alembic/versions/g9a0b3c4d5e6_add_external_id_to_project_and_entity.py

@@ -0,0 +1,173 @@
+"""Add external_id UUID column to project and entity tables
+
+Revision ID: g9a0b3c4d5e6
+Revises: f8a9b2c3d4e5
+Create Date: 2025-12-29 10:00:00.000000
+
+"""
+
+import uuid
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import text
+
+
+def column_exists(connection, table: str, column: str) -> bool:
+    """Check if a column exists in a table (idempotent migration support)."""
+    if connection.dialect.name == "postgresql":
+        result = connection.execute(
+            text(
+                "SELECT 1 FROM information_schema.columns "
+                "WHERE table_name = :table AND column_name = :column"
+            ),
+            {"table": table, "column": column},
+        )
+        return result.fetchone() is not None
+    else:
+        # SQLite
+        result = connection.execute(text(f"PRAGMA table_info({table})"))
+        columns = [row[1] for row in result]
+        return column in columns
+
+
+def index_exists(connection, index_name: str) -> bool:
+    """Check if an index exists (idempotent migration support)."""
+    if connection.dialect.name == "postgresql":
+        result = connection.execute(
+            text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
+            {"index_name": index_name},
+        )
+        return result.fetchone() is not None
+    else:
+        # SQLite
+        result = connection.execute(
+            text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"),
+            {"index_name": index_name},
+        )
+        return result.fetchone() is not None
+
+
+# revision identifiers, used by Alembic.
+revision: str = "g9a0b3c4d5e6"
+down_revision: Union[str, None] = "f8a9b2c3d4e5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add external_id UUID column to project and entity tables.
+
+    This migration:
+    1. Adds external_id column to project table
+    2. Adds external_id column to entity table
+    3. Generates UUIDs for existing rows
+    4. Creates unique indexes on both columns
+    """
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    # -------------------------------------------------------------------------
+    # Add external_id to project table
+    # -------------------------------------------------------------------------
+
+    if not column_exists(connection, "project", "external_id"):
+        # Step 1: Add external_id column as nullable first
+        op.add_column("project", sa.Column("external_id", sa.String(), nullable=True))
+
+        # Step 2: Generate UUIDs for existing rows
+        if dialect == "postgresql":
+            # Postgres has gen_random_uuid() function
+            op.execute("""
+                UPDATE project
+                SET external_id = gen_random_uuid()::text
+                WHERE external_id IS NULL
+            """)
+        else:
+            # SQLite: need to generate UUIDs in Python
+            result = connection.execute(text("SELECT id FROM project WHERE external_id IS NULL"))
+            for row in result:
+                new_uuid = str(uuid.uuid4())
+                connection.execute(
+                    text("UPDATE project SET external_id = :uuid WHERE id = :id"),
+                    {"uuid": new_uuid, "id": row[0]},
+                )
+
+        # Step 3: Make external_id NOT NULL
+        if dialect == "postgresql":
+            op.alter_column("project", "external_id", nullable=False)
+        else:
+            # SQLite requires batch operations for ALTER COLUMN
+            with op.batch_alter_table("project") as batch_op:
+                batch_op.alter_column("external_id", nullable=False)
+
+    # Step 4: Create unique index on project.external_id (idempotent)
+    if not index_exists(connection, "ix_project_external_id"):
+        op.create_index("ix_project_external_id", "project", ["external_id"], unique=True)
+
+    # -------------------------------------------------------------------------
+    # Add external_id to entity table
+    # -------------------------------------------------------------------------
+
+    if not column_exists(connection, "entity", "external_id"):
+        # Step 1: Add external_id column as nullable first
+        op.add_column("entity", sa.Column("external_id", sa.String(), nullable=True))
+
+        # Step 2: Generate UUIDs for existing rows
+        if dialect == "postgresql":
+            # Postgres has gen_random_uuid() function
+            op.execute("""
+                UPDATE entity
+                SET external_id = gen_random_uuid()::text
+                WHERE external_id IS NULL
+            """)
+        else:
+            # SQLite: need to generate UUIDs in Python
+            result = connection.execute(text("SELECT id FROM entity WHERE external_id IS NULL"))
+            for row in result:
+                new_uuid = str(uuid.uuid4())
+                connection.execute(
+                    text("UPDATE entity SET external_id = :uuid WHERE id = :id"),
+                    {"uuid": new_uuid, "id": row[0]},
+                )
+
+        # Step 3: Make external_id NOT NULL
+        if dialect == "postgresql":
+            op.alter_column("entity", "external_id", nullable=False)
+        else:
+            # SQLite requires batch operations for ALTER COLUMN
+            with op.batch_alter_table("entity") as batch_op:
+                batch_op.alter_column("external_id", nullable=False)
+
+    # Step 4: Create unique index on entity.external_id (idempotent)
+    if not index_exists(connection, "ix_entity_external_id"):
+        op.create_index("ix_entity_external_id", "entity", ["external_id"], unique=True)
+
+
+def downgrade() -> None:
+    """Remove external_id columns from project and entity tables."""
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    # Drop from entity table
+    if index_exists(connection, "ix_entity_external_id"):
+        op.drop_index("ix_entity_external_id", table_name="entity")
+
+    if column_exists(connection, "entity", "external_id"):
+        if dialect == "postgresql":
+            op.drop_column("entity", "external_id")
+        else:
+            with op.batch_alter_table("entity") as batch_op:
+                batch_op.drop_column("external_id")
+
+    # Drop from project table
+    if index_exists(connection, "ix_project_external_id"):
+        op.drop_index("ix_project_external_id", table_name="project")
+
+    if column_exists(connection, "project", "external_id"):
+        if dialect == "postgresql":
+            op.drop_column("project", "external_id")
+        else:
+            with op.batch_alter_table("project") as batch_op:
+                batch_op.drop_column("external_id")

basic_memory/api/app.py

@@ -1,6 +1,5 @@
 """FastAPI application for basic-memory knowledge graph API."""
 
-import asyncio
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, HTTPException
@@ -8,7 +7,7 @@ from fastapi.exception_handlers import http_exception_handler
 from loguru import logger
 
 from basic_memory import __version__ as version
-from basic_memory import db
+from basic_memory.api.container import ApiContainer, set_container
 from basic_memory.api.routers import (
     directory_router,
     importer_router,
@@ -20,42 +19,57 @@ from basic_memory.api.routers import (
     search,
     prompt_router,
 )
-from basic_memory.config import ConfigManager
-from basic_memory.services.initialization import initialize_file_sync, initialize_app
+from basic_memory.api.v2.routers import (
+    knowledge_router as v2_knowledge,
+    project_router as v2_project,
+    memory_router as v2_memory,
+    search_router as v2_search,
+    resource_router as v2_resource,
+    directory_router as v2_directory,
+    prompt_router as v2_prompt,
+    importer_router as v2_importer,
+)
+from basic_memory.config import init_api_logging
+from basic_memory.services.initialization import initialize_app
 
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):  # pragma: no cover
     """Lifecycle manager for the FastAPI app. Not called in stdio mcp mode"""
 
-    app_config = ConfigManager().config
-    logger.info("Starting Basic Memory API")
+    # Initialize logging for API (stdout in cloud mode, file otherwise)
+    init_api_logging()
+
+    # --- Composition Root ---
+    # Create container and read config (single point of config access)
+    container = ApiContainer.create()
+    set_container(container)
+    app.state.container = container
+
+    logger.info(f"Starting Basic Memory API (mode={container.mode.name})")
 
-    await initialize_app(app_config)
+    await initialize_app(container.config)
 
     # Cache database connections in app state for performance
     logger.info("Initializing database and caching connections...")
-    engine, session_maker = await db.get_or_create_db(app_config.database_path)
+    engine, session_maker = await container.init_database()
     app.state.engine = engine
     app.state.session_maker = session_maker
     logger.info("Database connections cached in app state")
 
-    logger.info(f"Sync changes enabled: {app_config.sync_changes}")
-    if app_config.sync_changes:
-        # start file sync task in background
-        app.state.sync_task = asyncio.create_task(initialize_file_sync(app_config))
-    else:
-        logger.info("Sync changes disabled. Skipping file sync service.")
+    # Create and start sync coordinator (lifecycle centralized in coordinator)
+    sync_coordinator = container.create_sync_coordinator()
+    await sync_coordinator.start()
+    app.state.sync_coordinator = sync_coordinator
 
-    # proceed with startup
+    # Proceed with startup
     yield
 
+    # Shutdown - coordinator handles clean task cancellation
     logger.info("Shutting down Basic Memory API")
-    if app.state.sync_task:
-        logger.info("Stopping sync...")
-        app.state.sync_task.cancel()  # pyright: ignore
+    await sync_coordinator.stop()
 
-    await db.shutdown_db()
+    await container.shutdown_database()
 
 
 # Initialize FastAPI app
@@ -66,8 +80,17 @@ app = FastAPI(
     lifespan=lifespan,
 )
 
-
-# Include routers
+# Include v2 routers FIRST (more specific paths must match before /{project} catch-all)
+app.include_router(v2_knowledge, prefix="/v2/projects/{project_id}")
+app.include_router(v2_memory, prefix="/v2/projects/{project_id}")
+app.include_router(v2_search, prefix="/v2/projects/{project_id}")
+app.include_router(v2_resource, prefix="/v2/projects/{project_id}")
+app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
+app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
+app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
+app.include_router(v2_project, prefix="/v2")
+
+# Include v1 routers (/{project} is a catch-all, must come after specific prefixes)
 app.include_router(knowledge.router, prefix="/{project}")
 app.include_router(memory.router, prefix="/{project}")
 app.include_router(resource.router, prefix="/{project}")
@@ -77,12 +100,10 @@ app.include_router(directory_router.router, prefix="/{project}")
 app.include_router(prompt_router.router, prefix="/{project}")
 app.include_router(importer_router.router, prefix="/{project}")
 
-# Project resource router works accross projects
+# Project resource router works across projects
 app.include_router(project.project_resource_router)
 app.include_router(management.router)
 
-# Auth routes are handled by FastMCP automatically when auth is enabled
-
 
 @app.exception_handler(Exception)
 async def exception_handler(request, exc):  # pragma: no cover

basic_memory/api/container.py

@@ -0,0 +1,133 @@
+"""API composition root for Basic Memory.
+
+This container owns reading ConfigManager and environment variables for the
+API entrypoint. Downstream modules receive config/dependencies explicitly
+rather than reading globals.
+
+Design principles:
+- Only this module reads ConfigManager directly
+- Runtime mode (cloud/local/test) is resolved here
+- Factories for services are provided, not singletons
+"""
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker, AsyncSession
+
+from basic_memory import db
+from basic_memory.config import BasicMemoryConfig, ConfigManager
+from basic_memory.runtime import RuntimeMode, resolve_runtime_mode
+
+if TYPE_CHECKING:  # pragma: no cover
+    from basic_memory.sync import SyncCoordinator
+
+
+@dataclass
+class ApiContainer:
+    """Composition root for the API entrypoint.
+
+    Holds resolved configuration and runtime context.
+    Created once at app startup, then used to wire dependencies.
+    """
+
+    config: BasicMemoryConfig
+    mode: RuntimeMode
+
+    # --- Database ---
+    # Cached database connections (set during lifespan startup)
+    engine: AsyncEngine | None = None
+    session_maker: async_sessionmaker[AsyncSession] | None = None
+
+    @classmethod
+    def create(cls) -> "ApiContainer":  # pragma: no cover
+        """Create container by reading ConfigManager.
+
+        This is the single point where API reads global config.
+        """
+        config = ConfigManager().config
+        mode = resolve_runtime_mode(
+            cloud_mode_enabled=config.cloud_mode_enabled,
+            is_test_env=config.is_test_env,
+        )
+        return cls(config=config, mode=mode)
+
+    # --- Runtime Mode Properties ---
+
+    @property
+    def should_sync_files(self) -> bool:
+        """Whether file sync should be started.
+
+        Sync is enabled when:
+        - sync_changes is True in config
+        - Not in test mode (tests manage their own sync)
+        """
+        return self.config.sync_changes and not self.mode.is_test
+
+    @property
+    def sync_skip_reason(self) -> str | None:  # pragma: no cover
+        """Reason why sync is skipped, or None if sync should run.
+
+        Useful for logging why sync was disabled.
+        """
+        if self.mode.is_test:
+            return "Test environment detected"
+        if not self.config.sync_changes:
+            return "Sync changes disabled"
+        return None
+
+    def create_sync_coordinator(self) -> "SyncCoordinator":  # pragma: no cover
+        """Create a SyncCoordinator with this container's settings.
+
+        Returns:
+            SyncCoordinator configured for this runtime environment
+        """
+        # Deferred import to avoid circular dependency
+        from basic_memory.sync import SyncCoordinator
+
+        return SyncCoordinator(
+            config=self.config,
+            should_sync=self.should_sync_files,
+            skip_reason=self.sync_skip_reason,
+        )
+
+    # --- Database Factory ---
+
+    async def init_database(  # pragma: no cover
+        self,
+    ) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
+        """Initialize and cache database connections.
+
+        Returns:
+            Tuple of (engine, session_maker)
+        """
+        engine, session_maker = await db.get_or_create_db(self.config.database_path)
+        self.engine = engine
+        self.session_maker = session_maker
+        return engine, session_maker
+
+    async def shutdown_database(self) -> None:  # pragma: no cover
+        """Clean up database connections."""
+        await db.shutdown_db()
+
+
+# Module-level container instance (set by lifespan)
+# This allows deps.py to access the container without reading ConfigManager
+_container: ApiContainer | None = None
+
+
+def get_container() -> ApiContainer:
+    """Get the current API container.
+
+    Raises:
+        RuntimeError: If container hasn't been initialized
+    """
+    if _container is None:
+        raise RuntimeError("API container not initialized. Call set_container() first.")
+    return _container
+
+
+def set_container(container: ApiContainer) -> None:
+    """Set the API container (called by lifespan)."""
+    global _container
+    _container = container