basic-memory 0.17.1__py3-none-any.whl

basic_memory/alembic/versions/f8a9b2c3d4e5_add_pg_trgm_for_fuzzy_link_resolution.py

@@ -0,0 +1,239 @@
+"""Add project_id to relation/observation and pg_trgm for fuzzy link resolution
+
+Revision ID: f8a9b2c3d4e5
+Revises: 314f1ea54dc4
+Create Date: 2025-12-01 12:00:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy import text
+
+
+def column_exists(connection, table: str, column: str) -> bool:
+    """Check if a column exists in a table (idempotent migration support)."""
+    if connection.dialect.name == "postgresql":
+        result = connection.execute(
+            text(
+                "SELECT 1 FROM information_schema.columns "
+                "WHERE table_name = :table AND column_name = :column"
+            ),
+            {"table": table, "column": column},
+        )
+        return result.fetchone() is not None
+    else:
+        # SQLite
+        result = connection.execute(text(f"PRAGMA table_info({table})"))
+        columns = [row[1] for row in result]
+        return column in columns
+
+
+def index_exists(connection, index_name: str) -> bool:
+    """Check if an index exists (idempotent migration support)."""
+    if connection.dialect.name == "postgresql":
+        result = connection.execute(
+            text("SELECT 1 FROM pg_indexes WHERE indexname = :index_name"),
+            {"index_name": index_name},
+        )
+        return result.fetchone() is not None
+    else:
+        # SQLite
+        result = connection.execute(
+            text("SELECT 1 FROM sqlite_master WHERE type='index' AND name = :index_name"),
+            {"index_name": index_name},
+        )
+        return result.fetchone() is not None
+
+
+# revision identifiers, used by Alembic.
+revision: str = "f8a9b2c3d4e5"
+down_revision: Union[str, None] = "314f1ea54dc4"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Add project_id to relation and observation tables, plus pg_trgm indexes.
+
+    This migration:
+    1. Adds project_id column to relation and observation tables (denormalization)
+    2. Backfills project_id from the associated entity
+    3. Enables pg_trgm extension for trigram-based fuzzy matching (Postgres only)
+    4. Creates GIN indexes on entity title and permalink for fast similarity searches
+    5. Creates partial index on unresolved relations for efficient bulk resolution
+    """
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    # -------------------------------------------------------------------------
+    # Add project_id to relation table
+    # -------------------------------------------------------------------------
+
+    # Step 1: Add project_id column as nullable first (idempotent)
+    if not column_exists(connection, "relation", "project_id"):
+        op.add_column("relation", sa.Column("project_id", sa.Integer(), nullable=True))
+
+        # Step 2: Backfill project_id from entity.project_id via from_id
+        if dialect == "postgresql":
+            op.execute("""
+                UPDATE relation
+                SET project_id = entity.project_id
+                FROM entity
+                WHERE relation.from_id = entity.id
+            """)
+        else:
+            # SQLite syntax
+            op.execute("""
+                UPDATE relation
+                SET project_id = (
+                    SELECT entity.project_id
+                    FROM entity
+                    WHERE entity.id = relation.from_id
+                )
+            """)
+
+        # Step 3: Make project_id NOT NULL and add foreign key
+        if dialect == "postgresql":
+            op.alter_column("relation", "project_id", nullable=False)
+            op.create_foreign_key(
+                "fk_relation_project_id",
+                "relation",
+                "project",
+                ["project_id"],
+                ["id"],
+            )
+        else:
+            # SQLite requires batch operations for ALTER COLUMN
+            with op.batch_alter_table("relation") as batch_op:
+                batch_op.alter_column("project_id", nullable=False)
+                batch_op.create_foreign_key(
+                    "fk_relation_project_id",
+                    "project",
+                    ["project_id"],
+                    ["id"],
+                )
+
+    # Step 4: Create index on relation.project_id (idempotent)
+    if not index_exists(connection, "ix_relation_project_id"):
+        op.create_index("ix_relation_project_id", "relation", ["project_id"])
+
+    # -------------------------------------------------------------------------
+    # Add project_id to observation table
+    # -------------------------------------------------------------------------
+
+    # Step 1: Add project_id column as nullable first (idempotent)
+    if not column_exists(connection, "observation", "project_id"):
+        op.add_column("observation", sa.Column("project_id", sa.Integer(), nullable=True))
+
+        # Step 2: Backfill project_id from entity.project_id via entity_id
+        if dialect == "postgresql":
+            op.execute("""
+                UPDATE observation
+                SET project_id = entity.project_id
+                FROM entity
+                WHERE observation.entity_id = entity.id
+            """)
+        else:
+            # SQLite syntax
+            op.execute("""
+                UPDATE observation
+                SET project_id = (
+                    SELECT entity.project_id
+                    FROM entity
+                    WHERE entity.id = observation.entity_id
+                )
+            """)
+
+        # Step 3: Make project_id NOT NULL and add foreign key
+        if dialect == "postgresql":
+            op.alter_column("observation", "project_id", nullable=False)
+            op.create_foreign_key(
+                "fk_observation_project_id",
+                "observation",
+                "project",
+                ["project_id"],
+                ["id"],
+            )
+        else:
+            # SQLite requires batch operations for ALTER COLUMN
+            with op.batch_alter_table("observation") as batch_op:
+                batch_op.alter_column("project_id", nullable=False)
+                batch_op.create_foreign_key(
+                    "fk_observation_project_id",
+                    "project",
+                    ["project_id"],
+                    ["id"],
+                )
+
+    # Step 4: Create index on observation.project_id (idempotent)
+    if not index_exists(connection, "ix_observation_project_id"):
+        op.create_index("ix_observation_project_id", "observation", ["project_id"])
+
+    # Postgres-specific: pg_trgm and GIN indexes
+    if dialect == "postgresql":
+        # Enable pg_trgm extension for fuzzy string matching
+        op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+        # Create trigram indexes on entity table for fuzzy matching
+        # GIN indexes with gin_trgm_ops support similarity searches
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_entity_title_trgm
+            ON entity USING gin (title gin_trgm_ops)
+        """)
+
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_entity_permalink_trgm
+            ON entity USING gin (permalink gin_trgm_ops)
+        """)
+
+        # Create partial index on unresolved relations for efficient bulk resolution
+        # This makes "WHERE to_id IS NULL AND project_id = X" queries very fast
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_relation_unresolved
+            ON relation (project_id, to_name)
+            WHERE to_id IS NULL
+        """)
+
+        # Create index on relation.to_name for join performance in bulk resolution
+        op.execute("""
+            CREATE INDEX IF NOT EXISTS idx_relation_to_name
+            ON relation (to_name)
+        """)
+
+
+def downgrade() -> None:
+    """Remove project_id from relation/observation and pg_trgm indexes."""
+    connection = op.get_bind()
+    dialect = connection.dialect.name
+
+    if dialect == "postgresql":
+        # Drop Postgres-specific indexes
+        op.execute("DROP INDEX IF EXISTS idx_relation_to_name")
+        op.execute("DROP INDEX IF EXISTS idx_relation_unresolved")
+        op.execute("DROP INDEX IF EXISTS idx_entity_permalink_trgm")
+        op.execute("DROP INDEX IF EXISTS idx_entity_title_trgm")
+        # Note: We don't drop the pg_trgm extension as other code may depend on it
+
+        # Drop project_id from observation
+        op.drop_index("ix_observation_project_id", table_name="observation")
+        op.drop_constraint("fk_observation_project_id", "observation", type_="foreignkey")
+        op.drop_column("observation", "project_id")
+
+        # Drop project_id from relation
+        op.drop_index("ix_relation_project_id", table_name="relation")
+        op.drop_constraint("fk_relation_project_id", "relation", type_="foreignkey")
+        op.drop_column("relation", "project_id")
+    else:
+        # SQLite requires batch operations
+        op.drop_index("ix_observation_project_id", table_name="observation")
+        with op.batch_alter_table("observation") as batch_op:
+            batch_op.drop_constraint("fk_observation_project_id", type_="foreignkey")
+            batch_op.drop_column("project_id")
+
+        op.drop_index("ix_relation_project_id", table_name="relation")
+        with op.batch_alter_table("relation") as batch_op:
+            batch_op.drop_constraint("fk_relation_project_id", type_="foreignkey")
+            batch_op.drop_column("project_id")

basic_memory/api/__init__.py

@@ -0,0 +1,5 @@
+"""Basic Memory API module."""
+
+from .app import app
+
+__all__ = ["app"]

basic_memory/api/app.py

@@ -0,0 +1,131 @@
+"""FastAPI application for basic-memory knowledge graph API."""
+
+import asyncio
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI, HTTPException
+from fastapi.exception_handlers import http_exception_handler
+from loguru import logger
+
+from basic_memory import __version__ as version
+from basic_memory import db
+from basic_memory.api.routers import (
+    directory_router,
+    importer_router,
+    knowledge,
+    management,
+    memory,
+    project,
+    resource,
+    search,
+    prompt_router,
+)
+from basic_memory.api.v2.routers import (
+    knowledge_router as v2_knowledge,
+    project_router as v2_project,
+    memory_router as v2_memory,
+    search_router as v2_search,
+    resource_router as v2_resource,
+    directory_router as v2_directory,
+    prompt_router as v2_prompt,
+    importer_router as v2_importer,
+)
+from basic_memory.config import ConfigManager, init_api_logging
+from basic_memory.services.initialization import initialize_file_sync, initialize_app
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):  # pragma: no cover
+    """Lifecycle manager for the FastAPI app. Not called in stdio mcp mode"""
+
+    # Initialize logging for API (stdout in cloud mode, file otherwise)
+    init_api_logging()
+
+    app_config = ConfigManager().config
+    logger.info("Starting Basic Memory API")
+
+    await initialize_app(app_config)
+
+    # Cache database connections in app state for performance
+    logger.info("Initializing database and caching connections...")
+    engine, session_maker = await db.get_or_create_db(app_config.database_path)
+    app.state.engine = engine
+    app.state.session_maker = session_maker
+    logger.info("Database connections cached in app state")
+
+    # Start file sync if enabled
+    if app_config.sync_changes and not app_config.is_test_env:
+        logger.info(f"Sync changes enabled: {app_config.sync_changes}")
+
+        # start file sync task in background
+        async def _file_sync_runner() -> None:
+            await initialize_file_sync(app_config)
+
+        app.state.sync_task = asyncio.create_task(_file_sync_runner())
+    else:
+        if app_config.is_test_env:
+            logger.info("Test environment detected. Skipping file sync service.")
+        else:
+            logger.info("Sync changes disabled. Skipping file sync service.")
+        app.state.sync_task = None
+
+    # proceed with startup
+    yield
+
+    logger.info("Shutting down Basic Memory API")
+    if app.state.sync_task:
+        logger.info("Stopping sync...")
+        app.state.sync_task.cancel()  # pyright: ignore
+        try:
+            await app.state.sync_task
+        except asyncio.CancelledError:
+            logger.info("Sync task cancelled successfully")
+
+    await db.shutdown_db()
+
+
+# Initialize FastAPI app
+app = FastAPI(
+    title="Basic Memory API",
+    description="Knowledge graph API for basic-memory",
+    version=version,
+    lifespan=lifespan,
+)
+
+# Include v1 routers
+app.include_router(knowledge.router, prefix="/{project}")
+app.include_router(memory.router, prefix="/{project}")
+app.include_router(resource.router, prefix="/{project}")
+app.include_router(search.router, prefix="/{project}")
+app.include_router(project.project_router, prefix="/{project}")
+app.include_router(directory_router.router, prefix="/{project}")
+app.include_router(prompt_router.router, prefix="/{project}")
+app.include_router(importer_router.router, prefix="/{project}")
+
+# Include v2 routers (ID-based paths)
+app.include_router(v2_knowledge, prefix="/v2/projects/{project_id}")
+app.include_router(v2_memory, prefix="/v2/projects/{project_id}")
+app.include_router(v2_search, prefix="/v2/projects/{project_id}")
+app.include_router(v2_resource, prefix="/v2/projects/{project_id}")
+app.include_router(v2_directory, prefix="/v2/projects/{project_id}")
+app.include_router(v2_prompt, prefix="/v2/projects/{project_id}")
+app.include_router(v2_importer, prefix="/v2/projects/{project_id}")
+app.include_router(v2_project, prefix="/v2")
+
+# Project resource router works across projects
+app.include_router(project.project_resource_router)
+app.include_router(management.router)
+
+
+@app.exception_handler(Exception)
+async def exception_handler(request, exc):  # pragma: no cover
+    logger.exception(
+        "API unhandled exception",
+        url=str(request.url),
+        method=request.method,
+        client=request.client.host if request.client else None,
+        path=request.url.path,
+        error_type=type(exc).__name__,
+        error=str(exc),
+    )
+    return await http_exception_handler(request, HTTPException(status_code=500, detail=str(exc)))

basic_memory/api/routers/__init__.py

@@ -0,0 +1,11 @@
+"""API routers."""
+
+from . import knowledge_router as knowledge
+from . import management_router as management
+from . import memory_router as memory
+from . import project_router as project
+from . import resource_router as resource
+from . import search_router as search
+from . import prompt_router as prompt
+
+__all__ = ["knowledge", "management", "memory", "project", "resource", "search", "prompt"]

basic_memory/api/routers/directory_router.py

@@ -0,0 +1,84 @@
+"""Router for directory tree operations."""
+
+from typing import List, Optional
+
+from fastapi import APIRouter, Query
+
+from basic_memory.deps import DirectoryServiceDep, ProjectIdDep
+from basic_memory.schemas.directory import DirectoryNode
+
+router = APIRouter(prefix="/directory", tags=["directory"])
+
+
+@router.get("/tree", response_model=DirectoryNode, response_model_exclude_none=True)
+async def get_directory_tree(
+    directory_service: DirectoryServiceDep,
+    project_id: ProjectIdDep,
+):
+    """Get hierarchical directory structure from the knowledge base.
+
+    Args:
+        directory_service: Service for directory operations
+        project_id: ID of the current project
+
+    Returns:
+        DirectoryNode representing the root of the hierarchical tree structure
+    """
+    # Get a hierarchical directory tree for the specific project
+    tree = await directory_service.get_directory_tree()
+
+    # Return the hierarchical tree
+    return tree
+
+
+@router.get("/structure", response_model=DirectoryNode, response_model_exclude_none=True)
+async def get_directory_structure(
+    directory_service: DirectoryServiceDep,
+    project_id: ProjectIdDep,
+):
+    """Get folder structure for navigation (no files).
+
+    Optimized endpoint for folder tree navigation. Returns only directory nodes
+    without file metadata. For full tree with files, use /directory/tree.
+
+    Args:
+        directory_service: Service for directory operations
+        project_id: ID of the current project
+
+    Returns:
+        DirectoryNode tree containing only folders (type="directory")
+    """
+    structure = await directory_service.get_directory_structure()
+    return structure
+
+
+@router.get("/list", response_model=List[DirectoryNode], response_model_exclude_none=True)
+async def list_directory(
+    directory_service: DirectoryServiceDep,
+    project_id: ProjectIdDep,
+    dir_name: str = Query("/", description="Directory path to list"),
+    depth: int = Query(1, ge=1, le=10, description="Recursion depth (1-10)"),
+    file_name_glob: Optional[str] = Query(
+        None, description="Glob pattern for filtering file names"
+    ),
+):
+    """List directory contents with filtering and depth control.
+
+    Args:
+        directory_service: Service for directory operations
+        project_id: ID of the current project
+        dir_name: Directory path to list (default: root "/")
+        depth: Recursion depth (1-10, default: 1 for immediate children only)
+        file_name_glob: Optional glob pattern for filtering file names (e.g., "*.md", "*meeting*")
+
+    Returns:
+        List of DirectoryNode objects matching the criteria
+    """
+    # Get directory listing with filtering
+    nodes = await directory_service.list_directory(
+        dir_name=dir_name,
+        depth=depth,
+        file_name_glob=file_name_glob,
+    )
+
+    return nodes

basic_memory/api/routers/importer_router.py

@@ -0,0 +1,152 @@
+"""Import router for Basic Memory API."""
+
+import json
+import logging
+
+from fastapi import APIRouter, Form, HTTPException, UploadFile, status
+
+from basic_memory.deps import (
+    ChatGPTImporterDep,
+    ClaudeConversationsImporterDep,
+    ClaudeProjectsImporterDep,
+    MemoryJsonImporterDep,
+)
+from basic_memory.importers import Importer
+from basic_memory.schemas.importer import (
+    ChatImportResult,
+    EntityImportResult,
+    ProjectImportResult,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/import", tags=["import"])
+
+
+@router.post("/chatgpt", response_model=ChatImportResult)
+async def import_chatgpt(
+    importer: ChatGPTImporterDep,
+    file: UploadFile,
+    folder: str = Form("conversations"),
+) -> ChatImportResult:
+    """Import conversations from ChatGPT JSON export.
+
+    Args:
+        file: The ChatGPT conversations.json file.
+        folder: The folder to place the files in.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        ChatImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    return await import_file(importer, file, folder)
+
+
+@router.post("/claude/conversations", response_model=ChatImportResult)
+async def import_claude_conversations(
+    importer: ClaudeConversationsImporterDep,
+    file: UploadFile,
+    folder: str = Form("conversations"),
+) -> ChatImportResult:
+    """Import conversations from Claude conversations.json export.
+
+    Args:
+        file: The Claude conversations.json file.
+        folder: The folder to place the files in.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        ChatImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    return await import_file(importer, file, folder)
+
+
+@router.post("/claude/projects", response_model=ProjectImportResult)
+async def import_claude_projects(
+    importer: ClaudeProjectsImporterDep,
+    file: UploadFile,
+    folder: str = Form("projects"),
+) -> ProjectImportResult:
+    """Import projects from Claude projects.json export.
+
+    Args:
+        file: The Claude projects.json file.
+        base_folder: The base folder to place the files in.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        ProjectImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    return await import_file(importer, file, folder)
+
+
+@router.post("/memory-json", response_model=EntityImportResult)
+async def import_memory_json(
+    importer: MemoryJsonImporterDep,
+    file: UploadFile,
+    folder: str = Form("conversations"),
+) -> EntityImportResult:
+    """Import entities and relations from a memory.json file.
+
+    Args:
+        file: The memory.json file.
+        destination_folder: Optional destination folder within the project.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        EntityImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    try:
+        file_data = []
+        file_bytes = await file.read()
+        file_str = file_bytes.decode("utf-8")
+        for line in file_str.splitlines():
+            json_data = json.loads(line)
+            file_data.append(json_data)
+
+        result = await importer.import_data(file_data, folder)
+        if not result.success:  # pragma: no cover
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=result.error_message or "Import failed",
+            )
+    except Exception as e:
+        logger.exception("Import failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Import failed: {str(e)}",
+        )
+    return result
+
+
+async def import_file(importer: Importer, file: UploadFile, destination_folder: str):
+    try:
+        # Process file
+        json_data = json.load(file.file)
+        result = await importer.import_data(json_data, destination_folder)
+        if not result.success:  # pragma: no cover
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=result.error_message or "Import failed",
+            )
+
+        return result
+
+    except Exception as e:
+        logger.exception("Import failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Import failed: {str(e)}",
+        )