basic-memory 0.2.12__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/alembic/versions/cc7172b46608_update_search_index_schema.py

@@ -0,0 +1,100 @@
+"""Update search index schema
+
+Revision ID: cc7172b46608
+Revises: 502b60eaa905
+Create Date: 2025-02-28 18:48:23.244941
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "cc7172b46608"
+down_revision: Union[str, None] = "502b60eaa905"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema to use new search index with content_stems and content_snippet."""
+
+    # First, drop the existing search_index table
+    op.execute("DROP TABLE IF EXISTS search_index")
+
+    # Create new search_index with updated schema
+    op.execute("""
+    CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+        -- Core entity fields
+        id UNINDEXED,          -- Row ID
+        title,                 -- Title for searching
+        content_stems,         -- Main searchable content split into stems
+        content_snippet,       -- File content snippet for display
+        permalink,             -- Stable identifier (now indexed for path search)
+        file_path UNINDEXED,   -- Physical location
+        type UNINDEXED,        -- entity/relation/observation
+        
+        -- Relation fields 
+        from_id UNINDEXED,     -- Source entity
+        to_id UNINDEXED,       -- Target entity
+        relation_type UNINDEXED, -- Type of relation
+        
+        -- Observation fields
+        entity_id UNINDEXED,   -- Parent entity
+        category UNINDEXED,    -- Observation category
+        
+        -- Common fields
+        metadata UNINDEXED,    -- JSON metadata
+        created_at UNINDEXED,  -- Creation timestamp
+        updated_at UNINDEXED,  -- Last update
+        
+        -- Configuration
+        tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+        prefix='1,2,3,4'                    -- Support longer prefixes for paths
+    );
+    """)
+
+
+def downgrade() -> None:
+    """Downgrade database schema to use old search index."""
+    # Drop the updated search_index table
+    op.execute("DROP TABLE IF EXISTS search_index")
+
+    # Recreate the original search_index schema
+    op.execute("""
+    CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+        -- Core entity fields
+        id UNINDEXED,          -- Row ID
+        title,                 -- Title for searching
+        content,               -- Main searchable content
+        permalink,             -- Stable identifier (now indexed for path search)
+        file_path UNINDEXED,   -- Physical location
+        type UNINDEXED,        -- entity/relation/observation
+        
+        -- Relation fields 
+        from_id UNINDEXED,     -- Source entity
+        to_id UNINDEXED,       -- Target entity
+        relation_type UNINDEXED, -- Type of relation
+        
+        -- Observation fields
+        entity_id UNINDEXED,   -- Parent entity
+        category UNINDEXED,    -- Observation category
+        
+        -- Common fields
+        metadata UNINDEXED,    -- JSON metadata
+        created_at UNINDEXED,  -- Creation timestamp
+        updated_at UNINDEXED,  -- Last update
+        
+        -- Configuration
+        tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+        prefix='1,2,3,4'                    -- Support longer prefixes for paths
+    );
+    """)
+
+    # Print instruction to manually reindex after migration
+    print("\n------------------------------------------------------------------")
+    print("IMPORTANT: After downgrade completes, manually run the reindex command:")
+    print("basic-memory sync")
+    print("------------------------------------------------------------------\n")

basic_memory/alembic/versions/e7e1f4367280_add_scan_watermark_tracking_to_project.py

@@ -0,0 +1,37 @@
+"""Add scan watermark tracking to Project
+
+Revision ID: e7e1f4367280
+Revises: 9d9c1cb7d8f5
+Create Date: 2025-10-20 16:42:46.625075
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "e7e1f4367280"
+down_revision: Union[str, None] = "9d9c1cb7d8f5"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("project", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("last_scan_timestamp", sa.Float(), nullable=True))
+        batch_op.add_column(sa.Column("last_file_count", sa.Integer(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("project", schema=None) as batch_op:
+        batch_op.drop_column("last_file_count")
+        batch_op.drop_column("last_scan_timestamp")
+
+    # ### end Alembic commands ###

basic_memory/api/app.py

@@ -1,45 +1,60 @@
 """FastAPI application for basic-memory knowledge graph API."""
 
+import asyncio
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, HTTPException
 from fastapi.exception_handlers import http_exception_handler
 from loguru import logger
 
+from basic_memory import __version__ as version
 from basic_memory import db
-from basic_memory.config import config as app_config
-from basic_memory.api.routers import knowledge, search, memory, resource
-from alembic import command
-from alembic.config import Config
+from basic_memory.api.routers import (
+    directory_router,
+    importer_router,
+    knowledge,
+    management,
+    memory,
+    project,
+    resource,
+    search,
+    prompt_router,
+)
+from basic_memory.config import ConfigManager
+from basic_memory.services.initialization import initialize_file_sync, initialize_app
 
-from basic_memory.db import DatabaseType
-from basic_memory.repository.search_repository import SearchRepository
 
+@asynccontextmanager
+async def lifespan(app: FastAPI):  # pragma: no cover
+    """Lifecycle manager for the FastAPI app. Not called in stdio mcp mode"""
 
-async def run_migrations():  # pragma: no cover
-    """Run any pending alembic migrations."""
-    logger.info("Running database migrations...")
-    try:
-        config = Config("alembic.ini")
-        command.upgrade(config, "head")
-        logger.info("Migrations completed successfully")
+    app_config = ConfigManager().config
+    logger.info("Starting Basic Memory API")
 
-        _, session_maker = await db.get_or_create_db(
-            app_config.database_path, DatabaseType.FILESYSTEM
-        )
-        await SearchRepository(session_maker).init_search_index()
-    except Exception as e:
-        logger.error(f"Error running migrations: {e}")
-        raise
+    await initialize_app(app_config)
 
+    # Cache database connections in app state for performance
+    logger.info("Initializing database and caching connections...")
+    engine, session_maker = await db.get_or_create_db(app_config.database_path)
+    app.state.engine = engine
+    app.state.session_maker = session_maker
+    logger.info("Database connections cached in app state")
 
-@asynccontextmanager
-async def lifespan(app: FastAPI):  # pragma: no cover
-    """Lifecycle manager for the FastAPI app."""
-    logger.info("Starting Basic Memory API")
-    await run_migrations()
+    logger.info(f"Sync changes enabled: {app_config.sync_changes}")
+    if app_config.sync_changes:
+        # start file sync task in background
+        app.state.sync_task = asyncio.create_task(initialize_file_sync(app_config))
+    else:
+        logger.info("Sync changes disabled. Skipping file sync service.")
+
+    # proceed with startup
     yield
+
     logger.info("Shutting down Basic Memory API")
+    if app.state.sync_task:
+        logger.info("Stopping sync...")
+        app.state.sync_task.cancel()  # pyright: ignore
+
     await db.shutdown_db()
 
 
@@ -47,20 +62,37 @@ async def lifespan(app: FastAPI):  # pragma: no cover
 app = FastAPI(
     title="Basic Memory API",
     description="Knowledge graph API for basic-memory",
-    version="0.1.0",
+    version=version,
     lifespan=lifespan,
 )
 
+
 # Include routers
-app.include_router(knowledge.router)
-app.include_router(search.router)
-app.include_router(memory.router)
-app.include_router(resource.router)
+app.include_router(knowledge.router, prefix="/{project}")
+app.include_router(memory.router, prefix="/{project}")
+app.include_router(resource.router, prefix="/{project}")
+app.include_router(search.router, prefix="/{project}")
+app.include_router(project.project_router, prefix="/{project}")
+app.include_router(directory_router.router, prefix="/{project}")
+app.include_router(prompt_router.router, prefix="/{project}")
+app.include_router(importer_router.router, prefix="/{project}")
+
+# Project resource router works accross projects
+app.include_router(project.project_resource_router)
+app.include_router(management.router)
+
+# Auth routes are handled by FastMCP automatically when auth is enabled
 
 
 @app.exception_handler(Exception)
 async def exception_handler(request, exc):  # pragma: no cover
     logger.exception(
-        f"An unhandled exception occurred for request '{request.url}', exception: {exc}"
+        "API unhandled exception",
+        url=str(request.url),
+        method=request.method,
+        client=request.client.host if request.client else None,
+        path=request.url.path,
+        error_type=type(exc).__name__,
+        error=str(exc),
     )
     return await http_exception_handler(request, HTTPException(status_code=500, detail=str(exc)))

basic_memory/api/routers/__init__.py

@@ -1,8 +1,11 @@
 """API routers."""
 
 from . import knowledge_router as knowledge
+from . import management_router as management
 from . import memory_router as memory
+from . import project_router as project
 from . import resource_router as resource
 from . import search_router as search
+from . import prompt_router as prompt
 
-__all__ = ["knowledge", "memory", "resource", "search"]
+__all__ = ["knowledge", "management", "memory", "project", "resource", "search", "prompt"]

basic_memory/api/routers/directory_router.py

@@ -0,0 +1,84 @@
+"""Router for directory tree operations."""
+
+from typing import List, Optional
+
+from fastapi import APIRouter, Query
+
+from basic_memory.deps import DirectoryServiceDep, ProjectIdDep
+from basic_memory.schemas.directory import DirectoryNode
+
+router = APIRouter(prefix="/directory", tags=["directory"])
+
+
+@router.get("/tree", response_model=DirectoryNode, response_model_exclude_none=True)
+async def get_directory_tree(
+    directory_service: DirectoryServiceDep,
+    project_id: ProjectIdDep,
+):
+    """Get hierarchical directory structure from the knowledge base.
+
+    Args:
+        directory_service: Service for directory operations
+        project_id: ID of the current project
+
+    Returns:
+        DirectoryNode representing the root of the hierarchical tree structure
+    """
+    # Get a hierarchical directory tree for the specific project
+    tree = await directory_service.get_directory_tree()
+
+    # Return the hierarchical tree
+    return tree
+
+
+@router.get("/structure", response_model=DirectoryNode, response_model_exclude_none=True)
+async def get_directory_structure(
+    directory_service: DirectoryServiceDep,
+    project_id: ProjectIdDep,
+):
+    """Get folder structure for navigation (no files).
+
+    Optimized endpoint for folder tree navigation. Returns only directory nodes
+    without file metadata. For full tree with files, use /directory/tree.
+
+    Args:
+        directory_service: Service for directory operations
+        project_id: ID of the current project
+
+    Returns:
+        DirectoryNode tree containing only folders (type="directory")
+    """
+    structure = await directory_service.get_directory_structure()
+    return structure
+
+
+@router.get("/list", response_model=List[DirectoryNode], response_model_exclude_none=True)
+async def list_directory(
+    directory_service: DirectoryServiceDep,
+    project_id: ProjectIdDep,
+    dir_name: str = Query("/", description="Directory path to list"),
+    depth: int = Query(1, ge=1, le=10, description="Recursion depth (1-10)"),
+    file_name_glob: Optional[str] = Query(
+        None, description="Glob pattern for filtering file names"
+    ),
+):
+    """List directory contents with filtering and depth control.
+
+    Args:
+        directory_service: Service for directory operations
+        project_id: ID of the current project
+        dir_name: Directory path to list (default: root "/")
+        depth: Recursion depth (1-10, default: 1 for immediate children only)
+        file_name_glob: Optional glob pattern for filtering file names (e.g., "*.md", "*meeting*")
+
+    Returns:
+        List of DirectoryNode objects matching the criteria
+    """
+    # Get directory listing with filtering
+    nodes = await directory_service.list_directory(
+        dir_name=dir_name,
+        depth=depth,
+        file_name_glob=file_name_glob,
+    )
+
+    return nodes

basic_memory/api/routers/importer_router.py

@@ -0,0 +1,152 @@
+"""Import router for Basic Memory API."""
+
+import json
+import logging
+
+from fastapi import APIRouter, Form, HTTPException, UploadFile, status
+
+from basic_memory.deps import (
+    ChatGPTImporterDep,
+    ClaudeConversationsImporterDep,
+    ClaudeProjectsImporterDep,
+    MemoryJsonImporterDep,
+)
+from basic_memory.importers import Importer
+from basic_memory.schemas.importer import (
+    ChatImportResult,
+    EntityImportResult,
+    ProjectImportResult,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/import", tags=["import"])
+
+
+@router.post("/chatgpt", response_model=ChatImportResult)
+async def import_chatgpt(
+    importer: ChatGPTImporterDep,
+    file: UploadFile,
+    folder: str = Form("conversations"),
+) -> ChatImportResult:
+    """Import conversations from ChatGPT JSON export.
+
+    Args:
+        file: The ChatGPT conversations.json file.
+        folder: The folder to place the files in.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        ChatImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    return await import_file(importer, file, folder)
+
+
+@router.post("/claude/conversations", response_model=ChatImportResult)
+async def import_claude_conversations(
+    importer: ClaudeConversationsImporterDep,
+    file: UploadFile,
+    folder: str = Form("conversations"),
+) -> ChatImportResult:
+    """Import conversations from Claude conversations.json export.
+
+    Args:
+        file: The Claude conversations.json file.
+        folder: The folder to place the files in.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        ChatImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    return await import_file(importer, file, folder)
+
+
+@router.post("/claude/projects", response_model=ProjectImportResult)
+async def import_claude_projects(
+    importer: ClaudeProjectsImporterDep,
+    file: UploadFile,
+    folder: str = Form("projects"),
+) -> ProjectImportResult:
+    """Import projects from Claude projects.json export.
+
+    Args:
+        file: The Claude projects.json file.
+        base_folder: The base folder to place the files in.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        ProjectImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    return await import_file(importer, file, folder)
+
+
+@router.post("/memory-json", response_model=EntityImportResult)
+async def import_memory_json(
+    importer: MemoryJsonImporterDep,
+    file: UploadFile,
+    folder: str = Form("conversations"),
+) -> EntityImportResult:
+    """Import entities and relations from a memory.json file.
+
+    Args:
+        file: The memory.json file.
+        destination_folder: Optional destination folder within the project.
+        markdown_processor: MarkdownProcessor instance.
+
+    Returns:
+        EntityImportResult with import statistics.
+
+    Raises:
+        HTTPException: If import fails.
+    """
+    try:
+        file_data = []
+        file_bytes = await file.read()
+        file_str = file_bytes.decode("utf-8")
+        for line in file_str.splitlines():
+            json_data = json.loads(line)
+            file_data.append(json_data)
+
+        result = await importer.import_data(file_data, folder)
+        if not result.success:  # pragma: no cover
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=result.error_message or "Import failed",
+            )
+    except Exception as e:
+        logger.exception("Import failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Import failed: {str(e)}",
+        )
+    return result
+
+
+async def import_file(importer: Importer, file: UploadFile, destination_folder: str):
+    try:
+        # Process file
+        json_data = json.load(file.file)
+        result = await importer.import_data(json_data, destination_folder)
+        if not result.success:  # pragma: no cover
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=result.error_message or "Import failed",
+            )
+
+        return result
+
+    except Exception as e:
+        logger.exception("Import failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Import failed: {str(e)}",
+        )