basic-memory 0.8.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

basic_memory/__init__.py

@@ -1,3 +1,3 @@
 """basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
 
-__version__ = "0.8.0"
+__version__ = "0.10.0"

basic_memory/alembic/migrations.py

@@ -1,6 +1,5 @@
 """Functions for managing database migrations."""
 
-import asyncio
 from pathlib import Path
 from loguru import logger
 from alembic.config import Config
@@ -10,20 +9,16 @@ from alembic import command
 def get_alembic_config() -> Config:  # pragma: no cover
     """Get alembic config with correct paths."""
     migrations_path = Path(__file__).parent
-    alembic_ini = migrations_path.parent.parent.parent / "alembic.ini"
+    alembic_ini = migrations_path / "alembic.ini"
 
     config = Config(alembic_ini)
     config.set_main_option("script_location", str(migrations_path))
     return config
 
 
-async def reset_database():  # pragma: no cover
+def reset_database():  # pragma: no cover
     """Drop and recreate all tables."""
     logger.info("Resetting database...")
     config = get_alembic_config()
-
-    def _reset(cfg):
-        command.downgrade(cfg, "base")
-        command.upgrade(cfg, "head")
-
-    await asyncio.get_event_loop().run_in_executor(None, _reset, config)
+    command.downgrade(config, "base")
+    command.upgrade(config, "head")

basic_memory/alembic/versions/cc7172b46608_update_search_index_schema.py

@@ -0,0 +1,106 @@
+"""Update search index schema
+
+Revision ID: cc7172b46608
+Revises: 502b60eaa905
+Create Date: 2025-02-28 18:48:23.244941
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "cc7172b46608"
+down_revision: Union[str, None] = "502b60eaa905"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade database schema to use new search index with content_stems and content_snippet."""
+
+    # First, drop the existing search_index table
+    op.execute("DROP TABLE IF EXISTS search_index")
+
+    # Create new search_index with updated schema
+    op.execute("""
+    CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+        -- Core entity fields
+        id UNINDEXED,          -- Row ID
+        title,                 -- Title for searching
+        content_stems,         -- Main searchable content split into stems
+        content_snippet,       -- File content snippet for display
+        permalink,             -- Stable identifier (now indexed for path search)
+        file_path UNINDEXED,   -- Physical location
+        type UNINDEXED,        -- entity/relation/observation
+        
+        -- Relation fields 
+        from_id UNINDEXED,     -- Source entity
+        to_id UNINDEXED,       -- Target entity
+        relation_type UNINDEXED, -- Type of relation
+        
+        -- Observation fields
+        entity_id UNINDEXED,   -- Parent entity
+        category UNINDEXED,    -- Observation category
+        
+        -- Common fields
+        metadata UNINDEXED,    -- JSON metadata
+        created_at UNINDEXED,  -- Creation timestamp
+        updated_at UNINDEXED,  -- Last update
+        
+        -- Configuration
+        tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+        prefix='1,2,3,4'                    -- Support longer prefixes for paths
+    );
+    """)
+
+    # Print instruction to manually reindex after migration
+    print("\n------------------------------------------------------------------")
+    print("IMPORTANT: After migration completes, manually run the reindex command:")
+    print("basic-memory sync")
+    print("------------------------------------------------------------------\n")
+
+
+def downgrade() -> None:
+    """Downgrade database schema to use old search index."""
+    # Drop the updated search_index table
+    op.execute("DROP TABLE IF EXISTS search_index")
+
+    # Recreate the original search_index schema
+    op.execute("""
+    CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
+        -- Core entity fields
+        id UNINDEXED,          -- Row ID
+        title,                 -- Title for searching
+        content,               -- Main searchable content
+        permalink,             -- Stable identifier (now indexed for path search)
+        file_path UNINDEXED,   -- Physical location
+        type UNINDEXED,        -- entity/relation/observation
+        
+        -- Relation fields 
+        from_id UNINDEXED,     -- Source entity
+        to_id UNINDEXED,       -- Target entity
+        relation_type UNINDEXED, -- Type of relation
+        
+        -- Observation fields
+        entity_id UNINDEXED,   -- Parent entity
+        category UNINDEXED,    -- Observation category
+        
+        -- Common fields
+        metadata UNINDEXED,    -- JSON metadata
+        created_at UNINDEXED,  -- Creation timestamp
+        updated_at UNINDEXED,  -- Last update
+        
+        -- Configuration
+        tokenize='unicode61 tokenchars 0x2F',  -- Hex code for /
+        prefix='1,2,3,4'                    -- Support longer prefixes for paths
+    );
+    """)
+
+    # Print instruction to manually reindex after migration
+    print("\n------------------------------------------------------------------")
+    print("IMPORTANT: After downgrade completes, manually run the reindex command:")
+    print("basic-memory sync")
+    print("------------------------------------------------------------------\n")

basic_memory/api/app.py

@@ -2,14 +2,13 @@
 
 from contextlib import asynccontextmanager
 
-import logfire
 from fastapi import FastAPI, HTTPException
 from fastapi.exception_handlers import http_exception_handler
 from loguru import logger
 
 from basic_memory import db
 from basic_memory.config import config as app_config
-from basic_memory.api.routers import knowledge, search, memory, resource
+from basic_memory.api.routers import knowledge, search, memory, resource, project_info
 
 
 @asynccontextmanager
@@ -29,20 +28,24 @@ app = FastAPI(
     lifespan=lifespan,
 )
 
-if app_config != "test":
-    logfire.instrument_fastapi(app)
-
 
 # Include routers
 app.include_router(knowledge.router)
 app.include_router(search.router)
 app.include_router(memory.router)
 app.include_router(resource.router)
+app.include_router(project_info.router)
 
 
 @app.exception_handler(Exception)
 async def exception_handler(request, exc):  # pragma: no cover
     logger.exception(
-        f"An unhandled exception occurred for request '{request.url}', exception: {exc}"
+        "API unhandled exception",
+        url=str(request.url),
+        method=request.method,
+        client=request.client.host if request.client else None,
+        path=request.url.path,
+        error_type=type(exc).__name__,
+        error=str(exc),
     )
     return await http_exception_handler(request, HTTPException(status_code=500, detail=str(exc)))

basic_memory/api/routers/__init__.py

@@ -4,5 +4,6 @@ from . import knowledge_router as knowledge
 from . import memory_router as memory
 from . import resource_router as resource
 from . import search_router as search
+from . import project_info_router as project_info
 
-__all__ = ["knowledge", "memory", "resource", "search"]
+__all__ = ["knowledge", "memory", "resource", "search", "project_info"]

basic_memory/api/routers/knowledge_router.py

@@ -33,7 +33,9 @@ async def create_entity(
     search_service: SearchServiceDep,
 ) -> EntityResponse:
     """Create an entity."""
-    logger.info(f"request: create_entity with data={data}")
+    logger.info(
+        "API request", endpoint="create_entity", entity_type=data.entity_type, title=data.title
+    )
 
     entity = await entity_service.create_entity(data)
 
@@ -41,7 +43,13 @@ async def create_entity(
     await search_service.index_entity(entity, background_tasks=background_tasks)
     result = EntityResponse.model_validate(entity)
 
-    logger.info(f"response: create_entity with result={result}")
+    logger.info(
+        "API response",
+        endpoint="create_entity",
+        title=result.title,
+        permalink=result.permalink,
+        status_code=201,
+    )
     return result
 
 
@@ -55,10 +63,23 @@ async def create_or_update_entity(
     search_service: SearchServiceDep,
 ) -> EntityResponse:
     """Create or update an entity. If entity exists, it will be updated, otherwise created."""
-    logger.info(f"request: create_or_update_entity with permalink={permalink}, data={data}")
+    logger.info(
+        "API request",
+        endpoint="create_or_update_entity",
+        permalink=permalink,
+        entity_type=data.entity_type,
+        title=data.title,
+    )
 
     # Validate permalink matches
     if data.permalink != permalink:
+        logger.warning(
+            "API validation error",
+            endpoint="create_or_update_entity",
+            permalink=permalink,
+            data_permalink=data.permalink,
+            error="Permalink mismatch",
+        )
         raise HTTPException(status_code=400, detail="Entity permalink must match URL path")
 
     # Try create_or_update operation
@@ -70,7 +91,12 @@ async def create_or_update_entity(
     result = EntityResponse.model_validate(entity)
 
     logger.info(
-        f"response: create_or_update_entity with result={result}, status_code={response.status_code}"
+        "API response",
+        endpoint="create_or_update_entity",
+        title=result.title,
+        permalink=result.permalink,
+        created=created,
+        status_code=response.status_code,
     )
     return result
 

basic_memory/api/routers/memory_router.py

@@ -32,6 +32,7 @@ async def to_graph_context(context, entity_repository: EntityRepository, page: i
                 return EntitySummary(
                     title=item.title,  # pyright: ignore
                     permalink=item.permalink,
+                    content=item.content,
                     file_path=item.file_path,
                     created_at=item.created_at,
                 )
@@ -52,8 +53,8 @@ async def to_graph_context(context, entity_repository: EntityRepository, page: i
                     file_path=item.file_path,
                     permalink=item.permalink,  # pyright: ignore
                     relation_type=item.type,
-                    from_id=from_entity.permalink,  # pyright: ignore
-                    to_id=to_entity.permalink if to_entity else None,
+                    from_entity=from_entity.permalink,  # pyright: ignore
+                    to_entity=to_entity.permalink if to_entity else None,
                     created_at=item.created_at,
                 )
             case _:  # pragma: no cover

basic_memory/api/routers/project_info_router.py

@@ -0,0 +1,274 @@
+"""Router for statistics and system information."""
+
+import json
+from datetime import datetime
+
+from basic_memory.config import config, config_manager
+from basic_memory.deps import (
+    ProjectInfoRepositoryDep,
+)
+from basic_memory.repository.project_info_repository import ProjectInfoRepository
+from basic_memory.schemas import (
+    ProjectInfoResponse,
+    ProjectStatistics,
+    ActivityMetrics,
+    SystemStatus,
+)
+from basic_memory.sync.watch_service import WATCH_STATUS_JSON
+from fastapi import APIRouter
+from sqlalchemy import text
+
+router = APIRouter(prefix="/stats", tags=["statistics"])
+
+
+@router.get("/project-info", response_model=ProjectInfoResponse)
+async def get_project_info(
+    repository: ProjectInfoRepositoryDep,
+) -> ProjectInfoResponse:
+    """Get comprehensive information about the current Basic Memory project."""
+    # Get statistics
+    statistics = await get_statistics(repository)
+
+    # Get activity metrics
+    activity = await get_activity_metrics(repository)
+
+    # Get system status
+    system = await get_system_status()
+
+    # Get project configuration information
+    project_name = config.project
+    project_path = str(config.home)
+    available_projects = config_manager.projects
+    default_project = config_manager.default_project
+
+    # Construct the response
+    return ProjectInfoResponse(
+        project_name=project_name,
+        project_path=project_path,
+        available_projects=available_projects,
+        default_project=default_project,
+        statistics=statistics,
+        activity=activity,
+        system=system,
+    )
+
+
+async def get_statistics(repository: ProjectInfoRepository) -> ProjectStatistics:
+    """Get statistics about the current project."""
+    # Get basic counts
+    entity_count_result = await repository.execute_query(text("SELECT COUNT(*) FROM entity"))
+    total_entities = entity_count_result.scalar() or 0
+
+    observation_count_result = await repository.execute_query(
+        text("SELECT COUNT(*) FROM observation")
+    )
+    total_observations = observation_count_result.scalar() or 0
+
+    relation_count_result = await repository.execute_query(text("SELECT COUNT(*) FROM relation"))
+    total_relations = relation_count_result.scalar() or 0
+
+    unresolved_count_result = await repository.execute_query(
+        text("SELECT COUNT(*) FROM relation WHERE to_id IS NULL")
+    )
+    total_unresolved = unresolved_count_result.scalar() or 0
+
+    # Get entity counts by type
+    entity_types_result = await repository.execute_query(
+        text("SELECT entity_type, COUNT(*) FROM entity GROUP BY entity_type")
+    )
+    entity_types = {row[0]: row[1] for row in entity_types_result.fetchall()}
+
+    # Get observation counts by category
+    category_result = await repository.execute_query(
+        text("SELECT category, COUNT(*) FROM observation GROUP BY category")
+    )
+    observation_categories = {row[0]: row[1] for row in category_result.fetchall()}
+
+    # Get relation counts by type
+    relation_types_result = await repository.execute_query(
+        text("SELECT relation_type, COUNT(*) FROM relation GROUP BY relation_type")
+    )
+    relation_types = {row[0]: row[1] for row in relation_types_result.fetchall()}
+
+    # Find most connected entities (most outgoing relations)
+    connected_result = await repository.execute_query(
+        text("""
+        SELECT e.id, e.title, e.permalink, COUNT(r.id) AS relation_count
+        FROM entity e
+        JOIN relation r ON e.id = r.from_id
+        GROUP BY e.id
+        ORDER BY relation_count DESC
+        LIMIT 10
+    """)
+    )
+    most_connected = [
+        {"id": row[0], "title": row[1], "permalink": row[2], "relation_count": row[3]}
+        for row in connected_result.fetchall()
+    ]
+
+    # Count isolated entities (no relations)
+    isolated_result = await repository.execute_query(
+        text("""
+        SELECT COUNT(e.id)
+        FROM entity e
+        LEFT JOIN relation r1 ON e.id = r1.from_id
+        LEFT JOIN relation r2 ON e.id = r2.to_id
+        WHERE r1.id IS NULL AND r2.id IS NULL
+    """)
+    )
+    isolated_count = isolated_result.scalar() or 0
+
+    return ProjectStatistics(
+        total_entities=total_entities,
+        total_observations=total_observations,
+        total_relations=total_relations,
+        total_unresolved_relations=total_unresolved,
+        entity_types=entity_types,
+        observation_categories=observation_categories,
+        relation_types=relation_types,
+        most_connected_entities=most_connected,
+        isolated_entities=isolated_count,
+    )
+
+
+async def get_activity_metrics(repository: ProjectInfoRepository) -> ActivityMetrics:
+    """Get activity metrics for the current project."""
+    # Get recently created entities
+    created_result = await repository.execute_query(
+        text("""
+        SELECT id, title, permalink, entity_type, created_at 
+        FROM entity
+        ORDER BY created_at DESC
+        LIMIT 10
+    """)
+    )
+    recently_created = [
+        {
+            "id": row[0],
+            "title": row[1],
+            "permalink": row[2],
+            "entity_type": row[3],
+            "created_at": row[4],
+        }
+        for row in created_result.fetchall()
+    ]
+
+    # Get recently updated entities
+    updated_result = await repository.execute_query(
+        text("""
+        SELECT id, title, permalink, entity_type, updated_at 
+        FROM entity
+        ORDER BY updated_at DESC
+        LIMIT 10
+    """)
+    )
+    recently_updated = [
+        {
+            "id": row[0],
+            "title": row[1],
+            "permalink": row[2],
+            "entity_type": row[3],
+            "updated_at": row[4],
+        }
+        for row in updated_result.fetchall()
+    ]
+
+    # Get monthly growth over the last 6 months
+    # Calculate the start of 6 months ago
+    now = datetime.now()
+    six_months_ago = datetime(
+        now.year - (1 if now.month <= 6 else 0), ((now.month - 6) % 12) or 12, 1
+    )
+
+    # Query for monthly entity creation
+    entity_growth_result = await repository.execute_query(
+        text(f"""
+        SELECT 
+            strftime('%Y-%m', created_at) AS month,
+            COUNT(*) AS count
+        FROM entity
+        WHERE created_at >= '{six_months_ago.isoformat()}'
+        GROUP BY month
+        ORDER BY month
+    """)
+    )
+    entity_growth = {row[0]: row[1] for row in entity_growth_result.fetchall()}
+
+    # Query for monthly observation creation
+    observation_growth_result = await repository.execute_query(
+        text(f"""
+        SELECT 
+            strftime('%Y-%m', created_at) AS month,
+            COUNT(*) AS count
+        FROM observation
+        INNER JOIN entity ON observation.entity_id = entity.id
+        WHERE entity.created_at >= '{six_months_ago.isoformat()}'
+        GROUP BY month
+        ORDER BY month
+    """)
+    )
+    observation_growth = {row[0]: row[1] for row in observation_growth_result.fetchall()}
+
+    # Query for monthly relation creation
+    relation_growth_result = await repository.execute_query(
+        text(f"""
+        SELECT 
+            strftime('%Y-%m', created_at) AS month,
+            COUNT(*) AS count
+        FROM relation
+        INNER JOIN entity ON relation.from_id = entity.id
+        WHERE entity.created_at >= '{six_months_ago.isoformat()}'
+        GROUP BY month
+        ORDER BY month
+    """)
+    )
+    relation_growth = {row[0]: row[1] for row in relation_growth_result.fetchall()}
+
+    # Combine all monthly growth data
+    monthly_growth = {}
+    for month in set(
+        list(entity_growth.keys()) + list(observation_growth.keys()) + list(relation_growth.keys())
+    ):
+        monthly_growth[month] = {
+            "entities": entity_growth.get(month, 0),
+            "observations": observation_growth.get(month, 0),
+            "relations": relation_growth.get(month, 0),
+            "total": (
+                entity_growth.get(month, 0)
+                + observation_growth.get(month, 0)
+                + relation_growth.get(month, 0)
+            ),
+        }
+
+    return ActivityMetrics(
+        recently_created=recently_created,
+        recently_updated=recently_updated,
+        monthly_growth=monthly_growth,
+    )
+
+
+async def get_system_status() -> SystemStatus:
+    """Get system status information."""
+    import basic_memory
+
+    # Get database information
+    db_path = config.database_path
+    db_size = db_path.stat().st_size if db_path.exists() else 0
+    db_size_readable = f"{db_size / (1024 * 1024):.2f} MB"
+
+    # Get watch service status if available
+    watch_status = None
+    watch_status_path = config.home / ".basic-memory" / WATCH_STATUS_JSON
+    if watch_status_path.exists():
+        try:
+            watch_status = json.loads(watch_status_path.read_text(encoding="utf-8"))
+        except Exception:  # pragma: no cover
+            pass
+
+    return SystemStatus(
+        version=basic_memory.__version__,
+        database_path=str(db_path),
+        database_size=db_size_readable,
+        watch_status=watch_status,
+        timestamp=datetime.now(),
+    )

basic_memory/api/routers/search_router.py

@@ -1,11 +1,9 @@
 """Router for search operations."""
 
-from dataclasses import asdict
-
 from fastapi import APIRouter, BackgroundTasks
 
 from basic_memory.schemas.search import SearchQuery, SearchResult, SearchResponse
-from basic_memory.deps import SearchServiceDep
+from basic_memory.deps import SearchServiceDep, EntityServiceDep
 
 router = APIRouter(prefix="/search", tags=["search"])
 
@@ -14,6 +12,7 @@ router = APIRouter(prefix="/search", tags=["search"])
 async def search(
     query: SearchQuery,
     search_service: SearchServiceDep,
+    entity_service: EntityServiceDep,
     page: int = 1,
     page_size: int = 10,
 ):
@@ -21,7 +20,26 @@ async def search(
     limit = page_size
     offset = (page - 1) * page_size
     results = await search_service.search(query, limit=limit, offset=offset)
-    search_results = [SearchResult.model_validate(asdict(r)) for r in results]
+
+    search_results = []
+    for r in results:
+        entities = await entity_service.get_entities_by_id([r.entity_id, r.from_id, r.to_id])  # pyright: ignore
+        search_results.append(
+            SearchResult(
+                title=r.title,  # pyright: ignore
+                type=r.type,  # pyright: ignore
+                permalink=r.permalink,
+                score=r.score,  # pyright: ignore
+                entity=entities[0].permalink if entities else None,
+                content=r.content,
+                file_path=r.file_path,
+                metadata=r.metadata,
+                category=r.category,
+                from_entity=entities[0].permalink if entities else None,
+                to_entity=entities[1].permalink if len(entities) > 1 else None,
+                relation_type=r.relation_type,
+            )
+        )
     return SearchResponse(
         results=search_results,
         current_page=page,

basic_memory/cli/app.py

@@ -1,4 +1,5 @@
 import asyncio
+from typing import Optional
 
 import typer
 
@@ -6,13 +7,63 @@ from basic_memory import db
 from basic_memory.config import config
 
 
-asyncio.run(db.run_migrations(config))
+def version_callback(value: bool) -> None:
+    """Show version and exit."""
+    if value:  # pragma: no cover
+        import basic_memory
+
+        typer.echo(f"Basic Memory version: {basic_memory.__version__}")
+        raise typer.Exit()
+
 
 app = typer.Typer(name="basic-memory")
 
-import_app = typer.Typer()
-app.add_typer(import_app, name="import")
 
+@app.callback()
+def app_callback(
+    project: Optional[str] = typer.Option(
+        None,
+        "--project",
+        "-p",
+        help="Specify which project to use",
+        envvar="BASIC_MEMORY_PROJECT",
+    ),
+    version: Optional[bool] = typer.Option(
+        None,
+        "--version",
+        "-v",
+        help="Show version and exit.",
+        callback=version_callback,
+        is_eager=True,
+    ),
+) -> None:
+    """Basic Memory - Local-first personal knowledge management."""
+    # We use the project option to set the BASIC_MEMORY_PROJECT environment variable
+    # The config module will pick this up when loading
+    if project:  # pragma: no cover
+        import os
+        import importlib
+        from basic_memory import config as config_module
+
+        # Set the environment variable
+        os.environ["BASIC_MEMORY_PROJECT"] = project
+
+        # Reload the config module to pick up the new project
+        importlib.reload(config_module)
+
+        # Update the local reference
+        global config
+        from basic_memory.config import config as new_config
+
+        config = new_config
+
+
+# Run database migrations
+asyncio.run(db.run_migrations(config))
+
+# Register sub-command groups
+import_app = typer.Typer(help="Import data from various sources")
+app.add_typer(import_app, name="import")
 
 claude_app = typer.Typer()
 import_app.add_typer(claude_app, name="claude")