basic-memory 0.0.0__py3-none-any.whl

basic_memory/cli/commands/sync.py

@@ -0,0 +1,254 @@
+"""Command module for basic-memory sync operations."""
+
+import asyncio
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Dict
+
+import typer
+from loguru import logger
+from rich.console import Console
+from rich.padding import Padding
+from rich.panel import Panel
+from rich.text import Text
+from rich.tree import Tree
+
+from basic_memory import db
+from basic_memory.cli.app import app
+from basic_memory.config import config
+from basic_memory.db import DatabaseType
+from basic_memory.markdown import EntityParser
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
+from basic_memory.repository import (
+    EntityRepository,
+    ObservationRepository,
+    RelationRepository,
+)
+from basic_memory.repository.search_repository import SearchRepository
+from basic_memory.services import EntityService, FileService
+from basic_memory.services.link_resolver import LinkResolver
+from basic_memory.services.search_service import SearchService
+from basic_memory.sync import SyncService, FileChangeScanner
+from basic_memory.sync.utils import SyncReport
+from basic_memory.sync.watch_service import WatchService
+
+console = Console()
+
+
+@dataclass
+class ValidationIssue:
+    file_path: str
+    error: str
+
+
+async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
+    """Get sync service instance with all dependencies."""
+    async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
+        engine,
+        session_maker,
+    ):
+        entity_parser = EntityParser(config.home)
+        markdown_processor = MarkdownProcessor(entity_parser)
+        file_service = FileService(config.home, markdown_processor)
+
+        # Initialize repositories
+        entity_repository = EntityRepository(session_maker)
+        observation_repository = ObservationRepository(session_maker)
+        relation_repository = RelationRepository(session_maker)
+        search_repository = SearchRepository(session_maker)
+
+        # Initialize services
+        search_service = SearchService(search_repository, entity_repository, file_service)
+        link_resolver = LinkResolver(entity_repository, search_service)
+
+        # Initialize scanner
+        file_change_scanner = FileChangeScanner(entity_repository)
+
+        # Initialize services
+        entity_service = EntityService(
+            entity_parser,
+            entity_repository,
+            observation_repository,
+            relation_repository,
+            file_service,
+            link_resolver,
+        )
+
+        # Create sync service
+        sync_service = SyncService(
+            scanner=file_change_scanner,
+            entity_service=entity_service,
+            entity_parser=entity_parser,
+            entity_repository=entity_repository,
+            relation_repository=relation_repository,
+            search_service=search_service,
+        )
+
+        return sync_service
+
+
+def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]:
+    """Group validation issues by directory."""
+    grouped = defaultdict(list)
+    for issue in issues:
+        dir_name = Path(issue.file_path).parent.name
+        grouped[dir_name].append(issue)
+    return dict(grouped)
+
+
+def display_validation_errors(issues: List[ValidationIssue]):
+    """Display validation errors in a rich, organized format."""
+    # Create header
+    console.print()
+    console.print(
+        Panel("[red bold]Error:[/red bold] Invalid frontmatter in knowledge files", expand=False)
+    )
+    console.print()
+
+    # Group issues by directory
+    grouped_issues = group_issues_by_directory(issues)
+
+    # Create tree structure
+    tree = Tree("Knowledge Files")
+    for dir_name, dir_issues in sorted(grouped_issues.items()):
+        # Create branch for directory
+        branch = tree.add(
+            f"[bold blue]{dir_name}/[/bold blue] ([yellow]{len(dir_issues)} files[/yellow])"
+        )
+
+        # Add each file issue
+        for issue in sorted(dir_issues, key=lambda x: x.file_path):
+            file_name = Path(issue.file_path).name
+            branch.add(
+                Text.assemble(("└─ ", "dim"), (file_name, "yellow"), ": ", (issue.error, "red"))
+            )
+
+    # Display tree
+    console.print(Padding(tree, (1, 2)))
+
+    # Add help text
+    console.print()
+    console.print(
+        Panel(
+            Text.assemble(
+                ("To fix:", "bold"),
+                "\n1. Add required frontmatter fields to each file",
+                "\n2. Run ",
+                ("basic-memory sync", "bold cyan"),
+                " again",
+            ),
+            expand=False,
+        )
+    )
+    console.print()
+
+
+def display_sync_summary(knowledge: SyncReport):
+    """Display a one-line summary of sync changes."""
+    total_changes = knowledge.total_changes
+    if total_changes == 0:
+        console.print("[green]Everything up to date[/green]")
+        return
+
+    # Format as: "Synced X files (A new, B modified, C moved, D deleted)"
+    changes = []
+    new_count = len(knowledge.new)
+    mod_count = len(knowledge.modified)
+    move_count = len(knowledge.moves)
+    del_count = len(knowledge.deleted)
+
+    if new_count:
+        changes.append(f"[green]{new_count} new[/green]")
+    if mod_count:
+        changes.append(f"[yellow]{mod_count} modified[/yellow]")
+    if move_count:
+        changes.append(f"[blue]{move_count} moved[/blue]")
+    if del_count:
+        changes.append(f"[red]{del_count} deleted[/red]")
+
+    console.print(f"Synced {total_changes} files ({', '.join(changes)})")
+
+
+def display_detailed_sync_results(knowledge: SyncReport):
+    """Display detailed sync results with trees."""
+    if knowledge.total_changes == 0:
+        console.print("\n[green]Everything up to date[/green]")
+        return
+
+    console.print("\n[bold]Sync Results[/bold]")
+
+    if knowledge.total_changes > 0:
+        knowledge_tree = Tree("[bold]Knowledge Files[/bold]")
+        if knowledge.new:
+            created = knowledge_tree.add("[green]Created[/green]")
+            for path in sorted(knowledge.new):
+                checksum = knowledge.checksums.get(path, "")
+                created.add(f"[green]{path}[/green] ({checksum[:8]})")
+        if knowledge.modified:
+            modified = knowledge_tree.add("[yellow]Modified[/yellow]")
+            for path in sorted(knowledge.modified):
+                checksum = knowledge.checksums.get(path, "")
+                modified.add(f"[yellow]{path}[/yellow] ({checksum[:8]})")
+        if knowledge.moves:
+            moved = knowledge_tree.add("[blue]Moved[/blue]")
+            for old_path, new_path in sorted(knowledge.moves.items()):
+                checksum = knowledge.checksums.get(new_path, "")
+                moved.add(f"[blue]{old_path}[/blue] → [blue]{new_path}[/blue] ({checksum[:8]})")
+        if knowledge.deleted:
+            deleted = knowledge_tree.add("[red]Deleted[/red]")
+            for path in sorted(knowledge.deleted):
+                deleted.add(f"[red]{path}[/red]")
+        console.print(knowledge_tree)
+
+
+async def run_sync(verbose: bool = False, watch: bool = False):
+    """Run sync operation."""
+
+    sync_service = await get_sync_service()
+
+    # Start watching if requested
+    if watch:
+        watch_service = WatchService(
+            sync_service=sync_service,
+            file_service=sync_service.entity_service.file_service,
+            config=config
+        )
+        await watch_service.handle_changes(config.home)
+        await watch_service.run()
+    else:
+        # one time sync
+        knowledge_changes = await sync_service.sync(config.home)
+        # Display results
+        if verbose:
+            display_detailed_sync_results(knowledge_changes)
+        else:
+            display_sync_summary(knowledge_changes)
+
+
+@app.command()
+def sync(
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Show detailed sync information.",
+    ),
+    watch: bool = typer.Option(
+        False,
+        "--watch",
+        "-w",
+        help="Start watching for changes after sync.",
+    ),
+) -> None:
+    """Sync knowledge files with the database."""
+    try:
+        # Run sync
+        asyncio.run(run_sync(verbose=verbose, watch=watch))
+
+    except Exception as e:
+        if not isinstance(e, typer.Exit):
+            logger.exception("Sync failed")
+            typer.echo(f"Error during sync: {e}", err=True)
+            raise typer.Exit(1)
+        raise

basic_memory/cli/main.py

@@ -0,0 +1,48 @@
+"""Main CLI entry point for basic-memory."""
+import sys
+
+import typer
+from loguru import logger
+
+from basic_memory.cli.app import app
+from basic_memory.cli.commands.init import init
+
+# Register commands
+from basic_memory.cli.commands import init, status, sync
+__all__ = ["init", "status", "sync"]
+
+from basic_memory.config import config
+
+
+def setup_logging(home_dir: str = config.home, log_file: str = "./basic-memory/basic-memory-tools.log"):
+    """Configure logging for the application."""
+
+    # Remove default handler and any existing handlers
+    logger.remove()
+
+    # Add file handler for debug level logs
+    log = f"{home_dir}/{log_file}"
+    logger.add(
+        log,
+        level="DEBUG",
+        rotation="100 MB",
+        retention="10 days",
+        backtrace=True,
+        diagnose=True,
+        enqueue=True,
+        colorize=False,
+    )
+
+    # Add stderr handler for warnings and errors only
+    logger.add(
+        sys.stderr,
+        level="WARNING",
+        backtrace=True,
+        diagnose=True
+    )
+
+# Set up logging when module is imported
+setup_logging()
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

basic_memory/config.py

@@ -0,0 +1,53 @@
+"""Configuration management for basic-memory."""
+
+from pathlib import Path
+
+from loguru import logger
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+DATABASE_NAME = "memory.db"
+DATA_DIR_NAME = ".basic-memory"
+
+
+class ProjectConfig(BaseSettings):
+    """Configuration for a specific basic-memory project."""
+
+    # Default to ~/basic-memory but allow override with env var: BASIC_MEMORY_HOME
+    home: Path = Field(
+        default_factory=lambda: Path.home() / "basic-memory",
+        description="Base path for basic-memory files",
+    )
+
+    # Name of the project
+    project: str = Field(default="default", description="Project name")
+
+    # Watch service configuration
+    sync_delay: int = Field(
+        default=500, description="Milliseconds to wait after changes before syncing", gt=0
+    )
+
+    model_config = SettingsConfigDict(
+        env_prefix="BASIC_MEMORY_",
+        extra="ignore",
+        env_file=".env",
+        env_file_encoding="utf-8",
+    )
+
+    @property
+    def database_path(self) -> Path:
+        """Get SQLite database path."""
+        return self.home / DATA_DIR_NAME / DATABASE_NAME
+
+    @field_validator("home")
+    @classmethod
+    def ensure_path_exists(cls, v: Path) -> Path:
+        """Ensure project path exists."""
+        if not v.exists():
+            v.mkdir(parents=True)
+        return v
+
+
+# Load project config
+config = ProjectConfig()
+logger.info(f"project config home: {config.home}")

basic_memory/db.py

@@ -0,0 +1,135 @@
+import asyncio
+from contextlib import asynccontextmanager
+from enum import Enum, auto
+from pathlib import Path
+from typing import AsyncGenerator, Optional
+
+from loguru import logger
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import (
+    create_async_engine,
+    async_sessionmaker,
+    AsyncSession,
+    AsyncEngine,
+    async_scoped_session,
+)
+
+from basic_memory.models import Base
+
+
+# Module level state
+_engine: Optional[AsyncEngine] = None
+_session_maker: Optional[async_sessionmaker[AsyncSession]] = None
+
+
+class DatabaseType(Enum):
+    """Types of supported databases."""
+
+    MEMORY = auto()
+    FILESYSTEM = auto()
+
+    @classmethod
+    def get_db_url(cls, db_path: Path, db_type: "DatabaseType") -> str:
+        """Get SQLAlchemy URL for database path."""
+        if db_type == cls.MEMORY:
+            logger.info("Using in-memory SQLite database")
+            return "sqlite+aiosqlite://"
+
+        return f"sqlite+aiosqlite:///{db_path}"
+
+
+def get_scoped_session_factory(
+    session_maker: async_sessionmaker[AsyncSession],
+) -> async_scoped_session:
+    """Create a scoped session factory scoped to current task."""
+    return async_scoped_session(session_maker, scopefunc=asyncio.current_task)
+
+
+@asynccontextmanager
+async def scoped_session(
+    session_maker: async_sessionmaker[AsyncSession],
+) -> AsyncGenerator[AsyncSession, None]:
+    """
+    Get a scoped session with proper lifecycle management.
+
+    Args:
+        session_maker: Session maker to create scoped sessions from
+    """
+    factory = get_scoped_session_factory(session_maker)
+    session = factory()
+    try:
+        await session.execute(text("PRAGMA foreign_keys=ON"))
+        yield session
+        await session.commit()
+    except Exception:
+        await session.rollback()
+        raise
+    finally:
+        await session.close()
+        await factory.remove()
+
+
+async def init_db(session: AsyncSession):
+    """Initialize database with required tables."""
+    await session.execute(text("PRAGMA foreign_keys=ON"))
+    conn = await session.connection()
+    await conn.run_sync(Base.metadata.create_all)
+    await session.commit()
+
+
+async def get_or_create_db(
+    db_path: Path,
+    db_type: DatabaseType = DatabaseType.FILESYSTEM,
+) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
+    """Get or create database engine and session maker."""
+    global _engine, _session_maker
+    
+    if _engine is None:
+        db_url = DatabaseType.get_db_url(db_path, db_type)
+        logger.debug(f"Creating engine for db_url: {db_url}")
+        _engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+        _session_maker = async_sessionmaker(_engine, expire_on_commit=False)
+
+        # Initialize database
+        logger.debug("Initializing database...")
+        async with scoped_session(_session_maker) as db_session:
+            await init_db(db_session)
+
+    return _engine, _session_maker
+
+
+async def shutdown_db():
+    """Clean up database connections."""
+    global _engine, _session_maker
+    
+    if _engine:
+        await _engine.dispose()
+        _engine = None
+        _session_maker = None
+
+
+@asynccontextmanager
+async def engine_session_factory(
+    db_path: Path,
+    db_type: DatabaseType = DatabaseType.FILESYSTEM,
+    init: bool = True,
+) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
+    """Create engine and session factory.
+    
+    Note: This is primarily used for testing where we want a fresh database
+    for each test. For production use, use get_or_create_db() instead.
+    """
+    db_url = DatabaseType.get_db_url(db_path, db_type)
+    logger.debug(f"Creating engine for db_url: {db_url}")
+    engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
+    try:
+        factory = async_sessionmaker(engine, expire_on_commit=False)
+
+        if init:
+            logger.debug("Initializing database...")
+            async with scoped_session(factory) as db_session:
+                await init_db(db_session)
+
+        yield engine, factory
+    finally:
+        await engine.dispose()

basic_memory/deps.py

@@ -0,0 +1,182 @@
+"""Dependency injection functions for basic-memory services."""
+
+from typing import Annotated
+
+from fastapi import Depends
+from sqlalchemy.ext.asyncio import (
+    AsyncSession,
+    AsyncEngine,
+    async_sessionmaker,
+)
+
+from basic_memory import db
+from basic_memory.config import ProjectConfig, config
+from basic_memory.markdown import EntityParser
+from basic_memory.markdown.markdown_processor import MarkdownProcessor
+from basic_memory.repository.entity_repository import EntityRepository
+from basic_memory.repository.observation_repository import ObservationRepository
+from basic_memory.repository.relation_repository import RelationRepository
+from basic_memory.repository.search_repository import SearchRepository
+from basic_memory.services import (
+    EntityService,
+)
+from basic_memory.services.context_service import ContextService
+from basic_memory.services.file_service import FileService
+from basic_memory.services.link_resolver import LinkResolver
+from basic_memory.services.search_service import SearchService
+
+
+## project
+
+
+def get_project_config() -> ProjectConfig:
+    return config
+
+
+ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]
+
+
+## sqlalchemy
+
+
+async def get_engine_factory(
+    project_config: ProjectConfigDep,
+) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
+    """Get engine and session maker."""
+    return await db.get_or_create_db(project_config.database_path)
+
+
+EngineFactoryDep = Annotated[
+    tuple[AsyncEngine, async_sessionmaker[AsyncSession]], Depends(get_engine_factory)
+]
+
+
+async def get_session_maker(engine_factory: EngineFactoryDep) -> async_sessionmaker[AsyncSession]:
+    """Get session maker."""
+    _, session_maker = engine_factory
+    return session_maker
+
+
+SessionMakerDep = Annotated[async_sessionmaker, Depends(get_session_maker)]
+
+
+## repositories
+
+
+async def get_entity_repository(
+    session_maker: SessionMakerDep,
+) -> EntityRepository:
+    """Create an EntityRepository instance."""
+    return EntityRepository(session_maker)
+
+
+EntityRepositoryDep = Annotated[EntityRepository, Depends(get_entity_repository)]
+
+
+async def get_observation_repository(
+    session_maker: SessionMakerDep,
+) -> ObservationRepository:
+    """Create an ObservationRepository instance."""
+    return ObservationRepository(session_maker)
+
+
+ObservationRepositoryDep = Annotated[ObservationRepository, Depends(get_observation_repository)]
+
+
+async def get_relation_repository(
+    session_maker: SessionMakerDep,
+) -> RelationRepository:
+    """Create a RelationRepository instance."""
+    return RelationRepository(session_maker)
+
+
+RelationRepositoryDep = Annotated[RelationRepository, Depends(get_relation_repository)]
+
+
+async def get_search_repository(
+    session_maker: SessionMakerDep,
+) -> SearchRepository:
+    """Create a SearchRepository instance."""
+    return SearchRepository(session_maker)
+
+
+SearchRepositoryDep = Annotated[SearchRepository, Depends(get_search_repository)]
+
+
+## services
+
+
+async def get_entity_parser(project_config: ProjectConfigDep) -> EntityParser:
+    return EntityParser(project_config.home)
+
+
+EntityParserDep = Annotated["EntityParser", Depends(get_entity_parser)]
+
+
+async def get_markdown_processor(entity_parser: EntityParserDep) -> MarkdownProcessor:
+    return MarkdownProcessor(entity_parser)
+
+
+MarkdownProcessorDep = Annotated[MarkdownProcessor, Depends(get_markdown_processor)]
+
+
+async def get_file_service(
+    project_config: ProjectConfigDep, markdown_processor: MarkdownProcessorDep
+) -> FileService:
+    return FileService(project_config.home, markdown_processor)
+
+
+FileServiceDep = Annotated[FileService, Depends(get_file_service)]
+
+
+
+async def get_entity_service(
+    entity_repository: EntityRepositoryDep,
+    observation_repository: ObservationRepositoryDep,
+    relation_repository: RelationRepositoryDep,
+    entity_parser: EntityParserDep,
+    file_service: FileServiceDep,
+    link_resolver: "LinkResolverDep",
+) -> EntityService:
+    """Create EntityService with repository."""
+    return EntityService(
+        entity_repository=entity_repository,
+        observation_repository=observation_repository,
+        relation_repository=relation_repository,
+        entity_parser=entity_parser,
+        file_service=file_service,
+        link_resolver=link_resolver,
+    )
+
+
+EntityServiceDep = Annotated[EntityService, Depends(get_entity_service)]
+
+
+async def get_search_service(
+    search_repository: SearchRepositoryDep,
+    entity_repository: EntityRepositoryDep,
+    file_service: FileServiceDep,
+) -> SearchService:
+    """Create SearchService with dependencies."""
+    return SearchService(search_repository, entity_repository, file_service)
+
+
+SearchServiceDep = Annotated[SearchService, Depends(get_search_service)]
+
+
+async def get_link_resolver(
+    entity_repository: EntityRepositoryDep, search_service: SearchServiceDep
+) -> LinkResolver:
+    return LinkResolver(entity_repository=entity_repository, search_service=search_service)
+
+
+LinkResolverDep = Annotated[LinkResolver, Depends(get_link_resolver)]
+
+
+async def get_context_service(
+    search_repository: SearchRepositoryDep, entity_repository: EntityRepositoryDep
+) -> ContextService:
+    return ContextService(search_repository, entity_repository)
+
+
+ContextServiceDep = Annotated[ContextService, Depends(get_context_service)]