basic-memory 0.2.12__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/cli/commands/cloud/upload.py

@@ -0,0 +1,233 @@
+"""WebDAV upload functionality for basic-memory projects."""
+
+import os
+from pathlib import Path
+
+import aiofiles
+import httpx
+
+from basic_memory.ignore_utils import load_gitignore_patterns, should_ignore_path
+from basic_memory.mcp.async_client import get_client
+from basic_memory.mcp.tools.utils import call_put
+
+# Archive file extensions that should be skipped during upload
+ARCHIVE_EXTENSIONS = {".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar", ".tgz", ".tbz2"}
+
+
+async def upload_path(
+    local_path: Path,
+    project_name: str,
+    verbose: bool = False,
+    use_gitignore: bool = True,
+    dry_run: bool = False,
+) -> bool:
+    """
+    Upload a file or directory to cloud project via WebDAV.
+
+    Args:
+        local_path: Path to local file or directory
+        project_name: Name of cloud project (destination)
+        verbose: Show detailed information about filtering and upload
+        use_gitignore: If False, skip .gitignore patterns (still use .bmignore)
+        dry_run: If True, show what would be uploaded without uploading
+
+    Returns:
+        True if upload succeeded, False otherwise
+    """
+    try:
+        # Resolve path
+        local_path = local_path.resolve()
+
+        # Check if path exists
+        if not local_path.exists():
+            print(f"Error: Path does not exist: {local_path}")
+            return False
+
+        # Get files to upload
+        if local_path.is_file():
+            files_to_upload = [(local_path, local_path.name)]
+            if verbose:
+                print(f"Uploading single file: {local_path.name}")
+        else:
+            files_to_upload = _get_files_to_upload(local_path, verbose, use_gitignore)
+
+        if not files_to_upload:
+            print("No files found to upload")
+            if verbose:
+                print(
+                    "\nTip: Use --verbose to see which files are being filtered, "
+                    "or --no-gitignore to skip .gitignore patterns"
+                )
+            return True
+
+        print(f"Found {len(files_to_upload)} file(s) to upload")
+
+        # Calculate total size
+        total_bytes = sum(file_path.stat().st_size for file_path, _ in files_to_upload)
+        skipped_count = 0
+
+        # If dry run, just show what would be uploaded
+        if dry_run:
+            print("\nFiles that would be uploaded:")
+            for file_path, relative_path in files_to_upload:
+                # Skip archive files
+                if _is_archive_file(file_path):
+                    print(f"  [SKIP] {relative_path} (archive file)")
+                    skipped_count += 1
+                    continue
+
+                size = file_path.stat().st_size
+                if size < 1024:
+                    size_str = f"{size} bytes"
+                elif size < 1024 * 1024:
+                    size_str = f"{size / 1024:.1f} KB"
+                else:
+                    size_str = f"{size / (1024 * 1024):.1f} MB"
+                print(f"  {relative_path} ({size_str})")
+        else:
+            # Upload files using httpx
+            async with get_client() as client:
+                for i, (file_path, relative_path) in enumerate(files_to_upload, 1):
+                    # Skip archive files (zip, tar, gz, etc.)
+                    if _is_archive_file(file_path):
+                        print(
+                            f"Skipping archive file: {relative_path} ({i}/{len(files_to_upload)})"
+                        )
+                        skipped_count += 1
+                        continue
+
+                    # Build remote path: /webdav/{project_name}/{relative_path}
+                    remote_path = f"/webdav/{project_name}/{relative_path}"
+                    print(f"Uploading {relative_path} ({i}/{len(files_to_upload)})")
+
+                    # Get file modification time
+                    file_stat = file_path.stat()
+                    mtime = int(file_stat.st_mtime)
+
+                    # Read file content asynchronously
+                    async with aiofiles.open(file_path, "rb") as f:
+                        content = await f.read()
+
+                    # Upload via HTTP PUT to WebDAV endpoint with mtime header
+                    # Using X-OC-Mtime (ownCloud/Nextcloud standard)
+                    response = await call_put(
+                        client, remote_path, content=content, headers={"X-OC-Mtime": str(mtime)}
+                    )
+                    response.raise_for_status()
+
+        # Format total size based on magnitude
+        if total_bytes < 1024:
+            size_str = f"{total_bytes} bytes"
+        elif total_bytes < 1024 * 1024:
+            size_str = f"{total_bytes / 1024:.1f} KB"
+        else:
+            size_str = f"{total_bytes / (1024 * 1024):.1f} MB"
+
+        uploaded_count = len(files_to_upload) - skipped_count
+        if dry_run:
+            print(f"\nTotal: {uploaded_count} file(s) ({size_str})")
+            if skipped_count > 0:
+                print(f"  Would skip {skipped_count} archive file(s)")
+        else:
+            print(f"✓ Upload complete: {uploaded_count} file(s) ({size_str})")
+            if skipped_count > 0:
+                print(f"  Skipped {skipped_count} archive file(s)")
+
+        return True
+
+    except httpx.HTTPStatusError as e:
+        print(f"Upload failed: HTTP {e.response.status_code} - {e.response.text}")
+        return False
+    except Exception as e:
+        print(f"Upload failed: {e}")
+        return False
+
+
+def _is_archive_file(file_path: Path) -> bool:
+    """
+    Check if a file is an archive file based on its extension.
+
+    Args:
+        file_path: Path to the file to check
+
+    Returns:
+        True if file is an archive, False otherwise
+    """
+    return file_path.suffix.lower() in ARCHIVE_EXTENSIONS
+
+
+def _get_files_to_upload(
+    directory: Path, verbose: bool = False, use_gitignore: bool = True
+) -> list[tuple[Path, str]]:
+    """
+    Get list of files to upload from directory.
+
+    Uses .bmignore and optionally .gitignore patterns for filtering.
+
+    Args:
+        directory: Directory to scan
+        verbose: Show detailed filtering information
+        use_gitignore: If False, skip .gitignore patterns (still use .bmignore)
+
+    Returns:
+        List of (absolute_path, relative_path) tuples
+    """
+    files = []
+    ignored_files = []
+
+    # Load ignore patterns from .bmignore and optionally .gitignore
+    ignore_patterns = load_gitignore_patterns(directory, use_gitignore=use_gitignore)
+
+    if verbose:
+        gitignore_path = directory / ".gitignore"
+        gitignore_exists = gitignore_path.exists() and use_gitignore
+        print(f"\nScanning directory: {directory}")
+        print("Using .bmignore: Yes")
+        print(f"Using .gitignore: {'Yes' if gitignore_exists else 'No'}")
+        print(f"Ignore patterns loaded: {len(ignore_patterns)}")
+        if ignore_patterns and len(ignore_patterns) <= 20:
+            print(f"Patterns: {', '.join(sorted(ignore_patterns))}")
+        print()
+
+    # Walk through directory
+    for root, dirs, filenames in os.walk(directory):
+        root_path = Path(root)
+
+        # Filter directories based on ignore patterns
+        filtered_dirs = []
+        for d in dirs:
+            dir_path = root_path / d
+            if should_ignore_path(dir_path, directory, ignore_patterns):
+                if verbose:
+                    rel_path = dir_path.relative_to(directory)
+                    print(f"  [IGNORED DIR] {rel_path}/")
+            else:
+                filtered_dirs.append(d)
+        dirs[:] = filtered_dirs
+
+        # Process files
+        for filename in filenames:
+            file_path = root_path / filename
+
+            # Calculate relative path for display/remote
+            rel_path = file_path.relative_to(directory)
+            remote_path = str(rel_path).replace("\\", "/")
+
+            # Check if file should be ignored
+            if should_ignore_path(file_path, directory, ignore_patterns):
+                ignored_files.append(remote_path)
+                if verbose:
+                    print(f"  [IGNORED] {remote_path}")
+                continue
+
+            if verbose:
+                print(f"  [INCLUDE] {remote_path}")
+
+            files.append((file_path, remote_path))
+
+    if verbose:
+        print("\nSummary:")
+        print(f"  Files to upload: {len(files)}")
+        print(f"  Files ignored: {len(ignored_files)}")
+
+    return files

basic_memory/cli/commands/cloud/upload_command.py

@@ -0,0 +1,124 @@
+"""Upload CLI commands for basic-memory projects."""
+
+import asyncio
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from basic_memory.cli.app import cloud_app
+from basic_memory.cli.commands.cloud.cloud_utils import (
+    create_cloud_project,
+    project_exists,
+    sync_project,
+)
+from basic_memory.cli.commands.cloud.upload import upload_path
+
+console = Console()
+
+
+@cloud_app.command("upload")
+def upload(
+    path: Path = typer.Argument(
+        ...,
+        help="Path to local file or directory to upload",
+        exists=True,
+        readable=True,
+        resolve_path=True,
+    ),
+    project: str = typer.Option(
+        ...,
+        "--project",
+        "-p",
+        help="Cloud project name (destination)",
+    ),
+    create_project: bool = typer.Option(
+        False,
+        "--create-project",
+        "-c",
+        help="Create project if it doesn't exist",
+    ),
+    sync: bool = typer.Option(
+        True,
+        "--sync/--no-sync",
+        help="Sync project after upload (default: true)",
+    ),
+    verbose: bool = typer.Option(
+        False,
+        "--verbose",
+        "-v",
+        help="Show detailed information about file filtering and upload",
+    ),
+    no_gitignore: bool = typer.Option(
+        False,
+        "--no-gitignore",
+        help="Skip .gitignore patterns (still respects .bmignore)",
+    ),
+    dry_run: bool = typer.Option(
+        False,
+        "--dry-run",
+        help="Show what would be uploaded without actually uploading",
+    ),
+) -> None:
+    """Upload local files or directories to cloud project via WebDAV.
+
+    Examples:
+      bm cloud upload ~/my-notes --project research
+      bm cloud upload notes.md --project research --create-project
+      bm cloud upload ~/docs --project work --no-sync
+      bm cloud upload ./history --project proto --verbose
+      bm cloud upload ./notes --project work --no-gitignore
+      bm cloud upload ./files --project test --dry-run
+    """
+
+    async def _upload():
+        # Check if project exists
+        if not await project_exists(project):
+            if create_project:
+                console.print(f"[blue]Creating cloud project '{project}'...[/blue]")
+                try:
+                    await create_cloud_project(project)
+                    console.print(f"[green]Created project '{project}'[/green]")
+                except Exception as e:
+                    console.print(f"[red]Failed to create project: {e}[/red]")
+                    raise typer.Exit(1)
+            else:
+                console.print(
+                    f"[red]Project '{project}' does not exist.[/red]\n"
+                    f"[yellow]Options:[/yellow]\n"
+                    f"  1. Create it first: bm project add {project}\n"
+                    f"  2. Use --create-project flag to create automatically"
+                )
+                raise typer.Exit(1)
+
+        # Perform upload (or dry run)
+        if dry_run:
+            console.print(
+                f"[yellow]DRY RUN: Showing what would be uploaded to '{project}'[/yellow]"
+            )
+        else:
+            console.print(f"[blue]Uploading {path} to project '{project}'...[/blue]")
+
+        success = await upload_path(
+            path, project, verbose=verbose, use_gitignore=not no_gitignore, dry_run=dry_run
+        )
+        if not success:
+            console.print("[red]Upload failed[/red]")
+            raise typer.Exit(1)
+
+        if dry_run:
+            console.print("[yellow]DRY RUN complete - no files were uploaded[/yellow]")
+        else:
+            console.print(f"[green]Successfully uploaded to '{project}'[/green]")
+
+        # Sync project if requested (skip on dry run)
+        # Force full scan after bisync to ensure database is up-to-date with synced files
+        if sync and not dry_run:
+            console.print(f"[blue]Syncing project '{project}'...[/blue]")
+            try:
+                await sync_project(project, force_full=True)
+            except Exception as e:
+                console.print(f"[yellow]Warning: Sync failed: {e}[/yellow]")
+                console.print("[dim]Files uploaded but may not be indexed yet[/dim]")
+
+    asyncio.run(_upload())

basic_memory/cli/commands/command_utils.py

@@ -0,0 +1,51 @@
+"""utility functions for commands"""
+
+from typing import Optional
+
+from mcp.server.fastmcp.exceptions import ToolError
+import typer
+
+from rich.console import Console
+
+from basic_memory.mcp.async_client import get_client
+
+from basic_memory.mcp.tools.utils import call_post, call_get
+from basic_memory.mcp.project_context import get_active_project
+from basic_memory.schemas import ProjectInfoResponse
+
+console = Console()
+
+
+async def run_sync(project: Optional[str] = None, force_full: bool = False):
+    """Run sync operation via API endpoint.
+
+    Args:
+        project: Optional project name
+        force_full: If True, force a full scan bypassing watermark optimization
+    """
+
+    try:
+        async with get_client() as client:
+            project_item = await get_active_project(client, project, None)
+            url = f"{project_item.project_url}/project/sync"
+            if force_full:
+                url += "?force_full=true"
+            response = await call_post(client, url)
+            data = response.json()
+            console.print(f"[green]{data['message']}[/green]")
+    except (ToolError, ValueError) as e:
+        console.print(f"[red]Sync failed: {e}[/red]")
+        raise typer.Exit(1)
+
+
+async def get_project_info(project: str):
+    """Get project information via API endpoint."""
+
+    try:
+        async with get_client() as client:
+            project_item = await get_active_project(client, project, None)
+            response = await call_get(client, f"{project_item.project_url}/project/info")
+            return ProjectInfoResponse.model_validate(response.json())
+    except (ToolError, ValueError) as e:
+        console.print(f"[red]Sync failed: {e}[/red]")
+        raise typer.Exit(1)

basic_memory/cli/commands/db.py

@@ -1,25 +1,44 @@
 """Database management commands."""
 
 import asyncio
+
 import typer
 from loguru import logger
 
-from basic_memory.alembic import migrations
+from basic_memory import db
 from basic_memory.cli.app import app
+from basic_memory.config import ConfigManager, BasicMemoryConfig, save_basic_memory_config
 
 
 @app.command()
 def reset(
-    reindex: bool = typer.Option(False, "--reindex", help="Rebuild indices from filesystem"),
+    reindex: bool = typer.Option(False, "--reindex", help="Rebuild db index from filesystem"),
 ):  # pragma: no cover
     """Reset database (drop all tables and recreate)."""
-    if typer.confirm("This will delete all data. Are you sure?"):
+    if typer.confirm("This will delete all data in your db. Are you sure?"):
         logger.info("Resetting database...")
-        asyncio.run(migrations.reset_database())
+        config_manager = ConfigManager()
+        app_config = config_manager.config
+        # Get database path
+        db_path = app_config.app_database_path
+
+        # Delete the database file if it exists
+        if db_path.exists():
+            db_path.unlink()
+            logger.info(f"Database file deleted: {db_path}")
+
+        # Reset project configuration
+        config = BasicMemoryConfig()
+        save_basic_memory_config(config_manager.config_file, config)
+        logger.info("Project configuration reset to default")
+
+        # Create a new empty database
+        asyncio.run(db.run_migrations(app_config))
+        logger.info("Database reset complete")
 
         if reindex:
-            # Import and run sync
-            from basic_memory.cli.commands.sync import sync
+            # Run database sync directly
+            from basic_memory.cli.commands.command_utils import run_sync
 
             logger.info("Rebuilding search index from filesystem...")
-            asyncio.run(sync())  # pyright: ignore
+            asyncio.run(run_sync(project=None))

basic_memory/cli/commands/import_chatgpt.py

@@ -0,0 +1,83 @@
+"""Import command for ChatGPT conversations."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from basic_memory.cli.app import import_app
+from basic_memory.config import get_project_config
+from basic_memory.importers import ChatGPTImporter
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+
+console = Console()
+
+
+async def get_markdown_processor() -> MarkdownProcessor:
+    """Get MarkdownProcessor instance."""
+    config = get_project_config()
+    entity_parser = EntityParser(config.home)
+    return MarkdownProcessor(entity_parser)
+
+
+@import_app.command(name="chatgpt", help="Import conversations from ChatGPT JSON export.")
+def import_chatgpt(
+    conversations_json: Annotated[
+        Path, typer.Argument(help="Path to ChatGPT conversations.json file")
+    ] = Path("conversations.json"),
+    folder: Annotated[
+        str, typer.Option(help="The folder to place the files in.")
+    ] = "conversations",
+):
+    """Import chat conversations from ChatGPT JSON format.
+
+    This command will:
+    1. Read the complex tree structure of messages
+    2. Convert them to linear markdown conversations
+    3. Save as clean, readable markdown files
+
+    After importing, run 'basic-memory sync' to index the new files.
+    """
+
+    try:
+        if not conversations_json.exists():  # pragma: no cover
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
+
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
+        config = get_project_config()
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
+
+        # Create importer and run import
+        importer = ChatGPTImporter(config.home, markdown_processor)
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
+
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
+            )
+        )
+
+        console.print("\nRun 'basic-memory sync' to index the new files.")
+
+    except Exception as e:
+        logger.error("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)

basic_memory/cli/commands/import_claude_conversations.py

@@ -0,0 +1,86 @@
+"""Import command for basic-memory CLI to import chat data from conversations2.json format."""
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+from basic_memory.cli.app import claude_app
+from basic_memory.config import get_project_config
+from basic_memory.importers.claude_conversations_importer import ClaudeConversationsImporter
+from basic_memory.markdown import EntityParser, MarkdownProcessor
+from loguru import logger
+from rich.console import Console
+from rich.panel import Panel
+
+console = Console()
+
+
+async def get_markdown_processor() -> MarkdownProcessor:
+    """Get MarkdownProcessor instance."""
+    config = get_project_config()
+    entity_parser = EntityParser(config.home)
+    return MarkdownProcessor(entity_parser)
+
+
+@claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.")
+def import_claude(
+    conversations_json: Annotated[
+        Path, typer.Argument(..., help="Path to conversations.json file")
+    ] = Path("conversations.json"),
+    folder: Annotated[
+        str, typer.Option(help="The folder to place the files in.")
+    ] = "conversations",
+):
+    """Import chat conversations from conversations2.json format.
+
+    This command will:
+    1. Read chat data and nested messages
+    2. Create markdown files for each conversation
+    3. Format content in clean, readable markdown
+
+    After importing, run 'basic-memory sync' to index the new files.
+    """
+
+    config = get_project_config()
+    try:
+        if not conversations_json.exists():
+            typer.echo(f"Error: File not found: {conversations_json}", err=True)
+            raise typer.Exit(1)
+
+        # Get markdown processor
+        markdown_processor = asyncio.run(get_markdown_processor())
+
+        # Create the importer
+        importer = ClaudeConversationsImporter(config.home, markdown_processor)
+
+        # Process the file
+        base_path = config.home / folder
+        console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
+
+        # Run the import
+        with conversations_json.open("r", encoding="utf-8") as file:
+            json_data = json.load(file)
+            result = asyncio.run(importer.import_data(json_data, folder))
+
+        if not result.success:  # pragma: no cover
+            typer.echo(f"Error during import: {result.error_message}", err=True)
+            raise typer.Exit(1)
+
+        # Show results
+        console.print(
+            Panel(
+                f"[green]Import complete![/green]\n\n"
+                f"Imported {result.conversations} conversations\n"
+                f"Containing {result.messages} messages",
+                expand=False,
+            )
+        )
+
+        console.print("\nRun 'basic-memory sync' to index the new files.")
+
+    except Exception as e:
+        logger.error("Import failed")
+        typer.echo(f"Error during import: {e}", err=True)
+        raise typer.Exit(1)