basic-memory 0.17.1__py3-none-any.whl

basic_memory/cli/commands/cloud/cloud_utils.py

@@ -0,0 +1,101 @@
+"""Shared utilities for cloud operations."""
+
+from basic_memory.cli.commands.cloud.api_client import make_api_request
+from basic_memory.config import ConfigManager
+from basic_memory.schemas.cloud import (
+    CloudProjectList,
+    CloudProjectCreateRequest,
+    CloudProjectCreateResponse,
+)
+from basic_memory.utils import generate_permalink
+
+
+class CloudUtilsError(Exception):
+    """Exception raised for cloud utility errors."""
+
+    pass
+
+
+async def fetch_cloud_projects() -> CloudProjectList:
+    """Fetch list of projects from cloud API.
+
+    Returns:
+        CloudProjectList with projects from cloud
+    """
+    try:
+        config_manager = ConfigManager()
+        config = config_manager.config
+        host_url = config.cloud_host.rstrip("/")
+
+        response = await make_api_request(method="GET", url=f"{host_url}/proxy/projects/projects")
+
+        return CloudProjectList.model_validate(response.json())
+    except Exception as e:
+        raise CloudUtilsError(f"Failed to fetch cloud projects: {e}") from e
+
+
+async def create_cloud_project(project_name: str) -> CloudProjectCreateResponse:
+    """Create a new project on cloud.
+
+    Args:
+        project_name: Name of project to create
+
+    Returns:
+        CloudProjectCreateResponse with project details from API
+    """
+    try:
+        config_manager = ConfigManager()
+        config = config_manager.config
+        host_url = config.cloud_host.rstrip("/")
+
+        # Use generate_permalink to ensure consistent naming
+        project_path = generate_permalink(project_name)
+
+        project_data = CloudProjectCreateRequest(
+            name=project_name,
+            path=project_path,
+            set_default=False,
+        )
+
+        response = await make_api_request(
+            method="POST",
+            url=f"{host_url}/proxy/projects/projects",
+            headers={"Content-Type": "application/json"},
+            json_data=project_data.model_dump(),
+        )
+
+        return CloudProjectCreateResponse.model_validate(response.json())
+    except Exception as e:
+        raise CloudUtilsError(f"Failed to create cloud project '{project_name}': {e}") from e
+
+
+async def sync_project(project_name: str, force_full: bool = False) -> None:
+    """Trigger sync for a specific project on cloud.
+
+    Args:
+        project_name: Name of project to sync
+        force_full: If True, force a full scan bypassing watermark optimization
+    """
+    try:
+        from basic_memory.cli.commands.command_utils import run_sync
+
+        await run_sync(project=project_name, force_full=force_full)
+    except Exception as e:
+        raise CloudUtilsError(f"Failed to sync project '{project_name}': {e}") from e
+
+
+async def project_exists(project_name: str) -> bool:
+    """Check if a project exists on cloud.
+
+    Args:
+        project_name: Name of project to check
+
+    Returns:
+        True if project exists, False otherwise
+    """
+    try:
+        projects = await fetch_cloud_projects()
+        project_names = {p.name for p in projects.projects}
+        return project_name in project_names
+    except Exception:
+        return False

basic_memory/cli/commands/cloud/core_commands.py

@@ -0,0 +1,195 @@
+"""Core cloud commands for Basic Memory CLI."""
+
+import asyncio
+
+import typer
+from rich.console import Console
+
+from basic_memory.cli.app import cloud_app
+from basic_memory.cli.auth import CLIAuth
+from basic_memory.config import ConfigManager
+from basic_memory.cli.commands.cloud.api_client import (
+    CloudAPIError,
+    SubscriptionRequiredError,
+    get_cloud_config,
+    make_api_request,
+)
+from basic_memory.cli.commands.cloud.bisync_commands import (
+    BisyncError,
+    generate_mount_credentials,
+    get_mount_info,
+)
+from basic_memory.cli.commands.cloud.rclone_config import configure_rclone_remote
+from basic_memory.cli.commands.cloud.rclone_installer import (
+    RcloneInstallError,
+    install_rclone,
+)
+
+console = Console()
+
+
+@cloud_app.command()
+def login():
+    """Authenticate with WorkOS using OAuth Device Authorization flow and enable cloud mode."""
+
+    async def _login():
+        client_id, domain, host_url = get_cloud_config()
+        auth = CLIAuth(client_id=client_id, authkit_domain=domain)
+
+        try:
+            success = await auth.login()
+            if not success:
+                console.print("[red]Login failed[/red]")
+                raise typer.Exit(1)
+
+            # Test subscription access by calling a protected endpoint
+            console.print("[dim]Verifying subscription access...[/dim]")
+            await make_api_request("GET", f"{host_url.rstrip('/')}/proxy/health")
+
+            # Enable cloud mode after successful login and subscription validation
+            config_manager = ConfigManager()
+            config = config_manager.load_config()
+            config.cloud_mode = True
+            config_manager.save_config(config)
+
+            console.print("[green]Cloud mode enabled[/green]")
+            console.print(f"[dim]All CLI commands now work against {host_url}[/dim]")
+
+        except SubscriptionRequiredError as e:
+            console.print("\n[red]Subscription Required[/red]\n")
+            console.print(f"[yellow]{e.args[0]}[/yellow]\n")
+            console.print(f"Subscribe at: [blue underline]{e.subscribe_url}[/blue underline]\n")
+            console.print(
+                "[dim]Once you have an active subscription, run [bold]bm cloud login[/bold] again.[/dim]"
+            )
+            raise typer.Exit(1)
+
+    asyncio.run(_login())
+
+
+@cloud_app.command()
+def logout():
+    """Disable cloud mode and return to local mode."""
+
+    # Disable cloud mode
+    config_manager = ConfigManager()
+    config = config_manager.load_config()
+    config.cloud_mode = False
+    config_manager.save_config(config)
+
+    console.print("[green]Cloud mode disabled[/green]")
+    console.print("[dim]All CLI commands now work locally[/dim]")
+
+
+@cloud_app.command("status")
+def status() -> None:
+    """Check cloud mode status and cloud instance health."""
+    # Check cloud mode
+    config_manager = ConfigManager()
+    config = config_manager.load_config()
+
+    console.print("[bold blue]Cloud Mode Status[/bold blue]")
+    if config.cloud_mode:
+        console.print("  Mode: [green]Cloud (enabled)[/green]")
+        console.print(f"  Host: {config.cloud_host}")
+        console.print("  [dim]All CLI commands work against cloud[/dim]")
+    else:
+        console.print("  Mode: [yellow]Local (disabled)[/yellow]")
+        console.print("  [dim]All CLI commands work locally[/dim]")
+        console.print("\n[dim]To enable cloud mode, run: bm cloud login[/dim]")
+        return
+
+    # Get cloud configuration
+    _, _, host_url = get_cloud_config()
+    host_url = host_url.rstrip("/")
+
+    # Prepare headers
+    headers = {}
+
+    try:
+        console.print("\n[blue]Checking cloud instance health...[/blue]")
+
+        # Make API request to check health
+        response = asyncio.run(
+            make_api_request(method="GET", url=f"{host_url}/proxy/health", headers=headers)
+        )
+
+        health_data = response.json()
+
+        console.print("[green]Cloud instance is healthy[/green]")
+
+        # Display status details
+        if "status" in health_data:
+            console.print(f"  Status: {health_data['status']}")
+        if "version" in health_data:
+            console.print(f"  Version: {health_data['version']}")
+        if "timestamp" in health_data:
+            console.print(f"  Timestamp: {health_data['timestamp']}")
+
+        console.print("\n[dim]To sync projects, use: bm project bisync --name <project>[/dim]")
+
+    except CloudAPIError as e:
+        console.print(f"[red]Error checking cloud health: {e}[/red]")
+        raise typer.Exit(1)
+    except Exception as e:
+        console.print(f"[red]Unexpected error: {e}[/red]")
+        raise typer.Exit(1)
+
+
+@cloud_app.command("setup")
+def setup() -> None:
+    """Set up cloud sync by installing rclone and configuring credentials.
+
+    SPEC-20: Simplified to project-scoped workflow.
+    After setup, use project commands for syncing:
+      bm project add <name> <path> --local-path ~/projects/<name>
+      bm project bisync --name <name> --resync  # First time
+      bm project bisync --name <name>            # Subsequent syncs
+    """
+    console.print("[bold blue]Basic Memory Cloud Setup[/bold blue]")
+    console.print("Setting up cloud sync with rclone...\n")
+
+    try:
+        # Step 1: Install rclone
+        console.print("[blue]Step 1: Installing rclone...[/blue]")
+        install_rclone()
+
+        # Step 2: Get tenant info
+        console.print("\n[blue]Step 2: Getting tenant information...[/blue]")
+        tenant_info = asyncio.run(get_mount_info())
+        console.print(f"[green]Found tenant: {tenant_info.tenant_id}[/green]")
+
+        # Step 3: Generate credentials
+        console.print("\n[blue]Step 3: Generating sync credentials...[/blue]")
+        creds = asyncio.run(generate_mount_credentials(tenant_info.tenant_id))
+        console.print("[green]Generated secure credentials[/green]")
+
+        # Step 4: Configure rclone remote
+        console.print("\n[blue]Step 4: Configuring rclone remote...[/blue]")
+        configure_rclone_remote(
+            access_key=creds.access_key,
+            secret_key=creds.secret_key,
+        )
+
+        console.print("\n[bold green]Cloud setup completed successfully![/bold green]")
+        console.print("\n[bold]Next steps:[/bold]")
+        console.print("1. Add a project with local sync path:")
+        console.print("   bm project add research --local-path ~/Documents/research")
+        console.print("\n   Or configure sync for an existing project:")
+        console.print("   bm project sync-setup research ~/Documents/research")
+        console.print("\n2. Preview the initial sync (recommended):")
+        console.print("   bm project bisync --name research --resync --dry-run")
+        console.print("\n3. If all looks good, run the actual sync:")
+        console.print("   bm project bisync --name research --resync")
+        console.print("\n4. Subsequent syncs (no --resync needed):")
+        console.print("   bm project bisync --name research")
+        console.print(
+            "\n[dim]Tip: Always use --dry-run first to preview changes before syncing[/dim]"
+        )
+
+    except (RcloneInstallError, BisyncError, CloudAPIError) as e:
+        console.print(f"\n[red]Setup failed: {e}[/red]")
+        raise typer.Exit(1)
+    except Exception as e:
+        console.print(f"\n[red]Unexpected error during setup: {e}[/red]")
+        raise typer.Exit(1)

basic_memory/cli/commands/cloud/rclone_commands.py

@@ -0,0 +1,371 @@
+"""Project-scoped rclone sync commands for Basic Memory Cloud.
+
+This module provides simplified, project-scoped rclone operations:
+- Each project syncs independently
+- Uses single "basic-memory-cloud" remote (not tenant-specific)
+- Balanced defaults from SPEC-8 Phase 4 testing
+- Per-project bisync state tracking
+
+Replaces tenant-wide sync with project-scoped workflows.
+"""
+
+import re
+import subprocess
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+from typing import Optional
+
+from loguru import logger
+from rich.console import Console
+
+from basic_memory.cli.commands.cloud.rclone_installer import is_rclone_installed
+from basic_memory.utils import normalize_project_path
+
+console = Console()
+
+# Minimum rclone version for --create-empty-src-dirs support
+MIN_RCLONE_VERSION_EMPTY_DIRS = (1, 64, 0)
+
+
+class RcloneError(Exception):
+    """Exception raised for rclone command errors."""
+
+    pass
+
+
+def check_rclone_installed() -> None:
+    """Check if rclone is installed and raise helpful error if not.
+
+    Raises:
+        RcloneError: If rclone is not installed with installation instructions
+    """
+    if not is_rclone_installed():
+        raise RcloneError(
+            "rclone is not installed.\n\n"
+            "Install rclone by running: bm cloud setup\n"
+            "Or install manually from: https://rclone.org/downloads/\n\n"
+            "Windows users: Ensure you have a package manager installed (winget, chocolatey, or scoop)"
+        )
+
+
+@lru_cache(maxsize=1)
+def get_rclone_version() -> tuple[int, int, int] | None:
+    """Get rclone version as (major, minor, patch) tuple.
+
+    Returns:
+        Version tuple like (1, 64, 2), or None if version cannot be determined.
+
+    Note:
+        Result is cached since rclone version won't change during runtime.
+    """
+    try:
+        result = subprocess.run(["rclone", "version"], capture_output=True, text=True, timeout=10)
+        # Parse "rclone v1.64.2" or "rclone v1.60.1-DEV"
+        match = re.search(r"v(\d+)\.(\d+)\.(\d+)", result.stdout)
+        if match:
+            version = (int(match.group(1)), int(match.group(2)), int(match.group(3)))
+            logger.debug(f"Detected rclone version: {version}")
+            return version
+    except Exception as e:
+        logger.warning(f"Could not determine rclone version: {e}")
+    return None
+
+
+def supports_create_empty_src_dirs() -> bool:
+    """Check if installed rclone supports --create-empty-src-dirs flag.
+
+    Returns:
+        True if rclone version >= 1.64.0, False otherwise.
+    """
+    version = get_rclone_version()
+    if version is None:
+        # If we can't determine version, assume older and skip the flag
+        return False
+    return version >= MIN_RCLONE_VERSION_EMPTY_DIRS
+
+
+@dataclass
+class SyncProject:
+    """Project configured for cloud sync.
+
+    Attributes:
+        name: Project name
+        path: Cloud path (e.g., "app/data/research")
+        local_sync_path: Local directory for syncing (optional)
+    """
+
+    name: str
+    path: str
+    local_sync_path: Optional[str] = None
+
+
+def get_bmignore_filter_path() -> Path:
+    """Get path to rclone filter file.
+
+    Uses ~/.basic-memory/.bmignore converted to rclone format.
+    File is automatically created with default patterns on first use.
+
+    Returns:
+        Path to rclone filter file
+    """
+    # Import here to avoid circular dependency
+    from basic_memory.cli.commands.cloud.bisync_commands import (
+        convert_bmignore_to_rclone_filters,
+    )
+
+    return convert_bmignore_to_rclone_filters()
+
+
+def get_project_bisync_state(project_name: str) -> Path:
+    """Get path to project's bisync state directory.
+
+    Args:
+        project_name: Name of the project
+
+    Returns:
+        Path to bisync state directory for this project
+    """
+    return Path.home() / ".basic-memory" / "bisync-state" / project_name
+
+
+def bisync_initialized(project_name: str) -> bool:
+    """Check if bisync has been initialized for this project.
+
+    Args:
+        project_name: Name of the project
+
+    Returns:
+        True if bisync state exists, False otherwise
+    """
+    state_path = get_project_bisync_state(project_name)
+    return state_path.exists() and any(state_path.iterdir())
+
+
+def get_project_remote(project: SyncProject, bucket_name: str) -> str:
+    """Build rclone remote path for project.
+
+    Args:
+        project: Project with cloud path
+        bucket_name: S3 bucket name
+
+    Returns:
+        Remote path like "basic-memory-cloud:bucket-name/basic-memory-llc"
+
+    Note:
+        The API returns paths like "/app/data/basic-memory-llc" because the S3 bucket
+        is mounted at /app/data on the fly machine. We need to strip the /app/data/
+        prefix to get the actual S3 path within the bucket.
+    """
+    # Normalize path to strip /app/data/ mount point prefix
+    cloud_path = normalize_project_path(project.path).lstrip("/")
+    return f"basic-memory-cloud:{bucket_name}/{cloud_path}"
+
+
+def project_sync(
+    project: SyncProject,
+    bucket_name: str,
+    dry_run: bool = False,
+    verbose: bool = False,
+) -> bool:
+    """One-way sync: local → cloud.
+
+    Makes cloud identical to local using rclone sync.
+
+    Args:
+        project: Project to sync
+        bucket_name: S3 bucket name
+        dry_run: Preview changes without applying
+        verbose: Show detailed output
+
+    Returns:
+        True if sync succeeded, False otherwise
+
+    Raises:
+        RcloneError: If project has no local_sync_path configured or rclone not installed
+    """
+    check_rclone_installed()
+
+    if not project.local_sync_path:
+        raise RcloneError(f"Project {project.name} has no local_sync_path configured")
+
+    local_path = Path(project.local_sync_path).expanduser()
+    remote_path = get_project_remote(project, bucket_name)
+    filter_path = get_bmignore_filter_path()
+
+    cmd = [
+        "rclone",
+        "sync",
+        str(local_path),
+        remote_path,
+        "--filter-from",
+        str(filter_path),
+    ]
+
+    if verbose:
+        cmd.append("--verbose")
+    else:
+        cmd.append("--progress")
+
+    if dry_run:
+        cmd.append("--dry-run")
+
+    result = subprocess.run(cmd, text=True)
+    return result.returncode == 0
+
+
+def project_bisync(
+    project: SyncProject,
+    bucket_name: str,
+    dry_run: bool = False,
+    resync: bool = False,
+    verbose: bool = False,
+) -> bool:
+    """Two-way sync: local ↔ cloud.
+
+    Uses rclone bisync with balanced defaults:
+    - conflict_resolve: newer (auto-resolve to most recent)
+    - max_delete: 25 (safety limit)
+    - compare: modtime (ignore size differences from line ending conversions)
+    - check_access: false (skip for performance)
+
+    Args:
+        project: Project to sync
+        bucket_name: S3 bucket name
+        dry_run: Preview changes without applying
+        resync: Force resync to establish new baseline
+        verbose: Show detailed output
+
+    Returns:
+        True if bisync succeeded, False otherwise
+
+    Raises:
+        RcloneError: If project has no local_sync_path, needs --resync, or rclone not installed
+    """
+    check_rclone_installed()
+
+    if not project.local_sync_path:
+        raise RcloneError(f"Project {project.name} has no local_sync_path configured")
+
+    local_path = Path(project.local_sync_path).expanduser()
+    remote_path = get_project_remote(project, bucket_name)
+    filter_path = get_bmignore_filter_path()
+    state_path = get_project_bisync_state(project.name)
+
+    # Ensure state directory exists
+    state_path.mkdir(parents=True, exist_ok=True)
+
+    cmd = [
+        "rclone",
+        "bisync",
+        str(local_path),
+        remote_path,
+        "--resilient",
+        "--conflict-resolve=newer",
+        "--max-delete=25",
+        "--compare=modtime",  # Ignore size differences from line ending conversions
+        "--filter-from",
+        str(filter_path),
+        "--workdir",
+        str(state_path),
+    ]
+
+    # Add --create-empty-src-dirs if rclone version supports it (v1.64+)
+    if supports_create_empty_src_dirs():
+        cmd.append("--create-empty-src-dirs")
+
+    if verbose:
+        cmd.append("--verbose")
+    else:
+        cmd.append("--progress")
+
+    if dry_run:
+        cmd.append("--dry-run")
+
+    if resync:
+        cmd.append("--resync")
+
+    # Check if first run requires resync
+    if not resync and not bisync_initialized(project.name) and not dry_run:
+        raise RcloneError(
+            f"First bisync for {project.name} requires --resync to establish baseline.\n"
+            f"Run: bm project bisync --name {project.name} --resync"
+        )
+
+    result = subprocess.run(cmd, text=True)
+    return result.returncode == 0
+
+
+def project_check(
+    project: SyncProject,
+    bucket_name: str,
+    one_way: bool = False,
+) -> bool:
+    """Check integrity between local and cloud.
+
+    Verifies files match without transferring data.
+
+    Args:
+        project: Project to check
+        bucket_name: S3 bucket name
+        one_way: Only check for missing files on destination (faster)
+
+    Returns:
+        True if files match, False if differences found
+
+    Raises:
+        RcloneError: If project has no local_sync_path configured or rclone not installed
+    """
+    check_rclone_installed()
+
+    if not project.local_sync_path:
+        raise RcloneError(f"Project {project.name} has no local_sync_path configured")
+
+    local_path = Path(project.local_sync_path).expanduser()
+    remote_path = get_project_remote(project, bucket_name)
+    filter_path = get_bmignore_filter_path()
+
+    cmd = [
+        "rclone",
+        "check",
+        str(local_path),
+        remote_path,
+        "--filter-from",
+        str(filter_path),
+    ]
+
+    if one_way:
+        cmd.append("--one-way")
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    return result.returncode == 0
+
+
+def project_ls(
+    project: SyncProject,
+    bucket_name: str,
+    path: Optional[str] = None,
+) -> list[str]:
+    """List files in remote project.
+
+    Args:
+        project: Project to list files from
+        bucket_name: S3 bucket name
+        path: Optional subdirectory within project
+
+    Returns:
+        List of file paths
+
+    Raises:
+        subprocess.CalledProcessError: If rclone command fails
+        RcloneError: If rclone is not installed
+    """
+    check_rclone_installed()
+
+    remote_path = get_project_remote(project, bucket_name)
+    if path:
+        remote_path = f"{remote_path}/{path}"
+
+    cmd = ["rclone", "ls", remote_path]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+    return result.stdout.splitlines()