basic-memory 0.2.12__py3-none-any.whl → 0.16.1__py3-none-any.whl

basic_memory/cli/commands/cloud/rclone_commands.py

@@ -0,0 +1,301 @@
+"""Project-scoped rclone sync commands for Basic Memory Cloud.
+
+This module provides simplified, project-scoped rclone operations:
+- Each project syncs independently
+- Uses single "basic-memory-cloud" remote (not tenant-specific)
+- Balanced defaults from SPEC-8 Phase 4 testing
+- Per-project bisync state tracking
+
+Replaces tenant-wide sync with project-scoped workflows.
+"""
+
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+from rich.console import Console
+
+from basic_memory.utils import normalize_project_path
+
+console = Console()
+
+
+class RcloneError(Exception):
+    """Exception raised for rclone command errors."""
+
+    pass
+
+
+@dataclass
+class SyncProject:
+    """Project configured for cloud sync.
+
+    Attributes:
+        name: Project name
+        path: Cloud path (e.g., "app/data/research")
+        local_sync_path: Local directory for syncing (optional)
+    """
+
+    name: str
+    path: str
+    local_sync_path: Optional[str] = None
+
+
+def get_bmignore_filter_path() -> Path:
+    """Get path to rclone filter file.
+
+    Uses ~/.basic-memory/.bmignore converted to rclone format.
+    File is automatically created with default patterns on first use.
+
+    Returns:
+        Path to rclone filter file
+    """
+    # Import here to avoid circular dependency
+    from basic_memory.cli.commands.cloud.bisync_commands import (
+        convert_bmignore_to_rclone_filters,
+    )
+
+    return convert_bmignore_to_rclone_filters()
+
+
+def get_project_bisync_state(project_name: str) -> Path:
+    """Get path to project's bisync state directory.
+
+    Args:
+        project_name: Name of the project
+
+    Returns:
+        Path to bisync state directory for this project
+    """
+    return Path.home() / ".basic-memory" / "bisync-state" / project_name
+
+
+def bisync_initialized(project_name: str) -> bool:
+    """Check if bisync has been initialized for this project.
+
+    Args:
+        project_name: Name of the project
+
+    Returns:
+        True if bisync state exists, False otherwise
+    """
+    state_path = get_project_bisync_state(project_name)
+    return state_path.exists() and any(state_path.iterdir())
+
+
+def get_project_remote(project: SyncProject, bucket_name: str) -> str:
+    """Build rclone remote path for project.
+
+    Args:
+        project: Project with cloud path
+        bucket_name: S3 bucket name
+
+    Returns:
+        Remote path like "basic-memory-cloud:bucket-name/basic-memory-llc"
+
+    Note:
+        The API returns paths like "/app/data/basic-memory-llc" because the S3 bucket
+        is mounted at /app/data on the fly machine. We need to strip the /app/data/
+        prefix to get the actual S3 path within the bucket.
+    """
+    # Normalize path to strip /app/data/ mount point prefix
+    cloud_path = normalize_project_path(project.path).lstrip("/")
+    return f"basic-memory-cloud:{bucket_name}/{cloud_path}"
+
+
+def project_sync(
+    project: SyncProject,
+    bucket_name: str,
+    dry_run: bool = False,
+    verbose: bool = False,
+) -> bool:
+    """One-way sync: local → cloud.
+
+    Makes cloud identical to local using rclone sync.
+
+    Args:
+        project: Project to sync
+        bucket_name: S3 bucket name
+        dry_run: Preview changes without applying
+        verbose: Show detailed output
+
+    Returns:
+        True if sync succeeded, False otherwise
+
+    Raises:
+        RcloneError: If project has no local_sync_path configured
+    """
+    if not project.local_sync_path:
+        raise RcloneError(f"Project {project.name} has no local_sync_path configured")
+
+    local_path = Path(project.local_sync_path).expanduser()
+    remote_path = get_project_remote(project, bucket_name)
+    filter_path = get_bmignore_filter_path()
+
+    cmd = [
+        "rclone",
+        "sync",
+        str(local_path),
+        remote_path,
+        "--filter-from",
+        str(filter_path),
+    ]
+
+    if verbose:
+        cmd.append("--verbose")
+    else:
+        cmd.append("--progress")
+
+    if dry_run:
+        cmd.append("--dry-run")
+
+    result = subprocess.run(cmd, text=True)
+    return result.returncode == 0
+
+
+def project_bisync(
+    project: SyncProject,
+    bucket_name: str,
+    dry_run: bool = False,
+    resync: bool = False,
+    verbose: bool = False,
+) -> bool:
+    """Two-way sync: local ↔ cloud.
+
+    Uses rclone bisync with balanced defaults:
+    - conflict_resolve: newer (auto-resolve to most recent)
+    - max_delete: 25 (safety limit)
+    - compare: modtime (ignore size differences from line ending conversions)
+    - check_access: false (skip for performance)
+
+    Args:
+        project: Project to sync
+        bucket_name: S3 bucket name
+        dry_run: Preview changes without applying
+        resync: Force resync to establish new baseline
+        verbose: Show detailed output
+
+    Returns:
+        True if bisync succeeded, False otherwise
+
+    Raises:
+        RcloneError: If project has no local_sync_path or needs --resync
+    """
+    if not project.local_sync_path:
+        raise RcloneError(f"Project {project.name} has no local_sync_path configured")
+
+    local_path = Path(project.local_sync_path).expanduser()
+    remote_path = get_project_remote(project, bucket_name)
+    filter_path = get_bmignore_filter_path()
+    state_path = get_project_bisync_state(project.name)
+
+    # Ensure state directory exists
+    state_path.mkdir(parents=True, exist_ok=True)
+
+    cmd = [
+        "rclone",
+        "bisync",
+        str(local_path),
+        remote_path,
+        "--create-empty-src-dirs",
+        "--resilient",
+        "--conflict-resolve=newer",
+        "--max-delete=25",
+        "--compare=modtime",  # Ignore size differences from line ending conversions
+        "--filter-from",
+        str(filter_path),
+        "--workdir",
+        str(state_path),
+    ]
+
+    if verbose:
+        cmd.append("--verbose")
+    else:
+        cmd.append("--progress")
+
+    if dry_run:
+        cmd.append("--dry-run")
+
+    if resync:
+        cmd.append("--resync")
+
+    # Check if first run requires resync
+    if not resync and not bisync_initialized(project.name) and not dry_run:
+        raise RcloneError(
+            f"First bisync for {project.name} requires --resync to establish baseline.\n"
+            f"Run: bm project bisync --name {project.name} --resync"
+        )
+
+    result = subprocess.run(cmd, text=True)
+    return result.returncode == 0
+
+
+def project_check(
+    project: SyncProject,
+    bucket_name: str,
+    one_way: bool = False,
+) -> bool:
+    """Check integrity between local and cloud.
+
+    Verifies files match without transferring data.
+
+    Args:
+        project: Project to check
+        bucket_name: S3 bucket name
+        one_way: Only check for missing files on destination (faster)
+
+    Returns:
+        True if files match, False if differences found
+
+    Raises:
+        RcloneError: If project has no local_sync_path configured
+    """
+    if not project.local_sync_path:
+        raise RcloneError(f"Project {project.name} has no local_sync_path configured")
+
+    local_path = Path(project.local_sync_path).expanduser()
+    remote_path = get_project_remote(project, bucket_name)
+    filter_path = get_bmignore_filter_path()
+
+    cmd = [
+        "rclone",
+        "check",
+        str(local_path),
+        remote_path,
+        "--filter-from",
+        str(filter_path),
+    ]
+
+    if one_way:
+        cmd.append("--one-way")
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    return result.returncode == 0
+
+
+def project_ls(
+    project: SyncProject,
+    bucket_name: str,
+    path: Optional[str] = None,
+) -> list[str]:
+    """List files in remote project.
+
+    Args:
+        project: Project to list files from
+        bucket_name: S3 bucket name
+        path: Optional subdirectory within project
+
+    Returns:
+        List of file paths
+
+    Raises:
+        subprocess.CalledProcessError: If rclone command fails
+    """
+    remote_path = get_project_remote(project, bucket_name)
+    if path:
+        remote_path = f"{remote_path}/{path}"
+
+    cmd = ["rclone", "ls", remote_path]
+    result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+    return result.stdout.splitlines()

basic_memory/cli/commands/cloud/rclone_config.py

@@ -0,0 +1,110 @@
+"""rclone configuration management for Basic Memory Cloud.
+
+This module provides simplified rclone configuration for SPEC-20.
+Uses a single "basic-memory-cloud" remote for all operations.
+"""
+
+import configparser
+import os
+import shutil
+from pathlib import Path
+from typing import Optional
+
+from rich.console import Console
+
+console = Console()
+
+
+class RcloneConfigError(Exception):
+    """Exception raised for rclone configuration errors."""
+
+    pass
+
+
+def get_rclone_config_path() -> Path:
+    """Get the path to rclone configuration file."""
+    config_dir = Path.home() / ".config" / "rclone"
+    config_dir.mkdir(parents=True, exist_ok=True)
+    return config_dir / "rclone.conf"
+
+
+def backup_rclone_config() -> Optional[Path]:
+    """Create a backup of existing rclone config."""
+    config_path = get_rclone_config_path()
+    if not config_path.exists():
+        return None
+
+    backup_path = config_path.with_suffix(f".conf.backup-{os.getpid()}")
+    shutil.copy2(config_path, backup_path)
+    console.print(f"[dim]Created backup: {backup_path}[/dim]")
+    return backup_path
+
+
+def load_rclone_config() -> configparser.ConfigParser:
+    """Load existing rclone configuration."""
+    config = configparser.ConfigParser()
+    config_path = get_rclone_config_path()
+
+    if config_path.exists():
+        config.read(config_path)
+
+    return config
+
+
+def save_rclone_config(config: configparser.ConfigParser) -> None:
+    """Save rclone configuration to file."""
+    config_path = get_rclone_config_path()
+
+    with open(config_path, "w") as f:
+        config.write(f)
+
+    console.print(f"[dim]Updated rclone config: {config_path}[/dim]")
+
+
+def configure_rclone_remote(
+    access_key: str,
+    secret_key: str,
+    endpoint: str = "https://fly.storage.tigris.dev",
+    region: str = "auto",
+) -> str:
+    """Configure single rclone remote named 'basic-memory-cloud'.
+
+    This is the simplified approach from SPEC-20 that uses one remote
+    for all Basic Memory cloud operations (not tenant-specific).
+
+    Args:
+        access_key: S3 access key ID
+        secret_key: S3 secret access key
+        endpoint: S3-compatible endpoint URL
+        region: S3 region (default: auto)
+
+    Returns:
+        The remote name: "basic-memory-cloud"
+    """
+    # Backup existing config
+    backup_rclone_config()
+
+    # Load existing config
+    config = load_rclone_config()
+
+    # Single remote name (not tenant-specific)
+    REMOTE_NAME = "basic-memory-cloud"
+
+    # Add/update the remote section
+    if not config.has_section(REMOTE_NAME):
+        config.add_section(REMOTE_NAME)
+
+    config.set(REMOTE_NAME, "type", "s3")
+    config.set(REMOTE_NAME, "provider", "Other")
+    config.set(REMOTE_NAME, "access_key_id", access_key)
+    config.set(REMOTE_NAME, "secret_access_key", secret_key)
+    config.set(REMOTE_NAME, "endpoint", endpoint)
+    config.set(REMOTE_NAME, "region", region)
+    # Prevent unnecessary encoding of filenames (only encode slashes and invalid UTF-8)
+    # This prevents files with spaces like "Hello World.md" from being quoted
+    config.set(REMOTE_NAME, "encoding", "Slash,InvalidUtf8")
+    # Save updated config
+    save_rclone_config(config)
+
+    console.print(f"[green]Configured rclone remote: {REMOTE_NAME}[/green]")
+    return REMOTE_NAME

basic_memory/cli/commands/cloud/rclone_installer.py

@@ -0,0 +1,249 @@
+"""Cross-platform rclone installation utilities."""
+
+import os
+import platform
+import shutil
+import subprocess
+from typing import Optional
+
+from rich.console import Console
+
+console = Console()
+
+
+class RcloneInstallError(Exception):
+    """Exception raised for rclone installation errors."""
+
+    pass
+
+
+def is_rclone_installed() -> bool:
+    """Check if rclone is already installed and available in PATH."""
+    return shutil.which("rclone") is not None
+
+
+def get_platform() -> str:
+    """Get the current platform identifier."""
+    system = platform.system().lower()
+    if system == "darwin":
+        return "macos"
+    elif system == "linux":
+        return "linux"
+    elif system == "windows":
+        return "windows"
+    else:
+        raise RcloneInstallError(f"Unsupported platform: {system}")
+
+
+def run_command(command: list[str], check: bool = True) -> subprocess.CompletedProcess:
+    """Run a command with proper error handling."""
+    try:
+        console.print(f"[dim]Running: {' '.join(command)}[/dim]")
+        result = subprocess.run(command, capture_output=True, text=True, check=check)
+        if result.stdout:
+            console.print(f"[dim]Output: {result.stdout.strip()}[/dim]")
+        return result
+    except subprocess.CalledProcessError as e:
+        console.print(f"[red]Command failed: {e}[/red]")
+        if e.stderr:
+            console.print(f"[red]Error output: {e.stderr}[/red]")
+        raise RcloneInstallError(f"Command failed: {e}") from e
+    except FileNotFoundError as e:
+        raise RcloneInstallError(f"Command not found: {' '.join(command)}") from e
+
+
+def install_rclone_macos() -> None:
+    """Install rclone on macOS using Homebrew or official script."""
+    # Try Homebrew first
+    if shutil.which("brew"):
+        try:
+            console.print("[blue]Installing rclone via Homebrew...[/blue]")
+            run_command(["brew", "install", "rclone"])
+            console.print("[green]rclone installed via Homebrew[/green]")
+            return
+        except RcloneInstallError:
+            console.print(
+                "[yellow]Homebrew installation failed, trying official script...[/yellow]"
+            )
+
+    # Fallback to official script
+    console.print("[blue]Installing rclone via official script...[/blue]")
+    try:
+        run_command(["sh", "-c", "curl https://rclone.org/install.sh | sudo bash"])
+        console.print("[green]rclone installed via official script[/green]")
+    except RcloneInstallError:
+        raise RcloneInstallError(
+            "Failed to install rclone. Please install manually: brew install rclone"
+        )
+
+
+def install_rclone_linux() -> None:
+    """Install rclone on Linux using package managers or official script."""
+    # Try snap first (most universal)
+    if shutil.which("snap"):
+        try:
+            console.print("[blue]Installing rclone via snap...[/blue]")
+            run_command(["sudo", "snap", "install", "rclone"])
+            console.print("[green]rclone installed via snap[/green]")
+            return
+        except RcloneInstallError:
+            console.print("[yellow]Snap installation failed, trying apt...[/yellow]")
+
+    # Try apt (Debian/Ubuntu)
+    if shutil.which("apt"):
+        try:
+            console.print("[blue]Installing rclone via apt...[/blue]")
+            run_command(["sudo", "apt", "update"])
+            run_command(["sudo", "apt", "install", "-y", "rclone"])
+            console.print("[green]rclone installed via apt[/green]")
+            return
+        except RcloneInstallError:
+            console.print("[yellow]apt installation failed, trying official script...[/yellow]")
+
+    # Fallback to official script
+    console.print("[blue]Installing rclone via official script...[/blue]")
+    try:
+        run_command(["sh", "-c", "curl https://rclone.org/install.sh | sudo bash"])
+        console.print("[green]rclone installed via official script[/green]")
+    except RcloneInstallError:
+        raise RcloneInstallError(
+            "Failed to install rclone. Please install manually: sudo snap install rclone"
+        )
+
+
+def install_rclone_windows() -> None:
+    """Install rclone on Windows using package managers."""
+    # Try winget first (built into Windows 10+)
+    if shutil.which("winget"):
+        try:
+            console.print("[blue]Installing rclone via winget...[/blue]")
+            run_command(
+                [
+                    "winget",
+                    "install",
+                    "Rclone.Rclone",
+                    "--accept-source-agreements",
+                    "--accept-package-agreements",
+                ]
+            )
+            console.print("[green]rclone installed via winget[/green]")
+            return
+        except RcloneInstallError:
+            console.print("[yellow]winget installation failed, trying chocolatey...[/yellow]")
+
+    # Try chocolatey
+    if shutil.which("choco"):
+        try:
+            console.print("[blue]Installing rclone via chocolatey...[/blue]")
+            run_command(["choco", "install", "rclone", "-y"])
+            console.print("[green]rclone installed via chocolatey[/green]")
+            return
+        except RcloneInstallError:
+            console.print("[yellow]chocolatey installation failed, trying scoop...[/yellow]")
+
+    # Try scoop
+    if shutil.which("scoop"):
+        try:
+            console.print("[blue]Installing rclone via scoop...[/blue]")
+            run_command(["scoop", "install", "rclone"])
+            console.print("[green]rclone installed via scoop[/green]")
+            return
+        except RcloneInstallError:
+            console.print("[yellow]scoop installation failed[/yellow]")
+
+    # No package manager available
+    raise RcloneInstallError(
+        "Could not install rclone automatically. Please install a package manager "
+        "(winget, chocolatey, or scoop) or install rclone manually from https://rclone.org/downloads/"
+    )
+
+
+def install_rclone(platform_override: Optional[str] = None) -> None:
+    """Install rclone for the current platform."""
+    if is_rclone_installed():
+        console.print("[green]rclone is already installed[/green]")
+        return
+
+    platform_name = platform_override or get_platform()
+    console.print(f"[blue]Installing rclone for {platform_name}...[/blue]")
+
+    try:
+        if platform_name == "macos":
+            install_rclone_macos()
+        elif platform_name == "linux":
+            install_rclone_linux()
+        elif platform_name == "windows":
+            install_rclone_windows()
+            refresh_windows_path()
+        else:
+            raise RcloneInstallError(f"Unsupported platform: {platform_name}")
+
+        # Verify installation
+        if not is_rclone_installed():
+            raise RcloneInstallError("rclone installation completed but command not found in PATH")
+
+        console.print("[green]rclone installation completed successfully[/green]")
+
+    except RcloneInstallError:
+        raise
+    except Exception as e:
+        raise RcloneInstallError(f"Unexpected error during installation: {e}") from e
+
+
+def refresh_windows_path() -> None:
+    """Refresh the Windows PATH environment variable for the current session."""
+    if platform.system().lower() != "windows":
+        return
+
+    # Importing here after performing platform detection. Also note that we have to ignore pylance/pyright
+    # warnings about winreg attributes so that "errors" don't appear on non-Windows platforms.
+    import winreg
+
+    user_key_path = r"Environment"
+    system_key_path = r"System\CurrentControlSet\Control\Session Manager\Environment"
+    new_path = ""
+
+    # Read user PATH
+    try:
+        reg_key = winreg.OpenKey(winreg.HKEY_CURRENT_USER, user_key_path, 0, winreg.KEY_READ)  # type: ignore[reportAttributeAccessIssue]
+        user_path, _ = winreg.QueryValueEx(reg_key, "PATH")  # type: ignore[reportAttributeAccessIssue]
+        winreg.CloseKey(reg_key)  # type: ignore[reportAttributeAccessIssue]
+    except Exception:
+        user_path = ""
+
+    # Read system PATH
+    try:
+        reg_key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, system_key_path, 0, winreg.KEY_READ)  # type: ignore[reportAttributeAccessIssue]
+        system_path, _ = winreg.QueryValueEx(reg_key, "PATH")  # type: ignore[reportAttributeAccessIssue]
+        winreg.CloseKey(reg_key)  # type: ignore[reportAttributeAccessIssue]
+    except Exception:
+        system_path = ""
+
+    # Merge user and system PATHs (system first, then user)
+    if system_path and user_path:
+        new_path = system_path + ";" + user_path
+    elif system_path:
+        new_path = system_path
+    elif user_path:
+        new_path = user_path
+
+    if new_path:
+        os.environ["PATH"] = new_path
+
+
+def get_rclone_version() -> Optional[str]:
+    """Get the installed rclone version."""
+    if not is_rclone_installed():
+        return None
+
+    try:
+        result = run_command(["rclone", "version"], check=False)
+        if result.returncode == 0:
+            # Parse version from output (format: "rclone v1.64.0")
+            lines = result.stdout.strip().split("\n")
+            for line in lines:
+                if line.startswith("rclone v"):
+                    return line.split()[1]
+        return "unknown"
+    except Exception:
+        return "unknown"