atdata 0.2.0a1__py3-none-any.whl → 0.2.3b1__py3-none-any.whl

atdata/cli/__init__.py

@@ -0,0 +1,222 @@
+"""Command-line interface for atdata.
+
+This module provides CLI commands for managing local development infrastructure
+and diagnosing configuration issues.
+
+Commands:
+    atdata local up     Start Redis and MinIO containers for local development
+    atdata local down   Stop local development containers
+    atdata diagnose     Check Redis configuration and connectivity
+    atdata version      Show version information
+
+Example:
+    $ atdata local up
+    Starting Redis on port 6379...
+    Starting MinIO on port 9000...
+    Local infrastructure ready.
+
+    $ atdata diagnose
+    Checking Redis configuration...
+    ✓ Redis connected
+    ✓ Persistence enabled (AOF)
+    ✓ Memory policy: noeviction
+"""
+
+import argparse
+import sys
+from typing import Sequence
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Main entry point for the atdata CLI.
+
+    Args:
+        argv: Command-line arguments. If None, uses sys.argv[1:].
+
+    Returns:
+        Exit code (0 for success, non-zero for errors).
+    """
+    parser = argparse.ArgumentParser(
+        prog="atdata",
+        description="A loose federation of distributed, typed datasets",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--version",
+        "-v",
+        action="store_true",
+        help="Show version information",
+    )
+
+    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+
+    # 'local' command group
+    local_parser = subparsers.add_parser(
+        "local",
+        help="Manage local development infrastructure",
+    )
+    local_subparsers = local_parser.add_subparsers(
+        dest="local_command",
+        help="Local infrastructure commands",
+    )
+
+    # 'local up' command
+    up_parser = local_subparsers.add_parser(
+        "up",
+        help="Start Redis and MinIO containers",
+    )
+    up_parser.add_argument(
+        "--redis-port",
+        type=int,
+        default=6379,
+        help="Redis port (default: 6379)",
+    )
+    up_parser.add_argument(
+        "--minio-port",
+        type=int,
+        default=9000,
+        help="MinIO API port (default: 9000)",
+    )
+    up_parser.add_argument(
+        "--minio-console-port",
+        type=int,
+        default=9001,
+        help="MinIO console port (default: 9001)",
+    )
+    up_parser.add_argument(
+        "--detach",
+        "-d",
+        action="store_true",
+        default=True,
+        help="Run containers in detached mode (default: True)",
+    )
+
+    # 'local down' command
+    down_parser = local_subparsers.add_parser(
+        "down",
+        help="Stop local development containers",
+    )
+    down_parser.add_argument(
+        "--volumes",
+        "-v",
+        action="store_true",
+        help="Also remove volumes (deletes all data)",
+    )
+
+    # 'local status' command
+    local_subparsers.add_parser(
+        "status",
+        help="Show status of local infrastructure",
+    )
+
+    # 'diagnose' command
+    diagnose_parser = subparsers.add_parser(
+        "diagnose",
+        help="Diagnose Redis configuration and connectivity",
+    )
+    diagnose_parser.add_argument(
+        "--host",
+        default="localhost",
+        help="Redis host (default: localhost)",
+    )
+    diagnose_parser.add_argument(
+        "--port",
+        type=int,
+        default=6379,
+        help="Redis port (default: 6379)",
+    )
+
+    # 'version' command (alternative to --version flag)
+    subparsers.add_parser(
+        "version",
+        help="Show version information",
+    )
+
+    args = parser.parse_args(argv)
+
+    # Handle --version flag
+    if args.version or args.command == "version":
+        return _cmd_version()
+
+    # Handle 'local' commands
+    if args.command == "local":
+        if args.local_command == "up":
+            return _cmd_local_up(
+                redis_port=args.redis_port,
+                minio_port=args.minio_port,
+                minio_console_port=args.minio_console_port,
+                detach=args.detach,
+            )
+        elif args.local_command == "down":
+            return _cmd_local_down(remove_volumes=args.volumes)
+        elif args.local_command == "status":
+            return _cmd_local_status()
+        else:
+            local_parser.print_help()
+            return 1
+
+    # Handle 'diagnose' command
+    if args.command == "diagnose":
+        return _cmd_diagnose(host=args.host, port=args.port)
+
+    # No command given
+    parser.print_help()
+    return 0
+
+
+def _cmd_version() -> int:
+    """Show version information."""
+    try:
+        from atdata import __version__
+
+        version = __version__
+    except ImportError:
+        # Fallback to package metadata
+        from importlib.metadata import version as pkg_version
+
+        version = pkg_version("atdata")
+
+    print(f"atdata {version}")
+    return 0
+
+
+def _cmd_local_up(
+    redis_port: int,
+    minio_port: int,
+    minio_console_port: int,
+    detach: bool,
+) -> int:
+    """Start local development infrastructure."""
+    from .local import local_up
+
+    return local_up(
+        redis_port=redis_port,
+        minio_port=minio_port,
+        minio_console_port=minio_console_port,
+        detach=detach,
+    )
+
+
+def _cmd_local_down(remove_volumes: bool) -> int:
+    """Stop local development infrastructure."""
+    from .local import local_down
+
+    return local_down(remove_volumes=remove_volumes)
+
+
+def _cmd_local_status() -> int:
+    """Show status of local infrastructure."""
+    from .local import local_status
+
+    return local_status()
+
+
+def _cmd_diagnose(host: str, port: int) -> int:
+    """Diagnose Redis configuration."""
+    from .diagnose import diagnose_redis
+
+    return diagnose_redis(host=host, port=port)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

atdata/cli/diagnose.py

@@ -0,0 +1,169 @@
+"""Diagnostic tools for atdata infrastructure.
+
+This module provides commands to diagnose configuration issues with Redis
+and other infrastructure components.
+"""
+
+import sys
+
+
+def _print_status(label: str, ok: bool, detail: str = "") -> None:
+    """Print a status line with checkmark or X."""
+    symbol = "✓" if ok else "✗"
+    status = f"{symbol} {label}"
+    if detail:
+        status += f": {detail}"
+    print(status)
+
+
+def diagnose_redis(host: str = "localhost", port: int = 6379) -> int:
+    """Diagnose Redis configuration and connectivity.
+
+    Checks for common issues that can cause data loss:
+    - Connection issues
+    - Persistence settings (AOF/RDB)
+    - Memory eviction policy
+    - Memory usage
+
+    Args:
+        host: Redis host (default: localhost)
+        port: Redis port (default: 6379)
+
+    Returns:
+        Exit code (0 if all checks pass, 1 if any issues found)
+    """
+    print(f"Diagnosing Redis at {host}:{port}...")
+    print()
+
+    issues_found = False
+
+    # Try to connect
+    try:
+        from redis import Redis
+
+        redis = Redis(host=host, port=port, socket_connect_timeout=5)
+        redis.ping()
+        _print_status("Connection", True, "connected")
+    except ImportError:
+        print("Error: redis package not installed", file=sys.stderr)
+        return 1
+    except Exception as e:
+        _print_status("Connection", False, str(e))
+        print()
+        print("Cannot connect to Redis. Make sure Redis is running:")
+        print("  atdata local up")
+        return 1
+
+    # Check Redis version
+    try:
+        info = redis.info()
+        version = info.get("redis_version", "unknown")
+        _print_status("Version", True, version)
+    except Exception as e:
+        _print_status("Version", False, str(e))
+        issues_found = True
+
+    # Check persistence - AOF
+    try:
+        aof_enabled = redis.config_get("appendonly").get("appendonly", "no")
+        aof_ok = aof_enabled == "yes"
+        _print_status(
+            "AOF Persistence",
+            aof_ok,
+            "enabled" if aof_ok else "DISABLED - data may be lost on restart!",
+        )
+        if not aof_ok:
+            issues_found = True
+    except Exception as e:
+        _print_status("AOF Persistence", False, f"check failed: {e}")
+        issues_found = True
+
+    # Check persistence - RDB
+    try:
+        save_config = redis.config_get("save").get("save", "")
+        rdb_ok = bool(save_config and save_config.strip())
+        _print_status(
+            "RDB Persistence",
+            rdb_ok,
+            f"configured ({save_config})" if rdb_ok else "DISABLED",
+        )
+        # RDB disabled is only a warning if AOF is enabled
+    except Exception as e:
+        _print_status("RDB Persistence", False, f"check failed: {e}")
+
+    # Check memory policy
+    try:
+        policy = redis.config_get("maxmemory-policy").get("maxmemory-policy", "unknown")
+        # Safe policies that won't evict index data
+        safe_policies = {
+            "noeviction",
+            "volatile-lru",
+            "volatile-lfu",
+            "volatile-ttl",
+            "volatile-random",
+        }
+        policy_ok = policy in safe_policies
+
+        if policy_ok:
+            _print_status("Memory Policy", True, policy)
+        else:
+            _print_status(
+                "Memory Policy",
+                False,
+                f"{policy} - may evict index data! Use 'noeviction' or 'volatile-*'",
+            )
+            issues_found = True
+    except Exception as e:
+        _print_status("Memory Policy", False, f"check failed: {e}")
+        issues_found = True
+
+    # Check maxmemory setting
+    try:
+        maxmemory = redis.config_get("maxmemory").get("maxmemory", "0")
+        maxmemory_bytes = int(maxmemory)
+        if maxmemory_bytes == 0:
+            _print_status("Max Memory", True, "unlimited")
+        else:
+            maxmemory_mb = maxmemory_bytes / (1024 * 1024)
+            _print_status("Max Memory", True, f"{maxmemory_mb:.0f} MB")
+    except Exception as e:
+        _print_status("Max Memory", False, f"check failed: {e}")
+
+    # Check current memory usage
+    try:
+        memory_info = redis.info("memory")
+        used_memory = memory_info.get("used_memory_human", "unknown")
+        peak_memory = memory_info.get("used_memory_peak_human", "unknown")
+        _print_status("Memory Usage", True, f"{used_memory} (peak: {peak_memory})")
+    except Exception as e:
+        _print_status("Memory Usage", False, f"check failed: {e}")
+
+    # Check number of atdata keys
+    try:
+        dataset_count = 0
+        schema_count = 0
+        for key in redis.scan_iter(match="LocalDatasetEntry:*", count=100):
+            dataset_count += 1
+        for key in redis.scan_iter(match="LocalSchema:*", count=100):
+            schema_count += 1
+        _print_status(
+            "atdata Keys", True, f"{dataset_count} datasets, {schema_count} schemas"
+        )
+    except Exception as e:
+        _print_status("atdata Keys", False, f"check failed: {e}")
+
+    print()
+
+    if issues_found:
+        print("Issues found! Recommended configuration:")
+        print()
+        print("  # In redis.conf or via CONFIG SET:")
+        print("  appendonly yes")
+        print("  maxmemory-policy noeviction")
+        print()
+        print("  # Or use atdata's preconfigured local setup:")
+        print("  atdata local up")
+        return 1
+    else:
+        print("All checks passed. Redis is properly configured for atdata.")
+        return 0

atdata/cli/local.py

@@ -0,0 +1,283 @@
+"""Local infrastructure management for atdata.
+
+This module provides commands to start and stop local development infrastructure:
+- Redis: For index storage and metadata
+- MinIO: S3-compatible object storage for dataset files
+
+The infrastructure runs in Docker containers managed via docker-compose or
+direct docker commands.
+"""
+
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from textwrap import dedent
+
+# Container names for tracking
+REDIS_CONTAINER = "atdata-redis"
+MINIO_CONTAINER = "atdata-minio"
+
+# Docker compose configuration
+COMPOSE_TEMPLATE = dedent("""\
+    version: '3.8'
+
+    services:
+      redis:
+        image: redis:7-alpine
+        container_name: {redis_container}
+        ports:
+          - "{redis_port}:6379"
+        volumes:
+          - atdata-redis-data:/data
+        command: redis-server --appendonly yes --maxmemory-policy noeviction
+        restart: unless-stopped
+        healthcheck:
+          test: ["CMD", "redis-cli", "ping"]
+          interval: 5s
+          timeout: 3s
+          retries: 3
+
+      minio:
+        image: minio/minio:latest
+        container_name: {minio_container}
+        ports:
+          - "{minio_port}:9000"
+          - "{minio_console_port}:9001"
+        volumes:
+          - atdata-minio-data:/data
+        environment:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+        command: server /data --console-address ":9001"
+        restart: unless-stopped
+        healthcheck:
+          test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+          interval: 5s
+          timeout: 3s
+          retries: 3
+
+    volumes:
+      atdata-redis-data:
+      atdata-minio-data:
+""")
+
+
+def _check_docker() -> bool:
+    """Check if Docker is available and running."""
+    if not shutil.which("docker"):
+        print("Error: Docker is not installed or not in PATH", file=sys.stderr)
+        return False
+
+    try:
+        result = subprocess.run(
+            ["docker", "info"],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode != 0:
+            print("Error: Docker daemon is not running", file=sys.stderr)
+            return False
+    except subprocess.TimeoutExpired:
+        print("Error: Docker daemon not responding", file=sys.stderr)
+        return False
+    except Exception as e:
+        print(f"Error checking Docker: {e}", file=sys.stderr)
+        return False
+
+    return True
+
+
+def _get_compose_file(
+    redis_port: int,
+    minio_port: int,
+    minio_console_port: int,
+) -> str:
+    """Generate docker-compose configuration."""
+    return COMPOSE_TEMPLATE.format(
+        redis_container=REDIS_CONTAINER,
+        minio_container=MINIO_CONTAINER,
+        redis_port=redis_port,
+        minio_port=minio_port,
+        minio_console_port=minio_console_port,
+    )
+
+
+def _container_running(name: str) -> bool:
+    """Check if a container is running."""
+    try:
+        result = subprocess.run(
+            ["docker", "inspect", "-f", "{{.State.Running}}", name],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        return result.returncode == 0 and result.stdout.strip() == "true"
+    except Exception:
+        return False
+
+
+def _run_compose(
+    compose_content: str,
+    command: list[str],
+    *,
+    capture_output: bool = False,
+) -> subprocess.CompletedProcess:
+    """Run a docker-compose command with the given configuration."""
+    # Write compose file to temp location
+    compose_dir = Path.home() / ".atdata"
+    compose_dir.mkdir(exist_ok=True)
+    compose_file = compose_dir / "docker-compose.yml"
+    compose_file.write_text(compose_content)
+
+    # Prefer 'docker compose' (v2) over 'docker-compose' (v1)
+    if shutil.which("docker"):
+        # Check if docker compose v2 is available
+        check = subprocess.run(
+            ["docker", "compose", "version"],
+            capture_output=True,
+            timeout=5,
+        )
+        if check.returncode == 0:
+            base_cmd = ["docker", "compose"]
+        elif shutil.which("docker-compose"):
+            base_cmd = ["docker-compose"]
+        else:
+            raise RuntimeError(
+                "Neither 'docker compose' nor 'docker-compose' available"
+            )
+    else:
+        raise RuntimeError("Docker not found")
+
+    full_cmd = base_cmd + ["-f", str(compose_file)] + command
+
+    return subprocess.run(
+        full_cmd,
+        capture_output=capture_output,
+        text=True,
+    )
+
+
+def local_up(
+    redis_port: int = 6379,
+    minio_port: int = 9000,
+    minio_console_port: int = 9001,
+    detach: bool = True,
+) -> int:
+    """Start local development infrastructure.
+
+    Args:
+        redis_port: Port for Redis (default: 6379)
+        minio_port: Port for MinIO API (default: 9000)
+        minio_console_port: Port for MinIO console (default: 9001)
+        detach: Run in background (default: True)
+
+    Returns:
+        Exit code (0 for success)
+    """
+    if not _check_docker():
+        return 1
+
+    print("Starting atdata local infrastructure...")
+
+    compose_content = _get_compose_file(redis_port, minio_port, minio_console_port)
+    command = ["up"]
+    if detach:
+        command.append("-d")
+
+    try:
+        result = _run_compose(compose_content, command)
+        if result.returncode != 0:
+            print("Error: Failed to start containers", file=sys.stderr)
+            return result.returncode
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    # Wait a moment for containers to be healthy
+    import time
+
+    time.sleep(2)
+
+    # Show status
+    print()
+    print("Local infrastructure started:")
+    print(f"  Redis:        localhost:{redis_port}")
+    print(f"  MinIO API:    http://localhost:{minio_port}")
+    print(f"  MinIO Console: http://localhost:{minio_console_port}")
+    print()
+    print("MinIO credentials: minioadmin / minioadmin")
+    print()
+    print("Example usage:")
+    print("  from atdata.local import Index, S3DataStore")
+    print("  ")
+    print("  store = S3DataStore.from_credentials({")
+    print(f"      'AWS_ENDPOINT': 'http://localhost:{minio_port}',")
+    print("      'AWS_ACCESS_KEY_ID': 'minioadmin',")
+    print("      'AWS_SECRET_ACCESS_KEY': 'minioadmin',")
+    print("  }, bucket='datasets')")
+    print("  index = Index(data_store=store)")
+
+    return 0
+
+
+def local_down(remove_volumes: bool = False) -> int:
+    """Stop local development infrastructure.
+
+    Args:
+        remove_volumes: Also remove data volumes (default: False)
+
+    Returns:
+        Exit code (0 for success)
+    """
+    if not _check_docker():
+        return 1
+
+    print("Stopping atdata local infrastructure...")
+
+    # Use default ports for compose file (actual ports don't matter for down)
+    compose_content = _get_compose_file(6379, 9000, 9001)
+    command = ["down"]
+    if remove_volumes:
+        command.append("-v")
+        print("Warning: This will delete all local data!")
+
+    try:
+        result = _run_compose(compose_content, command)
+        if result.returncode != 0:
+            print("Error: Failed to stop containers", file=sys.stderr)
+            return result.returncode
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        return 1
+
+    print("Local infrastructure stopped.")
+    return 0
+
+
+def local_status() -> int:
+    """Show status of local infrastructure.
+
+    Returns:
+        Exit code (0 for success)
+    """
+    if not _check_docker():
+        return 1
+
+    redis_running = _container_running(REDIS_CONTAINER)
+    minio_running = _container_running(MINIO_CONTAINER)
+
+    print("atdata local infrastructure status:")
+    print()
+    print(f"  Redis ({REDIS_CONTAINER}):  {'running' if redis_running else 'stopped'}")
+    print(f"  MinIO ({MINIO_CONTAINER}):  {'running' if minio_running else 'stopped'}")
+
+    if redis_running or minio_running:
+        print()
+        print("To stop: atdata local down")
+    else:
+        print()
+        print("To start: atdata local up")
+
+    return 0