atdata 0.2.3b1__py3-none-any.whl → 0.3.1b1__py3-none-any.whl

atdata/cli/__init__.py

@@ -1,221 +1,207 @@
 """Command-line interface for atdata.
 
-This module provides CLI commands for managing local development infrastructure
-and diagnosing configuration issues.
+This module provides CLI commands for managing development infrastructure,
+inspecting datasets, and diagnosing configuration issues.
 
 Commands:
-    atdata local up     Start Redis and MinIO containers for local development
-    atdata local down   Stop local development containers
-    atdata diagnose     Check Redis configuration and connectivity
-    atdata version      Show version information
-
-Example:
-    $ atdata local up
-    Starting Redis on port 6379...
-    Starting MinIO on port 9000...
-    Local infrastructure ready.
-
-    $ atdata diagnose
-    Checking Redis configuration...
-    ✓ Redis connected
-    ✓ Persistence enabled (AOF)
-    ✓ Memory policy: noeviction
+    atdata infra up      Start Redis and MinIO containers for development
+    atdata infra down    Stop development containers
+    atdata infra status  Show status of infrastructure
+    atdata diagnose      Check Redis configuration and connectivity
+    atdata inspect       Show dataset summary information
+    atdata schema show   Display dataset schema
+    atdata schema diff   Compare two dataset schemas
+    atdata preview       Preview first N samples of a dataset
+    atdata version       Show version information
 """
 
-import argparse
 import sys
-from typing import Sequence
 
+import typer
 
-def main(argv: Sequence[str] | None = None) -> int:
-    """Main entry point for the atdata CLI.
+# ---------------------------------------------------------------------------
+# App hierarchy
+# ---------------------------------------------------------------------------
 
-    Args:
-        argv: Command-line arguments. If None, uses sys.argv[1:].
+app = typer.Typer(
+    name="atdata",
+    help="A loose federation of distributed, typed datasets.",
+    add_completion=False,
+    no_args_is_help=True,
+)
 
-    Returns:
-        Exit code (0 for success, non-zero for errors).
-    """
-    parser = argparse.ArgumentParser(
-        prog="atdata",
-        description="A loose federation of distributed, typed datasets",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "--version",
-        "-v",
-        action="store_true",
-        help="Show version information",
-    )
+infra_app = typer.Typer(
+    name="infra",
+    help="Manage development infrastructure.",
+    no_args_is_help=True,
+)
+app.add_typer(infra_app, name="infra")
 
-    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+schema_app = typer.Typer(
+    name="schema",
+    help="Show or compare dataset schemas.",
+    no_args_is_help=True,
+)
+app.add_typer(schema_app, name="schema")
 
-    # 'local' command group
-    local_parser = subparsers.add_parser(
-        "local",
-        help="Manage local development infrastructure",
-    )
-    local_subparsers = local_parser.add_subparsers(
-        dest="local_command",
-        help="Local infrastructure commands",
-    )
 
-    # 'local up' command
-    up_parser = local_subparsers.add_parser(
-        "up",
-        help="Start Redis and MinIO containers",
-    )
-    up_parser.add_argument(
-        "--redis-port",
-        type=int,
-        default=6379,
-        help="Redis port (default: 6379)",
-    )
-    up_parser.add_argument(
-        "--minio-port",
-        type=int,
-        default=9000,
-        help="MinIO API port (default: 9000)",
-    )
-    up_parser.add_argument(
-        "--minio-console-port",
-        type=int,
-        default=9001,
-        help="MinIO console port (default: 9001)",
-    )
-    up_parser.add_argument(
-        "--detach",
-        "-d",
-        action="store_true",
-        default=True,
-        help="Run containers in detached mode (default: True)",
-    )
+# ---------------------------------------------------------------------------
+# Top-level commands
+# ---------------------------------------------------------------------------
 
-    # 'local down' command
-    down_parser = local_subparsers.add_parser(
-        "down",
-        help="Stop local development containers",
-    )
-    down_parser.add_argument(
-        "--volumes",
-        "-v",
-        action="store_true",
-        help="Also remove volumes (deletes all data)",
-    )
 
-    # 'local status' command
-    local_subparsers.add_parser(
-        "status",
-        help="Show status of local infrastructure",
-    )
+@app.command()
+def version() -> None:
+    """Show version information."""
+    try:
+        from atdata import __version__
 
-    # 'diagnose' command
-    diagnose_parser = subparsers.add_parser(
-        "diagnose",
-        help="Diagnose Redis configuration and connectivity",
-    )
-    diagnose_parser.add_argument(
-        "--host",
-        default="localhost",
-        help="Redis host (default: localhost)",
-    )
-    diagnose_parser.add_argument(
-        "--port",
-        type=int,
-        default=6379,
-        help="Redis port (default: 6379)",
-    )
+        ver = __version__
+    except ImportError:
+        from importlib.metadata import version as pkg_version
 
-    # 'version' command (alternative to --version flag)
-    subparsers.add_parser(
-        "version",
-        help="Show version information",
-    )
+        ver = pkg_version("atdata")
 
-    args = parser.parse_args(argv)
-
-    # Handle --version flag
-    if args.version or args.command == "version":
-        return _cmd_version()
-
-    # Handle 'local' commands
-    if args.command == "local":
-        if args.local_command == "up":
-            return _cmd_local_up(
-                redis_port=args.redis_port,
-                minio_port=args.minio_port,
-                minio_console_port=args.minio_console_port,
-                detach=args.detach,
-            )
-        elif args.local_command == "down":
-            return _cmd_local_down(remove_volumes=args.volumes)
-        elif args.local_command == "status":
-            return _cmd_local_status()
-        else:
-            local_parser.print_help()
-            return 1
+    print(f"atdata {ver}")
 
-    # Handle 'diagnose' command
-    if args.command == "diagnose":
-        return _cmd_diagnose(host=args.host, port=args.port)
 
-    # No command given
-    parser.print_help()
-    return 0
+@app.command()
+def inspect(
+    url: str = typer.Argument(help="Dataset URL, local path, or atmosphere URI"),
+) -> None:
+    """Show dataset summary (sample count, schema, shards)."""
+    from .inspect import inspect_dataset
 
+    code = inspect_dataset(url=url)
+    raise typer.Exit(code=code)
 
-def _cmd_version() -> int:
-    """Show version information."""
-    try:
-        from atdata import __version__
 
-        version = __version__
-    except ImportError:
-        # Fallback to package metadata
-        from importlib.metadata import version as pkg_version
+@app.command()
+def preview(
+    url: str = typer.Argument(help="Dataset URL, local path, or atmosphere URI"),
+    limit: int = typer.Option(5, help="Number of samples to preview."),
+) -> None:
+    """Preview first N samples of a dataset."""
+    from .preview import preview_dataset
+
+    code = preview_dataset(url=url, limit=limit)
+    raise typer.Exit(code=code)
+
+
+@app.command()
+def diagnose(
+    host: str = typer.Option("localhost", help="Redis host."),
+    port: int = typer.Option(6379, help="Redis port."),
+) -> None:
+    """Diagnose Redis configuration and connectivity."""
+    from .diagnose import diagnose_redis
 
-        version = pkg_version("atdata")
+    code = diagnose_redis(host=host, port=port)
+    raise typer.Exit(code=code)
 
-    print(f"atdata {version}")
-    return 0
 
+# ---------------------------------------------------------------------------
+# infra sub-commands
+# ---------------------------------------------------------------------------
 
-def _cmd_local_up(
-    redis_port: int,
-    minio_port: int,
-    minio_console_port: int,
-    detach: bool,
-) -> int:
-    """Start local development infrastructure."""
-    from .local import local_up
 
-    return local_up(
+@infra_app.command()
+def up(
+    redis_port: int = typer.Option(6379, help="Redis port."),
+    minio_port: int = typer.Option(9000, help="MinIO API port."),
+    minio_console_port: int = typer.Option(9001, help="MinIO console port."),
+    detach: bool = typer.Option(
+        True, "--detach", "-d", help="Run containers in detached mode."
+    ),
+) -> None:
+    """Start Redis and MinIO containers."""
+    from .infra import local_up
+
+    code = local_up(
         redis_port=redis_port,
         minio_port=minio_port,
         minio_console_port=minio_console_port,
         detach=detach,
     )
+    raise typer.Exit(code=code)
 
 
-def _cmd_local_down(remove_volumes: bool) -> int:
-    """Stop local development infrastructure."""
-    from .local import local_down
+@infra_app.command()
+def down(
+    volumes: bool = typer.Option(
+        False, "--volumes", "-v", help="Also remove volumes (deletes all data)."
+    ),
+) -> None:
+    """Stop local development containers."""
+    from .infra import local_down
 
-    return local_down(remove_volumes=remove_volumes)
+    code = local_down(remove_volumes=volumes)
+    raise typer.Exit(code=code)
 
 
-def _cmd_local_status() -> int:
-    """Show status of local infrastructure."""
-    from .local import local_status
+@infra_app.command()
+def status() -> None:
+    """Show status of infrastructure."""
+    from .infra import local_status
 
-    return local_status()
+    code = local_status()
+    raise typer.Exit(code=code)
 
 
-def _cmd_diagnose(host: str, port: int) -> int:
-    """Diagnose Redis configuration."""
-    from .diagnose import diagnose_redis
+# ---------------------------------------------------------------------------
+# schema sub-commands
+# ---------------------------------------------------------------------------
+
+
+@schema_app.command("show")
+def schema_show(
+    dataset_ref: str = typer.Argument(
+        help="Dataset URL, local path, or index reference."
+    ),
+) -> None:
+    """Display dataset schema."""
+    from .schema import schema_show as _schema_show
+
+    code = _schema_show(dataset_ref=dataset_ref)
+    raise typer.Exit(code=code)
+
+
+@schema_app.command("diff")
+def schema_diff(
+    url_a: str = typer.Argument(help="First dataset URL."),
+    url_b: str = typer.Argument(help="Second dataset URL."),
+) -> None:
+    """Compare two dataset schemas."""
+    from .schema import schema_diff as _schema_diff
 
-    return diagnose_redis(host=host, port=port)
+    code = _schema_diff(url_a=url_a, url_b=url_b)
+    raise typer.Exit(code=code)
+
+
+# ---------------------------------------------------------------------------
+# Entrypoint
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main entry point for the atdata CLI.
+
+    Args:
+        argv: Command-line arguments. If None, uses sys.argv[1:].
+
+    Returns:
+        Exit code (0 for success, non-zero for errors).
+    """
+    try:
+        if argv is not None:
+            app(args=argv, standalone_mode=False)
+        else:
+            app(standalone_mode=False)
+        return 0
+    except SystemExit as exc:
+        return exc.code if isinstance(exc.code, int) else 0
+    except Exception:
+        return 1
 
 
 if __name__ == "__main__":

atdata/cli/diagnose.py

@@ -51,7 +51,7 @@ def diagnose_redis(host: str = "localhost", port: int = 6379) -> int:
         _print_status("Connection", False, str(e))
         print()
         print("Cannot connect to Redis. Make sure Redis is running:")
-        print("  atdata local up")
+        print("  atdata infra up")
         return 1
 
     # Check Redis version
@@ -162,7 +162,7 @@ def diagnose_redis(host: str = "localhost", port: int = 6379) -> int:
         print("  maxmemory-policy noeviction")
         print()
         print("  # Or use atdata's preconfigured local setup:")
-        print("  atdata local up")
+        print("  atdata infra up")
         return 1
     else:
         print("All checks passed. Redis is properly configured for atdata.")

atdata/cli/{local.py → infra.py}

@@ -1,6 +1,6 @@
-"""Local infrastructure management for atdata.
+"""Infrastructure management for atdata.
 
-This module provides commands to start and stop local development infrastructure:
+This module provides commands to start and stop development infrastructure:
 - Redis: For index storage and metadata
 - MinIO: S3-compatible object storage for dataset files
 
@@ -114,7 +114,7 @@ def _container_running(name: str) -> bool:
             timeout=5,
         )
         return result.returncode == 0 and result.stdout.strip() == "true"
-    except Exception:
+    except (OSError, subprocess.SubprocessError):
         return False
 
 
@@ -179,7 +179,7 @@ def local_up(
     if not _check_docker():
         return 1
 
-    print("Starting atdata local infrastructure...")
+    print("Starting atdata infrastructure...")
 
     compose_content = _get_compose_file(redis_port, minio_port, minio_console_port)
     command = ["up"]
@@ -202,7 +202,7 @@ def local_up(
 
     # Show status
     print()
-    print("Local infrastructure started:")
+    print("Infrastructure started:")
     print(f"  Redis:        localhost:{redis_port}")
     print(f"  MinIO API:    http://localhost:{minio_port}")
     print(f"  MinIO Console: http://localhost:{minio_console_port}")
@@ -210,7 +210,7 @@ def local_up(
     print("MinIO credentials: minioadmin / minioadmin")
     print()
     print("Example usage:")
-    print("  from atdata.local import Index, S3DataStore")
+    print("  from atdata.stores import S3DataStore")
     print("  ")
     print("  store = S3DataStore.from_credentials({")
     print(f"      'AWS_ENDPOINT': 'http://localhost:{minio_port}',")
@@ -234,7 +234,7 @@ def local_down(remove_volumes: bool = False) -> int:
     if not _check_docker():
         return 1
 
-    print("Stopping atdata local infrastructure...")
+    print("Stopping atdata infrastructure...")
 
     # Use default ports for compose file (actual ports don't matter for down)
     compose_content = _get_compose_file(6379, 9000, 9001)
@@ -252,7 +252,7 @@ def local_down(remove_volumes: bool = False) -> int:
         print(f"Error: {e}", file=sys.stderr)
         return 1
 
-    print("Local infrastructure stopped.")
+    print("Infrastructure stopped.")
     return 0
 
 
@@ -268,16 +268,16 @@ def local_status() -> int:
     redis_running = _container_running(REDIS_CONTAINER)
     minio_running = _container_running(MINIO_CONTAINER)
 
-    print("atdata local infrastructure status:")
+    print("atdata infrastructure status:")
     print()
     print(f"  Redis ({REDIS_CONTAINER}):  {'running' if redis_running else 'stopped'}")
     print(f"  MinIO ({MINIO_CONTAINER}):  {'running' if minio_running else 'stopped'}")
 
     if redis_running or minio_running:
         print()
-        print("To stop: atdata local down")
+        print("To stop: atdata infra down")
     else:
         print()
-        print("To start: atdata local up")
+        print("To start: atdata infra up")
 
     return 0

atdata/cli/inspect.py

@@ -0,0 +1,69 @@
+"""``atdata inspect`` command — show dataset summary information."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+
+def inspect_dataset(url: str) -> int:
+    """Print summary information for a dataset at the given URL.
+
+    Args:
+        url: WebDataset URL, local path, or atmosphere URI.
+
+    Returns:
+        Exit code (0 success, 1 failure).
+    """
+    try:
+        from ..dataset import Dataset, DictSample
+
+        ds = Dataset[DictSample](url)
+    except Exception as exc:
+        print(f"Error opening dataset: {exc}", file=sys.stderr)
+        return 1
+
+    try:
+        shards = ds.list_shards()
+        print(f"URL:      {url}")
+        print(f"Shards:   {len(shards)}")
+        for shard in shards:
+            print(f"  - {shard}")
+
+        # Read first sample to infer schema
+        samples = ds.head(1)
+        if samples:
+            sample = samples[0]
+            print("Schema:   (inferred from first sample)")
+            for key in sample.keys():
+                val = sample[key]
+                print(f"  {key}: {_describe_value(val)}")
+        else:
+            print("Schema:   (no samples found)")
+
+        # Count samples — scan all shards
+        count = sum(1 for _ in ds.ordered())
+        print(f"Samples:  {count}")
+    except Exception as exc:
+        print(f"Error reading dataset: {exc}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+def _describe_value(val: Any) -> str:
+    """Human-readable type description for a sample field value."""
+    import numpy as np
+
+    if isinstance(val, np.ndarray):
+        return f"ndarray dtype={val.dtype} shape={val.shape}"
+    if isinstance(val, bytes):
+        return f"bytes len={len(val)}"
+    if isinstance(val, str):
+        truncated = val[:60] + ("..." if len(val) > 60 else "")
+        return f'str "{truncated}"'
+    if isinstance(val, (int, float, bool)):
+        return f"{type(val).__name__} {val}"
+    if isinstance(val, list):
+        return f"list len={len(val)}"
+    return type(val).__name__

atdata/cli/preview.py

@@ -0,0 +1,63 @@
+"""``atdata preview`` command — render first N samples of a dataset."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+
+def preview_dataset(url: str, limit: int = 5) -> int:
+    """Print a human-readable preview of the first *limit* samples.
+
+    Args:
+        url: Dataset URL, local path, or atmosphere URI.
+        limit: Number of samples to show. Default: 5.
+
+    Returns:
+        Exit code (0 success, 1 failure).
+    """
+    try:
+        from ..dataset import Dataset, DictSample
+
+        ds = Dataset[DictSample](url)
+    except Exception as exc:
+        print(f"Error opening dataset: {exc}", file=sys.stderr)
+        return 1
+
+    samples = ds.head(limit)
+    if not samples:
+        print("No samples found.", file=sys.stderr)
+        return 1
+
+    print(f"Preview of {url} ({len(samples)} sample(s)):")
+    print()
+
+    for i, sample in enumerate(samples):
+        print(f"--- Sample {i} ---")
+        for key in sample.keys():
+            val = sample[key]
+            print(f"  {key}: {_format_value(val)}")
+        print()
+
+    return 0
+
+
+def _format_value(val: Any) -> str:
+    """Format a value for preview, truncating large data."""
+    import numpy as np
+
+    if isinstance(val, np.ndarray):
+        return f"ndarray shape={val.shape} dtype={val.dtype}"
+    if isinstance(val, bytes):
+        if len(val) <= 40:
+            return repr(val)
+        return f"bytes[{len(val)}] {val[:20]!r}..."
+    if isinstance(val, str):
+        if len(val) <= 80:
+            return repr(val)
+        return repr(val[:77] + "...")
+    if isinstance(val, list):
+        if len(val) <= 5:
+            return repr(val)
+        return f"[{val[0]!r}, {val[1]!r}, ... ({len(val)} items)]"
+    return repr(val)