atdata 0.2.3b1__py3-none-any.whl → 0.3.0b1__py3-none-any.whl

atdata/_schema_codec.py

@@ -28,13 +28,14 @@ import hashlib
 
 from numpy.typing import NDArray
 
-# Import PackableSample for inheritance
+# Import PackableSample for inheritance in dynamic class generation
 from .dataset import PackableSample
+from ._protocols import Packable
 
 
 # Type cache to avoid regenerating identical types
 # Uses insertion order (Python 3.7+) for simple FIFO eviction
-_type_cache: dict[str, Type[PackableSample]] = {}
+_type_cache: dict[str, Type[Packable]] = {}
 _TYPE_CACHE_MAX_SIZE = 256
 
 
@@ -130,7 +131,7 @@ def schema_to_type(
     schema: dict,
     *,
     use_cache: bool = True,
-) -> Type[PackableSample]:
+) -> Type[Packable]:
     """Generate a PackableSample subclass from a schema record.
 
     This function dynamically creates a dataclass that inherits from PackableSample,
@@ -420,7 +421,7 @@ def clear_type_cache() -> None:
     _type_cache.clear()
 
 
-def get_cached_types() -> dict[str, Type[PackableSample]]:
+def get_cached_types() -> dict[str, Type[Packable]]:
     """Get a copy of the current type cache.
 
     Returns:

atdata/_type_utils.py

@@ -45,9 +45,13 @@ def numpy_dtype_to_string(dtype: Any) -> str:
         Schema dtype string (e.g., "float32", "int64"). Defaults to "float32".
     """
     dtype_str = str(dtype)
-    for key, value in NUMPY_DTYPE_MAP.items():
+    # Exact match first (handles "float32", "int64", etc.)
+    if dtype_str in NUMPY_DTYPE_MAP:
+        return NUMPY_DTYPE_MAP[dtype_str]
+    # Substring match, longest keys first to avoid "int8" matching "uint8"
+    for key in sorted(NUMPY_DTYPE_MAP, key=len, reverse=True):
         if key in dtype_str:
-            return value
+            return NUMPY_DTYPE_MAP[key]
     return "float32"
 
 
@@ -102,3 +106,25 @@ def extract_ndarray_dtype(python_type: Any) -> str:
         if dtype_arg is not None:
             return numpy_dtype_to_string(dtype_arg)
     return "float32"
+
+
+def parse_semver(version: str) -> tuple[int, int, int]:
+    """Parse a semantic version string into a comparable tuple.
+
+    Args:
+        version: A ``"major.minor.patch"`` version string.
+
+    Returns:
+        Tuple of (major, minor, patch) integers.
+
+    Raises:
+        ValueError: If the version string is not valid semver.
+
+    Examples:
+        >>> parse_semver("1.2.3")
+        (1, 2, 3)
+    """
+    parts = version.split(".")
+    if len(parts) != 3:
+        raise ValueError(f"Invalid semver: {version}")
+    return int(parts[0]), int(parts[1]), int(parts[2])

atdata/atmosphere/__init__.py

@@ -99,23 +99,25 @@ class AtmosphereIndexEntry:
 class AtmosphereIndex:
     """ATProto index implementing AbstractIndex protocol.
 
+    .. deprecated::
+        Use ``atdata.Index(atmosphere=client)`` instead.  ``AtmosphereIndex``
+        is retained for backwards compatibility and will be removed in a
+        future release.
+
     Wraps SchemaPublisher/Loader and DatasetPublisher/Loader to provide
-    a unified interface compatible with LocalIndex.
+    a unified interface compatible with Index.
 
     Optionally accepts a ``PDSBlobStore`` for writing dataset shards as
     ATProto blobs, enabling fully decentralized dataset storage.
 
     Examples:
-        >>> client = AtmosphereClient()
-        >>> client.login("handle.bsky.social", "app-password")
+        >>> # Preferred: use unified Index
+        >>> from atdata.local import Index
+        >>> from atdata.atmosphere import AtmosphereClient
+        >>> index = Index(atmosphere=client)
         >>>
-        >>> # Without blob storage (external URLs only)
+        >>> # Legacy (deprecated)
         >>> index = AtmosphereIndex(client)
-        >>>
-        >>> # With PDS blob storage
-        >>> store = PDSBlobStore(client)
-        >>> index = AtmosphereIndex(client, data_store=store)
-        >>> entry = index.insert_dataset(dataset, name="my-data")
     """
 
     def __init__(
@@ -131,6 +133,14 @@ class AtmosphereIndex:
             data_store: Optional PDSBlobStore for writing shards as blobs.
                 If provided, insert_dataset will upload shards to PDS.
         """
+        import warnings
+
+        warnings.warn(
+            "AtmosphereIndex is deprecated. Use atdata.Index(atmosphere=client) "
+            "instead for unified index access.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self.client = client
         self._schema_publisher = SchemaPublisher(client)
         self._schema_loader = SchemaLoader(client)

atdata/atmosphere/records.py

@@ -21,9 +21,10 @@ from ._types import (
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from ..dataset import PackableSample, Dataset
+    from ..dataset import Dataset
+    from .._protocols import Packable
 
-ST = TypeVar("ST", bound="PackableSample")
+ST = TypeVar("ST", bound="Packable")
 
 
 class DatasetPublisher:

atdata/atmosphere/schema.py

@@ -26,9 +26,9 @@ from .._type_utils import (
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from ..dataset import PackableSample
+    from .._protocols import Packable
 
-ST = TypeVar("ST", bound="PackableSample")
+ST = TypeVar("ST", bound="Packable")
 
 
 class SchemaPublisher:

atdata/cli/__init__.py

@@ -1,221 +1,207 @@
 """Command-line interface for atdata.
 
-This module provides CLI commands for managing local development infrastructure
-and diagnosing configuration issues.
+This module provides CLI commands for managing local development infrastructure,
+inspecting datasets, and diagnosing configuration issues.
 
 Commands:
-    atdata local up     Start Redis and MinIO containers for local development
-    atdata local down   Stop local development containers
-    atdata diagnose     Check Redis configuration and connectivity
-    atdata version      Show version information
-
-Example:
-    $ atdata local up
-    Starting Redis on port 6379...
-    Starting MinIO on port 9000...
-    Local infrastructure ready.
-
-    $ atdata diagnose
-    Checking Redis configuration...
-    ✓ Redis connected
-    ✓ Persistence enabled (AOF)
-    ✓ Memory policy: noeviction
+    atdata local up      Start Redis and MinIO containers for local development
+    atdata local down    Stop local development containers
+    atdata local status  Show status of local infrastructure
+    atdata diagnose      Check Redis configuration and connectivity
+    atdata inspect       Show dataset summary information
+    atdata schema show   Display dataset schema
+    atdata schema diff   Compare two dataset schemas
+    atdata preview       Preview first N samples of a dataset
+    atdata version       Show version information
 """
 
-import argparse
 import sys
-from typing import Sequence
 
+import typer
 
-def main(argv: Sequence[str] | None = None) -> int:
-    """Main entry point for the atdata CLI.
+# ---------------------------------------------------------------------------
+# App hierarchy
+# ---------------------------------------------------------------------------
 
-    Args:
-        argv: Command-line arguments. If None, uses sys.argv[1:].
+app = typer.Typer(
+    name="atdata",
+    help="A loose federation of distributed, typed datasets.",
+    add_completion=False,
+    no_args_is_help=True,
+)
 
-    Returns:
-        Exit code (0 for success, non-zero for errors).
-    """
-    parser = argparse.ArgumentParser(
-        prog="atdata",
-        description="A loose federation of distributed, typed datasets",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "--version",
-        "-v",
-        action="store_true",
-        help="Show version information",
-    )
+local_app = typer.Typer(
+    name="local",
+    help="Manage local development infrastructure.",
+    no_args_is_help=True,
+)
+app.add_typer(local_app, name="local")
 
-    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+schema_app = typer.Typer(
+    name="schema",
+    help="Show or compare dataset schemas.",
+    no_args_is_help=True,
+)
+app.add_typer(schema_app, name="schema")
 
-    # 'local' command group
-    local_parser = subparsers.add_parser(
-        "local",
-        help="Manage local development infrastructure",
-    )
-    local_subparsers = local_parser.add_subparsers(
-        dest="local_command",
-        help="Local infrastructure commands",
-    )
 
-    # 'local up' command
-    up_parser = local_subparsers.add_parser(
-        "up",
-        help="Start Redis and MinIO containers",
-    )
-    up_parser.add_argument(
-        "--redis-port",
-        type=int,
-        default=6379,
-        help="Redis port (default: 6379)",
-    )
-    up_parser.add_argument(
-        "--minio-port",
-        type=int,
-        default=9000,
-        help="MinIO API port (default: 9000)",
-    )
-    up_parser.add_argument(
-        "--minio-console-port",
-        type=int,
-        default=9001,
-        help="MinIO console port (default: 9001)",
-    )
-    up_parser.add_argument(
-        "--detach",
-        "-d",
-        action="store_true",
-        default=True,
-        help="Run containers in detached mode (default: True)",
-    )
+# ---------------------------------------------------------------------------
+# Top-level commands
+# ---------------------------------------------------------------------------
 
-    # 'local down' command
-    down_parser = local_subparsers.add_parser(
-        "down",
-        help="Stop local development containers",
-    )
-    down_parser.add_argument(
-        "--volumes",
-        "-v",
-        action="store_true",
-        help="Also remove volumes (deletes all data)",
-    )
 
-    # 'local status' command
-    local_subparsers.add_parser(
-        "status",
-        help="Show status of local infrastructure",
-    )
+@app.command()
+def version() -> None:
+    """Show version information."""
+    try:
+        from atdata import __version__
 
-    # 'diagnose' command
-    diagnose_parser = subparsers.add_parser(
-        "diagnose",
-        help="Diagnose Redis configuration and connectivity",
-    )
-    diagnose_parser.add_argument(
-        "--host",
-        default="localhost",
-        help="Redis host (default: localhost)",
-    )
-    diagnose_parser.add_argument(
-        "--port",
-        type=int,
-        default=6379,
-        help="Redis port (default: 6379)",
-    )
+        ver = __version__
+    except ImportError:
+        from importlib.metadata import version as pkg_version
 
-    # 'version' command (alternative to --version flag)
-    subparsers.add_parser(
-        "version",
-        help="Show version information",
-    )
+        ver = pkg_version("atdata")
 
-    args = parser.parse_args(argv)
-
-    # Handle --version flag
-    if args.version or args.command == "version":
-        return _cmd_version()
-
-    # Handle 'local' commands
-    if args.command == "local":
-        if args.local_command == "up":
-            return _cmd_local_up(
-                redis_port=args.redis_port,
-                minio_port=args.minio_port,
-                minio_console_port=args.minio_console_port,
-                detach=args.detach,
-            )
-        elif args.local_command == "down":
-            return _cmd_local_down(remove_volumes=args.volumes)
-        elif args.local_command == "status":
-            return _cmd_local_status()
-        else:
-            local_parser.print_help()
-            return 1
+    print(f"atdata {ver}")
 
-    # Handle 'diagnose' command
-    if args.command == "diagnose":
-        return _cmd_diagnose(host=args.host, port=args.port)
 
-    # No command given
-    parser.print_help()
-    return 0
+@app.command()
+def inspect(
+    url: str = typer.Argument(help="Dataset URL, local path, or atmosphere URI"),
+) -> None:
+    """Show dataset summary (sample count, schema, shards)."""
+    from .inspect import inspect_dataset
 
+    code = inspect_dataset(url=url)
+    raise typer.Exit(code=code)
 
-def _cmd_version() -> int:
-    """Show version information."""
-    try:
-        from atdata import __version__
 
-        version = __version__
-    except ImportError:
-        # Fallback to package metadata
-        from importlib.metadata import version as pkg_version
+@app.command()
+def preview(
+    url: str = typer.Argument(help="Dataset URL, local path, or atmosphere URI"),
+    limit: int = typer.Option(5, help="Number of samples to preview."),
+) -> None:
+    """Preview first N samples of a dataset."""
+    from .preview import preview_dataset
+
+    code = preview_dataset(url=url, limit=limit)
+    raise typer.Exit(code=code)
+
+
+@app.command()
+def diagnose(
+    host: str = typer.Option("localhost", help="Redis host."),
+    port: int = typer.Option(6379, help="Redis port."),
+) -> None:
+    """Diagnose Redis configuration and connectivity."""
+    from .diagnose import diagnose_redis
 
-        version = pkg_version("atdata")
+    code = diagnose_redis(host=host, port=port)
+    raise typer.Exit(code=code)
 
-    print(f"atdata {version}")
-    return 0
 
+# ---------------------------------------------------------------------------
+# local sub-commands
+# ---------------------------------------------------------------------------
 
-def _cmd_local_up(
-    redis_port: int,
-    minio_port: int,
-    minio_console_port: int,
-    detach: bool,
-) -> int:
-    """Start local development infrastructure."""
+
+@local_app.command()
+def up(
+    redis_port: int = typer.Option(6379, help="Redis port."),
+    minio_port: int = typer.Option(9000, help="MinIO API port."),
+    minio_console_port: int = typer.Option(9001, help="MinIO console port."),
+    detach: bool = typer.Option(
+        True, "--detach", "-d", help="Run containers in detached mode."
+    ),
+) -> None:
+    """Start Redis and MinIO containers."""
     from .local import local_up
 
-    return local_up(
+    code = local_up(
         redis_port=redis_port,
         minio_port=minio_port,
         minio_console_port=minio_console_port,
         detach=detach,
     )
+    raise typer.Exit(code=code)
 
 
-def _cmd_local_down(remove_volumes: bool) -> int:
-    """Stop local development infrastructure."""
+@local_app.command()
+def down(
+    volumes: bool = typer.Option(
+        False, "--volumes", "-v", help="Also remove volumes (deletes all data)."
+    ),
+) -> None:
+    """Stop local development containers."""
     from .local import local_down
 
-    return local_down(remove_volumes=remove_volumes)
+    code = local_down(remove_volumes=volumes)
+    raise typer.Exit(code=code)
 
 
-def _cmd_local_status() -> int:
+@local_app.command()
+def status() -> None:
     """Show status of local infrastructure."""
     from .local import local_status
 
-    return local_status()
+    code = local_status()
+    raise typer.Exit(code=code)
 
 
-def _cmd_diagnose(host: str, port: int) -> int:
-    """Diagnose Redis configuration."""
-    from .diagnose import diagnose_redis
+# ---------------------------------------------------------------------------
+# schema sub-commands
+# ---------------------------------------------------------------------------
+
+
+@schema_app.command("show")
+def schema_show(
+    dataset_ref: str = typer.Argument(
+        help="Dataset URL, local path, or index reference."
+    ),
+) -> None:
+    """Display dataset schema."""
+    from .schema import schema_show as _schema_show
+
+    code = _schema_show(dataset_ref=dataset_ref)
+    raise typer.Exit(code=code)
+
+
+@schema_app.command("diff")
+def schema_diff(
+    url_a: str = typer.Argument(help="First dataset URL."),
+    url_b: str = typer.Argument(help="Second dataset URL."),
+) -> None:
+    """Compare two dataset schemas."""
+    from .schema import schema_diff as _schema_diff
 
-    return diagnose_redis(host=host, port=port)
+    code = _schema_diff(url_a=url_a, url_b=url_b)
+    raise typer.Exit(code=code)
+
+
+# ---------------------------------------------------------------------------
+# Entrypoint
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main entry point for the atdata CLI.
+
+    Args:
+        argv: Command-line arguments. If None, uses sys.argv[1:].
+
+    Returns:
+        Exit code (0 for success, non-zero for errors).
+    """
+    try:
+        if argv is not None:
+            app(args=argv, standalone_mode=False)
+        else:
+            app(standalone_mode=False)
+        return 0
+    except SystemExit as exc:
+        return exc.code if isinstance(exc.code, int) else 0
+    except Exception:
+        return 1
 
 
 if __name__ == "__main__":

atdata/cli/inspect.py

@@ -0,0 +1,69 @@
+"""``atdata inspect`` command — show dataset summary information."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+
+def inspect_dataset(url: str) -> int:
+    """Print summary information for a dataset at the given URL.
+
+    Args:
+        url: WebDataset URL, local path, or atmosphere URI.
+
+    Returns:
+        Exit code (0 success, 1 failure).
+    """
+    try:
+        from ..dataset import Dataset, DictSample
+
+        ds = Dataset[DictSample](url)
+    except Exception as exc:
+        print(f"Error opening dataset: {exc}", file=sys.stderr)
+        return 1
+
+    try:
+        shards = ds.list_shards()
+        print(f"URL:      {url}")
+        print(f"Shards:   {len(shards)}")
+        for shard in shards:
+            print(f"  - {shard}")
+
+        # Read first sample to infer schema
+        samples = ds.head(1)
+        if samples:
+            sample = samples[0]
+            print("Schema:   (inferred from first sample)")
+            for key in sample.keys():
+                val = sample[key]
+                print(f"  {key}: {_describe_value(val)}")
+        else:
+            print("Schema:   (no samples found)")
+
+        # Count samples — scan all shards
+        count = sum(1 for _ in ds.ordered())
+        print(f"Samples:  {count}")
+    except Exception as exc:
+        print(f"Error reading dataset: {exc}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+def _describe_value(val: Any) -> str:
+    """Human-readable type description for a sample field value."""
+    import numpy as np
+
+    if isinstance(val, np.ndarray):
+        return f"ndarray dtype={val.dtype} shape={val.shape}"
+    if isinstance(val, bytes):
+        return f"bytes len={len(val)}"
+    if isinstance(val, str):
+        truncated = val[:60] + ("..." if len(val) > 60 else "")
+        return f'str "{truncated}"'
+    if isinstance(val, (int, float, bool)):
+        return f"{type(val).__name__} {val}"
+    if isinstance(val, list):
+        return f"list len={len(val)}"
+    return type(val).__name__

atdata/cli/local.py

@@ -114,7 +114,7 @@ def _container_running(name: str) -> bool:
             timeout=5,
         )
         return result.returncode == 0 and result.stdout.strip() == "true"
-    except Exception:
+    except (OSError, subprocess.SubprocessError):
         return False
 
 

atdata/cli/preview.py

@@ -0,0 +1,63 @@
+"""``atdata preview`` command — render first N samples of a dataset."""
+
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+
+def preview_dataset(url: str, limit: int = 5) -> int:
+    """Print a human-readable preview of the first *limit* samples.
+
+    Args:
+        url: Dataset URL, local path, or atmosphere URI.
+        limit: Number of samples to show. Default: 5.
+
+    Returns:
+        Exit code (0 success, 1 failure).
+    """
+    try:
+        from ..dataset import Dataset, DictSample
+
+        ds = Dataset[DictSample](url)
+    except Exception as exc:
+        print(f"Error opening dataset: {exc}", file=sys.stderr)
+        return 1
+
+    samples = ds.head(limit)
+    if not samples:
+        print("No samples found.", file=sys.stderr)
+        return 1
+
+    print(f"Preview of {url} ({len(samples)} sample(s)):")
+    print()
+
+    for i, sample in enumerate(samples):
+        print(f"--- Sample {i} ---")
+        for key in sample.keys():
+            val = sample[key]
+            print(f"  {key}: {_format_value(val)}")
+        print()
+
+    return 0
+
+
+def _format_value(val: Any) -> str:
+    """Format a value for preview, truncating large data."""
+    import numpy as np
+
+    if isinstance(val, np.ndarray):
+        return f"ndarray shape={val.shape} dtype={val.dtype}"
+    if isinstance(val, bytes):
+        if len(val) <= 40:
+            return repr(val)
+        return f"bytes[{len(val)}] {val[:20]!r}..."
+    if isinstance(val, str):
+        if len(val) <= 80:
+            return repr(val)
+        return repr(val[:77] + "...")
+    if isinstance(val, list):
+        if len(val) <= 5:
+            return repr(val)
+        return f"[{val[0]!r}, {val[1]!r}, ... ({len(val)} items)]"
+    return repr(val)