atdata 0.2.2b1__py3-none-any.whl → 0.3.0b1__py3-none-any.whl

atdata/atmosphere/records.py

@@ -19,10 +19,12 @@ from ._types import (
 
 # Import for type checking only to avoid circular imports
 from typing import TYPE_CHECKING
+
 if TYPE_CHECKING:
-    from ..dataset import PackableSample, Dataset
+    from ..dataset import Dataset
+    from .._protocols import Packable
 
-ST = TypeVar("ST", bound="PackableSample")
+ST = TypeVar("ST", bound="Packable")
 
 
 class DatasetPublisher:
@@ -31,21 +33,19 @@ class DatasetPublisher:
     This class creates dataset records that reference a schema and point to
     external storage (WebDataset URLs) or ATProto blobs.
 
-    Example:
-        ::
-
-            >>> dataset = atdata.Dataset[MySample]("s3://bucket/data-{000000..000009}.tar")
-            >>>
-            >>> client = AtmosphereClient()
-            >>> client.login("handle", "password")
-            >>>
-            >>> publisher = DatasetPublisher(client)
-            >>> uri = publisher.publish(
-            ...     dataset,
-            ...     name="My Training Data",
-            ...     description="Training data for my model",
-            ...     tags=["computer-vision", "training"],
-            ... )
+    Examples:
+        >>> dataset = atdata.Dataset[MySample]("s3://bucket/data-{000000..000009}.tar")
+        >>>
+        >>> client = AtmosphereClient()
+        >>> client.login("handle", "password")
+        >>>
+        >>> publisher = DatasetPublisher(client)
+        >>> uri = publisher.publish(
+        ...     dataset,
+        ...     name="My Training Data",
+        ...     description="Training data for my model",
+        ...     tags=["computer-vision", "training"],
+        ... )
     """
 
     def __init__(self, client: AtmosphereClient):
@@ -267,19 +267,17 @@ class DatasetLoader:
     from them. Note that loading a dataset requires having the corresponding
     Python class for the sample type.
 
-    Example:
-        ::
-
-            >>> client = AtmosphereClient()
-            >>> loader = DatasetLoader(client)
-            >>>
-            >>> # List available datasets
-            >>> datasets = loader.list()
-            >>> for ds in datasets:
-            ...     print(ds["name"], ds["schemaRef"])
-            >>>
-            >>> # Get a specific dataset record
-            >>> record = loader.get("at://did:plc:abc/ac.foundation.dataset.record/xyz")
+    Examples:
+        >>> client = AtmosphereClient()
+        >>> loader = DatasetLoader(client)
+        >>>
+        >>> # List available datasets
+        >>> datasets = loader.list()
+        >>> for ds in datasets:
+        ...     print(ds["name"], ds["schemaRef"])
+        >>>
+        >>> # Get a specific dataset record
+        >>> record = loader.get("at://did:plc:abc/ac.foundation.dataset.record/xyz")
     """
 
     def __init__(self, client: AtmosphereClient):
@@ -398,8 +396,7 @@ class DatasetLoader:
             return storage.get("blobs", [])
         elif "storageExternal" in storage_type:
             raise ValueError(
-                "Dataset uses external URL storage, not blobs. "
-                "Use get_urls() instead."
+                "Dataset uses external URL storage, not blobs. Use get_urls() instead."
             )
         else:
             raise ValueError(f"Unknown storage type: {storage_type}")
@@ -478,13 +475,11 @@ class DatasetLoader:
         Raises:
             ValueError: If no storage URLs can be resolved.
 
-        Example:
-            ::
-
-                >>> loader = DatasetLoader(client)
-                >>> dataset = loader.to_dataset(uri, MySampleType)
-                >>> for batch in dataset.shuffled(batch_size=32):
-                ...     process(batch)
+        Examples:
+            >>> loader = DatasetLoader(client)
+            >>> dataset = loader.to_dataset(uri, MySampleType)
+            >>> for batch in dataset.shuffled(batch_size=32):
+            ...     process(batch)
         """
         # Import here to avoid circular import
         from ..dataset import Dataset

atdata/atmosphere/schema.py

@@ -17,7 +17,6 @@ from ._types import (
     LEXICON_NAMESPACE,
 )
 from .._type_utils import (
-    numpy_dtype_to_string,
     unwrap_optional,
     is_ndarray_type,
     extract_ndarray_dtype,
@@ -25,10 +24,11 @@ from .._type_utils import (
 
 # Import for type checking only to avoid circular imports
 from typing import TYPE_CHECKING
+
 if TYPE_CHECKING:
-    from ..dataset import PackableSample
+    from .._protocols import Packable
 
-ST = TypeVar("ST", bound="PackableSample")
+ST = TypeVar("ST", bound="Packable")
 
 
 class SchemaPublisher:
@@ -37,21 +37,19 @@ class SchemaPublisher:
     This class introspects a PackableSample class to extract its field
     definitions and publishes them as an ATProto schema record.
 
-    Example:
-        ::
-
-            >>> @atdata.packable
-            ... class MySample:
-            ...     image: NDArray
-            ...     label: str
-            ...
-            >>> client = AtmosphereClient()
-            >>> client.login("handle", "password")
-            >>>
-            >>> publisher = SchemaPublisher(client)
-            >>> uri = publisher.publish(MySample, version="1.0.0")
-            >>> print(uri)
-            at://did:plc:.../ac.foundation.dataset.sampleSchema/...
+    Examples:
+        >>> @atdata.packable
+        ... class MySample:
+        ...     image: NDArray
+        ...     label: str
+        ...
+        >>> client = AtmosphereClient()
+        >>> client.login("handle", "password")
+        >>>
+        >>> publisher = SchemaPublisher(client)
+        >>> uri = publisher.publish(MySample, version="1.0.0")
+        >>> print(uri)
+        at://did:plc:.../ac.foundation.dataset.sampleSchema/...
     """
 
     def __init__(self, client: AtmosphereClient):
@@ -90,7 +88,9 @@ class SchemaPublisher:
             TypeError: If a field type is not supported.
         """
         if not is_dataclass(sample_type):
-            raise ValueError(f"{sample_type.__name__} must be a dataclass (use @packable)")
+            raise ValueError(
+                f"{sample_type.__name__} must be a dataclass (use @packable)"
+            )
 
         # Build the schema record
         schema_record = self._build_schema_record(
@@ -155,12 +155,18 @@ class SchemaPublisher:
             return FieldType(kind="primitive", primitive="bytes")
 
         if is_ndarray_type(python_type):
-            return FieldType(kind="ndarray", dtype=extract_ndarray_dtype(python_type), shape=None)
+            return FieldType(
+                kind="ndarray", dtype=extract_ndarray_dtype(python_type), shape=None
+            )
 
         origin = get_origin(python_type)
         if origin is list:
             args = get_args(python_type)
-            items = self._python_type_to_field_type(args[0]) if args else FieldType(kind="primitive", primitive="str")
+            items = (
+                self._python_type_to_field_type(args[0])
+                if args
+                else FieldType(kind="primitive", primitive="str")
+            )
             return FieldType(kind="array", items=items)
 
         if is_dataclass(python_type):
@@ -178,16 +184,14 @@ class SchemaLoader:
     This class fetches schema records from ATProto and can list available
     schemas from a repository.
 
-    Example:
-        ::
-
-            >>> client = AtmosphereClient()
-            >>> client.login("handle", "password")
-            >>>
-            >>> loader = SchemaLoader(client)
-            >>> schema = loader.get("at://did:plc:.../ac.foundation.dataset.sampleSchema/...")
-            >>> print(schema["name"])
-            'MySample'
+    Examples:
+        >>> client = AtmosphereClient()
+        >>> client.login("handle", "password")
+        >>>
+        >>> loader = SchemaLoader(client)
+        >>> schema = loader.get("at://did:plc:.../ac.foundation.dataset.sampleSchema/...")
+        >>> print(schema["name"])
+        'MySample'
     """
 
     def __init__(self, client: AtmosphereClient):

atdata/atmosphere/store.py

@@ -6,23 +6,20 @@ protocol that stores dataset shards as ATProto blobs in a Personal Data Server.
 This enables fully decentralized dataset storage where both metadata (records)
 and data (blobs) live on the AT Protocol network.
 
-Example:
-    ::
-
-        >>> from atdata.atmosphere import AtmosphereClient, PDSBlobStore
-        >>>
-        >>> client = AtmosphereClient()
-        >>> client.login("handle.bsky.social", "app-password")
-        >>>
-        >>> store = PDSBlobStore(client)
-        >>> urls = store.write_shards(dataset, prefix="mnist/v1")
-        >>> print(urls)
-        ['at://did:plc:.../blob/bafyrei...', ...]
+Examples:
+    >>> from atdata.atmosphere import AtmosphereClient, PDSBlobStore
+    >>>
+    >>> client = AtmosphereClient()
+    >>> client.login("handle.bsky.social", "app-password")
+    >>>
+    >>> store = PDSBlobStore(client)
+    >>> urls = store.write_shards(dataset, prefix="mnist/v1")
+    >>> print(urls)
+    ['at://did:plc:.../blob/bafyrei...', ...]
 """
 
 from __future__ import annotations
 
-import io
 import tempfile
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
@@ -31,6 +28,7 @@ import webdataset as wds
 
 if TYPE_CHECKING:
     from ..dataset import Dataset
+    from .._sources import BlobSource
     from .client import AtmosphereClient
 
 
@@ -48,13 +46,11 @@ class PDSBlobStore:
     Attributes:
         client: Authenticated AtmosphereClient instance.
 
-    Example:
-        ::
-
-            >>> store = PDSBlobStore(client)
-            >>> urls = store.write_shards(dataset, prefix="training/v1")
-            >>> # Returns AT URIs like:
-            >>> # ['at://did:plc:abc/blob/bafyrei...', ...]
+    Examples:
+        >>> store = PDSBlobStore(client)
+        >>> urls = store.write_shards(dataset, prefix="training/v1")
+        >>> # Returns AT URIs like:
+        >>> # ['at://did:plc:abc/blob/bafyrei...', ...]
     """
 
     client: "AtmosphereClient"

atdata/cli/__init__.py

@@ -1,212 +1,207 @@
 """Command-line interface for atdata.
 
-This module provides CLI commands for managing local development infrastructure
-and diagnosing configuration issues.
+This module provides CLI commands for managing local development infrastructure,
+inspecting datasets, and diagnosing configuration issues.
 
 Commands:
-    atdata local up     Start Redis and MinIO containers for local development
-    atdata local down   Stop local development containers
-    atdata diagnose     Check Redis configuration and connectivity
-    atdata version      Show version information
-
-Example:
-    $ atdata local up
-    Starting Redis on port 6379...
-    Starting MinIO on port 9000...
-    Local infrastructure ready.
-
-    $ atdata diagnose
-    Checking Redis configuration...
-    ✓ Redis connected
-    ✓ Persistence enabled (AOF)
-    ✓ Memory policy: noeviction
+    atdata local up      Start Redis and MinIO containers for local development
+    atdata local down    Stop local development containers
+    atdata local status  Show status of local infrastructure
+    atdata diagnose      Check Redis configuration and connectivity
+    atdata inspect       Show dataset summary information
+    atdata schema show   Display dataset schema
+    atdata schema diff   Compare two dataset schemas
+    atdata preview       Preview first N samples of a dataset
+    atdata version       Show version information
 """
 
-import argparse
 import sys
-from typing import Sequence
 
+import typer
 
-def main(argv: Sequence[str] | None = None) -> int:
-    """Main entry point for the atdata CLI.
+# ---------------------------------------------------------------------------
+# App hierarchy
+# ---------------------------------------------------------------------------
 
-    Args:
-        argv: Command-line arguments. If None, uses sys.argv[1:].
+app = typer.Typer(
+    name="atdata",
+    help="A loose federation of distributed, typed datasets.",
+    add_completion=False,
+    no_args_is_help=True,
+)
 
-    Returns:
-        Exit code (0 for success, non-zero for errors).
-    """
-    parser = argparse.ArgumentParser(
-        prog="atdata",
-        description="A loose federation of distributed, typed datasets",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-    parser.add_argument(
-        "--version", "-v",
-        action="store_true",
-        help="Show version information",
-    )
+local_app = typer.Typer(
+    name="local",
+    help="Manage local development infrastructure.",
+    no_args_is_help=True,
+)
+app.add_typer(local_app, name="local")
 
-    subparsers = parser.add_subparsers(dest="command", help="Available commands")
+schema_app = typer.Typer(
+    name="schema",
+    help="Show or compare dataset schemas.",
+    no_args_is_help=True,
+)
+app.add_typer(schema_app, name="schema")
 
-    # 'local' command group
-    local_parser = subparsers.add_parser(
-        "local",
-        help="Manage local development infrastructure",
-    )
-    local_subparsers = local_parser.add_subparsers(
-        dest="local_command",
-        help="Local infrastructure commands",
-    )
 
-    # 'local up' command
-    up_parser = local_subparsers.add_parser(
-        "up",
-        help="Start Redis and MinIO containers",
-    )
-    up_parser.add_argument(
-        "--redis-port",
-        type=int,
-        default=6379,
-        help="Redis port (default: 6379)",
-    )
-    up_parser.add_argument(
-        "--minio-port",
-        type=int,
-        default=9000,
-        help="MinIO API port (default: 9000)",
-    )
-    up_parser.add_argument(
-        "--minio-console-port",
-        type=int,
-        default=9001,
-        help="MinIO console port (default: 9001)",
-    )
-    up_parser.add_argument(
-        "--detach", "-d",
-        action="store_true",
-        default=True,
-        help="Run containers in detached mode (default: True)",
-    )
+# ---------------------------------------------------------------------------
+# Top-level commands
+# ---------------------------------------------------------------------------
 
-    # 'local down' command
-    down_parser = local_subparsers.add_parser(
-        "down",
-        help="Stop local development containers",
-    )
-    down_parser.add_argument(
-        "--volumes", "-v",
-        action="store_true",
-        help="Also remove volumes (deletes all data)",
-    )
 
-    # 'local status' command
-    local_subparsers.add_parser(
-        "status",
-        help="Show status of local infrastructure",
-    )
+@app.command()
+def version() -> None:
+    """Show version information."""
+    try:
+        from atdata import __version__
 
-    # 'diagnose' command
-    diagnose_parser = subparsers.add_parser(
-        "diagnose",
-        help="Diagnose Redis configuration and connectivity",
-    )
-    diagnose_parser.add_argument(
-        "--host",
-        default="localhost",
-        help="Redis host (default: localhost)",
-    )
-    diagnose_parser.add_argument(
-        "--port",
-        type=int,
-        default=6379,
-        help="Redis port (default: 6379)",
-    )
+        ver = __version__
+    except ImportError:
+        from importlib.metadata import version as pkg_version
 
-    # 'version' command (alternative to --version flag)
-    subparsers.add_parser(
-        "version",
-        help="Show version information",
-    )
+        ver = pkg_version("atdata")
 
-    args = parser.parse_args(argv)
-
-    # Handle --version flag
-    if args.version or args.command == "version":
-        return _cmd_version()
-
-    # Handle 'local' commands
-    if args.command == "local":
-        if args.local_command == "up":
-            return _cmd_local_up(
-                redis_port=args.redis_port,
-                minio_port=args.minio_port,
-                minio_console_port=args.minio_console_port,
-                detach=args.detach,
-            )
-        elif args.local_command == "down":
-            return _cmd_local_down(remove_volumes=args.volumes)
-        elif args.local_command == "status":
-            return _cmd_local_status()
-        else:
-            local_parser.print_help()
-            return 1
+    print(f"atdata {ver}")
 
-    # Handle 'diagnose' command
-    if args.command == "diagnose":
-        return _cmd_diagnose(host=args.host, port=args.port)
 
-    # No command given
-    parser.print_help()
-    return 0
+@app.command()
+def inspect(
+    url: str = typer.Argument(help="Dataset URL, local path, or atmosphere URI"),
+) -> None:
+    """Show dataset summary (sample count, schema, shards)."""
+    from .inspect import inspect_dataset
 
+    code = inspect_dataset(url=url)
+    raise typer.Exit(code=code)
+
+
+@app.command()
+def preview(
+    url: str = typer.Argument(help="Dataset URL, local path, or atmosphere URI"),
+    limit: int = typer.Option(5, help="Number of samples to preview."),
+) -> None:
+    """Preview first N samples of a dataset."""
+    from .preview import preview_dataset
+
+    code = preview_dataset(url=url, limit=limit)
+    raise typer.Exit(code=code)
+
+
+@app.command()
+def diagnose(
+    host: str = typer.Option("localhost", help="Redis host."),
+    port: int = typer.Option(6379, help="Redis port."),
+) -> None:
+    """Diagnose Redis configuration and connectivity."""
+    from .diagnose import diagnose_redis
+
+    code = diagnose_redis(host=host, port=port)
+    raise typer.Exit(code=code)
 
-def _cmd_version() -> int:
-    """Show version information."""
-    try:
-        from atdata import __version__
-        version = __version__
-    except ImportError:
-        # Fallback to package metadata
-        from importlib.metadata import version as pkg_version
-        version = pkg_version("atdata")
 
-    print(f"atdata {version}")
-    return 0
+# ---------------------------------------------------------------------------
+# local sub-commands
+# ---------------------------------------------------------------------------
 
 
-def _cmd_local_up(
-    redis_port: int,
-    minio_port: int,
-    minio_console_port: int,
-    detach: bool,
-) -> int:
-    """Start local development infrastructure."""
+@local_app.command()
+def up(
+    redis_port: int = typer.Option(6379, help="Redis port."),
+    minio_port: int = typer.Option(9000, help="MinIO API port."),
+    minio_console_port: int = typer.Option(9001, help="MinIO console port."),
+    detach: bool = typer.Option(
+        True, "--detach", "-d", help="Run containers in detached mode."
+    ),
+) -> None:
+    """Start Redis and MinIO containers."""
     from .local import local_up
-    return local_up(
+
+    code = local_up(
         redis_port=redis_port,
         minio_port=minio_port,
         minio_console_port=minio_console_port,
         detach=detach,
     )
+    raise typer.Exit(code=code)
 
 
-def _cmd_local_down(remove_volumes: bool) -> int:
-    """Stop local development infrastructure."""
+@local_app.command()
+def down(
+    volumes: bool = typer.Option(
+        False, "--volumes", "-v", help="Also remove volumes (deletes all data)."
+    ),
+) -> None:
+    """Stop local development containers."""
     from .local import local_down
-    return local_down(remove_volumes=remove_volumes)
+
+    code = local_down(remove_volumes=volumes)
+    raise typer.Exit(code=code)
 
 
-def _cmd_local_status() -> int:
+@local_app.command()
+def status() -> None:
     """Show status of local infrastructure."""
     from .local import local_status
-    return local_status()
 
+    code = local_status()
+    raise typer.Exit(code=code)
 
-def _cmd_diagnose(host: str, port: int) -> int:
-    """Diagnose Redis configuration."""
-    from .diagnose import diagnose_redis
-    return diagnose_redis(host=host, port=port)
+
+# ---------------------------------------------------------------------------
+# schema sub-commands
+# ---------------------------------------------------------------------------
+
+
+@schema_app.command("show")
+def schema_show(
+    dataset_ref: str = typer.Argument(
+        help="Dataset URL, local path, or index reference."
+    ),
+) -> None:
+    """Display dataset schema."""
+    from .schema import schema_show as _schema_show
+
+    code = _schema_show(dataset_ref=dataset_ref)
+    raise typer.Exit(code=code)
+
+
+@schema_app.command("diff")
+def schema_diff(
+    url_a: str = typer.Argument(help="First dataset URL."),
+    url_b: str = typer.Argument(help="Second dataset URL."),
+) -> None:
+    """Compare two dataset schemas."""
+    from .schema import schema_diff as _schema_diff
+
+    code = _schema_diff(url_a=url_a, url_b=url_b)
+    raise typer.Exit(code=code)
+
+
+# ---------------------------------------------------------------------------
+# Entrypoint
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Main entry point for the atdata CLI.
+
+    Args:
+        argv: Command-line arguments. If None, uses sys.argv[1:].
+
+    Returns:
+        Exit code (0 for success, non-zero for errors).
+    """
+    try:
+        if argv is not None:
+            app(args=argv, standalone_mode=False)
+        else:
+            app(standalone_mode=False)
+        return 0
+    except SystemExit as exc:
+        return exc.code if isinstance(exc.code, int) else 0
+    except Exception:
+        return 1
 
 
 if __name__ == "__main__":