earthscope-cli 1.1.2__tar.gz → 1.3.0__tar.gz

{earthscope_cli-1.1.2/src/earthscope_cli.egg-info → earthscope_cli-1.3.0}/PKG-INFO

@@ -1,24 +1,27 @@
 Metadata-Version: 2.4
 Name: earthscope-cli
-Version: 1.1.2
+Version: 1.3.0
 Summary: A CLI for the EarthScope API
 Author-email: EarthScope <data-help@earthscope.org>
 License-Expression: Apache-2.0
-Project-URL: Homepage, https://gitlab.com/earthscope/public/earthscope-cli
+Project-URL: Homepage, https://docs.earthscope.org/projects/CLI
+Project-URL: Repository, https://gitlab.com/earthscope/public/earthscope-cli
+Project-URL: Changelog, https://docs.earthscope.org/projects/CLI/page/changelog.html
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
 Classifier: Operating System :: OS Independent
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: typer<0.24.0
 Requires-Dist: typer-di>=0.1.3
-Requires-Dist: earthscope-sdk>=1.3.1
+Requires-Dist: earthscope-sdk>=1.10.0
 Requires-Dist: rich>=13.8.0
 Provides-Extra: dev
 Requires-Dist: black; extra == "dev"
 Requires-Dist: bumpver; extra == "dev"
 Requires-Dist: build; extra == "dev"
+Requires-Dist: pip>=22.0; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: twine>=6.0.1; extra == "dev"
@@ -36,7 +39,7 @@ A CLI for interacting with EarthScope's APIs
 To use the CLI you must have:
 
 - Registered an account with Earthscope ([sign up now](https://earthscope.org/user/login)). See [here](https://www.earthscope.org/data/authentication) for more information
-- Python >= 3.9
+- Python >= 3.10
 
 ### Installation
 

{earthscope_cli-1.1.2 → earthscope_cli-1.3.0}/README.md

@@ -9,7 +9,7 @@ A CLI for interacting with EarthScope's APIs
 To use the CLI you must have:
 
 - Registered an account with Earthscope ([sign up now](https://earthscope.org/user/login)). See [here](https://www.earthscope.org/data/authentication) for more information
-- Python >= 3.9
+- Python >= 3.10
 
 ### Installation
 

{earthscope_cli-1.1.2 → earthscope_cli-1.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "earthscope-cli"
-version = "1.1.2"
+version = "1.3.0"
 description = "A CLI for the EarthScope API"
 readme = "README.md"
 authors = [{ name = "EarthScope", email = "data-help@earthscope.org" }]
@@ -20,16 +20,17 @@ classifiers = [
 dependencies = [
     "typer<0.24.0",
     "typer-di>=0.1.3",
-    "earthscope-sdk>=1.3.1",
+    "earthscope-sdk>=1.10.0",
     "rich>=13.8.0",
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 [project.optional-dependencies]
 dev = [
     "black",
     "bumpver",
     "build",
+    "pip>=22.0",
     "pre-commit",
     "pytest",
     "twine>=6.0.1",
@@ -37,13 +38,15 @@ dev = [
 ]
 
 [project.urls]
-Homepage = "https://gitlab.com/earthscope/public/earthscope-cli"
+Homepage = "https://docs.earthscope.org/projects/CLI"
+Repository = "https://gitlab.com/earthscope/public/earthscope-cli"
+Changelog = "https://docs.earthscope.org/projects/CLI/page/changelog.html"
 
 [project.scripts]
 es = "earthscope_cli.main:app"
 
 [tool.bumpver]
-current_version = "1.1.2"
+current_version = "1.3.0"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message = "chore: bump version {old_version} -> {new_version}"
 commit = true

earthscope_cli-1.3.0/src/earthscope_cli/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.3.0"

{earthscope_cli-1.1.2 → earthscope_cli-1.3.0}/src/earthscope_cli/dropoff/render.py

@@ -1,8 +1,11 @@
 import threading
 from typing import Dict, List, Optional
 
-from earthscope_sdk.client.dropoff._multipart_uploader import UploadStatus
-from earthscope_sdk.client.dropoff.models import DropoffObject, DropoffSummary
+from earthscope_sdk.client.dropoff.models import (
+    DropoffObject,
+    DropoffSummary,
+    UploadStatus,
+)
 from rich import print
 from rich.console import Group
 from rich.live import Live
@@ -40,17 +43,34 @@ def _colorize_status(status: str) -> Text:
         return Text("ACCEPTED", style="green")
     if s == "FAILED":
         return Text("FAILED", style="red")
+    if s == "COPYING":
+        return Text("COPYING", style="yellow")
+    if s == "LOADING":
+        return Text("LOADING", style="yellow")
     if s == "INTERNAL_ERROR":
         return Text("INTERNAL_ERROR", style="red")
     return Text(f"{status or '-'}", style="dim")
 
 
+def _format_details(details: List[str], is_truncated: bool) -> Text:
+    """Format structured_messages (with truncation indicator) or fall back to status_message."""
+    if details:
+        content = "\n".join(details)
+        result = Text(content)
+        if is_truncated:
+            result.append("\n(truncated)", style="dim")
+        return result
+
+    return Text("-", style="dim")
+
+
 def print_dropoff_objects_table(
     objects: List[DropoffObject],
     *,
     title: str,
     has_next: bool,
     next_offset: int,
+    show_details: bool = False,
 ):
     if has_next:
         caption = f"Use --offset {next_offset} to view more results."
@@ -69,16 +89,28 @@ def print_dropoff_objects_table(
     table.add_column("Last Modified", justify="right")
     table.add_column("Status", justify="center")
     table.add_column("Message")
-
-    for o in objects:
-        table.add_row(
-            safe_str(o.path),
-            filesize.decimal(o.size),
-            safe_str(o.received_at),
-            _colorize_status(o.status),
-            safe_str(o.status_message),
-        )
-
+    if show_details:
+        table.add_column("Details")
+        for o in objects:
+            table.add_row(
+                safe_str(o.path),
+                filesize.decimal(o.size),
+                safe_str(o.received_at),
+                _colorize_status(o.status),
+                safe_str(o.status_message),
+                _format_details(
+                    o.structured_messages, o.structured_messages_were_truncated
+                ),
+            )
+    else:
+        for o in objects:
+            table.add_row(
+                safe_str(o.path),
+                filesize.decimal(o.size),
+                safe_str(o.received_at),
+                _colorize_status(o.status),
+                safe_str(o.status_message),
+            )
     print()
     print(table)
     print()
@@ -236,10 +268,10 @@ class ConcurrentUploadProgressDisplay:
     hidden to keep the display clean and focused on active uploads.
 
     Usage:
-        display = ConcurrentUploadProgressDisplay(num_files=10)
+        display = ConcurrentUploadProgressDisplay(num_collected=10)
         with display:
-            sdk.dropoff.put_dropoff_files(
-                files=file_list,
+            sdk.dropoff.put_objects(
+                objects=validated_files_generator(),
                 category="miniseed",
                 progress_cb=display.callback,
             )
@@ -247,7 +279,7 @@ class ConcurrentUploadProgressDisplay:
 
     def __init__(
         self,
-        num_files: int,
+        num_collected: int,
         hide_completed_threshold: int = 10,
         hide_delay_seconds: float = 1.5,
     ):
@@ -255,17 +287,29 @@ class ConcurrentUploadProgressDisplay:
         Initialize the progress display.
 
         Args:
-            num_files: Total number of files to upload
+            num_collected: Total number of files collected (to be validated then uploaded)
             hide_completed_threshold: Threshold for hiding completed progress bars
             hide_delay_seconds: Delay in seconds before hiding completed progress bars
         """
         self.hide_completed_threshold = hide_completed_threshold
         self.hide_delay_seconds = hide_delay_seconds
-        self.num_files = num_files
+        self.num_collected = num_collected
+        self.num_valid = 0
+        self.num_invalid = 0
+        self.validation_complete = False
         self.files_completed = 0
         self.upload_started = False  # Track if we've received any bytes yet
 
-        # Overall progress (files completed, plus aggregate throughput)
+        # Validation progress
+        self.validation_progress = Progress(
+            TextColumn("[bold blue]{task.description}"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            TextColumn("•"),
+            TimeElapsedColumn(),
+        )
+
+        # Overall upload progress (files completed, plus aggregate throughput)
         self.overall_progress = Progress(
             TextColumn("[bold blue]{task.description}"),
             BarColumn(),
@@ -291,6 +335,7 @@ class ConcurrentUploadProgressDisplay:
         self.file_tasks: Dict[str, TaskID] = {}
         self.file_bytes_done: Dict[str, int] = {}  # Track current bytes_done per file
         self.file_bytes_resumed: Dict[str, int] = {}  # Track bytes resumed per file
+        self.validation_task: Optional[TaskID] = None
         self.overall_task: Optional[TaskID] = None
         self.completed_files: set[str] = set()
         self.live: Optional[Live] = None
@@ -298,12 +343,17 @@ class ConcurrentUploadProgressDisplay:
 
     def __enter__(self):
         """Start the live progress display."""
-        # Create overall progress task tracking files completed
-        # (visual_bytes shows total progress, throughput_bytes for speed calculation)
-        # Don't start the timer yet - wait for first byte update
+        # Validation progress task
+        self.validation_task = self.validation_progress.add_task(
+            "[bold]Validating...",
+            total=self.num_collected,
+        )
+
+        # Upload progress task — total starts at num_collected, decremented as files fail validation
+        # Don't start the timer yet — wait for first byte update
         self.overall_task = self.overall_progress.add_task(
-            f"[bold]Uploading {self.num_files} files...",
-            total=self.num_files,
+            "[bold]Uploading...",
+            total=self.num_collected,
             visual_bytes=0,
             throughput_bytes=0,
             start=False,
@@ -311,6 +361,7 @@ class ConcurrentUploadProgressDisplay:
 
         # Create progress group
         progress_group = Group(
+            self.validation_progress,
             self.overall_progress,
             self.file_progress,
         )
@@ -329,10 +380,14 @@ class ConcurrentUploadProgressDisplay:
 
         if self.live:
             # Update final status
-            if exc_type is None and self.overall_task is not None:
+            if (
+                exc_type is None
+                and self.overall_task is not None
+                and self.num_valid > 0
+            ):
                 self.overall_progress.update(
                     self.overall_task,
-                    description=f"[bold green]✓ Completed {self.num_files} uploads",
+                    description=f"[bold green]✓ Completed {self.num_valid} uploads",
                 )
 
                 # Hide all individual file progress bars when complete
@@ -343,6 +398,62 @@ class ConcurrentUploadProgressDisplay:
             self.live.__exit__(exc_type, exc_val, exc_tb)
         return False
 
+    def validation_passed(self):
+        """Called when a file passes validation and will be uploaded."""
+        self.num_valid += 1
+        if self.validation_task is not None:
+            self.validation_progress.advance(self.validation_task, 1)
+
+    def validation_failed(self):
+        """Called when a file fails validation and will not be uploaded."""
+        self.num_invalid += 1
+        if self.validation_task is not None:
+            self.validation_progress.advance(self.validation_task, 1)
+            self.validation_progress.update(
+                self.validation_task,
+                description=f"[bold blue]Validating... [dim]({self.num_invalid} failed)",
+            )
+        # Decrement upload total since this file won't be uploaded
+        if self.overall_task is not None:
+            self.overall_progress.update(
+                self.overall_task,
+                total=self.num_collected - self.num_invalid,
+            )
+
+    def finalize_validation(self):
+        """Called when all validation is complete."""
+        self.validation_complete = True
+        if self.validation_task is not None:
+            self.validation_progress.stop_task(self.validation_task)
+            if self.num_invalid == 0:
+                self.validation_progress.update(
+                    self.validation_task,
+                    description=f"[bold green]✓ Validated {self.num_valid} files",
+                )
+            elif self.num_valid > 0:
+                self.validation_progress.update(
+                    self.validation_task,
+                    description=f"[bold yellow]✓ Validated {self.num_valid}/{self.num_collected} files ({self.num_invalid} failed)",
+                )
+            else:
+                self.validation_progress.update(
+                    self.validation_task,
+                    description=f"[bold red]✗ Validation failed for all {self.num_invalid} files",
+                )
+        # Update upload description with final count
+        if self.overall_task is not None and self.num_valid > 0:
+            self.overall_progress.update(
+                self.overall_task,
+                description=f"[bold]Uploading {self.num_valid} files...",
+            )
+        # Check if all uploads already completed
+        if (
+            self.overall_task is not None
+            and self.num_valid > 0
+            and len(self.completed_files) == self.num_valid
+        ):
+            self.overall_progress.stop_task(self.overall_task)
+
     def _hide_completed_task(self, task_id: TaskID, key: str):
         """
         Hide a completed task from the display.
@@ -426,11 +537,14 @@ class ConcurrentUploadProgressDisplay:
                 self.overall_progress.advance(self.overall_task, 1)
 
                 # If all files are complete, stop the timer to freeze throughput
-                if len(self.completed_files) == self.num_files:
+                if (
+                    self.validation_complete
+                    and len(self.completed_files) == self.num_valid
+                ):
                     self.overall_progress.stop_task(self.overall_task)
 
             # For large uploads, hide completed progress bars after a delay
-            if self.num_files > self.hide_completed_threshold:
+            if self.num_collected > self.hide_completed_threshold:
                 # Schedule hiding the task after a delay
                 timer = threading.Timer(
                     self.hide_delay_seconds,

{earthscope_cli-1.1.2 → earthscope_cli-1.3.0}/src/earthscope_cli/dropoff/typer.py

@@ -1,25 +1,17 @@
 from pathlib import Path
 from textwrap import dedent
-from typing import Annotated
+from typing import Annotated, Iterable
 
 import typer
 from earthscope_sdk import EarthScopeClient
-from earthscope_sdk.client.dropoff.models import DropoffCategory
-from earthscope_sdk.client.dropoff.validation import Validator
+from earthscope_sdk.client.dropoff.enums import DropoffCategory
 from rich import print
 from typer_di import Depends, TyperDI
 
 from earthscope_cli.common import get_sdk_refreshed
-from earthscope_cli.dropoff.render import (
-    ConcurrentUploadProgressDisplay,
-    UploadSummary,
-    print_dropoff_objects_table,
-    print_dropoff_summary,
-)
 from earthscope_cli.dropoff.util import (
     DropoffOptions,
     FilterOptions,
-    collect_files,
     get_category,
     get_dropoff_options,
     get_filter_options,
@@ -52,6 +44,9 @@ def get_object_history(
     es dropoff get-object-history -c miniseed -k demo/SAML.IU.2025.303
     es dropoff get-object-history -c miniseed -k demo/SAML.IU.2025.303 --offset 100 --limit 100
     """
+    # lazy import
+    from earthscope_cli.dropoff.render import print_dropoff_objects_table
+
     result = sdk.dropoff.get_object_history(
         category=options.category.value,
         key=key,
@@ -65,6 +60,7 @@ def get_object_history(
             title=f"History for: {key}",
             has_next=result.has_next,
             next_offset=result.offset + result.limit,
+            show_details=True,
         )
 
     cat_str = options.category.value
@@ -89,6 +85,9 @@ def get_summary(
     es dropoff get-summary -c miniseed --prefix demo/ --submitted-after 2025-01-01T00:00:00Z
     es dropoff get-summary -c miniseed --prefix demo/ --submitted-before 2025-12-31T23:59:59Z
     """
+    # lazy import
+    from earthscope_cli.dropoff.render import print_dropoff_summary
+
     result = sdk.dropoff.get_summary(
         category=options.category.value,
         prefix=prefix,
@@ -119,6 +118,9 @@ def list_objects(
     es dropoff list-objects -c miniseed --submitted-after 2025-01-01T00:00:00Z
     es dropoff list-objects -c miniseed --submitted-before 2025-12-31T23:59:59Z
     """
+    # lazy import
+    from earthscope_cli.dropoff.render import print_dropoff_objects_table
+
     result = sdk.dropoff.list_objects(
         category=options.category.value,
         prefix=prefix,
@@ -147,26 +149,53 @@ def list_objects(
     )
 
 
+def destinaton_callback(value: str) -> str:
+    if not value:
+        print("[red]Destination cannot be empty.")
+        raise typer.Exit(code=2)
+    return value.removeprefix("/")
+
+
 @app.command(name="upload")
 def upload(
     sdk: Annotated[EarthScopeClient, Depends(get_sdk_refreshed)],
     category: Annotated[DropoffCategory, Depends(get_category)],
-    paths: Annotated[
+    source: Annotated[
         list[Path],
-        typer.Argument(
+        typer.Option(
+            "--source",
+            "-s",
             exists=True,
             file_okay=True,
             dir_okay=True,
             resolve_path=True,
+            help="One or more local files or directories to upload.",
+        ),
+    ],
+    destination: Annotated[
+        str,
+        typer.Option(
+            "--destination",
+            "-d",
             help=dedent(
-                """
-                The local path of a file or directory to upload.
-
-                The path must include a directory e.g. 2026-01-01/file1 or 2025-01-01/
+                """\
+                Remote object key or prefix. A trailing '/' indicates a prefix
+                (source filename is appended); without a trailing '/' the
+                destination is used as the exact object key name. All keys are
+                relative to your dropoff home path.
                 """
             ),
+            callback=destinaton_callback,
         ),
     ],
+    recursive: Annotated[
+        bool,
+        typer.Option(
+            "--recursive",
+            "-r",
+            help="Recursively traverse source directories.",
+        ),
+    ] = False,
     object_concurrency: Annotated[
         int,
         typer.Option(
@@ -192,53 +221,88 @@ def upload(
     """
     Upload files or directories to EarthScope's dropoff system.
 
+    The source is one or more local files or directories. The destination
+    specifies the remote object key or prefix.
+
+    A destination with a trailing '/' is treated as a prefix -- the source
+    filename is appended. Without a trailing '/', the destination is the
+    exact object key name.
+
+    When the source is a directory or multiple files, the destination must
+    be a prefix (trailing '/').
+
+    Use -r to recursively include files in subdirectories.
+
     Examples:
-        es dropoff upload -c miniseed file.mseed
-        es dropoff upload -c miniseed mydata/
-        es dropoff upload -c miniseed project1/data.mseed project2/data.mseed
 
-    How S3 keys are generated:
+        Single file to exact key:
+            es dropoff upload -c miniseed -d dest/keyname -s file.mseed
 
-    - For individual files: The parent directory name is included in the S3 key
-      to provide context and prevent collisions.
-      Example: uploading 'project1/data.mseed' creates key 'project1/data.mseed'
+        Single file to prefix:
+            es dropoff upload -c miniseed -d dest/prefix/ -s file.mseed
 
-    - For directories: All files maintain their relative paths from the directory,
-      with the directory name as a prefix.
-      Example: uploading 'mydata/' containing 'subdir/file.mseed' creates
-      key 'mydata/subdir/file.mseed'
+        Directory (top-level files only):
+            es dropoff upload -c miniseed -d dest/prefix/ -s mydata/
 
-    - Multiple files: Each file preserves its parent directory context.
-      Example: uploading 'proj1/data.mseed' and 'proj2/data.mseed' creates
-      distinct keys 'proj1/data.mseed' and 'proj2/data.mseed'
+        Directory (recursive):
+            es dropoff upload -c miniseed -d dest/prefix/ -r -s mydata/
 
-    This behavior ensures files with the same name from different locations
-    don't overwrite each other and maintains meaningful organization in S3.
+        Multiple files:
+            es dropoff upload -c miniseed -d dest/prefix/ -s file1.mseed -s file2.mseed
     """
-    # Parse the input path to get a list of files
-    total_collected = 0
+    # lazy import
+    from earthscope_sdk.client.dropoff.files import (
+        UploadCandidate,
+        collect_files,
+        validate_files,
+    )
+
+    from earthscope_cli.dropoff.render import (
+        ConcurrentUploadProgressDisplay,
+        UploadSummary,
+    )
+
+    is_prefix = destination.endswith("/")
+
+    if not is_prefix and (len(source) > 1 or any(s.is_dir() for s in source)):
+        print(
+            "[red]Multiple sources require destination to be a prefix (trailing '/')."
+        )
+        raise typer.Exit(code=2)
+
+    # Collect all files first (fast filesystem traversal)
+    collected = list(collect_files(source, destination, recursive))
+    total_collected = len(collected)
     succeeded = 0
-    files: list[tuple[Path, str]] = []
-    for p, k in collect_files(paths):
-        total_collected += 1
-        try:
-            Validator(p, category=category).validate_all()
-        except Exception as e:
-            print(f"[red]{e}")
-        else:
-            files.append((p, k))
-
-    if not files:
-        paths_str = ", ".join(str(p) for p in paths)
-        print(f"[red]No valid {category.value} files found in paths: {paths_str}")
+
+    if not collected:
+        src_str = ", ".join(str(s) for s in source)
+        print(f"[red]No valid {category.value} files found in sources: {src_str}")
         UploadSummary(total=total_collected, succeeded=succeeded).print()
         raise typer.Exit(code=1)
 
-    display = ConcurrentUploadProgressDisplay(len(files))
+    # Validation runs inline in the generator. The SDK advances this iterator
+    # via asyncio.to_thread, so validation runs off the event loop and uploads
+    # can proceed concurrently with validation of remaining files.
+    validation_errors: list[str] = []
+    display = ConcurrentUploadProgressDisplay(num_collected=total_collected)
+
+    def _validated_files() -> Iterable[UploadCandidate]:
+        try:
+            for item in validate_files(collected, category):
+                if item.ok:
+                    display.validation_passed()
+                    yield item
+                else:
+                    validation_errors.append(str(item.error))
+                    display.validation_failed()
+        finally:
+            display.finalize_validation()
+
     try:
         with display:
             sdk.dropoff.put_objects(
-                objects=files,
+                objects=_validated_files(),
                 category=category,
                 object_concurrency=object_concurrency,
                 part_concurrency=part_concurrency,
@@ -249,5 +313,12 @@ def upload(
         print()
         print("[yellow]Upload interrupted. Use the same command to resume the upload.")
     finally:
+        for error in validation_errors:
+            print(f"[red]{error}")
         succeeded = len(display.completed_files)
         UploadSummary(total=total_collected, succeeded=succeeded).print()
+
+    if validation_errors and len(validation_errors) == total_collected:
+        paths_str = ", ".join(str(p) for p in source)
+        print(f"[red]No valid {category.value} files found in paths: {paths_str}")
+        raise typer.Exit(code=1)