usdata 0.32.0__tar.gz → 0.34.0__tar.gz

{usdata-0.32.0 → usdata-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.32.0
+Version: 0.34.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -113,7 +113,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-eight datasets are available today and eighteen more are planned, grouped
+Thirty-four datasets are available today and seventeen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.32.0 → usdata-0.34.0}/README.md

@@ -68,7 +68,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-eight datasets are available today and eighteen more are planned, grouped
+Thirty-four datasets are available today and seventeen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.32.0 → usdata-0.34.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.32.0"
+version = "0.34.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.32.0 → usdata-0.34.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.32.0"
+version = "0.34.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.32.0 → usdata-0.34.0}/src/usdata/__init__.py

@@ -13,6 +13,7 @@ except PackageNotFoundError:  # running from a source tree without an install
 from usdata._fetch import ChecksumMismatch, FetchedAsset, fetch, fetch_asset
 from usdata.cite import Citation, cite_dataset, cite_lockfile
 from usdata.inspect import inspect_asset, inspect_path
+from usdata.messages import MessageListing, list_messages
 from usdata.models import (
     Asset,
     BBox,
@@ -36,6 +37,7 @@ __all__ = [
     "Dataset",
     "DatasetNotFound",
     "FetchedAsset",
+    "MessageListing",
     "Place",
     "Provenance",
     "Query",
@@ -55,6 +57,7 @@ __all__ = [
     "get",
     "inspect_asset",
     "inspect_path",
+    "list_messages",
     "pull",
     "search",
     "select_by_time",

{usdata-0.32.0 → usdata-0.34.0}/src/usdata/_grib.py

@@ -96,7 +96,9 @@ def import_eccodes() -> Any:
     imported later in the same process, both copies free the same memory as the
     interpreter exits, which aborts it with "double free or corruption" and exit
     status 134 after the work is done. Loading pyproj's copy first avoids it.
-    A pyproj that fails to import is left for whoever uses it to report.
+    A pyproj that fails to import is left for whoever uses it to report. This is
+    a workaround for https://github.com/ecmwf/eckit/issues/354; once an eckitlib
+    wheel no longer clashes with pyproj's PROJ, it can go.
 
     Raises:
         ModuleNotFoundError: eccodes is not installed.
@@ -481,8 +483,11 @@ def open_grib2(
     needs_select = select is None and not fetched.provenance.is_partial
     object_messages = fetched.provenance.object_messages
     selectors = fetched.provenance.selectors if object_messages else []
+    fields_seen: dict[int, int] = {}
     for index, h in _messages(eccodes, fetched.path):
         count += 1
+        # Fields of one message share its index; the nth of them is the index's field n+1.
+        field = fields_seen[index] = fields_seen.get(index, -1) + 1
         if count > 1 and needs_select:
             continue
         hits = {key: _matched(eccodes, h, key, wanted) for key, wanted in options.items()}
@@ -536,7 +541,7 @@ def open_grib2(
             "step": attrs.get("step"),
         }
         if selectors:
-            message["selector"] = selectors[index]
+            message["selector"] = fetched.provenance.field_selector(index, field)
         short = base_name(
             message["shortName"], [attrs.get(key) for key in PARAMETER_KEYS], fetched.asset.id
         )

{usdata-0.32.0 → usdata-0.34.0}/src/usdata/cli/app.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from collections.abc import Iterable
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Any
 
 import httpx
 import typer
@@ -20,6 +20,7 @@ from usdata.cli.doctor import doctor
 from usdata.cli.inspect import inspect
 from usdata.cli.progress import progress
 from usdata.manifest import lockfile_path
+from usdata.messages import list_messages
 from usdata.models import READER_EXTRAS_TEXT, Dataset, Status, describe_duration
 from usdata.providers import adapter_class, load_adapter
 from usdata.providers.base import NotImplementedProvider
@@ -67,6 +68,33 @@ _QUERY_FLAGS = {
 }
 
 
+def _parse_params(
+    param: list[str] | None, command: str, flags: dict[str, str | None]
+) -> dict[str, Any]:
+    """``--param key=value`` pairs as a dict of strings; exit 2 on a malformed or reserved key.
+
+    ``flags`` maps each reserved query option to the flag ``command`` takes it
+    through, or to ``None`` when ``command`` does not take it.
+    """
+    params: dict[str, Any] = {}
+    for item in param or []:
+        key, sep, value = item.partition("=")
+        if not sep:
+            typer.secho(f"--param expects key=value, got {item!r}", err=True, fg="red")
+            raise typer.Exit(code=2)
+        if key in flags:
+            flag = flags[key]
+            hint = (
+                f"use {flag} instead of --param"
+                if flag is not None
+                else f"{command} does not support this option"
+            )
+            typer.secho(f"{key} is a reserved query option; {hint}", err=True, fg="red")
+            raise typer.Exit(code=2)
+        params[key] = value
+    return params
+
+
 def _size_phrase(known_bytes: int, unknown: int, *, sources: Iterable[str] = ()) -> str:
     """How many bytes a dry run would move, said honestly when some size is missing.
 
@@ -357,26 +385,7 @@ def fetch(
     ] = False,
 ) -> None:
     """Resolve a query against one dataset and download the matching assets."""
-    params: dict[str, str] = {}
-    for item in param or []:
-        key, sep, value = item.partition("=")
-        if not sep:
-            typer.secho(f"--param expects key=value, got {item!r}", err=True, fg="red")
-            raise typer.Exit(code=2)
-        if key in _QUERY_FLAGS:
-            flag = _QUERY_FLAGS[key]
-            hint = (
-                f"use {flag} instead of --param"
-                if flag is not None
-                else "fetch does not support this option"
-            )
-            typer.secho(
-                f"{key} is a reserved query option; {hint}",
-                err=True,
-                fg="red",
-            )
-            raise typer.Exit(code=2)
-        params[key] = value
+    params = _parse_params(param, "fetch", _QUERY_FLAGS)
     box = None
     if bbox:
         try:
@@ -437,6 +446,64 @@ def fetch(
         typer.echo(f"{f.path}\t{tag}\t{f.provenance.size} bytes")
 
 
+_MESSAGES_FLAGS: dict[str, str | None] = {
+    key: flag if key in ("start", "end") else None for key, flag in _QUERY_FLAGS.items()
+}
+"""The query options ``messages`` takes: a model run is selected by its window alone."""
+
+
+@app.command()
+def messages(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset id, e.g. noaa:hrrr")],
+    start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
+    end: Annotated[
+        str | None,
+        typer.Option(help="ISO date or datetime; a date alone runs to the end of that UTC day."),
+    ] = None,
+    param: Annotated[
+        list[str] | None,
+        typer.Option("--param", "-p", help="Provider-specific key=value, repeatable."),
+    ] = None,
+    as_json: Annotated[
+        bool,
+        typer.Option("--json", help="Emit a JSON array of listings and nothing else on stdout."),
+    ] = False,
+) -> None:
+    """List the GRIB2 messages each file of a query holds, spelled as the messages param takes them.
+
+    One tab-separated line per field: asset id, message number, bytes, and the
+    selector that names exactly that field. Only the index sidecars are read.
+    """
+    params = _parse_params(param, "messages", _MESSAGES_FLAGS)
+    try:
+        ds = default_registry().get(dataset_id)
+        listings = list_messages(ds, build_query(start=start, end=end, **params))
+    except (DatasetNotFound, ValueError) as e:
+        typer.secho(str(e), err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    except NotImplementedProvider as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=3) from None
+    except httpx.HTTPError as e:
+        typer.secho(f"request failed: {e}", err=True, fg="red")
+        raise typer.Exit(code=4) from None
+    if as_json:
+        typer.echo(json.dumps([listing.model_dump(mode="json") for listing in listings], indent=2))
+    else:
+        for listing in listings:
+            for entry in listing.messages:
+                number = entry.number if entry.field is None else f"{entry.number}.{entry.field}"
+                typer.echo(f"{listing.asset.id}\t{number}\t{entry.length}\t{entry.selector}")
+    count = sum(len(listing.messages) for listing in listings)
+    typer.echo(
+        f"{count} message(s) in {len(listings)} file(s); a selector may stop after its "
+        "level text, as CAPE:surface, to match every step",
+        err=True,
+    )
+    if not listings:
+        raise typer.Exit(code=1)
+
+
 def _print_plan(plan: Plan, *, as_json: bool, quiet: bool) -> None:
     """Report what a pull would fetch: assets, a subtotal per source, a total on stderr."""
     if as_json:
@@ -574,7 +641,10 @@ def verify(
         bool, typer.Option("--json", help="With --listing, emit the comparison as JSON on stdout.")
     ] = False,
 ) -> None:
-    """Check cached files against a manifest's lockfile. Exit 1 on any drift."""
+    """Check cached files, or with --listing today's source listings, against the lockfile.
+
+    Exit 1 on any drift.
+    """
     if as_json and not listing:
         typer.secho("--json applies to --listing only", err=True, fg="red")
         raise typer.Exit(code=2)
@@ -605,9 +675,6 @@ def _verify_listing(manifest: Path, *, as_json: bool) -> None:
     """Compare the lockfile with today's listings; exit 1 when a re-resolve would differ."""
     try:
         comparison = compare_manifest_listing(manifest)
-    except EmptySource as e:
-        typer.secho(str(e), err=True, fg="yellow")
-        raise typer.Exit(code=1) from None
     except (DatasetNotFound, UnknownDatasets, ManifestChanged, UnknownPlace, ValueError) as e:
         typer.secho(str(e), err=True, fg="red")
         raise typer.Exit(code=2) from None