usdata 0.32.0__tar.gz → 0.33.0__tar.gz

{usdata-0.32.0 → usdata-0.33.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.32.0
+Version: 0.33.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -113,7 +113,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-eight datasets are available today and eighteen more are planned, grouped
+Thirty-one datasets are available today and seventeen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.32.0 → usdata-0.33.0}/README.md

@@ -68,7 +68,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-eight datasets are available today and eighteen more are planned, grouped
+Thirty-one datasets are available today and seventeen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.32.0 → usdata-0.33.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.32.0"
+version = "0.33.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.32.0 → usdata-0.33.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.32.0"
+version = "0.33.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/__init__.py

@@ -13,6 +13,7 @@ except PackageNotFoundError:  # running from a source tree without an install
 from usdata._fetch import ChecksumMismatch, FetchedAsset, fetch, fetch_asset
 from usdata.cite import Citation, cite_dataset, cite_lockfile
 from usdata.inspect import inspect_asset, inspect_path
+from usdata.messages import MessageListing, list_messages
 from usdata.models import (
     Asset,
     BBox,
@@ -36,6 +37,7 @@ __all__ = [
     "Dataset",
     "DatasetNotFound",
     "FetchedAsset",
+    "MessageListing",
     "Place",
     "Provenance",
     "Query",
@@ -55,6 +57,7 @@ __all__ = [
     "get",
     "inspect_asset",
     "inspect_path",
+    "list_messages",
     "pull",
     "search",
     "select_by_time",

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/_grib.py

@@ -96,7 +96,9 @@ def import_eccodes() -> Any:
     imported later in the same process, both copies free the same memory as the
     interpreter exits, which aborts it with "double free or corruption" and exit
     status 134 after the work is done. Loading pyproj's copy first avoids it.
-    A pyproj that fails to import is left for whoever uses it to report.
+    A pyproj that fails to import is left for whoever uses it to report. This is
+    a workaround for https://github.com/ecmwf/eckit/issues/354; once an eckitlib
+    wheel no longer clashes with pyproj's PROJ, it can go.
 
     Raises:
         ModuleNotFoundError: eccodes is not installed.
@@ -481,8 +483,11 @@ def open_grib2(
     needs_select = select is None and not fetched.provenance.is_partial
     object_messages = fetched.provenance.object_messages
     selectors = fetched.provenance.selectors if object_messages else []
+    fields_seen: dict[int, int] = {}
     for index, h in _messages(eccodes, fetched.path):
         count += 1
+        # Fields of one message share its index; the nth of them is the index's field n+1.
+        field = fields_seen[index] = fields_seen.get(index, -1) + 1
         if count > 1 and needs_select:
             continue
         hits = {key: _matched(eccodes, h, key, wanted) for key, wanted in options.items()}
@@ -536,7 +541,7 @@ def open_grib2(
             "step": attrs.get("step"),
         }
         if selectors:
-            message["selector"] = selectors[index]
+            message["selector"] = fetched.provenance.field_selector(index, field)
         short = base_name(
             message["shortName"], [attrs.get(key) for key in PARAMETER_KEYS], fetched.asset.id
         )

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/cli/app.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import json
 from collections.abc import Iterable
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Any
 
 import httpx
 import typer
@@ -20,6 +20,7 @@ from usdata.cli.doctor import doctor
 from usdata.cli.inspect import inspect
 from usdata.cli.progress import progress
 from usdata.manifest import lockfile_path
+from usdata.messages import list_messages
 from usdata.models import READER_EXTRAS_TEXT, Dataset, Status, describe_duration
 from usdata.providers import adapter_class, load_adapter
 from usdata.providers.base import NotImplementedProvider
@@ -67,6 +68,33 @@ _QUERY_FLAGS = {
 }
 
 
+def _parse_params(
+    param: list[str] | None, command: str, flags: dict[str, str | None]
+) -> dict[str, Any]:
+    """``--param key=value`` pairs as a dict of strings; exit 2 on a malformed or reserved key.
+
+    ``flags`` maps each reserved query option to the flag ``command`` takes it
+    through, or to ``None`` when ``command`` does not take it.
+    """
+    params: dict[str, Any] = {}
+    for item in param or []:
+        key, sep, value = item.partition("=")
+        if not sep:
+            typer.secho(f"--param expects key=value, got {item!r}", err=True, fg="red")
+            raise typer.Exit(code=2)
+        if key in flags:
+            flag = flags[key]
+            hint = (
+                f"use {flag} instead of --param"
+                if flag is not None
+                else f"{command} does not support this option"
+            )
+            typer.secho(f"{key} is a reserved query option; {hint}", err=True, fg="red")
+            raise typer.Exit(code=2)
+        params[key] = value
+    return params
+
+
 def _size_phrase(known_bytes: int, unknown: int, *, sources: Iterable[str] = ()) -> str:
     """How many bytes a dry run would move, said honestly when some size is missing.
 
@@ -357,26 +385,7 @@ def fetch(
     ] = False,
 ) -> None:
     """Resolve a query against one dataset and download the matching assets."""
-    params: dict[str, str] = {}
-    for item in param or []:
-        key, sep, value = item.partition("=")
-        if not sep:
-            typer.secho(f"--param expects key=value, got {item!r}", err=True, fg="red")
-            raise typer.Exit(code=2)
-        if key in _QUERY_FLAGS:
-            flag = _QUERY_FLAGS[key]
-            hint = (
-                f"use {flag} instead of --param"
-                if flag is not None
-                else "fetch does not support this option"
-            )
-            typer.secho(
-                f"{key} is a reserved query option; {hint}",
-                err=True,
-                fg="red",
-            )
-            raise typer.Exit(code=2)
-        params[key] = value
+    params = _parse_params(param, "fetch", _QUERY_FLAGS)
     box = None
     if bbox:
         try:
@@ -437,6 +446,64 @@ def fetch(
         typer.echo(f"{f.path}\t{tag}\t{f.provenance.size} bytes")
 
 
+_MESSAGES_FLAGS: dict[str, str | None] = {
+    key: flag if key in ("start", "end") else None for key, flag in _QUERY_FLAGS.items()
+}
+"""The query options ``messages`` takes: a model run is selected by its window alone."""
+
+
+@app.command()
+def messages(
+    dataset_id: Annotated[str, typer.Argument(help="Dataset id, e.g. noaa:hrrr")],
+    start: Annotated[str | None, typer.Option(help="ISO date or datetime.")] = None,
+    end: Annotated[
+        str | None,
+        typer.Option(help="ISO date or datetime; a date alone runs to the end of that UTC day."),
+    ] = None,
+    param: Annotated[
+        list[str] | None,
+        typer.Option("--param", "-p", help="Provider-specific key=value, repeatable."),
+    ] = None,
+    as_json: Annotated[
+        bool,
+        typer.Option("--json", help="Emit a JSON array of listings and nothing else on stdout."),
+    ] = False,
+) -> None:
+    """List the GRIB2 messages each file of a query holds, spelled as the messages param takes them.
+
+    One tab-separated line per field: asset id, message number, bytes, and the
+    selector that names exactly that field. Only the index sidecars are read.
+    """
+    params = _parse_params(param, "messages", _MESSAGES_FLAGS)
+    try:
+        ds = default_registry().get(dataset_id)
+        listings = list_messages(ds, build_query(start=start, end=end, **params))
+    except (DatasetNotFound, ValueError) as e:
+        typer.secho(str(e), err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    except NotImplementedProvider as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=3) from None
+    except httpx.HTTPError as e:
+        typer.secho(f"request failed: {e}", err=True, fg="red")
+        raise typer.Exit(code=4) from None
+    if as_json:
+        typer.echo(json.dumps([listing.model_dump(mode="json") for listing in listings], indent=2))
+    else:
+        for listing in listings:
+            for entry in listing.messages:
+                number = entry.number if entry.field is None else f"{entry.number}.{entry.field}"
+                typer.echo(f"{listing.asset.id}\t{number}\t{entry.length}\t{entry.selector}")
+    count = sum(len(listing.messages) for listing in listings)
+    typer.echo(
+        f"{count} message(s) in {len(listings)} file(s); a selector may stop after its "
+        "level text, as CAPE:surface, to match every step",
+        err=True,
+    )
+    if not listings:
+        raise typer.Exit(code=1)
+
+
 def _print_plan(plan: Plan, *, as_json: bool, quiet: bool) -> None:
     """Report what a pull would fetch: assets, a subtotal per source, a total on stderr."""
     if as_json:
@@ -574,7 +641,10 @@ def verify(
         bool, typer.Option("--json", help="With --listing, emit the comparison as JSON on stdout.")
     ] = False,
 ) -> None:
-    """Check cached files against a manifest's lockfile. Exit 1 on any drift."""
+    """Check cached files, or with --listing today's source listings, against the lockfile.
+
+    Exit 1 on any drift.
+    """
     if as_json and not listing:
         typer.secho("--json applies to --listing only", err=True, fg="red")
         raise typer.Exit(code=2)
@@ -605,9 +675,6 @@ def _verify_listing(manifest: Path, *, as_json: bool) -> None:
     """Compare the lockfile with today's listings; exit 1 when a re-resolve would differ."""
     try:
         comparison = compare_manifest_listing(manifest)
-    except EmptySource as e:
-        typer.secho(str(e), err=True, fg="yellow")
-        raise typer.Exit(code=1) from None
     except (DatasetNotFound, UnknownDatasets, ManifestChanged, UnknownPlace, ValueError) as e:
         typer.secho(str(e), err=True, fg="red")
         raise typer.Exit(code=2) from None

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/data/registry.yaml

@@ -732,6 +732,52 @@ datasets:
       - { name: "TAVG", units: "degrees Celsius", description: "Monthly mean temperature (metric units)" }
     adapter: usdata.providers.noaa.gsom:GlobalSummaryMonthly
 
+  - id: noaa:gsom-station-files
+    provider: noaa
+    status: available
+    domain: surface-weather
+    since: "0.33"
+    title: Global Summary of the Month Station Files
+    description: >-
+      The same Global Summary of the Month as noaa:gsom, as NCEI's static per-station files:
+      one CSV per station holding its whole monthly record, every element with its attributes,
+      in metric units. Each station is one plain HTTPS GET that does not go through the Access
+      Data Service, which suits pulling many stations' long records. There is no selection by
+      time, place, or element; name stations and select months and columns after opening.
+      Variables are NCEI element codes, the catalog lists the common ones, and the delivered
+      set is open and depends on the station.
+    keywords: [climate, monthly, stations, snowfall, temperature, precipitation, gsom, ncei]
+    protocol: http
+    homepage: https://www.ncei.noaa.gov/data/global-summary-of-the-month/
+    license: US Government Work (public domain)
+    spatial_extent: { west: -180.0, south: -90.0, east: 180.0, north: 90.0 }
+    temporal_extent: { start: "1763-01-01T00:00:00Z" }
+    capabilities: { spatial_subset: false, temporal_subset: false, variable_subset: false }
+    summary: Whole monthly station records, one file per station
+    formats: [CSV]
+    selection: One whole file per named station, every month and element it holds
+    inputs: Station IDs; no dates, geographic filters, or variables
+    reader: pandas
+    guide: docs/providers/noaa-gsom-station-files.md
+    examples:
+      - examples/datasets/noaa-gsom-station-files/noaa-gsom-station-files.ipynb
+      - examples/studies/el-nino-snowfall/el-nino-snowfall.ipynb
+    resolution:
+      spatial: "Land surface stations, derived from GHCN-Daily"
+      temporal: "Monthly"
+    update_frequency: "Weekly"
+    citation: >-
+      Lawrimore, Jay H.; Ray, Ron; Applequist, Scott; Korzeniewski, Bryant; Menne, Matthew J.
+      (2016): Global Summary of the Month (GSOM), Version 1. NOAA National Centers for
+      Environmental Information. https://doi.org/10.7289/V5QV3JJ5
+    terms: https://www.ncei.noaa.gov/metadata/geoportal/rest/metadata/item/gov.noaa.ncdc:C00946/html
+    variables:
+      - { name: "PRCP", units: "mm", description: "Monthly precipitation total" }
+      - { name: "SNOW", units: "mm", description: "Monthly snowfall total" }
+      - { name: "SNOW_ATTRIBUTES", description: "Days missing, measurement, quality, and source flags for SNOW" }
+      - { name: "TAVG", units: "degrees Celsius", description: "Monthly mean temperature" }
+    adapter: usdata.providers.noaa.gsom_files:GsomStationFiles
+
   - id: noaa:gsoy
     provider: noaa
     status: available
@@ -1125,19 +1171,93 @@ datasets:
 
   - id: noaa:ersst
     provider: noaa
-    status: planned
+    status: available
     domain: ocean-physics
-    target: later
-    title: ERSST Monthly Sea Surface Temperature
+    since: "0.33"
+    title: ERSST v6 Monthly Sea Surface Temperature
     description: >-
-      Extended Reconstructed SST v5: monthly global 2 degree analysis since
-      1854, one NetCDF per month in an NCEI HTTPS directory.
-    keywords: [sst, sea surface temperature, ocean, climate, ersst, monthly, netcdf]
+      Extended Reconstructed Sea Surface Temperature version 6: NCEI's monthly global
+      analysis on a 2 degree grid since January 1850, built from ship, buoy, and Argo
+      observations. One NetCDF4 file per month in an anonymous NCEI HTTPS directory, holding
+      sea surface temperature and its anomaly from the 1991-2020 ERSSTv6 climatology. Every
+      month a window touches is selected whole; there is no server-side spatial or variable
+      subsetting. Each monthly update writes the month just ended and rewrites the month
+      before it, so the newest file is preliminary.
+    keywords: [sst, sea surface temperature, ocean, climate, ersst, monthly, netcdf, enso, el nino]
     protocol: http
     homepage: https://www.ncei.noaa.gov/products/extended-reconstructed-sst
     license: US Government Work (public domain)
-    temporal_extent: { start: "1854-01-01T00:00:00Z" }
+    spatial_extent: { west: -180.0, south: -89.0, east: 180.0, north: 89.0 }
+    temporal_extent: { start: "1850-01-01T00:00:00Z" }
     capabilities: { spatial_subset: false, temporal_subset: true, variable_subset: false }
+    summary: Monthly global 2 degree sea surface temperature since 1850
+    formats: [NetCDF4]
+    selection: Whole global monthly files, one for every month the window touches
+    inputs: Both dates (selects the months they span)
+    reader: netcdf
+    guide: docs/providers/noaa-ersst.md
+    examples:
+      - examples/datasets/noaa-ersst/noaa-ersst.ipynb
+    resolution:
+      spatial: "2.0 degree grid"
+      temporal: "Monthly means"
+    update_frequency: >-
+      Monthly, around the third: each update writes the month just ended and rewrites the
+      month before it
+    citation: >-
+      Huang, B., X. Yin, T. Boyer, C. Liu, M. Menne, Y. D. Rao, T. Smith, R. Vose, and
+      H.-M. Zhang, 2025: Extended Reconstructed Sea Surface Temperature Version 6 (ERSSTv6):
+      Part I. An Artificial Neural Network Approach. Journal of Climate, 38, 1105-1121,
+      doi:10.1175/JCLI-D-23-0707.1
+    terms: https://www.ncei.noaa.gov/products/extended-reconstructed-sst
+    variables:
+      - { name: "sst", units: "degree_C", description: "Extended reconstructed sea surface temperature" }
+      - { name: "ssta", units: "degree_C", description: "Anomaly from the 1991-2020 ERSSTv6 climatology" }
+    adapter: usdata.providers.noaa.ersst:Ersst
+
+  - id: noaa:enso-indices
+    provider: noaa
+    status: available
+    domain: climate
+    since: "0.33"
+    title: CPC ENSO Indices (RONI and ONI)
+    description: >-
+      The Climate Prediction Center's seasonal El Niño-Southern Oscillation indices: the
+      Relative Oceanic Niño Index (RONI), which NOAA has used for official ENSO monitoring
+      since 1 February 2026, and the traditional Oceanic Niño Index (ONI). Both are
+      three-month running means of Niño 3.4 sea surface temperature anomalies from ERSST v6;
+      RONI first subtracts the tropical mean anomaly. Each index is one whole plain-text table
+      from DJF 1950, rewritten in place each month, with no server-side selection.
+    keywords: [enso, el nino, la nina, oni, roni, nino 3.4, climate index, cpc, sea surface temperature]
+    protocol: http
+    homepage: https://www.cpc.ncep.noaa.gov/products/analysis_monitoring/enso/roni/
+    license: US Government Work (public domain)
+    spatial_extent: { west: -170.0, south: -5.0, east: -120.0, north: 5.0 }
+    temporal_extent: { start: "1949-12-01T00:00:00Z" }
+    capabilities: { spatial_subset: false, temporal_subset: false, variable_subset: false }
+    summary: Seasonal El Niño indices, the official RONI and the traditional ONI
+    formats: [whitespace-delimited text (no reader)]
+    selection: One whole index table, every season from DJF 1950; select seasons locally
+    inputs: Optional index (roni or oni); no dates or geographic filters
+    reader: null
+    guide: docs/providers/noaa-enso-indices.md
+    examples:
+      - examples/datasets/noaa-enso-indices/noaa-enso-indices.ipynb
+      - examples/studies/el-nino-snowfall/el-nino-snowfall.ipynb
+    resolution:
+      spatial: "One value for the Niño 3.4 region, 5N-5S and 170W-120W"
+      temporal: "Overlapping three-month seasons, one per month"
+    update_frequency: "Monthly, by the 5th"
+    citation: >-
+      NOAA Climate Prediction Center, Relative Oceanic Niño Index and Oceanic Niño Index,
+      accessed via usdata
+    terms: https://www.weather.gov/disclaimer
+    variables:
+      - { name: "SEAS", description: "Three-month season, such as DJF" }
+      - { name: "YR", description: "Year of the season's middle month" }
+      - { name: "ANOM", units: "degree_C", description: "Index value: the season's anomaly" }
+      - { name: "TOTAL", units: "degree_C", description: "Niño 3.4 mean temperature; ONI table only" }
+    adapter: usdata.providers.noaa.enso:EnsoIndices
 
   - id: noaa:etopo
     provider: noaa

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/inspect.py

@@ -137,7 +137,8 @@ class Grib2Summary(BaseModel):
             KeyError: No message here was fetched for that selector; the message
                 lists the selectors that were.
             ValueError: That selector fetched a message holding several fields,
-                as RAP publishes wind components, and the record cannot say
+                as RAP publishes wind components, and the record was written
+                before each field's selector was recorded, so it cannot say
                 which field it named; the message lists their variables.
         """
         from usdata._grib import variable_names
@@ -158,7 +159,7 @@ class Grib2Summary(BaseModel):
             if message.selector == selector
         ]
         if len(matched) > 1:
-            # The record keeps one selector per byte range, and one range can hold several fields.
+            # An older record keeps one selector per range, and one range can hold several fields.
             raise ValueError(
                 f"selector {selector!r} fetched one message holding {len(matched)} fields "
                 f"({', '.join(matched)}); pick the variable by name"
@@ -333,24 +334,26 @@ def _paired(messages: list[GribMessage], record: Provenance) -> list[GribMessage
     """Each message as the fetch that took it described it, where provenance says.
 
     The file itself carries neither number nor selector: a partial fetch recorded
-    one message number and one selector per range, and ``file_index`` counts the
-    file's messages the same way, so the fields of a message that holds several
-    all pair with its number. A whole file, or a record that does not pair,
-    leaves both unset.
+    one message number per range and one selector per field, and ``file_index``
+    counts the file's messages the same way, so the fields of a message that
+    holds several all pair with its number and each with its own selector. A
+    whole file, or a record that does not pair, leaves both unset.
     """
     numbers = record.object_messages
     if not numbers or any(message.file_index >= len(numbers) for message in messages):
         return messages
-    selectors = record.selectors if len(record.selectors) == len(numbers) else []
-    return [
-        message.model_copy(
-            update={
-                "object_index": numbers[message.file_index],
-                "selector": selectors[message.file_index] if selectors else None,
-            }
+    paired = len(record.selectors) == len(numbers)
+    fields_seen: dict[int, int] = {}
+    result = []
+    for message in messages:
+        field = fields_seen[message.file_index] = fields_seen.get(message.file_index, -1) + 1
+        selector = record.field_selector(message.file_index, field) if paired else None
+        result.append(
+            message.model_copy(
+                update={"object_index": numbers[message.file_index], "selector": selector}
+            )
         )
-        for message in messages
-    ]
+    return result
 
 
 def _csv_summary(path: Path, *, units_row: bool = False) -> CsvSummary:

usdata-0.33.0/src/usdata/messages.py

@@ -0,0 +1,60 @@
+"""The GRIB2 messages a query's files hold, read from their index sidecars.
+
+A dataset declaring ``partial_fetch`` takes a ``messages`` parameter spelled in
+its objects' index vocabulary, and this is where that vocabulary is listed.
+Listing resolves the query as a fetch would and reads each object's index,
+which is a few kilobytes; nothing else is downloaded, and nothing is cached or
+recorded. See ADR 0046.
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, Field
+
+from usdata.models import Asset, Dataset, IndexEntry, Query
+from usdata.providers import QueryError, load_adapter
+
+PARTIAL_PARAM = "messages"
+"""The parameter a ``partial_fetch`` dataset selects messages through."""
+
+
+class MessageListing(BaseModel):
+    """One file a query resolved to, and every field its index describes."""
+
+    asset: Asset = Field(description="The whole-file asset, as a fetch without messages lists it")
+    messages: list[IndexEntry] = Field(description="Every field in the object's index, in order")
+
+
+def list_messages(dataset: Dataset, query: Query) -> list[MessageListing]:
+    """List each file a query resolves to with the fields its index describes.
+
+    Each entry's ``selector`` is a ``messages`` value that names exactly that
+    field. A selector may stop after its level text, as ``CAPE:surface``, to
+    name that field at whatever step each file holds.
+
+    Args:
+        dataset: A dataset declaring ``partial_fetch``.
+        query: The query a fetch would take, without ``messages``.
+
+    Returns:
+        One listing per file, in the order a fetch would list them.
+
+    Raises:
+        QueryError: The dataset selects no messages, the query names
+            ``messages``, or a file publishes no readable index.
+    """
+    if not dataset.capabilities.partial_fetch:
+        raise QueryError(
+            f"{dataset.id} fetches whole files, so it has no messages to list; "
+            "only datasets that declare partial_fetch do"
+        )
+    if PARTIAL_PARAM in query.params:
+        raise QueryError(
+            f"drop {PARTIAL_PARAM}: the listing shows every message each file holds, "
+            "whatever a selection would keep"
+        )
+    with load_adapter(dataset) as adapter:
+        return [
+            MessageListing(asset=asset, messages=adapter.list_messages(asset))
+            for asset in adapter.list_assets(query)
+        ]

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/models.py

@@ -560,6 +560,42 @@ class ByteRange(BaseModel):
         return f"bytes={self.start}-{self.end}"
 
 
+class IndexEntry(BaseModel):
+    """One field of one GRIB2 object, as the object's wgrib2 ``.idx`` sidecar describes it.
+
+    ``number`` is the GRIB2 message the field belongs to and ``field`` its
+    position inside a message that holds several (``12.2``), or ``None`` for a
+    message holding one. Fields of one message share an offset and a length.
+    ``selector`` is the ``messages`` text that names exactly this field.
+    """
+
+    number: int = Field(ge=1, description="One-based GRIB2 message number in the object")
+    offset: int = Field(ge=0, description="Byte offset of the message in the object")
+    length: int = Field(gt=0, description="Bytes the message occupies")
+    short_name: str = Field(description="wgrib2 short name, such as CAPE")
+    level: str = Field(description="Level text, such as '2 m above ground'")
+    step: str = Field(description="Step text, such as 'anl' or '1 hour fcst'")
+    field: int | None = Field(
+        default=None, ge=1, description="Position inside a message holding several fields"
+    )
+    extra: str = Field(
+        default="", description="Further text some products add, such as 'ens std dev'"
+    )
+
+    @property
+    def byte_range(self) -> ByteRange:
+        """The inclusive byte interval this field's message occupies in the object."""
+        return ByteRange(start=self.offset, end=self.offset + self.length - 1)
+
+    @property
+    def selector(self) -> str:
+        """The ``messages`` selector naming exactly this field, short name through the last text."""
+        parts = [self.short_name, self.level, self.step]
+        if self.extra:
+            parts.append(self.extra)
+        return ":".join(parts)
+
+
 PARTIAL_FRAGMENT = "messages"
 """Href fragment key naming the GRIB2 messages a partial asset selects."""
 
@@ -583,6 +619,13 @@ class PartialFetch(BaseModel):
         default_factory=list,
         description="Index selector naming each of those messages, in the same order",
     )
+    field_selectors: list[list[str]] = Field(
+        default_factory=list,
+        description=(
+            "Index selector naming each field of each of those messages, in the same order; "
+            "a message holding several fields, as RAP packs wind components, has one per field"
+        ),
+    )
 
     @model_validator(mode="after")
     def _aligned(self) -> PartialFetch:
@@ -590,6 +633,10 @@ class PartialFetch(BaseModel):
             raise ValueError("a partial fetch needs one byte range per selected message")
         if self.selectors and len(self.selectors) != len(self.messages):
             raise ValueError("a partial fetch needs one selector per selected message")
+        if self.field_selectors and (
+            len(self.field_selectors) != len(self.messages) or not all(self.field_selectors)
+        ):
+            raise ValueError("a partial fetch needs one or more field selectors per message")
         if sorted(set(self.messages)) != self.messages:
             raise ValueError("selected messages must be ascending and distinct")
         return self
@@ -655,6 +702,13 @@ class Provenance(BaseModel):
         default_factory=list,
         description="Index selector each of those ranges was fetched for, in the same order",
     )
+    field_selectors: list[list[str]] = Field(
+        default_factory=list,
+        description=(
+            "Index selector naming each field each of those ranges holds, in the same order; "
+            "empty in a record written before fields were recorded"
+        ),
+    )
     object_size: int | None = Field(
         default=None, ge=0, description="Size of the whole object those ranges came from"
     )
@@ -680,6 +734,19 @@ class Provenance(BaseModel):
         """Whether this record describes selected byte ranges rather than a whole object."""
         return any(entry.startswith(PARTIAL_TRANSFORMATION) for entry in self.transformations)
 
+    def field_selector(self, message: int, field: int) -> str | None:
+        """The selector one field on disk was fetched for, or ``None`` where none was recorded.
+
+        ``message`` counts the local file's messages from zero and ``field`` the
+        fields inside that message from zero, as a reader meets them. A record
+        that names no fields, as one written before they were recorded does,
+        gives every field of a message that message's one selector.
+        """
+        fields = self.field_selectors[message] if message < len(self.field_selectors) else []
+        if field < len(fields):
+            return fields[field]
+        return self.selectors[message] if message < len(self.selectors) else None
+
     @property
     def object_messages(self) -> list[int]:
         """The source object's own message numbers, in the order the local file holds them.

{usdata-0.32.0 → usdata-0.33.0}/src/usdata/provenance.py

@@ -50,6 +50,7 @@ def record(
         index_checksum=None if partial is None else partial.index_checksum,
         ranges=[] if partial is None else list(partial.ranges),
         selectors=[] if partial is None else list(partial.selectors),
+        field_selectors=[] if partial is None else [list(f) for f in partial.field_selectors],
         object_size=None if partial is None else partial.object_size,
         object_etag=None if partial is None else partial.object_etag,
         credentials=[] if dataset.credentials is None else list(dataset.credentials.variables),