usdata 0.19.0__tar.gz → 0.20.0__tar.gz

{usdata-0.19.0 → usdata-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.19.0
+Version: 0.20.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -93,7 +93,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty datasets are available today and twenty-five more are planned, grouped
+Twenty-three datasets are available today and twenty-one more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.19.0 → usdata-0.20.0}/README.md

@@ -48,7 +48,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty datasets are available today and twenty-five more are planned, grouped
+Twenty-three datasets are available today and twenty-one more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.19.0 → usdata-0.20.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.19.0"
+version = "0.20.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.19.0 → usdata-0.20.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.19.0"
+version = "0.20.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.19.0 → usdata-0.20.0}/src/usdata/_fetch.py

@@ -130,6 +130,8 @@ def _fetch_asset(
             and prov.provider == dataset.provider
             and prov.source_url == asset.href
             and prov.size == path.stat().st_size
+            # A listing that now reports another size describes a rebuilt file.
+            and (asset.size is None or prov.size == asset.size)
             and (asset.checksum is None or prov.checksum == asset.checksum)
             # Trust an untouched cached file; hash whenever anything is unclear.
             and (_sidecar_not_older(path) or sha256_file(path) == prov.checksum)
@@ -138,7 +140,10 @@ def _fetch_asset(
             return FetchedAsset(asset=asset, path=path, provenance=prov, from_cache=True)
     with staged_path(path) as tmp:
         partial = adapter.prepare_fetch(asset, pinned)
-        adapter.fetch(asset, tmp)
+        if partial is None:
+            adapter.fetch(asset, tmp)
+        else:
+            adapter.fetch_partial(asset, tmp, partial)
         prov = provenance.record(dataset, asset, tmp, partial)
         if asset.checksum and prov.checksum != asset.checksum:
             raise ChecksumMismatch(f"{asset.id}: expected {asset.checksum}, got {prov.checksum}")

{usdata-0.19.0 → usdata-0.20.0}/src/usdata/_grib.py

@@ -81,6 +81,9 @@ def _modules() -> tuple[Any, Any, Any]:
     except (RuntimeError, OSError) as error:
         # eccodes imports, then findlibs fails to locate the shared library.
         raise MissingReaderDependency(f"{LIBRARY_HINT} ({error})") from error
+    # A GRIB2 message may hold several fields (RAP packs wind components that way);
+    # without this ecCodes yields only the first and silently drops the rest.
+    eccodes.codes_grib_multi_support_on()
     return eccodes, xarray, numpy
 
 
@@ -92,9 +95,15 @@ def _handle(eccodes: Any, handle: int) -> Iterator[int]:
         eccodes.codes_release(handle)
 
 
-def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[int]:
-    """Handles for each message in an in-memory GRIB byte string, using section 0 lengths."""
+def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[tuple[int, int]]:
+    """Message index and handle for each field in an in-memory GRIB byte string.
+
+    Messages are walked by their section 0 lengths; an in-memory message yields
+    its first field only, which is every field of the single-field products
+    that arrive compressed.
+    """
     offset = 0
+    index = 0
     while offset < len(data):
         if data[offset : offset + 4] != b"GRIB":
             raise ValueError(f"not a GRIB message at byte {offset}")
@@ -104,12 +113,18 @@ def _messages_from_bytes(eccodes: Any, data: bytes) -> Iterator[int]:
         if length < 16 or offset + length > len(data):
             raise ValueError(f"truncated GRIB message at byte {offset}")
         with _handle(eccodes, eccodes.codes_new_from_message(data[offset : offset + length])) as h:
-            yield h
+            yield index, h
         offset += length
+        index += 1
 
 
-def _messages(eccodes: Any, path: Path) -> Iterator[int]:
-    """Handles for each message, decompressing gzip in memory without changing cached bytes."""
+def _messages(eccodes: Any, path: Path) -> Iterator[tuple[int, int]]:
+    """Message index and handle for each field, decompressing gzip in memory.
+
+    The index counts GRIB2 messages from zero, so the fields of one message
+    that holds several share it; that is the number a partial fetch's record
+    pairs with, and the ``file_index`` the reader and ``inspect`` report.
+    """
     with path.open("rb") as probe:
         compressed = probe.read(2) == b"\x1f\x8b"
     if compressed:
@@ -117,12 +132,18 @@ def _messages(eccodes: Any, path: Path) -> Iterator[int]:
         yield from _messages_from_bytes(eccodes, data)
         return
     # ecCodes reads through the descriptor, so it gets a file object it alone positions.
+    index = -1
+    last_offset: int | None = None
     with path.open("rb") as raw:
         while (handle := eccodes.codes_grib_new_from_file(raw)) is not None:
             with _handle(eccodes, handle) as h:
                 if eccodes.codes_get(h, "edition") != 2:
                     raise ValueError("only GRIB edition 2 is supported")
-                yield h
+                offset = _get(eccodes, h, "offset", int)
+                if offset is None or offset != last_offset:
+                    index += 1
+                    last_offset = offset
+                yield index, h
 
 
 def _get(eccodes: Any, h: int, key: str, ktype: type | None = None) -> Any:
@@ -203,7 +224,7 @@ def inventory(path: Path) -> list[GribMessage]:
     """Every message in a local GRIB2 file, in file order, without decoding its values."""
     eccodes = _modules()[0]
     messages = []
-    for index, h in enumerate(_messages(eccodes, path)):
+    for index, h in _messages(eccodes, path):
         keys = {key: _get(eccodes, h, key, str) for key in INVENTORY_KEYS}
         messages.append(
             GribMessage(
@@ -373,7 +394,7 @@ def open_grib2(
     needs_select = select is None and not fetched.provenance.is_partial
     object_messages = fetched.provenance.object_messages
     selectors = fetched.provenance.selectors if object_messages else []
-    for h in _messages(eccodes, fetched.path):
+    for index, h in _messages(eccodes, fetched.path):
         count += 1
         if count > 1 and needs_select:
             continue
@@ -415,15 +436,15 @@ def open_grib2(
             if stamp:
                 attrs[label] = stamp
         message: dict[str, Any] = {
-            "file_index": count - 1,
-            "object_index": object_messages[count - 1] if object_messages else None,
+            "file_index": index,
+            "object_index": object_messages[index] if object_messages else None,
             "shortName": _get(eccodes, h, "shortName", str),
             "typeOfLevel": attrs.get("typeOfLevel"),
             "level": attrs.get("level"),
             "step": attrs.get("step"),
         }
         if selectors:
-            message["selector"] = selectors[count - 1]
+            message["selector"] = selectors[index]
         short = message["shortName"]
         if short in (None, "", "unknown", "~"):
             short = _product_name(fetched.asset.id) or (

{usdata-0.19.0 → usdata-0.20.0}/src/usdata/cli/inspect.py

@@ -94,7 +94,8 @@ def _echo_detail(summary: Summary, width: int) -> None:
         else:
             typer.echo("  columns:")
             for name in frame.columns:
-                typer.echo(f"    {name}")
+                unit = frame.units.get(name)
+                typer.echo(f"    {name} ({unit})" if unit else f"    {name}")
     if (scene := summary.netcdf) is not None:
         _echo_table(
             "variables",

{usdata-0.19.0 → usdata-0.20.0}/src/usdata/data/registry.yaml

@@ -473,21 +473,77 @@ datasets:
 
   - id: noaa:ibtracs
     provider: noaa
-    status: planned
+    status: available
     domain: tropical-cyclones
-    target: later
+    since: "0.20"
     title: IBTrACS Global Tropical Cyclone Tracks
     description: >-
-      International Best Track Archive for Climate Stewardship: merged best
-      tracks from all agencies worldwide since 1842. CSV and NetCDF files by
-      basin or period from an NCEI HTTPS directory (v04r01).
+      International Best Track Archive for Climate Stewardship: every agency's
+      tropical cyclone best tracks merged into one global record since 1842,
+      published by NCEI as whole files under an anonymous HTTPS directory. A
+      subset parameter names one of the eleven files NCEI builds (the complete
+      record, active storms, the last three years, everything since 1980, or
+      one of seven basins), a format parameter picks the CSV list or the
+      NetCDF file, and the newest product version wins unless one is pinned.
+      There is no query interface, so dates and geographic filters are
+      rejected; the files are rebuilt in place under unchanging names, so a
+      lockfile's checksum is what pins the bytes. The CSV opens with the units-row reader
+      and the NetCDF with xarray; filter the table locally.
     keywords: [tropical cyclone, hurricane, typhoon, best track, ibtracs, global]
     protocol: http
     homepage: https://www.ncei.noaa.gov/products/international-best-track-archive
     license: US Government Work (public domain)
-    temporal_extent: { start: "1842-01-01T00:00:00Z" }
+    temporal_extent: { start: "1842-10-25T03:00:00Z" }
     capabilities: { spatial_subset: false, temporal_subset: false, variable_subset: false }
-
+    summary: Global tropical cyclone best tracks
+    formats: [CSV with a units row, NetCDF4]
+    selection: One whole subset file per query, from the newest or a pinned product version
+    inputs: Required subset; optional format and version; no dates or geographic filters
+    reader: pandas
+    guide: docs/providers/noaa-ibtracs.md
+    examples:
+      - examples/ibtracs/README.md
+    resolution:
+      spatial: "Positions to a tenth of a degree"
+      temporal: "Three-hourly points, interpolated between the agencies' six-hourly best tracks"
+    update_frequency: >-
+      Files are rebuilt in place as agencies deliver best tracks, stamped with the build
+      date; the v04r01 change log records twenty-three additions and corrections between
+      2024-06-21 and 2026-07-31
+    citation: >-
+      Gahtan, J., K. R. Knapp, C. J. Schreck, H. J. Diamond, J. P. Kossin, and M. C. Kruk, 2024:
+      International Best Track Archive for Climate Stewardship (IBTrACS) Project, Version 4r01.
+      NOAA National Centers for Environmental Information, doi:10.25921/82ty-9e16. Knapp, K. R.,
+      M. C. Kruk, D. H. Levinson, H. J. Diamond, and C. J. Neumann, 2010: The International Best
+      Track Archive for Climate Stewardship (IBTrACS). Bull. Amer. Meteor. Soc., 91, 363-376
+    terms: https://www.ncei.noaa.gov/products/international-best-track-archive
+    variables:
+      - { name: "sid", description: "IBTrACS serial id, such as 2004086S29318; SID in the CSV" }
+      - { name: "season", units: "year", description: "Season the storm is assigned to" }
+      - { name: "number", description: "Storm number within the season" }
+      - { name: "basin", description: "Basin at this point: NA, EP, WP, NI, SI, SP, or SA" }
+      - { name: "subbasin", description: "Sub-basin at this point, or MM when none applies" }
+      - { name: "name", description: "Storm name, or UNNAMED" }
+      - { name: "time", units: "UTC", description: "Track-point time; ISO_TIME in the CSV" }
+      - { name: "nature", description: "Nature of the cyclone: TS, ET, SS, DS, NR, or MX" }
+      - { name: "lat", units: "degrees_north", description: "Track-point latitude" }
+      - { name: "lon", units: "degrees_east", description: "Track-point longitude in [-180, 180]" }
+      - { name: "wmo_wind", units: "knots", description: "Maximum sustained wind from the responsible WMO agency, in that agency's averaging period" }
+      - { name: "wmo_pres", units: "millibars", description: "Minimum central pressure from the responsible WMO agency" }
+      - { name: "wmo_agency", description: "The WMO agency responsible for the storm at this point" }
+      - { name: "track_type", description: "main, PROVISIONAL, or a spur track merged, split, or otherwise set aside" }
+      - { name: "dist2land", units: "km", description: "Distance to land at this point" }
+      - { name: "landfall", units: "km", description: "Minimum distance to land before the next point; 0 marks a landfall" }
+      - { name: "iflag", description: "One interpolation flag per agency: original, interpolated, or filled" }
+      - { name: "usa_wind", units: "knots", description: "Maximum sustained 1-minute wind from the U.S. agency" }
+      - { name: "usa_pres", units: "millibars", description: "Minimum central pressure from the U.S. agency" }
+      - { name: "usa_status", description: "U.S. agency status, such as TD, TS, HU, or EX" }
+      - { name: "usa_sshs", description: "Saffir-Simpson category from the U.S. wind; -5 to 5, with the negatives for non-hurricane statuses" }
+      - { name: "usa_rmw", units: "nautical miles", description: "Radius of maximum wind from the U.S. agency" }
+      - { name: "usa_r34", units: "nautical miles", description: "34 kt wind extent by quadrant; USA_R34_NE through USA_R34_NW in the CSV" }
+      - { name: "storm_speed", units: "knots", description: "Storm translation speed" }
+      - { name: "storm_dir", units: "degrees", description: "Storm translation direction" }
+    adapter: usdata.providers.noaa.ibtracs:Ibtracs
   - id: noaa:coops-water-levels
     provider: noaa
     status: available
@@ -946,19 +1002,50 @@ datasets:
 
   - id: noaa:nbm
     provider: noaa
-    status: planned
+    status: available
+    system: noaa:ncep-models
     domain: weather-models
-    target: later
+    since: "0.20"
     title: National Blend of Models
     description: >-
-      Statistically blended forecast guidance in GRIB2 from the public
-      noaa-nbm-grib2-pds S3 bucket, laid out as blend.YYYYMMDD/HH/core/ with
-      files per region and forecast hour.
-    keywords: [forecast, blend, nbm, grib2, guidance]
+      NOAA's statistically blended, bias-corrected forecast guidance in GRIB2 from the anonymous
+      noaa-nbm-grib2-pds S3 bucket, one whole core file per hourly run, forecast hour, and
+      region. Select a run by initialization window and cycle hour, the forecast hours from 1 to
+      264, and the CONUS, Alaska, Hawaii, Puerto Rico, or Guam grid. The CONUS file is about
+      170 MB per hour; the server subsets nothing, so fields are chosen either by naming GRIB2
+      messages, which fetches their byte ranges through the object's index sidecar, or after
+      download with the GRIB2 reader. Each file holds hundreds of fields; the catalog lists the
+      handful the guide and example select.
+    keywords: [forecast, blend, nbm, grib2, guidance, temperature, precipitation, nwp]
     protocol: s3
     homepage: https://registry.opendata.aws/noaa-nbm/
     license: US Government Work (public domain)
-    capabilities: { spatial_subset: false, temporal_subset: true, variable_subset: false }
+    temporal_extent: { start: "2020-09-29T12:00:00Z" }
+    capabilities: { spatial_subset: false, temporal_subset: true, variable_subset: false, partial_fetch: true }
+    summary: NBM forecast guidance
+    formats: [GRIB2]
+    selection: Regional core files, whole or by named GRIB2 message, chosen by run initialization window, cycle hour, forecast hours, and region
+    inputs: Both timestamps, cycle, and forecast_hour
+    reader: grib
+    guide: docs/providers/noaa-nbm.md
+    examples:
+      - examples/nbm-forecast/README.md
+    resolution:
+      spatial: "2.5 km Lambert conformal CONUS grid, 2345 x 1597 points; separate Alaska, Hawaii, Puerto Rico, and Guam grids"
+      temporal: >-
+        Hourly runs; forecast hours hourly to 36, then every 3 hours to about 190, then every
+        6 hours to 264, on a schedule that varies by cycle
+    update_frequency: "Hourly"
+    citation: >-
+      NOAA National Blend of Models (NBM) was accessed on [date] from
+      https://registry.opendata.aws/noaa-nbm
+    terms: https://www.noaa.gov/information-technology/open-data-dissemination
+    variables:
+      - { name: "2t", units: "K", description: "2 m temperature" }
+      - { name: "2d", units: "K", description: "2 m dewpoint temperature" }
+      - { name: "tp", units: "kg/m2", description: "Total precipitation accumulated over the step" }
+    limits: { max_window: P1D }
+    adapter: usdata.providers.noaa.nbm:Nbm
 
   - id: noaa:oisst
     provider: noaa
@@ -1378,33 +1465,52 @@ datasets:
 # Documentation composition keyed by stable dataset ID; validated by just docs.
   - id: noaa:rap
     provider: noaa
-    status: planned
+    status: available
+    system: noaa:ncep-models
     domain: weather-models
-    target: later
+    since: "0.20"
     title: RAP Model Output
     description: >-
-      Rapid Refresh 13 km hourly analyses and forecasts in GRIB2 from the public
-      noaa-rap-pds S3 bucket, laid out as rap.YYYYMMDD/ with one file per cycle,
-      forecast hour, and grid family (awp130 is the 13 km CONUS grid, about 18 MB
-      per analysis). The bucket begins in December 2020; it is the environmental
-      source for years HRRR's archive does not reach only where NCEI holds older
-      runs. Same cycle and forecast-hour selection as HRRR.
-    keywords:
-      - forecast
-      - model
-      - rap
-      - grib2
-      - weather
-      - nwp
+      Rapid Refresh 13 km hourly analyses and forecasts in GRIB2 from the anonymous noaa-rap-pds
+      S3 bucket, one whole file per run, forecast hour, and grid family. Select a run by
+      initialization window and cycle hour, the forecast hours, and the file family: the 13 km
+      CONUS pressure-level grid (about 18 MB per file), its secondary fields, or the native-grid
+      pressure and hybrid-level files. Runs reach 21 hours, or 51 from the 03, 09, 15, and 21
+      UTC runs. The server subsets nothing, so fields are chosen either by naming GRIB2 messages,
+      which fetches their byte ranges through the object's index sidecar, or after download with
+      the GRIB2 reader. Each file holds hundreds of fields; the catalog lists the handful the
+      guide and example select.
+    keywords: [forecast, model, rap, grib2, weather, nwp, cape, helicity, convective]
     protocol: s3
     homepage: https://registry.opendata.aws/noaa-rap/
     license: US Government Work (public domain)
-    temporal_extent:
-      start: "2020-12-01T00:00:00Z"
-    capabilities:
-      spatial_subset: false
-      temporal_subset: true
-      variable_subset: false
+    temporal_extent: { start: "2021-02-22T00:00:00Z" }
+    capabilities: { spatial_subset: false, temporal_subset: true, variable_subset: false, partial_fetch: true }
+    summary: RAP model output
+    formats: [GRIB2]
+    selection: Files, whole or by named GRIB2 message, chosen by run initialization window, cycle hour, forecast hours, and file family
+    inputs: Both timestamps, cycle, and forecast_hour
+    reader: grib
+    guide: docs/providers/noaa-rap.md
+    examples:
+      - examples/rap-environment/README.md
+    resolution:
+      spatial: "13 km Lambert conformal CONUS grid, 451 x 337 points, for the awp130 family"
+      temporal: >-
+        Hourly runs; forecast hours to 21, or to 51 from the 03, 09, 15, and 21 UTC runs
+    update_frequency: "Hourly"
+    citation: >-
+      NOAA Rapid Refresh (RAP) was accessed on [date] from https://registry.opendata.aws/noaa-rap
+    terms: https://www.noaa.gov/information-technology/open-data-dissemination
+    variables:
+      - { name: "cape", units: "J/kg", description: "Convective available potential energy, surface or a ground layer" }
+      - { name: "cin", units: "J/kg", description: "Convective inhibition, surface or a ground layer" }
+      - { name: "hlcy", units: "m2/s2", description: "Storm-relative helicity over the 0-1 km and 0-3 km layers" }
+      - { name: "refc", units: "dBZ", description: "Composite reflectivity" }
+      - { name: "2t", units: "K", description: "2 m temperature" }
+      - { name: "2d", units: "K", description: "2 m dewpoint temperature" }
+    limits: { max_window: P1D }
+    adapter: usdata.providers.noaa.rap:Rap
 
   - id: noaa:igra
     provider: noaa

{usdata-0.19.0 → usdata-0.20.0}/src/usdata/inspect.py

@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
 from pydantic import BaseModel, Field
 
 from usdata import provenance, readers
-from usdata.models import Provenance
+from usdata.models import Protocol, Provenance
 
 if TYPE_CHECKING:
     from usdata._fetch import FetchedAsset
@@ -47,7 +47,16 @@ class CsvSummary(BaseModel):
     """A delimited text file's header and how many data rows the scan counted."""
 
     columns: list[str] = Field(description="Header fields, in file order")
-    row_count: int = Field(ge=0, description="Data rows counted, excluding the header")
+    units: dict[str, str] = Field(
+        default_factory=dict,
+        description=(
+            "Units per column, from the units row an ERDDAP or IBTrACS CSV lays under its "
+            "header; empty for a CSV with no such row"
+        ),
+    )
+    row_count: int = Field(
+        ge=0, description="Data rows counted, excluding the header and any units row"
+    )
     row_limit: int = Field(gt=0, description="Rows the scan reads before it stops")
     truncated: bool = Field(description="The scan stopped early, so row_count is a lower bound")
 
@@ -77,7 +86,10 @@ class GribMessage(BaseModel):
     ``selector``, the index selector the message was fetched for.
     """
 
-    file_index: int = Field(ge=0, description="Zero-based position in the local file")
+    file_index: int = Field(
+        ge=0,
+        description="Zero-based message position in the local file; fields of one message share it",
+    )
     object_index: int | None = Field(
         default=None,
         ge=1,
@@ -186,6 +198,7 @@ def inspect_asset(fetched: FetchedAsset) -> Summary:
         fetched.asset.id,
         fetched.provenance,
         _detect(fetched.asset.id, fetched.asset.media_type),
+        fetched.asset.protocol,
     )
 
 
@@ -198,23 +211,39 @@ def inspect_path(path: str | os.PathLike[str]) -> Summary:
 
     Returns:
         The summary ``inspect_asset`` builds, with the format taken from the file
-        name because a sidecar records no media type.
+        name because a sidecar records no media type, and the protocol from the
+        registry entry of the dataset the sidecar names, for the same reason.
 
     Raises:
         OSError: The file or its provenance sidecar is missing or unreadable.
         ValueError: The sidecar is not a provenance record.
     """
     local = Path(path)
-    return _summarize(local, local.name, provenance.read(local), _detect(local.name, None))
+    record = provenance.read(local)
+    return _summarize(
+        local, local.name, record, _detect(local.name, None), _registered_protocol(record)
+    )
+
+
+def _registered_protocol(record: Provenance) -> Protocol | None:
+    """The protocol the registry gives the dataset a sidecar names, or None when it has none."""
+    from usdata.registry import DatasetNotFound, default_registry
+
+    try:
+        return default_registry().get(record.dataset_id).protocol
+    except DatasetNotFound:
+        return None
 
 
-def _summarize(path: Path, asset_id: str, record: Provenance, fmt: AssetFormat) -> Summary:
+def _summarize(
+    path: Path, asset_id: str, record: Provenance, fmt: AssetFormat, protocol: Protocol | None
+) -> Summary:
     """One summary, with the detail its format allows and a note where it allows none."""
     detail: CsvSummary | NetcdfSummary | Grib2Summary | None = None
     note: str | None = None
     if fmt is not AssetFormat.BYTES:
         try:
-            detail = _detail(path, fmt, record)
+            detail = _detail(path, fmt, record, protocol)
         except readers.MissingReaderDependency as error:
             note = str(error)
         # A cached file that no longer decodes is reported, not raised: a summary
@@ -259,11 +288,11 @@ def _detect(name: str, media_type: str | None) -> AssetFormat:
 
 
 def _detail(
-    path: Path, fmt: AssetFormat, record: Provenance
+    path: Path, fmt: AssetFormat, record: Provenance, protocol: Protocol | None
 ) -> CsvSummary | NetcdfSummary | Grib2Summary:
     """The detail for one recognized format, reading only what that format needs."""
     if fmt is AssetFormat.CSV:
-        return _csv_summary(path)
+        return _csv_summary(path, units_row=readers.has_units_row(record.dataset_id, protocol))
     if fmt is AssetFormat.NETCDF:
         from usdata._netcdf import variables
 
@@ -275,23 +304,35 @@ def _paired(messages: list[GribMessage], record: Provenance) -> list[GribMessage
     """Each message as the fetch that took it described it, where provenance says.
 
     The file itself carries neither number nor selector: a partial fetch recorded
-    one message number and one selector per range, so the lists pair up in order.
-    A whole file, or a record that does not pair, leaves both unset.
+    one message number and one selector per range, and ``file_index`` counts the
+    file's messages the same way, so the fields of a message that holds several
+    all pair with its number. A whole file, or a record that does not pair,
+    leaves both unset.
     """
     numbers = record.object_messages
-    if len(numbers) != len(messages):
+    if not numbers or any(message.file_index >= len(numbers) for message in messages):
         return messages
-    selectors = (
-        record.selectors if len(record.selectors) == len(messages) else [None] * len(messages)
-    )
+    selectors = record.selectors if len(record.selectors) == len(numbers) else []
     return [
-        message.model_copy(update={"object_index": number, "selector": selector})
-        for message, number, selector in zip(messages, numbers, selectors, strict=True)
+        message.model_copy(
+            update={
+                "object_index": numbers[message.file_index],
+                "selector": selectors[message.file_index] if selectors else None,
+            }
+        )
+        for message in messages
     ]
 
 
-def _csv_summary(path: Path) -> CsvSummary:
-    """Header and data-row count through the standard library, stopping at ``ROW_LIMIT`` rows."""
+def _csv_summary(path: Path, *, units_row: bool = False) -> CsvSummary:
+    """Header and data-row count through the standard library, stopping at ``ROW_LIMIT`` rows.
+
+    With ``units_row`` the row under the header is read as units, as ``open``
+    reads it, rather than counted as data.
+
+    Raises:
+        ValueError: A units row was expected and does not match the header.
+    """
     with path.open("rb") as raw:
         compressed = raw.read(2) == b"\x1f\x8b"
         raw.seek(0)
@@ -299,13 +340,21 @@ def _csv_summary(path: Path) -> CsvSummary:
         with io.TextIOWrapper(binary, encoding="utf-8-sig", newline="") as stream:
             records = csv.reader(stream)
             columns = next(records, [])
+            units: dict[str, str] = {}
+            if units_row and columns:
+                values = next(records, [])
+                if len(values) != len(columns):
+                    raise ValueError("CSV must have a units row matching the header")
+                units = dict(zip(columns, (value.strip() for value in values), strict=True))
             rows = 0
             for _ in records:
                 rows += 1
                 if rows == ROW_LIMIT:
                     break
             truncated = next(records, None) is not None
-    return CsvSummary(columns=columns, row_count=rows, row_limit=ROW_LIMIT, truncated=truncated)
+    return CsvSummary(
+        columns=columns, units=units, row_count=rows, row_limit=ROW_LIMIT, truncated=truncated
+    )
 
 
 __all__ = [

{usdata-0.19.0 → usdata-0.20.0}/src/usdata/protocols/http.py

@@ -174,7 +174,8 @@ def download_ranges(
         url: The object's URL; ``s3://`` callers map it with ``s3.object_url`` first.
         dest: Path the concatenated bytes are moved to once every run has arrived.
         ranges: Ascending, non-overlapping inclusive byte ranges to fetch.
-        etag: The object's ETag, sent as ``If-Match`` so a republished object fails.
+        etag: The object's ETag, bare or quoted, sent as a quoted ``If-Match`` so a
+            republished object fails.
         total: The object's size, checked against every ``Content-Range``.
         http: Client to use; one is created and closed here when it is omitted.
 
@@ -191,7 +192,8 @@ def download_ranges(
     own = http is None
     active = http or client()
     attempt = 0
-    headers = {"If-Match": etag.strip('"')}
+    # An entity-tag is a quoted string (RFC 9110); records keep the bare value.
+    headers = {"If-Match": '"' + etag.strip('"') + '"'}
 
     def request() -> Path:
         nonlocal attempt