usdata 0.26.0__tar.gz → 0.27.0__tar.gz

{usdata-0.26.0 → usdata-0.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.26.0
+Version: 0.27.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -34,7 +34,7 @@ Requires-Python: >=3.11
 Project-URL: Homepage, https://usdata.dev/
 Project-URL: Documentation, https://docs.usdata.dev/
 Project-URL: Repository, https://github.com/jakeryderv/usdata
-Project-URL: Examples, https://usdata.dev/examples/
+Project-URL: Examples, https://usdata.dev/studies/
 Project-URL: Changelog, https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md
 Project-URL: Issues, https://github.com/jakeryderv/usdata/issues
 Provides-Extra: grib
@@ -55,12 +55,13 @@ Description-Content-Type: text/markdown
 [![usdata.dev: public science, reproducible inputs](https://usdata.dev/og.png)](https://usdata.dev/)
 
 Reproducible acquisition of U.S. public scientific data. One Python SDK and
-CLI discovers curated NOAA and USGS datasets, fetches their files, and keeps a
-record of every input: a manifest names them, a lockfile pins them by checksum,
-and the record of what was fetched is what a methods section cites. Analysis
-stays in pandas and xarray; usdata only acquires. If you need every product
-one agency publishes, that agency's own library is the better tool; usdata is
-for pinning inputs across sources and proving later that they have not changed.
+CLI discovers curated datasets from agencies such as NOAA, USGS, and EPA,
+fetches their files, and keeps a record of every input: a manifest names them,
+a lockfile pins them by checksum, and the record of what was fetched is what a
+methods section cites. Analysis stays in pandas and xarray; usdata only
+acquires. If you need every product one agency publishes, that agency's own
+library is the better tool; usdata is for pinning inputs across sources and
+proving later that they have not changed.
 
 ```sh
 pip install "usdata[pandas]"
@@ -108,9 +109,9 @@ breaking changes.
 
 | | |
 | --- | --- |
-| [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/examples/) with saved results |
+| [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/studies/) with saved results |
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
-| [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
+| [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
 Twenty-seven datasets are available today and nineteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).

{usdata-0.26.0 → usdata-0.27.0}/README.md

@@ -10,12 +10,13 @@
 [![usdata.dev: public science, reproducible inputs](https://usdata.dev/og.png)](https://usdata.dev/)
 
 Reproducible acquisition of U.S. public scientific data. One Python SDK and
-CLI discovers curated NOAA and USGS datasets, fetches their files, and keeps a
-record of every input: a manifest names them, a lockfile pins them by checksum,
-and the record of what was fetched is what a methods section cites. Analysis
-stays in pandas and xarray; usdata only acquires. If you need every product
-one agency publishes, that agency's own library is the better tool; usdata is
-for pinning inputs across sources and proving later that they have not changed.
+CLI discovers curated datasets from agencies such as NOAA, USGS, and EPA,
+fetches their files, and keeps a record of every input: a manifest names them,
+a lockfile pins them by checksum, and the record of what was fetched is what a
+methods section cites. Analysis stays in pandas and xarray; usdata only
+acquires. If you need every product one agency publishes, that agency's own
+library is the better tool; usdata is for pinning inputs across sources and
+proving later that they have not changed.
 
 ```sh
 pip install "usdata[pandas]"
@@ -63,9 +64,9 @@ breaking changes.
 
 | | |
 | --- | --- |
-| [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/examples/) with saved results |
+| [usdata.dev](https://usdata.dev/) | What it is for: the [dataset browser](https://usdata.dev/datasets/) and [worked examples](https://usdata.dev/studies/) with saved results |
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
-| [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
+| [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
 Twenty-seven datasets are available today and nineteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).

{usdata-0.26.0 → usdata-0.27.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.26.0"
+version = "0.27.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"
@@ -59,7 +59,7 @@ grib = [
 Homepage = "https://usdata.dev/"
 Documentation = "https://docs.usdata.dev/"
 Repository = "https://github.com/jakeryderv/usdata"
-Examples = "https://usdata.dev/examples/"
+Examples = "https://usdata.dev/studies/"
 Changelog = "https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md"
 Issues = "https://github.com/jakeryderv/usdata/issues"
 

{usdata-0.26.0 → usdata-0.27.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.26.0"
+version = "0.27.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"
@@ -56,7 +56,7 @@ grib = [
 Homepage = "https://usdata.dev/"
 Documentation = "https://docs.usdata.dev/"
 Repository = "https://github.com/jakeryderv/usdata"
-Examples = "https://usdata.dev/examples/"
+Examples = "https://usdata.dev/studies/"
 Changelog = "https://github.com/jakeryderv/usdata/blob/main/CHANGELOG.md"
 Issues = "https://github.com/jakeryderv/usdata/issues"
 

{usdata-0.26.0 → usdata-0.27.0}/src/usdata/_fetch.py

@@ -17,6 +17,10 @@ from usdata.models import Asset, Dataset, Provenance, Query
 from usdata.providers import Provider, load_adapter
 
 if TYPE_CHECKING:
+    # Each is an optional extra: without it installed the result is simply untyped.
+    import pandas as pd  # pyright: ignore[reportMissingImports]
+    import xarray as xr  # pyright: ignore[reportMissingImports]
+
     from usdata.inspect import Summary
 
 
@@ -32,46 +36,74 @@ class FetchedAsset(BaseModel):
     provenance: Provenance
     from_cache: bool
 
-    def open(
+    def open(self) -> Any:
+        """Open local data with the reader its format implies, with that reader's defaults.
+
+        CSV and ERDDAP CSV, HURDAT2 and AQS daily JSON return a pandas DataFrame;
+        NetCDF4 and GRIB2 a loaded xarray Dataset; NEXRAD Level II an xarray
+        DataTree. Provenance is kept in the result's ``attrs["usdata"]``. A file
+        that needs options, or whose metadata leaves its format ambiguous, is
+        opened with the method for its format instead: ``open_csv``,
+        ``open_nexrad``, ``open_grib2``, or ``open_netcdf``. Cached files and
+        provenance sidecars are never changed. See ``usdata.readers.open_asset``.
+        """
+        from usdata.readers import open_asset
+
+        return open_asset(self)
+
+    def open_csv(
         self,
         *,
-        reader: str | None = None,
         dtype: dict[str, str] | None = None,
         parse_dates: list[str] | None = None,
         usecols: list[str] | None = None,
         nrows: int | None = None,
-        sweep: int | list[int] | None = None,
-        select: Mapping[str, Any] | None = None,
-        strict: bool = False,
-    ) -> Any:
-        """Open local data with an optional ``pandas``, ``radar``, ``netcdf``, or ``grib`` reader.
-
-        ERDDAP units are kept in ``frame.attrs["units"]`` and source provenance
-        in ``frame.attrs["usdata"]``. NEXRAD returns a xarray DataTree with provenance
-        in ``radar.attrs["usdata"]``. NetCDF4 and GRIB2 return a loaded xarray Dataset
-        with matching provenance in its attributes, and with units and long names
-        the file leaves unstated filled from the registry entry's variables. See
-        ``usdata.readers.open_asset`` for options. Use ``sweep=0`` or ``sweep=[0, 2]``
-        to load selected zero-based radar sweeps, and
-        ``select={"shortName": "cape", "typeOfLevel": "surface"}``
-        to choose GRIB2 messages; ``strict=True`` raises instead of warning when a
-        GRIB2 select value matches none of the selected messages. Cached files and
-        provenance sidecars are never changed.
+        units_row: bool | None = None,
+    ) -> pd.DataFrame:
+        """Open a CSV as a pandas DataFrame; see ``usdata.readers.open_csv``.
+
+        ``units_row`` says whether a units row follows the header, as in ERDDAP
+        CSV, and is inferred when ``None``; the units go to ``frame.attrs["units"]``.
         """
-        from usdata.readers import open_asset
+        from usdata.readers import open_csv
 
-        return open_asset(
+        return open_csv(
             self,
-            reader=reader,
             dtype=dtype,
             parse_dates=parse_dates,
             usecols=usecols,
             nrows=nrows,
-            sweep=sweep,
-            select=select,
-            strict=strict,
+            units_row=units_row,
         )
 
+    def open_nexrad(self, *, sweep: int | list[int] | None = None) -> xr.DataTree:
+        """Open a NEXRAD Level II volume as an xarray DataTree; see ``usdata.readers.open_nexrad``.
+
+        Use ``sweep=0`` or ``sweep=[0, 2]`` to load selected zero-based sweeps.
+        """
+        from usdata.readers import open_nexrad
+
+        return open_nexrad(self, sweep=sweep)
+
+    def open_grib2(
+        self, *, select: Mapping[str, Any] | None = None, strict: bool = False
+    ) -> xr.Dataset:
+        """Open GRIB2 messages as one xarray Dataset; see ``usdata.readers.open_grib2``.
+
+        ``select={"shortName": "cape", "typeOfLevel": "surface"}`` chooses
+        messages; ``strict=True`` raises instead of warning when a select value
+        matches none of the selected messages.
+        """
+        from usdata.readers import open_grib2
+
+        return open_grib2(self, select=select, strict=strict)
+
+    def open_netcdf(self) -> xr.Dataset:
+        """Open a NetCDF4 file as an xarray Dataset; see ``usdata.readers.open_netcdf``."""
+        from usdata.readers import open_netcdf
+
+        return open_netcdf(self)
+
     def inspect(self) -> Summary:
         """Summarize this file: provenance, format, and what that format holds.
 
@@ -108,12 +140,18 @@ def _fetch_asset(
     root: Path | None = None,
     force: bool = False,
     pinned: Provenance | None = None,
+    staging: Path | None = None,
 ) -> FetchedAsset:
     """Fetch one asset through the cache, passing any pinned record to the adapter.
 
     ``pinned`` is the provenance a lockfile holds for this asset. The adapter sees
     it through ``prepare_fetch``, so an asset that pins byte ranges is reproduced
     from the record rather than by resolving its query again.
+
+    ``staging`` is a root a download is written under instead of the cache. The
+    cache at ``root`` is still checked, and a hit is returned from there; a
+    miss comes back with its ``path`` under ``staging``, for the caller to move
+    home once it can pin it (ADR 0031).
     """
     if asset.dataset_id != dataset.id:
         raise ValueError(f"asset dataset {asset.dataset_id!r} does not match {dataset.id!r}")
@@ -138,6 +176,8 @@ def _fetch_asset(
         ):
             _progress.emit(_progress.AssetProgress(asset.id, "cached", prov.size))
             return FetchedAsset(asset=asset, path=path, provenance=prov, from_cache=True)
+    if staging is not None:
+        path = asset_path(asset, staging)
     with staged_path(path) as tmp:
         partial = adapter.prepare_fetch(asset, pinned)
         if partial is None:
@@ -174,15 +214,19 @@ def _fetch_with(
     *,
     root: Path | None = None,
     force: bool = False,
+    staging: Path | None = None,
 ) -> list[FetchedAsset]:
     """Run the loop on an adapter the caller opened, so one adapter can serve many queries.
 
     The listing is put in the order every result promises, by ``asset.time.start``
     then id, so no adapter has to sort and no caller has to sort defensively.
+    ``staging`` is passed to each fetch; see ``_fetch_asset``.
     """
     assets = ordered(adapter.list_assets(query))
     _progress.batch([asset.size for asset in assets])
-    return [_fetch_asset(dataset, a, adapter, root=root, force=force) for a in assets]
+    return [
+        _fetch_asset(dataset, a, adapter, root=root, force=force, staging=staging) for a in assets
+    ]
 
 
 def ordered(assets: list[Asset]) -> list[Asset]:

{usdata-0.26.0 → usdata-0.27.0}/src/usdata/_grib.py

@@ -9,7 +9,8 @@ repeat, so the same select always yields the same names. Values arrive from
 ecCodes as float64, are masked to NaN where the message's bitmap marks them
 missing, and are stored as float32; the float64 array is released before the
 Dataset is returned. Rows are ordered north to south and columns west to east
-regardless of the message's scanning mode. See ADR 0022.
+regardless of the message's scanning mode, including grids whose adjacent rows
+scan in opposite directions. See ADR 0022.
 """
 
 from __future__ import annotations
@@ -279,6 +280,8 @@ class _Grid:
         rows, cols = shape
         self.flip_rows = bool(_get(eccodes, h, "jScansPositively", int))
         self.flip_cols = bool(_get(eccodes, h, "iScansNegatively", int))
+        # Adjacent rows scan in opposite directions (NBM's CONUS grid does this).
+        self.alternating = bool(_get(eccodes, h, "alternativeRowScanning", int))
         if _get(eccodes, h, "jPointsAreConsecutive", int):
             raise ValueError("grids with consecutive j points are not supported")
         self.regular = self.grid_type == "regular_ll"
@@ -305,15 +308,25 @@ class _Grid:
                 value = _get(eccodes, h, key)
                 if value is not None:
                     self.attrs[key] = value
-        self.key = (self.grid_type, self.shape, self.flip_rows, self.flip_cols)
+        self.key = (self.grid_type, self.shape, self.flip_rows, self.flip_cols, self.alternating)
 
     def _orient(self, numpy: Any, axis: Any, *, rows: bool) -> Any:
         return axis[::-1].copy() if (self.flip_rows if rows else self.flip_cols) else axis
 
-    def reshape(self, numpy: Any, flat: Any) -> Any:
+    def reshape(self, numpy: Any, flat: Any, *, stored: bool = False) -> Any:
+        """Order a flat array north to south and west to east.
+
+        ``stored`` marks data values, which ecCodes returns in the order the
+        message stores them: with alternative row scanning every second row
+        runs the other way and is reversed here. The latitudes and longitudes
+        ecCodes computes do not alternate, so coordinates skip that step.
+        """
         if flat.size != self.shape[0] * self.shape[1]:
             raise ValueError(f"message has {flat.size} values for a {self.shape} grid")
         grid = flat.reshape(self.shape)
+        if stored and self.alternating:
+            grid = grid.copy()
+            grid[1::2] = grid[1::2, ::-1]
         if self.flip_rows:
             grid = grid[::-1, :]
         if self.flip_cols:
@@ -423,7 +436,7 @@ def open_grib2(
             missing = _get(eccodes, h, "missingValue", float)
             if missing is not None:
                 values[values == missing] = numpy.nan
-        data = grid.reshape(numpy, values).astype(numpy.float32)
+        data = grid.reshape(numpy, values, stored=True).astype(numpy.float32)
         del values
         attrs = {
             key: value for key in VARIABLE_KEYS if (value := _get(eccodes, h, key)) is not None
@@ -454,8 +467,8 @@ def open_grib2(
         selected.append(_Field(short=short, data=data, attrs=attrs, message=message))
     if count > 1 and needs_select:
         raise ValueError(
-            f"{count} messages; pass select={{...}} with ecCodes keys to choose, "
-            f"for example select={{'shortName': ..., 'typeOfLevel': ...}}. "
+            f"{count} messages; pass select={{...}} to open_grib2 with ecCodes keys to "
+            f"choose, for example select={{'shortName': ..., 'typeOfLevel': ...}}. "
             f"Available (shortName, typeOfLevel, level): {_available(fetched.path)}"
         )
     if grid is None or not selected:

{usdata-0.26.0 → usdata-0.27.0}/src/usdata/_radar.py

@@ -67,8 +67,9 @@ def _check_sweeps(content: bytes, sweep: int | list[int] | None) -> None:
             ):
                 raise RadarDecodeError(
                     f"cannot safely decode sweep {index}: NEXRAD moment and coordinate records "
-                    "do not agree; select an unaffected sweep explicitly with open(sweep=...) "
-                    "or use another decoder. No sweeps were silently dropped."
+                    "do not agree; select an unaffected sweep explicitly with "
+                    "open_nexrad(sweep=...) or use another decoder. "
+                    "No sweeps were silently dropped."
                 )