usdata 0.16.0__tar.gz → 0.17.0__tar.gz

{usdata-0.16.0 → usdata-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.16.0
+Version: 0.17.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,nasa,open-data,scientific-data,provenance
 Author: Jake Van Slyke
@@ -108,7 +108,8 @@ the `check-*` commands install their respective extras.
 
 Releases: `just release minor` prepares a release branch. Update release
 notices, then `just release-pr` validates and opens a draft for review. Merging
-the reviewed PR publishes to PyPI and creates the tag and GitHub release. See
+the reviewed PR publishes to PyPI and creates the tag and GitHub release, then
+walks the published package through the getting-started guide. See
 [versioning](docs/versioning.md).
 
 The main way the project grows is a new dataset: one registry entry and one

{usdata-0.16.0 → usdata-0.17.0}/README.md

@@ -74,7 +74,8 @@ the `check-*` commands install their respective extras.
 
 Releases: `just release minor` prepares a release branch. Update release
 notices, then `just release-pr` validates and opens a draft for review. Merging
-the reviewed PR publishes to PyPI and creates the tag and GitHub release. See
+the reviewed PR publishes to PyPI and creates the tag and GitHub release, then
+walks the published package through the getting-started guide. See
 [versioning](docs/versioning.md).
 
 The main way the project grows is a new dataset: one registry entry and one

{usdata-0.16.0 → usdata-0.17.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.16.0"
+version = "0.17.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.16.0 → usdata-0.17.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.16.0"
+version = "0.17.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

usdata-0.17.0/src/usdata/__init__.py

@@ -0,0 +1,98 @@
+"""usdata: unified access and provenance for U.S. public scientific data."""
+
+from __future__ import annotations
+
+from importlib.metadata import PackageNotFoundError, version
+from typing import Any
+
+try:
+    __version__ = version("usdata")
+except PackageNotFoundError:  # running from a source tree without an install
+    __version__ = "0.0.0"
+
+from usdata._fetch import ChecksumMismatch, FetchedAsset, fetch, fetch_asset
+from usdata.cite import cite_dataset, cite_lockfile
+from usdata.inspect import inspect_asset, inspect_path
+from usdata.models import Asset, BBox, Dataset, Provenance, Query, TemporalSelection, TimeRange
+from usdata.pull import pull, verify
+from usdata.query import build_query
+from usdata.registry import DatasetNotFound, Registry, SearchResult, default_registry
+from usdata.selection import select_by_time
+
+__all__ = [
+    "Asset",
+    "BBox",
+    "ChecksumMismatch",
+    "Dataset",
+    "DatasetNotFound",
+    "FetchedAsset",
+    "Provenance",
+    "Query",
+    "Registry",
+    "SearchResult",
+    "TemporalSelection",
+    "TimeRange",
+    "__version__",
+    "build_query",
+    "cite_dataset",
+    "cite_lockfile",
+    "datasets",
+    "default_registry",
+    "fetch",
+    "fetch_asset",
+    "get",
+    "inspect_asset",
+    "inspect_path",
+    "pull",
+    "search",
+    "select_by_time",
+    "verify",
+]
+
+
+def datasets(
+    *,
+    provider: str | None = None,
+    domain: str | None = None,
+    format: str | None = None,
+    reader: str | None = None,
+    capability: str | None = None,
+    status: str = "available",
+) -> list[Dataset]:
+    """List curated registry datasets. Keyword arguments match ``Registry.list``."""
+    return default_registry().list(
+        provider=provider,
+        domain=domain,
+        format=format,
+        reader=reader,
+        capability=capability,
+        status=status,
+    )
+
+
+def search(
+    text: str | None = None,
+    *,
+    include_planned: bool = False,
+    domain: str | None = None,
+    format: str | None = None,
+    reader: str | None = None,
+    capability: str | None = None,
+    status: str | None = None,
+    **kwargs: Any,
+) -> list[SearchResult]:
+    """Search the curated registry. Remaining keyword arguments match ``build_query``."""
+    return default_registry().search(
+        build_query(text, **kwargs),
+        include_planned=include_planned,
+        domain=domain,
+        format=format,
+        reader=reader,
+        capability=capability,
+        status=status,
+    )
+
+
+def get(dataset_id: str) -> Dataset:
+    """Look up a dataset by ``provider:name`` id."""
+    return default_registry().get(dataset_id)

{usdata-0.16.0 → usdata-0.17.0}/src/usdata/_fetch.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from pydantic import BaseModel
 
@@ -14,6 +14,9 @@ from usdata.cache import asset_path, sha256_file
 from usdata.models import Asset, Dataset, Provenance, Query
 from usdata.providers import Provider, load_adapter
 
+if TYPE_CHECKING:
+    from usdata.inspect import Summary
+
 
 class ChecksumMismatch(RuntimeError):
     """A fetched file's sha256 did not match the checksum the adapter declared."""
@@ -37,16 +40,21 @@ class FetchedAsset(BaseModel):
         nrows: int | None = None,
         sweep: int | list[int] | None = None,
         select: Mapping[str, Any] | None = None,
+        strict: bool = False,
     ) -> Any:
         """Open local data with an optional ``pandas``, ``radar``, ``netcdf``, or ``grib`` reader.
 
         ERDDAP units are kept in ``frame.attrs["units"]`` and source provenance
         in ``frame.attrs["usdata"]``. NEXRAD returns a xarray DataTree with provenance
         in ``radar.attrs["usdata"]``. NetCDF4 and GRIB2 return a loaded xarray Dataset
-        with matching provenance in its attributes. See ``usdata.readers.open_asset``
-        for options. Use ``sweep=0`` or ``sweep=[0, 2]`` to load selected zero-based
-        radar sweeps, and ``select={"shortName": "cape", "typeOfLevel": "surface"}``
-        to choose GRIB2 messages. Cached files and provenance sidecars are never changed.
+        with matching provenance in its attributes, and with units and long names
+        the file leaves unstated filled from the registry entry's variables. See
+        ``usdata.readers.open_asset`` for options. Use ``sweep=0`` or ``sweep=[0, 2]``
+        to load selected zero-based radar sweeps, and
+        ``select={"shortName": "cape", "typeOfLevel": "surface"}``
+        to choose GRIB2 messages; ``strict=True`` raises instead of warning when a
+        GRIB2 select value matches none of the selected messages. Cached files and
+        provenance sidecars are never changed.
         """
         from usdata.readers import open_asset
 
@@ -59,8 +67,22 @@ class FetchedAsset(BaseModel):
             nrows=nrows,
             sweep=sweep,
             select=select,
+            strict=strict,
         )
 
+    def inspect(self) -> Summary:
+        """Summarize this file: provenance, format, and what that format holds.
+
+        CSV columns and a capped row count need no extra; NetCDF4 variables and
+        GRIB2 messages need the same extras ``open`` does, and a missing one
+        yields a summary with no detail and a note naming it. Like ``open``,
+        this reads the cached file and changes nothing. See
+        ``usdata.inspect.inspect_asset``.
+        """
+        from usdata.inspect import inspect_asset
+
+        return inspect_asset(self)
+
 
 def _sidecar_not_older(path: Path) -> bool:
     """Whether ``path`` still carries the mtime it had when its sidecar was written.

{usdata-0.16.0 → usdata-0.17.0}/src/usdata/_grib.py

@@ -14,14 +14,17 @@ from __future__ import annotations
 
 import gzip
 import re
+import warnings
 from collections.abc import Iterator, Mapping
 from contextlib import contextmanager
 from datetime import UTC, datetime
 from importlib import import_module
+from inspect import currentframe
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-from usdata.readers import MissingReaderDependency
+from usdata.inspect import GribMessage
+from usdata.readers import MissingReaderDependency, fill_registry_attrs
 
 if TYPE_CHECKING:
     from usdata._fetch import FetchedAsset
@@ -33,7 +36,7 @@ LIBRARY_HINT = (
     "`conda install -c conda-forge eccodes` or `brew install eccodes`"
 )
 MRMS_NAME = re.compile(r"^MRMS_(?P<product>.+?)_\d{2}\.\d{2}_\d{8}-\d{6}\.grib2(?:\.gz)?$")
-SUMMARY_KEYS = ("shortName", "typeOfLevel", "level")
+INVENTORY_KEYS = ("shortName", "name", "typeOfLevel", "level", "step", "units")
 VARIABLE_KEYS = (
     "name",
     "units",
@@ -129,28 +132,98 @@ def _get(eccodes: Any, h: int, key: str, ktype: type | None = None) -> Any:
         return None
 
 
-def _matches(eccodes: Any, h: int, select: Mapping[str, Any]) -> bool:
-    for key, wanted in select.items():
-        options = wanted if isinstance(wanted, list | tuple | set) else [wanted]
-        if not options:
-            raise ValueError(f"select[{key!r}] must not be empty")
-        text = _get(eccodes, h, key, str)
-        number = _get(eccodes, h, key, float) if text is not None else None
-        matched = False
-        for option in options:
-            if isinstance(option, bool) or not isinstance(option, int | float | str):
-                raise ValueError(f"select[{key!r}] values must be strings or numbers")
-            if isinstance(option, str):
-                matched |= text == option
-            elif number is not None:
-                matched |= number == option
-        if not matched:
-            return False
-    return True
+def _options(key: str, wanted: Any) -> list[Any]:
+    """One key's wanted values as a validated list, whether one value or several were given."""
+    options = list(wanted) if isinstance(wanted, list | tuple | set) else [wanted]
+    if not options:
+        raise ValueError(f"select[{key!r}] must not be empty")
+    for option in options:
+        if isinstance(option, bool) or not isinstance(option, int | float | str):
+            raise ValueError(f"select[{key!r}] values must be strings or numbers")
+    return options
 
 
-def _summary(eccodes: Any, h: int) -> tuple[str, str, Any]:
-    return tuple(_get(eccodes, h, key, str) for key in SUMMARY_KEYS)  # type: ignore[return-value]
+def _matched(eccodes: Any, h: int, key: str, options: list[Any]) -> set[int]:
+    """Positions in ``options`` that this message's value for ``key`` matches."""
+    text = _get(eccodes, h, key, str)
+    number = _get(eccodes, h, key, float) if text is not None else None
+    hits = set()
+    for index, option in enumerate(options):
+        if isinstance(option, str):
+            if text == option:
+                hits.add(index)
+        elif number is not None and number == option:
+            hits.add(index)
+    return hits
+
+
+def _unmatched_report(
+    options: Mapping[str, list[Any]],
+    matched: Mapping[str, set[int]],
+    present: Mapping[str, list[str]],
+) -> str:
+    """Text naming every select value that matched no message, with what was there instead."""
+    parts = []
+    for key, wanted in options.items():
+        missing = [option for index, option in enumerate(wanted) if index not in matched[key]]
+        if not missing:
+            continue
+        available = ", ".join(repr(value) for value in present[key]) or "none"
+        parts.append(
+            f"select[{key!r}] matched no message for "
+            f"{', '.join(repr(option) for option in missing)}; "
+            f"{key} values available with the other select keys: {available}."
+        )
+    return " ".join(parts)
+
+
+def _caller_stacklevel() -> int:
+    """Stack level of the first frame outside usdata, so a warning points at the caller."""
+    package = Path(__file__).resolve().parent
+    frame = currentframe()
+    frame = frame.f_back if frame is not None else None
+    level = 1
+    while frame is not None and Path(frame.f_code.co_filename).resolve().is_relative_to(package):
+        level += 1
+        frame = frame.f_back
+    return level
+
+
+def _shape(eccodes: Any, h: int) -> tuple[int, int] | None:
+    """A message's grid rows and columns under either key pair, or None when neither is set."""
+    rows, cols = _get(eccodes, h, "Nj", int), _get(eccodes, h, "Ni", int)
+    if rows is None or cols is None:
+        rows, cols = _get(eccodes, h, "Ny", int), _get(eccodes, h, "Nx", int)
+    return None if rows is None or cols is None else (rows, cols)
+
+
+def inventory(path: Path) -> list[GribMessage]:
+    """Every message in a local GRIB2 file, in file order, without decoding its values."""
+    eccodes = _modules()[0]
+    messages = []
+    for index, h in enumerate(_messages(eccodes, path)):
+        keys = {key: _get(eccodes, h, key, str) for key in INVENTORY_KEYS}
+        messages.append(
+            GribMessage(
+                index=index,
+                short_name=keys["shortName"],
+                name=keys["name"],
+                type_of_level=keys["typeOfLevel"],
+                level=keys["level"],
+                step=keys["step"],
+                units=keys["units"],
+                shape=_shape(eccodes, h),
+            )
+        )
+    return messages
+
+
+def _available(path: Path) -> str:
+    """Every message as a (shortName, typeOfLevel, level) triple, for a reader error to list."""
+    return ", ".join(
+        f"({message.short_name!r}, {message.type_of_level!r}, {message.level!r})"
+        for message in inventory(path)
+    )
 
 
 def _product_name(asset_id: str) -> str | None:
@@ -176,13 +249,11 @@ class _Grid:
 
     def __init__(self, eccodes: Any, numpy: Any, h: int) -> None:
         self.grid_type = _get(eccodes, h, "gridType", str) or "unknown"
-        rows = _get(eccodes, h, "Nj", int)
-        cols = _get(eccodes, h, "Ni", int)
-        if rows is None or cols is None:
-            rows, cols = _get(eccodes, h, "Ny", int), _get(eccodes, h, "Nx", int)
-        if rows is None or cols is None:
+        shape = _shape(eccodes, h)
+        if shape is None:
             raise ValueError(f"cannot determine the grid shape of a {self.grid_type} message")
-        self.shape = (rows, cols)
+        self.shape = shape
+        rows, cols = shape
         self.flip_rows = bool(_get(eccodes, h, "jScansPositively", int))
         self.flip_cols = bool(_get(eccodes, h, "iScansNegatively", int))
         if _get(eccodes, h, "jPointsAreConsecutive", int):
@@ -227,21 +298,41 @@ class _Grid:
         return numpy.ascontiguousarray(grid)
 
 
-def open_grib2(fetched: FetchedAsset, select: Mapping[str, Any] | None = None) -> Any:
-    """Decode selected GRIB2 messages into one loaded xarray Dataset."""
+def open_grib2(
+    fetched: FetchedAsset, select: Mapping[str, Any] | None = None, *, strict: bool = False
+) -> Any:
+    """Decode selected GRIB2 messages into one loaded xarray Dataset.
+
+    A select value that matches none of the selected messages warns, or raises
+    ``ValueError`` when ``strict``; a select that matches nothing always raises.
+    """
     eccodes, xarray, numpy = _modules()
     if select is not None and not isinstance(select, Mapping):
         raise ValueError("select must be a mapping of ecCodes key names to values")
+    options = {key: _options(key, wanted) for key, wanted in (select or {}).items()}
+    matched: dict[str, set[int]] = {key: set() for key in options}
+    present: dict[str, list[str]] = {key: [] for key in options}
     fields: dict[str, tuple[Any, dict[str, Any]]] = {}
     grid: _Grid | None = None
-    available: list[tuple[str, str, Any]] = []
+    count = 0
     names: list[str] = []
     for h in _messages(eccodes, fetched.path):
-        available.append(_summary(eccodes, h))
-        if len(available) > 1 and select is None:
+        count += 1
+        if count > 1 and select is None:
             continue
-        if select is not None and not _matches(eccodes, h, select):
+        hits = {key: _matched(eccodes, h, key, wanted) for key, wanted in options.items()}
+        missed = [key for key, indexes in hits.items() if not indexes]
+        for key in options:
+            # What this key could have been with the rest of the select held fixed.
+            if missed not in ([], [key]):
+                continue
+            value = _get(eccodes, h, key, str)
+            if value is not None and value not in present[key]:
+                present[key].append(value)
+        if missed:
             continue
+        for key, indexes in hits.items():
+            matched[key] |= indexes
         message_grid = _Grid(eccodes, numpy, h)
         if grid is None:
             grid = message_grid
@@ -274,20 +365,23 @@ def open_grib2(fetched: FetchedAsset, select: Mapping[str, Any] | None = None) -
             )
         names.append(short)
         fields[f"{short}#{len(names)}"] = (data, attrs)
-    if len(available) > 1 and select is None:
-        listing = ", ".join(f"({s!r}, {t!r}, {lv!r})" for s, t, lv in available)
+    if count > 1 and select is None:
         raise ValueError(
-            f"{len(available)} messages; pass select={{...}} with ecCodes keys to choose, "
+            f"{count} messages; pass select={{...}} with ecCodes keys to choose, "
             f"for example select={{'shortName': ..., 'typeOfLevel': ...}}. "
-            f"Available (shortName, typeOfLevel, level): {listing}"
+            f"Available (shortName, typeOfLevel, level): {_available(fetched.path)}"
         )
     if grid is None or not fields:
-        if not available:
+        if not count:
             raise ValueError("no GRIB2 messages found")
-        listing = ", ".join(f"({s!r}, {t!r}, {lv!r})" for s, t, lv in available)
         raise ValueError(
-            f"select matched no messages. Available (shortName, typeOfLevel, level): {listing}"
+            "select matched no messages. Available (shortName, typeOfLevel, level): "
+            f"{_available(fetched.path)}"
         )
+    if report := _unmatched_report(options, matched, present):
+        if strict:
+            raise ValueError(report)
+        warnings.warn(report, UserWarning, stacklevel=_caller_stacklevel())
     variables: dict[str, Any] = {}
     duplicates = {name for name in names if names.count(name) > 1}
     for (data, attrs), name in zip(fields.values(), names, strict=True):
@@ -305,4 +399,5 @@ def open_grib2(fetched: FetchedAsset, select: Mapping[str, Any] | None = None) -
         "asset_id": fetched.asset.id,
         "provenance": fetched.provenance.model_dump(mode="json"),
     }
+    fill_registry_attrs(fetched, dataset)
     return dataset

usdata-0.17.0/src/usdata/_netcdf.py

@@ -0,0 +1,69 @@
+"""Local, eagerly loaded NetCDF4 reading behind the netcdf extra."""
+
+from __future__ import annotations
+
+from importlib import import_module
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from usdata.inspect import NetcdfVariable
+from usdata.readers import MissingReaderDependency, fill_registry_attrs
+
+if TYPE_CHECKING:
+    from usdata._fetch import FetchedAsset
+
+PIP_HINT = 'NetCDF4 reading requires xarray and h5netcdf; install: pip install "usdata[netcdf]"'
+
+
+def _xarray() -> Any:
+    """The xarray module once h5netcdf can back it, or a hint naming the extra."""
+    try:
+        xarray = import_module("xarray")
+        import_module("h5netcdf")
+        import_module("h5py")
+    except ModuleNotFoundError as error:
+        if error.name not in {"xarray", "h5netcdf", "h5py"}:
+            raise
+        raise MissingReaderDependency(PIP_HINT) from error
+    return xarray
+
+
+def _text(value: Any) -> str | None:
+    """One attribute as a single string, or None where the file states nothing."""
+    return None if value is None else str(value)
+
+
+def open_netcdf(fetched: FetchedAsset) -> Any:
+    """Load a NetCDF4 root Dataset, then close every source file handle."""
+    xarray = _xarray()
+    # A local file object and fixed engine prevent interpretation as an OPeNDAP URL.
+    with (
+        fetched.path.open("rb") as stream,
+        xarray.open_dataset(stream, engine="h5netcdf", chunks=None) as dataset,
+    ):
+        dataset.load()
+    dataset.attrs["usdata"] = {
+        "asset_id": fetched.asset.id,
+        "provenance": fetched.provenance.model_dump(mode="json"),
+    }
+    fill_registry_attrs(fetched, dataset)
+    return dataset
+
+
+def variables(path: Path) -> list[NetcdfVariable]:
+    """Describe a local NetCDF4 file's data variables without loading their values."""
+    xarray = _xarray()
+    with (
+        path.open("rb") as stream,
+        xarray.open_dataset(stream, engine="h5netcdf", chunks=None) as dataset,
+    ):
+        return [
+            NetcdfVariable(
+                name=str(name),
+                dims=[str(dim) for dim in array.dims],
+                shape=[int(size) for size in array.shape],
+                units=_text(array.attrs.get("units")),
+                long_name=_text(array.attrs.get("long_name")),
+            )
+            for name, array in dataset.data_vars.items()
+        ]

{usdata-0.16.0 → usdata-0.17.0}/src/usdata/cache.py

@@ -23,14 +23,31 @@ def cache_dir() -> Path:
 
 def asset_path(asset: Asset, root: Path | None = None) -> Path:
     """Where an asset lives in the cache: <root>/<provider>/<dataset>/<asset id>."""
-    provider, sep, name = asset.dataset_id.partition(":")
+    return cached_path(asset.dataset_id, asset.id, root)
+
+
+def cached_path(dataset_id: str, asset_id: str, root: Path | None = None) -> Path:
+    """Where the cache keeps one asset, addressed by its ids rather than the asset itself.
+
+    Args:
+        dataset_id: The owning dataset's ``provider:name`` id.
+        asset_id: The asset's id; slashes become underscores, as they do on fetch.
+        root: Cache root to resolve against, or the configured cache directory.
+
+    Returns:
+        The path the cache uses for that asset, whether or not it exists.
+
+    Raises:
+        ValueError: Either id is unsafe, or the path would escape the cache root.
+    """
+    provider, sep, name = dataset_id.partition(":")
     if not sep or any(
         not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*", part) for part in (provider, name)
     ):
-        raise ValueError(f"unsafe dataset id: {asset.dataset_id!r}")
-    safe_id = asset.id.replace("/", "_")
+        raise ValueError(f"unsafe dataset id: {dataset_id!r}")
+    safe_id = asset_id.replace("/", "_")
     if not safe_id or safe_id in {".", ".."} or "\\" in safe_id or "\x00" in safe_id:
-        raise ValueError(f"unsafe asset id: {asset.id!r}")
+        raise ValueError(f"unsafe asset id: {asset_id!r}")
     base = (root or cache_dir()).expanduser().resolve()
     path = base / provider / name / safe_id
     for candidate in (path, path.with_name(path.name + ".provenance.json")):