usdata 0.27.0__tar.gz → 0.28.0__tar.gz

{usdata-0.27.0 → usdata-0.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.27.0
+Version: 0.28.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke

{usdata-0.27.0 → usdata-0.28.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.27.0"
+version = "0.28.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.27.0 → usdata-0.28.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.27.0"
+version = "0.28.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/_aqs.py

@@ -13,7 +13,7 @@ import json
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-from usdata.readers import MissingReaderDependency
+from usdata.readers import MissingReaderDependency, source_attrs
 
 if TYPE_CHECKING:
     from usdata._fetch import FetchedAsset
@@ -43,9 +43,5 @@ def open_aqs(fetched: FetchedAsset) -> Any:
     for column in DATE_COLUMNS:
         if column in frame:
             frame[column] = pandas.to_datetime(frame[column], format="%Y-%m-%d")
-    frame.attrs["usdata"] = {
-        "asset_id": fetched.asset.id,
-        "provenance": fetched.provenance.model_dump(mode="json"),
-        "header": body.get("Header", []),
-    }
+    frame.attrs["usdata"] = {**source_attrs(fetched), "header": body.get("Header", [])}
     return frame

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/_grib.py

@@ -22,12 +22,16 @@ from collections.abc import Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from datetime import UTC, datetime
 from importlib import import_module
-from inspect import currentframe
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, NamedTuple
 
 from usdata.inspect import GribMessage
-from usdata.readers import MissingReaderDependency, fill_registry_attrs
+from usdata.readers import (
+    MissingReaderDependency,
+    caller_stacklevel,
+    fill_registry_attrs,
+    source_attrs,
+)
 
 if TYPE_CHECKING:
     from usdata._fetch import FetchedAsset
@@ -40,6 +44,11 @@ LIBRARY_HINT = (
 )
 MRMS_NAME = re.compile(r"^MRMS_(?P<product>.+?)_\d{2}\.\d{2}_\d{8}-\d{6}\.grib2(?:\.gz)?$")
 INVENTORY_KEYS = ("shortName", "name", "typeOfLevel", "level", "step", "units")
+SYMBOL_POWER = re.compile(r"(?<=[A-Za-z])\*\*")
+"""A power written after a unit symbol, as ecCodes writes ``kg**-1``."""
+NUMERIC_POWER = re.compile(r"(?<![A-Za-z])\*\*")
+"""A power written after anything else, such as the ``10**-3`` of a scale factor."""
+
 VARIABLE_KEYS = (
     "name",
     "units",
@@ -201,18 +210,6 @@ def _unmatched_report(
     return " ".join(parts)
 
 
-def _caller_stacklevel() -> int:
-    """Stack level of the first frame outside usdata, so a warning points at the caller."""
-    package = Path(__file__).resolve().parent
-    frame = currentframe()
-    frame = frame.f_back if frame is not None else None
-    level = 1
-    while frame is not None and Path(frame.f_code.co_filename).resolve().is_relative_to(package):
-        level += 1
-        frame = frame.f_back
-    return level
-
-
 def _shape(eccodes: Any, h: int) -> tuple[int, int] | None:
     """A message's grid rows and columns under either key pair, or None when neither is set."""
     rows, cols = _get(eccodes, h, "Nj", int), _get(eccodes, h, "Ni", int)
@@ -242,6 +239,20 @@ def inventory(path: Path) -> list[GribMessage]:
     return messages
 
 
+def udunits(units: str) -> str:
+    """EcCodes ``units`` in the UDUNITS notation CF metadata uses, or unchanged.
+
+    ecCodes writes powers as ``**``: ``J kg**-1``, ``m**2 s**-2``. UDUNITS and
+    CF write them as a trailing signed integer, ``J kg-1`` and ``m2 s-2``,
+    which is what xarray-based tools expect. Only the notation changes. A power
+    of a number, such as ``10**-3``, has no such spelling, so a string holding
+    one is returned as it is rather than half rewritten.
+    """
+    if NUMERIC_POWER.search(units):
+        return units
+    return SYMBOL_POWER.sub("", units)
+
+
 def _available(path: Path) -> str:
     """Every message as a (shortName, typeOfLevel, level) triple, for a reader error to list."""
     return ", ".join(
@@ -441,6 +452,10 @@ def open_grib2(
         attrs = {
             key: value for key in VARIABLE_KEYS if (value := _get(eccodes, h, key)) is not None
         }
+        if isinstance(units := attrs.get("units"), str) and (plain := udunits(units)) != units:
+            # The file's own spelling stays beside the rewritten one, as cfgrib keeps it.
+            attrs["GRIB_units"] = units
+            attrs["units"] = plain
         for label, date_key, time_key in (
             ("reference_time", "dataDate", "dataTime"),
             ("valid_time", "validityDate", "validityTime"),
@@ -481,7 +496,7 @@ def open_grib2(
     if report := _unmatched_report(options, matched, present):
         if strict:
             raise ValueError(report)
-        warnings.warn(report, UserWarning, stacklevel=_caller_stacklevel())
+        warnings.warn(report, UserWarning, stacklevel=caller_stacklevel())
     variables: dict[str, Any] = {}
     messages: dict[str, dict[str, Any]] = {}
     names = variable_names(
@@ -497,10 +512,6 @@ def open_grib2(
     dataset.latitude.attrs["units"] = "degrees_north"
     dataset.longitude.attrs["units"] = "degrees_east"
     dataset.attrs.update(grid.attrs)
-    dataset.attrs["usdata"] = {
-        "asset_id": fetched.asset.id,
-        "provenance": fetched.provenance.model_dump(mode="json"),
-        "messages": messages,
-    }
+    dataset.attrs["usdata"] = {**source_attrs(fetched), "messages": messages}
     fill_registry_attrs(fetched, dataset)
     return dataset

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/_hurdat2.py

@@ -15,7 +15,7 @@ from datetime import UTC, datetime
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-from usdata.readers import Hurdat2FormatError, MissingReaderDependency
+from usdata.readers import Hurdat2FormatError, MissingReaderDependency, source_attrs
 
 if TYPE_CHECKING:
     from usdata._fetch import FetchedAsset
@@ -174,8 +174,5 @@ def open_hurdat2(fetched: FetchedAsset) -> Any:
     data["time"] = pandas.to_datetime(columns["time"], utc=True)
     data.update({name: pandas.array(columns[name], dtype="float64") for name in NUMERIC_COLUMNS})
     frame = pandas.DataFrame(data, columns=COLUMNS)
-    frame.attrs["usdata"] = {
-        "asset_id": fetched.asset.id,
-        "provenance": fetched.provenance.model_dump(mode="json"),
-    }
+    frame.attrs["usdata"] = source_attrs(fetched)
     return frame

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/_netcdf.py

@@ -7,7 +7,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from usdata.inspect import NetcdfVariable
-from usdata.readers import MissingReaderDependency, fill_registry_attrs
+from usdata.readers import MissingReaderDependency, fill_registry_attrs, source_attrs
 
 if TYPE_CHECKING:
     from usdata._fetch import FetchedAsset
@@ -42,10 +42,7 @@ def open_netcdf(fetched: FetchedAsset) -> Any:
         xarray.open_dataset(stream, engine="h5netcdf", chunks=None) as dataset,
     ):
         dataset.load()
-    dataset.attrs["usdata"] = {
-        "asset_id": fetched.asset.id,
-        "provenance": fetched.provenance.model_dump(mode="json"),
-    }
+    dataset.attrs["usdata"] = source_attrs(fetched)
     fill_registry_attrs(fetched, dataset)
     return dataset
 

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/_radar.py

@@ -7,7 +7,7 @@ import gzip
 from importlib import import_module
 from typing import TYPE_CHECKING, Any
 
-from usdata.readers import MissingReaderDependency, RadarDecodeError
+from usdata.readers import MissingReaderDependency, RadarDecodeError, source_attrs
 
 # NOAA RDA/RPG ICD 2620002Y, Table XVII-I notes 21 and 30.
 MOMENT_FLAG_COUNTS = {
@@ -117,8 +117,7 @@ def open_nexrad(fetched: FetchedAsset, *, sweep: int | list[int] | None = None)
                     masked.encoding = data.encoding.copy()
                     node[name] = masked
     radar.attrs["usdata"] = {
-        "asset_id": fetched.asset.id,
-        "provenance": fetched.provenance.model_dump(mode="json"),
+        **source_attrs(fetched),
         "sweeps": [name.lstrip("/") for name in radar.groups if name.startswith("/sweep_")],
     }
     return radar

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/models.py

@@ -500,6 +500,14 @@ class Asset(BaseModel):
     checksum: str | None = Field(default=None, description="'<algo>:<hex>', e.g. 'sha256:ab12...'")
     time: TimeRange | None = None
     bbox: BBox | None = None
+    properties: dict[str, str] = Field(
+        default_factory=dict,
+        description=(
+            "Facts of the request the bytes do not state, such as the unit system, datum, "
+            "or station, keyed in lowercase snake case; readers copy them into "
+            "attrs['usdata']['properties'] (ADR 0043)"
+        ),
+    )
 
 
 class ByteRange(BaseModel):

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/providers/noaa/coops.py

@@ -227,6 +227,8 @@ class _CoopsStation(HttpProvider):
             protocol=Protocol.HTTP,
             media_type="text/csv",
             time=TimeRange(start=start, end=end),
+            # The CSV states none of these, so they travel with the asset (ADR 0043).
+            properties={"station": station, **params, "time_zone": "gmt"},
         )
 
     def _validate(self, path: Path, asset: Asset) -> None:

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/providers/noaa/ghcnd.py

@@ -163,6 +163,8 @@ class GhcnDaily(HttpProvider):
                     media_type="text/csv",
                     time=window,
                     bbox=query.bbox,
+                    # The CSV has no units row; its stations are in a column.
+                    properties={"units": params.units},
                 )
             )
         return assets

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/providers/noaa/normals.py

@@ -117,6 +117,7 @@ class ClimateNormals(GhcnDaily):
                     media_type="text/csv",
                     time=NORMALS_PERIOD,
                     bbox=query.bbox,
+                    properties={"units": params.units},
                 )
             )
         return assets

{usdata-0.27.0 → usdata-0.28.0}/src/usdata/readers.py

@@ -7,8 +7,10 @@ import gzip
 import io
 import os
 import re
+import warnings
 from collections.abc import Mapping
 from importlib import import_module
+from inspect import currentframe
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
@@ -226,6 +228,32 @@ def fill_registry_attrs(fetched: FetchedAsset, data: Any) -> None:
         data.attrs["usdata"]["registry_attrs"] = filled
 
 
+def caller_stacklevel() -> int:
+    """Stack level of the first frame outside usdata, so a warning points at the caller."""
+    package = Path(__file__).resolve().parent
+    frame = currentframe()
+    frame = frame.f_back if frame is not None else None
+    level = 1
+    while frame is not None and Path(frame.f_code.co_filename).resolve().is_relative_to(package):
+        level += 1
+        frame = frame.f_back
+    return level
+
+
+def source_attrs(fetched: FetchedAsset) -> dict[str, Any]:
+    """The ``attrs["usdata"]`` every reader starts from: the asset, its request facts, its record.
+
+    ``properties`` are the facts of the request the bytes do not state, such as
+    a unit system or datum, always a mapping and empty when the asset records
+    none (ADR 0043).
+    """
+    return {
+        "asset_id": fetched.asset.id,
+        "properties": dict(fetched.asset.properties),
+        "provenance": fetched.provenance.model_dump(mode="json"),
+    }
+
+
 def _local_timestamps(pandas: Any, values: Any) -> Any:
     """Storm Events local timestamps as tz-naive datetimes, unparsable strings as NaT.
 
@@ -259,15 +287,29 @@ def derive_storm_events_utc(pandas: Any, frame: Any) -> None:
     timestamp does not yield an instant gets ``NaT``, and each derived column,
     its source, the rule, that row count, and the labels that gave no offset
     with how many rows carry each are listed under
-    ``frame.attrs["usdata"]["derived"]``. A frame missing any of the three
-    source columns is left alone.
+    ``frame.attrs["usdata"]["derived"]``.
+
+    Each UTC column needs only its own local column and ``CZ_TIMEZONE``, so a
+    ``usecols`` that keeps ``BEGIN_DATE_TIME`` but not ``END_DATE_TIME`` still
+    gets ``BEGIN_UTC``. A frame that keeps a local column without
+    ``CZ_TIMEZONE`` cannot be converted, and says so with a ``UserWarning``
+    rather than coming back without the column it would otherwise have.
 
     Args:
         pandas: The imported pandas module.
         frame: The DataFrame read from a Storm Events details CSV.
     """
-    required = {*STORM_EVENTS_UTC_COLUMNS, STORM_EVENTS_TIMEZONE_COLUMN}
-    if not required.issubset(frame.columns):
+    sources = [source for source in STORM_EVENTS_UTC_COLUMNS if source in frame.columns]
+    if not sources:
+        return
+    if STORM_EVENTS_TIMEZONE_COLUMN not in frame.columns:
+        derivable = ", ".join(STORM_EVENTS_UTC_COLUMNS[source] for source in sources)
+        warnings.warn(
+            f"{', '.join(sources)} read without {STORM_EVENTS_TIMEZONE_COLUMN}, so "
+            f"{derivable} cannot be derived; add {STORM_EVENTS_TIMEZONE_COLUMN} to usecols",
+            UserWarning,
+            stacklevel=caller_stacklevel(),
+        )
         return
     labels = frame[STORM_EVENTS_TIMEZONE_COLUMN].astype("string").str.strip()
     stated = pandas.to_numeric(
@@ -279,7 +321,8 @@ def derive_storm_events_utc(pandas: Any, frame: Any) -> None:
     without_offset = labels[hours.isna() & labels.notna()].value_counts()
     unconverted = {str(label): int(count) for label, count in sorted(without_offset.items())}
     derived = []
-    for source, column in STORM_EVENTS_UTC_COLUMNS.items():
+    for source in sources:
+        column = STORM_EVENTS_UTC_COLUMNS[source]
         local = _local_timestamps(pandas, frame[source])
         frame[column] = (local - offsets).dt.tz_localize("UTC")
         derived.append(
@@ -520,10 +563,7 @@ def open_csv(
             )
     if units:
         frame.attrs["units"] = {name: units[name] for name in frame.columns}
-    frame.attrs["usdata"] = {
-        "asset_id": fetched.asset.id,
-        "provenance": fetched.provenance.model_dump(mode="json"),
-    }
+    frame.attrs["usdata"] = source_attrs(fetched)
     if fetched.asset.dataset_id == STORM_EVENTS_DATASET:
         derive_storm_events_utc(pandas, frame)
     return frame