usdata 0.29.0__tar.gz → 0.31.0__tar.gz

{usdata-0.29.0 → usdata-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.29.0
+Version: 0.31.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -113,7 +113,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-seven datasets are available today and nineteen more are planned, grouped
+Twenty-eight datasets are available today and eighteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.29.0 → usdata-0.31.0}/README.md

@@ -68,7 +68,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-seven datasets are available today and nineteen more are planned, grouped
+Twenty-eight datasets are available today and eighteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.29.0 → usdata-0.31.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.29.0"
+version = "0.31.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.29.0 → usdata-0.31.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.29.0"
+version = "0.31.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

usdata-0.31.0/src/usdata/_census.py

@@ -0,0 +1,51 @@
+"""Local reading of Census Data API tables into a pandas DataFrame, behind pandas.
+
+A fetched ``census:acs-5year`` file is the service's response as served: a JSON
+array whose first row names the columns and whose other rows are one geography
+each, every value a string. The table is those rows under those names, with the
+estimate and margin-of-error columns as numbers and everything else, geography
+codes and their leading zeros included, as text.
+
+Negative annotation codes such as ``-555555555`` stay the numbers they are.
+Each means something different (too few sample cases, not applicable, a
+controlled estimate), so they are not turned into missing values (ADR 0045).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from usdata.readers import MissingReaderDependency, source_attrs
+
+if TYPE_CHECKING:
+    from usdata._fetch import FetchedAsset
+
+NUMERIC = re.compile(r"[A-Z]\d{5}[A-Z0-9]*_\d{3}[EM]")
+"""An estimate (``...E``) or margin of error (``...M``) variable, such as ``B01003_001E``.
+
+Annotation columns (``...EA``, ``...MA``) and ``NAME`` do not match and stay text.
+"""
+
+
+def open_acs(fetched: FetchedAsset) -> Any:
+    """Parse a fetched ACS table into a pandas DataFrame, one row per geography."""
+    try:
+        pandas = import_module("pandas")
+    except ModuleNotFoundError as error:
+        if error.name != "pandas":
+            raise
+        raise MissingReaderDependency(
+            'ACS reading requires pandas; install it with: pip install "usdata[pandas]" '
+            '(or uv add "usdata[pandas]")'
+        ) from error
+    # Reading a fetched asset is strictly local; the cached file is never rewritten.
+    header, *rows = json.loads(fetched.path.read_text(encoding="utf-8"))
+    frame = pandas.DataFrame(rows, columns=header, dtype="string")
+    for column in header:
+        if NUMERIC.fullmatch(column):
+            frame[column] = pandas.to_numeric(frame[column])
+    frame.attrs["usdata"] = source_attrs(fetched)
+    return frame

{usdata-0.29.0 → usdata-0.31.0}/src/usdata/_fetch.py

@@ -39,7 +39,7 @@ class FetchedAsset(BaseModel):
     def open(self) -> Any:
         """Open local data with the reader its format implies, with that reader's defaults.
 
-        CSV and ERDDAP CSV, HURDAT2 and AQS daily JSON return a pandas DataFrame;
+        CSV and ERDDAP CSV, HURDAT2, AQS daily JSON, and ACS tables return a pandas DataFrame;
         NetCDF4 and GRIB2 a loaded xarray Dataset; NEXRAD Level II an xarray
         DataTree. Provenance is kept in the result's ``attrs["usdata"]``. A file
         that needs options, or whose metadata leaves its format ambiguous, is
@@ -234,6 +234,19 @@ def _fetch_with(
     ``staging`` is passed to each fetch; see ``_fetch_asset``.
     """
     assets = ordered(adapter.list_assets(query))
+    return _fetch_listed(adapter, dataset, assets, root=root, force=force, staging=staging)
+
+
+def _fetch_listed(
+    adapter: Provider,
+    dataset: Dataset,
+    assets: list[Asset],
+    *,
+    root: Path | None = None,
+    force: bool = False,
+    staging: Path | None = None,
+) -> list[FetchedAsset]:
+    """Fetch assets already listed and put in order, as the loop does once it has listed them."""
     _progress.batch([asset.size for asset in assets])
     return [
         _fetch_asset(dataset, a, adapter, root=root, force=force, staging=staging) for a in assets

{usdata-0.29.0 → usdata-0.31.0}/src/usdata/_grib.py

@@ -304,15 +304,19 @@ def base_name(short_name: str | None, parameter: Sequence[Any], source: str) ->
     return _product_name(source) or f"parameter_{discipline}_{category}_{number}"
 
 
-def _time(date: Any, time: Any) -> str | None:
+def _time(date: Any, time: Any, second: Any = None) -> str | None:
+    """An ecCodes date and ``HHMM`` time as ISO 8601 UTC, with the message's seconds.
+
+    ecCodes spells times to the minute and keeps the reference time's seconds
+    in a key of their own, which MRMS sets: a file stamped ``20:00:41`` would
+    otherwise read as ``20:00:00``. A validity time is the reference time plus
+    a whole-minute step, so it carries the same seconds.
+    """
     if date is None or time is None:
         return None
     try:
-        return (
-            datetime.strptime(f"{int(date):08d}{int(time):04d}", "%Y%m%d%H%M")
-            .replace(tzinfo=UTC)
-            .isoformat()
-        )
+        stamp = datetime.strptime(f"{int(date):08d}{int(time):04d}", "%Y%m%d%H%M")
+        return stamp.replace(second=int(second or 0), tzinfo=UTC).isoformat()
     except ValueError:
         return None
 
@@ -494,11 +498,12 @@ def open_grib2(
             # The file's own spelling stays beside the rewritten one, as cfgrib keeps it.
             attrs["GRIB_units"] = units
             attrs["units"] = plain
+        second = _get(eccodes, h, "second", int)
         for label, date_key, time_key in (
             ("reference_time", "dataDate", "dataTime"),
             ("valid_time", "validityDate", "validityTime"),
         ):
-            stamp = _time(attrs.get(date_key), attrs.get(time_key))
+            stamp = _time(attrs.get(date_key), attrs.get(time_key), second)
             if stamp:
                 attrs[label] = stamp
         message: dict[str, Any] = {

{usdata-0.29.0 → usdata-0.31.0}/src/usdata/cache.py

@@ -10,6 +10,8 @@ from pathlib import Path
 from usdata.models import Asset
 
 ENV_VAR = "USDATA_CACHE_DIR"
+SIDECAR_SUFFIX = ".provenance.json"
+"""What a provenance sidecar's name adds to its data file's; ``provenance`` names it from here."""
 
 
 def cache_dir() -> Path:
@@ -38,7 +40,9 @@ def cached_path(dataset_id: str, asset_id: str, root: Path | None = None) -> Pat
         The path the cache uses for that asset, whether or not it exists.
 
     Raises:
-        ValueError: Either id is unsafe, or the path would escape the cache root.
+        ValueError: Either id is unsafe, names a sidecar or temporary file, or the
+            path would escape the cache root. Symlinks inside the root are
+            followed like any directory, so part of the cache can live elsewhere.
     """
     provider, sep, name = dataset_id.partition(":")
     if not sep or any(
@@ -48,11 +52,16 @@ def cached_path(dataset_id: str, asset_id: str, root: Path | None = None) -> Pat
     safe_id = asset_id.replace("/", "_")
     if not safe_id or safe_id in {".", ".."} or "\\" in safe_id or "\x00" in safe_id:
         raise ValueError(f"unsafe asset id: {asset_id!r}")
+    # A file of this name would be taken for another asset's sidecar, or for a temporary file.
+    if safe_id.endswith(SIDECAR_SUFFIX) or (safe_id.startswith(".") and safe_id.endswith(".part")):
+        raise ValueError(f"asset id {asset_id!r} names a file the cache keeps for itself")
     base = (root or cache_dir()).expanduser().resolve()
-    path = base / provider / name / safe_id
-    for candidate in (path, path.with_name(path.name + ".provenance.json")):
-        if not candidate.resolve().is_relative_to(base):
-            raise ValueError(f"cache path escapes root: {candidate}")
+    home = base / provider / name
+    path = home / safe_id
+    # Checked on the path as written, so a directory the user symlinked elsewhere is still
+    # inside the cache; only an id that re-anchors the path, such as C:x on Windows, is not.
+    if path.parent != home:
+        raise ValueError(f"cache path escapes root: {path}")
     return path