usdata 0.28.0__tar.gz → 0.30.0__tar.gz

{usdata-0.28.0 → usdata-0.30.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.28.0
+Version: 0.30.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke

{usdata-0.28.0 → usdata-0.30.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.28.0"
+version = "0.30.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.28.0 → usdata-0.30.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.28.0"
+version = "0.30.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.28.0 → usdata-0.30.0}/src/usdata/_fetch.py

@@ -132,6 +132,29 @@ def _sidecar_not_older(path: Path) -> bool:
         return False
 
 
+def _cached(dataset: Dataset, asset: Asset, path: Path) -> Provenance | None:
+    """The record of a file at ``path`` that still describes ``asset``, or None if there is none."""
+    if not path.is_file():
+        return None
+    try:
+        prov = provenance.read(path)
+    except (ValueError, OSError):
+        return None
+    if (
+        prov.dataset_id == dataset.id
+        and prov.provider == dataset.provider
+        and prov.source_url == asset.href
+        and prov.size == path.stat().st_size
+        # A listing that now reports another size describes a rebuilt file.
+        and (asset.size is None or prov.size == asset.size)
+        and (asset.checksum is None or prov.checksum == asset.checksum)
+        # Trust an untouched cached file; hash whenever anything is unclear.
+        and (_sidecar_not_older(path) or sha256_file(path) == prov.checksum)
+    ):
+        return prov
+    return None
+
+
 def _fetch_asset(
     dataset: Dataset,
     asset: Asset,
@@ -149,35 +172,23 @@ def _fetch_asset(
     from the record rather than by resolving its query again.
 
     ``staging`` is a root a download is written under instead of the cache. The
-    cache at ``root`` is still checked, and a hit is returned from there; a
-    miss comes back with its ``path`` under ``staging``, for the caller to move
-    home once it can pin it (ADR 0031).
+    cache at ``root`` is still checked, and a hit is returned from there; so is
+    ``staging``, where an earlier source in the same run may have fetched the
+    same asset. A miss comes back with its ``path`` under ``staging``, for the
+    caller to move home once it can pin it (ADR 0031).
     """
     if asset.dataset_id != dataset.id:
         raise ValueError(f"asset dataset {asset.dataset_id!r} does not match {dataset.id!r}")
     _progress.emit(_progress.AssetProgress(asset.id, "start", asset.size))
     path = asset_path(asset, root)
-    if not force and path.is_file():
-        try:
-            prov = provenance.read(path)
-        except (ValueError, OSError):
-            prov = None
-        if (
-            prov is not None
-            and prov.dataset_id == dataset.id
-            and prov.provider == dataset.provider
-            and prov.source_url == asset.href
-            and prov.size == path.stat().st_size
-            # A listing that now reports another size describes a rebuilt file.
-            and (asset.size is None or prov.size == asset.size)
-            and (asset.checksum is None or prov.checksum == asset.checksum)
-            # Trust an untouched cached file; hash whenever anything is unclear.
-            and (_sidecar_not_older(path) or sha256_file(path) == prov.checksum)
-        ):
+    staged = None if staging is None else asset_path(asset, staging)
+    # A staged run looks in its staging too, so an asset two sources share is fetched once.
+    for candidate in () if force else (path, staged):
+        if candidate is not None and (prov := _cached(dataset, asset, candidate)) is not None:
             _progress.emit(_progress.AssetProgress(asset.id, "cached", prov.size))
-            return FetchedAsset(asset=asset, path=path, provenance=prov, from_cache=True)
-    if staging is not None:
-        path = asset_path(asset, staging)
+            return FetchedAsset(asset=asset, path=candidate, provenance=prov, from_cache=True)
+    if staged is not None:
+        path = staged
     with staged_path(path) as tmp:
         partial = adapter.prepare_fetch(asset, pinned)
         if partial is None:
@@ -223,6 +234,19 @@ def _fetch_with(
     ``staging`` is passed to each fetch; see ``_fetch_asset``.
     """
     assets = ordered(adapter.list_assets(query))
+    return _fetch_listed(adapter, dataset, assets, root=root, force=force, staging=staging)
+
+
+def _fetch_listed(
+    adapter: Provider,
+    dataset: Dataset,
+    assets: list[Asset],
+    *,
+    root: Path | None = None,
+    force: bool = False,
+    staging: Path | None = None,
+) -> list[FetchedAsset]:
+    """Fetch assets already listed and put in order, as the loop does once it has listed them."""
     _progress.batch([asset.size for asset in assets])
     return [
         _fetch_asset(dataset, a, adapter, root=root, force=force, staging=staging) for a in assets

usdata-0.30.0/src/usdata/_files.py

@@ -0,0 +1,64 @@
+"""Atomic replacement helpers for downloads and JSON records."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+TEMP_SUFFIX = ".part"
+
+
+def is_temporary(path: Path) -> bool:
+    """Whether ``path`` is a temporary file ``staged_path`` made, finished or abandoned."""
+    return path.name.startswith(".") and path.name.endswith(TEMP_SUFFIX)
+
+
+@contextmanager
+def staged_path(dest: Path) -> Iterator[Path]:
+    """Replace dest only after successful work in a unique sibling temporary file.
+
+    The file is flushed to disk before it replaces ``dest``, and the rename
+    after, so once this returns the new ``dest`` survives a crash; without that
+    a crash could keep the rename but lose the bytes, or reorder two renames.
+    """
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    with NamedTemporaryFile(
+        prefix=f".{dest.name}.", suffix=TEMP_SUFFIX, dir=dest.parent, delete=False
+    ) as f:
+        tmp = Path(f.name)
+    try:
+        yield tmp
+        _fsync(tmp)
+        tmp.replace(dest)
+        _fsync_directory(dest.parent)
+    finally:
+        tmp.unlink(missing_ok=True)
+
+
+def atomic_write_text(dest: Path, text: str) -> None:
+    """Write UTF-8 text without exposing an incomplete record to readers."""
+    with staged_path(dest) as tmp:
+        tmp.write_text(text, encoding="utf-8")
+
+
+def _fsync(path: Path) -> None:
+    # Windows flushes only through a descriptor opened for writing.
+    fd = os.open(path, os.O_RDWR)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)
+
+
+def _fsync_directory(path: Path) -> None:
+    """Flush a directory's entries, where the platform allows opening a directory."""
+    if os.name == "nt":
+        return
+    fd = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(fd)
+    finally:
+        os.close(fd)

{usdata-0.28.0 → usdata-0.30.0}/src/usdata/_grib.py

@@ -42,12 +42,19 @@ LIBRARY_HINT = (
     "on macOS and on Windows with Python 3.14 install it separately, for example "
     "`conda install -c conda-forge eccodes` or `brew install eccodes`"
 )
-MRMS_NAME = re.compile(r"^MRMS_(?P<product>.+?)_\d{2}\.\d{2}_\d{8}-\d{6}\.grib2(?:\.gz)?$")
+MRMS_NAME = re.compile(r"^MRMS_(?P<product>.+?)(?:_\d{2}\.\d{2})?_\d{8}-\d{6}\.grib2(?:\.gz)?$")
+"""An MRMS filename, whose product segment names a variable ecCodes cannot.
+
+Most products end in a level such as ``_00.50``, which the name drops; one,
+``LightningProbabilityNext30minGrid_scale_1``, has none and is named whole.
+"""
 INVENTORY_KEYS = ("shortName", "name", "typeOfLevel", "level", "step", "units")
-SYMBOL_POWER = re.compile(r"(?<=[A-Za-z])\*\*")
-"""A power written after a unit symbol, as ecCodes writes ``kg**-1``."""
-NUMERIC_POWER = re.compile(r"(?<![A-Za-z])\*\*")
-"""A power written after anything else, such as the ``10**-3`` of a scale factor."""
+PARAMETER_KEYS = ("discipline", "parameterCategory", "parameterNumber")
+"""The keys that identify a parameter ecCodes has no short name for."""
+UNNAMED = frozenset({"", "unknown", "~"})
+"""Short names ecCodes gives a parameter its tables do not name."""
+SYMBOL_POWER = re.compile(r"(?<=[A-Za-z])\*\*(?=-?\d+(?![\d.]))")
+"""An integer power of a unit symbol, as ecCodes writes ``kg**-1``."""
 
 VARIABLE_KEYS = (
     "name",
@@ -218,16 +225,23 @@ def _shape(eccodes: Any, h: int) -> tuple[int, int] | None:
     return None if rows is None or cols is None else (rows, cols)
 
 
-def inventory(path: Path) -> list[GribMessage]:
-    """Every message in a local GRIB2 file, in file order, without decoding its values."""
+def inventory(path: Path, source: str | None = None) -> list[GribMessage]:
+    """Every message in a local GRIB2 file, in file order, without decoding its values.
+
+    ``source`` is the asset id the file was fetched as, which names a message
+    ecCodes cannot; it defaults to the file's name, which is the asset id for a
+    cached file.
+    """
     eccodes = _modules()[0]
     messages = []
     for index, h in _messages(eccodes, path):
         keys = {key: _get(eccodes, h, key, str) for key in INVENTORY_KEYS}
+        parameter = [_get(eccodes, h, key) for key in PARAMETER_KEYS]
         messages.append(
             GribMessage(
                 file_index=index,
                 short_name=keys["shortName"],
+                base_name=base_name(keys["shortName"], parameter, source or path.name),
                 name=keys["name"],
                 type_of_level=keys["typeOfLevel"],
                 level=keys["level"],
@@ -244,11 +258,13 @@ def udunits(units: str) -> str:
 
     ecCodes writes powers as ``**``: ``J kg**-1``, ``m**2 s**-2``. UDUNITS and
     CF write them as a trailing signed integer, ``J kg-1`` and ``m2 s-2``,
-    which is what xarray-based tools expect. Only the notation changes. A power
-    of a number, such as ``10**-3``, has no such spelling, so a string holding
-    one is returned as it is rather than half rewritten.
+    which is what xarray-based tools expect. Only the notation changes, and only
+    for an integer power of a unit symbol. Any other power has no such
+    spelling: of a number, as in ``10**-3``, of a group, as in ``(m s**-1)**2``,
+    or not an integer, as in ``m**(2/3)``. A string holding one is returned as
+    it is rather than half rewritten.
     """
-    if NUMERIC_POWER.search(units):
+    if len(SYMBOL_POWER.findall(units)) != units.count("**"):
         return units
     return SYMBOL_POWER.sub("", units)
 
@@ -266,15 +282,41 @@ def _product_name(asset_id: str) -> str | None:
     return match["product"] if match else None
 
 
-def _time(date: Any, time: Any) -> str | None:
+def base_name(short_name: str | None, parameter: Sequence[Any], source: str) -> str:
+    """The name one message's variable starts from, before ``variable_names`` qualifies it.
+
+    That is the ecCodes ``shortName`` where its tables name the parameter.
+    Where they do not, it is the MRMS product the asset id names, or else
+    ``parameter_<discipline>_<category>_<number>``. The reader and
+    ``Grib2Summary.variable_for`` both name messages through here.
+
+    Args:
+        short_name: The message's ecCodes ``shortName``, or None when it has none.
+        parameter: Its ``discipline``, ``parameterCategory``, and ``parameterNumber``.
+        source: The asset id the file was fetched as.
+
+    Returns:
+        The unqualified variable name.
+    """
+    if short_name is not None and short_name not in UNNAMED:
+        return short_name
+    discipline, category, number = parameter
+    return _product_name(source) or f"parameter_{discipline}_{category}_{number}"
+
+
+def _time(date: Any, time: Any, second: Any = None) -> str | None:
+    """An ecCodes date and ``HHMM`` time as ISO 8601 UTC, with the message's seconds.
+
+    ecCodes spells times to the minute and keeps the reference time's seconds
+    in a key of their own, which MRMS sets: a file stamped ``20:00:41`` would
+    otherwise read as ``20:00:00``. A validity time is the reference time plus
+    a whole-minute step, so it carries the same seconds.
+    """
     if date is None or time is None:
         return None
     try:
-        return (
-            datetime.strptime(f"{int(date):08d}{int(time):04d}", "%Y%m%d%H%M")
-            .replace(tzinfo=UTC)
-            .isoformat()
-        )
+        stamp = datetime.strptime(f"{int(date):08d}{int(time):04d}", "%Y%m%d%H%M")
+        return stamp.replace(second=int(second or 0), tzinfo=UTC).isoformat()
     except ValueError:
         return None
 
@@ -456,11 +498,12 @@ def open_grib2(
             # The file's own spelling stays beside the rewritten one, as cfgrib keeps it.
             attrs["GRIB_units"] = units
             attrs["units"] = plain
+        second = _get(eccodes, h, "second", int)
         for label, date_key, time_key in (
             ("reference_time", "dataDate", "dataTime"),
             ("valid_time", "validityDate", "validityTime"),
         ):
-            stamp = _time(attrs.get(date_key), attrs.get(time_key))
+            stamp = _time(attrs.get(date_key), attrs.get(time_key), second)
             if stamp:
                 attrs[label] = stamp
         message: dict[str, Any] = {
@@ -473,12 +516,9 @@ def open_grib2(
         }
         if selectors:
             message["selector"] = selectors[index]
-        short = message["shortName"]
-        if short in (None, "", "unknown", "~"):
-            short = _product_name(fetched.asset.id) or (
-                f"parameter_{attrs.get('discipline')}_{attrs.get('parameterCategory')}"
-                f"_{attrs.get('parameterNumber')}"
-            )
+        short = base_name(
+            message["shortName"], [attrs.get(key) for key in PARAMETER_KEYS], fetched.asset.id
+        )
         selected.append(_Field(short=short, data=data, attrs=attrs, message=message))
     if count > 1 and needs_select:
         raise ValueError(

{usdata-0.28.0 → usdata-0.30.0}/src/usdata/cache.py

@@ -10,6 +10,8 @@ from pathlib import Path
 from usdata.models import Asset
 
 ENV_VAR = "USDATA_CACHE_DIR"
+SIDECAR_SUFFIX = ".provenance.json"
+"""What a provenance sidecar's name adds to its data file's; ``provenance`` names it from here."""
 
 
 def cache_dir() -> Path:
@@ -38,7 +40,9 @@ def cached_path(dataset_id: str, asset_id: str, root: Path | None = None) -> Pat
         The path the cache uses for that asset, whether or not it exists.
 
     Raises:
-        ValueError: Either id is unsafe, or the path would escape the cache root.
+        ValueError: Either id is unsafe, names a sidecar or temporary file, or the
+            path would escape the cache root. Symlinks inside the root are
+            followed like any directory, so part of the cache can live elsewhere.
     """
     provider, sep, name = dataset_id.partition(":")
     if not sep or any(
@@ -48,11 +52,16 @@ def cached_path(dataset_id: str, asset_id: str, root: Path | None = None) -> Pat
     safe_id = asset_id.replace("/", "_")
     if not safe_id or safe_id in {".", ".."} or "\\" in safe_id or "\x00" in safe_id:
         raise ValueError(f"unsafe asset id: {asset_id!r}")
+    # A file of this name would be taken for another asset's sidecar, or for a temporary file.
+    if safe_id.endswith(SIDECAR_SUFFIX) or (safe_id.startswith(".") and safe_id.endswith(".part")):
+        raise ValueError(f"asset id {asset_id!r} names a file the cache keeps for itself")
     base = (root or cache_dir()).expanduser().resolve()
-    path = base / provider / name / safe_id
-    for candidate in (path, path.with_name(path.name + ".provenance.json")):
-        if not candidate.resolve().is_relative_to(base):
-            raise ValueError(f"cache path escapes root: {candidate}")
+    home = base / provider / name
+    path = home / safe_id
+    # Checked on the path as written, so a directory the user symlinked elsewhere is still
+    # inside the cache; only an id that re-anchors the path, such as C:x on Windows, is not.
+    if path.parent != home:
+        raise ValueError(f"cache path escapes root: {path}")
     return path
 
 

{usdata-0.28.0 → usdata-0.30.0}/src/usdata/cache_ops.py

@@ -15,9 +15,10 @@ from pathlib import Path
 from pydantic import BaseModel, Field, ValidationError
 
 from usdata import provenance
+from usdata._files import is_temporary
 from usdata.cache import asset_path, cache_dir
 from usdata.manifest import Lockfile, lockfile_path
-from usdata.models import Provenance
+from usdata.models import AwareUTC, Provenance, as_utc
 from usdata.provenance import SIDECAR_SUFFIX
 
 MANIFEST_GLOBS = ("*.yaml", "*.yml")
@@ -37,8 +38,8 @@ class CacheEntry(BaseModel):
     asset_id: str
     path: Path
     size: int = Field(ge=0, description="Bytes of the data file, excluding its sidecar")
-    modified_at: datetime = Field(description="Filesystem modification time, in UTC")
-    retrieved_at: datetime | None = Field(
+    modified_at: AwareUTC = Field(description="Filesystem modification time, in UTC")
+    retrieved_at: AwareUTC | None = Field(
         default=None,
         description="When the sidecar says the file was fetched; unset without a readable sidecar",
     )
@@ -90,6 +91,7 @@ def entries(root: Path | None = None) -> list[CacheEntry]:
         One entry per file at ``<root>/<provider>/<name>/<asset id>``. A file
         with no readable sidecar is still listed, as unrecorded, with its
         dataset id taken from the directories it sits in. Sidecars themselves,
+        temporary files a download or a killed run left, the staging directory,
         and files at any other depth, are skipped.
     """
     base = _root(root)
@@ -98,9 +100,11 @@ def entries(root: Path | None = None) -> list[CacheEntry]:
     found = [
         _entry(path)
         for provider in _subdirectories(base)
+        # No provider id starts with a dot; the staging directory does.
+        if not provider.name.startswith(".")
         for name in _subdirectories(provider)
         for path in sorted(name.iterdir())
-        if path.is_file() and not path.name.endswith(SIDECAR_SUFFIX)
+        if path.is_file() and not path.name.endswith(SIDECAR_SUFFIX) and not is_temporary(path)
     ]
     return sorted(found, key=lambda entry: (entry.dataset_id, entry.asset_id))
 
@@ -131,11 +135,12 @@ def candidates(
         older_than: Keep entries younger than this age. Unfiltered when None.
         dataset: Keep only entries of this dataset id. Unfiltered when None.
         now: The moment ages are measured from; the current UTC time when omitted.
+            A naive value means UTC.
 
     Returns:
         The matching entries, in the order :func:`entries` returns them.
     """
-    moment = now or datetime.now(UTC)
+    moment = datetime.now(UTC) if now is None else as_utc(now)
     return [
         entry
         for entry in entries(root)

{usdata-0.28.0 → usdata-0.30.0}/src/usdata/cli/app.py

@@ -404,6 +404,8 @@ def fetch(
             typer.echo(f"{len(assets)} asset(s) matched, {sizes}", err=True)
             with progress(disabled=no_progress):
                 batch([asset.size for asset in assets])
+            if not assets:
+                raise typer.Exit(code=1)
             return
         with progress(disabled=no_progress):
             fetched = fetch_query(ds, query, root=cache_dir, force=force)
@@ -507,7 +509,8 @@ def pull(
             typer.echo(f"{d.asset_id}\t{d.problem}\t{d.path}")
         typer.secho(
             f"{len(e.drift)} asset(s) changed upstream; lockfile unchanged. "
-            "Pass --update <asset or dataset id> to accept the new bytes.",
+            "Pass --update <asset or dataset id> to accept the new bytes, "
+            "or --force to re-resolve.",
             err=True,
             fg="red",
         )