usdata 0.31.0__tar.gz → 0.32.0__tar.gz

{usdata-0.31.0 → usdata-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.31.0
+Version: 0.32.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke

{usdata-0.31.0 → usdata-0.32.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.31.0"
+version = "0.32.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.31.0 → usdata-0.32.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.31.0"
+version = "0.32.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.31.0 → usdata-0.32.0}/src/usdata/__init__.py

@@ -23,7 +23,7 @@ from usdata.models import (
     TemporalSelection,
     TimeRange,
 )
-from usdata.pull import pull, verify
+from usdata.pull import compare_listing, pull, verify
 from usdata.query import build_query
 from usdata.registry import DatasetNotFound, Registry, SearchResult, default_registry
 from usdata.selection import select_by_time
@@ -47,6 +47,7 @@ __all__ = [
     "build_query",
     "cite_dataset",
     "cite_lockfile",
+    "compare_listing",
     "datasets",
     "default_registry",
     "fetch",

{usdata-0.31.0 → usdata-0.32.0}/src/usdata/_grib.py

@@ -19,9 +19,10 @@ import gzip
 import re
 import warnings
 from collections.abc import Iterator, Mapping, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from datetime import UTC, datetime
 from importlib import import_module
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, NamedTuple
 
@@ -86,9 +87,29 @@ PROJECTION_KEYS = (
 )
 
 
+def import_eccodes() -> Any:
+    """Import the ecCodes bindings, loading pyproj first when it is installed.
+
+    The eckitlib wheel that the ecCodes library wheel depends on bundles its own
+    copy of the PROJ library, and so does pyproj's wheel, which xradar and many
+    analysis stacks bring in. When eckitlib's copy is loaded first and pyproj is
+    imported later in the same process, both copies free the same memory as the
+    interpreter exits, which aborts it with "double free or corruption" and exit
+    status 134 after the work is done. Loading pyproj's copy first avoids it.
+    A pyproj that fails to import is left for whoever uses it to report.
+
+    Raises:
+        ModuleNotFoundError: eccodes is not installed.
+    """
+    if find_spec("pyproj") is not None:
+        with suppress(ImportError, OSError):
+            import_module("pyproj")
+    return import_module("eccodes")
+
+
 def _modules() -> tuple[Any, Any, Any]:
     try:
-        eccodes = import_module("eccodes")
+        eccodes = import_eccodes()
         xarray = import_module("xarray")
         numpy = import_module("numpy")
     except ModuleNotFoundError as error:

{usdata-0.31.0 → usdata-0.32.0}/src/usdata/cli/app.py

@@ -23,7 +23,15 @@ from usdata.manifest import lockfile_path
 from usdata.models import READER_EXTRAS_TEXT, Dataset, Status, describe_duration
 from usdata.providers import adapter_class, load_adapter
 from usdata.providers.base import NotImplementedProvider
-from usdata.pull import EmptySource, ManifestChanged, Plan, UnknownDatasets, UpstreamChanged
+from usdata.pull import (
+    EmptySource,
+    ListingComparison,
+    ManifestChanged,
+    Plan,
+    UnknownDatasets,
+    UpstreamChanged,
+)
+from usdata.pull import compare_listing as compare_manifest_listing
 from usdata.pull import plan as plan_manifest
 from usdata.pull import pull as pull_manifest
 from usdata.pull import verify as verify_manifest
@@ -554,12 +562,32 @@ def pull(
 def verify(
     manifest: Annotated[Path, typer.Argument(exists=True, dir_okay=False)],
     cache_dir: Annotated[Path | None, typer.Option(help="Override the cache directory.")] = None,
+    listing: Annotated[
+        bool,
+        typer.Option(
+            "--listing",
+            help="Instead of the cache, compare the lockfile with what each source lists today; "
+            "downloads nothing, but asks every source.",
+        ),
+    ] = False,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="With --listing, emit the comparison as JSON on stdout.")
+    ] = False,
 ) -> None:
     """Check cached files against a manifest's lockfile. Exit 1 on any drift."""
+    if as_json and not listing:
+        typer.secho("--json applies to --listing only", err=True, fg="red")
+        raise typer.Exit(code=2)
+    if listing and cache_dir is not None:
+        typer.secho("--listing reads no cache; drop --cache-dir", err=True, fg="red")
+        raise typer.Exit(code=2)
     lock = lockfile_path(manifest)
     if not lock.exists():
         typer.secho(f"no lockfile at {lock}; run pull first", err=True, fg="red")
         raise typer.Exit(code=2)
+    if listing:
+        _verify_listing(manifest, as_json=as_json)
+        return
     try:
         drift = verify_manifest(manifest, root=cache_dir)
     except (ManifestChanged, ValueError, OSError) as e:
@@ -573,4 +601,55 @@ def verify(
     typer.echo(f"all assets match {lock.name}", err=True)
 
 
+def _verify_listing(manifest: Path, *, as_json: bool) -> None:
+    """Compare the lockfile with today's listings; exit 1 when a re-resolve would differ."""
+    try:
+        comparison = compare_manifest_listing(manifest)
+    except EmptySource as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=1) from None
+    except (DatasetNotFound, UnknownDatasets, ManifestChanged, UnknownPlace, ValueError) as e:
+        typer.secho(str(e), err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    except NotImplementedProvider as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=3) from None
+    except httpx.HTTPError as e:
+        typer.secho(f"listing failed: {e}", err=True, fg="red")
+        raise typer.Exit(code=4) from None
+    _print_listing(comparison, as_json=as_json)
+    if not comparison.matches:
+        raise typer.Exit(code=1)
+
+
+def _print_listing(comparison: ListingComparison, *, as_json: bool) -> None:
+    """One line per difference, a summary on stderr; or the whole comparison as JSON."""
+    name = comparison.lockfile.name
+    if as_json:
+        typer.echo(comparison.model_dump_json(indent=2))
+    else:
+        for source in comparison.sources:
+            for asset in source.added:
+                typer.echo(f"{source.source}\tadded\t{asset.id}\t{asset.href}")
+            for asset in source.removed:
+                typer.echo(f"{source.source}\tremoved\t{asset.id}\t{asset.href}")
+            for change in source.changed:
+                sizes = f"{_size(change.locked_size)} -> {_size(change.listed_size)} bytes"
+                typer.echo(f"{source.source}\tchanged\t{change.asset_id}\t{sizes}")
+    differing = [source.source for source in comparison.sources if not source.matches]
+    if differing:
+        typer.secho(
+            f"{len(differing)} source(s) list something other than {name} pins: "
+            f"{', '.join(differing)}. Pass pull --force to re-resolve.",
+            err=True,
+            fg="red",
+        )
+    else:
+        typer.echo(f"every source lists what {name} pins", err=True)
+
+
+def _size(value: int | None) -> str:
+    return "?" if value is None else str(value)
+
+
 app.command()(doctor)

{usdata-0.31.0 → usdata-0.32.0}/src/usdata/doctor.py

@@ -28,7 +28,7 @@ import httpx
 from pydantic import BaseModel
 
 from usdata import __version__
-from usdata._grib import LIBRARY_HINT
+from usdata._grib import LIBRARY_HINT, import_eccodes
 from usdata.cache import ENV_VAR, cache_dir
 from usdata.mirror import ENV_VAR as MIRROR_ENV_VAR
 from usdata.models import READER_EXTRAS, Status
@@ -145,7 +145,8 @@ def _reader_check(extra: str) -> Check:
     modules: list[Any] = []
     for module in READER_MODULES[extra]:
         try:
-            modules.append(import_module(module))
+            # ecCodes goes through the reader's import, which orders it after pyproj.
+            modules.append(import_eccodes() if module == "eccodes" else import_module(module))
         except (ImportError, RuntimeError, OSError) as error:
             # eccodes imports and then fails to load its shared library.
             return Check(

{usdata-0.31.0 → usdata-0.32.0}/src/usdata/pull.py

@@ -157,6 +157,22 @@ class PullResult(BaseModel):
         return items[0]
 
 
+def _source_keys(entries: Iterable[LockedAsset]) -> list[str]:
+    """The manifest source key each lockfile entry belongs to, in entry order.
+
+    A lockfile written before sources had keys records none; such an entry falls
+    back to the one-based position of its dataset id among the datasets seen.
+    """
+    positions: dict[str, str] = {}
+    keys: list[str] = []
+    for entry in entries:
+        dataset_id = entry.asset.dataset_id
+        if dataset_id not in positions:
+            positions[dataset_id] = str(len(positions) + 1)
+        keys.append(entry.source or positions[dataset_id])
+    return keys
+
+
 def _by_source(pairs: Iterable[tuple[LockedAsset, FetchedAsset]]) -> dict[str, list[FetchedAsset]]:
     """Group fetched assets by the source key their lockfile entry records.
 
@@ -169,12 +185,9 @@ def _by_source(pairs: Iterable[tuple[LockedAsset, FetchedAsset]]) -> dict[str, l
     them and a forced pull rewrites the lockfile.
     """
     grouped: dict[str, list[FetchedAsset]] = {}
-    positions: dict[str, str] = {}
-    for entry, item in pairs:
-        dataset_id = entry.asset.dataset_id
-        if dataset_id not in positions:
-            positions[dataset_id] = str(len(positions) + 1)
-        grouped.setdefault(entry.source or positions[dataset_id], []).append(item)
+    pairs = list(pairs)
+    for key, (_, item) in zip(_source_keys([entry for entry, _ in pairs]), pairs, strict=True):
+        grouped.setdefault(key, []).append(item)
     by_id: dict[str, FetchedAsset] = {}
     for items in grouped.values():
         by_id.update({item.asset.id: item for item in items})
@@ -381,6 +394,123 @@ def plan(manifest_path: str | os.PathLike[str], *, registry: Registry | None = N
     return Plan(manifest=manifest.name, sources=sources)
 
 
+class AssetChange(BaseModel):
+    """A pinned asset that today's listing still names, but describes differently."""
+
+    asset_id: str
+    locked_size: int | None = Field(description="Size the listing gave when the pin was written")
+    listed_size: int | None = Field(description="Size today's listing gives")
+    locked_href: str
+    listed_href: str
+
+
+class SourceComparison(BaseModel):
+    """How one source's pinned assets compare with what its adapter lists today."""
+
+    source: str = Field(description="The source's manifest key: its name, or its position")
+    dataset_id: str
+    added: list[Asset] = Field(description="Listed today and not pinned")
+    removed: list[Asset] = Field(description="Pinned and no longer listed")
+    changed: list[AssetChange] = Field(
+        description="Pinned and listed, with a different href, or a different size where both "
+        "the pin and the listing give one"
+    )
+    unchanged: int = Field(description="How many pinned assets are listed exactly as pinned")
+
+    @computed_field(description="Whether a re-resolve of this source would pin the same assets")
+    @property
+    def matches(self) -> bool:
+        """True when nothing was added, removed, or changed."""
+        return not (self.added or self.removed or self.changed)
+
+
+class ListingComparison(BaseModel):
+    """A lockfile against today's listings: whether a re-resolve would pin the same assets.
+
+    Nothing is downloaded, so the bytes are not compared: restore checks those,
+    and ``verify`` checks the cache. What this sees is the set of files, an
+    asset added, removed, or re-described since the lockfile was written.
+    """
+
+    manifest: str
+    lockfile: Path
+    sources: list[SourceComparison] = Field(description="One entry per manifest source, in order")
+
+    @computed_field(description="Whether a re-resolve of every source would pin the same assets")
+    @property
+    def matches(self) -> bool:
+        """True when every source matches."""
+        return all(source.matches for source in self.sources)
+
+
+def _changed(pinned: Asset, listed: Asset) -> AssetChange | None:
+    """How ``listed`` differs from the pin of the same id, or None when it does not."""
+    sizes_differ = None not in (pinned.size, listed.size) and pinned.size != listed.size
+    if not sizes_differ and pinned.href == listed.href:
+        return None
+    return AssetChange(
+        asset_id=pinned.id,
+        locked_size=pinned.size,
+        listed_size=listed.size,
+        locked_href=pinned.href,
+        listed_href=listed.href,
+    )
+
+
+def compare_listing(
+    manifest_path: str | os.PathLike[str], *, registry: Registry | None = None
+) -> ListingComparison:
+    """Compare a manifest's lockfile with what its sources list today; download nothing.
+
+    Every source is listed as ``plan`` lists it, ``select`` rules included, and
+    its assets are matched to the lockfile's entries for that source by id. An
+    asset only in the listing is ``added``, one only in the lockfile ``removed``,
+    and one in both with a different href, or a different size where both give
+    one, ``changed``. A listing that reports no size cannot show a size change;
+    restoring the pin checks the bytes themselves.
+
+    Args:
+        manifest_path: Path of the manifest YAML file, as a string or any ``os.PathLike``.
+        registry: Registry to resolve dataset ids against; the default one if omitted.
+
+    Returns:
+        A ``ListingComparison`` with one entry per manifest source.
+
+    Raises:
+        FileNotFoundError: The manifest has no lockfile yet.
+        ManifestChanged: The manifest changed since its lockfile was written.
+    """
+    manifest_path = Path(manifest_path)
+    lock_path = lockfile_path(manifest_path)
+    if not lock_path.exists():
+        raise FileNotFoundError(f"no lockfile at {lock_path}; run pull first")
+    lock = Lockfile.load(lock_path)
+    _check_manifest(manifest_path, lock)
+    pinned: dict[str, dict[str, Asset]] = {}
+    for key, entry in zip(_source_keys(lock.assets), lock.assets, strict=True):
+        pinned.setdefault(key, {})[entry.asset.id] = entry.asset
+    sources: list[SourceComparison] = []
+    for listed in plan(manifest_path, registry=registry).sources:
+        locked = pinned.get(listed.source, {})
+        today = {asset.id: asset for asset in listed.assets}
+        changes = [
+            change
+            for asset_id, asset in locked.items()
+            if asset_id in today and (change := _changed(asset, today[asset_id])) is not None
+        ]
+        sources.append(
+            SourceComparison(
+                source=listed.source,
+                dataset_id=listed.dataset_id,
+                added=[asset for asset in listed.assets if asset.id not in locked],
+                removed=[asset for asset_id, asset in locked.items() if asset_id not in today],
+                changed=changes,
+                unchanged=sum(1 for asset_id in locked if asset_id in today) - len(changes),
+            )
+        )
+    return ListingComparison(manifest=lock.manifest, lockfile=lock_path, sources=sources)
+
+
 def _selected(lock: Lockfile, update: Iterable[str]) -> set[str]:
     """Ids of the locked assets that ``update`` selectors (asset or dataset ids) name."""
     selectors = set(update)