usdata 0.30.0__tar.gz → 0.32.0__tar.gz

{usdata-0.30.0 → usdata-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.30.0
+Version: 0.32.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -113,7 +113,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-seven datasets are available today and nineteen more are planned, grouped
+Twenty-eight datasets are available today and eighteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.30.0 → usdata-0.32.0}/README.md

@@ -68,7 +68,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-seven datasets are available today and nineteen more are planned, grouped
+Twenty-eight datasets are available today and eighteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.30.0 → usdata-0.32.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.30.0"
+version = "0.32.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.30.0 → usdata-0.32.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.30.0"
+version = "0.32.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/__init__.py

@@ -23,7 +23,7 @@ from usdata.models import (
     TemporalSelection,
     TimeRange,
 )
-from usdata.pull import pull, verify
+from usdata.pull import compare_listing, pull, verify
 from usdata.query import build_query
 from usdata.registry import DatasetNotFound, Registry, SearchResult, default_registry
 from usdata.selection import select_by_time
@@ -47,6 +47,7 @@ __all__ = [
     "build_query",
     "cite_dataset",
     "cite_lockfile",
+    "compare_listing",
     "datasets",
     "default_registry",
     "fetch",

usdata-0.32.0/src/usdata/_census.py

@@ -0,0 +1,51 @@
+"""Local reading of Census Data API tables into a pandas DataFrame, behind pandas.
+
+A fetched ``census:acs-5year`` file is the service's response as served: a JSON
+array whose first row names the columns and whose other rows are one geography
+each, every value a string. The table is those rows under those names, with the
+estimate and margin-of-error columns as numbers and everything else, geography
+codes and their leading zeros included, as text.
+
+Negative annotation codes such as ``-555555555`` stay the numbers they are.
+Each means something different (too few sample cases, not applicable, a
+controlled estimate), so they are not turned into missing values (ADR 0045).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from usdata.readers import MissingReaderDependency, source_attrs
+
+if TYPE_CHECKING:
+    from usdata._fetch import FetchedAsset
+
+NUMERIC = re.compile(r"[A-Z]\d{5}[A-Z0-9]*_\d{3}[EM]")
+"""An estimate (``...E``) or margin of error (``...M``) variable, such as ``B01003_001E``.
+
+Annotation columns (``...EA``, ``...MA``) and ``NAME`` do not match and stay text.
+"""
+
+
+def open_acs(fetched: FetchedAsset) -> Any:
+    """Parse a fetched ACS table into a pandas DataFrame, one row per geography."""
+    try:
+        pandas = import_module("pandas")
+    except ModuleNotFoundError as error:
+        if error.name != "pandas":
+            raise
+        raise MissingReaderDependency(
+            'ACS reading requires pandas; install it with: pip install "usdata[pandas]" '
+            '(or uv add "usdata[pandas]")'
+        ) from error
+    # Reading a fetched asset is strictly local; the cached file is never rewritten.
+    header, *rows = json.loads(fetched.path.read_text(encoding="utf-8"))
+    frame = pandas.DataFrame(rows, columns=header, dtype="string")
+    for column in header:
+        if NUMERIC.fullmatch(column):
+            frame[column] = pandas.to_numeric(frame[column])
+    frame.attrs["usdata"] = source_attrs(fetched)
+    return frame

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/_fetch.py

@@ -39,7 +39,7 @@ class FetchedAsset(BaseModel):
     def open(self) -> Any:
         """Open local data with the reader its format implies, with that reader's defaults.
 
-        CSV and ERDDAP CSV, HURDAT2 and AQS daily JSON return a pandas DataFrame;
+        CSV and ERDDAP CSV, HURDAT2, AQS daily JSON, and ACS tables return a pandas DataFrame;
         NetCDF4 and GRIB2 a loaded xarray Dataset; NEXRAD Level II an xarray
         DataTree. Provenance is kept in the result's ``attrs["usdata"]``. A file
         that needs options, or whose metadata leaves its format ambiguous, is

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/_grib.py

@@ -19,9 +19,10 @@ import gzip
 import re
 import warnings
 from collections.abc import Iterator, Mapping, Sequence
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from datetime import UTC, datetime
 from importlib import import_module
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, NamedTuple
 
@@ -86,9 +87,29 @@ PROJECTION_KEYS = (
 )
 
 
+def import_eccodes() -> Any:
+    """Import the ecCodes bindings, loading pyproj first when it is installed.
+
+    The eckitlib wheel that the ecCodes library wheel depends on bundles its own
+    copy of the PROJ library, and so does pyproj's wheel, which xradar and many
+    analysis stacks bring in. When eckitlib's copy is loaded first and pyproj is
+    imported later in the same process, both copies free the same memory as the
+    interpreter exits, which aborts it with "double free or corruption" and exit
+    status 134 after the work is done. Loading pyproj's copy first avoids it.
+    A pyproj that fails to import is left for whoever uses it to report.
+
+    Raises:
+        ModuleNotFoundError: eccodes is not installed.
+    """
+    if find_spec("pyproj") is not None:
+        with suppress(ImportError, OSError):
+            import_module("pyproj")
+    return import_module("eccodes")
+
+
 def _modules() -> tuple[Any, Any, Any]:
     try:
-        eccodes = import_module("eccodes")
+        eccodes = import_eccodes()
         xarray = import_module("xarray")
         numpy = import_module("numpy")
     except ModuleNotFoundError as error:

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/cli/app.py

@@ -23,7 +23,15 @@ from usdata.manifest import lockfile_path
 from usdata.models import READER_EXTRAS_TEXT, Dataset, Status, describe_duration
 from usdata.providers import adapter_class, load_adapter
 from usdata.providers.base import NotImplementedProvider
-from usdata.pull import EmptySource, ManifestChanged, Plan, UnknownDatasets, UpstreamChanged
+from usdata.pull import (
+    EmptySource,
+    ListingComparison,
+    ManifestChanged,
+    Plan,
+    UnknownDatasets,
+    UpstreamChanged,
+)
+from usdata.pull import compare_listing as compare_manifest_listing
 from usdata.pull import plan as plan_manifest
 from usdata.pull import pull as pull_manifest
 from usdata.pull import verify as verify_manifest
@@ -554,12 +562,32 @@ def pull(
 def verify(
     manifest: Annotated[Path, typer.Argument(exists=True, dir_okay=False)],
     cache_dir: Annotated[Path | None, typer.Option(help="Override the cache directory.")] = None,
+    listing: Annotated[
+        bool,
+        typer.Option(
+            "--listing",
+            help="Instead of the cache, compare the lockfile with what each source lists today; "
+            "downloads nothing, but asks every source.",
+        ),
+    ] = False,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="With --listing, emit the comparison as JSON on stdout.")
+    ] = False,
 ) -> None:
     """Check cached files against a manifest's lockfile. Exit 1 on any drift."""
+    if as_json and not listing:
+        typer.secho("--json applies to --listing only", err=True, fg="red")
+        raise typer.Exit(code=2)
+    if listing and cache_dir is not None:
+        typer.secho("--listing reads no cache; drop --cache-dir", err=True, fg="red")
+        raise typer.Exit(code=2)
     lock = lockfile_path(manifest)
     if not lock.exists():
         typer.secho(f"no lockfile at {lock}; run pull first", err=True, fg="red")
         raise typer.Exit(code=2)
+    if listing:
+        _verify_listing(manifest, as_json=as_json)
+        return
     try:
         drift = verify_manifest(manifest, root=cache_dir)
     except (ManifestChanged, ValueError, OSError) as e:
@@ -573,4 +601,55 @@ def verify(
     typer.echo(f"all assets match {lock.name}", err=True)
 
 
+def _verify_listing(manifest: Path, *, as_json: bool) -> None:
+    """Compare the lockfile with today's listings; exit 1 when a re-resolve would differ."""
+    try:
+        comparison = compare_manifest_listing(manifest)
+    except EmptySource as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=1) from None
+    except (DatasetNotFound, UnknownDatasets, ManifestChanged, UnknownPlace, ValueError) as e:
+        typer.secho(str(e), err=True, fg="red")
+        raise typer.Exit(code=2) from None
+    except NotImplementedProvider as e:
+        typer.secho(str(e), err=True, fg="yellow")
+        raise typer.Exit(code=3) from None
+    except httpx.HTTPError as e:
+        typer.secho(f"listing failed: {e}", err=True, fg="red")
+        raise typer.Exit(code=4) from None
+    _print_listing(comparison, as_json=as_json)
+    if not comparison.matches:
+        raise typer.Exit(code=1)
+
+
+def _print_listing(comparison: ListingComparison, *, as_json: bool) -> None:
+    """One line per difference, a summary on stderr; or the whole comparison as JSON."""
+    name = comparison.lockfile.name
+    if as_json:
+        typer.echo(comparison.model_dump_json(indent=2))
+    else:
+        for source in comparison.sources:
+            for asset in source.added:
+                typer.echo(f"{source.source}\tadded\t{asset.id}\t{asset.href}")
+            for asset in source.removed:
+                typer.echo(f"{source.source}\tremoved\t{asset.id}\t{asset.href}")
+            for change in source.changed:
+                sizes = f"{_size(change.locked_size)} -> {_size(change.listed_size)} bytes"
+                typer.echo(f"{source.source}\tchanged\t{change.asset_id}\t{sizes}")
+    differing = [source.source for source in comparison.sources if not source.matches]
+    if differing:
+        typer.secho(
+            f"{len(differing)} source(s) list something other than {name} pins: "
+            f"{', '.join(differing)}. Pass pull --force to re-resolve.",
+            err=True,
+            fg="red",
+        )
+    else:
+        typer.echo(f"every source lists what {name} pins", err=True)
+
+
+def _size(value: int | None) -> str:
+    return "?" if value is None else str(value)
+
+
 app.command()(doctor)

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/data/registry.yaml

@@ -1556,20 +1556,55 @@ datasets:
   # ---------------------------------------------------------------- Census
   - id: census:acs-5year
     provider: census
-    status: planned
+    status: available
     domain: demographics
-    target: later
+    since: "0.31"
     title: American Community Survey 5-Year Estimates
     description: >-
-      Population, housing, income, and demographic estimates for every
-      geography down to block group, via the Census Data API, selected by state
-      and county FIPS code rather than by box. Requires a free API key on every
-      data request; only the dataset and variable metadata are served without one.
-    keywords: [population, demographics, housing, income, acs, census]
+      Population, housing, income, and demographic estimates from the American
+      Community Survey's detailed tables, through the Census Data API: one row
+      per geography, for one to fifty API variables, including margins of error
+      and annotations. A query names a vintage, the last year of the five-year
+      period, rather than a window, and a state or county rather than a box;
+      a state can return its own row or one row per county. The delivered
+      variable set is open, tens of thousands of variables per vintage.
+      Requires a free API key on every data request; the response is stored as
+      served, since it carries no copy of the key.
+    keywords: [population, demographics, housing, income, acs, census, american community survey]
     protocol: http
     homepage: https://www.census.gov/data/developers/data-sets/acs-5year.html
     license: US Government Work (public domain)
-    capabilities: { spatial_subset: false, place_subset: true, temporal_subset: true, variable_subset: true }
+    capabilities: { spatial_subset: false, place_subset: true, temporal_subset: false, variable_subset: true }
+    credentials:
+      variables: [USDATA_CENSUS_KEY]
+      signup: https://api.census.gov/data/key_signup.html
+    summary: Five-year population, housing, and income estimates by state or county
+    formats: [JSON]
+    selection: One vintage's estimates for up to fifty variables over a state, its counties, or one county
+    inputs: A vintage; one to fifty variables; a state or county location
+    reader: pandas
+    guide: docs/providers/census-acs-5year.md
+    examples:
+      - examples/datasets/census-acs-5year/census-acs-5year.ipynb
+    resolution:
+      spatial: Detailed tables are available down to the block group; usdata selects states and counties
+      temporal: Five-year period estimates, one vintage per year
+    update_frequency: >-
+      Released once a year; the 2020-2024 estimates were released on 8 January 2026
+    citation: >-
+      U.S. Census Bureau, American Community Survey 5-Year Estimates, Detailed Tables, Census Data
+      API, accessed via usdata
+    terms: https://www.census.gov/data/developers/about/terms-of-service.html
+    variables:
+      - { name: "NAME", description: "Geographic area name, such as Osage County, Oklahoma" }
+      - { name: "B01003_001E", description: "Total population, estimate" }
+      - { name: "B01003_001M", description: "Total population, margin of error" }
+      - { name: "B25001_001E", description: "Housing units, estimate" }
+      - { name: "B25001_001M", description: "Housing units, margin of error" }
+      - { name: "B19013_001E", units: "US dollars of the vintage year", description: "Median household income in the past 12 months, inflation-adjusted to the vintage year, estimate" }
+      - { name: "state", description: "FIPS code of the state, appended by the service" }
+      - { name: "county", description: "FIPS code of the county within the state, appended by the service for county rows" }
+    adapter: usdata.providers.census.acs:Acs5Year
 
   # ---------------------------------------------------------------- USDA
   - id: usda:cropland-data-layer

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/doctor.py

@@ -28,7 +28,7 @@ import httpx
 from pydantic import BaseModel
 
 from usdata import __version__
-from usdata._grib import LIBRARY_HINT
+from usdata._grib import LIBRARY_HINT, import_eccodes
 from usdata.cache import ENV_VAR, cache_dir
 from usdata.mirror import ENV_VAR as MIRROR_ENV_VAR
 from usdata.models import READER_EXTRAS, Status
@@ -145,7 +145,8 @@ def _reader_check(extra: str) -> Check:
     modules: list[Any] = []
     for module in READER_MODULES[extra]:
         try:
-            modules.append(import_module(module))
+            # ecCodes goes through the reader's import, which orders it after pyproj.
+            modules.append(import_eccodes() if module == "eccodes" else import_module(module))
         except (ImportError, RuntimeError, OSError) as error:
             # eccodes imports and then fails to load its shared library.
             return Check(

usdata-0.32.0/src/usdata/providers/census/__init__.py

@@ -0,0 +1 @@
+"""Census Bureau dataset adapters."""

usdata-0.32.0/src/usdata/providers/census/acs.py

@@ -0,0 +1,243 @@
+"""American Community Survey 5-year estimates from the Census Data API, which needs a free key.
+
+The service answers one query per request with a JSON array of arrays: a header
+row naming the columns, then one row per geography, every value a string. The
+requested variables come first and the geography columns (``state``,
+``county``) follow. Every data request carries a ``key`` query parameter, which
+usdata reads from ``USDATA_CENSUS_KEY`` and adds only as each request is sent
+(ADR 0039).
+
+Selection (ADR 0045):
+
+- ``vintage`` names the estimates by the last year of their five-year period, so
+  2023 is 2019 through 2023. A time window is refused: a five-year estimate
+  describes a period, and choosing one from a window would be a guess.
+- The query's ``variables`` name one to fifty API variables, ``NAME``
+  included, the service's limit per request. Margins of error and annotations
+  are variables too, such as ``B01003_001M`` and ``B01003_001EA``.
+- A ``location`` names one state or county. ``geography`` says what each row
+  is: the place itself by default, or, for a state, every county in it. A box
+  names no place and is refused.
+- Connecticut's codes depend on the vintage: its eight counties through 2021,
+  its nine planning regions from 2022. A place the vintage does not have is
+  refused with the ones that it does.
+
+The response is written as served. It carries no copy of the key and identical
+requests return identical bytes, so there is nothing to canonicalize. The
+service answers a geography it does not have with an empty ``204`` and a bad
+key with a redirect; both are refused rather than written.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import UTC, datetime, time
+from pathlib import Path
+from typing import Annotated
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field
+
+from usdata.models import Asset, Place, Protocol, Query, TimeRange
+from usdata.protocols import http
+from usdata.providers.base import QueryError
+from usdata.providers.http import HttpProvider
+from usdata.providers.params import choice, int_range
+from usdata.query import legacy_counties, planning_regions
+
+SERVICE_URL = "https://api.census.gov/data"
+KEY = "USDATA_CENSUS_KEY"
+FIRST_VINTAGE = 2009
+"""The first ACS 5-year vintage the API lists, covering 2005 through 2009."""
+
+REGIONS_FROM = 2022
+"""The first vintage keyed by Connecticut's planning regions rather than its old counties."""
+
+MAX_VARIABLES = 50
+"""The most variables, ``NAME`` included, the service returns for one request."""
+
+_VARIABLE = re.compile(r"[A-Z][A-Z0-9_]*")
+
+
+class CensusError(QueryError):
+    """The Census Data API refused the request and said why, or refused the key."""
+
+
+class AcsParams(BaseModel):
+    """Which estimates one ACS query names, and what each of its rows is."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    vintage: Annotated[int, int_range(FIRST_VINTAGE, 9999)] = Field(
+        description=(
+            "Required last year of the five-year period, such as 2023 for 2019-2023; "
+            "the estimates are selected by vintage, not by a date window."
+        )
+    )
+    geography: Annotated[str, choice("state", "county")] | None = Field(
+        default=None,
+        description=(
+            "What each row is: state or county. Defaults to the kind of place the location "
+            "names; county with a state location gives every county in it."
+        ),
+    )
+
+
+def _variables(names: list[str], dataset_id: str) -> list[str]:
+    """The query's variables upper-cased, repeats dropped, in the order given."""
+    ordered = list(dict.fromkeys(name.strip().upper() for name in names))
+    if not ordered or not all(ordered):
+        raise QueryError(
+            f"{dataset_id} needs variables naming the estimates to return, such as "
+            "NAME,B01003_001E,B01003_001M"
+        )
+    if bad := [name for name in ordered if not _VARIABLE.fullmatch(name)]:
+        raise QueryError(f"{dataset_id} variable {bad[0]!r} is not a Census API variable name")
+    if len(ordered) > MAX_VARIABLES:
+        raise QueryError(
+            f"{dataset_id} was given {len(ordered)} variables; the service returns at most "
+            f"{MAX_VARIABLES} per request, NAME included, so split them across manifest sources"
+        )
+    return ordered
+
+
+def _labels(places: tuple[Place, ...]) -> str:
+    return "; ".join(place.label for place in places)
+
+
+class Acs5Year(HttpProvider):
+    """ACS 5-year detailed-table estimates for one vintage, variable list, and place."""
+
+    params_model = AcsParams
+
+    def list_assets(self, query: Query) -> list[Asset]:
+        """One JSON asset for the query's vintage, variables, and place; nothing is requested."""
+        params = self.parse_params(query, AcsParams)
+        self.reject(
+            query,
+            "text",
+            "time",
+            hint="choose the five-year period with vintage, such as vintage=2023 for 2019-2023",
+        )
+        variables = _variables(query.variables, self.dataset.id)
+        place = self.place_of(query, hint="the estimates are keyed by state and county")
+        if place is None:
+            raise QueryError(f"{self.dataset.id} needs a location naming a state or county")
+        self._check_connecticut(place, params.vintage)
+        geography = params.geography or place.kind
+        if geography == "state" and place.kind == "county":
+            raise QueryError(
+                f"{self.dataset.id} geography=state needs a state location, not {place.label}"
+            )
+        selection: dict[str, str]
+        if place.kind == "state" and geography == "state":
+            selection, label = {"for": f"state:{place.state_fips}"}, place.geoid
+        elif place.kind == "state":
+            selection = {"for": "county:*", "in": f"state:{place.state_fips}"}
+            label = f"{place.geoid}-counties"
+        else:
+            selection = {"for": f"county:{place.county_fips}", "in": f"state:{place.state_fips}"}
+            label = place.geoid
+        url = httpx.URL(
+            f"{SERVICE_URL}/{params.vintage}/acs/acs5",
+            params={"get": ",".join(variables), **selection},
+        )
+        digest = hashlib.sha256(",".join(variables).encode()).hexdigest()[:12]
+        first = params.vintage - 4
+        return [
+            Asset(
+                id=f"acs5_{params.vintage}_{label}_{digest}.json",
+                dataset_id=self.dataset.id,
+                href=str(url),
+                protocol=Protocol.HTTP,
+                media_type="application/json",
+                time=TimeRange(
+                    start=datetime(first, 1, 1, tzinfo=UTC),
+                    end=datetime.combine(datetime(params.vintage, 12, 31), time.max, UTC),
+                ),
+                bbox=query.bbox,
+                properties={"vintage": str(params.vintage), "period": f"{first}-{params.vintage}"},
+            )
+        ]
+
+    def _check_connecticut(self, place: Place, vintage: int) -> None:
+        """Refuse a Connecticut place the vintage's geography does not have (ADR 0045)."""
+        if vintage < REGIONS_FROM and (counties := legacy_counties(place)):
+            raise QueryError(
+                f"{self.dataset.id} vintage {vintage} keys Connecticut by its eight counties "
+                f"before 2022, and {place.label} is a planning region that replaced them; name "
+                f"a county it overlaps with location: {_labels(counties)}"
+            )
+        if vintage >= REGIONS_FROM and (regions := planning_regions(place)):
+            raise QueryError(
+                f"{self.dataset.id} vintage {vintage} keys Connecticut by the planning regions "
+                f"that replaced its counties in 2022, and {place.label} is one of those "
+                f"counties; name a region it overlaps with location: {_labels(regions)}"
+            )
+
+    def fetch(self, asset: Asset, dest: Path) -> Path:
+        """Request the estimates with this adapter's key and write the response as served."""
+        vintage = asset.properties.get("vintage", "?")
+        with self.redacted_errors():
+            keyed = httpx.URL(asset.href).copy_merge_params({"key": self.credentials[KEY]})
+            try:
+                # A bad key is a redirect to an explanation page, which is refused, not followed.
+                response = http.get(keyed, self._http(), follow_redirects=False)
+            except httpx.HTTPStatusError as error:
+                # Any other status, such as a 503, stays an upstream failure.
+                if (refusal := _refusal(error.response, vintage)) is None:
+                    raise
+                raise refusal from error
+            if response.status_code == 204:
+                raise CensusError(
+                    f"the Census Data API has no rows for {_where(asset.href)} in ACS 5-year "
+                    f"vintage {vintage}; the place's code may differ in that vintage"
+                )
+            _check_table(response)
+        dest.write_bytes(response.content)
+        return dest
+
+
+def _refusal(response: httpx.Response, vintage: str) -> CensusError | None:
+    """The service's refusal as an error that says why, or None for an upstream failure."""
+    status = response.status_code
+    location = response.headers.get("location", "")
+    if response.is_redirect and "invalid_key" in location:
+        return CensusError(
+            f"the Census Data API refused the key in {KEY} as invalid; check it, and that the "
+            "activation link in the signup email was followed"
+        )
+    if response.is_redirect and "missing_key" in location:
+        return CensusError(f"the Census Data API answered as if no key was sent; check {KEY}")
+    if status == 400 and response.headers.get("content-type", "").startswith("text/plain"):
+        reason = response.text.strip().removeprefix("error:").strip()
+        return CensusError(f"the Census Data API refused the request: {reason}")
+    if status == 404:
+        return CensusError(f"the Census Data API publishes no ACS 5-year vintage {vintage}")
+    return None
+
+
+def _where(href: str) -> str:
+    """The geography a request names, as the service's ``for`` and ``in`` spell it."""
+    params = httpx.URL(href).params
+    place = params.get("for", "")
+    return f"{place} in {params['in']}" if "in" in params else place
+
+
+def _check_table(response: httpx.Response) -> None:
+    """Raise unless the body is the array of rows, header first, that the service promises."""
+    try:
+        body = json.loads(response.content)
+    except ValueError:
+        body = None
+    if not (
+        isinstance(body, list)
+        and body
+        and all(isinstance(row, list) and len(row) == len(body[0]) for row in body)
+    ):
+        raise httpx.DecodingError(
+            "the Census Data API answered with something other than a table of rows",
+            request=response.request,
+        )

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/pull.py

@@ -157,6 +157,22 @@ class PullResult(BaseModel):
         return items[0]
 
 
+def _source_keys(entries: Iterable[LockedAsset]) -> list[str]:
+    """The manifest source key each lockfile entry belongs to, in entry order.
+
+    A lockfile written before sources had keys records none; such an entry falls
+    back to the one-based position of its dataset id among the datasets seen.
+    """
+    positions: dict[str, str] = {}
+    keys: list[str] = []
+    for entry in entries:
+        dataset_id = entry.asset.dataset_id
+        if dataset_id not in positions:
+            positions[dataset_id] = str(len(positions) + 1)
+        keys.append(entry.source or positions[dataset_id])
+    return keys
+
+
 def _by_source(pairs: Iterable[tuple[LockedAsset, FetchedAsset]]) -> dict[str, list[FetchedAsset]]:
     """Group fetched assets by the source key their lockfile entry records.
 
@@ -169,12 +185,9 @@ def _by_source(pairs: Iterable[tuple[LockedAsset, FetchedAsset]]) -> dict[str, l
     them and a forced pull rewrites the lockfile.
     """
     grouped: dict[str, list[FetchedAsset]] = {}
-    positions: dict[str, str] = {}
-    for entry, item in pairs:
-        dataset_id = entry.asset.dataset_id
-        if dataset_id not in positions:
-            positions[dataset_id] = str(len(positions) + 1)
-        grouped.setdefault(entry.source or positions[dataset_id], []).append(item)
+    pairs = list(pairs)
+    for key, (_, item) in zip(_source_keys([entry for entry, _ in pairs]), pairs, strict=True):
+        grouped.setdefault(key, []).append(item)
     by_id: dict[str, FetchedAsset] = {}
     for items in grouped.values():
         by_id.update({item.asset.id: item for item in items})
@@ -381,6 +394,123 @@ def plan(manifest_path: str | os.PathLike[str], *, registry: Registry | None = N
     return Plan(manifest=manifest.name, sources=sources)
 
 
+class AssetChange(BaseModel):
+    """A pinned asset that today's listing still names, but describes differently."""
+
+    asset_id: str
+    locked_size: int | None = Field(description="Size the listing gave when the pin was written")
+    listed_size: int | None = Field(description="Size today's listing gives")
+    locked_href: str
+    listed_href: str
+
+
+class SourceComparison(BaseModel):
+    """How one source's pinned assets compare with what its adapter lists today."""
+
+    source: str = Field(description="The source's manifest key: its name, or its position")
+    dataset_id: str
+    added: list[Asset] = Field(description="Listed today and not pinned")
+    removed: list[Asset] = Field(description="Pinned and no longer listed")
+    changed: list[AssetChange] = Field(
+        description="Pinned and listed, with a different href, or a different size where both "
+        "the pin and the listing give one"
+    )
+    unchanged: int = Field(description="How many pinned assets are listed exactly as pinned")
+
+    @computed_field(description="Whether a re-resolve of this source would pin the same assets")
+    @property
+    def matches(self) -> bool:
+        """True when nothing was added, removed, or changed."""
+        return not (self.added or self.removed or self.changed)
+
+
+class ListingComparison(BaseModel):
+    """A lockfile against today's listings: whether a re-resolve would pin the same assets.
+
+    Nothing is downloaded, so the bytes are not compared: restore checks those,
+    and ``verify`` checks the cache. What this sees is the set of files, an
+    asset added, removed, or re-described since the lockfile was written.
+    """
+
+    manifest: str
+    lockfile: Path
+    sources: list[SourceComparison] = Field(description="One entry per manifest source, in order")
+
+    @computed_field(description="Whether a re-resolve of every source would pin the same assets")
+    @property
+    def matches(self) -> bool:
+        """True when every source matches."""
+        return all(source.matches for source in self.sources)
+
+
+def _changed(pinned: Asset, listed: Asset) -> AssetChange | None:
+    """How ``listed`` differs from the pin of the same id, or None when it does not."""
+    sizes_differ = None not in (pinned.size, listed.size) and pinned.size != listed.size
+    if not sizes_differ and pinned.href == listed.href:
+        return None
+    return AssetChange(
+        asset_id=pinned.id,
+        locked_size=pinned.size,
+        listed_size=listed.size,
+        locked_href=pinned.href,
+        listed_href=listed.href,
+    )
+
+
+def compare_listing(
+    manifest_path: str | os.PathLike[str], *, registry: Registry | None = None
+) -> ListingComparison:
+    """Compare a manifest's lockfile with what its sources list today; download nothing.
+
+    Every source is listed as ``plan`` lists it, ``select`` rules included, and
+    its assets are matched to the lockfile's entries for that source by id. An
+    asset only in the listing is ``added``, one only in the lockfile ``removed``,
+    and one in both with a different href, or a different size where both give
+    one, ``changed``. A listing that reports no size cannot show a size change;
+    restoring the pin checks the bytes themselves.
+
+    Args:
+        manifest_path: Path of the manifest YAML file, as a string or any ``os.PathLike``.
+        registry: Registry to resolve dataset ids against; the default one if omitted.
+
+    Returns:
+        A ``ListingComparison`` with one entry per manifest source.
+
+    Raises:
+        FileNotFoundError: The manifest has no lockfile yet.
+        ManifestChanged: The manifest changed since its lockfile was written.
+    """
+    manifest_path = Path(manifest_path)
+    lock_path = lockfile_path(manifest_path)
+    if not lock_path.exists():
+        raise FileNotFoundError(f"no lockfile at {lock_path}; run pull first")
+    lock = Lockfile.load(lock_path)
+    _check_manifest(manifest_path, lock)
+    pinned: dict[str, dict[str, Asset]] = {}
+    for key, entry in zip(_source_keys(lock.assets), lock.assets, strict=True):
+        pinned.setdefault(key, {})[entry.asset.id] = entry.asset
+    sources: list[SourceComparison] = []
+    for listed in plan(manifest_path, registry=registry).sources:
+        locked = pinned.get(listed.source, {})
+        today = {asset.id: asset for asset in listed.assets}
+        changes = [
+            change
+            for asset_id, asset in locked.items()
+            if asset_id in today and (change := _changed(asset, today[asset_id])) is not None
+        ]
+        sources.append(
+            SourceComparison(
+                source=listed.source,
+                dataset_id=listed.dataset_id,
+                added=[asset for asset in listed.assets if asset.id not in locked],
+                removed=[asset for asset_id, asset in locked.items() if asset_id not in today],
+                changed=changes,
+                unchanged=sum(1 for asset_id in locked if asset_id in today) - len(changes),
+            )
+        )
+    return ListingComparison(manifest=lock.manifest, lockfile=lock_path, sources=sources)
+
+
 def _selected(lock: Lockfile, update: Iterable[str]) -> set[str]:
     """Ids of the locked assets that ``update`` selectors (asset or dataset ids) name."""
     selectors = set(update)

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/query.py

@@ -89,6 +89,24 @@ def legacy_counties(place: Place) -> tuple[Place, ...]:
     return tuple(table.places[geoid][0] for geoid in table.legacy.get(place.geoid, ()))
 
 
+def planning_regions(place: Place) -> tuple[Place, ...]:
+    """The Connecticut planning regions a pre-2022 county overlaps, or () for any other place.
+
+    The reverse of ``legacy_counties``, for a source that switched to the
+    regions in 2022, such as Census geography from that vintage on, and so has
+    no rows under an old county's code.
+
+    Args:
+        place: Any place from ``find_place`` or ``build_query``.
+
+    Returns:
+        The planning regions, in FIPS order, when ``place`` is a legacy county.
+    """
+    table = _places()
+    regions = sorted(region for region, counties in table.legacy.items() if place.geoid in counties)
+    return tuple(table.places[geoid][0] for geoid in regions)
+
+
 def resolve_place(name: str) -> BBox:
     """Resolve a state, qualified county name, or quoted two/five-digit FIPS code to its box."""
     return find_place(name)[1]

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/readers.py

@@ -47,6 +47,12 @@ AQS_DATASET = "epa:aqs-daily"
 AQS_PREFIX = "aqs-daily_"
 """How an ``epa:aqs-daily`` asset id begins, so a copied file is still recognized."""
 
+ACS_DATASET = "census:acs-5year"
+"""The dataset whose Census Data API tables the ``acs`` reader opens."""
+
+ACS_PREFIX = "acs5_"
+"""How a ``census:acs-5year`` asset id begins, so a copied file is still recognized."""
+
 STORM_EVENTS_DATASET = "noaa:storm-events"
 """The dataset whose local timestamps the CSV reader pairs with UTC columns."""
 
@@ -388,12 +394,13 @@ def open_asset(fetched: FetchedAsset) -> Any:
 
     The reader is inferred from the asset's dataset, media type, and id:
     ``noaa:nexrad-level2`` volumes open as with ``open_nexrad``, HURDAT2 best
-    tracks and AQS daily-summary JSON by their dataset or filename, NetCDF4 as
+    tracks, AQS daily-summary JSON, and ACS tables by their dataset or filename, NetCDF4 as
     with ``open_netcdf``, GRIB2 as with ``open_grib2``, and CSV as with
     ``open_csv``, with its units row detected. Each takes its defaults; a file
     that needs options is opened with the function for its format instead.
-    HURDAT2 best-track text returns one row per track point, and AQS daily-summary
-    JSON one row per monitor, day, and standard; neither takes options.
+    HURDAT2 best-track text returns one row per track point, AQS daily-summary
+    JSON one row per monitor, day, and standard, and an ACS table one row per
+    geography; none takes options.
 
     Raises:
         UnsupportedFormat: Nothing about the asset names a format usdata reads.
@@ -411,6 +418,10 @@ def open_asset(fetched: FetchedAsset) -> Any:
         from usdata._aqs import open_aqs
 
         return open_aqs(fetched)
+    if dataset_id == ACS_DATASET or (name.startswith(ACS_PREFIX) and name.endswith(".json")):
+        from usdata._census import open_acs
+
+        return open_acs(fetched)
     if media_type in NETCDF_MEDIA_TYPES:
         return open_netcdf(fetched)
     if media_type in GRIB2_MEDIA_TYPES or (
@@ -424,7 +435,7 @@ def open_asset(fetched: FetchedAsset) -> Any:
     raise UnsupportedFormat(
         f"no reader for {fetched.asset.media_type!r}; supported formats are CSV, "
         "ERDDAP CSV, NetCDF4, GRIB2, NEXRAD Level II, HURDAT2 best tracks, "
-        "and AQS daily JSON. For a CSV with ambiguous metadata, use open_csv; "
+        "AQS daily JSON, and ACS tables. For a CSV with ambiguous metadata, use open_csv; "
         "otherwise use fetched.path with a format-specific reader"
     )
 

{usdata-0.30.0 → usdata-0.32.0}/src/usdata/testing.py

@@ -335,7 +335,8 @@ def check_declared_capabilities(
     variable = dataset.variables[0].name
     with _patched_client(unexpected_client), adapter_factory() as adapter:
         base = scenario_query
-        boxed = base.model_copy(update={"bbox": PROBE_BBOX})
+        # A scenario may name a place of its own; the bare-box probe must not keep it.
+        boxed = base.model_copy(update={"bbox": PROBE_BBOX, "place": None})
         placed = boxed.model_copy(update={"place": PROBE_PLACE})
         bare_box_refused = _refuses_bare_box(adapter, boxed)
         bbox_refused = bare_box_refused or _refuses(adapter, boxed, "bbox")