usdata 0.30.0__tar.gz → 0.31.0__tar.gz

{usdata-0.30.0 → usdata-0.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.30.0
+Version: 0.31.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -113,7 +113,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-seven datasets are available today and nineteen more are planned, grouped
+Twenty-eight datasets are available today and eighteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.30.0 → usdata-0.31.0}/README.md

@@ -68,7 +68,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/studies/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-seven datasets are available today and nineteen more are planned, grouped
+Twenty-eight datasets are available today and eighteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.30.0 → usdata-0.31.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.30.0"
+version = "0.31.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.30.0 → usdata-0.31.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.30.0"
+version = "0.31.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

usdata-0.31.0/src/usdata/_census.py

@@ -0,0 +1,51 @@
+"""Local reading of Census Data API tables into a pandas DataFrame, behind pandas.
+
+A fetched ``census:acs-5year`` file is the service's response as served: a JSON
+array whose first row names the columns and whose other rows are one geography
+each, every value a string. The table is those rows under those names, with the
+estimate and margin-of-error columns as numbers and everything else, geography
+codes and their leading zeros included, as text.
+
+Negative annotation codes such as ``-555555555`` stay the numbers they are.
+Each means something different (too few sample cases, not applicable, a
+controlled estimate), so they are not turned into missing values (ADR 0045).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from usdata.readers import MissingReaderDependency, source_attrs
+
+if TYPE_CHECKING:
+    from usdata._fetch import FetchedAsset
+
+NUMERIC = re.compile(r"[A-Z]\d{5}[A-Z0-9]*_\d{3}[EM]")
+"""An estimate (``...E``) or margin of error (``...M``) variable, such as ``B01003_001E``.
+
+Annotation columns (``...EA``, ``...MA``) and ``NAME`` do not match and stay text.
+"""
+
+
+def open_acs(fetched: FetchedAsset) -> Any:
+    """Parse a fetched ACS table into a pandas DataFrame, one row per geography."""
+    try:
+        pandas = import_module("pandas")
+    except ModuleNotFoundError as error:
+        if error.name != "pandas":
+            raise
+        raise MissingReaderDependency(
+            'ACS reading requires pandas; install it with: pip install "usdata[pandas]" '
+            '(or uv add "usdata[pandas]")'
+        ) from error
+    # Reading a fetched asset is strictly local; the cached file is never rewritten.
+    header, *rows = json.loads(fetched.path.read_text(encoding="utf-8"))
+    frame = pandas.DataFrame(rows, columns=header, dtype="string")
+    for column in header:
+        if NUMERIC.fullmatch(column):
+            frame[column] = pandas.to_numeric(frame[column])
+    frame.attrs["usdata"] = source_attrs(fetched)
+    return frame

{usdata-0.30.0 → usdata-0.31.0}/src/usdata/_fetch.py

@@ -39,7 +39,7 @@ class FetchedAsset(BaseModel):
     def open(self) -> Any:
         """Open local data with the reader its format implies, with that reader's defaults.
 
-        CSV and ERDDAP CSV, HURDAT2 and AQS daily JSON return a pandas DataFrame;
+        CSV and ERDDAP CSV, HURDAT2, AQS daily JSON, and ACS tables return a pandas DataFrame;
         NetCDF4 and GRIB2 a loaded xarray Dataset; NEXRAD Level II an xarray
         DataTree. Provenance is kept in the result's ``attrs["usdata"]``. A file
         that needs options, or whose metadata leaves its format ambiguous, is

{usdata-0.30.0 → usdata-0.31.0}/src/usdata/data/registry.yaml

@@ -1556,20 +1556,55 @@ datasets:
   # ---------------------------------------------------------------- Census
   - id: census:acs-5year
     provider: census
-    status: planned
+    status: available
     domain: demographics
-    target: later
+    since: "0.31"
     title: American Community Survey 5-Year Estimates
     description: >-
-      Population, housing, income, and demographic estimates for every
-      geography down to block group, via the Census Data API, selected by state
-      and county FIPS code rather than by box. Requires a free API key on every
-      data request; only the dataset and variable metadata are served without one.
-    keywords: [population, demographics, housing, income, acs, census]
+      Population, housing, income, and demographic estimates from the American
+      Community Survey's detailed tables, through the Census Data API: one row
+      per geography, for one to fifty API variables, including margins of error
+      and annotations. A query names a vintage, the last year of the five-year
+      period, rather than a window, and a state or county rather than a box;
+      a state can return its own row or one row per county. The delivered
+      variable set is open, tens of thousands of variables per vintage.
+      Requires a free API key on every data request; the response is stored as
+      served, since it carries no copy of the key.
+    keywords: [population, demographics, housing, income, acs, census, american community survey]
     protocol: http
     homepage: https://www.census.gov/data/developers/data-sets/acs-5year.html
     license: US Government Work (public domain)
-    capabilities: { spatial_subset: false, place_subset: true, temporal_subset: true, variable_subset: true }
+    capabilities: { spatial_subset: false, place_subset: true, temporal_subset: false, variable_subset: true }
+    credentials:
+      variables: [USDATA_CENSUS_KEY]
+      signup: https://api.census.gov/data/key_signup.html
+    summary: Five-year population, housing, and income estimates by state or county
+    formats: [JSON]
+    selection: One vintage's estimates for up to fifty variables over a state, its counties, or one county
+    inputs: A vintage; one to fifty variables; a state or county location
+    reader: pandas
+    guide: docs/providers/census-acs-5year.md
+    examples:
+      - examples/datasets/census-acs-5year/census-acs-5year.ipynb
+    resolution:
+      spatial: Detailed tables are available down to the block group; usdata selects states and counties
+      temporal: Five-year period estimates, one vintage per year
+    update_frequency: >-
+      Released once a year; the 2020-2024 estimates were released on 8 January 2026
+    citation: >-
+      U.S. Census Bureau, American Community Survey 5-Year Estimates, Detailed Tables, Census Data
+      API, accessed via usdata
+    terms: https://www.census.gov/data/developers/about/terms-of-service.html
+    variables:
+      - { name: "NAME", description: "Geographic area name, such as Osage County, Oklahoma" }
+      - { name: "B01003_001E", description: "Total population, estimate" }
+      - { name: "B01003_001M", description: "Total population, margin of error" }
+      - { name: "B25001_001E", description: "Housing units, estimate" }
+      - { name: "B25001_001M", description: "Housing units, margin of error" }
+      - { name: "B19013_001E", units: "US dollars of the vintage year", description: "Median household income in the past 12 months, inflation-adjusted to the vintage year, estimate" }
+      - { name: "state", description: "FIPS code of the state, appended by the service" }
+      - { name: "county", description: "FIPS code of the county within the state, appended by the service for county rows" }
+    adapter: usdata.providers.census.acs:Acs5Year
 
   # ---------------------------------------------------------------- USDA
   - id: usda:cropland-data-layer

usdata-0.31.0/src/usdata/providers/census/__init__.py

@@ -0,0 +1 @@
+"""Census Bureau dataset adapters."""

usdata-0.31.0/src/usdata/providers/census/acs.py

@@ -0,0 +1,243 @@
+"""American Community Survey 5-year estimates from the Census Data API, which needs a free key.
+
+The service answers one query per request with a JSON array of arrays: a header
+row naming the columns, then one row per geography, every value a string. The
+requested variables come first and the geography columns (``state``,
+``county``) follow. Every data request carries a ``key`` query parameter, which
+usdata reads from ``USDATA_CENSUS_KEY`` and adds only as each request is sent
+(ADR 0039).
+
+Selection (ADR 0045):
+
+- ``vintage`` names the estimates by the last year of their five-year period, so
+  2023 is 2019 through 2023. A time window is refused: a five-year estimate
+  describes a period, and choosing one from a window would be a guess.
+- The query's ``variables`` name one to fifty API variables, ``NAME``
+  included, the service's limit per request. Margins of error and annotations
+  are variables too, such as ``B01003_001M`` and ``B01003_001EA``.
+- A ``location`` names one state or county. ``geography`` says what each row
+  is: the place itself by default, or, for a state, every county in it. A box
+  names no place and is refused.
+- Connecticut's codes depend on the vintage: its eight counties through 2021,
+  its nine planning regions from 2022. A place the vintage does not have is
+  refused with the ones that it does.
+
+The response is written as served. It carries no copy of the key and identical
+requests return identical bytes, so there is nothing to canonicalize. The
+service answers a geography it does not have with an empty ``204`` and a bad
+key with a redirect; both are refused rather than written.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import UTC, datetime, time
+from pathlib import Path
+from typing import Annotated
+
+import httpx
+from pydantic import BaseModel, ConfigDict, Field
+
+from usdata.models import Asset, Place, Protocol, Query, TimeRange
+from usdata.protocols import http
+from usdata.providers.base import QueryError
+from usdata.providers.http import HttpProvider
+from usdata.providers.params import choice, int_range
+from usdata.query import legacy_counties, planning_regions
+
+SERVICE_URL = "https://api.census.gov/data"
+KEY = "USDATA_CENSUS_KEY"
+FIRST_VINTAGE = 2009
+"""The first ACS 5-year vintage the API lists, covering 2005 through 2009."""
+
+REGIONS_FROM = 2022
+"""The first vintage keyed by Connecticut's planning regions rather than its old counties."""
+
+MAX_VARIABLES = 50
+"""The most variables, ``NAME`` included, the service returns for one request."""
+
+_VARIABLE = re.compile(r"[A-Z][A-Z0-9_]*")
+
+
+class CensusError(QueryError):
+    """The Census Data API refused the request and said why, or refused the key."""
+
+
+class AcsParams(BaseModel):
+    """Which estimates one ACS query names, and what each of its rows is."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    vintage: Annotated[int, int_range(FIRST_VINTAGE, 9999)] = Field(
+        description=(
+            "Required last year of the five-year period, such as 2023 for 2019-2023; "
+            "the estimates are selected by vintage, not by a date window."
+        )
+    )
+    geography: Annotated[str, choice("state", "county")] | None = Field(
+        default=None,
+        description=(
+            "What each row is: state or county. Defaults to the kind of place the location "
+            "names; county with a state location gives every county in it."
+        ),
+    )
+
+
+def _variables(names: list[str], dataset_id: str) -> list[str]:
+    """The query's variables upper-cased, repeats dropped, in the order given."""
+    ordered = list(dict.fromkeys(name.strip().upper() for name in names))
+    if not ordered or not all(ordered):
+        raise QueryError(
+            f"{dataset_id} needs variables naming the estimates to return, such as "
+            "NAME,B01003_001E,B01003_001M"
+        )
+    if bad := [name for name in ordered if not _VARIABLE.fullmatch(name)]:
+        raise QueryError(f"{dataset_id} variable {bad[0]!r} is not a Census API variable name")
+    if len(ordered) > MAX_VARIABLES:
+        raise QueryError(
+            f"{dataset_id} was given {len(ordered)} variables; the service returns at most "
+            f"{MAX_VARIABLES} per request, NAME included, so split them across manifest sources"
+        )
+    return ordered
+
+
+def _labels(places: tuple[Place, ...]) -> str:
+    return "; ".join(place.label for place in places)
+
+
+class Acs5Year(HttpProvider):
+    """ACS 5-year detailed-table estimates for one vintage, variable list, and place."""
+
+    params_model = AcsParams
+
+    def list_assets(self, query: Query) -> list[Asset]:
+        """One JSON asset for the query's vintage, variables, and place; nothing is requested."""
+        params = self.parse_params(query, AcsParams)
+        self.reject(
+            query,
+            "text",
+            "time",
+            hint="choose the five-year period with vintage, such as vintage=2023 for 2019-2023",
+        )
+        variables = _variables(query.variables, self.dataset.id)
+        place = self.place_of(query, hint="the estimates are keyed by state and county")
+        if place is None:
+            raise QueryError(f"{self.dataset.id} needs a location naming a state or county")
+        self._check_connecticut(place, params.vintage)
+        geography = params.geography or place.kind
+        if geography == "state" and place.kind == "county":
+            raise QueryError(
+                f"{self.dataset.id} geography=state needs a state location, not {place.label}"
+            )
+        selection: dict[str, str]
+        if place.kind == "state" and geography == "state":
+            selection, label = {"for": f"state:{place.state_fips}"}, place.geoid
+        elif place.kind == "state":
+            selection = {"for": "county:*", "in": f"state:{place.state_fips}"}
+            label = f"{place.geoid}-counties"
+        else:
+            selection = {"for": f"county:{place.county_fips}", "in": f"state:{place.state_fips}"}
+            label = place.geoid
+        url = httpx.URL(
+            f"{SERVICE_URL}/{params.vintage}/acs/acs5",
+            params={"get": ",".join(variables), **selection},
+        )
+        digest = hashlib.sha256(",".join(variables).encode()).hexdigest()[:12]
+        first = params.vintage - 4
+        return [
+            Asset(
+                id=f"acs5_{params.vintage}_{label}_{digest}.json",
+                dataset_id=self.dataset.id,
+                href=str(url),
+                protocol=Protocol.HTTP,
+                media_type="application/json",
+                time=TimeRange(
+                    start=datetime(first, 1, 1, tzinfo=UTC),
+                    end=datetime.combine(datetime(params.vintage, 12, 31), time.max, UTC),
+                ),
+                bbox=query.bbox,
+                properties={"vintage": str(params.vintage), "period": f"{first}-{params.vintage}"},
+            )
+        ]
+
+    def _check_connecticut(self, place: Place, vintage: int) -> None:
+        """Refuse a Connecticut place the vintage's geography does not have (ADR 0045)."""
+        if vintage < REGIONS_FROM and (counties := legacy_counties(place)):
+            raise QueryError(
+                f"{self.dataset.id} vintage {vintage} keys Connecticut by its eight counties "
+                f"before 2022, and {place.label} is a planning region that replaced them; name "
+                f"a county it overlaps with location: {_labels(counties)}"
+            )
+        if vintage >= REGIONS_FROM and (regions := planning_regions(place)):
+            raise QueryError(
+                f"{self.dataset.id} vintage {vintage} keys Connecticut by the planning regions "
+                f"that replaced its counties in 2022, and {place.label} is one of those "
+                f"counties; name a region it overlaps with location: {_labels(regions)}"
+            )
+
+    def fetch(self, asset: Asset, dest: Path) -> Path:
+        """Request the estimates with this adapter's key and write the response as served."""
+        vintage = asset.properties.get("vintage", "?")
+        with self.redacted_errors():
+            keyed = httpx.URL(asset.href).copy_merge_params({"key": self.credentials[KEY]})
+            try:
+                # A bad key is a redirect to an explanation page, which is refused, not followed.
+                response = http.get(keyed, self._http(), follow_redirects=False)
+            except httpx.HTTPStatusError as error:
+                # Any other status, such as a 503, stays an upstream failure.
+                if (refusal := _refusal(error.response, vintage)) is None:
+                    raise
+                raise refusal from error
+            if response.status_code == 204:
+                raise CensusError(
+                    f"the Census Data API has no rows for {_where(asset.href)} in ACS 5-year "
+                    f"vintage {vintage}; the place's code may differ in that vintage"
+                )
+            _check_table(response)
+        dest.write_bytes(response.content)
+        return dest
+
+
+def _refusal(response: httpx.Response, vintage: str) -> CensusError | None:
+    """The service's refusal as an error that says why, or None for an upstream failure."""
+    status = response.status_code
+    location = response.headers.get("location", "")
+    if response.is_redirect and "invalid_key" in location:
+        return CensusError(
+            f"the Census Data API refused the key in {KEY} as invalid; check it, and that the "
+            "activation link in the signup email was followed"
+        )
+    if response.is_redirect and "missing_key" in location:
+        return CensusError(f"the Census Data API answered as if no key was sent; check {KEY}")
+    if status == 400 and response.headers.get("content-type", "").startswith("text/plain"):
+        reason = response.text.strip().removeprefix("error:").strip()
+        return CensusError(f"the Census Data API refused the request: {reason}")
+    if status == 404:
+        return CensusError(f"the Census Data API publishes no ACS 5-year vintage {vintage}")
+    return None
+
+
+def _where(href: str) -> str:
+    """The geography a request names, as the service's ``for`` and ``in`` spell it."""
+    params = httpx.URL(href).params
+    place = params.get("for", "")
+    return f"{place} in {params['in']}" if "in" in params else place
+
+
+def _check_table(response: httpx.Response) -> None:
+    """Raise unless the body is the array of rows, header first, that the service promises."""
+    try:
+        body = json.loads(response.content)
+    except ValueError:
+        body = None
+    if not (
+        isinstance(body, list)
+        and body
+        and all(isinstance(row, list) and len(row) == len(body[0]) for row in body)
+    ):
+        raise httpx.DecodingError(
+            "the Census Data API answered with something other than a table of rows",
+            request=response.request,
+        )

{usdata-0.30.0 → usdata-0.31.0}/src/usdata/query.py

@@ -89,6 +89,24 @@ def legacy_counties(place: Place) -> tuple[Place, ...]:
     return tuple(table.places[geoid][0] for geoid in table.legacy.get(place.geoid, ()))
 
 
+def planning_regions(place: Place) -> tuple[Place, ...]:
+    """The Connecticut planning regions a pre-2022 county overlaps, or () for any other place.
+
+    The reverse of ``legacy_counties``, for a source that switched to the
+    regions in 2022, such as Census geography from that vintage on, and so has
+    no rows under an old county's code.
+
+    Args:
+        place: Any place from ``find_place`` or ``build_query``.
+
+    Returns:
+        The planning regions, in FIPS order, when ``place`` is a legacy county.
+    """
+    table = _places()
+    regions = sorted(region for region, counties in table.legacy.items() if place.geoid in counties)
+    return tuple(table.places[geoid][0] for geoid in regions)
+
+
 def resolve_place(name: str) -> BBox:
     """Resolve a state, qualified county name, or quoted two/five-digit FIPS code to its box."""
     return find_place(name)[1]

{usdata-0.30.0 → usdata-0.31.0}/src/usdata/readers.py

@@ -47,6 +47,12 @@ AQS_DATASET = "epa:aqs-daily"
 AQS_PREFIX = "aqs-daily_"
 """How an ``epa:aqs-daily`` asset id begins, so a copied file is still recognized."""
 
+ACS_DATASET = "census:acs-5year"
+"""The dataset whose Census Data API tables the ``acs`` reader opens."""
+
+ACS_PREFIX = "acs5_"
+"""How a ``census:acs-5year`` asset id begins, so a copied file is still recognized."""
+
 STORM_EVENTS_DATASET = "noaa:storm-events"
 """The dataset whose local timestamps the CSV reader pairs with UTC columns."""
 
@@ -388,12 +394,13 @@ def open_asset(fetched: FetchedAsset) -> Any:
 
     The reader is inferred from the asset's dataset, media type, and id:
     ``noaa:nexrad-level2`` volumes open as with ``open_nexrad``, HURDAT2 best
-    tracks and AQS daily-summary JSON by their dataset or filename, NetCDF4 as
+    tracks, AQS daily-summary JSON, and ACS tables by their dataset or filename, NetCDF4 as
     with ``open_netcdf``, GRIB2 as with ``open_grib2``, and CSV as with
     ``open_csv``, with its units row detected. Each takes its defaults; a file
     that needs options is opened with the function for its format instead.
-    HURDAT2 best-track text returns one row per track point, and AQS daily-summary
-    JSON one row per monitor, day, and standard; neither takes options.
+    HURDAT2 best-track text returns one row per track point, AQS daily-summary
+    JSON one row per monitor, day, and standard, and an ACS table one row per
+    geography; none takes options.
 
     Raises:
         UnsupportedFormat: Nothing about the asset names a format usdata reads.
@@ -411,6 +418,10 @@ def open_asset(fetched: FetchedAsset) -> Any:
         from usdata._aqs import open_aqs
 
         return open_aqs(fetched)
+    if dataset_id == ACS_DATASET or (name.startswith(ACS_PREFIX) and name.endswith(".json")):
+        from usdata._census import open_acs
+
+        return open_acs(fetched)
     if media_type in NETCDF_MEDIA_TYPES:
         return open_netcdf(fetched)
     if media_type in GRIB2_MEDIA_TYPES or (
@@ -424,7 +435,7 @@ def open_asset(fetched: FetchedAsset) -> Any:
     raise UnsupportedFormat(
         f"no reader for {fetched.asset.media_type!r}; supported formats are CSV, "
         "ERDDAP CSV, NetCDF4, GRIB2, NEXRAD Level II, HURDAT2 best tracks, "
-        "and AQS daily JSON. For a CSV with ambiguous metadata, use open_csv; "
+        "AQS daily JSON, and ACS tables. For a CSV with ambiguous metadata, use open_csv; "
         "otherwise use fetched.path with a format-specific reader"
     )
 

{usdata-0.30.0 → usdata-0.31.0}/src/usdata/testing.py

@@ -335,7 +335,8 @@ def check_declared_capabilities(
     variable = dataset.variables[0].name
     with _patched_client(unexpected_client), adapter_factory() as adapter:
         base = scenario_query
-        boxed = base.model_copy(update={"bbox": PROBE_BBOX})
+        # A scenario may name a place of its own; the bare-box probe must not keep it.
+        boxed = base.model_copy(update={"bbox": PROBE_BBOX, "place": None})
         placed = boxed.model_copy(update={"place": PROBE_PLACE})
         bare_box_refused = _refuses_bare_box(adapter, boxed)
         bbox_refused = bare_box_refused or _refuses(adapter, boxed, "bbox")