usdata 0.25.0__tar.gz → 0.26.0__tar.gz

{usdata-0.25.0 → usdata-0.26.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.25.0
+Version: 0.26.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,open-data,scientific-data,provenance,reproducible-research,weather,climate,meteorology
 Author: Jake Van Slyke
@@ -112,7 +112,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-six datasets are available today and twenty more are planned, grouped
+Twenty-seven datasets are available today and nineteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.25.0 → usdata-0.26.0}/README.md

@@ -67,7 +67,7 @@ breaking changes.
 | [docs.usdata.dev](https://docs.usdata.dev/) | How to use it: [install](https://docs.usdata.dev/install/), [getting started](https://docs.usdata.dev/getting-started/), guides, dataset notes, and reference |
 | [Severe-weather case study](https://usdata.dev/examples/severe-weather-case-study/) | One tornado, six sources, one manifest and lockfile, ending in a citation |
 
-Twenty-six datasets are available today and twenty more are planned, grouped
+Twenty-seven datasets are available today and nineteen more are planned, grouped
 by agency and product family in the [catalog](docs/providers/README.md).
 
 ## How this compares

{usdata-0.25.0 → usdata-0.26.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.25.0"
+version = "0.26.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.25.0 → usdata-0.26.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.25.0"
+version = "0.26.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

usdata-0.26.0/src/usdata/_aqs.py

@@ -0,0 +1,51 @@
+"""Local reading of AQS daily-summary JSON into a tidy table, behind pandas.
+
+A fetched ``epa:aqs-daily`` file is the canonical form the adapter writes: the
+service's JSON with the echoed request left out of its header and the rows in a
+fixed order (ADR 0039). Each element of ``Data`` is one monitor's summary for one
+local calendar day under one pollutant standard, so the table is one row per
+element, with the columns as the service names them.
+"""
+
+from __future__ import annotations
+
+import json
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+from usdata.readers import MissingReaderDependency
+
+if TYPE_CHECKING:
+    from usdata._fetch import FetchedAsset
+
+DATE_COLUMNS = ("date_local", "date_of_last_change")
+"""Calendar dates as the service writes them, ``YYYY-MM-DD``, parsed without a timezone.
+
+``date_local`` is a day in the monitor's local standard time, not a UTC instant,
+so it is left naive rather than given a zone it does not have.
+"""
+
+
+def open_aqs(fetched: FetchedAsset) -> Any:
+    """Parse a fetched AQS daily-summary file into a pandas DataFrame, one row per summary."""
+    try:
+        pandas = import_module("pandas")
+    except ModuleNotFoundError as error:
+        if error.name != "pandas":
+            raise
+        raise MissingReaderDependency(
+            'AQS reading requires pandas; install it with: pip install "usdata[pandas]" '
+            '(or uv add "usdata[pandas]")'
+        ) from error
+    # Reading a fetched asset is strictly local; the cached file is never rewritten.
+    body = json.loads(fetched.path.read_text(encoding="utf-8"))
+    frame = pandas.DataFrame(body.get("Data") or [])
+    for column in DATE_COLUMNS:
+        if column in frame:
+            frame[column] = pandas.to_datetime(frame[column], format="%Y-%m-%d")
+    frame.attrs["usdata"] = {
+        "asset_id": fetched.asset.id,
+        "provenance": fetched.provenance.model_dump(mode="json"),
+        "header": body.get("Header", []),
+    }
+    return frame

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/_fetch.py

@@ -144,7 +144,7 @@ def _fetch_asset(
             adapter.fetch(asset, tmp)
         else:
             adapter.fetch_partial(asset, tmp, partial)
-        prov = provenance.record(dataset, asset, tmp, partial)
+        prov = provenance.record(dataset, asset, tmp, partial, adapter.transformations)
         if asset.checksum and prov.checksum != asset.checksum:
             raise ChecksumMismatch(f"{asset.id}: expected {asset.checksum}, got {prov.checksum}")
     # A crash between replacements leaves a detectable mismatch, never a trusted partial file.

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/_hurdat2.py

@@ -160,7 +160,14 @@ def open_hurdat2(fetched: FetchedAsset) -> Any:
             '(or uv add "usdata[pandas]")'
         ) from error
     # Reading a fetched asset is strictly local; the source file is never rewritten.
-    columns = parse(fetched.path.read_text(encoding="utf-8"))
+    try:
+        columns = parse(fetched.path.read_text(encoding="utf-8"))
+    except Hurdat2FormatError as error:
+        # NHC typos come and go between revisions, so name the way to pick another.
+        raise Hurdat2FormatError(
+            f"{fetched.asset.id}: {error}; an upstream typo is usually absent from the "
+            "neighbouring revisions, so select one with the 'revision' parameter"
+        ) from error
     data: dict[str, Any] = {
         name: pandas.array(columns[name], dtype="string") for name in TEXT_COLUMNS
     }

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/cli/app.py

@@ -21,7 +21,7 @@ from usdata.cli.inspect import inspect
 from usdata.cli.progress import progress
 from usdata.manifest import lockfile_path
 from usdata.models import READER_EXTRAS_TEXT, Dataset, Status, describe_duration
-from usdata.providers import load_adapter
+from usdata.providers import adapter_class, load_adapter
 from usdata.providers.base import NotImplementedProvider
 from usdata.pull import EmptySource, ManifestChanged, Plan, UnknownDatasets, UpstreamChanged
 from usdata.pull import plan as plan_manifest
@@ -260,8 +260,7 @@ def info(
         )
     if ds.status is not Status.AVAILABLE:
         return  # Planned entries have no adapter or usage metadata to show.
-    with load_adapter(ds) as adapter:
-        declared = dict(adapter.accepted_params)
+    declared = dict(adapter_class(ds).accepted_params)
     if declared:
         typer.echo("  params:")
         width = max(len(name) for name in declared)
@@ -285,6 +284,9 @@ def _echo_usage(ds: Dataset) -> None:
         typer.echo(f"  selection: {ds.selection}")
     if ds.inputs:
         typer.echo(f"  inputs:    {ds.inputs}")
+    if ds.credentials:
+        typer.echo(f"  credentials: {', '.join(ds.credentials.variables)} (environment)")
+        typer.echo(f"  key:       {ds.credentials.signup}")
     if ds.examples:
         typer.echo(f"  examples:  {', '.join(ds.examples)}")
 
@@ -528,13 +530,21 @@ def pull(
     else:
         mode = "restored from" if result.from_lockfile else "wrote"
     typer.echo(f"{len(result.fetched)} asset(s); {mode} {result.lockfile_path}", err=True)
-    if result.mirrored:
+    changed = len(result.mirrored) - len(result.unchecked)
+    if changed:
         typer.secho(
-            f"{len(result.mirrored)} asset(s) changed upstream and were restored from the "
+            f"{changed} asset(s) changed upstream and were restored from the "
             "mirror; the lockfile still pins the source. Pass --update to accept new bytes.",
             err=True,
             fg="yellow",
         )
+    if result.unchecked:
+        typer.secho(
+            f"{len(result.unchecked)} asset(s) were restored from the mirror without asking "
+            "their source, whose credentials are not set; upstream was not checked for changes.",
+            err=True,
+            fg="yellow",
+        )
 
 
 @app.command()

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/data/registry.yaml

@@ -424,7 +424,8 @@ datasets:
       intensity, pressure, and wind radii for Atlantic (since 1851) and
       northeast/north-central Pacific (since 1949) tropical cyclones. One
       fixed-format text file per basin, revised after each season; a basin
-      parameter picks the file and the newest revision wins. There is no
+      parameter picks the file and the newest revision wins unless a revision
+      date names another. There is no
       query interface, so dates and geographic filters are rejected and the
       local reader turns the whole file into one row per track point.
     keywords: [hurricane, tropical cyclone, best track, nhc, atlantic, pacific]
@@ -435,8 +436,8 @@ datasets:
     capabilities: { spatial_subset: false, temporal_subset: false, variable_subset: false }
     summary: Tropical cyclone best tracks
     formats: [HURDAT2 fixed-format text]
-    selection: The newest revision of one whole basin file; filter track points locally
-    inputs: Optional basin (atlantic or pacific); no dates or geographic filters
+    selection: The newest or a named revision of one whole basin file; filter track points locally
+    inputs: Optional basin (atlantic or pacific) and revision date; no dates or geographic filters
     reader: pandas
     guide: docs/providers/noaa-hurdat2.md
     examples:
@@ -1450,18 +1451,80 @@ datasets:
   # ---------------------------------------------------------------- EPA
   - id: epa:aqs-daily
     provider: epa
-    status: planned
+    status: available
     domain: air-quality
-    target: later
+    since: "0.26"
     title: Air Quality System Daily Summaries
     description: >-
-      Daily pollutant summaries (ozone, PM2.5, NO2, and others) from regulatory
-      monitors via the AQS Data API. Requires a free API key issued by email.
-    keywords: [air quality, pollution, ozone, pm2.5, monitors, aqs]
+      Daily summaries of regulatory air monitoring data from EPA's Air Quality
+      System through the AQS Data API: one row per monitor, local day, and
+      pollutant standard, for one to five AQS parameter codes such as PM2.5
+      and ozone, over named sites, a state or county, or a box. Each request
+      covers at most one calendar year, so a window becomes one JSON file per
+      year. Requires a free key issued by email; the response is stored in a
+      canonical form without the echoed request.
+    keywords: [air quality, pollution, ozone, pm2.5, monitors, aqs, smoke, epa]
     protocol: http
-    homepage: https://aqs.epa.gov/aqsweb/documents/data_api.html
+    homepage: https://aqs.epa.gov/aqsweb/documents/data_api.html#daily
     license: US Government Work (public domain)
-    capabilities: { spatial_subset: true, temporal_subset: true, variable_subset: true }
+    capabilities: { spatial_subset: true, temporal_subset: true, variable_subset: false }
+    credentials:
+      variables: [USDATA_AQS_EMAIL, USDATA_AQS_KEY]
+      signup: https://aqs.epa.gov/aqsweb/documents/data_api.html#signup
+    summary: Daily air pollutant summaries from regulatory monitors
+    formats: [JSON]
+    selection: Local days within inclusive UTC calendar dates for one to five pollutants; one file per year
+    inputs: Both dates; one to five parameter codes; site ids, a state or county, or a box
+    reader: pandas
+    guide: docs/providers/epa-aqs-daily.md
+    examples:
+      - examples/wildfire-smoke/README.md
+    resolution:
+      spatial: Regulatory monitoring sites operated by state, local, and tribal agencies
+      temporal: Daily summaries of each monitor's samples, one row per pollutant standard
+    update_frequency: >-
+      As monitoring agencies submit each quarter (40 CFR 58.16) and certify the previous year by
+      May 1 (40 CFR 58.15); submitted values can be revised later
+    latency: >-
+      Agencies must submit each calendar quarter's data within 90 days after it ends (40 CFR 58.16)
+    citation: >-
+      U.S. Environmental Protection Agency, Air Quality System (AQS) daily summary data, AQS Data
+      API, accessed via usdata
+    terms: https://aqs.epa.gov/aqsweb/documents/data_api.html#terms
+    variables:
+      - { name: "state_code", description: "FIPS code of the state the monitor is in; 80 for Mexico, CC for Canada at border sites" }
+      - { name: "county_code", description: "FIPS code of the county, parish, or independent city within the state" }
+      - { name: "site_number", description: "Four-digit site number, unique within the county" }
+      - { name: "parameter_code", description: "AQS code of the parameter measured" }
+      - { name: "poc", description: "Parameter occurrence code distinguishing instruments measuring the same parameter at one site" }
+      - { name: "latitude", units: "degrees_north", description: "Site latitude, WGS84" }
+      - { name: "longitude", units: "degrees_east", description: "Site longitude, WGS84" }
+      - { name: "datum", description: "Datum of the coordinates, always WGS84" }
+      - { name: "parameter", description: "Name of the parameter measured" }
+      - { name: "sample_duration_code", description: "Code of the sample duration" }
+      - { name: "sample_duration", description: "Averaging period: observed, such as 1 HOUR or 24 HOUR, or calculated, such as 24-HR BLK AVG" }
+      - { name: "pollutant_standard", description: "National ambient air quality standard the row's statistics are calculated for; empty for none" }
+      - { name: "date_local", description: "Day the sample was taken, in local standard time" }
+      - { name: "units_of_measure", description: "Units of every statistic on the row" }
+      - { name: "event_type", description: "Whether exceptional-event data are included: No Events, Events Included, Events Excluded, or Concurred Events Excluded" }
+      - { name: "observation_count", description: "Number of observations in the averaging period" }
+      - { name: "observation_percent", units: "percent", description: "Share of scheduled values for the day that were reported" }
+      - { name: "validity_indicator", description: "Y where the value meets all completeness criteria" }
+      - { name: "arithmetic_mean", description: "Mean of the day's values, in units_of_measure" }
+      - { name: "first_max_value", description: "Highest value at the row's duration or standard, in units_of_measure" }
+      - { name: "first_max_hour", description: "Hour of the day, 24-hour local standard time, of the highest value" }
+      - { name: "aqi", description: "Air Quality Index for the day, where the pollutant has one" }
+      - { name: "method_code", description: "Three-digit measurement method code, unique within a parameter" }
+      - { name: "method", description: "Collection and analysis method" }
+      - { name: "local_site_name", description: "Site name in the operating agency's own nomenclature" }
+      - { name: "site_address", description: "Approximate street address of the site" }
+      - { name: "county", description: "Name of the county the site is in" }
+      - { name: "state", description: "Name of the state the site is in" }
+      - { name: "city", description: "Incorporated city the site is in, if any" }
+      - { name: "cbsa_code", description: "Code of the core-based statistical (metropolitan) area" }
+      - { name: "cbsa", description: "Name of the core-based statistical (metropolitan) area" }
+      - { name: "date_of_last_change", description: "Date the underlying data were last changed in AQS" }
+    adapter: usdata.providers.epa.aqs:AqsDaily
 
   # ---------------------------------------------------------------- FEMA
   - id: fema:nfhl

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/doctor.py

@@ -1,7 +1,8 @@
-"""Read-only environment report: interpreter, reader extras, cache, and endpoints.
+"""Read-only environment report: interpreter, reader extras, cache, credentials, and endpoints.
 
 ``diagnose`` only observes. It imports optional modules, reads environment
-variables, stats the cache directory, and - when asked - makes one bounded
+variables, stats the cache directory, reports whether each dataset that needs
+credentials has them without ever printing a value, and - when asked - makes one bounded
 request per upstream host family. It never creates, moves, or repairs anything;
 the CLI prints what it finds and leaves the fixing to the reader.
 """
@@ -30,9 +31,10 @@ from usdata import __version__
 from usdata._grib import LIBRARY_HINT
 from usdata.cache import ENV_VAR, cache_dir
 from usdata.mirror import ENV_VAR as MIRROR_ENV_VAR
-from usdata.models import READER_EXTRAS
+from usdata.models import READER_EXTRAS, Status
 from usdata.protocols import http
-from usdata.registry import default_registry
+from usdata.providers.credentials import Credentials
+from usdata.registry import Registry, default_registry
 
 READER_MODULES: dict[str, tuple[str, ...]] = {
     "pandas": ("pandas",),
@@ -86,7 +88,13 @@ def diagnose(*, network: bool = False) -> Report:
         cache, environment, endpoints. A missing optional extra is ``warn``;
         a cache directory that cannot be written is ``fail``.
     """
-    checks = [*_runtime_checks(), *_reader_checks(), *_cache_checks(), *_environment_checks()]
+    checks = [
+        *_runtime_checks(),
+        *_reader_checks(),
+        *_cache_checks(),
+        *_environment_checks(),
+        *_credential_checks(),
+    ]
     if network:
         checks.extend(_endpoint_checks())
     return Report(checks=checks)
@@ -198,6 +206,31 @@ def _environment_checks() -> Iterator[Check]:
             yield Check(name=f"env:{name}", status=CheckStatus.OK, detail=value)
 
 
+def _credential_checks(registry: Registry | None = None) -> Iterator[Check]:
+    """Whether each fetchable dataset that needs credentials has them; values are never read out.
+
+    An unset variable is a warning, not a failure: every other dataset still
+    works, and a locked restore can still come from the cache or the mirror.
+    """
+    for dataset in (registry or default_registry()).list():
+        if dataset.credentials is None or dataset.status is Status.PLANNED:
+            continue
+        names = dataset.credentials.variables
+        present = Credentials.from_environment(names)
+        if missing := [name for name in names if name not in present]:
+            yield Check(
+                name=f"credentials:{dataset.id}",
+                status=CheckStatus.WARN,
+                detail=f"{', '.join(missing)} unset; request a key at {dataset.credentials.signup}",
+            )
+        else:
+            yield Check(
+                name=f"credentials:{dataset.id}",
+                status=CheckStatus.OK,
+                detail=f"{', '.join(names)} set",
+            )
+
+
 def _endpoint_checks() -> Iterator[Check]:
     """One bounded request per upstream host; an unreachable host is a failure."""
     hosts = probe_hosts()

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/models.py

@@ -230,6 +230,37 @@ class Variable(BaseModel):
         return f"{self.name} ({self.units})" if self.units else self.name
 
 
+CREDENTIAL_VARIABLE = re.compile(r"USDATA_[A-Z0-9]+(?:_[A-Z0-9]+)+")
+"""How a credential variable is named: ``USDATA_<SYSTEM>_<FIELD>``, such as ``USDATA_AQS_KEY``."""
+
+
+class CredentialSpec(BaseModel):
+    """The environment variables a source needs before it can be contacted, and where to get a key.
+
+    Declared in the registry entry so the catalog, ``info``, and ``doctor`` can
+    say what a dataset needs, and so the core can check it before any request
+    (ADR 0039). Values never appear here or anywhere else the core writes.
+    """
+
+    variables: list[str] = Field(
+        min_length=1, description="Environment variables that must be set, USDATA_<SYSTEM>_<FIELD>"
+    )
+    signup: str = Field(description="https URL where the agency issues a key")
+
+    @model_validator(mode="after")
+    def _named(self) -> CredentialSpec:
+        for name in self.variables:
+            if not CREDENTIAL_VARIABLE.fullmatch(name):
+                raise ValueError(
+                    f"credential variable {name!r} must be named USDATA_<SYSTEM>_<FIELD>"
+                )
+        if len(set(self.variables)) != len(self.variables):
+            raise ValueError("credentials.variables entries must be distinct")
+        if not self.signup.startswith("https://"):
+            raise ValueError("credentials.signup must be an https URL")
+        return self
+
+
 class Limits(BaseModel):
     """Request limits the adapter enforces, declared here and verified by the adapter tests."""
 
@@ -304,6 +335,10 @@ class Dataset(BaseModel):
     limits: Limits | None = Field(
         default=None, description="Request limits the adapter enforces, such as the longest window"
     )
+    credentials: CredentialSpec | None = Field(
+        default=None,
+        description="Environment variables the source needs before it can be contacted (ADR 0039)",
+    )
     system: str | None = Field(
         default=None,
         description="Id of a system declared in the registry, for datasets that belong to one",
@@ -595,7 +630,16 @@ class Provenance(BaseModel):
     )
     mirror: str | None = Field(
         default=None,
-        description="Mirror object that served these bytes after the source stopped (ADR 0030)",
+        description=(
+            "Mirror object that served these bytes in place of the source (ADR 0030, ADR 0039)"
+        ),
+    )
+    credentials: list[str] = Field(
+        default_factory=list,
+        description=(
+            "Environment variables the source requires to fetch this file, never their values "
+            "(ADR 0039); empty for an anonymous source"
+        ),
     )
 
     @property

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/provenance.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -14,7 +15,11 @@ SIDECAR_SUFFIX = ".provenance.json"
 
 
 def record(
-    dataset: Dataset, asset: Asset, path: Path, partial: PartialFetch | None = None
+    dataset: Dataset,
+    asset: Asset,
+    path: Path,
+    partial: PartialFetch | None = None,
+    transformations: Sequence[str] = (),
 ) -> Provenance:
     """Build a provenance record for a file that was just fetched to ``path``.
 
@@ -26,6 +31,9 @@ def record(
             Its index, ranges, selectors, and object identity are recorded
             alongside the checksum of the local file, which still covers exactly
             these bytes.
+        transformations: How the adapter's ``fetch`` changed the bytes the
+            source sent (``Provider.transformations``), recorded after any
+            partial-fetch entry.
 
     Returns:
         The record to write beside ``path``.
@@ -39,13 +47,14 @@ def record(
         size=path.stat().st_size,
         license=dataset.license,
         usdata_version=__version__,
-        transformations=[] if partial is None else [partial.describe()],
+        transformations=[*([] if partial is None else [partial.describe()]), *transformations],
         index_url=None if partial is None else partial.index_url,
         index_checksum=None if partial is None else partial.index_checksum,
         ranges=[] if partial is None else list(partial.ranges),
         selectors=[] if partial is None else list(partial.selectors),
         object_size=None if partial is None else partial.object_size,
         object_etag=None if partial is None else partial.object_etag,
+        credentials=[] if dataset.credentials is None else list(dataset.credentials.variables),
     )
 
 

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/providers/__init__.py

@@ -5,11 +5,21 @@ inside this repository or outside it, is written against. ``Provider`` is the
 interface, ``HttpProvider`` the client lifecycle HTTP-backed adapters inherit,
 ``QueryError`` the refusal every adapter raises, and the coercions come from
 ``usdata.providers.params`` so parameter models read the same everywhere.
+``Credentials`` carries the values a source that needs a key receives, and
+``MissingCredentials`` is the refusal when one is unset (ADR 0039).
 ``usdata.testing`` checks an adapter against this contract. See
 [ADR 0027](https://github.com/jakeryderv/usdata/blob/main/docs/adr/0027-provider-contract.md).
 """
 
-from usdata.providers.base import NotImplementedProvider, Provider, QueryError, load_adapter
+from usdata.providers.base import (
+    MissingCredentials,
+    NotImplementedProvider,
+    Provider,
+    QueryError,
+    adapter_class,
+    load_adapter,
+)
+from usdata.providers.credentials import Credentials
 from usdata.providers.http import HttpProvider
 from usdata.providers.params import (
     OptionalUpperStrList,
@@ -23,13 +33,16 @@ from usdata.providers.params import (
 )
 
 __all__ = [
+    "Credentials",
     "HttpProvider",
+    "MissingCredentials",
     "NotImplementedProvider",
     "OptionalUpperStrList",
     "Provider",
     "QueryError",
     "StrList",
     "UpperStrList",
+    "adapter_class",
     "choice",
     "flag",
     "int_list",

{usdata-0.25.0 → usdata-0.26.0}/src/usdata/providers/base.py

@@ -14,6 +14,7 @@ from pydantic import BaseModel, ValidationError
 from pydantic_core import ErrorDetails
 
 from usdata.models import Asset, Dataset, PartialFetch, Place, Provenance, Query
+from usdata.providers.credentials import Credentials
 
 QueryField = Literal["text", "bbox", "variables", "time"]
 Params = TypeVar("Params", bound=BaseModel)
@@ -37,6 +38,36 @@ class QueryError(ValueError):
     """The query cannot be satisfied by this dataset (missing or unsupported constraints)."""
 
 
+class MissingCredentials(QueryError):
+    """A source that needs credentials was about to be contacted without them.
+
+    Raised when the adapter is built, so no request is ever sent without them.
+    ``missing`` names the unset variables, and the message says where to get a key.
+    """
+
+    def __init__(self, dataset: Dataset, missing: list[str], note: str = "") -> None:
+        self.dataset_id = dataset.id
+        self.missing = missing
+        signup = dataset.credentials.signup if dataset.credentials else None
+        message = f"{dataset.id} needs {', '.join(missing)} set in the environment"
+        if signup:
+            message = f"{message}; request a key at {signup}"
+        super().__init__(f"{message}; {note}" if note else message)
+
+
+def required_credentials(dataset: Dataset, credentials: Credentials | None) -> Credentials:
+    """``credentials``, checked against what ``dataset`` declares; empty for an anonymous source.
+
+    Raises:
+        MissingCredentials: A declared variable has no value in ``credentials``.
+    """
+    given = credentials if credentials is not None else Credentials()
+    declared = [] if dataset.credentials is None else dataset.credentials.variables
+    if missing := [name for name in declared if not given.get(name)]:
+        raise MissingCredentials(dataset, missing)
+    return given
+
+
 def to_utc(value: datetime) -> datetime:
     """Apply the shared time policy: naive datetimes mean UTC, aware ones convert to it."""
     return value.replace(tzinfo=value.tzinfo or UTC).astimezone(UTC)
@@ -115,14 +146,34 @@ class Provider(ABC):
     subclassing the parent's model.
     """
 
+    transformations: ClassVar[tuple[str, ...]] = ()
+    """How ``fetch`` changes the bytes the source sent, one line each, or nothing for exact bytes.
+
+    The core records these in every provenance sidecar the adapter's fetches
+    write, beside any partial-fetch entry. Only an adapter whose source cannot be
+    pinned as received declares one: a response that echoes the request's
+    credentials, or differs between identical requests (ADR 0039).
+    """
+
     def __init_subclass__(cls, **kwargs: Any) -> None:
         """Derive ``accepted_params`` from a declared model, so one declaration feeds both."""
         super().__init_subclass__(**kwargs)
         if cls.params_model is not None:
             cls.accepted_params = described_params(cls.params_model)
 
-    def __init__(self, dataset: Dataset) -> None:
+    def __init__(self, dataset: Dataset, *, credentials: Credentials | None = None) -> None:
+        """Bind the adapter to its registry entry, refusing to exist without declared credentials.
+
+        Args:
+            dataset: The registry entry this adapter serves.
+            credentials: Values for the variables ``dataset.credentials`` declares;
+                ``load_adapter`` reads them from the environment.
+
+        Raises:
+            MissingCredentials: The dataset declares a variable ``credentials`` lacks.
+        """
         self.dataset = dataset
+        self.credentials = required_credentials(dataset, credentials)
 
     def __enter__(self) -> Self:
         return self
@@ -265,8 +316,15 @@ class Provider(ABC):
         )
 
 
-def load_adapter(dataset: Dataset) -> Provider:
-    """Instantiate the Provider named by a dataset's ``adapter`` dotted path."""
+def adapter_class(dataset: Dataset) -> type[Provider]:
+    """The Provider class a dataset's ``adapter`` dotted path names, without instantiating it.
+
+    For what a class declares, such as ``accepted_params``, which needs no
+    credentials and no transport.
+
+    Raises:
+        NotImplementedProvider: The dataset is planned and names no adapter.
+    """
     if dataset.adapter is None:
         raise NotImplementedProvider(
             f"{dataset.id} is {dataset.status.value}; no adapter exists yet"
@@ -276,4 +334,22 @@ def load_adapter(dataset: Dataset) -> Provider:
     cls = getattr(module, class_name)
     if not (isinstance(cls, type) and issubclass(cls, Provider)):
         raise TypeError(f"{dataset.adapter} is not a Provider subclass")
-    return cls(dataset)
+    return cls
+
+
+def load_adapter(dataset: Dataset) -> Provider:
+    """Instantiate the Provider named by a dataset's ``adapter`` dotted path.
+
+    A dataset that declares credentials gets their values from the environment,
+    and one that is missing any is refused here, before a request can be made.
+    An anonymous dataset's adapter is built from the dataset alone, so an
+    adapter that predates credentials needs no change.
+
+    Raises:
+        NotImplementedProvider: The dataset is planned and names no adapter.
+        MissingCredentials: A variable the dataset declares is unset or blank.
+    """
+    cls = adapter_class(dataset)
+    if dataset.credentials is None:
+        return cls(dataset)
+    return cls(dataset, credentials=Credentials.from_environment(dataset.credentials.variables))