usdata 0.12.0__tar.gz → 0.13.0__tar.gz

{usdata-0.12.0 → usdata-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: usdata
-Version: 0.12.0
+Version: 0.13.0
 Summary: Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data
 Keywords: noaa,usgs,nasa,open-data,scientific-data,provenance
 Author: Jake Van Slyke

{usdata-0.12.0 → usdata-0.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.12.0"
+version = "0.13.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.12.0 → usdata-0.13.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "usdata"
-version = "0.12.0"
+version = "0.13.0"
 description = "Unified Python SDK and CLI for discovering, fetching, and tracking provenance of U.S. public scientific data"
 readme = "README.md"
 license = "Apache-2.0"

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/data/registry.yaml

@@ -743,6 +743,7 @@ catalog:
     reader_extra: radar
     examples:
     - examples/radar-reflectivity/example.ipynb
+    - examples/event-context/example.ipynb
   noaa:goes-abi:
     guide: docs/providers/noaa-goes.md
     summary: GOES CONUS imagery
@@ -753,6 +754,7 @@ catalog:
     reader_extra: netcdf
     examples:
     - examples/goes-imagery/example.ipynb
+    - examples/event-context/example.ipynb
   noaa:storm-events:
     guide: docs/providers/noaa-storm-events.md
     summary: Storm Events details
@@ -763,6 +765,7 @@ catalog:
     reader_extra: pandas
     examples:
     - examples/storm-events/example.ipynb
+    - examples/event-context/example.ipynb
   noaa:hurdat2:
     guide: docs/providers/noaa-hurdat2.md
     summary: Tropical cyclone best tracks

usdata-0.13.0/src/usdata/protocols/listing.py

@@ -0,0 +1,61 @@
+"""Read file names and byte sizes from Apache-style HTML directory indexes."""
+
+from __future__ import annotations
+
+import re
+from html.parser import HTMLParser
+
+
+class _Index(HTMLParser):
+    """Collect (name, size) for anchors whose href is a literal local file name.
+
+    Only hrefs matching ``pattern`` in full are kept, so a listing can never
+    steer a download to an arbitrary link. The size is the first all-digit
+    table cell after the anchor's cell in the same row, or None.
+    """
+
+    def __init__(self, pattern: re.Pattern[str]) -> None:
+        super().__init__()
+        self.pattern = pattern
+        self.entries: list[tuple[str, int | None]] = []
+        self._in_row = False
+        self._in_cell = False
+        self._cells: list[str] = []
+        self._name: str | None = None
+        self._name_cell: int | None = None
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag == "tr":
+            self._in_row, self._cells, self._name, self._name_cell = True, [], None, None
+        elif tag == "td":
+            self._cells.append("")
+            self._in_cell = True
+        elif tag == "a":
+            href = dict(attrs).get("href", "") or ""
+            if not self.pattern.fullmatch(href):
+                return
+            if self._in_row:
+                self._name, self._name_cell = href, len(self._cells) - 1
+            else:
+                self.entries.append((href, None))
+
+    def handle_data(self, data: str) -> None:
+        if self._in_cell:
+            self._cells[-1] += data
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "td":
+            self._in_cell = False
+        elif tag == "tr":
+            if self._name is not None:
+                later = self._cells[(self._name_cell or 0) + 1 :]
+                sizes = [c.strip() for c in later if c.strip().isascii() and c.strip().isdigit()]
+                self.entries.append((self._name, int(sizes[0]) if sizes else None))
+            self._in_row = False
+
+
+def directory_entries(html: str, pattern: re.Pattern[str]) -> list[tuple[str, int | None]]:
+    """File names matching ``pattern`` in a directory index, with byte sizes when listed."""
+    index = _Index(pattern)
+    index.feed(html)
+    return index.entries

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/protocols/s3.py

@@ -61,7 +61,6 @@ def list_objects(
     try:
         while True:
             resp = http.get(https_url(bucket), client, params=params)
-            resp.raise_for_status()
             root = ET.fromstring(resp.text)
             truncated = _text(root, "IsTruncated") == "true"
             token = _text(root, "NextContinuationToken")

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/base.py

@@ -5,12 +5,21 @@ from __future__ import annotations
 import importlib
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
+from datetime import UTC, datetime
 from pathlib import Path
 from types import MappingProxyType, TracebackType
-from typing import ClassVar, Self
+from typing import ClassVar, Literal, Self
 
 from usdata.models import Asset, Dataset, Query
 
+QueryField = Literal["text", "bbox", "variables", "time"]
+_LABELS: dict[QueryField, str] = {
+    "text": "text",
+    "bbox": "location/bbox",
+    "variables": "variables",
+    "time": "start/end",
+}
+
 
 class NotImplementedProvider(NotImplementedError):
     """Raised by adapters that are registered but not yet built."""
@@ -20,8 +29,30 @@ class QueryError(ValueError):
     """The query cannot be satisfied by this dataset (missing or unsupported constraints)."""
 
 
+def to_utc(value: datetime) -> datetime:
+    """Apply the shared time policy: naive datetimes mean UTC, aware ones convert to it."""
+    return value.replace(tzinfo=value.tzinfo or UTC).astimezone(UTC)
+
+
+def _is_set(query: Query, field: QueryField) -> bool:
+    if field == "text":
+        return bool(query.text)
+    if field == "bbox":
+        return query.bbox is not None
+    if field == "variables":
+        return bool(query.variables)
+    return query.time is not None
+
+
 class Provider(ABC):
-    """One adapter per dataset. Translates a normalized query into concrete assets."""
+    """One adapter per dataset. Translates a normalized query into concrete assets.
+
+    ``list_assets`` implementations validate with the shared helpers before any
+    transport: ``check_params`` for provider-specific keys, ``reject`` for query
+    fields the source cannot honour, and ``utc_window`` for the time bounds.
+    Every adapter then reports the same errors for the same mistakes, and no
+    query field is silently ignored.
+    """
 
     accepted_params: ClassVar[Mapping[str, str]] = MappingProxyType({})
     """Accepted ``query.params`` keys, each mapped to a one-line description.
@@ -49,6 +80,28 @@ class Provider(ABC):
         """Release owned resources. Providers with resources override this method."""
         return None
 
+    def check_params(self, query: Query) -> None:
+        """Reject every ``query.params`` key outside ``accepted_params``, naming it."""
+        if unknown := set(query.params) - set(self.accepted_params):
+            raise QueryError(f"unsupported {self.dataset.id} params: {', '.join(sorted(unknown))}")
+
+    def reject(self, query: Query, *fields: QueryField, hint: str = "") -> None:
+        """Raise ``QueryError`` when the query sets a field this dataset cannot honour.
+
+        Sources that ignore a filter must say so rather than return unfiltered
+        data; ``hint`` tells the caller what to do instead.
+        """
+        present = [_LABELS[field] for field in fields if _is_set(query, field)]
+        if present:
+            message = f"{self.dataset.id} does not support {', '.join(present)}"
+            raise QueryError(f"{message}; {hint}" if hint else message)
+
+    def utc_window(self, query: Query) -> tuple[datetime, datetime]:
+        """Both time bounds, required, in UTC. Naive bounds are read as UTC."""
+        if query.time is None or query.time.start is None or query.time.end is None:
+            raise QueryError(f"{self.dataset.id} requires both start and end")
+        return to_utc(query.time.start), to_utc(query.time.end)
+
     @abstractmethod
     def list_assets(self, query: Query) -> list[Asset]:
         """Resolve a query to the concrete objects that satisfy it, without downloading."""

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/coastwatch.py

@@ -53,17 +53,11 @@ class CoastwatchSst(_HttpProvider):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Resolve a valid grid intersection into one stable, bounded CSV request."""
-        if (
-            query.bbox is None
-            or query.time is None
-            or query.time.start is None
-            or query.time.end is None
-        ):
-            raise QueryError(
-                f"{self.dataset.id} requires a bbox/location and both start and end times"
-            )
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported CoastWatch params: {', '.join(sorted(unknown))}")
+        self.check_params(query)
+        self.reject(query, "text", hint="select a bbox/location, time window, and variables")
+        if query.bbox is None:
+            raise QueryError(f"{self.dataset.id} requires a bbox/location and both start and end")
+        start, end = self.utc_window(query)
         stride = query.params.get("stride", 1)
         if isinstance(stride, str) and stride.isascii() and stride.isdigit():
             stride = int(stride)
@@ -102,8 +96,8 @@ class CoastwatchSst(_HttpProvider):
                 raise ValueError("unexpected time axis")
         except (KeyError, TypeError, ValueError) as error:
             raise httpx.DecodingError("CoastWatch grid metadata changed or is invalid") from error
-        first = bisect_left(times, query.time.start)
-        last = bisect_right(times, query.time.end) - 1
+        first = bisect_left(times, start)
+        last = bisect_right(times, end) - 1
         if first > last:
             return []
         if (last - first + 1) * lat[1] * lon[1] > MAX_ROWS:

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/coops.py

@@ -89,12 +89,8 @@ class CoopsWaterLevels(_HttpProvider):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Describe one bounded CSV request; availability is checked during fetching."""
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported {self.dataset.id} params: {', '.join(sorted(unknown))}")
-        if query.bbox is not None or query.text is not None or query.variables:
-            raise QueryError(
-                "CO-OPS requires an explicit station; location, text and variables are unsupported"
-            )
+        self.check_params(query)
+        self.reject(query, "bbox", "text", "variables", hint="CO-OPS requires an explicit station")
         station = query.params.get("station")
         if (
             not isinstance(station, str)
@@ -109,10 +105,7 @@ class CoopsWaterLevels(_HttpProvider):
         units = query.params.get("units", "metric")
         if units not in ("metric", "english"):
             raise QueryError("units must be metric or english")
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end timestamps")
-        start = query.time.start.replace(tzinfo=query.time.start.tzinfo or UTC).astimezone(UTC)
-        end = query.time.end.replace(tzinfo=query.time.end.tzinfo or UTC).astimezone(UTC)
+        start, end = self.utc_window(query)
         if any(value.second or value.microsecond for value in (start, end)):
             raise QueryError("CO-OPS timestamps must have minute precision (zero seconds)")
         if end - start > MAX_INTERVAL:

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/ghcnd.py

@@ -79,7 +79,6 @@ class GhcnDaily(_HttpProvider):
         seen: set[str] = set()
         while True:
             resp = http.get(SEARCH_URL, self._http(), params=params)
-            resp.raise_for_status()
             body = resp.json()
             results = body.get("results", [])
             before = len(found)
@@ -111,24 +110,35 @@ class GhcnDaily(_HttpProvider):
                 break
         return found
 
-    def list_assets(self, query: Query) -> list[Asset]:
-        """One CSV asset per chunk of up to STATIONS_PER_ASSET stations for the query window."""
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end dates")
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported {self.dataset.id} params: {', '.join(sorted(unknown))}")
-        if query.params.get("units", "metric") not in ("metric", "standard"):
+    def _units(self, query: Query) -> str:
+        units = query.params.get("units", "metric")
+        if units not in ("metric", "standard"):
             raise QueryError("units must be metric or standard")
+        return units
+
+    def _stations(self, query: Query) -> list[str]:
+        """Explicit ``stations`` or the ones found inside the bbox, never a mix of both."""
+        if "stations" in query.params and query.bbox is not None:
+            raise QueryError("pass stations or a location/bbox, not both")
         if "stations" in query.params:
-            stations = _stations_param(query.params["stations"])
-        elif query.bbox is not None:
-            stations = self.find_stations(query)
-        else:
-            raise QueryError(f"{self.dataset.id} needs a location, bbox, or stations=...")
+            return _stations_param(query.params["stations"])
+        if query.bbox is not None:
+            return self.find_stations(query)
+        raise QueryError(f"{self.dataset.id} needs a location, bbox, or stations=...")
+
+    def list_assets(self, query: Query) -> list[Asset]:
+        """One CSV asset per chunk of up to STATIONS_PER_ASSET stations for the query window."""
+        self.check_params(query)
+        self.reject(query, "text", hint="search the registry instead")
+        start_at, end_at = self.utc_window(query)
+        window = TimeRange(start=start_at, end=end_at)
+        query = query.model_copy(update={"time": window})
+        units = self._units(query)
+        stations = self._stations(query)
         if not stations:
             return []
 
-        start, end = _date(query.time.start), _date(query.time.end)
+        start, end = _date(window.start), _date(window.end)
         assets: list[Asset] = []
         for i in range(0, len(stations), STATIONS_PER_ASSET):
             chunk = stations[i : i + STATIONS_PER_ASSET]
@@ -138,7 +148,7 @@ class GhcnDaily(_HttpProvider):
                 "startDate": start,
                 "endDate": end,
                 "format": "csv",
-                "units": query.params.get("units", "metric"),
+                "units": units,
                 "includeStationLocation": "1",
             }
             if query.variables:
@@ -152,7 +162,7 @@ class GhcnDaily(_HttpProvider):
                     href=url,
                     protocol=Protocol.HTTP,
                     media_type="text/csv",
-                    time=TimeRange(start=query.time.start, end=query.time.end),
+                    time=window,
                     bbox=query.bbox,
                 )
             )

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/goes.py

@@ -1,8 +1,9 @@
 """GOES ABI CONUS Cloud and Moisture Imagery from anonymous NOAA S3 buckets.
 
 Require ``satellite`` (16, 17, 18, or 19), ``channel`` (1--16 or C01--C16), and
-both timestamps. The optional ``product`` must be ``ABI-L2-CMIPC``. Select whole
-single-channel NetCDF files by inclusive scan-start time, never by scan overlap.
+both timestamps at most seven days apart. The optional ``product`` must be
+``ABI-L2-CMIPC``. Select whole single-channel NetCDF files by inclusive
+scan-start time, never by scan overlap.
 Geographic and variable subsetting are not available for these archived files.
 """
 
@@ -21,6 +22,7 @@ from usdata.providers.base import QueryError
 
 PRODUCT = "ABI-L2-CMIPC"
 PUBLIC_START = datetime(2017, 2, 28, tzinfo=UTC)
+MAX_WINDOW = timedelta(days=7)
 KEY_RE = re.compile(
     r"OR_ABI-L2-CMIPC-M[346]C(?P<channel>0[1-9]|1[0-6])_G(?P<satellite>1[6-9])"
     r"_s(?P<start>\d{14})_e(?P<end>\d{14})_c\d{14}\.nc"
@@ -57,24 +59,21 @@ class GoesAbi(_HttpProvider):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """List complete scenes whose scan starts fall inside the inclusive UTC interval."""
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported GOES params: {', '.join(sorted(unknown))}")
+        self.check_params(query)
         if query.params.get("product", PRODUCT) != PRODUCT:
             raise QueryError(f"only product={PRODUCT} is supported")
-        if query.bbox is not None:
-            raise QueryError(
-                "GOES imagery has no geographic subsetting; omit location/bbox/lat/lon"
-            )
-        if query.text:
-            raise QueryError("GOES imagery has no text filtering; select satellite and channel")
-        if query.variables:
-            raise QueryError("GOES imagery has no variable subsetting; select a channel instead")
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end times")
+        self.reject(
+            query,
+            "bbox",
+            "text",
+            "variables",
+            hint="archived GOES scenes are whole files; select satellite and channel",
+        )
+        start, end = self.utc_window(query)
+        if end - start > MAX_WINDOW:
+            raise QueryError("GOES requests must span at most 7 days; split longer intervals")
         satellite = _number(query.params.get("satellite"), "satellite", 16, 19)
         channel = _number(query.params.get("channel"), "channel", 1, 16)
-        start = query.time.start.replace(tzinfo=query.time.start.tzinfo or UTC).astimezone(UTC)
-        end = query.time.end.replace(tzinfo=query.time.end.tzinfo or UTC).astimezone(UTC)
         if end < PUBLIC_START:
             raise QueryError("GOES CMIPC public observations begin on 2017-02-28")
         start = max(start, PUBLIC_START)

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/gsom.py

@@ -6,10 +6,8 @@ selected in full. Geographic discovery uses the same normalized month bounds.
 """
 
 from calendar import monthrange
-from datetime import UTC
 
 from usdata.models import Asset, Query, TimeRange
-from usdata.providers.base import QueryError
 from usdata.providers.noaa.ghcnd import GhcnDaily
 
 
@@ -19,10 +17,7 @@ class GlobalSummaryMonthly(GhcnDaily):
     ncei_dataset = "global-summary-of-the-month"
 
     def _monthly_query(self, query: Query) -> Query:
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end dates")
-        start = query.time.start.replace(tzinfo=query.time.start.tzinfo or UTC).astimezone(UTC)
-        end = query.time.end.replace(tzinfo=query.time.end.tzinfo or UTC).astimezone(UTC)
+        start, end = self.utc_window(query)
         return query.model_copy(
             update={
                 "time": TimeRange(
@@ -44,6 +39,4 @@ class GlobalSummaryMonthly(GhcnDaily):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Resolve monthly CSVs with stable URLs and complete-month asset bounds."""
-        if "stations" in query.params and query.bbox is not None:
-            raise QueryError("pass stations or a location/bbox, not both")
         return super().list_assets(self._monthly_query(query))

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/gsoy.py

@@ -5,10 +5,7 @@ Both dates are required; every UTC calendar year touched by the interval is
 selected in full. Geographic discovery uses the same normalized year bounds.
 """
 
-from datetime import UTC
-
 from usdata.models import Asset, Query, TimeRange
-from usdata.providers.base import QueryError
 from usdata.providers.noaa.ghcnd import GhcnDaily
 
 
@@ -18,14 +15,7 @@ class GlobalSummaryYearly(GhcnDaily):
     ncei_dataset = "global-summary-of-the-year"
 
     def _yearly_query(self, query: Query) -> Query:
-        if query.text is not None:
-            raise QueryError(f"{self.dataset.id} does not support text queries")
-        if "stations" in query.params and query.bbox is not None:
-            raise QueryError("pass stations or a location/bbox, not both")
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end dates")
-        start = query.time.start.replace(tzinfo=query.time.start.tzinfo or UTC).astimezone(UTC)
-        end = query.time.end.replace(tzinfo=query.time.end.tzinfo or UTC).astimezone(UTC)
+        start, end = self.utc_window(query)
         return query.model_copy(
             update={
                 "time": TimeRange(

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/hurdat2.py

@@ -17,12 +17,12 @@ from __future__ import annotations
 import re
 from collections.abc import Mapping
 from datetime import UTC, date, datetime
-from html.parser import HTMLParser
 from pathlib import Path
 from typing import ClassVar
 
 from usdata.models import Asset, Protocol, Query, TimeRange
 from usdata.protocols import http
+from usdata.protocols.listing import directory_entries
 from usdata.providers._http import _HttpProvider
 from usdata.providers.base import QueryError
 
@@ -33,20 +33,6 @@ FILE_NAME = re.compile(r"hurdat2-(?:(atl|nepac)-)?(\d{4})-(\d{4})-(\d{6}|\d{8})(
 BASIN_TOKENS = {"atlantic": frozenset({"", "atl"}), "pacific": frozenset({"nepac"})}
 
 
-class _Directory(HTMLParser):
-    """Collect HURDAT2 filenames from the NHC Apache directory listing."""
-
-    def __init__(self) -> None:
-        super().__init__()
-        self.names: list[str] = []
-
-    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        href = dict(attrs).get("href", "") or "" if tag == "a" else ""
-        # Only literal local filenames; never follow arbitrary links from a listing.
-        if FILE_NAME.fullmatch(href):
-            self.names.append(href)
-
-
 def _revision(value: str) -> date | None:
     """The revision date encoded in a filename, or None if it is not a real date."""
     try:
@@ -64,39 +50,40 @@ class Hurdat2(_HttpProvider):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Select the newest revision of the requested basin's complete database."""
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported HURDAT2 params: {', '.join(sorted(unknown))}")
+        self.check_params(query)
         requested = query.params.get("basin", "atlantic")
         basin = requested.strip().casefold() if isinstance(requested, str) else ""
         if basin not in BASIN_TOKENS:
             raise QueryError(
                 f"unsupported HURDAT2 basin {requested!r}; use 'atlantic' (default) or 'pacific'"
             )
-        if query.bbox is not None or query.variables or query.text:
-            raise QueryError(
-                "HURDAT2 publishes one whole text file per basin; location/bbox, variables, "
-                "and text filters are unsupported. Filter the parsed track table locally."
-            )
-        if query.time is not None:
-            raise QueryError(
-                "HURDAT2 does not accept dates: every revision holds the complete basin "
-                "record, so a date range would not change the download. Remove start/end "
-                "and filter the parsed 'time' column locally."
-            )
-        listing = _Directory()
-        listing.feed(http.get(DIRECTORY_URL, self._http()).text)
+        self.reject(
+            query,
+            "bbox",
+            "variables",
+            "text",
+            hint="HURDAT2 publishes one whole text file per basin; filter the parsed track "
+            "table locally",
+        )
+        self.reject(
+            query,
+            "time",
+            hint="every revision holds the complete basin record, so a date range would not "
+            "change the download; remove start/end and filter the parsed 'time' column locally",
+        )
+        page = http.get(DIRECTORY_URL, self._http()).text
         candidates = []
-        for name in listing.names:
+        for name, size in directory_entries(page, FILE_NAME):
             match = FILE_NAME.fullmatch(name)
             assert match is not None
             first, last, revised = int(match[2]), int(match[3]), _revision(match[4])
             if (match[1] or "") not in BASIN_TOKENS[basin] or revised is None or first > last:
                 continue
             # Prefer the latest season covered, then the newest revision of that span.
-            candidates.append(((last, revised, name), first, last))
+            candidates.append(((last, revised, name), first, last, size))
         if not candidates:
             raise QueryError(f"no HURDAT2 {basin} best-track file in the NHC directory listing")
-        (_, _, name), first, last = max(candidates)
+        (_, _, name), first, last, size = max(candidates)
         return [
             Asset(
                 id=name,
@@ -104,6 +91,7 @@ class Hurdat2(_HttpProvider):
                 href=DIRECTORY_URL + name,
                 protocol=Protocol.HTTP,
                 media_type="text/plain",
+                size=size,
                 time=TimeRange(
                     start=datetime(first, 1, 1, tzinfo=UTC),
                     end=datetime(last, 12, 31, 23, 59, 59, 999999, tzinfo=UTC),

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/nexrad.py

@@ -2,8 +2,8 @@
 
 Key layout: ``YYYY/MM/DD/SITE/SITEYYYYMMDD_HHMMSS[_V06][.gz]``. One object per
 volume scan. ``_MDM`` objects are metadata sidecars and are skipped. There is
-no server-side subsetting; a query selects sites and a time window and every
-whole scan in that window is an asset.
+no server-side subsetting; a query selects sites and a time window of at most
+31 days, and every whole scan in that window is an asset.
 
 Site selection, in order: ``site=``/``sites=`` params, then radars located
 inside the query bbox, then the single radar nearest the bbox centre.
@@ -25,6 +25,7 @@ from usdata.providers.base import QueryError
 from usdata.providers.noaa import sites
 
 BUCKET = "unidata-nexrad-level2"
+MAX_WINDOW = timedelta(days=31)
 KEY_RE = re.compile(r"^(?P<site>[A-Z]{4})(?P<stamp>\d{8}_\d{6})(?:_V0[36])?(?:\.gz)?$")
 
 
@@ -58,8 +59,7 @@ class NexradLevel2(_HttpProvider):
 
     def select_sites(self, query: Query) -> list[str]:
         """Radar site ids the query refers to; see the module docstring for the rules."""
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported NEXRAD params: {', '.join(sorted(unknown))}")
+        self.check_params(query)
         if "site" in query.params and "sites" in query.params:
             raise QueryError("pass only one of site or sites")
         if "nearest" in query.params and {"site", "sites"}.intersection(query.params):
@@ -95,10 +95,12 @@ class NexradLevel2(_HttpProvider):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Every volume scan for the selected sites inside the query's UTC time window."""
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end times")
-        start = query.time.start.replace(tzinfo=query.time.start.tzinfo or UTC).astimezone(UTC)
-        end = query.time.end.replace(tzinfo=query.time.end.tzinfo or UTC).astimezone(UTC)
+        self.reject(
+            query, "text", "variables", hint="whole volume scans are selected by site and time"
+        )
+        start, end = self.utc_window(query)
+        if end - start > MAX_WINDOW:
+            raise QueryError("NEXRAD requests must span at most 31 days; split longer intervals")
         assets: list[Asset] = []
         for site in self.select_sites(query):
             day = start.date()

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/normals.py

@@ -21,8 +21,8 @@ from typing import Any, ClassVar
 import httpx
 
 from usdata.models import Asset, Protocol, Query, TimeRange
-from usdata.providers.base import QueryError
-from usdata.providers.noaa.ghcnd import DATA_URL, STATIONS_PER_ASSET, GhcnDaily, _stations_param
+from usdata.providers.base import QueryError, to_utc
+from usdata.providers.noaa.ghcnd import DATA_URL, STATIONS_PER_ASSET, GhcnDaily
 
 PERIODS = {
     "monthly": "normals-monthly-1991-2020",
@@ -55,8 +55,7 @@ def _window(query: Query, period: str) -> tuple[str, str] | None:
         return (f"{PLACEHOLDER_YEAR}-01-01", f"{PLACEHOLDER_YEAR}-12-31")
     if start is None or end is None:
         raise QueryError("pass both start and end dates, or neither for the whole year")
-    first = start.replace(tzinfo=start.tzinfo or UTC).astimezone(UTC)
-    last = end.replace(tzinfo=end.tzinfo or UTC).astimezone(UTC)
+    first, last = to_utc(start), to_utc(end)
     if (first.month, first.day) > (last.month, last.day):
         raise QueryError("normals windows cannot cross the new year; split the query")
     return (
@@ -80,23 +79,12 @@ class ClimateNormals(GhcnDaily):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """One CSV asset per chunk of stations for the selected period and window."""
-        if query.text is not None:
-            raise QueryError(f"{self.dataset.id} does not support text queries")
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported {self.dataset.id} params: {', '.join(sorted(unknown))}")
+        self.check_params(query)
+        self.reject(query, "text", hint="search the registry instead")
         period = _period(query)
-        units = query.params.get("units", "metric")
-        if units not in ("metric", "standard"):
-            raise QueryError("units must be metric or standard")
-        if "stations" in query.params and query.bbox is not None:
-            raise QueryError("pass stations or a location/bbox, not both")
+        units = self._units(query)
         window = _window(query, period)
-        if "stations" in query.params:
-            stations = _stations_param(query.params["stations"])
-        elif query.bbox is not None:
-            stations = self.find_stations(query)
-        else:
-            raise QueryError(f"{self.dataset.id} needs a location, bbox, or stations=...")
+        stations = self._stations(query)
         if not stations:
             return []
 

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/noaa/storm_events.py

@@ -9,11 +9,11 @@ from __future__ import annotations
 
 import re
 from datetime import UTC, datetime
-from html.parser import HTMLParser
 from pathlib import Path
 
 from usdata.models import Asset, Protocol, Query, TimeRange
 from usdata.protocols import http
+from usdata.protocols.listing import directory_entries
 from usdata.providers._http import _HttpProvider
 from usdata.providers.base import QueryError
 
@@ -21,65 +21,27 @@ DIRECTORY_URL = "https://www.ncei.noaa.gov/pub/data/swdi/stormevents/csvfiles/"
 DETAILS_NAME = re.compile(r"StormEvents_details-ftp_v1\.0_d(\d{4})_c(\d{8})\.csv\.gz", re.ASCII)
 
 
-class _Directory(HTMLParser):
-    """Read filenames and exact byte sizes from NCEI's HTML directory table."""
-
-    def __init__(self) -> None:
-        super().__init__()
-        self.files: list[tuple[str, int | None]] = []
-        self._cells: list[str] = []
-        self._name: str | None = None
-        self._in_cell = False
-
-    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
-        if tag == "tr":
-            self._cells, self._name = [], None
-        elif tag == "td":
-            self._cells.append("")
-            self._in_cell = True
-        elif tag == "a":
-            href = dict(attrs).get("href", "") or ""
-            # Only literal local filenames; never follow arbitrary links from a listing.
-            if DETAILS_NAME.fullmatch(href):
-                self._name = href
-
-    def handle_data(self, data: str) -> None:
-        if self._in_cell:
-            self._cells[-1] += data
-
-    def handle_endtag(self, tag: str) -> None:
-        if tag == "td":
-            self._in_cell = False
-        elif tag == "tr" and self._name is not None:
-            size = self._cells[2].strip() if len(self._cells) > 2 else ""
-            self.files.append(
-                (self._name, int(size) if size.isascii() and size.isdigit() else None)
-            )
-
-
 class StormEvents(_HttpProvider):
     """Resolve whole-year details archives; preserve the original gzip bytes."""
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Select the latest supported details revision for every requested year."""
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported Storm Events params: {', '.join(sorted(unknown))}")
-        if query.bbox is not None or query.variables or query.text:
-            raise QueryError(
-                "Storm Events downloads whole annual details files; location/bbox, variables, "
-                "and text filters are unsupported. Filter locally after opening the CSV."
-            )
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end dates")
-        start = query.time.start.replace(tzinfo=query.time.start.tzinfo or UTC).astimezone(UTC)
-        end = query.time.end.replace(tzinfo=query.time.end.tzinfo or UTC).astimezone(UTC)
+        self.check_params(query)
+        self.reject(
+            query,
+            "bbox",
+            "variables",
+            "text",
+            hint="Storm Events downloads whole annual details files; filter locally after "
+            "opening the CSV",
+        )
+        start, end = self.utc_window(query)
         if start.year < 1950:
             raise QueryError("Storm Events annual details files start in 1950")
         years = range(start.year, end.year + 1)
-        listing = _Directory()
-        listing.feed(http.get(DIRECTORY_URL, self._http()).text)
+        page = http.get(DIRECTORY_URL, self._http()).text
         selected: dict[int, tuple[str, int | None]] = {}
-        for name, size in listing.files:
+        for name, size in directory_entries(page, DETAILS_NAME):
             match = DETAILS_NAME.fullmatch(name)
             assert match is not None
             year = int(match[1])

{usdata-0.12.0 → usdata-0.13.0}/src/usdata/providers/usgs/daily.py

@@ -5,8 +5,10 @@ prefix), and ``statistic_id`` (default 00003: daily mean). Variables are
 five-digit parameter codes, such as 00060 for streamflow. Dates select inclusive
 local calendar days; time-of-day information is discarded for daily values.
 
-Each result page is fetched as the service's CSV representation, preserving
-units, qualifiers, and approval status without the volatile GeoJSON timeStamp.
+Listing probes one row at each page offset to learn how many pages exist without
+downloading them. Each page is then fetched as the service's CSV representation,
+preserving units, qualifiers, and approval status without the volatile GeoJSON
+timeStamp.
 """
 
 from __future__ import annotations
@@ -20,7 +22,7 @@ from typing import ClassVar
 
 import httpx
 
-from usdata.models import Asset, Protocol, Query
+from usdata.models import Asset, Protocol, Query, TimeRange
 from usdata.protocols import http
 from usdata.providers._http import _HttpProvider
 from usdata.providers.base import QueryError
@@ -53,10 +55,10 @@ class WaterDaily(_HttpProvider):
 
     def list_assets(self, query: Query) -> list[Asset]:
         """Resolve site or bbox queries to paginated CSV requests."""
-        if query.time is None or query.time.start is None or query.time.end is None:
-            raise QueryError(f"{self.dataset.id} requires both start and end dates")
-        if unknown := set(query.params) - set(self.accepted_params):
-            raise QueryError(f"unsupported USGS params: {', '.join(sorted(unknown))}")
+        self.check_params(query)
+        self.reject(query, "text", hint="select sites, a location, or parameter codes")
+        start, end = self.utc_window(query)
+        query = query.model_copy(update={"time": TimeRange(start=start, end=end)})
         if "site" in query.params and "sites" in query.params:
             raise QueryError("pass only one of site or sites")
         raw_sites = query.params.get("sites", query.params.get("site"))
@@ -77,7 +79,7 @@ class WaterDaily(_HttpProvider):
             raise QueryError("variables must be five-digit USGS parameter codes, for example 00060")
         params = {
             "f": "json",
-            "time": f"{query.time.start.date().isoformat()}/{query.time.end.date().isoformat()}",
+            "time": f"{start.date().isoformat()}/{end.date().isoformat()}",
             "statistic_id": statistic,
             "limit": str(PAGE_SIZE),
         }
@@ -97,20 +99,18 @@ class WaterDaily(_HttpProvider):
         assets: list[Asset] = []
         offset = 0
         while True:
-            url = httpx.URL(ITEMS_URL, params={**params, "offset": str(offset)})
-            response = http.get(url, self._http())
-            response.raise_for_status()
+            # A one-row probe says whether a page starts here; the CSV asset is the page.
+            probe = httpx.URL(ITEMS_URL, params={**params, "limit": "1", "offset": str(offset)})
+            response = http.get(probe, self._http())
             try:
-                body = response.json()
-                features = body["features"]
+                features = response.json()["features"]
                 if not isinstance(features, list):
                     raise ValueError("features must be a list")
-                links = body.get("links", [])
-                next_url = next((link["href"] for link in links if link.get("rel") == "next"), None)
             except (ValueError, KeyError, TypeError, AttributeError) as e:
                 raise httpx.DecodingError("invalid USGS page", request=response.request) from e
             if not features:
                 break
+            url = httpx.URL(ITEMS_URL, params={**params, "offset": str(offset)})
             href = str(url.copy_set_param("f", "csv"))
             digest = hashlib.sha256(href.encode()).hexdigest()[:20]
             assets.append(
@@ -124,12 +124,10 @@ class WaterDaily(_HttpProvider):
                     bbox=query.bbox,
                 )
             )
-            if next_url is None:
-                break
             # The live service can fall back from a cursor link to offset=1 at the
             # end of a page, repeating records. Absolute offsets preserve the original
             # filters and avoid mixing pagination modes; sortby is not supported here.
-            offset += len(features)
+            offset += PAGE_SIZE
         return assets
 
     def fetch(self, asset: Asset, dest: Path) -> Path: