forcingkit 0.6.0__tar.gz → 0.7.1__tar.gz

{forcingkit-0.6.0 → forcingkit-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: forcingkit
-Version: 0.6.0
+Version: 0.7.1
 Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
 Project-URL: Documentation, https://forcingkit.docs.lhzn.io
 Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues

{forcingkit-0.6.0 → forcingkit-0.7.1}/docs/source/atmospheric_forcing.rst

@@ -147,6 +147,27 @@ coastal domains from about 15 to 85 km across. Hourly radiation and precipitatio
 *Limits.* CONUS only. A forecast product, not a reanalysis: it carries forecast-model biases and is
 not homogeneous across HRRR versions (v4 since December 2020). Radiation is instantaneous.
 
+HRRR forecasts at points
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+``forcingkit.fetchers.hrrr_points.fetch_points(cycles, leads, points, fields)`` returns HRRR forecasts
+at caller-supplied points for any cycles and lead hours, one row per (issue time, valid time, lead,
+point, field, value). Keeping both times makes the record bitemporal: a forecast can be compared
+with what was later observed, by how far ahead it was made. Fields are 10 m wind (rotated to east
+and north), gust, 2 m temperature and dew point, sea-level pressure and visibility, in the units
+HRRR publishes. Each point takes an inverse-distance-squared mean of up to four HRRR water cells
+within 6 km, so a coastal point never takes a land cell's temperature or wind.
+
+Leads a cycle does not carry are skipped (HRRR v4 runs to 48 h at 00, 06, 12 and 18 UTC and to
+18 h otherwise); a missing field in a published file raises. Three leads of seven fields at four
+points took 28 s on a warm connection, so one cycle's leads 1 to 18 take about three minutes.
+
+.. code-block:: python
+
+   from forcingkit.fetchers.hrrr_points import fetch_points
+
+   df = fetch_points(["2026-10-14T08:00"], [1, 6, 18], {"buoy": (-73.58, 40.96)})
+
 ERA5 (fallback)
 ~~~~~~~~~~~~~~~
 

{forcingkit-0.6.0 → forcingkit-0.7.1}/docs/source/conf.py

@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
 project = "forcingkit"
 copyright = "2026, Long Horizon Observatory"
 author = "Daniel Fry"
-release = "0.6.0"
+release = "0.7.1"
 
 extensions = [
     "sphinx.ext.autodoc",

{forcingkit-0.6.0 → forcingkit-0.7.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "forcingkit"
-version = "0.6.0"
+version = "0.7.1"
 description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
 readme = "README.md"
 requires-python = ">=3.11"

{forcingkit-0.6.0 → forcingkit-0.7.1}/src/forcingkit/fetchers/hrrr.py

@@ -57,16 +57,13 @@ def _fetch_s3_byte_ranges(
     Download specific byte ranges from an S3 object and concatenate into a
     single valid GRIB2 file. Each GRIB2 message is self-contained, so
     concatenating selected messages produces a valid file.
+
+    Each range is one ranged GET of exactly its bytes. A buffered file
+    (``fs.open`` then ``seek`` and ``read``) fetches whole blocks with
+    read-ahead, which for 1 to 2 MB messages downloaded about 38 times the
+    bytes needed (4.0 GB for 106 MB of messages in one HRRR cycle).
     """
     with open(output_path, "wb") as f:
         for start, end in ranges:
-            if end is not None:
-                length = end - start
-                with fs.open(s3_path, "rb") as s3f:
-                    s3f.seek(start)
-                    f.write(s3f.read(length))
-            else:
-                # Read from start to EOF
-                with fs.open(s3_path, "rb") as s3f:
-                    s3f.seek(start)
-                    f.write(s3f.read())
+            # end=None reads to the end of the object (the file's last message)
+            f.write(fs.cat_file(s3_path, start=start, end=end))

forcingkit-0.7.1/src/forcingkit/fetchers/hrrr_points.py

@@ -0,0 +1,224 @@
+"""HRRR forecasts at points, kept with both times: when each forecast was issued and the time it is for.
+
+For each requested cycle (the issue time) and lead hour, the surface fields are read by byte range from that
+cycle's `wrfsfcfNN` file on the anonymous AWS bucket (`noaa-hrrr-bdp-pds`, which holds every run since 2014-07-30),
+and the value at each point is taken from the nearest HRRR water cells: an inverse-distance-squared mean of up to
+four water cells within `max_km`, so a coastal point never takes a land cell's temperature or wind. Winds are
+rotated from the Lambert grid axes to east and north. Values stay in the units HRRR publishes (m/s, K, Pa, m).
+
+The result is one row per (issue_time, valid_time, lead_h, point, field, value): a bitemporal record, so a
+forecast's error can later be measured by how far ahead it was made. A lead a cycle does not carry (HRRR runs to
+48 h only at 00, 06, 12 and 18 UTC) is skipped; a missing field in a file that exists is an error.
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import os
+import tempfile
+from typing import Iterable
+
+import numpy as np
+import pandas as pd
+
+from .hrrr import _fetch_s3_byte_ranges, _parse_idx, fs
+from .hrrr_atmosphere import _read_message, earth_relative_winds
+
+logger = logging.getLogger(__name__)
+
+# GRIB variable and level of each field in wrfsfcfNN.
+FIELDS = {
+    "u10": ("UGRD", "10 m above ground"),
+    "v10": ("VGRD", "10 m above ground"),
+    "gust": ("GUST", "surface"),
+    "t2m": ("TMP", "2 m above ground"),
+    "d2m": ("DPT", "2 m above ground"),
+    "mslp": ("MSLMA", "mean sea level"),
+    "vis": ("VIS", "surface"),
+}
+DEFAULT_FIELDS = tuple(FIELDS)
+UNITS = {
+    "u10": "m s-1",
+    "v10": "m s-1",
+    "gust": "m s-1",
+    "t2m": "K",
+    "d2m": "K",
+    "mslp": "Pa",
+    "vis": "m",
+}
+
+
+def s3_key(cycle: pd.Timestamp, lead: int) -> str:
+    return f"noaa-hrrr-bdp-pds/hrrr.{cycle.strftime('%Y%m%d')}/conus/hrrr.t{cycle.strftime('%H')}z.wrfsfcf{lead:02d}.grib2"
+
+
+def idx_pattern(field: str, lead: int) -> str:
+    """The .idx text of one field at one lead: "anl" at 0, "N hour fcst" after (instantaneous fields only)."""
+    var, level = FIELDS[field]
+    return f":{var}:{level}:{'anl' if lead == 0 else f'{lead} hour fcst'}:"
+
+
+def max_lead(cycle: pd.Timestamp) -> int:
+    """The longest lead a cycle carries: 48 h at 00, 06, 12 and 18 UTC (36 h before HRRR v4, 2020-12-02), else 18 h."""
+    if cycle.hour % 6:
+        return 18
+    return 48 if cycle >= pd.Timestamp("2020-12-02") else 36
+
+
+class PointWeights:
+    """Inverse-distance-squared weights from the nearest water cells to each point, built once per grid."""
+
+    def __init__(
+        self,
+        lon,
+        lat,
+        land,
+        points: dict[str, tuple[float, float]],
+        k: int = 4,
+        max_km: float = 6.0,
+    ):
+        lon = ((np.asarray(lon, dtype=float) + 180.0) % 360.0) - 180.0
+        lat = np.asarray(lat, dtype=float)
+        water = np.asarray(land) < 0.5
+        self.names = list(points)
+        self.cells: list[np.ndarray] = []
+        self.weights: list[np.ndarray] = []
+        flat_lon, flat_lat, flat_water = lon.ravel(), lat.ravel(), water.ravel()
+        for name in self.names:
+            plon, plat = points[name]
+            dx = (flat_lon - plon) * 111.32 * np.cos(np.deg2rad(plat))
+            dy = (flat_lat - plat) * 110.57
+            d = np.hypot(dx, dy)
+            d[~flat_water] = np.inf
+            near = np.argsort(d)[:k]
+            near = near[d[near] <= max_km]
+            if not len(near):
+                logger.warning(
+                    "HRRR points: no water cell within %.0f km of %s", max_km, name
+                )
+                self.cells.append(np.array([], dtype=int))
+                self.weights.append(np.array([]))
+                continue
+            dist = d[near]
+            if dist[0] < 1e-6:  # on a cell centre
+                w = np.zeros(len(near))
+                w[0] = 1.0
+            else:
+                w = 1.0 / dist**2
+            self.cells.append(near)
+            self.weights.append(w / w.sum())
+
+    def __call__(self, field) -> dict[str, float]:
+        flat = np.asarray(field, dtype=float).ravel()
+        return {
+            name: float(np.dot(flat[c], w)) if len(c) else float("nan")
+            for name, c, w in zip(self.names, self.cells, self.weights)
+        }
+
+
+def _fetch_lead(cycle: pd.Timestamp, lead: int, fields: Iterable[str], workdir: str):
+    """The requested fields (and the land mask) of one cycle and lead, on HRRR's grid; None if the file is absent."""
+    key = s3_key(cycle, lead)
+    try:
+        with fs.open(key + ".idx", "r") as f:
+            idx = f.read()
+    except FileNotFoundError:
+        return None
+    patterns = {name: idx_pattern(name, lead) for name in fields} | {
+        "land": ":LAND:surface:"
+    }
+    out = {}
+    for name, pattern in patterns.items():
+        ranges = _parse_idx(idx, (pattern,))
+        if name == "land":
+            ranges = ranges[:1]
+        if len(ranges) != 1:
+            raise FileNotFoundError(
+                f"HRRR s3://{key}: expected one message for {pattern!r}, found {len(ranges)}"
+            )
+        path = os.path.join(
+            workdir, f"{name}_{cycle.strftime('%Y%m%d%H')}_f{lead:02d}.grib2"
+        )
+        _fetch_s3_byte_ranges(key, ranges, path)
+        try:
+            out[name] = _read_message(path)
+        finally:
+            os.remove(path)
+    return out
+
+
+_weights_cache: dict = {}
+
+
+def _weights(native, points, k, max_km) -> PointWeights:
+    u = native[next(iter(n for n in native if n != "land"))]
+    key = (u.shape, tuple(sorted(points.items())), k, max_km)
+    if key not in _weights_cache:
+        _weights_cache[key] = PointWeights(
+            u["longitude"].values,
+            u["latitude"].values,
+            native["land"].values,
+            points,
+            k,
+            max_km,
+        )
+    return _weights_cache[key]
+
+
+def fetch_points(
+    cycles: Iterable,
+    leads: Iterable[int],
+    points: dict[str, tuple[float, float]],
+    fields: Iterable[str] = DEFAULT_FIELDS,
+    k: int = 4,
+    max_km: float = 6.0,
+    max_workers: int = 6,
+) -> pd.DataFrame:
+    """HRRR forecasts at `points` ({name: (lon, lat)}) for every cycle and lead, as rows of
+    (issue_time, valid_time, lead_h, point, field, value). Leads past a cycle's `max_lead` are skipped."""
+    fields = tuple(fields)
+    unknown = set(fields) - set(FIELDS)
+    if unknown:
+        raise ValueError(f"unknown HRRR point fields: {sorted(unknown)}")
+    wind = "u10" in fields and "v10" in fields
+    tasks = []
+    for c in cycles:
+        cycle = pd.Timestamp(c)
+        if cycle.tzinfo is not None:
+            cycle = cycle.tz_convert("UTC").tz_localize(None)
+        tasks += [(cycle, int(lead)) for lead in leads if int(lead) <= max_lead(cycle)]
+    rows = []
+    with (
+        tempfile.TemporaryDirectory() as workdir,
+        concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as ex,
+    ):
+        for (cycle, lead), native in zip(
+            tasks, ex.map(lambda t: _fetch_lead(t[0], t[1], fields, workdir), tasks)
+        ):
+            if native is None:
+                logger.info("HRRR %s f%02d not published; skipped", cycle, lead)
+                continue
+            w = _weights(native, points, k, max_km)
+            values = {name: w(native[name].values) for name in fields}
+            if wind:
+                a = native["u10"].attrs
+                if int(a.get("GRIB_uvRelativeToGrid", 1)) == 1:
+                    cone = float(np.sin(np.deg2rad(a["GRIB_Latin1InDegrees"])))
+                    lov = float(a["GRIB_LoVInDegrees"])
+                    for p in points:
+                        u, v = earth_relative_winds(
+                            np.array([values["u10"][p]]),
+                            np.array([values["v10"][p]]),
+                            np.array([points[p][0]]),
+                            cone,
+                            lov,
+                        )
+                        values["u10"][p], values["v10"][p] = float(u[0]), float(v[0])
+            valid = cycle + pd.Timedelta(hours=lead)
+            for name in fields:
+                for p, v in values[name].items():
+                    rows.append((cycle, valid, lead, p, name, v))
+    return pd.DataFrame(
+        rows, columns=["issue_time", "valid_time", "lead_h", "point", "field", "value"]
+    )

forcingkit-0.7.1/tests/unit/test_hrrr_points.py

@@ -0,0 +1,134 @@
+"""HRRR at points: .idx patterns, leads per cycle, water-cell weights, wind rotation and the bitemporal rows. No
+network."""
+
+import numpy as np
+import pandas as pd
+import pytest
+import xarray as xr
+
+from forcingkit.fetchers import hrrr_points as hp
+
+LOV = 262.5
+
+
+def test_idx_patterns_name_the_lead():
+    assert hp.idx_pattern("u10", 0) == ":UGRD:10 m above ground:anl:"
+    assert hp.idx_pattern("u10", 6) == ":UGRD:10 m above ground:6 hour fcst:"
+    assert hp.idx_pattern("vis", 18) == ":VIS:surface:18 hour fcst:"
+
+
+def test_leads_per_cycle():
+    assert hp.max_lead(pd.Timestamp("2026-10-14T06:00")) == 48
+    assert hp.max_lead(pd.Timestamp("2026-10-14T07:00")) == 18
+    assert hp.max_lead(pd.Timestamp("2019-06-01T12:00")) == 36
+
+
+def test_key_for_a_cycle_and_lead():
+    assert hp.s3_key(pd.Timestamp("2026-10-14T07:00"), 3).endswith(
+        "hrrr.20261014/conus/hrrr.t07z.wrfsfcf03.grib2"
+    )
+
+
+def _grid():
+    j, i = np.meshgrid(np.arange(30), np.arange(30), indexing="ij")
+    lon = 286.0 + 0.03 * i  # 0..360 longitudes, as HRRR's
+    lat = 40.5 + 0.03 * j
+    land = (i < 10).astype(float)  # the western third is land
+    return lon, lat, land
+
+
+def test_weights_take_water_cells_only():
+    lon, lat, land = _grid()
+    # A point on the coast: its nearest cells are land, so the weights must reach east to water.
+    p = {"coast": (-74.0 + 0.03 * 9, 40.5 + 0.03 * 15)}
+    w = hp.PointWeights(lon, lat, land, p)
+    field = np.where(land > 0.5, -999.0, 7.0)
+    assert w(field)["coast"] == pytest.approx(7.0)
+
+
+def test_weights_are_exact_on_a_cell_and_linear_fields_stay_close():
+    lon, lat, land = _grid()
+    on = {"on": (-74.0 + 0.03 * 20, 40.5 + 0.03 * 12)}
+    w = hp.PointWeights(lon, lat, land, on)
+    field = 3.0 * (lon - 360.0) + 2.0 * lat
+    assert w(field)["on"] == pytest.approx(3.0 * on["on"][0] + 2.0 * on["on"][1])
+
+
+def test_no_water_within_reach_gives_nan():
+    lon, lat, _ = _grid()
+    w = hp.PointWeights(lon, lat, np.ones_like(lon), {"inland": (-73.7, 40.9)})
+    assert np.isnan(w(np.zeros_like(lon))["inland"])
+
+
+def _fake_native(cycle, lead, fields, workdir, grid_relative=0):
+    lon, lat, land = _grid()
+    out = {}
+    for n, name in enumerate(fields):
+        out[name] = xr.DataArray(
+            np.full(lon.shape, float(n + lead)),
+            dims=("y", "x"),
+            coords={"longitude": (("y", "x"), lon), "latitude": (("y", "x"), lat)},
+            attrs={
+                "GRIB_Latin1InDegrees": 38.5,
+                "GRIB_LoVInDegrees": LOV,
+                "GRIB_uvRelativeToGrid": grid_relative,
+            },
+        )
+    out["land"] = xr.DataArray(land, dims=("y", "x"))
+    return out
+
+
+def test_rows_are_bitemporal_and_skip_leads_a_cycle_lacks(monkeypatch):
+    monkeypatch.setattr(hp, "_fetch_lead", _fake_native)
+    hp._weights_cache.clear()
+    df = hp.fetch_points(
+        ["2026-10-14T07:00"],
+        [1, 18, 24],
+        {"buoy": (-73.4, 40.8)},
+        fields=("t2m", "vis"),
+    )
+    # f24 is past a 07Z cycle's 18 h: skipped.
+    assert sorted(df["lead_h"].unique()) == [1, 18]
+    row = df[(df.lead_h == 18) & (df.field == "vis")].iloc[0]
+    assert row.issue_time == pd.Timestamp("2026-10-14T07:00")
+    assert row.valid_time == pd.Timestamp("2026-10-15T01:00")
+    assert row.value == pytest.approx(1 + 18)  # second field, offset by the lead
+
+
+def test_an_absent_file_is_skipped(monkeypatch):
+    monkeypatch.setattr(
+        hp,
+        "_fetch_lead",
+        lambda c, lead, f, w: None if lead == 2 else _fake_native(c, lead, f, w),
+    )
+    hp._weights_cache.clear()
+    df = hp.fetch_points(
+        ["2026-10-14T07:00"], [1, 2], {"buoy": (-73.4, 40.8)}, fields=("t2m",)
+    )
+    assert list(df["lead_h"].unique()) == [1]
+
+
+def test_grid_relative_winds_are_rotated(monkeypatch):
+    # A grid-relative wind along +x at 73.4 W is rotated by cone * (lon - lov) to east and north.
+    monkeypatch.setattr(
+        hp,
+        "_fetch_lead",
+        lambda c, lead, f, w: _fake_native(c, lead, f, w, grid_relative=1),
+    )
+    hp._weights_cache.clear()
+    df = hp.fetch_points(
+        ["2026-10-14T07:00"], [1], {"buoy": (-73.4, 40.8)}, fields=("u10", "v10")
+    )
+    u = df[df.field == "u10"].value.iloc[0]
+    v = df[df.field == "v10"].value.iloc[0]
+    # The fake gives grid u = 0 + 1 and grid v = 1 + 1 (field order plus lead).
+    alpha = np.deg2rad(np.sin(np.deg2rad(38.5)) * (-73.4 - (LOV - 360.0)))
+    assert u == pytest.approx(np.cos(alpha) * 1.0 + np.sin(alpha) * 2.0)
+    assert v == pytest.approx(-np.sin(alpha) * 1.0 + np.cos(alpha) * 2.0)
+
+
+def test_unknown_fields_are_refused():
+    with pytest.raises(ValueError):
+        hp.fetch_points(
+            ["2026-10-14T07:00"], [1], {"b": (-73.4, 40.8)}, fields=("cape",)
+        )

{forcingkit-0.6.0 → forcingkit-0.7.1}/uv.lock

@@ -873,7 +873,7 @@ wheels = [
 
 [[package]]
 name = "forcingkit"
-version = "0.6.0"
+version = "0.7.1"
 source = { editable = "." }
 dependencies = [
     { name = "bottleneck" },