forcingkit 0.5.0__tar.gz → 0.6.0__tar.gz

{forcingkit-0.5.0 → forcingkit-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: forcingkit
-Version: 0.5.0
+Version: 0.6.0
 Summary: Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance.
 Project-URL: Documentation, https://forcingkit.docs.lhzn.io
 Project-URL: Issues, https://github.com/lhzn-io/forcingkit/issues

{forcingkit-0.5.0 → forcingkit-0.6.0}/docs/source/conf.py

@@ -9,7 +9,7 @@ sys.path.insert(0, os.path.abspath("../../src"))
 project = "forcingkit"
 copyright = "2026, Long Horizon Observatory"
 author = "Daniel Fry"
-release = "0.5.0"
+release = "0.6.0"
 
 extensions = [
     "sphinx.ext.autodoc",

{forcingkit-0.5.0 → forcingkit-0.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "forcingkit"
-version = "0.5.0"
+version = "0.6.0"
 description = "Spatiotemporal forcing for computational Earth-system models: selects, regrids and serves model-ready time series with provenance."
 readme = "README.md"
 requires-python = ">=3.11"

{forcingkit-0.5.0 → forcingkit-0.6.0}/src/forcingkit/fetchers/coastwatch.py

@@ -292,6 +292,16 @@ def latest_time(product_id: str) -> pd.Timestamp | None:
     return _utc(lines[-1]) if len(lines) >= 3 else None  # header, units row, value
 
 
+def first_time(product_id: str) -> pd.Timestamp | None:
+    """The product's earliest time step (the near-real-time sectors keep a rolling window of a few months)."""
+    p = PRODUCTS[product_id]
+    resp = _get(f"{p.server}/griddap/{p.dataset}.csv?time[0]")
+    if resp is None:
+        return None
+    lines = [line for line in resp.text.strip().splitlines() if line]
+    return _utc(lines[-1]) if len(lines) >= 3 else None
+
+
 def fetch(
     product_id: str, bbox: list[float], start: datetime, end: datetime
 ) -> xr.Dataset:
@@ -300,8 +310,18 @@ def fetch(
     The window is trimmed to the product's latest time step. An empty window returns a Dataset with a zero-length
     time axis rather than raising.
     """
+    return _fetch_window(product_id, bbox, start, end, latest_time(product_id))
+
+
+def _fetch_window(
+    product_id: str,
+    bbox: list[float],
+    start: datetime,
+    end: datetime,
+    last: pd.Timestamp | None,
+) -> xr.Dataset:
+    """`fetch` with the product's latest time already known (None: the product has no data)."""
     p = PRODUCTS[product_id]
-    last = latest_time(product_id)
     if last is not None and _utc(end) > last:
         end = last.to_pydatetime()
     resp = (
@@ -428,6 +448,155 @@ def composite(
     return out
 
 
+def fetch_range(
+    product_id: str,
+    bbox: list[float],
+    start: datetime,
+    end: datetime,
+    chunk_days: int = 10,
+) -> xr.Dataset:
+    """`fetch` over a long window in chunks of `chunk_days`, so each ERDDAP response stays small (the servers'
+    proxies refuse long requests); the chunks are joined on time.
+
+    The product's time range is looked up once, and the window is clamped to it (the near-real-time sectors keep
+    only a few months). A chunk that still fails after its retries is logged and skipped, so one bad request costs
+    those days rather than the whole series. Raises only when the time range itself cannot be read.
+    """
+    last = latest_time(product_id)
+    first = first_time(product_id) if last is not None else None
+    t0 = max(_utc(start), first) if first is not None else _utc(start)
+    t1 = min(_utc(end), last) if last is not None else _utc(end)
+    parts = []
+    while last is not None and t0 <= t1:
+        t_end = min(t1, t0 + pd.Timedelta(days=chunk_days) - pd.Timedelta(seconds=1))
+        try:
+            ds = _fetch_window(
+                product_id, bbox, t0.to_pydatetime(), t_end.to_pydatetime(), last
+            )
+        except Exception as exc:
+            logger.warning(
+                "satellite chunk skipped for %s, %s to %s: %s",
+                product_id,
+                _iso(t0),
+                _iso(t_end),
+                exc,
+            )
+        else:
+            if ds.sizes.get("time", 0):
+                parts.append(ds)
+        t0 = t_end + pd.Timedelta(seconds=1)
+    if not parts:
+        return _fetch_window(
+            product_id,
+            bbox,
+            _utc(end).to_pydatetime(),
+            _utc(start).to_pydatetime(),
+            None,
+        )  # empty
+    out = xr.concat(parts, dim="time") if len(parts) > 1 else parts[0]
+    _, keep = np.unique(out["time"].values, return_index=True)
+    return out.isel(time=np.sort(keep))
+
+
+def composite_series(
+    product_ids: list[str],
+    bbox: list[float],
+    frame_ends: list[datetime],
+    days: int = 7,
+) -> xr.Dataset:
+    """`composite` at each of `frame_ends` (ascending), from one fetch per product: for every frame, each pixel's
+    most recent valid value across the products within `days` before the frame's end, with its age in days.
+
+    Returns a Dataset with `value` and `age_days` (time, lat, lon; NaN where no pixel was valid within the window),
+    `time` being the frame ends, on the first available product's grid. Sensors that fail are skipped, as in
+    `composite`; a frame with no valid pixel is all NaN rather than an error.
+    """
+    if not frame_ends:
+        raise ValueError("no frame ends given")
+    ends = [_utc(t) for t in frame_ends]
+    if any(b <= a for a, b in zip(ends, ends[1:])):
+        raise ValueError("frame_ends must be strictly ascending")
+    start = ends[0] - pd.Timedelta(days=days)
+    slices: list[tuple[np.datetime64, int, np.ndarray]] = []
+    used: list[str] = []
+    lat = lon = None
+    name = PRODUCTS[product_ids[0]].name
+    units = ""
+    for pid in product_ids:
+        try:
+            ds = fetch_range(pid, bbox, start.to_pydatetime(), ends[-1].to_pydatetime())
+        except Exception as exc:  # one sensor's outage should not drop the others
+            logger.warning("satellite fetch failed for %s: %s", pid, exc)
+            continue
+        if ds.sizes.get("time", 0) == 0:
+            continue
+        da = ds[PRODUCTS[pid].name]
+        if lat is None or lon is None:
+            lat, lon = da["lat"].values, da["lon"].values
+            units = da.attrs.get("units", "")
+        elif not (
+            np.array_equal(da["lat"].values, lat)
+            and np.array_equal(da["lon"].values, lon)
+        ):
+            dlat = float(np.abs(np.diff(lat)).mean()) if lat.size > 1 else 0.01
+            da = da.reindex(lat=lat, lon=lon, method="nearest", tolerance=dlat / 2)
+        k = len(used)
+        used.append(pid)
+        for ti in range(da.sizes["time"]):
+            slices.append((_naive64(da["time"].values[ti]), k, da.isel(time=ti).values))
+    if lat is None or lon is None:
+        raise ValueError(
+            f"no valid data for {product_ids} between {_iso(start)} and {_iso(ends[-1])}"
+        )
+    slices.sort(key=lambda item: (item[0], item[1]))
+    shape = (lat.size, lon.size)
+    best = np.full(shape, np.nan, dtype=np.float32)
+    best_t = np.full(shape, np.datetime64("NaT"), dtype="datetime64[ns]")
+    values = np.full((len(ends),) + shape, np.nan, dtype=np.float32)
+    ages = np.full((len(ends),) + shape, np.nan, dtype=np.float32)
+    window = np.timedelta64(days * 86400, "s")
+    i = 0
+    for f, end in enumerate(ends):
+        t_end = _naive64(end)
+        while i < len(slices) and slices[i][0] <= t_end:
+            t, _, v = slices[i]
+            newer = np.isfinite(v) & (np.isnat(best_t) | (best_t <= t))
+            best[newer] = v[newer]
+            best_t[newer] = t
+            i += 1
+        age = t_end - best_t
+        ok = ~np.isnat(best_t) & (age <= window)
+        values[f][ok] = best[ok]
+        ages[f][ok] = (
+            age[ok].astype("timedelta64[s]").astype(np.float64) / 86400.0
+        ).astype(np.float32)
+    times = pd.DatetimeIndex([e.tz_localize(None) for e in ends])
+    return xr.Dataset(
+        {
+            "value": (
+                ("time", "lat", "lon"),
+                values,
+                {"units": units, "long_name": name},
+            ),
+            "age_days": (
+                ("time", "lat", "lon"),
+                ages,
+                {
+                    "units": "days",
+                    "long_name": "age of the pixel's value at the frame's end",
+                },
+            ),
+        },
+        coords={"time": times, "lat": lat, "lon": lon},
+        attrs={
+            "quantity": name,
+            "products": ",".join(used),
+            "attribution": "; ".join(sorted({PRODUCTS[p].attribution for p in used})),
+            "window_days": days,
+        },
+    )
+
+
 def mask_coast(ds: xr.Dataset, pixels: int = 1, var: str = "value") -> xr.Dataset:
     """Drop `pixels` of water along the shore, where land adjacency biases ocean colour.
 
@@ -439,7 +608,12 @@ def mask_coast(ds: xr.Dataset, pixels: int = 1, var: str = "value") -> xr.Datase
     if pixels <= 0:
         return ds
     water = np.isfinite(ds[var].values)
-    keep = binary_erosion(water, iterations=pixels, border_value=1)
+    if water.ndim == 3:  # a series: each frame on its own
+        keep = np.stack(
+            [binary_erosion(w, iterations=pixels, border_value=1) for w in water]
+        )
+    else:
+        keep = binary_erosion(water, iterations=pixels, border_value=1)
     out = ds.copy()
     for name in out.data_vars:
         if out[name].dims == ds[var].dims:

{forcingkit-0.5.0 → forcingkit-0.6.0}/tests/unit/test_coastwatch.py

@@ -75,6 +75,8 @@ class FakeERDDAP:
         times, values = self.data[ds_id]
         if "time[(last)]" in url:
             return FakeResponse(200, text=f"time\nUTC\n{times[-1]}\n")
+        if "time[0]" in url:
+            return FakeResponse(200, text=f"time\nUTC\n{times[0]}\n")
         var = url.split("?")[1].split("[")[0]
         return FakeResponse(200, content=griddap_nc(var, times, values))
 
@@ -220,3 +222,80 @@ def test_cache_key_is_stable_within_the_hour():
     b = coastwatch.cache_key(["sst_mur"], BBOX, datetime(2026, 10, 8, 12, 55), 7, 1)
     c = coastwatch.cache_key(["sst_mur"], BBOX, datetime(2026, 10, 8, 13, 5), 7, 1)
     assert a == b != c and a.startswith("sat_")
+
+
+def _three_days():
+    """Three daily passes: day 1 clear (1.0), day 2 cloudy in the first row (2.0 elsewhere), day 3 all cloud."""
+    v = np.stack(
+        [np.full((5, 5), 1.0), np.full((5, 5), 2.0), np.full((5, 5), np.nan)]
+    ).astype(np.float32)
+    v[1, 0, :] = np.nan
+    times = ["2026-10-01T12:00:00Z", "2026-10-02T12:00:00Z", "2026-10-03T12:00:00Z"]
+    return times, v
+
+
+def test_composite_series_rolls_the_window(erddap):
+    erddap({"noaacwNPPVIIRSchlaSectorVYDaily": _three_days()})
+    ends = [datetime(2026, 10, d, 23, 59, tzinfo=timezone.utc) for d in (1, 2, 3, 5)]
+    s = coastwatch.composite_series(["chl_viirs_snpp"], BBOX, ends, days=2)
+    assert s["value"].dims == ("time", "lat", "lon") and s.sizes["time"] == 4
+    value, age = s["value"].values, s["age_days"].values
+    # Latitude ascends, so the descending input's first row is the last row here.
+    assert value[0, 0, 0] == pytest.approx(1.0)  # Oct 1: the first pass
+    assert value[1, 0, 0] == pytest.approx(2.0) and value[1, -1, 0] == pytest.approx(
+        1.0
+    )  # Oct 2: cloud keeps day 1
+    assert age[1, -1, 0] == pytest.approx(1.0 + 11.98 / 24, abs=1e-3)
+    assert value[2, 0, 0] == pytest.approx(
+        2.0
+    )  # Oct 3 is all cloud: Oct 2 carries over
+    assert np.isnan(value[3]).all()  # Oct 5: nothing within 2 days
+    assert np.isnan(
+        value[2, -1, 0]
+    )  # Oct 3: the cloudy row's Oct 1 value is 2.5 days old, outside the window
+
+
+def test_composite_series_rejects_unsorted_ends(erddap):
+    erddap({"noaacwNPPVIIRSchlaSectorVYDaily": _three_days()})
+    ends = [
+        datetime(2026, 10, 3, tzinfo=timezone.utc),
+        datetime(2026, 10, 2, tzinfo=timezone.utc),
+    ]
+    with pytest.raises(ValueError):
+        coastwatch.composite_series(["chl_viirs_snpp"], BBOX, ends, days=2)
+
+
+def test_mask_coast_on_a_series():
+    v = np.ones((2, 5, 5), dtype=np.float32)
+    v[:, :, 0] = np.nan  # land on the west edge
+    ds = xr.Dataset({"value": (("time", "lat", "lon"), v)})
+    m = coastwatch.mask_coast(ds, pixels=1)
+    assert np.isnan(m["value"].values[:, :, 1]).all()  # the band next to land
+    assert np.isfinite(m["value"].values[:, 2:4, 2:4]).all()
+
+
+def test_fetch_range_skips_a_failing_chunk(erddap, monkeypatch):
+    erddap({"noaacwNPPVIIRSchlaSectorVYDaily": _three_days()})
+    real = coastwatch._fetch_window
+    calls = {"n": 0}
+
+    def flaky(*args, **kwargs):
+        calls["n"] += 1
+        if calls["n"] == 1:
+            raise RuntimeError("ERDDAP: giving up after 3 attempts")
+        return real(*args, **kwargs)
+
+    monkeypatch.setattr(coastwatch, "_fetch_window", flaky)
+    ds = coastwatch.fetch_range(
+        "chl_viirs_snpp",
+        BBOX,
+        datetime(2026, 9, 1, tzinfo=timezone.utc),
+        datetime(2026, 10, 3, 23, tzinfo=timezone.utc),
+        chunk_days=1,
+    )
+    assert (
+        calls["n"] == 3
+    )  # the window is clamped to the archive (Oct 1 to 3): three 1-day chunks, all tried
+    assert (
+        ds.sizes["time"] == 3
+    )  # the later chunks still delivered (the fake serves every pass to each)

{forcingkit-0.5.0 → forcingkit-0.6.0}/uv.lock

@@ -873,7 +873,7 @@ wheels = [
 
 [[package]]
 name = "forcingkit"
-version = "0.5.0"
+version = "0.6.0"
 source = { editable = "." }
 dependencies = [
     { name = "bottleneck" },