acmadDL 0.2.0__py3-none-any.whl

acmaddl/__init__.py

@@ -0,0 +1,61 @@
+"""acmadDL — federated data integration for seasonal climate forecasting.
+
+Provides a unified fetch() API across CDS, OPeNDAP, NCEI, S3, HTTP, and
+Sheerwater data sources. All outputs are CF-aligned xarray Datasets.
+
+Quick start:
+    import acmaddl
+    ds = acmaddl.fetch("c3s/ecmwf", variable="precip", init="2025-02",
+                       target="MAM", region=[-12, 6, 28, 42], hindcast=(2010, 2015))
+"""
+
+import os as _os
+from pathlib import Path as _Path
+
+# Pin nuthatch's cache to a local directory at import time.
+#
+# nuthatch discovers its config by walking *up* the filesystem from the calling
+# module and adopting the first pyproject.toml / nuthatch.toml it finds. When
+# acmaddl is pip-installed, that search leaves site-packages/acmaddl/ and can pick
+# up a nuthatch.toml shipped by a co-installed package — notably `sheerwater`, which
+# points the cache root at a private GCS bucket (gs://sheerwater-datalake/...).
+# Anyone without those credentials then hits a 401 / interactive prompt on their
+# first fetch(). We defend against that on two fronts:
+#   1. Ship src/acmaddl/nuthatch.toml (see there) so the upward search finds *our*
+#      local config first and never reaches the ambient one.
+#   2. Pin the root/local cache filesystem here via the standard NUTHATCH_* env
+#      vars, since an installed package's own config is demoted to a "mirror" and
+#      cannot set the actual root.
+# The file:// prefix is required: without it fsspec's split_protocol() mis-parses
+# the '://' that adapter source URLs embed in cache-key paths. setdefault() means
+# ACMADDL_CACHE_DIR, the NUTHATCH_* env vars, and ~/.nuthatch.toml all still win.
+from ._paths import env_with_legacy as _env_with_legacy
+
+_cache_dir = _Path(
+    _env_with_legacy("ACMADDL_CACHE_DIR") or _Path.home() / ".nuthatch" / "caches"
+).expanduser()
+_cache_uri = f"file://{_cache_dir}"
+_os.environ.setdefault("NUTHATCH_ROOT_FILESYSTEM", _cache_uri)
+_os.environ.setdefault("NUTHATCH_LOCAL_FILESYSTEM", _cache_uri)
+
+from . import catalog
+from .errors import VariableNotSupported
+from .fetch import fetch, parse_target, parse_init, season_to_months
+from .cpc_nmme import cpc_nmme_predictor  # noqa: F401
+from .assemble import assemble, obs_predictor
+from .zonal import zonal
+from .health import check_product, check_all_products
+
+__all__ = [
+    "catalog",
+    "fetch",
+    "VariableNotSupported",
+    "zonal",
+    "parse_target",
+    "season_to_months",
+    "parse_init",
+    "assemble",
+    "obs_predictor",
+    "check_product",
+    "check_all_products",
+]

acmaddl/_paths.py

@@ -0,0 +1,51 @@
+"""Filesystem locations used by acmaddl adapters."""
+import os
+import warnings
+
+# Pre-rename spellings of the env vars below. The package was renamed
+# rosetta -> acmaddl in 0.2.0; the old names are still honored so an exported
+# ROSETTA_CACHE_DIR does not silently relocate an existing cache.
+_LEGACY_ENV = {
+    "ACMADDL_CACHE_DIR": "ROSETTA_CACHE_DIR",
+    "ACMADDL_TMP_DIR": "ROSETTA_TMP_DIR",
+}
+
+
+def env_with_legacy(name: str) -> str | None:
+    """Read ``name``, falling back to its deprecated ``ROSETTA_*`` spelling.
+
+    Returns ``None`` if neither is set. Reading the legacy name emits a
+    ``DeprecationWarning`` — it still works, but it is on its way out.
+    """
+    value = os.environ.get(name)
+    if value:
+        return value
+    legacy = _LEGACY_ENV[name]
+    value = os.environ.get(legacy)
+    if value:
+        warnings.warn(
+            f"{legacy} is deprecated and will be removed in a future release; "
+            f"use {name} instead.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+    return value or None
+
+
+def get_tmpdir() -> str:
+    """Return the directory adapters should use for scratch downloads.
+
+    Defaults to ``~/.nuthatch/acmaddl/_tmp/`` (co-located with the nuthatch
+    cache so scratch and cache live in the same tree). Override via the
+    ``ACMADDL_TMP_DIR`` environment variable (or the deprecated
+    ``ROSETTA_TMP_DIR``).
+
+    Adapters MUST use this rather than the system tempdir: macOS reaps
+    ``/var/folders/.../T/`` on its own schedule, which broke cache entries
+    that pickled lazy datasets referencing those temp files (issue #24).
+    """
+    path = env_with_legacy("ACMADDL_TMP_DIR") or os.path.expanduser(
+        "~/.nuthatch/acmaddl/_tmp"
+    )
+    os.makedirs(path, exist_ok=True)
+    return path

acmaddl/adapters/__init__.py

@@ -0,0 +1,34 @@
+from .cds import CDSAdapter
+from .opendap import OPeNDAPAdapter
+from .http import HTTPAdapter
+from .iridl import IRIDLAdapter
+from .mars import MARSAdapter
+from .ncei import NCEIAdapter
+from .s3 import S3Adapter
+from .sheerwater import SheerwaterAdapter
+from .ccsr import CCSRAdapter
+from .cpc_binary import CPCBinaryAdapter
+# IcechunkAdapter imports the `icechunk` package lazily (inside its methods), so
+# registering it here is safe without that optional dependency installed; a
+# fetch only needs it when an icechunk-backed product is actually used.
+from .icechunk import IcechunkAdapter
+
+_ADAPTERS = {
+    "cds": CDSAdapter,
+    "opendap": OPeNDAPAdapter,
+    "http": HTTPAdapter,
+    "iridl": IRIDLAdapter,
+    "mars": MARSAdapter,
+    "ncei": NCEIAdapter,
+    "s3": S3Adapter,
+    "sheerwater": SheerwaterAdapter,
+    "ccsr": CCSRAdapter,
+    "cpc_binary": CPCBinaryAdapter,
+    "icechunk": IcechunkAdapter,
+}
+
+
+def get_adapter(name):
+    if name not in _ADAPTERS:
+        raise KeyError(f"Unknown adapter: {name}")
+    return _ADAPTERS[name]()

acmaddl/adapters/_issuance.py

@@ -0,0 +1,124 @@
+"""Enumerating the files of an issuance-keyed forecast archive.
+
+Observational archives are keyed by the time an observation describes. Forecast
+archives are keyed by *two* times: when the forecast was issued, and what it is
+about. CHC's CHIRPS-GEFS puts the first in the directory path and the second in
+the filename::
+
+    .../CHIRPS-GEFS/v3/daily/global/2026/07/05/c3g_2026.07.20.tif
+                                    ^^^^^^^^^^        ^^^^^^^^^^
+                                    issued 5 July     valid 20 July, lead 15
+
+Other archives put the lead number in the filename, or the init date in both
+places, or serve one accumulated file per issuance with no lead at all. Rather
+than a new adapter per layout, a catalog entry declares an ``issuance`` block
+and gets `strftime` templating over three names — ``init``, ``valid`` and
+``lead``::
+
+    issuance:
+      path_pattern: "{init:%Y}/{init:%m}/{init:%d}"
+      file_pattern: "c3g_{valid:%Y}.{valid:%m}.{valid:%d}.tif"
+      leads: [0, 15]
+      lead_units: days
+
+This module turns that declaration plus a list of issuance dates into the file
+list, and the file list back into the ``(init_time, lead_time)`` coordinates the
+rest of acmadDL expects. It knows nothing about HTTP, so an S3 or OPeNDAP
+archive with the same shape can reuse it.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+import numpy as np
+
+_LEAD_UNITS = {
+    "days": timedelta(days=1),
+    "hours": timedelta(hours=1),
+}
+
+
+@dataclass(frozen=True)
+class IssuanceFile:
+    """One remote file, and the coordinates it will occupy."""
+
+    url: str
+    init: datetime
+    lead: int
+
+
+def issuance_config(product_config: dict) -> dict | None:
+    """The validated ``issuance`` block, or None for a plain time-series product."""
+    block = product_config.get("issuance")
+    if block is None:
+        return None
+    if "file_pattern" not in block:
+        raise ValueError("issuance block requires 'file_pattern'")
+    leads = block.get("leads", [0, 0])
+    if len(leads) != 2 or leads[0] > leads[1]:
+        raise ValueError(
+            f"issuance 'leads' must be an inclusive [min, max] pair, got {leads!r}"
+        )
+    units = block.get("lead_units", "days")
+    if units not in _LEAD_UNITS:
+        raise ValueError(
+            f"issuance 'lead_units' must be one of {sorted(_LEAD_UNITS)}, got {units!r}"
+        )
+    return {
+        "path_pattern": block.get("path_pattern", ""),
+        "file_pattern": block["file_pattern"],
+        "leads": list(range(int(leads[0]), int(leads[1]) + 1)),
+        "lead_units": units,
+        "lead_step": _LEAD_UNITS[units],
+    }
+
+
+def parse_init_dates(init_dates) -> list[datetime]:
+    """Normalize whatever the caller passed as issuance dates to datetimes."""
+    parsed = []
+    for value in init_dates:
+        if isinstance(value, datetime):
+            parsed.append(value)
+        else:
+            text = str(value)
+            if len(text) != 10:
+                raise ValueError(
+                    f"issuance dates must be full YYYY-MM-DD dates, got {value!r}. "
+                    "An issuance-keyed forecast has a day, not just a month."
+                )
+            parsed.append(datetime.strptime(text, "%Y-%m-%d"))
+    return sorted(set(parsed))
+
+
+def enumerate_files(base_url: str, issuance: dict, init_dates) -> list[IssuanceFile]:
+    """Every (issuance, lead) file implied by the config, in a stable order."""
+    inits = parse_init_dates(init_dates)
+    if not inits:
+        raise ValueError("no issuance dates given")
+
+    step = issuance["lead_step"]
+    files = []
+    for init in inits:
+        for lead in issuance["leads"]:
+            valid = init + lead * step
+            fields = {"init": init, "valid": valid, "lead": lead}
+            parts = [base_url.rstrip("/")]
+            path = issuance["path_pattern"].format(**fields)
+            if path:
+                parts.append(path.strip("/"))
+            parts.append(issuance["file_pattern"].format(**fields))
+            files.append(IssuanceFile("/".join(parts), init, lead))
+    return files
+
+
+def lead_timedelta(leads, lead_units: str) -> np.ndarray:
+    """Lead numbers -> numpy timedeltas, so ``lead_time`` carries its own units.
+
+    An integer ``lead_time`` is ambiguous the moment two products disagree on
+    whether it counts days or hours; a timedelta cannot be misread.
+    """
+    unit = "D" if lead_units == "days" else "h"
+    return np.array([np.timedelta64(int(lead), unit) for lead in leads]).astype(
+        "timedelta64[ns]"
+    )

acmaddl/adapters/_robust.py

@@ -0,0 +1,152 @@
+"""Shared retry / rate-limit infrastructure for acmadDL adapters.
+
+Extracted from the HTTP adapter so that OPeNDAP and CCSR adapters can also
+retry transient network failures (e.g. ``OSError: NetCDF: I/O failure`` from
+a brief IRIDL hiccup) without duplicating logic.
+"""
+import random
+import re
+import threading
+import time
+
+_DEFAULT_MAX_RETRIES = 3
+_DEFAULT_RETRY_BACKOFF = 1.0
+
+
+class _RateLimiter:
+    """Thread-safe minimum-interval gate between requests.
+
+    Used to pace per-file opens so we don't blow through a server's
+    requests-per-second budget (CHIRPS rate-limits at >~60 requests/sec when
+    GDAL/vsicurl drives many range reads in quick succession).
+    """
+
+    def __init__(self, min_interval):
+        self.min_interval = float(min_interval or 0.0)
+        self._last = 0.0
+        self._lock = threading.Lock()
+
+    def wait(self):
+        if self.min_interval <= 0:
+            return
+        with self._lock:
+            elapsed = time.monotonic() - self._last
+            if elapsed < self.min_interval:
+                time.sleep(self.min_interval - elapsed)
+            self._last = time.monotonic()
+
+
+# HTTP 4xx ("client error") responses indicate a problem with our request
+# (URL doesn't exist, unauthorized, etc.) and won't fix themselves on retry.
+# 5xx and network errors are the ones worth retrying. GDAL/rasterio and
+# urllib surface them with different wording, so we match both.
+_PERMANENT_HTTP_PATTERNS = (
+    re.compile(r"HTTP response code: 4\d{2}"),    # rasterio / vsicurl
+    re.compile(r"HTTP Error 4\d{2}"),             # urllib
+)
+
+
+def _is_transient(exc):
+    """Return True if ``exc`` is worth retrying.
+
+    Any exception whose message does NOT match a known-permanent HTTP 4xx
+    pattern is considered transient — this covers ``OSError`` (netcdf I/O
+    failure), ``TimeoutError``, and server-side 5xx errors alike.
+    """
+    msg = str(exc)
+    for pat in _PERMANENT_HTTP_PATTERNS:
+        if pat.search(msg):
+            return False
+    return True
+
+
+def _with_retry(fn, max_retries, backoff, label, verbose=True, on_failure=None,
+                is_transient=_is_transient):
+    """Call fn(), retrying transient failures with exponential backoff + jitter.
+
+    Only retries when `is_transient(exc)` is True — by default that means
+    "anything except an HTTP 4xx response," since 4xx errors are permanent
+    (404 won't become 200 in 2 seconds). Jitter (random factor in [0.5, 1.5))
+    desynchronizes retries when many workers fail simultaneously against a
+    rate-limited server. `on_failure` runs after each failed attempt before
+    the sleep.
+
+    Covers both HTTP adapter downloads and OPeNDAP/CCSR ``xr.open_dataset``
+    opens, since ``OSError`` (e.g. ``NetCDF: I/O failure``) is not a 4xx and
+    is therefore treated as transient.
+    """
+    last_err = None
+    for attempt in range(max_retries + 1):
+        try:
+            return fn()
+        except Exception as e:
+            last_err = e
+            if not is_transient(e):
+                raise
+            if attempt >= max_retries:
+                raise
+            if on_failure is not None:
+                try:
+                    on_failure()
+                except Exception:
+                    pass
+            wait = backoff * (2 ** attempt) * (0.5 + random.random())
+            if verbose:
+                print(f"[acmaddl] {label} failed "
+                      f"(attempt {attempt + 1}/{max_retries + 1}): {e}; "
+                      f"retrying in {wait:.2f}s")
+            time.sleep(wait)
+    raise last_err  # unreachable, but keeps type-checkers honest
+
+
+# ---------------------------------------------------------------------------
+# Degenerate-response detection
+# ---------------------------------------------------------------------------
+# A large OPeNDAP/DAP request can fail mid-transfer and the netCDF/DAP client
+# silently substitutes a constant (typically all-zero) array instead of
+# raising. That corruption then gets memoized by the fetch cache and poisons
+# every downstream computation (a zero-variance predictor makes CCA singular,
+# NaN-ing out every skill score). This detector lets the fetch path reject such
+# a response *before* it is cached, for any adapter — generalizing what the
+# OPeNDAP adapter previously did only for its own obs-chunk path.
+import numpy as _np
+
+
+class DegenerateResponseError(RuntimeError):
+    """A fetched field is bitwise-constant (or all-NaN): a truncated/zero-filled response, not data."""
+
+
+def reject_if_degenerate(ds, variable, label, *, reject_all_nan=True):
+    """Raise :class:`DegenerateResponseError` if a multi-value field is degenerate.
+
+    A multi-cell, multi-timestep geophysical field is never a single repeated value, so a
+    bitwise-constant response (the common all-zero case included) is a truncated DAP packet or a
+    zero-filled server error, not real data.
+
+    ``reject_all_nan`` controls the all-NaN case: keep it True where a variable is expected present
+    over its domain (the OPeNDAP obs path); set it False on the general fetch path, where an
+    all-NaN result can be a legitimately land-masked region (e.g. SST over a land-only box).
+    """
+    if variable not in getattr(ds, "data_vars", {}) and variable not in getattr(ds, "coords", {}):
+        # tolerate callers passing the sole data var implicitly
+        try:
+            values = ds[variable].values
+        except Exception:
+            return
+    else:
+        values = ds[variable].values
+    if values.size < 2:
+        return
+    finite = _np.isfinite(values)
+    if not finite.any():
+        if reject_all_nan:
+            raise DegenerateResponseError(
+                f"{label}: {variable!r} returned no finite values — a truncated or empty response."
+            )
+        return
+    if _np.unique(values[finite]).size == 1:
+        raise DegenerateResponseError(
+            f"{label}: constant {variable!r} field (every value is {values[finite].flat[0]}). "
+            "This is the signature of a truncated DAP response or a zero-filled server error, not "
+            "real data."
+        )

acmaddl/adapters/base.py

@@ -0,0 +1,60 @@
+from abc import ABC, abstractmethod
+
+
+class AdapterBase(ABC):
+    @abstractmethod
+    def fetch_data(self, product_config, variable, date_range=None, region=None):
+        ...
+
+    def _resolve_streams(self, product_config, date_range):
+        """Resolve a request into [(stream, sub_range)] segments.
+
+        Mirrors today's start-year stream selection. When the product sets
+        ``append_streams`` and the requested range spans the end of
+        ``grid.hindcast_range``, return both a hindcast and a forecast segment so
+        the adapter can fetch and concatenate them on time.
+
+        The forecast segment's start is ``grid.forecast_range[0]`` when the
+        product declares one, else ``hindcast_range[1] + 1`` (legacy default for
+        products that don't set forecast_range). For most products these two
+        streams are disjoint (forecast starts the year after the hindcast ends),
+        so this is a no-op vs. the old ``hr[1] + 1`` logic. But some datasets
+        (e.g. IRI NMME CFSv2) split their two OPeNDAP streams by MONTH, not by
+        year -- the hindcast stream runs through ~March of its final year and the
+        forecast stream starts ~April of that SAME year. For those, the catalog
+        sets ``hindcast_range`` and ``forecast_range`` to OVERLAP at that split
+        year (e.g. hindcast_range=[1982, 2011], forecast_range=[2011, null]), so
+        the year appears in BOTH segments below; each stream's own init-time (S)
+        filter then naturally contributes only the inits it actually has, and the
+        adapter unions the two fetches. Backward-compatible: when the ranges do
+        NOT overlap (forecast_range[0] > hindcast_range[1], or forecast_range is
+        absent), the segments are exactly as before -- no shared year.
+        """
+        if not product_config.get("split_streams"):
+            return [("hindcast", date_range)]
+        grid = product_config.get("grid") or {}
+        hr = grid.get("hindcast_range")
+        fr = grid.get("forecast_range")
+        fstart = fr[0] if fr else (hr[1] + 1 if hr else None)
+        if (product_config.get("append_streams") and hr and fstart is not None and date_range
+                and date_range[0] <= hr[1] and fstart <= date_range[1]):
+            segs = []
+            if date_range[0] <= hr[1]:
+                segs.append(("hindcast", (date_range[0], min(date_range[1], hr[1]))))
+            if date_range[1] >= fstart:
+                segs.append(("forecast", (max(date_range[0], fstart), date_range[1])))
+            return segs
+        is_forecast = bool(date_range and hr and date_range[0] > hr[1])
+        return [("forecast" if is_forecast else "hindcast", date_range)]
+
+    def health_check(self, product_config, probe_remote=False):
+        """Return adapter health metadata.
+
+        Subclasses should override this to provide adapter-specific checks.
+        """
+        return {
+            "healthy": True,
+            "kind": "config",
+            "message": "No adapter-specific health check implemented.",
+            "probe_remote": bool(probe_remote),
+        }