redlem-force 0.13.1__py3-none-any.whl

redlem_force/__init__.py

@@ -0,0 +1,19 @@
+"""redlem-force: DLEM forcing-data extraction from Google Earth Engine."""
+
+import logging
+
+from .cells import select_urma_cells
+from .config import RunConfig
+from .extract import extract_urma
+from .figures import plot_cells
+from .prism_cells import select_prism_cells
+from .quota import (QuotaExceededError, QuotaUnavailableError,
+                    estimate_eecu_hours, quota_status)
+
+logging.getLogger('redlem_force').addHandler(logging.NullHandler())
+
+__version__ = "0.13.1"
+__all__ = ['RunConfig', 'extract_urma', 'select_urma_cells',
+           'select_prism_cells', 'plot_cells',
+           'quota_status', 'estimate_eecu_hours', 'QuotaExceededError',
+           'QuotaUnavailableError']

redlem_force/cells.py

@@ -0,0 +1,317 @@
+"""URMA cell selection: native reservoir polygon -> the URMA grid cells used
+for the met variables.
+
+Everything is computed locally on the URMA grid of the reference image
+(redlem_force.grids: the first URMA daily image of 2026). Earth Engine is
+used for two things only:
+  * the grid definition (CRS + affine transform), one small request
+  * rtma_elev and rtma_water_mask read ONCE as arrays on that same grid
+    (they sit on the same lattice; checked 2026-10-13), one value
+    per URMA cell
+No grid-cell asset and no per-reservoir Earth Engine requests.
+
+Selection rule (same logic as urma_cells.py; cell values are now the pixel
+values of the cell itself):
+  * candidate cells = URMA cells the polygon intersects (sharing area;
+    touching only at an edge / corner does not count)
+  * water-mask flag: a column you pass, or the rule max(water mask of the
+    candidate cells) == 1 (urma_cells.py used the max at 100 m over the
+    polygon, i.e. the cells it intersects)
+  * flag 1 -> candidates whose water mask is 1; if there are none, the rule
+    below (cell_method = water_mask_fallback)
+  * otherwise -> candidates with overlap area >= the median overlap, then of
+    those, the ones with elevation <= their median elevation
+    (overlap_elevation). Medians: numpy median of the candidates' values.
+  * output geometry = union of the selected cells (exact pixel squares)
+
+Output columns:
+  <id_col>      str
+  cell_geometry WKB, EPSG:4326
+  n_cells       cells selected
+  n_touching    candidate cells (intersecting the reservoir)
+  wm_value      flag value used (rule: max water mask of the candidates;
+                null = no water-mask data there)
+  cell_method   water_mask | water_mask_fallback | overlap_elevation
+  cell_elev_m   mean rtma_elev of the selected cells (-> ELEV_M)
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+
+import ee
+import numpy as np
+import polars as pl
+import shapely
+
+from . import grids
+from .config import ELEV_ID, WM_ID, RunConfig
+from .ee_utils import init_ee, with_retries
+from .geometry import load_geometries, reproject
+from .grid_cells import Grid, cell_geometry
+
+log = logging.getLogger('redlem_force')
+
+CELL_SCHEMA = {
+    'cell_geometry': pl.Binary,
+    'n_cells': pl.Int64,
+    'n_touching': pl.Int64,
+    'wm_value': pl.Float64,
+    'cell_method': pl.Utf8,
+    'cell_elev_m': pl.Float64,
+}
+ROWS_PER_REQUEST = 256        # computePixels tile height (rows of the grid)
+
+
+# ---------------------------------------------------------------------------
+# Per-cell values on the URMA grid (one download per run)
+# ---------------------------------------------------------------------------
+class CellValues:
+    """rtma_elev / rtma_water_mask per URMA cell, as arrays indexed by the
+    reference grid's (col, row). NaN = no data."""
+
+    def __init__(self, grid: Grid, col0, row0, elev, wm):
+        self.grid, self.col0, self.row0 = grid, col0, row0
+        self.elev, self.wm = elev, wm            # [row - row0, col - col0]
+
+    def lookup(self, cols, rows):
+        r = np.asarray(rows) - self.row0
+        c = np.asarray(cols) - self.col0
+        ok = (r >= 0) & (r < self.elev.shape[0]) & (c >= 0) & \
+            (c < self.elev.shape[1])
+        e = np.full(len(r), np.nan)
+        w = np.full(len(r), np.nan)
+        e[ok] = self.elev[r[ok], c[ok]]
+        w[ok] = self.wm[r[ok], c[ok]]
+        return e, w, ok
+
+
+def _value_image():
+    elev = ee.Image(ELEV_ID).select(0)
+    wm = ee.Image(WM_ID).select(0)
+    # data values, and 1/0 masks so missing pixels can be told apart
+    return ee.Image.cat([
+        elev.unmask(0, False).toFloat().rename('elev'),
+        elev.mask().unmask(0, False).toFloat().rename('elev_ok'),
+        wm.unmask(0, False).toFloat().rename('wm'),
+        wm.mask().unmask(0, False).toFloat().rename('wm_ok'),
+    ])
+
+
+def fetch_cell_values(config: RunConfig) -> CellValues:
+    """Read rtma_elev / rtma_water_mask on the URMA reference grid, over the
+    reference image's pixel extent, in tiles of ROWS_PER_REQUEST rows."""
+    info = grids.grid_info('urma', config)
+    grid, w, h = info['grid'], info['width'], info['height']
+    col0, row0 = info['col0'], info['row0']
+    if not w or not h:
+        raise RuntimeError(f'URMA reference image has no dimensions: {info}')
+    init_ee(config.project, config.base_url)
+    img = _value_image()
+    crs = grid.ee_crs or grid.crs      # Earth Engine reads its own text
+    elev = np.empty((h, w), dtype=np.float64)
+    wm = np.empty((h, w), dtype=np.float64)
+    for r in range(0, h, ROWS_PER_REQUEST):
+        n = min(ROWS_PER_REQUEST, h - r)
+        req = {
+            'expression': img,
+            'fileFormat': 'NUMPY_NDARRAY',
+            'grid': {
+                'dimensions': {'width': w, 'height': n},
+                'affineTransform': {
+                    'scaleX': grid.a, 'shearX': 0,
+                    'translateX': grid.c + col0 * grid.a,
+                    'shearY': 0, 'scaleY': grid.e,
+                    'translateY': grid.f + (row0 + r) * grid.e},
+                'crsWkt' if crs.lstrip().upper().startswith(
+                    ('PROJCS', 'GEOGCS', 'PROJCRS', 'GEOGCRS'))
+                else 'crsCode': crs,
+            },
+        }
+        arr = with_retries(lambda: ee.data.computePixels(req),
+                           config.max_retries, config.retry_wait_secs)
+        elev[r:r + n] = np.where(arr['elev_ok'] > 0, arr['elev'], np.nan)
+        wm[r:r + n] = np.where(arr['wm_ok'] > 0, arr['wm'], np.nan)
+    log.info('URMA cell values: %d x %d cells; elevation for %d, water mask '
+             'for %d (%d water cells)', w, h, int(np.isfinite(elev).sum()),
+             int(np.isfinite(wm).sum()), int((wm == 1).sum()))
+    return CellValues(grid, col0, row0, elev, wm)
+
+
+# ---------------------------------------------------------------------------
+# Selection (local)
+# ---------------------------------------------------------------------------
+def candidates(geom, grid: Grid):
+    """(cols, rows, overlap areas) of cells geom intersects (grid CRS)."""
+    minx, miny, maxx, maxy = geom.bounds
+    c0, r0 = grid.col_row(minx, maxy)
+    c1, r1 = grid.col_row(maxx, miny)
+    cc, rr = np.meshgrid(np.arange(min(c0, c1), max(c0, c1) + 1),
+                         np.arange(min(r0, r1), max(r0, r1) + 1))
+    cc, rr = cc.ravel(), rr.ravel()
+    sq = grid.squares(cc, rr)
+    shapely.prepare(geom)
+    hit = shapely.intersects(geom, sq) & ~shapely.touches(geom, sq)
+    cc, rr, sq = cc[hit], rr[hit], sq[hit]
+    area = shapely.area(shapely.intersection(geom, sq))
+    keep = area > 0
+    return cc[keep], rr[keep], area[keep]
+
+
+def select_one(geom, values: CellValues, flag=None):
+    """Selection for one polygon in the grid CRS. flag: 0/1 from a column,
+    or None for the rule. Returns a dict of CELL_SCHEMA values (geometry in
+    the grid CRS, not yet WKB), plus 'cells': the selected (col, row)
+    pairs when any were selected."""
+    grid = values.grid
+    cols, rows, area = candidates(geom, grid)
+    n_touch = len(cols)
+    if n_touch == 0:
+        return {'geom': None, 'n_cells': 0, 'n_touching': 0,
+                'wm_value': None if flag is None else float(flag),
+                'cell_method': None, 'cell_elev_m': None}
+    elev, wm, _ = values.lookup(cols, rows)
+    if flag is None:
+        wm_value = float(np.nanmax(wm)) if np.isfinite(wm).any() else None
+    else:
+        wm_value = float(flag)
+    use_wm = wm_value == 1
+
+    water = wm == 1
+    if use_wm and water.any():
+        sel, method = water, 'water_mask'
+    else:
+        method = 'water_mask_fallback' if use_wm else 'overlap_elevation'
+        sel = area >= np.median(area)
+        e = elev[sel]
+        if np.isfinite(e).any():
+            cut = np.median(e[np.isfinite(e)])
+            sub = np.isfinite(elev) & (elev <= cut)
+            sel = sel & sub
+    n = int(sel.sum())
+    if n == 0:
+        return {'geom': None, 'n_cells': 0, 'n_touching': n_touch,
+                'wm_value': wm_value, 'cell_method': method,
+                'cell_elev_m': None}
+    g = cell_geometry(list(zip(cols[sel].tolist(), rows[sel].tolist())),
+                      grid)                          # valid, or raises
+    e = elev[sel]
+    return {'geom': g, 'cells': list(zip(cols[sel].tolist(),
+                                         rows[sel].tolist())),
+            'n_cells': n, 'n_touching': n_touch,
+            'wm_value': wm_value, 'cell_method': method,
+            'cell_elev_m': float(np.nanmean(e)) if np.isfinite(e).any()
+            else None}
+
+
+def _log_summary(df: pl.DataFrame, id_col: str):
+    zero = df.filter(pl.col('cell_geometry').is_null())[id_col].to_list()
+    if zero:
+        log.warning('%d reservoirs got zero cells: %s', len(zero), zero[:20])
+    fb = df.filter(pl.col('cell_method') == 'water_mask_fallback')[id_col]
+    if len(fb):
+        log.warning('%d water-mask reservoirs had no water cells and used the '
+                    'overlap/elevation rule: %s', len(fb), fb.to_list()[:20])
+    log.info('cell_method counts: %s', df['cell_method'].value_counts().rows())
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+def _select_cells_frame(native: pl.DataFrame, *, id_col: str, crs,
+                        config: RunConfig, water_mask_col: str | None = None,
+                        geometry_col: str = 'geometry',
+                        values: CellValues | None = None) -> pl.DataFrame:
+    """URMA cell selection for reservoirs given as a polars DataFrame.
+
+    native         : id_col and geometry_col (WKB or WKT, in `crs`).
+    water_mask_col : Boolean column (no nulls): True -> water-mask cells,
+                     False -> overlap/elevation rule. None (default) -> the
+                     rule (max water mask of the intersecting cells == 1).
+    values         : CellValues to reuse (default: fetched here).
+    Returns id_col + CELL_SCHEMA, sorted by id.
+    """
+    gs = load_geometries(native, id_col, geometry_col, crs,
+                         label='native reservoir')
+    if water_mask_col is None:
+        flags = [None] * len(gs.ids)
+    else:
+        if water_mask_col not in native.columns:
+            raise KeyError(f'water mask column {water_mask_col!r} not in '
+                           f'{native.columns}')
+        wm = native.get_column(water_mask_col)
+        if wm.dtype != pl.Boolean:
+            raise TypeError(f'{water_mask_col!r} must be Boolean, got '
+                            f'{wm.dtype}')
+        if wm.null_count():
+            raise ValueError(f'{wm.null_count()} null values in '
+                             f'{water_mask_col!r}')
+        flags = [int(w) for w in wm.to_list()]
+
+    values = values or fetch_cell_values(config)
+    grid = values.grid
+    polys = reproject(gs.geoms, gs.crs, grid.crs)
+    rows = [select_one(g, values, f) for g, f in zip(polys, flags)]
+    geoms = np.array([r['geom'] for r in rows], dtype=object)
+    ok = np.array([g is not None for g in geoms])
+    wkb = [None] * len(geoms)
+    if ok.any():
+        wgs = reproject(geoms[ok], grid.crs, 'EPSG:4326')
+        for k, g in zip(np.flatnonzero(ok), wgs):
+            wkb[k] = shapely.to_wkb(g)
+    out = pl.DataFrame({
+        id_col: gs.ids,
+        'cell_geometry': wkb,
+        **{c: [r[c] for r in rows] for c in CELL_SCHEMA
+           if c != 'cell_geometry'},
+    }, schema={id_col: pl.Utf8, **CELL_SCHEMA}).sort(id_col)
+    log.info('select_urma_cells: %d reservoirs, water mask from %s',
+             out.height, water_mask_col or 'the rule')
+    _log_summary(out, id_col)
+    return out
+
+
+def select_urma_cells(geometry, *, id_col: str, config: RunConfig,
+                      layer: str | None = None,
+                      sample: int | None = None, sample_seed: int = 1,
+                      crs=None, geometry_col: str = 'geometry',
+                      water_mask_col: str | None = None):
+    """URMA cell selection (URMA met variables).
+
+    geometry : the reservoir polygons -- a geopandas GeoDataFrame (e.g.
+               gpd.read_file of a GeoPackage, or a database read), a path
+               to a .shp / .gpkg, or a polars DataFrame with id_col and
+               WKB/WKT geometry_col (then `crs` is required).
+
+    GeoDataFrame / path -> GeoDataFrame: every original column + N_CELLS, ELEV_M (mean elevation of the
+    selected cells, m), WATER_MASK (1 = water-mask rule, 0 =
+    overlap/elevation rule);
+    geometry = the selected cells, in the input CRS. Nothing is written to
+    disk; saving it is up to the caller (see scripts/run_*_cells.py).
+    polars DataFrame -> polars DataFrame: id_col, cell_geometry (WKB,
+    EPSG:4326), n_cells, n_touching,
+    wm_value, cell_method, cell_elev_m.
+
+    layer          : layer to read from a multi-layer GeoPackage (path input)
+    sample         : random N reservoirs, for a trial (path / GeoDataFrame)
+    water_mask_col : existing 0/1 / Boolean column for the flag; None = the
+                     rule (max water mask of the intersecting cells == 1)
+    """
+    if isinstance(geometry, pl.DataFrame):
+        if crs is None:
+            raise ValueError('crs is required when geometry is a DataFrame')
+        return _select_cells_frame(geometry, id_col=id_col, crs=crs,
+                                   config=config,
+                                   water_mask_col=water_mask_col,
+                                   geometry_col=geometry_col)
+    from .files import is_geodataframe, urma_cells
+    if not (is_geodataframe(geometry)
+            or isinstance(geometry, (str, os.PathLike))):
+        raise TypeError('geometry must be a GeoDataFrame, a file path (.shp / '
+                        '.gpkg) or a polars DataFrame, got '
+                        f'{type(geometry).__name__}')
+    return urma_cells(geometry, id_col=id_col, config=config,
+                      layer=layer,
+                      water_mask_col=water_mask_col, sample=sample,
+                      sample_seed=sample_seed)

redlem_force/config.py

@@ -0,0 +1,142 @@
+"""Dataset IDs, band/column maps, and the user-facing run configuration."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+# ---------------------------------------------------------------------------
+# Earth Engine assets
+# ---------------------------------------------------------------------------
+URMA_ID = 'projects/climate-engine-pro/assets/noaa-urma/daily'
+PRISM_800_ID = 'projects/sat-io/open-datasets/OREGONSTATE/PRISM_800_DAILY'
+PRISM_4K_ID = 'OREGONSTATE/PRISM/ANd'
+
+# Per-cell values for URMA cell selection, read on the URMA grid.
+# rtma_elev: same lattice as the URMA grid (checked 2026-10-13).
+# Water mask: URMA ancillary asset (public); its grid not yet checked here.
+ELEV_ID = 'projects/bor-evap/assets/rtma_elev'
+WM_ID = 'projects/climate-engine-pro/assets/noaa-urma/ancillary/water_mask'
+
+HIGH_VOLUME_URL = 'https://earthengine-highvolume.googleapis.com'
+
+# The ONLY source of grid information (redlem_force.grids): the reference
+# image of each source as (collection, band, year of the first image by
+# system:index, or None = first image of the collection).
+GRID_REFERENCE = {
+    'urma': (URMA_ID, 'TMAX', 2026),
+    'prism800': (PRISM_800_ID, 'ppt', None),
+}
+
+# Property name used for the reservoir ID inside Earth Engine requests.
+# The user's own ID column name is restored on output.
+ID_PROP = 'dlem_id'
+
+# ---------------------------------------------------------------------------
+# URMA met variables (same bands, reducers and output names as
+# urma_download_v2.py)
+# ---------------------------------------------------------------------------
+MEAN_BANDS = ['PRES', 'TMIN', 'TMAX', 'vs2m', 'SRAD_TCDC', 'vpd', 'SPFH',
+              'ETo', 'ETr']
+MEDIAN_BANDS = ['WDIR']
+
+# reduceRegions output name -> output column name. Output names follow the
+# DLEM variable config (key = output name, band_name = source band):
+#   tmax TMAX degC   tmin TMIN degC   vpd vpd kpa (derived)   vs2m m/s
+#   (derived from WIND)   srad SRAD_TCDC wm2   etr ETr mm/day   eto ETo
+#   mm/day   th WDIR deg (median)   sph SPFH kg/kg   pres PRES pa
+# pr (mm) is the precip column: PRISM 800 m ppt, falling back to PRISM 4 km
+# ppt, then URMA PCP (pr_source says which).
+URMA_OUTPUT = {
+    'TMAX_mean': 'tmax',
+    'TMIN_mean': 'tmin',
+    'vpd_mean': 'vpd',
+    'vs2m_mean': 'vs2m',
+    'SRAD_TCDC_mean': 'srad',
+    'ETr_mean': 'etr',
+    'ETo_mean': 'eto',
+    'WDIR_median': 'th',
+    'SPFH_mean': 'sph',
+    'PRES_mean': 'pres',
+}
+
+# ---------------------------------------------------------------------------
+# Precip fallback chain, in priority order: (name, collection, band).
+# All three are matched by date label (system:index == YYYYMMDD), never by
+# filterDate: the 4 km collection stamps images at 12:00 UTC on the day
+# BEFORE their label. Verified: 4 km vs 800 m corr 0.998 when matched by
+# label, ~0.17 when off by a day.
+# ---------------------------------------------------------------------------
+PRECIP_SOURCES = (
+    ('prism800', PRISM_800_ID, 'ppt'),
+    ('prism4k', PRISM_4K_ID, 'ppt'),
+    ('urma', URMA_ID, 'PCP'),
+)
+
+# Output columns, in order. The ID column takes the user's name.
+FLOAT_COLS = ['pr'] + list(URMA_OUTPUT.values())
+OUTPUT_COLS_AFTER_ID = (['date'] + FLOAT_COLS
+                        + ['pr_source', 'urma_status', 'processed_date'])
+
+
+@dataclass
+class RunConfig:
+    """Settings for one run.
+
+    project              : Google Cloud project used for Earth Engine.
+    n_workers            : parallel worker processes. 1 = run in this
+                           process (useful for debugging / Spyder).
+    base_url             : Earth Engine endpoint. Default is the high-volume
+                           endpoint used by urma_download_v2.py. None = the
+                           standard endpoint.
+    tile_scale           : reduceRegions tileScale. Raise to 2 or 4 only on
+                           an Earth Engine memory error.
+    features_per_request : max reservoirs per extraction request.
+    days_per_request     : consecutive dates stacked into one request (all
+                           days of a block as bands of one image). 5 was
+                           benchmarked (stacking 5 days: ~1.35x faster for
+                           URMA met, ~2.7x for precip); larger blocks are
+                           NOT yet benchmarked.
+    max_request_bytes    : max geometry bytes (GeoJSON) per extraction
+                           request. Earth Engine rejects requests over
+                           10,485,760 bytes ("Request payload size exceeds the
+                           limit", seen with 1000 detailed reservoirs); the
+                           4 MB default leaves room for request overhead.
+    max_retries          : attempts per Earth Engine call / download.
+    retry_wait_secs      : base wait for exponential backoff.
+    download_timeout_secs: HTTP timeout for CSV downloads.
+    check_quota          : before extracting, estimate the run's compute
+                           and stop if it exceeds the project's remaining
+                           monthly Earth Engine quota (needs
+                           google-cloud-monitoring).
+    eecu_s_per_reservoir_day : cost estimate used by that check. Default
+                           0.012 = measured ~0.0102 (2026-10-15, URMA met +
+                           PRISM precip, 5-day blocks), rounded up.
+    quota_fraction       : stop if the estimate exceeds this fraction of the
+                           remaining quota (1.0 = all of it).
+    """
+    project: str
+    n_workers: int = 8
+    base_url: str | None = HIGH_VOLUME_URL
+    tile_scale: float = 1
+    features_per_request: int = 1000
+    days_per_request: int = 5
+    max_request_bytes: int = 4_000_000
+    max_retries: int = 5
+    retry_wait_secs: float = 10.0
+    download_timeout_secs: float = 600.0
+    check_quota: bool = True
+    eecu_s_per_reservoir_day: float = 0.012
+    quota_fraction: float = 1.0
+
+    def __post_init__(self):
+        if not self.project:
+            raise ValueError('project is required')
+        for name in ('n_workers', 'features_per_request', 'days_per_request',
+                     'max_retries',
+                     'max_request_bytes'):
+            if int(getattr(self, name)) < 1:
+                raise ValueError(f'{name} must be >= 1')
+        if self.eecu_s_per_reservoir_day <= 0:
+            raise ValueError('eecu_s_per_reservoir_day must be > 0')
+        if not 0 < self.quota_fraction <= 1:
+            raise ValueError('quota_fraction must be in (0, 1]')

redlem_force/dates.py

@@ -0,0 +1,50 @@
+"""Date input handling."""
+
+from __future__ import annotations
+
+import datetime as dt
+
+
+def to_date(value) -> dt.date:
+    """Accept datetime.date, datetime.datetime, 'YYYY-MM-DD' or 'YYYYMMDD'."""
+    if isinstance(value, dt.datetime):
+        return value.date()
+    if isinstance(value, dt.date):
+        return value
+    if isinstance(value, str):
+        s = value.strip()
+        for fmt in ('%Y-%m-%d', '%Y%m%d'):
+            try:
+                return dt.datetime.strptime(s, fmt).date()
+            except ValueError:
+                pass
+    raise ValueError(f'Unrecognized date {value!r}; use YYYY-MM-DD, YYYYMMDD '
+                     'or datetime.date')
+
+
+def normalize_dates(dates=None, start=None, end=None) -> list[dt.date]:
+    """Sorted, de-duplicated list of dates.
+
+    Either `dates` (one date or an iterable of dates), or `start` and `end`
+    (both INCLUSIVE). Not both.
+    """
+    if dates is not None and (start is not None or end is not None):
+        raise ValueError('Pass either dates or start/end, not both')
+    if dates is not None:
+        items = [dates] if isinstance(dates, (str, dt.date)) else list(dates)
+        out = sorted({to_date(d) for d in items})
+    elif start is not None and end is not None:
+        s, e = to_date(start), to_date(end)
+        if e < s:
+            raise ValueError(f'end {e} is before start {s}')
+        out = [s + dt.timedelta(days=i) for i in range((e - s).days + 1)]
+    else:
+        raise ValueError('Pass dates, or both start and end')
+    if not out:
+        raise ValueError('No dates given')
+    return out
+
+
+def label(d: dt.date) -> str:
+    """Date label used by every collection's system:index: 'YYYYMMDD'."""
+    return d.strftime('%Y%m%d')

redlem_force/ee_utils.py

@@ -0,0 +1,150 @@
+"""Earth Engine session, retries, downloads and the worker pool."""
+
+from __future__ import annotations
+
+import io
+import logging
+import multiprocessing
+import random
+import time
+
+import ee
+import polars as pl
+import requests
+
+from .config import ID_PROP
+
+log = logging.getLogger('redlem_force')
+
+# Substrings that mark an error worth retrying as-is (quota, transient server
+# or network trouble). Network timeouts are caught by type below.
+_RETRYABLE = ('Too Many Requests', '429', 'Quota exceeded',
+              'Service Unavailable', '503', '502', 'Internal error',
+              'Connection reset', 'Connection aborted')
+
+# Substrings that mean the REQUEST IS TOO BIG. Retrying the same request
+# will fail the same way; the remedy is a smaller request.
+_TOO_BIG = ('Computation timed out', 'User memory limit exceeded',
+            'Too many concurrent aggregations', 'payload size',
+            'Request payload')
+
+
+def is_too_big(exc: Exception) -> bool:
+    msg = str(exc)
+    return any(tok in msg for tok in _TOO_BIG)
+
+
+def solve_split(fn, items: list):
+    """Call fn(items). If it fails because the request is too big, split
+    items in half and recurse, down to single items.
+
+    Returns (results, failures): results is the concatenation of fn's list
+    results; failures is [(item, message)] for items that still failed.
+    """
+    try:
+        return fn(items), []
+    except Exception as e:
+        msg = f'{type(e).__name__}: {e}'
+        if len(items) > 1 and is_too_big(e):
+            mid = len(items) // 2
+            r1, f1 = solve_split(fn, items[:mid])
+            r2, f2 = solve_split(fn, items[mid:])
+            return r1 + r2, f1 + f2
+        return [], [(it, msg) for it in items]
+
+
+def is_retryable(exc: Exception) -> bool:
+    if isinstance(exc, (requests.ConnectionError, requests.Timeout)):
+        return True
+    if isinstance(exc, requests.HTTPError) and exc.response is not None:
+        return exc.response.status_code == 429 or exc.response.status_code >= 500
+    msg = str(exc)
+    return any(tok in msg for tok in _RETRYABLE)
+
+
+def with_retries(fn, max_retries: int = 5, wait: float = 10.0):
+    """Call fn(); on a retryable error wait (exponential backoff with jitter)
+    and try again, up to max_retries attempts in total."""
+    for attempt in range(1, max_retries + 1):
+        try:
+            return fn()
+        except Exception as e:
+            if attempt == max_retries or not is_retryable(e):
+                raise
+            sleep = wait * 2 ** (attempt - 1) * (0.5 + random.random())
+            log.warning('retryable error (attempt %d/%d), sleeping %.0fs: %s',
+                        attempt, max_retries, sleep, e)
+            time.sleep(sleep)
+
+
+def init_ee(project: str, base_url: str | None) -> None:
+    """Initialize Earth Engine (used as the worker-pool initializer)."""
+    def go():
+        if base_url:
+            ee.Initialize(project=project, opt_url=base_url)
+        else:
+            ee.Initialize(project=project)
+    with_retries(go)
+
+
+def get_info(obj, cfg):
+    return with_retries(obj.getInfo, cfg.max_retries, cfg.retry_wait_secs)
+
+
+def download_csv(fc, selectors: list, cfg) -> pl.DataFrame:
+    """Download a FeatureCollection's properties as CSV into polars.
+
+    ID_PROP is always read as text. Other columns are left for the caller to
+    cast; empty CSV fields become nulls.
+    """
+    def go():
+        url = fc.getDownloadURL(filetype='CSV', selectors=selectors)
+        r = requests.get(url, timeout=cfg.download_timeout_secs)
+        r.raise_for_status()
+        return r.content
+
+    content = with_retries(go, cfg.max_retries, cfg.retry_wait_secs)
+    if not content.strip():
+        return pl.DataFrame(schema={c: pl.Utf8 for c in selectors})
+    return pl.read_csv(io.BytesIO(content), schema_overrides={ID_PROP: pl.Utf8},
+                       infer_schema_length=None)
+
+
+def run_jobs(fn, jobs: list, cfg):
+    """Yield fn(job) for each job, in completion order.
+
+    n_workers == 1 (or a single job) runs in this process. Otherwise a
+    multiprocessing pool is used, each worker initializing Earth Engine once
+    -- the same pattern as urma_download_v2.py. Scripts that call this must
+    sit under `if __name__ == '__main__':` (required by multiprocessing on
+    macOS/Windows).
+    """
+    if not jobs:
+        return
+    if cfg.n_workers == 1 or len(jobs) == 1:
+        init_ee(cfg.project, cfg.base_url)
+        for j in jobs:
+            yield fn(j)
+        return
+    n = min(cfg.n_workers, len(jobs))
+    with multiprocessing.Pool(processes=n, initializer=init_ee,
+                              initargs=(cfg.project, cfg.base_url)) as pool:
+        yield from pool.imap_unordered(fn, jobs)
+
+
+class Progress:
+    """Log progress about every 5% of jobs."""
+
+    def __init__(self, total: int, what: str):
+        self.total, self.what = total, what
+        self.done = 0
+        self.t0 = time.time()
+        self.step = max(1, total // 20)
+
+    def tick(self):
+        self.done += 1
+        if self.done % self.step == 0 or self.done == self.total:
+            el = time.time() - self.t0
+            rate = self.done / el * 3600 if el > 0 else float('nan')
+            log.info('%s: %d/%d done, %.1f min, %.0f jobs/hr', self.what,
+                     self.done, self.total, el / 60, rate)