coarsegate 0.1.0__tar.gz

coarsegate-0.1.0/.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+*.egg-info/
+dist/
+build/
+.venv/
+.pytest_cache/

coarsegate-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Joshua Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

coarsegate-0.1.0/PKG-INFO

@@ -0,0 +1,147 @@
+Metadata-Version: 2.5
+Name: coarsegate
+Version: 0.1.0
+Summary: Measure what a coarsening removes, before interpreting a spatial-resolution comparison
+Author: Joshua Wang
+License-Expression: MIT
+License-File: LICENSE
+Keywords: change of support,cross-validation,downscaling,ecology,spatial resolution,statistical power
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: pandas>=2.0
+Provides-Extra: dev
+Requires-Dist: pyarrow>=14; extra == 'dev'
+Requires-Dist: pytest>=7; extra == 'dev'
+Provides-Extra: sim
+Requires-Dist: scikit-learn>=1.3; extra == 'sim'
+Requires-Dist: scipy>=1.10; extra == 'sim'
+Description-Content-Type: text/markdown
+
+# coarsegate
+
+Measure what a coarsening removes, before you interpret a spatial-resolution
+comparison.
+
+## The problem
+
+A common design asks whether finer predictors beat coarser ones: take a
+predictor field, block-average it onto a coarse grid, fit the same model to
+both arms, compare. The design looks airtight, because resolution is the only
+thing that varies.
+
+It can still fail to test its own hypothesis. Over a small extent, block
+averaging barely changes the predictors. The two arms are fed nearly the same
+numbers, the comparison cannot come out either way, and it returns a null that
+means nothing. Nothing in the output distinguishes that case from a real
+negative result.
+
+The condition is measurable from the predictors alone, with no model fitted, so
+it is available while the design can still be changed.
+
+## Install
+
+```bash
+pip install coarsegate
+```
+
+## Use
+
+```python
+from coarsegate import precondition_report
+
+report = precondition_report(
+    df, features=["tmax", "ppt", "cmd"],
+    cell_size=4000,          # same units as x and y
+    x="x_m", y="y_m",        # projected coordinates, not degrees
+    time="year",             # optional; a panel needs it, one slice does not
+)
+print(report)
+```
+
+```
+coarsening 153 sites to 25 cells of 4000 units
+  spatial variance removed (median over 14 features): f = 0.123
+  fine-coarse correlation (median):            r = 0.998
+  verdict: UNINFORMATIVE
+
+f = 0.123 -> detection rate 0.08 [0.04, 0.15] at k = 5
+  f is below the 0.4 floor: a null here says nothing about the hypothesis.
+
+The two predictor sets are close to the same numbers, so a comparison between them
+cannot come out either way. Report this as unanswerable at this design, not as a null.
+```
+
+`report.passes` is the verdict as a boolean. `report.per_feature` and
+`report.correlations` carry the detail behind the two summary numbers, in case
+one predictor is behaving differently from the rest.
+
+The pieces are usable on their own:
+
+```python
+from coarsegate import variance_removed, correlation_retained, power_at, upscale
+
+variance_removed(df, features, cell_size=25_000, x="x_m", y="y_m", time="year")
+power_at(0.484)                    # what that fraction buys in detection
+upscale(df, features, cell_size=25_000, x="x_m", y="y_m", time="year")
+```
+
+## What the floor is, and what it is not
+
+`f` is the fraction of between-site predictor variance that block averaging
+destroys. It is the within-cell share of the change-of-support decomposition,
+and geostatistics has computed that quantity from the variogram since long
+before this package existed. That part is not new.
+
+What is not available analytically is the probability that a comparison at a
+given `f` returns a verdict at all. That has to be measured, and the shipped
+calibration is the measurement: 800 simulated panels across eight correlation
+lengths, plus a block-count sweep, all under the same spatially-blocked
+cross-validation and paired bootstrap the method is meant to be used with.
+
+```python
+from coarsegate import calibration_table
+calibration_table("f")        # the correlation-length sweep, k = 5
+calibration_table("blocks")   # the block-count sweep, at the two highest f
+```
+
+Read the floor in one direction only. Below `f` of about 0.4, a null carries no
+information, so the diagnostic disqualifies the comparison. Above it, detection
+depends on the number of validation blocks as much as on `f`, so clearing the
+floor licenses interpretation rather than guaranteeing power. Detection rates
+below the floor drift back up, and those are not detections in any useful sense.
+They are what a bootstrap does with two nearly identical predictor sets, which
+is exactly why the floor is a floor and not a point on a curve.
+
+## Caveats worth reading before you cite a number
+
+- The calibration is measured at five validation blocks and 300 sites. A design
+  far from that should re-run the simulation rather than read the table.
+- `x`, `y` and `cell_size` must be projected and share units. Averaging degrees
+  gives cells whose ground area varies with latitude, which puts a spurious
+  north-south gradient into `f`.
+- Pass `time` for a panel. Without it the coarse arm loses the interannual
+  signal too, so `f` counts variance the coarsening was never meant to touch and
+  comes out too **high**, which is the dangerous direction: it makes an
+  unanswerable design look answerable. Repeated coordinates with no `time`
+  raise a warning.
+- `f` is summarised as the median over features, so one nearly flat predictor
+  cannot drag the verdict.
+
+## Citation
+
+The method, the calibration and the two worked extents:
+
+> Wang, J. Measure what the coarsening removes: a precondition for interpreting
+> spatial-resolution comparisons in ecological models. EcoEvoRxiv.
+> https://doi.org/10.32942/X2ZH5C
+
+That is a preprint. It has not been peer reviewed.
+
+## License
+
+MIT.

coarsegate-0.1.0/README.md

@@ -0,0 +1,123 @@
+# coarsegate
+
+Measure what a coarsening removes, before you interpret a spatial-resolution
+comparison.
+
+## The problem
+
+A common design asks whether finer predictors beat coarser ones: take a
+predictor field, block-average it onto a coarse grid, fit the same model to
+both arms, compare. The design looks airtight, because resolution is the only
+thing that varies.
+
+It can still fail to test its own hypothesis. Over a small extent, block
+averaging barely changes the predictors. The two arms are fed nearly the same
+numbers, the comparison cannot come out either way, and it returns a null that
+means nothing. Nothing in the output distinguishes that case from a real
+negative result.
+
+The condition is measurable from the predictors alone, with no model fitted, so
+it is available while the design can still be changed.
+
+## Install
+
+```bash
+pip install coarsegate
+```
+
+## Use
+
+```python
+from coarsegate import precondition_report
+
+report = precondition_report(
+    df, features=["tmax", "ppt", "cmd"],
+    cell_size=4000,          # same units as x and y
+    x="x_m", y="y_m",        # projected coordinates, not degrees
+    time="year",             # optional; a panel needs it, one slice does not
+)
+print(report)
+```
+
+```
+coarsening 153 sites to 25 cells of 4000 units
+  spatial variance removed (median over 14 features): f = 0.123
+  fine-coarse correlation (median):            r = 0.998
+  verdict: UNINFORMATIVE
+
+f = 0.123 -> detection rate 0.08 [0.04, 0.15] at k = 5
+  f is below the 0.4 floor: a null here says nothing about the hypothesis.
+
+The two predictor sets are close to the same numbers, so a comparison between them
+cannot come out either way. Report this as unanswerable at this design, not as a null.
+```
+
+`report.passes` is the verdict as a boolean. `report.per_feature` and
+`report.correlations` carry the detail behind the two summary numbers, in case
+one predictor is behaving differently from the rest.
+
+The pieces are usable on their own:
+
+```python
+from coarsegate import variance_removed, correlation_retained, power_at, upscale
+
+variance_removed(df, features, cell_size=25_000, x="x_m", y="y_m", time="year")
+power_at(0.484)                    # what that fraction buys in detection
+upscale(df, features, cell_size=25_000, x="x_m", y="y_m", time="year")
+```
+
+## What the floor is, and what it is not
+
+`f` is the fraction of between-site predictor variance that block averaging
+destroys. It is the within-cell share of the change-of-support decomposition,
+and geostatistics has computed that quantity from the variogram since long
+before this package existed. That part is not new.
+
+What is not available analytically is the probability that a comparison at a
+given `f` returns a verdict at all. That has to be measured, and the shipped
+calibration is the measurement: 800 simulated panels across eight correlation
+lengths, plus a block-count sweep, all under the same spatially-blocked
+cross-validation and paired bootstrap the method is meant to be used with.
+
+```python
+from coarsegate import calibration_table
+calibration_table("f")        # the correlation-length sweep, k = 5
+calibration_table("blocks")   # the block-count sweep, at the two highest f
+```
+
+Read the floor in one direction only. Below `f` of about 0.4, a null carries no
+information, so the diagnostic disqualifies the comparison. Above it, detection
+depends on the number of validation blocks as much as on `f`, so clearing the
+floor licenses interpretation rather than guaranteeing power. Detection rates
+below the floor drift back up, and those are not detections in any useful sense.
+They are what a bootstrap does with two nearly identical predictor sets, which
+is exactly why the floor is a floor and not a point on a curve.
+
+## Caveats worth reading before you cite a number
+
+- The calibration is measured at five validation blocks and 300 sites. A design
+  far from that should re-run the simulation rather than read the table.
+- `x`, `y` and `cell_size` must be projected and share units. Averaging degrees
+  gives cells whose ground area varies with latitude, which puts a spurious
+  north-south gradient into `f`.
+- Pass `time` for a panel. Without it the coarse arm loses the interannual
+  signal too, so `f` counts variance the coarsening was never meant to touch and
+  comes out too **high**, which is the dangerous direction: it makes an
+  unanswerable design look answerable. Repeated coordinates with no `time`
+  raise a warning.
+- `f` is summarised as the median over features, so one nearly flat predictor
+  cannot drag the verdict.
+
+## Citation
+
+The method, the calibration and the two worked extents:
+
+> Wang, J. Measure what the coarsening removes: a precondition for interpreting
+> spatial-resolution comparisons in ecological models. EcoEvoRxiv.
+> https://doi.org/10.32942/X2ZH5C
+
+That is a preprint. It has not been peer reviewed.
+
+## License
+
+MIT.

coarsegate-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "coarsegate"
+version = "0.1.0"
+description = "Measure what a coarsening removes, before interpreting a spatial-resolution comparison"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "Joshua Wang" }]
+keywords = ["change of support", "spatial resolution", "downscaling",
+            "statistical power", "cross-validation", "ecology"]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering",
+]
+dependencies = ["numpy>=1.24", "pandas>=2.0"]
+
+[project.optional-dependencies]
+sim = ["scikit-learn>=1.3", "scipy>=1.10"]
+dev = ["pytest>=7", "pyarrow>=14"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/coarsegate"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

coarsegate-0.1.0/src/coarsegate/__init__.py

@@ -0,0 +1,26 @@
+"""Measure what a coarsening removes, before interpreting a resolution comparison.
+
+A comparison between a fine and a coarse version of the same predictor field can
+fail to test its own hypothesis. Over a small extent, block averaging barely
+changes the predictors, the two models are fed nearly the same numbers, and the
+comparison returns a null that means nothing. This package measures that
+condition first, from the predictors alone, with no model fitted.
+
+    >>> from coarsegate import precondition_report
+    >>> print(precondition_report(df, features, cell_size=4000,
+    ...                           x="x_m", y="y_m", time="year"))
+
+Method and calibration: Wang, J. Measure what the coarsening removes.
+EcoEvoRxiv, doi:10.32942/X2ZH5C
+"""
+from ._upscale import n_cells, upscale
+from .calibration import INFORMATIVE_FLOOR, Power, calibration_table, power_at
+from .diagnostics import correlation_retained, variance_removed
+from .report import Precondition, precondition_report
+
+__version__ = "0.1.0"
+__all__ = [
+    "INFORMATIVE_FLOOR", "Power", "Precondition", "calibration_table",
+    "correlation_retained", "n_cells", "power_at", "precondition_report",
+    "upscale", "variance_removed",
+]

coarsegate-0.1.0/src/coarsegate/_upscale.py

@@ -0,0 +1,75 @@
+"""Block-average a predictor field onto a coarse grid."""
+from __future__ import annotations
+
+import warnings
+
+import numpy as np
+import pandas as pd
+
+
+def upscale(
+    df: pd.DataFrame,
+    features: list[str],
+    *,
+    cell_size: float,
+    x: str = "x",
+    y: str = "y",
+    time: str | None = None,
+) -> pd.DataFrame:
+    """Replace each site's predictors with its coarse cell's mean.
+
+    Averaging within (cell, time) rather than within cell alone keeps the
+    temporal signal intact, so only the spatial detail is destroyed. That is the
+    single thing a resolution comparison is allowed to vary: if the coarse arm
+    also loses temporal detail, a difference between the two arms no longer
+    isolates resolution.
+
+    ``x``, ``y`` and ``cell_size`` must share units, and the coordinates must be
+    projected. Averaging degrees gives cells whose ground area varies with
+    latitude, which puts a spurious north-south gradient into ``f``.
+    """
+    _require_columns(df, [x, y] + list(features) + ([time] if time else []))
+    if cell_size <= 0:
+        raise ValueError(f"cell_size must be positive, got {cell_size}")
+
+    out = df.copy()
+    cell = (np.floor(out[x] / cell_size).astype("int64").astype(str) + "_"
+            + np.floor(out[y] / cell_size).astype("int64").astype(str))
+    by = [cell] if time is None else [cell, out[time]]
+    out[features] = out.groupby(by, sort=False)[features].transform("mean")
+    return out
+
+
+def n_cells(df: pd.DataFrame, *, cell_size: float, x: str = "x", y: str = "y") -> int:
+    """How many occupied coarse cells the sites fall into."""
+    _require_columns(df, [x, y])
+    cell = (np.floor(df[x] / cell_size).astype("int64").astype(str) + "_"
+            + np.floor(df[y] / cell_size).astype("int64").astype(str))
+    return int(cell.nunique())
+
+
+def _require_columns(df: pd.DataFrame, cols: list[str]) -> None:
+    missing = [c for c in cols if c not in df.columns]
+    if missing:
+        raise KeyError(f"columns not in the frame: {missing}")
+
+
+def _warn_if_panel_without_time(df: pd.DataFrame, *, x: str, y: str,
+                                time: str | None) -> None:
+    """A panel measured without a time column inflates f. Say so, loudly.
+
+    Repeated coordinates mean several observations per site. Averaging across
+    them destroys the temporal signal along with the spatial detail, so the
+    fraction removed counts variance the coarsening was never supposed to touch.
+    """
+    if time is not None:
+        return
+    n_sites = len(df.drop_duplicates([x, y]))
+    if n_sites < len(df):
+        warnings.warn(
+            f"{len(df)} rows over {n_sites} distinct coordinates but no `time` "
+            "column was given. On a panel this counts temporal variance as "
+            "spatial and overstates the fraction removed. Pass time= to measure "
+            "the spatial part alone.",
+            UserWarning, stacklevel=3,
+        )

coarsegate-0.1.0/src/coarsegate/calibration.py

@@ -0,0 +1,107 @@
+"""What a measured f buys you in detection power.
+
+The tables shipped here are simulation output, not theory. The change-of-support
+literature gives the variance decomposition analytically; what it does not give
+is the probability that a resolution comparison at a given f returns a verdict.
+That has to be measured, and these are the measurements.
+
+Both tables come from Gaussian random fields sampled at 300 sites over four
+time slices, upscaled to 25 km cells, with the two arms compared under
+spatially-blocked cross-validation and a paired percentile bootstrap. A
+"detection" is the paper's own signed criterion: the paired interval excludes
+zero *in the direction of the fine arm*. Detecting the wrong direction is not a
+detection.
+
+Two limits worth stating before anyone reads a number off these:
+
+* The f table is calibrated at five validation blocks and 300 sites. It is not a
+  general power curve, and a design far from that one should re-run the
+  simulation rather than read this table.
+* Below f of about 0.4 the detection rate stops falling and drifts back up. Those
+  are not detections in any useful sense, they are what a bootstrap does with two
+  nearly identical predictor sets. This is why the floor is a floor rather than
+  a point on a curve, and why ``power_at`` reports ``informative`` separately
+  from the rate.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, asdict
+from functools import lru_cache
+from importlib.resources import files
+
+import numpy as np
+import pandas as pd
+
+#: Below this fraction a null result carries no information about the hypothesis.
+INFORMATIVE_FLOOR = 0.4
+
+
+@dataclass(frozen=True)
+class Power:
+    """An interpolated detection rate, with everything needed to distrust it."""
+
+    f: float
+    detection_rate: float
+    ci_low: float
+    ci_high: float
+    informative: bool
+    extrapolated: bool
+    n_blocks: int
+    basis: str
+
+    def __str__(self) -> str:
+        head = (f"f = {self.f:.3f} -> detection rate {self.detection_rate:.2f} "
+                f"[{self.ci_low:.2f}, {self.ci_high:.2f}] at k = {self.n_blocks}")
+        if not self.informative:
+            head += (f"\n  f is below the {INFORMATIVE_FLOOR:g} floor: a null here "
+                     "says nothing about the hypothesis.")
+        if self.extrapolated:
+            head += "\n  f is outside the calibrated range; the rate is clamped."
+        return head
+
+    def as_dict(self) -> dict:
+        return asdict(self)
+
+
+@lru_cache(maxsize=None)
+def calibration_table(axis: str = "f") -> pd.DataFrame:
+    """The raw simulation output behind ``power_at``.
+
+    ``axis="f"`` is the correlation-length sweep at five blocks; ``axis="blocks"``
+    is the block-count sweep at the two highest-f levels. Read them directly
+    rather than trusting an interpolation when your design is unusual.
+    """
+    names = {"f": "calibration_f.csv", "blocks": "calibration_blocks.csv"}
+    if axis not in names:
+        raise ValueError(f"axis must be one of {sorted(names)}, got {axis!r}")
+    with files("coarsegate.data").joinpath(names[axis]).open() as fh:
+        return pd.read_csv(fh)
+
+
+def power_at(f: float, *, n_blocks: int = 5) -> Power:
+    """Detection rate at a measured variance-removed fraction.
+
+    Linear interpolation over the shipped sweep. ``n_blocks`` is carried into
+    the result and checked against the block-count table, but it does not move
+    the rate: the sweep resolves f at k = 5, and the k axis was measured at only
+    two f levels, which is not enough to interpolate a surface from.
+    """
+    if not np.isfinite(f):
+        raise ValueError("f must be finite")
+    tab = calibration_table("f").sort_values("f_mean")
+    lo, hi = float(tab["f_mean"].iloc[0]), float(tab["f_mean"].iloc[-1])
+    extrapolated = not (lo <= f <= hi)
+    fc = float(np.clip(f, lo, hi))
+
+    rate, ci_lo, ci_hi = (
+        float(np.interp(fc, tab["f_mean"], tab[c]))
+        for c in ("detection_rate", "det_lo", "det_hi")
+    )
+    basis = (f"{int(tab['n'].iloc[0])} simulated panels per level, "
+             f"{len(tab)} levels, k = 5 blocks, n = 300 sites")
+    if n_blocks != 5:
+        basis += (f"; requested k = {n_blocks}, which the f sweep does not "
+                  "resolve (see calibration_table('blocks'))")
+    return Power(f=float(f), detection_rate=rate, ci_low=ci_lo, ci_high=ci_hi,
+                 informative=f >= INFORMATIVE_FLOOR, extrapolated=extrapolated,
+                 n_blocks=n_blocks, basis=basis)

coarsegate-0.1.0/src/coarsegate/data/calibration_blocks.csv

@@ -0,0 +1,7 @@
+range_m,n_blocks,n_sites,f_mean,f_sd,detection_rate,n,det_lo,det_hi
+3500,5,300,0.810033,0.0239791,0.68,50,0.541894,0.79242
+3500,10,300,0.810033,0.0239791,0.78,50,0.647582,0.872462
+3500,20,300,0.810033,0.0239791,0.84,50,0.714855,0.916627
+5000,5,300,0.70606,0.0359356,0.58,50,0.442332,0.706252
+5000,10,300,0.70606,0.0359356,0.56,50,0.423058,0.68838
+5000,20,300,0.70606,0.0359356,0.6,50,0.461812,0.723918

coarsegate-0.1.0/src/coarsegate/data/calibration_f.csv

@@ -0,0 +1,9 @@
+range_m,n_blocks,n_sites,f_mean,f_sd,detection_rate,det_lo,det_hi,n
+40000,5,300,0.0595642,0.00756533,0.19,0.125147,0.27779,100
+28000,5,300,0.0860021,0.0148084,0.1,0.0552285,0.174367,100
+20000,5,300,0.135776,0.0234062,0.07,0.0343188,0.137497,100
+14000,5,300,0.23796,0.0370467,0.11,0.0625413,0.186315,100
+10000,5,300,0.388117,0.0505486,0.2,0.133366,0.288831,100
+7000,5,300,0.561183,0.0518677,0.28,0.201396,0.374882,100
+5000,5,300,0.700381,0.0409765,0.54,0.442647,0.634394,100
+3500,5,300,0.805313,0.0289198,0.63,0.532203,0.718178,100

coarsegate-0.1.0/src/coarsegate/diagnostics.py

@@ -0,0 +1,86 @@
+"""How much spatial variance a coarsening actually removes."""
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from ._upscale import upscale, _require_columns, _warn_if_panel_without_time
+
+
+def variance_removed(
+    df: pd.DataFrame,
+    features: list[str],
+    *,
+    cell_size: float,
+    x: str = "x",
+    y: str = "y",
+    time: str | None = None,
+) -> pd.DataFrame:
+    """Per feature, the fraction of spatial variance the coarsening destroys.
+
+    Returns one row per feature with the spatial variance before and after
+    block averaging and their ratio, sorted by the fraction removed.
+
+    The reference point is the mean within each time slice, not the grand mean,
+    so what is measured is variance *between sites* and interannual variation
+    never enters it. With ``time=None`` the grand mean is used instead, which is
+    correct for a single-slice frame and wrong for a panel: on a panel it counts
+    temporal variance as spatial and *inflates* the fraction removed, which is
+    the dangerous direction, since it makes an unanswerable comparison look
+    answerable. Repeated coordinates with no ``time`` therefore raise a warning.
+
+    A fraction near zero means the two predictor sets are nearly the same
+    numbers. A comparison between them cannot come out either way, so its result
+    is uninformative rather than negative, and that is worth being able to read
+    off the data rather than argue about afterwards.
+    """
+    features = list(features)
+    if not features:
+        raise ValueError("no features given")
+    _require_columns(df, features)
+
+    _warn_if_panel_without_time(df, x=x, y=y, time=time)
+    coarse = upscale(df, features, cell_size=cell_size, x=x, y=y, time=time)
+    rows = []
+    for f in features:
+        ref = (df[f].mean() if time is None
+               else df.groupby(time)[f].transform("mean"))
+        var_fine = float(((df[f] - ref) ** 2).mean())
+        var_coarse = float(((coarse[f] - ref) ** 2).mean())
+        rows.append({
+            "feature": f,
+            "spatial_var_fine": var_fine,
+            "spatial_var_coarse": var_coarse,
+            "var_removed_frac": 1 - var_coarse / var_fine if var_fine > 0 else np.nan,
+        })
+    return (pd.DataFrame(rows)
+            .sort_values("var_removed_frac", ascending=False)
+            .reset_index(drop=True))
+
+
+def correlation_retained(
+    df: pd.DataFrame,
+    features: list[str],
+    *,
+    cell_size: float,
+    x: str = "x",
+    y: str = "y",
+    time: str | None = None,
+) -> pd.DataFrame:
+    """Per feature, the correlation between the fine and coarse arms.
+
+    The companion to ``variance_removed`` and the more legible number of the
+    two: a fraction removed of 0.12 is abstract, whereas r = 0.998 between the
+    two arms says plainly that the models are being fed the same predictor.
+    """
+    features = list(features)
+    _require_columns(df, features)
+    coarse = upscale(df, features, cell_size=cell_size, x=x, y=y, time=time)
+    rows = []
+    for f in features:
+        a, b = df[f].to_numpy(float), coarse[f].to_numpy(float)
+        ok = np.isfinite(a) & np.isfinite(b)
+        r = (np.corrcoef(a[ok], b[ok])[0, 1]
+             if ok.sum() > 1 and a[ok].std() > 0 and b[ok].std() > 0 else np.nan)
+        rows.append({"feature": f, "r_fine_coarse": float(r)})
+    return pd.DataFrame(rows).sort_values("r_fine_coarse").reset_index(drop=True)

coarsegate-0.1.0/src/coarsegate/report.py

@@ -0,0 +1,88 @@
+"""The precondition, run end to end and stated as a verdict."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pandas as pd
+
+from ._upscale import n_cells
+from .calibration import INFORMATIVE_FLOOR, Power, power_at
+from .diagnostics import correlation_retained, variance_removed
+
+
+@dataclass(frozen=True)
+class Precondition:
+    """Whether a resolution comparison at this design can answer its question."""
+
+    f: float
+    verdict: str
+    power: Power
+    n_cells: int
+    n_sites: int
+    cell_size: float
+    per_feature: pd.DataFrame
+    correlations: pd.DataFrame
+
+    @property
+    def passes(self) -> bool:
+        return self.verdict == "informative"
+
+    def __str__(self) -> str:
+        med_r = float(self.correlations["r_fine_coarse"].median())
+        lines = [
+            f"coarsening {self.n_sites} sites to {self.n_cells} cells "
+            f"of {self.cell_size:g} units",
+            f"  spatial variance removed (median over "
+            f"{len(self.per_feature)} features): f = {self.f:.3f}",
+            f"  fine-coarse correlation (median):            r = {med_r:.3f}",
+            f"  verdict: {self.verdict.upper()}",
+            "",
+            str(self.power),
+        ]
+        if not self.passes:
+            lines += [
+                "",
+                "The two predictor sets are close to the same numbers, so a "
+                "comparison between them",
+                "cannot come out either way. Report this as unanswerable at this "
+                "design, not as a null.",
+            ]
+        return "\n".join(lines)
+
+
+def precondition_report(
+    df: pd.DataFrame,
+    features: list[str],
+    *,
+    cell_size: float,
+    x: str = "x",
+    y: str = "y",
+    time: str | None = None,
+    n_blocks: int = 5,
+) -> Precondition:
+    """Measure f, look up what it buys, and say whether the comparison is worth
+    interpreting.
+
+    Run this before fitting either model. The whole point of the diagnostic is
+    that it needs no model: it is a property of the predictor field and the cell
+    size, so it is available while the design can still be changed.
+
+    ``f`` is summarised as the **median** over features rather than the mean, so
+    one predictor that happens to be nearly flat in space cannot drag the
+    verdict.
+    """
+    per_feature = variance_removed(df, features, cell_size=cell_size,
+                                   x=x, y=y, time=time)
+    corr = correlation_retained(df, features, cell_size=cell_size,
+                                x=x, y=y, time=time)
+    f = float(per_feature["var_removed_frac"].median())
+    return Precondition(
+        f=f,
+        verdict="informative" if f >= INFORMATIVE_FLOOR else "uninformative",
+        power=power_at(f, n_blocks=n_blocks),
+        n_cells=n_cells(df, cell_size=cell_size, x=x, y=y),
+        n_sites=int(len(df.drop_duplicates([x, y]))),
+        cell_size=float(cell_size),
+        per_feature=per_feature,
+        correlations=corr,
+    )

coarsegate-0.1.0/tests/test_calibration.py

@@ -0,0 +1,70 @@
+import numpy as np
+import pytest
+
+from coarsegate import INFORMATIVE_FLOOR, calibration_table, power_at
+
+
+def test_both_tables_ship_and_carry_their_provenance():
+    f = calibration_table("f")
+    assert len(f) == 8 and (f["n"] == 100).all() and (f["n_blocks"] == 5).all()
+    b = calibration_table("blocks")
+    assert set(b["n_blocks"]) == {5, 10, 20}
+
+
+def test_unknown_axis_is_refused():
+    with pytest.raises(ValueError):
+        calibration_table("elevation")
+
+
+def test_detection_rises_with_f_across_the_informative_range():
+    f = calibration_table("f").sort_values("f_mean")
+    high = f[f["f_mean"] >= INFORMATIVE_FLOOR]["detection_rate"].tolist()
+    assert all(b > a for a, b in zip(high, high[1:]))
+
+
+def test_the_low_f_end_is_not_monotone_which_is_why_there_is_a_floor():
+    """A rate at low f is bootstrap noise, not power. If this ever becomes
+    monotone the floor's justification has changed and the docs must follow."""
+    f = calibration_table("f").sort_values("f_mean")
+    low = f[f["f_mean"] < INFORMATIVE_FLOOR]["detection_rate"].tolist()
+    assert not all(b > a for a, b in zip(low, low[1:]))
+
+
+def test_power_at_interpolates_between_measured_levels():
+    f = calibration_table("f").sort_values("f_mean")
+    lo, hi = f.iloc[-2], f.iloc[-1]
+    mid = power_at((lo["f_mean"] + hi["f_mean"]) / 2)
+    assert lo["detection_rate"] < mid.detection_rate < hi["detection_rate"]
+    assert not mid.extrapolated and mid.informative
+
+
+def test_power_at_reproduces_a_measured_level_exactly():
+    row = calibration_table("f").iloc[0]
+    p = power_at(float(row["f_mean"]))
+    assert p.detection_rate == pytest.approx(row["detection_rate"])
+
+
+def test_the_two_extents_from_the_paper_land_either_side_of_the_floor():
+    surrey, transect = power_at(0.123), power_at(0.484)
+    assert not surrey.informative
+    assert transect.informative
+    assert "floor" in str(surrey) and "floor" not in str(transect)
+
+
+@pytest.mark.parametrize("f", [0.0, 1.0])
+def test_outside_the_calibrated_range_the_rate_is_clamped_and_says_so(f):
+    p = power_at(f)
+    assert p.extrapolated and "clamped" in str(p)
+    assert 0.0 <= p.detection_rate <= 1.0
+
+
+def test_a_block_count_the_sweep_cannot_resolve_is_declared_not_silently_used():
+    p = power_at(0.7, n_blocks=20)
+    assert p.n_blocks == 20
+    assert "does not" in p.basis
+    assert p.detection_rate == power_at(0.7).detection_rate
+
+
+def test_non_finite_f_is_refused():
+    with pytest.raises(ValueError):
+        power_at(np.nan)

coarsegate-0.1.0/tests/test_diagnostics.py

@@ -0,0 +1,107 @@
+import numpy as np
+import pandas as pd
+import pytest
+
+from coarsegate import (correlation_retained, n_cells, upscale,
+                        variance_removed)
+
+
+@pytest.fixture
+def panel():
+    """Nine sites on a 3x3 grid at 1 km spacing, three time slices."""
+    rng = np.random.default_rng(0)
+    xs, ys = np.meshgrid(np.arange(3) * 1000.0, np.arange(3) * 1000.0)
+    rows = []
+    for t in (2020, 2021, 2022):
+        rows.append(pd.DataFrame({
+            "x": xs.ravel(), "y": ys.ravel(), "year": t,
+            "a": rng.normal(size=9) + t,      # a different mean per year
+            "b": rng.normal(size=9) * 5,
+        }))
+    return pd.concat(rows, ignore_index=True)
+
+
+def test_upscale_preserves_the_mean_within_each_time_slice(panel):
+    coarse = upscale(panel, ["a", "b"], cell_size=5000, x="x", y="y", time="year")
+    for col in ("a", "b"):
+        pd.testing.assert_series_equal(
+            panel.groupby("year")[col].mean(),
+            coarse.groupby("year")[col].mean(),
+        )
+
+
+def test_upscale_leaves_other_columns_alone(panel):
+    coarse = upscale(panel, ["a"], cell_size=5000, x="x", y="y", time="year")
+    pd.testing.assert_series_equal(panel["b"], coarse["b"])
+    assert "_cell" not in coarse.columns
+
+
+def test_nothing_removed_when_every_site_owns_its_cell(panel):
+    v = variance_removed(panel, ["a", "b"], cell_size=500,
+                         x="x", y="y", time="year")
+    assert n_cells(panel, cell_size=500) == 9
+    assert v["var_removed_frac"].abs().max() < 1e-12
+
+
+def test_everything_removed_when_one_cell_holds_them_all(panel):
+    v = variance_removed(panel, ["a", "b"], cell_size=1e6,
+                         x="x", y="y", time="year")
+    assert n_cells(panel, cell_size=1e6) == 1
+    assert np.allclose(v["var_removed_frac"], 1.0)
+
+
+def test_f_rises_with_cell_size(panel):
+    fs = [variance_removed(panel, ["a", "b"], cell_size=c,
+                           x="x", y="y", time="year")["var_removed_frac"].median()
+          for c in (500, 1500, 1e6)]
+    assert fs[0] < fs[1] < fs[2]
+
+
+def test_dropping_time_on_a_panel_inflates_f_in_the_dangerous_direction(panel):
+    """Without `time`, the coarse arm loses the interannual signal too, so the
+    fraction removed counts variance the coarsening should never have touched.
+    It comes out HIGHER, which makes an unanswerable design look answerable."""
+    with_time = variance_removed(panel, ["a"], cell_size=1500,
+                                 x="x", y="y", time="year")
+    with pytest.warns(UserWarning, match="overstates"):
+        without = variance_removed(panel, ["a"], cell_size=1500, x="x", y="y")
+    assert without["spatial_var_fine"][0] > with_time["spatial_var_fine"][0]
+    assert without["var_removed_frac"][0] > with_time["var_removed_frac"][0]
+
+
+def test_a_single_time_slice_needs_no_time_column_and_warns_about_nothing(panel):
+    import warnings
+    one = panel[panel["year"] == 2020]
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        v = variance_removed(one, ["a"], cell_size=1500, x="x", y="y")
+    assert 0.0 <= v["var_removed_frac"][0] <= 1.0
+
+
+def test_correlation_is_one_when_nothing_is_removed(panel):
+    c = correlation_retained(panel, ["a", "b"], cell_size=500,
+                             x="x", y="y", time="year")
+    assert np.allclose(c["r_fine_coarse"], 1.0)
+
+
+def test_constant_feature_gives_nan_not_a_divide_by_zero(panel):
+    panel = panel.assign(flat=1.0)
+    v = variance_removed(panel, ["flat"], cell_size=1500,
+                         x="x", y="y", time="year")
+    assert np.isnan(v["var_removed_frac"][0])
+
+
+@pytest.mark.parametrize("kwargs, exc", [
+    (dict(features=["a"], cell_size=0), ValueError),
+    (dict(features=["a"], cell_size=-1), ValueError),
+    (dict(features=[], cell_size=1000), ValueError),
+    (dict(features=["nope"], cell_size=1000), KeyError),
+])
+def test_bad_input_is_refused(panel, kwargs, exc):
+    with pytest.raises(exc):
+        variance_removed(panel, x="x", y="y", time="year", **kwargs)
+
+
+def test_missing_coordinate_column_is_refused(panel):
+    with pytest.raises(KeyError):
+        variance_removed(panel, ["a"], cell_size=1000, x="easting", y="y")

coarsegate-0.1.0/tests/test_published_numbers.py

@@ -0,0 +1,47 @@
+"""Reproduce the two fractions the preprint reports, from the panels themselves.
+
+The panels are not redistributed with this package, so these skip unless
+``COARSEGATE_PANELS`` points at a directory holding ``features.parquet`` and
+``features_phase3b.parquet``. They exist so that a change to the generalised
+API cannot silently move a published number: the values below are what
+``src.pipeline.experiment.upscaling_diagnostics`` returned when the preprint
+was written.
+"""
+import os
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from coarsegate import precondition_report
+
+PANELS = Path(os.environ["COARSEGATE_PANELS"]) if os.environ.get("COARSEGATE_PANELS") else None
+
+pytestmark = pytest.mark.skipif(
+    PANELS is None or not PANELS.is_dir(),
+    reason="set COARSEGATE_PANELS to the directory holding the analysis panels",
+)
+
+CLIMATE = ["CMD_sm", "CMD_sm_anom", "DD18_sm", "Eref_sm", "Eref_sm_anom",
+           "PPT_sm", "PPT_sm_anom", "Rad_sm", "Tmax_sm", "Tmax_sm_anom",
+           "Tmin_sm", "Tmin_sm_anom", "logPPT_sm", "novelty"]
+
+
+@pytest.mark.parametrize("panel, cell_size, published, informative", [
+    ("features.parquet", 4_000, 0.123, False),          # Surrey, 12.3%
+    ("features_phase3b.parquet", 25_000, 0.484, True),  # transect, 48.4%
+])
+def test_the_published_fraction_reproduces(panel, cell_size, published, informative):
+    df = pd.read_parquet(PANELS / panel).reset_index(drop=True)
+    r = precondition_report(df, CLIMATE, cell_size=cell_size,
+                            x="x_m", y="y_m", time="year")
+    assert round(r.f, 3) == published
+    assert r.passes is informative
+
+
+def test_surrey_leaves_the_two_arms_all_but_identical():
+    """The 0.998 median correlation the abstract reports."""
+    df = pd.read_parquet(PANELS / "features.parquet").reset_index(drop=True)
+    r = precondition_report(df, CLIMATE, cell_size=4_000,
+                            x="x_m", y="y_m", time="year")
+    assert round(float(r.correlations["r_fine_coarse"].median()), 3) == 0.998

coarsegate-0.1.0/tests/test_report.py

@@ -0,0 +1,38 @@
+import numpy as np
+import pandas as pd
+
+from coarsegate import precondition_report
+
+
+def _panel(spacing, n=6, seed=0):
+    rng = np.random.default_rng(seed)
+    xs, ys = np.meshgrid(np.arange(n) * spacing, np.arange(n) * spacing)
+    return pd.concat([
+        pd.DataFrame({"x": xs.ravel(), "y": ys.ravel(), "year": t,
+                      "p": rng.normal(size=n * n), "q": rng.normal(size=n * n)})
+        for t in (2020, 2021)
+    ], ignore_index=True)
+
+
+def test_a_coarsening_that_changes_nothing_is_reported_as_unanswerable():
+    r = precondition_report(_panel(10_000), ["p", "q"], cell_size=1000,
+                            x="x", y="y", time="year")
+    assert r.f < 0.4 and not r.passes and r.verdict == "uninformative"
+    assert "unanswerable" in str(r)
+    assert r.n_cells == r.n_sites == 36
+
+
+def test_a_coarsening_that_destroys_the_field_passes():
+    r = precondition_report(_panel(1000), ["p", "q"], cell_size=100_000,
+                            x="x", y="y", time="year")
+    assert r.f > 0.4 and r.passes
+    assert "unanswerable" not in str(r)
+    assert r.n_cells == 1
+
+
+def test_the_report_carries_the_per_feature_detail_not_just_the_summary():
+    r = precondition_report(_panel(1000), ["p", "q"], cell_size=2000,
+                            x="x", y="y", time="year")
+    assert list(r.per_feature["feature"]) and len(r.per_feature) == 2
+    assert r.f == r.per_feature["var_removed_frac"].median()
+    assert len(r.correlations) == 2