PyCDFt 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pycdft/__init__.py +59 -0
- pycdft/api.py +199 -0
- pycdft/core.py +489 -0
- pycdft/gpu/__init__.py +42 -0
- pycdft/gpu/_torch.py +29 -0
- pycdft/gpu/api.py +634 -0
- pycdft/gpu/devices.py +160 -0
- pycdft/gpu/kernels.py +554 -0
- pycdft/math.py +561 -0
- pycdft/py.typed +0 -0
- pycdft-0.1.0.dist-info/METADATA +289 -0
- pycdft-0.1.0.dist-info/RECORD +15 -0
- pycdft-0.1.0.dist-info/WHEEL +5 -0
- pycdft-0.1.0.dist-info/licenses/LICENSE +201 -0
- pycdft-0.1.0.dist-info/top_level.txt +1 -0
pycdft/core.py
ADDED
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
"""CDF-t calibration objects and the transfer pipeline (numpy only).
|
|
2
|
+
|
|
3
|
+
The CDF-t composition (Michelangeli, Vrac & Loukos 2009)
|
|
4
|
+
|
|
5
|
+
F_o^tar = F_o^cal o (F_m^cal)^{-1} o F_m^tar
|
|
6
|
+
y(x) = q_m^tar( F_m^cal( q_o^cal( F_m^tar(x) ) ) )
|
|
7
|
+
|
|
8
|
+
is computed per series (per grid cell) on a uniform probability grid
|
|
9
|
+
``p = linspace(0, 1, Q)`` (default Q = 1001, endpoints included), with:
|
|
10
|
+
|
|
11
|
+
- step ECDFs (``F(x) = #{finite <= x} / n``, NaN-excluded, right-continuous);
|
|
12
|
+
- type-7 linear quantiles (numpy/R convention);
|
|
13
|
+
- an optional per-series (per-cell) mean delta shift applied to the model
|
|
14
|
+
series (calibration and target) before the transfer -- the standard CDF-t
|
|
15
|
+
practice (R ``CDFt`` package, ibicus);
|
|
16
|
+
- the transfer curve ``(x_grid, T) = (q_m^tar(p), q_m^tar(R))`` evaluated on
|
|
17
|
+
the target's own value grid, interpolated per cell in x-space, with the
|
|
18
|
+
``clip`` (constant) or ``linear`` (windowed least-squares extrapolation,
|
|
19
|
+
slope capped at 2x the median interior slope) tail options;
|
|
20
|
+
- NaN robustness: NaNs never enter the counts or the interpolation; cells
|
|
21
|
+
whose calibration is empty (all-NaN) give NaN output.
|
|
22
|
+
|
|
23
|
+
Pipeline::
|
|
24
|
+
|
|
25
|
+
calib = calibrate(cm_hist, obs, p, delta_shift) # (R, delta, mode)
|
|
26
|
+
pred = apply_cdft(cm_future, calib, p, tail) # same shape
|
|
27
|
+
|
|
28
|
+
``calibrate`` keeps the calibration arrays in the caller's dtype (float32
|
|
29
|
+
storage with float64 per-chunk math is exact: float32 values convert exactly)
|
|
30
|
+
and processes cell chunks so that the ``(Q, chunk)`` float64 intermediates
|
|
31
|
+
stay below ``chunk_bytes`` (default 2 GiB) -- the memory-lean design of the
|
|
32
|
+
reference grid-scale implementation. The result is identical to a pure
|
|
33
|
+
float64 computation.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import warnings
|
|
39
|
+
from dataclasses import dataclass
|
|
40
|
+
from typing import Callable, Tuple
|
|
41
|
+
|
|
42
|
+
import numpy as np
|
|
43
|
+
|
|
44
|
+
from . import math as _m
|
|
45
|
+
|
|
46
|
+
__all__ = [
|
|
47
|
+
"Calibration",
|
|
48
|
+
"DEFAULT_CHUNK_BYTES",
|
|
49
|
+
"SHIFT_MODES",
|
|
50
|
+
"delta_from_means",
|
|
51
|
+
"apply_shift",
|
|
52
|
+
"calibrate",
|
|
53
|
+
"calibrate_hist",
|
|
54
|
+
"histograms",
|
|
55
|
+
"transfer",
|
|
56
|
+
"apply_cdft",
|
|
57
|
+
"apply_cdft_hist",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
DEFAULT_CHUNK_BYTES = 2 ** 31 # 2 GiB per (Q, chunk) float64 block
|
|
61
|
+
SHIFT_MODES = ("additive", "multiplicative", "none")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Calibration:
|
|
66
|
+
"""The target-independent CDF-t calibration.
|
|
67
|
+
|
|
68
|
+
Attributes
|
|
69
|
+
----------
|
|
70
|
+
R : (Q, N) float64
|
|
71
|
+
The calibration mapping ``R_k = F_m^cal(q_o^cal(p_k))`` (model
|
|
72
|
+
ECDF of the obs calibration quantiles; the model series in the
|
|
73
|
+
shifted frame).
|
|
74
|
+
delta : (N,) float64
|
|
75
|
+
Per-cell shift (``mean(obs) - mean(cm_hist)`` for additive,
|
|
76
|
+
``mean(obs) / mean(cm_hist)`` for multiplicative, ``0`` for none).
|
|
77
|
+
mode : str
|
|
78
|
+
One of ``{"additive", "multiplicative", "none"}``.
|
|
79
|
+
n_quantiles : int
|
|
80
|
+
Size of the probability grid ``p``.
|
|
81
|
+
"""
|
|
82
|
+
|
|
83
|
+
R: np.ndarray
|
|
84
|
+
delta: np.ndarray
|
|
85
|
+
mode: str
|
|
86
|
+
n_quantiles: int
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# ---------------------------------------------------------------------------
|
|
90
|
+
# Delta shift
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
|
|
93
|
+
def delta_from_means(mean_lr: np.ndarray, mean_hr: np.ndarray,
|
|
94
|
+
mode: str) -> np.ndarray:
|
|
95
|
+
"""Per-cell shift ``d`` from the calibration means (float64 arrays).
|
|
96
|
+
|
|
97
|
+
``mean(HR_cal) - mean(LR_cal)`` (additive) or
|
|
98
|
+
``mean(HR_cal) / mean(LR_cal)`` (multiplicative); ``0`` for ``"none"``.
|
|
99
|
+
Multiplicative falls back to ``1.0`` where the LR mean is ~0, the ratio
|
|
100
|
+
is non-finite or non-positive (signed variables).
|
|
101
|
+
"""
|
|
102
|
+
if mode == "additive":
|
|
103
|
+
return mean_hr - mean_lr
|
|
104
|
+
if mode == "multiplicative":
|
|
105
|
+
ok = np.abs(mean_lr) > 1e-12
|
|
106
|
+
d = np.where(ok, mean_hr / np.where(ok, mean_lr, 1.0), 1.0)
|
|
107
|
+
return np.where(np.isfinite(d) & (d > 0), d, 1.0)
|
|
108
|
+
return np.zeros_like(mean_lr)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def apply_shift(x, d, mode: str) -> np.ndarray:
|
|
112
|
+
"""Shift a (T, N) array by the per-cell delta (additive/multiplicative)."""
|
|
113
|
+
x = np.asarray(x, np.float64)
|
|
114
|
+
d = np.asarray(d, np.float64)
|
|
115
|
+
if mode == "additive":
|
|
116
|
+
return x + d[None, :]
|
|
117
|
+
if mode == "multiplicative":
|
|
118
|
+
return x * d[None, :]
|
|
119
|
+
return x
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
# ---------------------------------------------------------------------------
|
|
123
|
+
# Helpers
|
|
124
|
+
# ---------------------------------------------------------------------------
|
|
125
|
+
|
|
126
|
+
def _as_matrix(x, name: str = "array") -> Tuple[np.ndarray, Tuple[int, ...]]:
|
|
127
|
+
"""(T, ...) or (T,) input -> ((T, N) float64 matrix, original shape)."""
|
|
128
|
+
a = np.asarray(x, np.float64)
|
|
129
|
+
if a.ndim < 1:
|
|
130
|
+
raise ValueError(f"{name} must be at least 1-D, got shape {a.shape}")
|
|
131
|
+
return a.reshape(a.shape[0], -1), a.shape
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _mean_axis0(a: np.ndarray, nv: np.ndarray) -> np.ndarray:
|
|
135
|
+
"""Per-column mean of the finite values (NaN where a column is empty)."""
|
|
136
|
+
with np.errstate(invalid="ignore"):
|
|
137
|
+
s = np.where(np.isfinite(a), a, 0.0).sum(axis=0, dtype=np.float64)
|
|
138
|
+
return np.where(nv > 0, s / np.maximum(nv, 1), np.nan)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _cell_chunks(n: int, t_or_q: int,
|
|
142
|
+
chunk_bytes: int = DEFAULT_CHUNK_BYTES):
|
|
143
|
+
"""Column-chunk bounds [(s, e), ...] keeping the (Q, chunk) float64
|
|
144
|
+
intermediates below ``chunk_bytes``."""
|
|
145
|
+
chunk = max(1, min(n, chunk_bytes // (8 * max(t_or_q, 1))))
|
|
146
|
+
return [(s, min(s + chunk, n)) for s in range(0, n, chunk)]
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _check_grid(p, n_quantiles: int = 1001) -> np.ndarray:
|
|
150
|
+
"""Validate / default the probability grid."""
|
|
151
|
+
if p is None:
|
|
152
|
+
return _m.quantile_grid(n_quantiles)
|
|
153
|
+
p = np.asarray(p, np.float64)
|
|
154
|
+
if p.ndim != 1 or p.size < 2:
|
|
155
|
+
raise ValueError(f"p must be a 1-D grid with >= 2 points, got {p.shape}")
|
|
156
|
+
if not np.all(np.diff(p) > 0):
|
|
157
|
+
raise ValueError("p must be strictly increasing")
|
|
158
|
+
return p
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ---------------------------------------------------------------------------
|
|
162
|
+
# Exact calibration
|
|
163
|
+
# ---------------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
def calibrate(lr_cal, hr_cal, p=None, delta_shift: str = "additive",
|
|
166
|
+
workers: int = 1, chunk_bytes: int = DEFAULT_CHUNK_BYTES,
|
|
167
|
+
n_quantiles: int = 1001) -> Calibration:
|
|
168
|
+
"""Build the CDF-t calibration ``R = F_m^cal(q_o^cal(p))`` + per-cell delta.
|
|
169
|
+
|
|
170
|
+
Parameters
|
|
171
|
+
----------
|
|
172
|
+
lr_cal : (T, N) or (T,) array
|
|
173
|
+
Model (coarse) series over the calibration period.
|
|
174
|
+
hr_cal : (T, N) or (T,) array
|
|
175
|
+
Observation (fine) series over the calibration period (same T).
|
|
176
|
+
p : (Q,) array, optional
|
|
177
|
+
Probability grid (default ``quantile_grid(1001)``, endpoints 0 and 1).
|
|
178
|
+
delta_shift : {"additive", "multiplicative", "none"}
|
|
179
|
+
Per-cell mean shift applied to the model series before the transfer.
|
|
180
|
+
workers : int
|
|
181
|
+
Threads for the per-cell kernels.
|
|
182
|
+
chunk_bytes : int
|
|
183
|
+
Memory budget for the (Q, chunk) float64 intermediates.
|
|
184
|
+
n_quantiles : int
|
|
185
|
+
Default grid size when ``p`` is None.
|
|
186
|
+
|
|
187
|
+
Returns
|
|
188
|
+
-------
|
|
189
|
+
Calibration
|
|
190
|
+
``R`` (Q, N), ``delta`` (N,), ``mode``, ``n_quantiles``. Cells whose
|
|
191
|
+
calibration is empty (all-NaN in either series) get ``R = NaN`` and
|
|
192
|
+
produce NaN output downstream.
|
|
193
|
+
|
|
194
|
+
Notes
|
|
195
|
+
-----
|
|
196
|
+
The arrays are sorted in place on a private copy (the caller's data is
|
|
197
|
+
never modified); the sort keeps the input dtype (float32 stays float32,
|
|
198
|
+
the per-chunk math is float64 and exact -- bit-identical to sorting
|
|
199
|
+
float64 first), and all per-cell math runs on cell chunks so that the
|
|
200
|
+
peak memory is ``O((T + Q) * chunk_bytes / 8)``, not ``O(T * N)``
|
|
201
|
+
float64.
|
|
202
|
+
"""
|
|
203
|
+
if delta_shift not in SHIFT_MODES:
|
|
204
|
+
raise ValueError(f"delta_shift must be one of {SHIFT_MODES}, "
|
|
205
|
+
f"got {delta_shift!r}")
|
|
206
|
+
p = _check_grid(p, n_quantiles)
|
|
207
|
+
lr = np.asarray(lr_cal)
|
|
208
|
+
hr = np.asarray(hr_cal)
|
|
209
|
+
if lr.ndim == 1:
|
|
210
|
+
lr = lr[:, None]
|
|
211
|
+
if hr.ndim == 1:
|
|
212
|
+
hr = hr[:, None]
|
|
213
|
+
if lr.ndim < 2 or hr.ndim < 2:
|
|
214
|
+
raise ValueError("lr_cal/hr_cal must be 1-D or (T, ...) arrays")
|
|
215
|
+
if lr.shape[0] != hr.shape[0]:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"calibration series lengths differ: {lr.shape[0]} vs "
|
|
218
|
+
f"{hr.shape[0]}")
|
|
219
|
+
# private copies: np.sort is done in place and must not touch the
|
|
220
|
+
# caller's arrays; the input dtype is kept (float32 stays float32 ->
|
|
221
|
+
# memory-lean sort; the per-chunk math is float64 and exact)
|
|
222
|
+
lr = np.array(lr, copy=True).reshape(lr.shape[0], -1)
|
|
223
|
+
hr = np.array(hr, copy=True).reshape(hr.shape[0], -1)
|
|
224
|
+
N = lr.shape[1]
|
|
225
|
+
nv_lr = np.isfinite(lr).sum(axis=0).astype(np.float64)
|
|
226
|
+
nv_hr = np.isfinite(hr).sum(axis=0).astype(np.float64)
|
|
227
|
+
mean_lr = _mean_axis0(lr, nv_lr)
|
|
228
|
+
mean_hr = _mean_axis0(hr, nv_hr)
|
|
229
|
+
delta = delta_from_means(mean_lr, mean_hr, delta_shift)
|
|
230
|
+
lr.sort(axis=0) # NaNs to the end, excluded by counts
|
|
231
|
+
hr.sort(axis=0)
|
|
232
|
+
Q = p.size
|
|
233
|
+
R = np.empty((Q, N), dtype=np.float64)
|
|
234
|
+
for s, e in _cell_chunks(N, max(lr.shape[0], Q), chunk_bytes):
|
|
235
|
+
# per-chunk float64 math (bit-identical to a full float64 run)
|
|
236
|
+
q_hr = _m.quantile_from_sorted(
|
|
237
|
+
np.asarray(hr[:, s:e], np.float64), p) # (Q, chunk)
|
|
238
|
+
lc = np.asarray(lr[:, s:e], np.float64) # (T, chunk)
|
|
239
|
+
if delta_shift == "additive":
|
|
240
|
+
lc = lc + delta[None, s:e]
|
|
241
|
+
elif delta_shift == "multiplicative":
|
|
242
|
+
lc = lc * delta[None, s:e]
|
|
243
|
+
R[:, s:e] = _m._ecdf_at_sorted(lc, q_hr)
|
|
244
|
+
return Calibration(R=R, delta=delta, mode=delta_shift, n_quantiles=Q)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
# ---------------------------------------------------------------------------
|
|
248
|
+
# Histogram calibration
|
|
249
|
+
# ---------------------------------------------------------------------------
|
|
250
|
+
|
|
251
|
+
def histograms(values, n_bins: int = 2000):
|
|
252
|
+
"""Per-cell histograms with per-cell edges (in-memory, 2 passes).
|
|
253
|
+
|
|
254
|
+
Parameters
|
|
255
|
+
----------
|
|
256
|
+
values : (T, N) array.
|
|
257
|
+
n_bins : int, number of bins per cell.
|
|
258
|
+
|
|
259
|
+
Returns
|
|
260
|
+
-------
|
|
261
|
+
((B + 1, N) edges, (B, N) counts): NaNs excluded from the binning;
|
|
262
|
+
degenerate (constant) cells get a ±1 padding, empty (all-NaN) cells get
|
|
263
|
+
zero counts (and NaN edges internally, quantiles -> NaN).
|
|
264
|
+
"""
|
|
265
|
+
values = np.asarray(values, np.float64)
|
|
266
|
+
if values.ndim == 1:
|
|
267
|
+
values = values[:, None]
|
|
268
|
+
if values.ndim != 2:
|
|
269
|
+
raise ValueError(f"expected a 2-D (T, N) array, got {values.shape}")
|
|
270
|
+
valid = np.isfinite(values)
|
|
271
|
+
nv = valid.sum(axis=0)
|
|
272
|
+
with np.errstate(invalid="ignore"):
|
|
273
|
+
lo = np.where(valid, values, np.inf).min(axis=0)
|
|
274
|
+
hi = np.where(valid, values, -np.inf).max(axis=0)
|
|
275
|
+
degenerate = hi <= lo
|
|
276
|
+
lo = np.where(degenerate, lo - 1.0, lo)
|
|
277
|
+
hi = np.where(degenerate, hi + 1.0, hi)
|
|
278
|
+
empty = nv == 0
|
|
279
|
+
lo = np.where(empty, 0.0, lo)
|
|
280
|
+
hi = np.where(empty, 1.0, hi)
|
|
281
|
+
edges = np.linspace(lo, hi, n_bins + 1) # (B+1, N)
|
|
282
|
+
width = (hi - lo)[None, :]
|
|
283
|
+
with warnings.catch_warnings():
|
|
284
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
285
|
+
idx = np.clip(((values - lo[None, :])
|
|
286
|
+
/ np.where(width > 0, width, 1.0)
|
|
287
|
+
* n_bins).astype(np.int64), 0, n_bins - 1)
|
|
288
|
+
idx = np.where(valid, idx, n_bins) # sentinel bin
|
|
289
|
+
stride = n_bins + 1
|
|
290
|
+
N = values.shape[1]
|
|
291
|
+
counts = np.zeros((n_bins + 1, N), dtype=np.float64)
|
|
292
|
+
flat = idx + (np.arange(N) * stride)[None]
|
|
293
|
+
counts += np.bincount(flat.ravel(), minlength=stride * N) \
|
|
294
|
+
.reshape(N, stride).T
|
|
295
|
+
return edges, counts[:-1]
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def calibrate_hist(hist_lr, hist_hr, means, p=None, delta_shift: str = "additive",
|
|
299
|
+
workers: int = 1, n_quantiles: int = 1001) -> Calibration:
|
|
300
|
+
"""Histogram calibration (same :class:`Calibration` object as ``calibrate``).
|
|
301
|
+
|
|
302
|
+
Parameters
|
|
303
|
+
----------
|
|
304
|
+
hist_lr, hist_hr : (edges, counts)
|
|
305
|
+
Per-cell histograms of the model / observation calibration series
|
|
306
|
+
(from :func:`histograms`; edges (B + 1, N), counts (B, N)).
|
|
307
|
+
means : (mean_lr, mean_hr)
|
|
308
|
+
Exact per-cell means (float64 (N,)) accumulated while streaming.
|
|
309
|
+
p : (Q,) array, optional probability grid.
|
|
310
|
+
delta_shift : {"additive", "multiplicative", "none"}.
|
|
311
|
+
|
|
312
|
+
Returns
|
|
313
|
+
-------
|
|
314
|
+
Calibration
|
|
315
|
+
"""
|
|
316
|
+
if delta_shift not in SHIFT_MODES:
|
|
317
|
+
raise ValueError(f"delta_shift must be one of {SHIFT_MODES}, "
|
|
318
|
+
f"got {delta_shift!r}")
|
|
319
|
+
p = _check_grid(p, n_quantiles)
|
|
320
|
+
(edges_lr, counts_lr), (edges_hr, counts_hr) = hist_lr, hist_hr
|
|
321
|
+
mean_lr, mean_hr = means
|
|
322
|
+
mean_lr = np.atleast_1d(np.asarray(mean_lr, np.float64))
|
|
323
|
+
mean_hr = np.atleast_1d(np.asarray(mean_hr, np.float64))
|
|
324
|
+
if mean_lr.size == 1 and counts_hr.shape[1] != 1:
|
|
325
|
+
mean_lr = np.full(counts_hr.shape[1], mean_lr[0])
|
|
326
|
+
mean_hr = np.full(counts_hr.shape[1], mean_hr[0])
|
|
327
|
+
delta = np.atleast_1d(delta_from_means(mean_lr, mean_hr, delta_shift))
|
|
328
|
+
p_grid = np.broadcast_to(p[:, None], (p.size, counts_hr.shape[1]))
|
|
329
|
+
q_hr = _m.hist_quantile_pc(edges_hr, counts_hr, p_grid, workers=workers)
|
|
330
|
+
if delta_shift == "additive":
|
|
331
|
+
q_unshift = q_hr - delta[None, :]
|
|
332
|
+
elif delta_shift == "multiplicative":
|
|
333
|
+
q_unshift = q_hr / np.where(delta[None, :] != 0, delta[None, :], 1.0)
|
|
334
|
+
else:
|
|
335
|
+
q_unshift = q_hr
|
|
336
|
+
R = _m.ecdf_hist(edges_lr, counts_lr, q_unshift, workers=workers)
|
|
337
|
+
return Calibration(R=R, delta=delta, mode=delta_shift, n_quantiles=p.size)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
# ---------------------------------------------------------------------------
|
|
341
|
+
# Transfer
|
|
342
|
+
# ---------------------------------------------------------------------------
|
|
343
|
+
|
|
344
|
+
def transfer(q_tar: Callable[[np.ndarray, int, int], np.ndarray], R, p,
|
|
345
|
+
workers: int = 1, chunk_bytes: int = DEFAULT_CHUNK_BYTES):
|
|
346
|
+
"""Build the transfer curve on the target value grid.
|
|
347
|
+
|
|
348
|
+
Parameters
|
|
349
|
+
----------
|
|
350
|
+
q_tar : callable ``q_tar(q, s, e) -> (Q, e - s)``
|
|
351
|
+
Quantile function of the (shifted) target series for cell columns
|
|
352
|
+
``[s, e)``, evaluated at the probabilities ``q`` (Q, chunk).
|
|
353
|
+
R : (Q, N) calibration mapping (may contain NaN for empty cells).
|
|
354
|
+
p : (Q,) probability grid.
|
|
355
|
+
|
|
356
|
+
Returns
|
|
357
|
+
-------
|
|
358
|
+
(x_grid (Q, N), T (Q, N), k_lo (N,), k_hi (N,), bad (N,))
|
|
359
|
+
``x_grid = q_m^tar(p)``, ``T = q_m^tar(R)`` (target value grid and
|
|
360
|
+
mapped values), the first / last unsaturated ranks of ``R`` and the
|
|
361
|
+
flag of cells with a NaN calibration (R replaced by 0.5 in the
|
|
362
|
+
lookup, output must be set to NaN downstream).
|
|
363
|
+
"""
|
|
364
|
+
R = np.asarray(R, np.float64)
|
|
365
|
+
p = np.asarray(p, np.float64)
|
|
366
|
+
Q, N = R.shape
|
|
367
|
+
if p.size != Q:
|
|
368
|
+
raise ValueError(f"p size {p.size} != R rows {Q}")
|
|
369
|
+
bad = ~np.isfinite(R).all(axis=0) # NaN calibration cells
|
|
370
|
+
R = np.where(bad[None, :], 0.5, R)
|
|
371
|
+
x_grid = np.empty((Q, N), dtype=np.float64)
|
|
372
|
+
T = np.empty((Q, N), dtype=np.float64)
|
|
373
|
+
k_lo = np.empty(N, dtype=np.int64)
|
|
374
|
+
k_hi = np.empty(N, dtype=np.int64)
|
|
375
|
+
for s, e in _cell_chunks(N, Q, chunk_bytes):
|
|
376
|
+
T[:, s:e] = q_tar(R[:, s:e], s, e)
|
|
377
|
+
x_grid[:, s:e] = q_tar(np.broadcast_to(p[:, None], (Q, e - s)), s, e)
|
|
378
|
+
k_lo[s:e] = np.argmax(R[:, s:e] > 0, axis=0)
|
|
379
|
+
k_hi[s:e] = Q - 1 - np.argmax((R[:, s:e] < 1)[::-1], axis=0)
|
|
380
|
+
return x_grid, T, k_lo, k_hi, bad
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
# ---------------------------------------------------------------------------
|
|
384
|
+
# Apply (target side)
|
|
385
|
+
# ---------------------------------------------------------------------------
|
|
386
|
+
|
|
387
|
+
def apply_cdft(lr_tar, calib: Calibration, p=None, tail: str = "linear",
|
|
388
|
+
workers: int = 1, chunk_bytes: int = DEFAULT_CHUNK_BYTES,
|
|
389
|
+
n_quantiles: int = 1001):
|
|
390
|
+
"""Apply a precomputed calibration to the target model series.
|
|
391
|
+
|
|
392
|
+
Parameters
|
|
393
|
+
----------
|
|
394
|
+
lr_tar : (T, N) or (T, ...) array
|
|
395
|
+
Model series to downscale / bias-correct (any shape; the first axis
|
|
396
|
+
is time).
|
|
397
|
+
calib : Calibration
|
|
398
|
+
From :func:`calibrate` or :func:`calibrate_hist`.
|
|
399
|
+
p : (Q,) array, optional probability grid (must match the calibration).
|
|
400
|
+
tail : {"linear", "clip"}
|
|
401
|
+
workers : int, threads for the per-cell kernels.
|
|
402
|
+
chunk_bytes : int, memory budget for the (Q, chunk) intermediates.
|
|
403
|
+
|
|
404
|
+
Returns
|
|
405
|
+
-------
|
|
406
|
+
Array of the same shape as ``lr_tar``, in the shifted frame (the mean
|
|
407
|
+
bias model-vs-observation is removed, the model change between
|
|
408
|
+
calibration and target is preserved). NaN inputs map to NaN; cells with
|
|
409
|
+
an empty (all-NaN) calibration map entirely to NaN.
|
|
410
|
+
"""
|
|
411
|
+
if tail not in ("linear", "clip"):
|
|
412
|
+
raise ValueError(f"tail must be 'linear' or 'clip', got {tail!r}")
|
|
413
|
+
if p is None:
|
|
414
|
+
p = _m.quantile_grid(calib.R.shape[0])
|
|
415
|
+
else:
|
|
416
|
+
p = _check_grid(p, n_quantiles)
|
|
417
|
+
if p.size != calib.R.shape[0]:
|
|
418
|
+
raise ValueError(
|
|
419
|
+
f"probability grid size {p.size} does not match the "
|
|
420
|
+
f"calibration ({calib.R.shape[0]}); pass the same grid used "
|
|
421
|
+
f"in calibrate()")
|
|
422
|
+
x, shape = _as_matrix(lr_tar, "lr_tar")
|
|
423
|
+
delta, mode = calib.delta, calib.mode
|
|
424
|
+
if mode == "additive":
|
|
425
|
+
xs = x + delta[None, :]
|
|
426
|
+
elif mode == "multiplicative":
|
|
427
|
+
xs = x * delta[None, :]
|
|
428
|
+
else:
|
|
429
|
+
xs = x
|
|
430
|
+
v_tar = np.sort(xs, axis=0) # NaNs to the end
|
|
431
|
+
x_grid, T, k_lo, k_hi, bad = transfer(
|
|
432
|
+
lambda q, s, e: _m.quantile_from_sorted(v_tar[:, s:e], q),
|
|
433
|
+
calib.R, p, workers, chunk_bytes)
|
|
434
|
+
pred = _m.interp_transfer(x_grid, T, k_lo, k_hi, xs, tail=tail,
|
|
435
|
+
workers=workers)
|
|
436
|
+
if bad.any():
|
|
437
|
+
pred[:, bad] = np.nan # empty (NaN) calibration cells -> NaN
|
|
438
|
+
return pred.reshape(shape)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def apply_cdft_hist(lr_tar, calib: Calibration, p=None, n_bins: int = 2000,
|
|
442
|
+
tail: str = "linear", workers: int = 1,
|
|
443
|
+
n_quantiles: int = 1001):
|
|
444
|
+
"""Same as :func:`apply_cdft`, with the target CDF accumulated as a
|
|
445
|
+
per-cell histogram (bounded memory for very long records).
|
|
446
|
+
|
|
447
|
+
Parameters
|
|
448
|
+
----------
|
|
449
|
+
lr_tar : (T, N) or (T, ...) array
|
|
450
|
+
calib : Calibration
|
|
451
|
+
p : (Q,) array, optional probability grid.
|
|
452
|
+
n_bins : int, bins for the target histogram (global edges).
|
|
453
|
+
tail : {"linear", "clip"}
|
|
454
|
+
workers : int
|
|
455
|
+
|
|
456
|
+
Returns
|
|
457
|
+
-------
|
|
458
|
+
Array of the same shape as ``lr_tar`` (histogram approximation of
|
|
459
|
+
:func:`apply_cdft`, exact in the limit of many bins).
|
|
460
|
+
"""
|
|
461
|
+
if tail not in ("linear", "clip"):
|
|
462
|
+
raise ValueError(f"tail must be 'linear' or 'clip', got {tail!r}")
|
|
463
|
+
if p is None:
|
|
464
|
+
p = _m.quantile_grid(calib.R.shape[0])
|
|
465
|
+
else:
|
|
466
|
+
p = _check_grid(p, n_quantiles)
|
|
467
|
+
if p.size != calib.R.shape[0]:
|
|
468
|
+
raise ValueError(
|
|
469
|
+
f"probability grid size {p.size} does not match the "
|
|
470
|
+
f"calibration ({calib.R.shape[0]}); pass the same grid used "
|
|
471
|
+
f"in calibrate()")
|
|
472
|
+
x, shape = _as_matrix(lr_tar, "lr_tar")
|
|
473
|
+
delta, mode = calib.delta, calib.mode
|
|
474
|
+
if mode == "additive":
|
|
475
|
+
xs = x + delta[None, :]
|
|
476
|
+
elif mode == "multiplicative":
|
|
477
|
+
xs = x * delta[None, :]
|
|
478
|
+
else:
|
|
479
|
+
xs = x
|
|
480
|
+
edges, counts = _m.histogram_accumulate(xs, n_bins=n_bins)
|
|
481
|
+
x_grid, T, k_lo, k_hi, bad = transfer(
|
|
482
|
+
lambda q, s, e: _m.hist_quantile(edges, counts[:, s:e], q,
|
|
483
|
+
workers=workers),
|
|
484
|
+
calib.R, p, workers)
|
|
485
|
+
pred = _m.interp_transfer(x_grid, T, k_lo, k_hi, xs, tail=tail,
|
|
486
|
+
workers=workers)
|
|
487
|
+
if bad.any():
|
|
488
|
+
pred[:, bad] = np.nan
|
|
489
|
+
return pred.reshape(shape)
|
pycdft/gpu/__init__.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""pycdft.gpu -- torch (device-agnostic) backend for CDF-t.
|
|
2
|
+
|
|
3
|
+
A mirror of the :mod:`pycdft` numpy API implemented with torch operators, so
|
|
4
|
+
that the same CDF-t math can run on CUDA / AMD ROCm / Apple MPS / CPU
|
|
5
|
+
devices::
|
|
6
|
+
|
|
7
|
+
from pycdft.gpu import CDFt # mirrors pycdft.CDFt
|
|
8
|
+
ds = CDFt(device="cuda").fit_transform(obs, cm_hist, cm_future)
|
|
9
|
+
|
|
10
|
+
Torch is an optional dependency: importing this module never fails without
|
|
11
|
+
torch (``TORCH_AVAILABLE`` is then ``False`` and the functions raise a clear
|
|
12
|
+
``ImportError`` when called). See ``docs/gpu.md`` for the supported devices
|
|
13
|
+
and the verified numerical equivalence with the numpy backend.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from ._torch import TORCH_AVAILABLE
|
|
17
|
+
from .devices import (available_devices, describe, is_rocm_build,
|
|
18
|
+
resolve_device, rocm_available)
|
|
19
|
+
from .api import (CDFt, Calibration, apply_cdft, apply_cdft_hist, calibrate,
|
|
20
|
+
calibrate_hist, cdft_transform, delta_from_means,
|
|
21
|
+
histograms, transfer)
|
|
22
|
+
from . import kernels
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"TORCH_AVAILABLE",
|
|
26
|
+
"available_devices",
|
|
27
|
+
"resolve_device",
|
|
28
|
+
"rocm_available",
|
|
29
|
+
"is_rocm_build",
|
|
30
|
+
"describe",
|
|
31
|
+
"kernels",
|
|
32
|
+
"Calibration",
|
|
33
|
+
"calibrate",
|
|
34
|
+
"calibrate_hist",
|
|
35
|
+
"apply_cdft",
|
|
36
|
+
"apply_cdft_hist",
|
|
37
|
+
"histograms",
|
|
38
|
+
"delta_from_means",
|
|
39
|
+
"transfer",
|
|
40
|
+
"CDFt",
|
|
41
|
+
"cdft_transform",
|
|
42
|
+
]
|
pycdft/gpu/_torch.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Internal torch accessor for :mod:`pycdft.gpu` (optional dependency).
|
|
2
|
+
|
|
3
|
+
Torch is an *optional* dependency of pycdft: the package itself is pure
|
|
4
|
+
numpy and imports fine without torch. This module centralizes the guarded
|
|
5
|
+
import so that every other module of the GPU backend can rely on
|
|
6
|
+
:func:`require_torch` to raise a clear error only when torch is actually
|
|
7
|
+
needed.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
try: # pragma: no cover - exercised only
|
|
13
|
+
import torch # when torch is missing
|
|
14
|
+
TORCH_AVAILABLE = True
|
|
15
|
+
except ImportError: # pragma: no cover
|
|
16
|
+
torch = None # type: ignore[assignment]
|
|
17
|
+
TORCH_AVAILABLE = False
|
|
18
|
+
|
|
19
|
+
__all__ = ["TORCH_AVAILABLE", "require_torch"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def require_torch():
|
|
23
|
+
"""Return the ``torch`` module or raise an informative ``ImportError``."""
|
|
24
|
+
if not TORCH_AVAILABLE:
|
|
25
|
+
raise ImportError(
|
|
26
|
+
"pycdft.gpu requires torch: install it with "
|
|
27
|
+
"`pip install pycdft[gpu]` (or a torch build matching your "
|
|
28
|
+
"hardware, see docs/gpu.md)")
|
|
29
|
+
return torch
|