PyCDFt 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pycdft/__init__.py +59 -0
- pycdft/api.py +199 -0
- pycdft/core.py +489 -0
- pycdft/gpu/__init__.py +42 -0
- pycdft/gpu/_torch.py +29 -0
- pycdft/gpu/api.py +634 -0
- pycdft/gpu/devices.py +160 -0
- pycdft/gpu/kernels.py +554 -0
- pycdft/math.py +561 -0
- pycdft/py.typed +0 -0
- pycdft-0.1.0.dist-info/METADATA +289 -0
- pycdft-0.1.0.dist-info/RECORD +15 -0
- pycdft-0.1.0.dist-info/WHEEL +5 -0
- pycdft-0.1.0.dist-info/licenses/LICENSE +201 -0
- pycdft-0.1.0.dist-info/top_level.txt +1 -0
pycdft/gpu/api.py
ADDED
|
@@ -0,0 +1,634 @@
|
|
|
1
|
+
"""High-level torch CDF-t API (``pycdft.gpu``).
|
|
2
|
+
|
|
3
|
+
A device-agnostic mirror of :mod:`pycdft.api` / :mod:`pycdft.core`: the same
|
|
4
|
+
CDF-t math (Michelangeli, Vrac & Loukos 2009, with the delta-shift and tail
|
|
5
|
+
conventions of the R ``CDFt`` package and ibicus) implemented with torch
|
|
6
|
+
operators, so that the per-cell kernels can run on a GPU (CUDA, AMD ROCm,
|
|
7
|
+
Apple MPS) or on CPU. The high-level :class:`CDFt` class mirrors
|
|
8
|
+
:class:`pycdft.CDFt` (same parameters, same shapes, numpy in / numpy out by
|
|
9
|
+
default) and the low-level functions (:func:`calibrate`,
|
|
10
|
+
:func:`apply_cdft`, ...) mirror :mod:`pycdft.core`.
|
|
11
|
+
|
|
12
|
+
Devices
|
|
13
|
+
-------
|
|
14
|
+
- :func:`pycdft.gpu.available_devices` reports what torch exposes on the
|
|
15
|
+
machine; :func:`pycdft.gpu.resolve_device` picks the best available
|
|
16
|
+
device (``cuda`` -> ``mps`` -> ``cpu``) or validates an explicit request
|
|
17
|
+
(an explicit unavailable device raises ``ValueError``, never a silent CPU
|
|
18
|
+
fallback).
|
|
19
|
+
- On AMD hardware, a *ROCm build* of torch is required: ROCm builds expose
|
|
20
|
+
the AMD GPUs through the ``cuda`` API (``torch.cuda.is_available()`` is
|
|
21
|
+
true, ``torch.version.hip`` is set). A CUDA-built torch simply reports no
|
|
22
|
+
device and the backend runs on CPU.
|
|
23
|
+
|
|
24
|
+
Dtype / device handling
|
|
25
|
+
-----------------------
|
|
26
|
+
Inputs (numpy arrays or torch tensors) are converted to float64 tensors on
|
|
27
|
+
the target device -- the float64 CDF-t kernels then produce results that
|
|
28
|
+
match the numpy backend to the last significant digit for the same data
|
|
29
|
+
(see ``tests/test_gpu.py`` and ``docs/gpu.md`` for the verified tolerance).
|
|
30
|
+
Outputs are numpy float64 arrays by default (``as_numpy=True``), i.e. a
|
|
31
|
+
drop-in replacement for the numpy API; ``as_numpy=False`` returns torch
|
|
32
|
+
tensors on the compute device.
|
|
33
|
+
|
|
34
|
+
Memory notes
|
|
35
|
+
------------
|
|
36
|
+
The kernels are fully vectorized: peak GPU memory for a field is
|
|
37
|
+
``O((T + Q) * N * 8 bytes)`` (plus one sorted copy of the input), i.e. the
|
|
38
|
+
whole field is processed at once, unlike the chunked numpy core. For very
|
|
39
|
+
large fields, pass ``chunk_cols`` to process the cells in column chunks
|
|
40
|
+
(identical results, bounded memory, e.g. ``chunk_cols=4096``).
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
from __future__ import annotations
|
|
44
|
+
|
|
45
|
+
from dataclasses import dataclass
|
|
46
|
+
from typing import TYPE_CHECKING
|
|
47
|
+
|
|
48
|
+
import numpy as np
|
|
49
|
+
|
|
50
|
+
from ._torch import require_torch
|
|
51
|
+
from .devices import resolve_device
|
|
52
|
+
from . import kernels as _k
|
|
53
|
+
|
|
54
|
+
if TYPE_CHECKING: # annotations only, never runtime
|
|
55
|
+
import torch
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
__all__ = [
|
|
59
|
+
"Calibration",
|
|
60
|
+
"SHIFT_MODES",
|
|
61
|
+
"delta_from_means",
|
|
62
|
+
"apply_shift",
|
|
63
|
+
"calibrate",
|
|
64
|
+
"calibrate_hist",
|
|
65
|
+
"histograms",
|
|
66
|
+
"transfer",
|
|
67
|
+
"apply_cdft",
|
|
68
|
+
"apply_cdft_hist",
|
|
69
|
+
"CDFt",
|
|
70
|
+
"cdft_transform",
|
|
71
|
+
]
|
|
72
|
+
|
|
73
|
+
SHIFT_MODES = ("additive", "multiplicative", "none")
|
|
74
|
+
_TAILS = ("linear", "clip")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _as_matrix(x, device, name: str = "array"):
|
|
78
|
+
"""(T, ...) / (T,) input -> ((T, N) float64 tensor on ``device``, shape).
|
|
79
|
+
|
|
80
|
+
Accepts numpy arrays (converted, never shared) and torch tensors (moved
|
|
81
|
+
to ``device`` and converted to float64).
|
|
82
|
+
"""
|
|
83
|
+
torch = require_torch()
|
|
84
|
+
if isinstance(x, torch.Tensor):
|
|
85
|
+
t = x.to(device=device, dtype=torch.float64)
|
|
86
|
+
else:
|
|
87
|
+
t = torch.as_tensor(x, device=device, dtype=torch.float64)
|
|
88
|
+
if t.ndim < 1:
|
|
89
|
+
raise ValueError(f"{name} must be at least 1-D, got shape {tuple(t.shape)}")
|
|
90
|
+
shape = tuple(t.shape)
|
|
91
|
+
return t.reshape(t.shape[0], -1), shape
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _as_1d(x, device, name: str = "array"):
|
|
95
|
+
"""Convert a 1-D array / tensor (numpy or torch) to a device tensor."""
|
|
96
|
+
torch = require_torch()
|
|
97
|
+
if isinstance(x, torch.Tensor):
|
|
98
|
+
t = x.to(device=device, dtype=torch.float64)
|
|
99
|
+
else:
|
|
100
|
+
t = torch.as_tensor(x, device=device, dtype=torch.float64)
|
|
101
|
+
if t.ndim != 1:
|
|
102
|
+
raise ValueError(f"{name} must be 1-D, got shape {tuple(t.shape)}")
|
|
103
|
+
return t
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _check_grid(p, n_quantiles: int = 1001, device=None):
|
|
107
|
+
"""Validate / default the probability grid (tensor on ``device``)."""
|
|
108
|
+
torch = require_torch()
|
|
109
|
+
if p is None:
|
|
110
|
+
return _k.quantile_grid(n_quantiles, device=device)
|
|
111
|
+
p = _as_1d(p, device, "p")
|
|
112
|
+
if p.size(0) < 2:
|
|
113
|
+
raise ValueError(f"p must be a 1-D grid with >= 2 points, got {p.size(0)}")
|
|
114
|
+
if not bool(torch.all(torch.diff(p) > 0)):
|
|
115
|
+
raise ValueError("p must be strictly increasing")
|
|
116
|
+
return p
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _mean_axis0(a, nv):
|
|
120
|
+
"""Per-column mean of the finite values (NaN where a column is empty)."""
|
|
121
|
+
torch = require_torch()
|
|
122
|
+
s = torch.where(torch.isfinite(a), a, torch.zeros_like(a)).sum(dim=0)
|
|
123
|
+
return torch.where(nv > 0, s / nv.clamp(min=1.0), torch.full_like(s, float("nan")))
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _cell_chunks(n: int, chunk_cols=None):
|
|
127
|
+
"""Column-chunk bounds [(s, e), ...] (single chunk when None)."""
|
|
128
|
+
if chunk_cols is None or chunk_cols <= 0:
|
|
129
|
+
return [(0, n)]
|
|
130
|
+
chunk = max(1, int(chunk_cols))
|
|
131
|
+
return [(s, min(s + chunk, n)) for s in range(0, n, chunk)]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# Delta shift
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
|
|
138
|
+
def delta_from_means(mean_lr, mean_hr, mode: str):
|
|
139
|
+
"""Per-cell shift ``d`` from the calibration means (float64 tensors).
|
|
140
|
+
|
|
141
|
+
``mean(HR_cal) - mean(LR_cal)`` (additive) or
|
|
142
|
+
``mean(HR_cal) / mean(LR_cal)`` (multiplicative); ``0`` for ``"none"``.
|
|
143
|
+
Multiplicative falls back to ``1.0`` where the LR mean is ~0, the ratio
|
|
144
|
+
is non-finite or non-positive (signed variables). Identical to
|
|
145
|
+
:func:`pycdft.core.delta_from_means`.
|
|
146
|
+
"""
|
|
147
|
+
torch = require_torch()
|
|
148
|
+
if mode == "additive":
|
|
149
|
+
return mean_hr - mean_lr
|
|
150
|
+
if mode == "multiplicative":
|
|
151
|
+
ok = mean_lr.abs() > 1e-12
|
|
152
|
+
d = torch.where(ok, mean_hr / torch.where(ok, mean_lr,
|
|
153
|
+
torch.ones_like(mean_lr)),
|
|
154
|
+
torch.ones_like(mean_lr))
|
|
155
|
+
return torch.where(torch.isfinite(d) & (d > 0), d, torch.ones_like(d))
|
|
156
|
+
return torch.zeros_like(mean_lr)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def apply_shift(x, d, mode: str):
|
|
160
|
+
"""Shift a (T, N) tensor by the per-cell delta (additive/multiplicative)."""
|
|
161
|
+
if mode == "additive":
|
|
162
|
+
return x + d[None, :]
|
|
163
|
+
if mode == "multiplicative":
|
|
164
|
+
return x * d[None, :]
|
|
165
|
+
return x
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# ---------------------------------------------------------------------------
|
|
169
|
+
# Calibration object
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
@dataclass
|
|
173
|
+
class Calibration:
|
|
174
|
+
"""The target-independent CDF-t calibration (torch tensors).
|
|
175
|
+
|
|
176
|
+
Attributes
|
|
177
|
+
----------
|
|
178
|
+
R : (Q, N) float64 tensor
|
|
179
|
+
The calibration mapping ``R_k = F_m^cal(q_o^cal(p_k))``.
|
|
180
|
+
delta : (N,) float64 tensor
|
|
181
|
+
Per-cell shift (``mean(obs) - mean(cm_hist)`` for additive,
|
|
182
|
+
``mean(obs) / mean(cm_hist)`` for multiplicative, ``0`` for none).
|
|
183
|
+
mode : str
|
|
184
|
+
One of ``{"additive", "multiplicative", "none"}``.
|
|
185
|
+
n_quantiles : int
|
|
186
|
+
Size of the probability grid ``p``.
|
|
187
|
+
"""
|
|
188
|
+
|
|
189
|
+
R: torch.Tensor
|
|
190
|
+
delta: torch.Tensor
|
|
191
|
+
mode: str
|
|
192
|
+
n_quantiles: int
|
|
193
|
+
|
|
194
|
+
def to(self, device):
|
|
195
|
+
"""Return a copy of the calibration on another device."""
|
|
196
|
+
return Calibration(self.R.to(device), self.delta.to(device),
|
|
197
|
+
self.mode, self.n_quantiles)
|
|
198
|
+
|
|
199
|
+
def cpu(self):
|
|
200
|
+
"""Return a copy of the calibration on CPU."""
|
|
201
|
+
return self.to("cpu")
|
|
202
|
+
|
|
203
|
+
def numpy(self):
|
|
204
|
+
"""Convert to a numpy-backed :class:`pycdft.core.Calibration`."""
|
|
205
|
+
from ..core import Calibration as NumpyCalibration
|
|
206
|
+
return NumpyCalibration(
|
|
207
|
+
R=self.R.detach().cpu().numpy(),
|
|
208
|
+
delta=self.delta.detach().cpu().numpy(),
|
|
209
|
+
mode=self.mode, n_quantiles=self.n_quantiles)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
# ---------------------------------------------------------------------------
|
|
213
|
+
# Exact calibration
|
|
214
|
+
# ---------------------------------------------------------------------------
|
|
215
|
+
|
|
216
|
+
def calibrate(lr_cal, hr_cal, p=None, delta_shift: str = "additive",
|
|
217
|
+
device=None, chunk_cols=None, n_quantiles: int = 1001):
|
|
218
|
+
"""Build the CDF-t calibration ``R = F_m^cal(q_o^cal(p))`` + per-cell delta.
|
|
219
|
+
|
|
220
|
+
Torch equivalent of :func:`pycdft.core.calibrate` (same maths, same
|
|
221
|
+
conventions): per-column type-7 quantiles of the observation
|
|
222
|
+
calibration series, per-column step ECDF of the shifted model series.
|
|
223
|
+
Inputs may be numpy arrays or torch tensors.
|
|
224
|
+
|
|
225
|
+
Parameters
|
|
226
|
+
----------
|
|
227
|
+
lr_cal, hr_cal : (T, N) or (T,) array / tensor
|
|
228
|
+
Model (coarse) / observation (fine) series over the calibration
|
|
229
|
+
period (same T).
|
|
230
|
+
p : (Q,) array / tensor, optional probability grid (default
|
|
231
|
+
``quantile_grid(1001)``).
|
|
232
|
+
delta_shift : {"additive", "multiplicative", "none"}.
|
|
233
|
+
device : str, torch.device or None
|
|
234
|
+
Target device (default: best available, see
|
|
235
|
+
:func:`pycdft.gpu.resolve_device`).
|
|
236
|
+
chunk_cols : int, optional
|
|
237
|
+
Process the cells in column chunks to bound the peak memory (the
|
|
238
|
+
results are identical).
|
|
239
|
+
|
|
240
|
+
Returns
|
|
241
|
+
-------
|
|
242
|
+
Calibration
|
|
243
|
+
"""
|
|
244
|
+
torch = require_torch()
|
|
245
|
+
if delta_shift not in SHIFT_MODES:
|
|
246
|
+
raise ValueError(f"delta_shift must be one of {SHIFT_MODES}, "
|
|
247
|
+
f"got {delta_shift!r}")
|
|
248
|
+
device = resolve_device(device)
|
|
249
|
+
p = _check_grid(p, n_quantiles, device)
|
|
250
|
+
lr, _ = _as_matrix(lr_cal, device, "lr_cal")
|
|
251
|
+
hr, _ = _as_matrix(hr_cal, device, "hr_cal")
|
|
252
|
+
if lr.shape[0] != hr.shape[0]:
|
|
253
|
+
raise ValueError(
|
|
254
|
+
f"calibration series lengths differ: {lr.shape[0]} vs "
|
|
255
|
+
f"{hr.shape[0]}")
|
|
256
|
+
N = lr.shape[1]
|
|
257
|
+
nv_lr = torch.isfinite(lr).sum(dim=0).to(torch.float64)
|
|
258
|
+
nv_hr = torch.isfinite(hr).sum(dim=0).to(torch.float64)
|
|
259
|
+
mean_lr = _mean_axis0(lr, nv_lr)
|
|
260
|
+
mean_hr = _mean_axis0(hr, nv_hr)
|
|
261
|
+
delta = delta_from_means(mean_lr, mean_hr, delta_shift)
|
|
262
|
+
lr_s, _ = torch.sort(lr, dim=0) # NaNs to the end, private copy
|
|
263
|
+
hr_s, _ = torch.sort(hr, dim=0)
|
|
264
|
+
Q = p.size(0)
|
|
265
|
+
R = torch.empty((Q, N), dtype=torch.float64, device=device)
|
|
266
|
+
for s, e in _cell_chunks(N, chunk_cols):
|
|
267
|
+
q_hr = _k.quantile_from_sorted(hr_s[:, s:e], p) # (Q, chunk)
|
|
268
|
+
lc = apply_shift(lr_s[:, s:e], delta[s:e], delta_shift)
|
|
269
|
+
R[:, s:e] = _k._ecdf_at_sorted(lc, q_hr)
|
|
270
|
+
return Calibration(R=R, delta=delta, mode=delta_shift, n_quantiles=Q)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
# ---------------------------------------------------------------------------
|
|
274
|
+
# Histogram calibration
|
|
275
|
+
# ---------------------------------------------------------------------------
|
|
276
|
+
|
|
277
|
+
def histograms(values, n_bins: int = 2000, device=None):
|
|
278
|
+
"""Per-cell histograms with per-cell edges (in-memory, 2 passes).
|
|
279
|
+
|
|
280
|
+
Torch equivalent of :func:`pycdft.core.histograms`: returns
|
|
281
|
+
``((B + 1, N) edges, (B, N) counts)``; NaNs excluded from the binning,
|
|
282
|
+
degenerate (constant) cells get a +/-1 padding, empty (all-NaN) cells
|
|
283
|
+
get zero counts.
|
|
284
|
+
"""
|
|
285
|
+
torch = require_torch()
|
|
286
|
+
values, _ = _as_matrix(values, resolve_device(device), "values")
|
|
287
|
+
n_bins = int(n_bins)
|
|
288
|
+
valid = torch.isfinite(values)
|
|
289
|
+
nv = valid.sum(dim=0)
|
|
290
|
+
lo = torch.where(valid, values, torch.full_like(values, float("inf"))).min(dim=0).values
|
|
291
|
+
hi = torch.where(valid, values, torch.full_like(values, float("-inf"))).max(dim=0).values
|
|
292
|
+
degenerate = hi <= lo
|
|
293
|
+
lo = torch.where(degenerate, lo - 1.0, lo)
|
|
294
|
+
hi = torch.where(degenerate, hi + 1.0, hi)
|
|
295
|
+
empty = nv == 0
|
|
296
|
+
lo = torch.where(empty, torch.zeros_like(lo), lo)
|
|
297
|
+
hi = torch.where(empty, torch.ones_like(hi), hi)
|
|
298
|
+
edges = (torch.arange(n_bins + 1, dtype=torch.float64,
|
|
299
|
+
device=values.device)[:, None]
|
|
300
|
+
* ((hi - lo)[None, :] / n_bins)) + lo[None, :]
|
|
301
|
+
width = (hi - lo)[None, :]
|
|
302
|
+
idx = (((values - lo[None, :])
|
|
303
|
+
/ torch.where(width > 0, width, torch.ones_like(width))
|
|
304
|
+
* n_bins).to(torch.int64)).clamp(0, n_bins - 1)
|
|
305
|
+
idx = torch.where(valid, idx, torch.full_like(idx, n_bins)) # sentinel bin
|
|
306
|
+
N = values.shape[1]
|
|
307
|
+
counts = torch.zeros((n_bins + 1, N), dtype=torch.float64,
|
|
308
|
+
device=values.device)
|
|
309
|
+
counts.scatter_add_(0, idx, torch.ones_like(values, dtype=torch.float64))
|
|
310
|
+
return edges, counts[:-1]
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def calibrate_hist(hist_lr, hist_hr, means, p=None, delta_shift: str = "additive",
|
|
314
|
+
device=None, n_quantiles: int = 1001):
|
|
315
|
+
"""Histogram calibration (same :class:`Calibration` object as ``calibrate``).
|
|
316
|
+
|
|
317
|
+
Torch equivalent of :func:`pycdft.core.calibrate_hist`; ``means`` is the
|
|
318
|
+
``(mean_lr, mean_hr)`` pair of per-cell float64 means.
|
|
319
|
+
"""
|
|
320
|
+
torch = require_torch()
|
|
321
|
+
if delta_shift not in SHIFT_MODES:
|
|
322
|
+
raise ValueError(f"delta_shift must be one of {SHIFT_MODES}, "
|
|
323
|
+
f"got {delta_shift!r}")
|
|
324
|
+
device = resolve_device(device)
|
|
325
|
+
p = _check_grid(p, n_quantiles, device)
|
|
326
|
+
(edges_lr, counts_lr), (edges_hr, counts_hr) = hist_lr, hist_hr
|
|
327
|
+
mean_lr, mean_hr = means
|
|
328
|
+
mean_lr = _as_1d(mean_lr, device, "mean_lr")
|
|
329
|
+
mean_hr = _as_1d(mean_hr, device, "mean_hr")
|
|
330
|
+
if mean_lr.size(0) == 1 and counts_hr.shape[1] != 1:
|
|
331
|
+
mean_lr = mean_lr.expand(counts_hr.shape[1])
|
|
332
|
+
mean_hr = mean_hr.expand(counts_hr.shape[1])
|
|
333
|
+
delta = delta_from_means(mean_lr, mean_hr, delta_shift)
|
|
334
|
+
p_grid = p[:, None].expand(p.size(0), counts_hr.shape[1])
|
|
335
|
+
q_hr = _k.hist_quantile_pc(edges_hr, counts_hr, p_grid)
|
|
336
|
+
if delta_shift == "additive":
|
|
337
|
+
q_unshift = q_hr - delta[None, :]
|
|
338
|
+
elif delta_shift == "multiplicative":
|
|
339
|
+
q_unshift = q_hr / torch.where(delta[None, :] != 0, delta[None, :],
|
|
340
|
+
torch.ones_like(delta[None, :]))
|
|
341
|
+
else:
|
|
342
|
+
q_unshift = q_hr
|
|
343
|
+
R = _k.ecdf_hist(edges_lr, counts_lr, q_unshift)
|
|
344
|
+
return Calibration(R=R, delta=delta, mode=delta_shift, n_quantiles=p.size(0))
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
# ---------------------------------------------------------------------------
|
|
348
|
+
# Transfer
|
|
349
|
+
# ---------------------------------------------------------------------------
|
|
350
|
+
|
|
351
|
+
def transfer(q_tar, R, p, chunk_cols=None):
|
|
352
|
+
"""Build the transfer curve on the target value grid.
|
|
353
|
+
|
|
354
|
+
Torch equivalent of :func:`pycdft.core.transfer`: ``q_tar(q, s, e) ->
|
|
355
|
+
(Q, e - s)`` is the quantile function of the (shifted) target series for
|
|
356
|
+
the cell columns ``[s, e)``. Returns ``(x_grid, T, k_lo, k_hi, bad)``.
|
|
357
|
+
"""
|
|
358
|
+
torch = require_torch()
|
|
359
|
+
R = R.to(torch.float64)
|
|
360
|
+
p = p.to(torch.float64)
|
|
361
|
+
Q, N = R.shape
|
|
362
|
+
if p.size(0) != Q:
|
|
363
|
+
raise ValueError(f"p size {p.size(0)} != R rows {Q}")
|
|
364
|
+
bad = ~torch.isfinite(R).all(dim=0) # NaN calibration cells
|
|
365
|
+
R = torch.where(bad[None, :], torch.full_like(R, 0.5), R)
|
|
366
|
+
x_grid = torch.empty((Q, N), dtype=torch.float64, device=R.device)
|
|
367
|
+
T = torch.empty((Q, N), dtype=torch.float64, device=R.device)
|
|
368
|
+
k_lo = torch.empty(N, dtype=torch.int64, device=R.device)
|
|
369
|
+
k_hi = torch.empty(N, dtype=torch.int64, device=R.device)
|
|
370
|
+
for s, e in _cell_chunks(N, chunk_cols):
|
|
371
|
+
T[:, s:e] = q_tar(R[:, s:e], s, e)
|
|
372
|
+
x_grid[:, s:e] = q_tar(p, s, e)
|
|
373
|
+
k_lo[s:e] = torch.argmax((R[:, s:e] > 0).to(torch.int64), dim=0)
|
|
374
|
+
k_hi[s:e] = Q - 1 - torch.argmax((R[:, s:e] < 1).flip(0).to(torch.int64),
|
|
375
|
+
dim=0)
|
|
376
|
+
return x_grid, T, k_lo, k_hi, bad
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
# ---------------------------------------------------------------------------
|
|
380
|
+
# Apply (target side)
|
|
381
|
+
# ---------------------------------------------------------------------------
|
|
382
|
+
|
|
383
|
+
def apply_cdft(lr_tar, calib: Calibration, p=None, tail: str = "linear",
|
|
384
|
+
device=None, chunk_cols=None, as_numpy: bool = True,
|
|
385
|
+
n_quantiles: int = 1001):
|
|
386
|
+
"""Apply a precomputed calibration to the target model series.
|
|
387
|
+
|
|
388
|
+
Torch equivalent of :func:`pycdft.core.apply_cdft` (same maths, same
|
|
389
|
+
conventions, same shape semantics: the output matches ``lr_tar`` and is
|
|
390
|
+
in the shifted frame). Inputs may be numpy arrays or torch tensors.
|
|
391
|
+
|
|
392
|
+
Parameters
|
|
393
|
+
----------
|
|
394
|
+
lr_tar : (T, N) or (T, ...) array / tensor
|
|
395
|
+
Model series to downscale / bias-correct.
|
|
396
|
+
calib : Calibration
|
|
397
|
+
From :func:`calibrate` or :func:`calibrate_hist`.
|
|
398
|
+
p : (Q,) array / tensor, optional probability grid.
|
|
399
|
+
tail : {"linear", "clip"}.
|
|
400
|
+
device : str, torch.device or None
|
|
401
|
+
Target device for the computation (default: the calibration's
|
|
402
|
+
device).
|
|
403
|
+
chunk_cols : int, optional
|
|
404
|
+
Process the cells in column chunks to bound the peak memory.
|
|
405
|
+
as_numpy : bool, default True
|
|
406
|
+
Return a numpy float64 array (as the numpy backend); ``False``
|
|
407
|
+
returns a torch tensor on the compute device.
|
|
408
|
+
|
|
409
|
+
Returns
|
|
410
|
+
-------
|
|
411
|
+
Array / tensor of the same shape as ``lr_tar`` (NaN inputs map to NaN;
|
|
412
|
+
cells with an empty calibration map entirely to NaN).
|
|
413
|
+
"""
|
|
414
|
+
torch = require_torch()
|
|
415
|
+
if tail not in _TAILS:
|
|
416
|
+
raise ValueError(f"tail must be 'linear' or 'clip', got {tail!r}")
|
|
417
|
+
if device is None:
|
|
418
|
+
device = calib.R.device
|
|
419
|
+
else:
|
|
420
|
+
device = resolve_device(device)
|
|
421
|
+
if p is None:
|
|
422
|
+
p = _k.quantile_grid(calib.R.shape[0], device=device)
|
|
423
|
+
else:
|
|
424
|
+
p = _check_grid(p, n_quantiles, device)
|
|
425
|
+
if p.size(0) != calib.R.shape[0]:
|
|
426
|
+
raise ValueError(
|
|
427
|
+
f"probability grid size {p.size(0)} does not match the "
|
|
428
|
+
f"calibration ({calib.R.shape[0]}); pass the same grid used "
|
|
429
|
+
f"in calibrate()")
|
|
430
|
+
x, shape = _as_matrix(lr_tar, device, "lr_tar")
|
|
431
|
+
xs = apply_shift(x, calib.delta, calib.mode)
|
|
432
|
+
v_tar, _ = torch.sort(xs, dim=0) # NaNs to the end
|
|
433
|
+
x_grid, T, k_lo, k_hi, bad = transfer(
|
|
434
|
+
lambda q, s, e: _k.quantile_from_sorted(v_tar[:, s:e], q),
|
|
435
|
+
calib.R, p, chunk_cols)
|
|
436
|
+
pred = _k.interp_transfer(x_grid, T, k_lo, k_hi, xs, tail=tail)
|
|
437
|
+
if bool(bad.any()):
|
|
438
|
+
pred = pred.masked_fill(bad[None, :], float("nan"))
|
|
439
|
+
pred = pred.reshape(shape)
|
|
440
|
+
if as_numpy:
|
|
441
|
+
return pred.detach().cpu().numpy()
|
|
442
|
+
return pred
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def apply_cdft_hist(lr_tar, calib: Calibration, p=None, n_bins: int = 2000,
|
|
446
|
+
tail: str = "linear", device=None, as_numpy: bool = True,
|
|
447
|
+
n_quantiles: int = 1001):
|
|
448
|
+
"""Same as :func:`apply_cdft`, with the target CDF accumulated as a
|
|
449
|
+
per-cell histogram (bounded memory for very long records).
|
|
450
|
+
|
|
451
|
+
Torch equivalent of :func:`pycdft.core.apply_cdft_hist` (approximate,
|
|
452
|
+
exact in the limit of many bins).
|
|
453
|
+
"""
|
|
454
|
+
if tail not in _TAILS:
|
|
455
|
+
raise ValueError(f"tail must be 'linear' or 'clip', got {tail!r}")
|
|
456
|
+
if device is None:
|
|
457
|
+
device = calib.R.device
|
|
458
|
+
else:
|
|
459
|
+
device = resolve_device(device)
|
|
460
|
+
if p is None:
|
|
461
|
+
p = _k.quantile_grid(calib.R.shape[0], device=device)
|
|
462
|
+
else:
|
|
463
|
+
p = _check_grid(p, n_quantiles, device)
|
|
464
|
+
if p.size(0) != calib.R.shape[0]:
|
|
465
|
+
raise ValueError(
|
|
466
|
+
f"probability grid size {p.size(0)} does not match the "
|
|
467
|
+
f"calibration ({calib.R.shape[0]}); pass the same grid used "
|
|
468
|
+
f"in calibrate()")
|
|
469
|
+
x, shape = _as_matrix(lr_tar, device, "lr_tar")
|
|
470
|
+
xs = apply_shift(x, calib.delta, calib.mode)
|
|
471
|
+
edges, counts = _k.histogram_accumulate(xs, n_bins=n_bins)
|
|
472
|
+
x_grid, T, k_lo, k_hi, bad = transfer(
|
|
473
|
+
lambda q, s, e: _k.hist_quantile(edges, counts[:, s:e], q),
|
|
474
|
+
calib.R, p, None)
|
|
475
|
+
pred = _k.interp_transfer(x_grid, T, k_lo, k_hi, xs, tail=tail)
|
|
476
|
+
if bool(bad.any()):
|
|
477
|
+
pred = pred.masked_fill(bad[None, :], float("nan"))
|
|
478
|
+
pred = pred.reshape(shape)
|
|
479
|
+
if as_numpy:
|
|
480
|
+
return pred.detach().cpu().numpy()
|
|
481
|
+
return pred
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
# ---------------------------------------------------------------------------
|
|
485
|
+
# High-level API
|
|
486
|
+
# ---------------------------------------------------------------------------
|
|
487
|
+
|
|
488
|
+
class CDFt:
|
|
489
|
+
"""CDF-t downscaling / bias correction on a torch device (GPU backend).
|
|
490
|
+
|
|
491
|
+
Mirrors :class:`pycdft.CDFt` (same parameters, same ``fit`` /
|
|
492
|
+
``transform`` / ``fit_transform`` interface, numpy in / numpy out by
|
|
493
|
+
default) but runs the per-cell kernels with torch on the device selected
|
|
494
|
+
by :func:`pycdft.gpu.resolve_device`.
|
|
495
|
+
|
|
496
|
+
Parameters
|
|
497
|
+
----------
|
|
498
|
+
n_quantiles : int, default 1001
|
|
499
|
+
Number of points of the uniform probability grid ``p`` (endpoints 0
|
|
500
|
+
and 1 included).
|
|
501
|
+
tail : {"linear", "clip"}, default "linear"
|
|
502
|
+
Behaviour outside the saturated region of the transfer.
|
|
503
|
+
delta_shift : {"additive", "multiplicative", "none"}, default "additive"
|
|
504
|
+
Per-cell mean shift applied to the model series.
|
|
505
|
+
use_histograms : bool, default False
|
|
506
|
+
Accumulate the target CDF as a per-cell histogram instead of an
|
|
507
|
+
exact sort (approximate; bounded memory for very long records).
|
|
508
|
+
n_bins : int, default 2000
|
|
509
|
+
Number of bins per cell in histogram mode.
|
|
510
|
+
device : str, torch.device or None, default None
|
|
511
|
+
Compute device (default: best available -- ``cuda`` -> ``mps`` ->
|
|
512
|
+
``cpu``; an explicit unavailable device raises ``ValueError``).
|
|
513
|
+
chunk_cols : int, optional
|
|
514
|
+
Process the cells in column chunks to bound the peak memory.
|
|
515
|
+
as_numpy : bool, default True
|
|
516
|
+
Return numpy float64 arrays from ``transform`` /
|
|
517
|
+
``fit_transform`` (``False`` returns torch tensors on the compute
|
|
518
|
+
device).
|
|
519
|
+
|
|
520
|
+
Attributes
|
|
521
|
+
----------
|
|
522
|
+
calibration_ : Calibration or None
|
|
523
|
+
The calibration from the last :meth:`fit` / :meth:`fit_transform`.
|
|
524
|
+
device : torch.device
|
|
525
|
+
The resolved compute device.
|
|
526
|
+
|
|
527
|
+
Notes
|
|
528
|
+
-----
|
|
529
|
+
NaN handling is identical to the numpy backend: NaNs are excluded from
|
|
530
|
+
the ECDF counts and the quantile interpolation, NaN inputs pass through
|
|
531
|
+
as NaN, and cells whose calibration is empty map entirely to NaN. The
|
|
532
|
+
math is float64 on the device: for the same input data the results match
|
|
533
|
+
:class:`pycdft.CDFt` to the last significant digit (see
|
|
534
|
+
``tests/test_gpu.py``).
|
|
535
|
+
"""
|
|
536
|
+
|
|
537
|
+
def __init__(self, n_quantiles: int = 1001, tail: str = "linear",
|
|
538
|
+
delta_shift: str = "additive", use_histograms: bool = False,
|
|
539
|
+
n_bins: int = 2000, device=None, chunk_cols=None,
|
|
540
|
+
as_numpy: bool = True):
|
|
541
|
+
n_quantiles = int(n_quantiles)
|
|
542
|
+
if n_quantiles < 2:
|
|
543
|
+
raise ValueError(f"n_quantiles must be >= 2, got {n_quantiles}")
|
|
544
|
+
if delta_shift not in SHIFT_MODES:
|
|
545
|
+
raise ValueError(f"delta_shift must be one of {SHIFT_MODES}, "
|
|
546
|
+
f"got {delta_shift!r}")
|
|
547
|
+
if tail not in _TAILS:
|
|
548
|
+
raise ValueError(f"tail must be one of {_TAILS}, got {tail!r}")
|
|
549
|
+
if n_bins < 2:
|
|
550
|
+
raise ValueError(f"n_bins must be >= 2, got {n_bins}")
|
|
551
|
+
self.n_quantiles = n_quantiles
|
|
552
|
+
self.tail = tail
|
|
553
|
+
self.delta_shift = delta_shift
|
|
554
|
+
self.use_histograms = bool(use_histograms)
|
|
555
|
+
self.n_bins = int(n_bins)
|
|
556
|
+
self.chunk_cols = chunk_cols
|
|
557
|
+
self.as_numpy = bool(as_numpy)
|
|
558
|
+
self.device = resolve_device(device)
|
|
559
|
+
self.calibration_ = None
|
|
560
|
+
|
|
561
|
+
# ------------------------------------------------------------------
|
|
562
|
+
@property
|
|
563
|
+
def probability_grid(self) -> np.ndarray:
|
|
564
|
+
"""The uniform probability grid ``linspace(0, 1, n_quantiles)``
|
|
565
|
+
(numpy, identical to the numpy backend's grid)."""
|
|
566
|
+
return np.linspace(0.0, 1.0, self.n_quantiles)
|
|
567
|
+
|
|
568
|
+
# ------------------------------------------------------------------
|
|
569
|
+
def to(self, device):
|
|
570
|
+
"""Move the fitted calibration to another device and return self."""
|
|
571
|
+
if self.calibration_ is not None:
|
|
572
|
+
self.calibration_ = self.calibration_.to(resolve_device(device))
|
|
573
|
+
self.device = resolve_device(device)
|
|
574
|
+
return self
|
|
575
|
+
|
|
576
|
+
# ------------------------------------------------------------------
|
|
577
|
+
def fit(self, obs, cm_hist) -> "CDFt":
|
|
578
|
+
"""Fit the calibration on ``(obs, cm_hist)`` (numpy arrays or
|
|
579
|
+
torch tensors; returns self)."""
|
|
580
|
+
self.calibration_ = calibrate(cm_hist, obs,
|
|
581
|
+
p=_k.quantile_grid(self.n_quantiles,
|
|
582
|
+
device=self.device),
|
|
583
|
+
delta_shift=self.delta_shift,
|
|
584
|
+
device=self.device,
|
|
585
|
+
chunk_cols=self.chunk_cols)
|
|
586
|
+
return self
|
|
587
|
+
|
|
588
|
+
def transform(self, cm_future):
|
|
589
|
+
"""Apply the fitted calibration to the target model series.
|
|
590
|
+
|
|
591
|
+
Returns an array of the same shape as ``cm_future`` (numpy float64
|
|
592
|
+
by default, or a tensor on the compute device with
|
|
593
|
+
``as_numpy=False``).
|
|
594
|
+
"""
|
|
595
|
+
if self.calibration_ is None:
|
|
596
|
+
raise RuntimeError("CDFt is not fitted: call fit() or "
|
|
597
|
+
"fit_transform() first")
|
|
598
|
+
if self.use_histograms:
|
|
599
|
+
return apply_cdft_hist(cm_future, self.calibration_,
|
|
600
|
+
p=_k.quantile_grid(self.n_quantiles,
|
|
601
|
+
device=self.device),
|
|
602
|
+
n_bins=self.n_bins, tail=self.tail,
|
|
603
|
+
device=self.device, as_numpy=self.as_numpy)
|
|
604
|
+
return apply_cdft(cm_future, self.calibration_,
|
|
605
|
+
p=_k.quantile_grid(self.n_quantiles,
|
|
606
|
+
device=self.device),
|
|
607
|
+
tail=self.tail, device=self.device,
|
|
608
|
+
chunk_cols=self.chunk_cols, as_numpy=self.as_numpy)
|
|
609
|
+
|
|
610
|
+
def fit_transform(self, obs, cm_hist, cm_future):
|
|
611
|
+
"""Fit on ``(obs, cm_hist)`` and transform ``cm_future`` in one call."""
|
|
612
|
+
return self.fit(obs, cm_hist).transform(cm_future)
|
|
613
|
+
|
|
614
|
+
# ------------------------------------------------------------------
|
|
615
|
+
def __repr__(self) -> str:
|
|
616
|
+
return (f"CDFt(n_quantiles={self.n_quantiles}, tail={self.tail!r}, "
|
|
617
|
+
f"delta_shift={self.delta_shift!r}, "
|
|
618
|
+
f"use_histograms={self.use_histograms}, "
|
|
619
|
+
f"n_bins={self.n_bins}, device={self.device!s})")
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def cdft_transform(obs, cm_hist, cm_future, n_quantiles: int = 1001,
|
|
623
|
+
tail: str = "linear", delta_shift: str = "additive",
|
|
624
|
+
use_histograms: bool = False, n_bins: int = 2000,
|
|
625
|
+
device=None, chunk_cols=None, as_numpy: bool = True):
|
|
626
|
+
"""One-shot CDF-t transform (no persistent calibration object).
|
|
627
|
+
|
|
628
|
+
Equivalent to ``CDFt(...).fit_transform(obs, cm_hist, cm_future)``;
|
|
629
|
+
returns the same-shaped output array.
|
|
630
|
+
"""
|
|
631
|
+
return CDFt(n_quantiles=n_quantiles, tail=tail, delta_shift=delta_shift,
|
|
632
|
+
use_histograms=use_histograms, n_bins=n_bins, device=device,
|
|
633
|
+
chunk_cols=chunk_cols, as_numpy=as_numpy) \
|
|
634
|
+
.fit_transform(obs, cm_hist, cm_future)
|