PyCDFt 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pycdft/__init__.py +59 -0
- pycdft/api.py +199 -0
- pycdft/core.py +489 -0
- pycdft/gpu/__init__.py +42 -0
- pycdft/gpu/_torch.py +29 -0
- pycdft/gpu/api.py +634 -0
- pycdft/gpu/devices.py +160 -0
- pycdft/gpu/kernels.py +554 -0
- pycdft/math.py +561 -0
- pycdft/py.typed +0 -0
- pycdft-0.1.0.dist-info/METADATA +289 -0
- pycdft-0.1.0.dist-info/RECORD +15 -0
- pycdft-0.1.0.dist-info/WHEEL +5 -0
- pycdft-0.1.0.dist-info/licenses/LICENSE +201 -0
- pycdft-0.1.0.dist-info/top_level.txt +1 -0
pycdft/math.py
ADDED
|
@@ -0,0 +1,561 @@
|
|
|
1
|
+
"""Low-level pure-numpy kernels for the CDF-t transform.
|
|
2
|
+
|
|
3
|
+
This module contains the numerical building blocks of the CDF-t method,
|
|
4
|
+
vectorized over an arbitrary number of independent series (columns):
|
|
5
|
+
|
|
6
|
+
- ECDF: right-continuous step function ``F(x) = #{finite values <= x} / n``
|
|
7
|
+
(NaNs excluded from the count and sorted to the end);
|
|
8
|
+
- quantiles: numpy / R ``type-7`` linear interpolation
|
|
9
|
+
(``np.quantile(..., method="linear")`` == R ``quantile(..., type=7)``),
|
|
10
|
+
re-implemented bit-for-bit on pre-sorted arrays (including numpy's
|
|
11
|
+
``_lerp`` ``w >= 0.5`` branch), with the index grid scaled by the per-cell
|
|
12
|
+
number of *valid* values so that NaNs can never enter the interpolation;
|
|
13
|
+
- the transfer interpolation (``np.interp`` per cell) with the two tail
|
|
14
|
+
options (``clip`` / ``linear``);
|
|
15
|
+
- histogram-based ECDF / quantile variants.
|
|
16
|
+
|
|
17
|
+
All kernels accept 2-D arrays ``(T, N)`` (or ``(Q, N)``) of float64 and
|
|
18
|
+
operate independently per column (per series / per grid cell). The 1-D
|
|
19
|
+
convenience wrappers (:func:`ecdf`, :func:`quantile`) accept ``(T,)`` input.
|
|
20
|
+
|
|
21
|
+
The conventions are identical to the reference grid-scale implementation the
|
|
22
|
+
package is derived from (a production CERRA-downscaling CDF-t baseline) and
|
|
23
|
+
to the R ``CDFt`` package with the same options.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import warnings
|
|
29
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
30
|
+
|
|
31
|
+
import numpy as np
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"quantile_grid",
|
|
35
|
+
"searchsorted_columns",
|
|
36
|
+
"ecdf",
|
|
37
|
+
"ecdf_at",
|
|
38
|
+
"quantile",
|
|
39
|
+
"nanquantile",
|
|
40
|
+
"quantile_from_sorted",
|
|
41
|
+
"windowed_slope",
|
|
42
|
+
"interp_transfer",
|
|
43
|
+
"histogram_accumulate",
|
|
44
|
+
"hist_quantile",
|
|
45
|
+
"hist_quantile_pc",
|
|
46
|
+
"ecdf_hist",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
# Probability grid
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
def quantile_grid(n_quantiles: int = 1001) -> np.ndarray:
|
|
55
|
+
"""Uniform probability grid ``linspace(0, 1, n_quantiles)``.
|
|
56
|
+
|
|
57
|
+
Parameters
|
|
58
|
+
----------
|
|
59
|
+
n_quantiles : int
|
|
60
|
+
Number of grid points (``>= 2``); the endpoints 0 and 1 are always
|
|
61
|
+
included (the grid is the support of the transfer curve).
|
|
62
|
+
|
|
63
|
+
Returns
|
|
64
|
+
-------
|
|
65
|
+
np.ndarray of shape ``(n_quantiles,)``.
|
|
66
|
+
"""
|
|
67
|
+
n_quantiles = int(n_quantiles)
|
|
68
|
+
if n_quantiles < 2:
|
|
69
|
+
raise ValueError(f"n_quantiles must be >= 2, got {n_quantiles}")
|
|
70
|
+
return np.linspace(0.0, 1.0, n_quantiles)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ---------------------------------------------------------------------------
|
|
74
|
+
# Batched per-column searchsorted
|
|
75
|
+
# ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
def searchsorted_columns(a, v, side="right", workers: int = 1) -> np.ndarray:
|
|
78
|
+
"""Per-column ``np.searchsorted`` on the (T, N) / (M, N) layouts.
|
|
79
|
+
|
|
80
|
+
Equivalent to looping ``np.searchsorted(a[:, j], v[:, j], side=side)``
|
|
81
|
+
over columns (identical results), optionally with a thread pool (numpy
|
|
82
|
+
releases the GIL, so ``workers > 1`` scales the per-cell kernels).
|
|
83
|
+
|
|
84
|
+
Parameters
|
|
85
|
+
----------
|
|
86
|
+
a : (T, N) array, sorted along axis 0 (NaNs allowed, sorted last).
|
|
87
|
+
v : (M, N) array of query points.
|
|
88
|
+
side : {"left", "right"}, default "right" (``F(x) = #(<= x)``).
|
|
89
|
+
workers : int, number of threads (``<= 1`` = single-threaded).
|
|
90
|
+
|
|
91
|
+
Returns
|
|
92
|
+
-------
|
|
93
|
+
(M, N) int64 array of insertion indices.
|
|
94
|
+
"""
|
|
95
|
+
a = np.asarray(a)
|
|
96
|
+
v = np.asarray(v)
|
|
97
|
+
if a.ndim != 2 or v.ndim != 2 or a.shape[1] != v.shape[1]:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
f"expected (T, N) and (M, N) arrays with the same N, "
|
|
100
|
+
f"got {a.shape} and {v.shape}")
|
|
101
|
+
M, N = v.shape
|
|
102
|
+
out = np.empty((M, N), dtype=np.int64)
|
|
103
|
+
if workers > 1 and N >= 256:
|
|
104
|
+
n_chunks = min(workers * 4, N)
|
|
105
|
+
edges = np.linspace(0, N, n_chunks + 1).astype(np.int64)
|
|
106
|
+
|
|
107
|
+
def _run(lo: int, hi: int) -> None:
|
|
108
|
+
for j in range(lo, hi):
|
|
109
|
+
out[:, j] = np.searchsorted(a[:, j], v[:, j], side=side)
|
|
110
|
+
|
|
111
|
+
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
112
|
+
list(ex.map(_run, edges[:-1], edges[1:]))
|
|
113
|
+
else:
|
|
114
|
+
for j in range(N):
|
|
115
|
+
out[:, j] = np.searchsorted(a[:, j], v[:, j], side=side)
|
|
116
|
+
return out
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
# ---------------------------------------------------------------------------
|
|
120
|
+
# ECDF
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
|
|
123
|
+
def _ecdf_at_sorted(v_sorted, points) -> np.ndarray:
|
|
124
|
+
"""Step ECDF of a sorted (T, N) array (NaNs at the end) at (M, N) points.
|
|
125
|
+
|
|
126
|
+
``F(x) = #{finite values <= x} / n_valid`` (right-continuous, side="right");
|
|
127
|
+
NaNs excluded from the count; NaN query points and empty (all-NaN)
|
|
128
|
+
columns give NaN.
|
|
129
|
+
"""
|
|
130
|
+
v = np.asarray(v_sorted, np.float64)
|
|
131
|
+
pts = np.asarray(points, np.float64)
|
|
132
|
+
nv = np.isfinite(v).sum(axis=0).astype(np.float64)
|
|
133
|
+
idx = searchsorted_columns(v, pts, side="right")
|
|
134
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
135
|
+
out = np.where(nv[None, :] > 0, idx / np.maximum(nv[None, :], 1.0),
|
|
136
|
+
np.nan)
|
|
137
|
+
# NaN query points must give NaN, not the end-index (numpy treats NaN
|
|
138
|
+
# queries as +inf in searchsorted)
|
|
139
|
+
out = np.where(np.isfinite(pts), out, np.nan)
|
|
140
|
+
return np.clip(out, 0.0, 1.0)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def ecdf_at(values, points) -> np.ndarray:
|
|
144
|
+
"""Step ECDF of ``values`` (T, N) evaluated at ``points`` (M, N).
|
|
145
|
+
|
|
146
|
+
Returns the (M, N) matrix ``F(points)`` with
|
|
147
|
+
``F(x) = #{finite values <= x} / n`` (NaN-excluded); NaN points and
|
|
148
|
+
all-NaN columns give NaN.
|
|
149
|
+
"""
|
|
150
|
+
values = np.asarray(values, np.float64)
|
|
151
|
+
points = np.asarray(points, np.float64)
|
|
152
|
+
if values.ndim != 2 or points.ndim != 2:
|
|
153
|
+
raise ValueError("expected 2-D (T, N) values and (M, N) points")
|
|
154
|
+
v = np.sort(values, axis=0) # NaNs sort to the end
|
|
155
|
+
return _ecdf_at_sorted(v, points)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def ecdf(values, points=None) -> np.ndarray:
|
|
159
|
+
"""Empirical (step) CDF of a sample, at the sample or at given points.
|
|
160
|
+
|
|
161
|
+
``F(x) = #{finite values <= x} / n`` (right-continuous step; NaNs
|
|
162
|
+
excluded from the count). With ``points=None`` returns the ECDF
|
|
163
|
+
evaluated at the (finite) sample values themselves.
|
|
164
|
+
|
|
165
|
+
Accepts 1-D ``(T,)`` (returns ``(T,)``) or 2-D ``(T, N)`` (returns
|
|
166
|
+
``(T, N)``) input.
|
|
167
|
+
"""
|
|
168
|
+
values = np.asarray(values, np.float64)
|
|
169
|
+
if values.ndim == 1:
|
|
170
|
+
v = np.sort(values, axis=0)[:, None]
|
|
171
|
+
pts = (values if points is None else np.asarray(points, np.float64))
|
|
172
|
+
pts = np.asarray(pts, np.float64)[:, None]
|
|
173
|
+
return _ecdf_at_sorted(v, pts)[:, 0]
|
|
174
|
+
v = np.sort(values, axis=0)
|
|
175
|
+
if points is None:
|
|
176
|
+
return _ecdf_at_sorted(v, values)
|
|
177
|
+
return ecdf_at(values, np.asarray(points, np.float64))
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
# ---------------------------------------------------------------------------
|
|
181
|
+
# Quantiles (type-7 linear, bit-identical to np.quantile(..., method="linear"))
|
|
182
|
+
# ---------------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
def quantile_from_sorted(v_sorted, q) -> np.ndarray:
|
|
185
|
+
"""Type-7 linear quantiles of a sorted (T, N) array at probabilities ``q``.
|
|
186
|
+
|
|
187
|
+
The array is sorted along axis 0 with NaNs at the end (e.g. the output of
|
|
188
|
+
``np.sort(..., axis=0)``); NaNs are excluded via the per-column valid
|
|
189
|
+
counts, so they can never enter the interpolation (empty columns -> NaN).
|
|
190
|
+
|
|
191
|
+
``q`` may be ``(Q,)`` (same probabilities for all columns) or ``(Q, N)``
|
|
192
|
+
(per-column probabilities); the result is ``(Q, N)``.
|
|
193
|
+
|
|
194
|
+
Bit-identical to ``np.quantile(v, q, method="linear", axis=0)`` on the
|
|
195
|
+
NaN-free prefix (numpy's ``_lerp`` ``w >= 0.5`` branch is replicated).
|
|
196
|
+
"""
|
|
197
|
+
v = np.asarray(v_sorted, np.float64)
|
|
198
|
+
q = np.asarray(q, np.float64)
|
|
199
|
+
if v.ndim != 2:
|
|
200
|
+
raise ValueError(f"expected a 2-D (T, N) sorted array, got {v.shape}")
|
|
201
|
+
if q.ndim == 1:
|
|
202
|
+
q = np.broadcast_to(q[:, None], (q.size, v.shape[1]))
|
|
203
|
+
elif q.ndim != 2 or q.shape[1] != v.shape[1]:
|
|
204
|
+
raise ValueError("q must be (Q,) or (Q, N) with the same N as v")
|
|
205
|
+
nv = np.isfinite(v).sum(axis=0).astype(np.float64)
|
|
206
|
+
scale = np.maximum(nv - 1.0, 0.0)[None, :]
|
|
207
|
+
idx = np.clip(q * scale, 0.0, scale)
|
|
208
|
+
i0 = np.floor(idx).astype(np.int64)
|
|
209
|
+
i1 = np.clip(np.ceil(idx).astype(np.int64), 0,
|
|
210
|
+
np.maximum(nv.astype(np.int64) - 1, 0)[None, :])
|
|
211
|
+
w = idx - i0
|
|
212
|
+
a = np.take_along_axis(v, i0, axis=0)
|
|
213
|
+
b = np.take_along_axis(v, i1, axis=0)
|
|
214
|
+
diff = b - a
|
|
215
|
+
out = a + diff * w
|
|
216
|
+
out = np.where(w >= 0.5, b - diff * (1.0 - w), out)
|
|
217
|
+
return np.where(nv[None, :] > 0, out, np.nan)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def quantile(values, p) -> np.ndarray:
|
|
221
|
+
"""Type-7 quantiles of ``values`` (T, N) at probabilities ``p`` (Q,).
|
|
222
|
+
|
|
223
|
+
Returns ``(Q, N)``; identical to ``np.quantile(values, p, method="linear",
|
|
224
|
+
axis=0)`` with NaNs excluded per column (empty columns -> NaN).
|
|
225
|
+
"""
|
|
226
|
+
values = np.asarray(values, np.float64)
|
|
227
|
+
if values.ndim != 2:
|
|
228
|
+
raise ValueError(f"expected a 2-D (T, N) array, got {values.shape}")
|
|
229
|
+
v = np.sort(values, axis=0) # NaNs to the end
|
|
230
|
+
return quantile_from_sorted(v, np.asarray(p, np.float64))
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def nanquantile(values, p) -> np.ndarray:
|
|
234
|
+
"""Alias of :func:`quantile` (NaN-excluded type-7 quantiles)."""
|
|
235
|
+
return quantile(values, p)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _nanmedian_axis0(a: np.ndarray) -> np.ndarray:
|
|
239
|
+
"""Column-wise nanmedian without RuntimeWarnings on all-NaN columns."""
|
|
240
|
+
with warnings.catch_warnings():
|
|
241
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
242
|
+
return np.nanmedian(a, axis=0)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
# ---------------------------------------------------------------------------
|
|
246
|
+
# Windowed least-squares slope (tail extrapolation)
|
|
247
|
+
# ---------------------------------------------------------------------------
|
|
248
|
+
|
|
249
|
+
def windowed_slope(x_grid, T, lo, hi, W: int = 30) -> np.ndarray:
|
|
250
|
+
"""Least-squares slope of ``T`` vs ``x_grid`` over the rank window per cell.
|
|
251
|
+
|
|
252
|
+
For each column j, the slope of the linear least-squares fit of
|
|
253
|
+
``T[lo[j]:hi[j]+1, j]`` on ``x_grid[lo[j]:hi[j]+1, j]`` (closed form,
|
|
254
|
+
equivalent to ``np.polyfit(..., 1)[0]`` to ~1e-12 relative, without
|
|
255
|
+
RankWarnings on degenerate windows: 0 when the window is degenerate).
|
|
256
|
+
|
|
257
|
+
Parameters
|
|
258
|
+
----------
|
|
259
|
+
x_grid, T : (Q, N) transfer arrays.
|
|
260
|
+
lo, hi : (N,) rank bounds (inclusive; the window is the full ``[lo, hi]``
|
|
261
|
+
rank range per cell -- the ``W`` argument is kept for interface
|
|
262
|
+
compatibility with the reference implementation, where the callers
|
|
263
|
+
pass ``hi = min(lo + W, Q - 1)``).
|
|
264
|
+
W : typical window length (informational; the effective window is the
|
|
265
|
+
full ``[lo, hi]`` range of each cell).
|
|
266
|
+
|
|
267
|
+
Returns
|
|
268
|
+
-------
|
|
269
|
+
(N,) slopes (``0`` where the window is degenerate or empty).
|
|
270
|
+
"""
|
|
271
|
+
x_grid = np.asarray(x_grid, np.float64)
|
|
272
|
+
T = np.asarray(T, np.float64)
|
|
273
|
+
lo = np.asarray(lo, np.int64)
|
|
274
|
+
hi = np.asarray(hi, np.int64)
|
|
275
|
+
Q, N = x_grid.shape
|
|
276
|
+
lo = np.maximum(lo, 0)
|
|
277
|
+
hi = np.minimum(hi, Q - 1)
|
|
278
|
+
r = np.arange(Q)[:, None]
|
|
279
|
+
mask = (r >= lo[None, :]) & (r <= hi[None, :]) # (Q, N)
|
|
280
|
+
gw = np.where(mask, x_grid, np.nan)
|
|
281
|
+
tw = np.where(mask, T, np.nan)
|
|
282
|
+
with warnings.catch_warnings():
|
|
283
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
284
|
+
with np.errstate(invalid="ignore"):
|
|
285
|
+
gm = np.nansum(gw, axis=0) / np.maximum(mask.sum(axis=0), 1)
|
|
286
|
+
g0 = gw - gm[None, :]
|
|
287
|
+
tm = np.nansum(tw, axis=0) / np.maximum(mask.sum(axis=0), 1)
|
|
288
|
+
t0 = tw - tm[None, :]
|
|
289
|
+
num = np.nansum(g0 * t0, axis=0)
|
|
290
|
+
den = np.nansum(g0 * g0, axis=0)
|
|
291
|
+
slope = np.where(den > 0, num / np.where(den > 0, den, 1.0), 0.0)
|
|
292
|
+
return np.where(np.isfinite(slope), slope, 0.0)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
# ---------------------------------------------------------------------------
|
|
296
|
+
# Transfer interpolation (per cell) + tail handling
|
|
297
|
+
# ---------------------------------------------------------------------------
|
|
298
|
+
|
|
299
|
+
def interp_transfer(x_grid, T, k_lo, k_hi, x, tail: str = "linear",
|
|
300
|
+
workers: int = 1) -> np.ndarray:
|
|
301
|
+
"""Evaluate the CDF-t transfer at the target values ``x`` (T, N).
|
|
302
|
+
|
|
303
|
+
``y(x)`` is the per-cell piecewise-linear interpolation of the transfer
|
|
304
|
+
curve ``(x_grid, T)`` (``np.interp``), with ``T[0]`` / ``T[-1]`` as the
|
|
305
|
+
constant (clip) extrapolation below / above the grid.
|
|
306
|
+
|
|
307
|
+
Parameters
|
|
308
|
+
----------
|
|
309
|
+
x_grid, T : (Q, N) transfer arrays (``x_grid`` = target quantiles at the
|
|
310
|
+
probability grid, ``T`` = target quantiles at the mapping ``R``).
|
|
311
|
+
k_lo, k_hi : (N,) first / last *unsaturated* ranks of ``R`` per cell
|
|
312
|
+
(``k_lo = argmax(R > 0)``, ``k_hi = last rank with ``R < 1``).
|
|
313
|
+
x : (T, N) target values (already delta-shifted).
|
|
314
|
+
tail : {"linear", "clip"}
|
|
315
|
+
- "linear": outside ``[x_grid[k_lo], x_grid[k_hi]]`` the transfer is
|
|
316
|
+
extrapolated linearly with the 30-rank windowed least-squares slope
|
|
317
|
+
at the boundary, clipped to ``[0, 2 x median interior slope]``;
|
|
318
|
+
- "clip": constant at ``T[0]`` / ``T[-1]`` (np.interp defaults).
|
|
319
|
+
workers : threads for the per-cell interpolation.
|
|
320
|
+
|
|
321
|
+
Returns
|
|
322
|
+
-------
|
|
323
|
+
(T, N) mapped values (NaN inputs propagate to NaN outputs).
|
|
324
|
+
"""
|
|
325
|
+
x_grid = np.asarray(x_grid, np.float64)
|
|
326
|
+
T = np.asarray(T, np.float64)
|
|
327
|
+
x = np.asarray(x, np.float64)
|
|
328
|
+
if not (x_grid.ndim == T.ndim == 2 and x.ndim == 2
|
|
329
|
+
and x_grid.shape == T.shape and x_grid.shape[1] == x.shape[1]):
|
|
330
|
+
raise ValueError(
|
|
331
|
+
f"expected (Q, N) x_grid/T and (T, N) x, got "
|
|
332
|
+
f"{x_grid.shape}, {T.shape}, {x.shape}")
|
|
333
|
+
Q, N = x_grid.shape
|
|
334
|
+
y = np.empty(x.shape, dtype=np.float64)
|
|
335
|
+
|
|
336
|
+
def _run(cols) -> None:
|
|
337
|
+
for j in cols:
|
|
338
|
+
y[:, j] = np.interp(x[:, j], x_grid[:, j], T[:, j],
|
|
339
|
+
left=T[0, j], right=T[-1, j])
|
|
340
|
+
|
|
341
|
+
if workers > 1 and N >= 256:
|
|
342
|
+
n_chunks = min(workers * 4, N)
|
|
343
|
+
edges = np.linspace(0, N, n_chunks + 1).astype(np.int64)
|
|
344
|
+
with ThreadPoolExecutor(max_workers=workers) as ex:
|
|
345
|
+
list(ex.map(_run, [range(a, b) for a, b in
|
|
346
|
+
zip(edges[:-1], edges[1:])]))
|
|
347
|
+
else:
|
|
348
|
+
_run(range(N))
|
|
349
|
+
|
|
350
|
+
if tail == "linear":
|
|
351
|
+
# ---- vectorized interior median slope + 2x cap (as in the
|
|
352
|
+
# reference implementation) ---------------------------------------
|
|
353
|
+
a, b = int(0.2 * (Q - 1)), int(0.8 * (Q - 1)) + 1
|
|
354
|
+
dg = np.diff(x_grid[a:b], axis=0)
|
|
355
|
+
dT = np.diff(T[a:b], axis=0)
|
|
356
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
357
|
+
ratio = np.where(dg > 0, dT / np.where(dg > 0, dg, np.nan),
|
|
358
|
+
np.nan)
|
|
359
|
+
s_med = _nanmedian_axis0(ratio)
|
|
360
|
+
s_med = np.where(np.isnan(s_med), 0.0, s_med)
|
|
361
|
+
cap = np.where(s_med > 0, 2.0 * s_med, 0.0)
|
|
362
|
+
s_lo = np.clip(windowed_slope(x_grid, T, k_lo,
|
|
363
|
+
np.minimum(k_lo + 30, Q - 1)),
|
|
364
|
+
0.0, cap)
|
|
365
|
+
s_hi = np.clip(windowed_slope(x_grid, T, np.maximum(k_hi - 30, 0),
|
|
366
|
+
k_hi),
|
|
367
|
+
0.0, cap)
|
|
368
|
+
# ---- linear tail override (same where() as the reference) -------
|
|
369
|
+
g_lo = np.take_along_axis(x_grid, k_lo[None, :], axis=0)
|
|
370
|
+
T_lo = np.take_along_axis(T, k_lo[None, :], axis=0)
|
|
371
|
+
g_hi = np.take_along_axis(x_grid, k_hi[None, :], axis=0)
|
|
372
|
+
T_hi = np.take_along_axis(T, k_hi[None, :], axis=0)
|
|
373
|
+
ok_lo = (k_lo > 0) & (k_lo < Q - 1)
|
|
374
|
+
ok_hi = (k_hi > 0) & (k_hi < Q - 1)
|
|
375
|
+
y = np.where((x < g_lo) & ok_lo[None, :],
|
|
376
|
+
T_lo + s_lo[None, :] * (x - g_lo), y)
|
|
377
|
+
y = np.where((x > g_hi) & ok_hi[None, :],
|
|
378
|
+
T_hi + s_hi[None, :] * (x - g_hi), y)
|
|
379
|
+
elif tail != "clip":
|
|
380
|
+
raise ValueError(f"tail must be 'linear' or 'clip', got {tail!r}")
|
|
381
|
+
return y
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
# ---------------------------------------------------------------------------
|
|
385
|
+
# Histogram-based ECDF / quantile variants (bounded memory)
|
|
386
|
+
# ---------------------------------------------------------------------------
|
|
387
|
+
|
|
388
|
+
def histogram_accumulate(values, n_bins: int = 2000):
|
|
389
|
+
"""Per-cell histogram with global (field-wide) edges.
|
|
390
|
+
|
|
391
|
+
Parameters
|
|
392
|
+
----------
|
|
393
|
+
values : (T, N) array.
|
|
394
|
+
n_bins : int, number of bins (the edges are ``n_bins + 1``).
|
|
395
|
+
|
|
396
|
+
Returns
|
|
397
|
+
-------
|
|
398
|
+
(edges (n_bins + 1,), counts (n_bins, N)): NaNs are excluded from the
|
|
399
|
+
binning (sentinel bin); all-NaN fields give ``edges = linspace(0, 1, ...)``
|
|
400
|
+
and zero counts.
|
|
401
|
+
"""
|
|
402
|
+
values = np.asarray(values, np.float64)
|
|
403
|
+
if values.ndim != 2:
|
|
404
|
+
raise ValueError(f"expected a 2-D (T, N) array, got {values.shape}")
|
|
405
|
+
with warnings.catch_warnings():
|
|
406
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
407
|
+
lo = float(np.nanmin(values))
|
|
408
|
+
hi = float(np.nanmax(values))
|
|
409
|
+
if not (np.isfinite(lo) and np.isfinite(hi)):
|
|
410
|
+
lo, hi = 0.0, 1.0 # all-NaN field: neutral edges (counts 0)
|
|
411
|
+
elif hi <= lo:
|
|
412
|
+
lo, hi = lo - 1.0, hi + 1.0
|
|
413
|
+
edges = np.linspace(lo, hi, n_bins + 1)
|
|
414
|
+
with warnings.catch_warnings():
|
|
415
|
+
warnings.simplefilter("ignore", RuntimeWarning)
|
|
416
|
+
idx = np.clip(((values - lo) / (hi - lo) * n_bins).astype(np.int64),
|
|
417
|
+
0, n_bins - 1)
|
|
418
|
+
idx = np.where(np.isfinite(values), idx, n_bins) # sentinel bin
|
|
419
|
+
N = values.shape[1]
|
|
420
|
+
counts = np.zeros((n_bins + 1, N), dtype=np.float64)
|
|
421
|
+
stride = n_bins + 1
|
|
422
|
+
for s in range(0, values.shape[0], 500):
|
|
423
|
+
block = idx[s:s + 500]
|
|
424
|
+
flat = block + (np.arange(N) * stride)[None]
|
|
425
|
+
counts += np.bincount(flat.ravel(),
|
|
426
|
+
minlength=stride * N).reshape(N, stride).T
|
|
427
|
+
return edges, counts[:-1]
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def hist_quantile(edges, counts, q, workers: int = 1) -> np.ndarray:
|
|
431
|
+
"""Quantile function of per-cell histograms with GLOBAL 1-D edges.
|
|
432
|
+
|
|
433
|
+
Same convention as :func:`quantile_from_sorted` (type-7-style linear
|
|
434
|
+
within-bin interpolation scaled by ``total - 1``); ``q=0`` interpolates
|
|
435
|
+
inside the first non-empty bin instead of extrapolating below the global
|
|
436
|
+
minimum; all-empty cells give NaN.
|
|
437
|
+
|
|
438
|
+
Parameters
|
|
439
|
+
----------
|
|
440
|
+
edges : (B + 1,) bin edges.
|
|
441
|
+
counts : (B, N) bin counts.
|
|
442
|
+
q : (Q,) or (Q, N) probabilities.
|
|
443
|
+
workers : int, threads for the per-cell searchsorted.
|
|
444
|
+
|
|
445
|
+
Returns
|
|
446
|
+
-------
|
|
447
|
+
(Q, N) quantile values.
|
|
448
|
+
"""
|
|
449
|
+
edges = np.asarray(edges, np.float64)
|
|
450
|
+
counts = np.asarray(counts, np.float64)
|
|
451
|
+
q = np.asarray(q, np.float64)
|
|
452
|
+
B, N = counts.shape
|
|
453
|
+
if edges.ndim != 1 or edges.size != B + 1:
|
|
454
|
+
raise ValueError(f"expected edges of shape ({B + 1},), got {edges.shape}")
|
|
455
|
+
if q.ndim == 1:
|
|
456
|
+
q = np.broadcast_to(q[:, None], (q.size, N))
|
|
457
|
+
C = np.cumsum(counts, axis=0)
|
|
458
|
+
total = C[-1]
|
|
459
|
+
pos = q * (total - 1.0)[None]
|
|
460
|
+
b = searchsorted_columns(C, pos, side="right", workers=workers)
|
|
461
|
+
b_safe = np.clip(b, 1, B - 1)
|
|
462
|
+
lo = np.take_along_axis(C, b_safe - 1, axis=0)
|
|
463
|
+
hi = np.take_along_axis(C, b_safe, axis=0)
|
|
464
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
465
|
+
frac = np.where(hi > lo, (pos - lo) / np.where(hi > lo, hi - lo, 1.0),
|
|
466
|
+
0.0)
|
|
467
|
+
out = edges[b_safe] + frac * (edges[b_safe + 1] - edges[b_safe])
|
|
468
|
+
# q=0 / below the first non-empty bin: interpolate within bin 0
|
|
469
|
+
c0 = np.maximum(C[0], 1.0)[None, :]
|
|
470
|
+
frac0 = np.where(C[0][None, :] > 0, pos / c0, 0.0)
|
|
471
|
+
out0 = edges[0] + frac0 * (edges[1] - edges[0])
|
|
472
|
+
out = np.where(b == 0, out0, out)
|
|
473
|
+
return np.where(total[None, :] > 0, out, np.nan)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def hist_quantile_pc(edges, counts, q, workers: int = 1) -> np.ndarray:
|
|
477
|
+
"""Quantile function of per-cell histograms with PER-CELL edges.
|
|
478
|
+
|
|
479
|
+
Same convention as :func:`hist_quantile`, but ``edges`` is ``(B + 1, N)``
|
|
480
|
+
(a different edge vector per column).
|
|
481
|
+
|
|
482
|
+
Parameters
|
|
483
|
+
----------
|
|
484
|
+
edges : (B + 1, N) bin edges.
|
|
485
|
+
counts : (B, N) bin counts.
|
|
486
|
+
q : (Q,) or (Q, N) probabilities.
|
|
487
|
+
workers : int, threads for the per-cell searchsorted.
|
|
488
|
+
|
|
489
|
+
Returns
|
|
490
|
+
-------
|
|
491
|
+
(Q, N) quantile values.
|
|
492
|
+
"""
|
|
493
|
+
edges = np.asarray(edges, np.float64)
|
|
494
|
+
counts = np.asarray(counts, np.float64)
|
|
495
|
+
q = np.asarray(q, np.float64)
|
|
496
|
+
B, N = counts.shape
|
|
497
|
+
if edges.shape != (B + 1, N):
|
|
498
|
+
raise ValueError(f"expected edges of shape ({B + 1}, {N}), "
|
|
499
|
+
f"got {edges.shape}")
|
|
500
|
+
if q.ndim == 1:
|
|
501
|
+
q = np.broadcast_to(q[:, None], (q.size, N))
|
|
502
|
+
C = np.cumsum(counts, axis=0)
|
|
503
|
+
total = C[-1]
|
|
504
|
+
pos = q * (total - 1.0)[None]
|
|
505
|
+
b = searchsorted_columns(C, pos, side="right", workers=workers)
|
|
506
|
+
b_safe = np.clip(b, 1, B - 1)
|
|
507
|
+
lo = np.take_along_axis(C, b_safe - 1, axis=0)
|
|
508
|
+
hi = np.take_along_axis(C, b_safe, axis=0)
|
|
509
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
510
|
+
frac = np.where(hi > lo, (pos - lo) / np.where(hi > lo, hi - lo, 1.0),
|
|
511
|
+
0.0)
|
|
512
|
+
e_lo = np.take_along_axis(edges, b_safe, axis=0)
|
|
513
|
+
e_hi = np.take_along_axis(edges, b_safe + 1, axis=0)
|
|
514
|
+
out = e_lo + frac * (e_hi - e_lo)
|
|
515
|
+
# q=0 / below the first non-empty bin: interpolate within bin 0
|
|
516
|
+
c0 = np.maximum(C[0], 1.0)[None, :]
|
|
517
|
+
frac0 = np.where(C[0][None, :] > 0, pos / c0, 0.0)
|
|
518
|
+
out0 = edges[0][None, :] + frac0 * (edges[1][None, :] - edges[0][None, :])
|
|
519
|
+
out = np.where(b == 0, out0, out)
|
|
520
|
+
return np.where(total[None, :] > 0, out, np.nan)
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def ecdf_hist(edges, counts, points, workers: int = 1) -> np.ndarray:
|
|
524
|
+
"""Empirical CDF of per-cell histograms at ``points`` (M, N) -> (M, N).
|
|
525
|
+
|
|
526
|
+
Linear interpolation inside a bin (consistent with
|
|
527
|
+
:func:`hist_quantile_pc`): ``F(v) = (C[b-1] + frac * counts[b]) / total``;
|
|
528
|
+
``F = 0`` below the first edge, ``1`` at/above the last edge; all-empty
|
|
529
|
+
cells and NaN points give NaN. ``edges`` may be 1-D (global) or
|
|
530
|
+
``(B + 1, N)`` (per cell).
|
|
531
|
+
"""
|
|
532
|
+
edges = np.asarray(edges, np.float64)
|
|
533
|
+
counts = np.asarray(counts, np.float64)
|
|
534
|
+
points = np.asarray(points, np.float64)
|
|
535
|
+
B, N = counts.shape
|
|
536
|
+
if edges.ndim == 1:
|
|
537
|
+
if edges.size != B + 1:
|
|
538
|
+
raise ValueError(f"expected edges of shape ({B + 1},), "
|
|
539
|
+
f"got {edges.shape}")
|
|
540
|
+
edges = np.broadcast_to(edges[:, None], (B + 1, N))
|
|
541
|
+
if edges.shape != (B + 1, N):
|
|
542
|
+
raise ValueError(f"expected edges of shape ({B + 1}, {N}), "
|
|
543
|
+
f"got {edges.shape}")
|
|
544
|
+
C = np.cumsum(counts, axis=0)
|
|
545
|
+
total = C[-1]
|
|
546
|
+
idx = searchsorted_columns(edges, points, side="right",
|
|
547
|
+
workers=workers) - 1
|
|
548
|
+
idx_safe = np.clip(idx, 0, B - 1)
|
|
549
|
+
lo = np.take_along_axis(edges, idx_safe, axis=0)
|
|
550
|
+
hi = np.take_along_axis(edges, idx_safe + 1, axis=0)
|
|
551
|
+
with np.errstate(divide="ignore", invalid="ignore"):
|
|
552
|
+
frac = np.where(hi > lo, (points - lo) / np.where(hi > lo, hi - lo, 1.0),
|
|
553
|
+
0.0)
|
|
554
|
+
c_below = np.take_along_axis(C, np.clip(idx_safe - 1, 0, B - 1), axis=0)
|
|
555
|
+
c_below = np.where(idx_safe > 0, c_below, 0.0)
|
|
556
|
+
cnt = np.take_along_axis(counts, idx_safe, axis=0)
|
|
557
|
+
out = (c_below + frac * cnt) / np.where(total > 0, total, 1.0)
|
|
558
|
+
out = np.where(points <= edges[0], 0.0, out)
|
|
559
|
+
out = np.where(points >= edges[-1], 1.0, out)
|
|
560
|
+
out = np.where(np.isfinite(points), out, np.nan)
|
|
561
|
+
return np.where(total[None, :] > 0, np.clip(out, 0.0, 1.0), np.nan)
|
pycdft/py.typed
ADDED
|
File without changes
|