quantmet-tools 0.4.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
quantmet/stats.py ADDED
@@ -0,0 +1,368 @@
1
+ """Significance tools for geophysical time series and composites: false
2
+ discovery rate, regression with AR(1) errors, the variance inflation of a
3
+ mean of autocorrelated values, event-block bootstrap composites, and
4
+ permutation / bootstrap tests for correlations and classifiers.
5
+
6
+ Numerical notes that matter in practice:
7
+ - FALSE DISCOVERY RATE, NOT PER-POINT 5 %. A map or section of p-values
8
+ tested point by point at 5 % shades 5 % of a pure-noise field, in
9
+ spatially coherent blobs that look like signal. Benjamini-Hochberg (as
10
+ recommended for field significance by Wilks 2016, BAMS, with
11
+ alpha_FDR = 0.10) controls the expected share of false discoveries and
12
+ stays valid under the positive spatial correlation of geophysical fields.
13
+ NaN p-values (masked points) are excluded from the count, not treated as
14
+ 1: a land mask should not make the ocean test stricter.
15
+ - AR(1) ERRORS (prais_winsten). Monthly or daily anomalies are persistent
16
+ (lag-1 autocorrelation 0.8-0.95 for stratospheric age of air), so OLS
17
+ standard errors are too small by a factor ~sqrt((1+rho)/(1-rho)). Prais-
18
+ Winsten GLS keeps the first observation (scaled by sqrt(1 - rho^2))
19
+ instead of dropping it as Cochrane-Orcutt does, and `start` marks rows
20
+ that begin a new contiguous segment -- after an excluded volcanic window,
21
+ say -- which are scaled like the first row instead of being differenced
22
+ against a row that is not their predecessor.
23
+ - BARTLETT, NOT AR(1), FOR THE NOISE OF A MEAN (acf_inflation). The
24
+ variance of an n-day mean is var/n x (1 + 2 sum_k (1 - k/n) rho_k). An
25
+ AR(1) model fills that sum from rho_1 alone, which fails for oscillating
26
+ series: a daily index with rho_1 = 0.59 but a ~6-day oscillation
27
+ (rho_3 = -0.44, rho_6 = +0.31) has an AR(1) inflation of 3.9 and a
28
+ Bartlett-window inflation (L = 10) of ~1.3 -- the AR(1) model overstates
29
+ the noise of a 30-day mean about threefold. The Bartlett sum uses the
30
+ measured autocorrelations with weights 1 - k/(L+1).
31
+ - RESAMPLE EVENTS, NOT DAYS (event_bootstrap). Days of one event (an MJO
32
+ phase passage, an SSW) are not independent; a day bootstrap understates
33
+ the composite's spread by the square root of the event length. Resampling
34
+ whole events with replacement, each carrying all of its days, keeps the
35
+ within-event correlation. The composite is the mean over resampled DAYS
36
+ (events weighted by their length), as the composite itself is.
37
+ - FREEDMAN-LANE (partial_corr_test). To test x against y given covariates
38
+ Z, permute the residuals of the REDUCED model (y on Z), add them back to
39
+ its fitted values, and recompute the partial correlation. Permuting the
40
+ residuals of x and correlating them with those of y (Kennedy 1995)
41
+ ignores that the permuted residuals are no longer orthogonal to Z and is
42
+ anti-conservative in small samples with correlated covariates.
43
+
44
+ Consolidated from the significance code of the stratosphere, MJO and
45
+ age-of-air products on https://scorvec.com/stratosphere.html and
46
+ https://scorvec.com/mjo.html.
47
+
48
+ Requires: numpy, scipy (distributions, ranks).
49
+ """
50
+
51
+ from __future__ import annotations
52
+
53
+ from dataclasses import dataclass
54
+
55
+ import numpy as np
56
+
57
+ __all__ = [
58
+ "fdr_adjust",
59
+ "benjamini_hochberg",
60
+ "prais_winsten",
61
+ "PraisWinsten",
62
+ "segment_starts",
63
+ "acf_inflation",
64
+ "label_events",
65
+ "event_bootstrap",
66
+ "pearson",
67
+ "residualize",
68
+ "partial_corr_test",
69
+ "compare_r",
70
+ "auc",
71
+ ]
72
+
73
+
74
+ # --------------------------------------------------------------------------
75
+ # False discovery rate
76
+ # --------------------------------------------------------------------------
77
+ def fdr_adjust(p) -> np.ndarray:
78
+ """Benjamini-Hochberg adjusted p-values (q-values), any shape; NaN stays
79
+ NaN and is excluded from the count. Same as
80
+ scipy.stats.false_discovery_control(p, method="bh") on the finite p."""
81
+ p = np.asarray(p, float)
82
+ out = np.full(p.shape, np.nan)
83
+ ok = np.isfinite(p)
84
+ q = p[ok]
85
+ n = q.size
86
+ if n == 0:
87
+ return out
88
+ o = np.argsort(q)
89
+ adj = q[o] * n / np.arange(1, n + 1)
90
+ adj = np.minimum.accumulate(adj[::-1])[::-1]
91
+ res = np.empty(n)
92
+ res[o] = np.clip(adj, 0, 1)
93
+ out[ok] = res
94
+ return out
95
+
96
+
97
+ def benjamini_hochberg(p, alpha: float = 0.10) -> np.ndarray:
98
+ """Boolean mask (p's shape) of discoveries at false discovery rate alpha:
99
+ every finite p at or below the largest p_(i) with p_(i) <= alpha i / n."""
100
+ p = np.asarray(p, float)
101
+ ok = np.isfinite(p)
102
+ out = np.zeros(p.shape, bool)
103
+ s = np.sort(p[ok])
104
+ n = s.size
105
+ if n == 0:
106
+ return out
107
+ below = s <= alpha * np.arange(1, n + 1) / n
108
+ if below.any():
109
+ out = ok & (p <= s[np.flatnonzero(below).max()])
110
+ return out
111
+
112
+
113
+ # --------------------------------------------------------------------------
114
+ # Regression with AR(1) errors
115
+ # --------------------------------------------------------------------------
116
+ @dataclass
117
+ class PraisWinsten:
118
+ """beta (k, m), se (k, m), rho (m,), dof; m = number of series."""
119
+
120
+ beta: np.ndarray
121
+ se: np.ndarray
122
+ rho: np.ndarray
123
+ dof: int
124
+
125
+
126
+ def segment_starts(keep) -> np.ndarray:
127
+ """For the retained rows of a boolean mask over a regular time axis:
128
+ True where the previous time step was not retained (a new segment)."""
129
+ idx = np.flatnonzero(np.asarray(keep, bool))
130
+ return np.r_[True, np.diff(idx) > 1] if idx.size else np.zeros(0, bool)
131
+
132
+
133
+ def prais_winsten(y, X, start=None, iters: int = 2, rho_max: float = 0.97) -> PraisWinsten:
134
+ """GLS with AR(1) errors (Prais-Winsten), iterated `iters` times.
135
+
136
+ y : (n,) or (n, m) -- m series sharing one design
137
+ X : (n, k) design matrix (include the intercept column yourself)
138
+ start : (n,) bool, rows that begin a contiguous segment (row 0 always
139
+ does); they are scaled by sqrt(1 - rho^2) like the first row
140
+ instead of being quasi-differenced. See segment_starts.
141
+ rho is estimated from residual pairs within segments only and clipped to
142
+ +-rho_max. se is the GLS standard error with dof = n - k."""
143
+ y = np.asarray(y, float)
144
+ one = y.ndim == 1
145
+ Y = y[:, None] if one else y
146
+ X = np.asarray(X, float)
147
+ n, k = X.shape
148
+ st = np.zeros(n, bool) if start is None else np.asarray(start, bool).copy()
149
+ st[0] = True
150
+ pair = ~st[1:]
151
+ nx = np.flatnonzero(~st)
152
+ beta = np.linalg.lstsq(X, Y, rcond=None)[0]
153
+ rho = np.zeros(Y.shape[1])
154
+ se = np.zeros_like(beta)
155
+ for _ in range(max(1, iters)):
156
+ r = Y - X @ beta
157
+ num = (r[1:][pair] * r[:-1][pair]).sum(0)
158
+ den = (r[:-1][pair] ** 2).sum(0)
159
+ rho = np.clip(np.divide(num, den, out=np.zeros_like(num), where=den > 0), -rho_max, rho_max)
160
+ beta = np.empty((k, Y.shape[1]))
161
+ se = np.empty((k, Y.shape[1]))
162
+ for j in range(Y.shape[1]):
163
+ c = np.sqrt(1 - rho[j] ** 2)
164
+ ys = np.empty(n)
165
+ Xs = np.empty_like(X)
166
+ ys[st] = c * Y[st, j]
167
+ Xs[st] = c * X[st]
168
+ ys[nx] = Y[nx, j] - rho[j] * Y[nx - 1, j]
169
+ Xs[nx] = X[nx] - rho[j] * X[nx - 1]
170
+ b = np.linalg.lstsq(Xs, ys, rcond=None)[0]
171
+ e = ys - Xs @ b
172
+ s2 = (e @ e) / (n - k)
173
+ beta[:, j] = b
174
+ se[:, j] = np.sqrt(np.diag(s2 * np.linalg.inv(Xs.T @ Xs)))
175
+ if one:
176
+ return PraisWinsten(beta[:, 0], se[:, 0], rho[:1], n - k)
177
+ return PraisWinsten(beta, se, rho, n - k)
178
+
179
+
180
+ # --------------------------------------------------------------------------
181
+ # Autocorrelation
182
+ # --------------------------------------------------------------------------
183
+ def acf_inflation(res, times=None, lags: int = 10, clip=(1.0, None)) -> np.ndarray:
184
+ """Variance inflation of a mean of autocorrelated values,
185
+ 1 + 2 sum_{k=1..L} (1 - k/(L+1)) rho_k (Bartlett window, L = lags),
186
+ per series along axis 0 of `res`.
187
+
188
+ times : optional datetime64-like array (one per row); the rows are then
189
+ placed on a daily grid with NaN gaps, so the lags are in days
190
+ even when some days are missing. None: rows are consecutive.
191
+ clip : (lo, hi) bounds on the result (None = open)."""
192
+ R = np.asarray(res, float)
193
+ if times is not None:
194
+ d = np.asarray(times, dtype="datetime64[D]")
195
+ k = (d - d.min()).astype(int)
196
+ full = np.full((k.max() + 1, *R.shape[1:]), np.nan)
197
+ full[k] = R
198
+ R = full
199
+ R = R - np.nanmean(R, 0)
200
+ v = np.nanmean(R * R, 0)
201
+ infl = np.ones(R.shape[1:])
202
+ for k in range(1, lags + 1):
203
+ rk = np.nanmean(R[k:] * R[:-k], 0) / v
204
+ infl = infl + 2 * (1 - k / (lags + 1)) * np.nan_to_num(rk)
205
+ lo, hi = clip
206
+ return np.clip(infl, lo, hi) if (lo is not None or hi is not None) else infl
207
+
208
+
209
+ # --------------------------------------------------------------------------
210
+ # Composites
211
+ # --------------------------------------------------------------------------
212
+ def label_events(key, active) -> np.ndarray:
213
+ """Event id per time step: a run of consecutive active steps with the
214
+ same key (e.g. the MJO phase) is one event; -1 where inactive."""
215
+ key = np.asarray(key)
216
+ active = np.asarray(active, bool)
217
+ ev = np.full(key.shape[0], -1)
218
+ k = -1
219
+ for i in range(key.shape[0]):
220
+ if active[i]:
221
+ if i == 0 or not active[i - 1] or key[i] != key[i - 1]:
222
+ k += 1
223
+ ev[i] = k
224
+ return ev
225
+
226
+
227
+ def event_bootstrap(X, events, nboot: int = 1000, rng=None, return_samples: bool = False):
228
+ """Composite mean of X (ndays, ...) over its rows and its standard error
229
+ from an EVENT bootstrap: events (one id per row) are resampled with
230
+ replacement and carry all their rows. NaN in X is skipped per cell.
231
+
232
+ Returns (mean, se), plus the (nboot, ...) bootstrap means if
233
+ return_samples. se is NaN with fewer than two events."""
234
+ X = np.asarray(X, float)
235
+ ev = np.asarray(events)
236
+ if ev.shape[0] != X.shape[0]:
237
+ raise ValueError("one event id per row of X")
238
+ rng = np.random.default_rng(rng)
239
+ flat = X.reshape(X.shape[0], -1)
240
+ ok = np.isfinite(flat)
241
+ with np.errstate(invalid="ignore"):
242
+ mean = np.where(ok.any(0), np.nansum(flat, 0) / ok.sum(0), np.nan)
243
+ ue, inv = np.unique(ev, return_inverse=True)
244
+ ne = ue.size
245
+ shape = X.shape[1:]
246
+ if ne < 2:
247
+ nan = np.full(shape, np.nan)
248
+ return (mean.reshape(shape), nan) + ((np.full((0, *shape), np.nan),) if return_samples else ())
249
+ S = np.zeros((ne, flat.shape[1]))
250
+ C = np.zeros((ne, flat.shape[1]))
251
+ np.add.at(S, inv, np.where(ok, flat, 0.0))
252
+ np.add.at(C, inv, ok.astype(float))
253
+ R = rng.integers(0, ne, size=(nboot, ne))
254
+ W = np.zeros((nboot, ne))
255
+ np.add.at(W, (np.repeat(np.arange(nboot), ne), R.ravel()), 1.0)
256
+ den = W @ C
257
+ with np.errstate(invalid="ignore", divide="ignore"):
258
+ bm = (W @ S) / np.where(den > 0, den, np.nan)
259
+ se = np.nanstd(bm, axis=0)
260
+ out = (mean.reshape(shape), se.reshape(shape))
261
+ return out + ((bm.reshape((nboot, *shape)),) if return_samples else ())
262
+
263
+
264
+ # --------------------------------------------------------------------------
265
+ # Correlation tests
266
+ # --------------------------------------------------------------------------
267
+ def pearson(x, y) -> float:
268
+ x = np.asarray(x, float) - np.mean(x)
269
+ y = np.asarray(y, float) - np.mean(y)
270
+ return float((x * y).sum() / np.sqrt((x * x).sum() * (y * y).sum()))
271
+
272
+
273
+ def _design(covs, n):
274
+ cols = [np.ones(n)] + [np.asarray(c, float) for c in (covs or [])]
275
+ return np.column_stack(cols)
276
+
277
+
278
+ def residualize(y, covs):
279
+ """Residual of y after OLS on an intercept and the covariates."""
280
+ y = np.asarray(y, float)
281
+ Z = _design(covs, y.size)
282
+ return y - Z @ np.linalg.lstsq(Z, y, rcond=None)[0]
283
+
284
+
285
+ def partial_corr_test(x, y, covs=(), nperm: int = 9999, nboot: int = 9999, rng=None) -> dict:
286
+ """Partial correlation of x with y given covariates, with
287
+
288
+ p_perm : two-sided Freedman-Lane permutation p (reduced-model
289
+ residuals of y permuted, added back to its fitted values)
290
+ ci : 95 % bootstrap interval (rows resampled, covariates refit)
291
+ p_F : nested-OLS F-test p for adding x to the covariates
292
+ dR2 : the R^2 that x adds.
293
+ """
294
+ from scipy import stats
295
+
296
+ rng = np.random.default_rng(rng)
297
+ x = np.asarray(x, float)
298
+ y = np.asarray(y, float)
299
+ n = x.size
300
+ covs = [np.asarray(c, float) for c in covs]
301
+ Z = _design(covs, n)
302
+ Hz = Z @ np.linalg.pinv(Z) # hat matrix of the reduced model
303
+ rx = x - Hz @ x
304
+ yfit = Hz @ y
305
+ ry = y - yfit
306
+ r = pearson(rx, ry)
307
+ count = 0
308
+ for _ in range(nperm):
309
+ ys = yfit + rng.permutation(ry)
310
+ rs = pearson(rx, ys - Hz @ ys)
311
+ count += abs(rs) >= abs(r) - 1e-12
312
+ p_perm = (count + 1) / (nperm + 1)
313
+ bs = []
314
+ for _ in range(nboot):
315
+ i = rng.integers(0, n, n)
316
+ a = residualize(x[i], [c[i] for c in covs])
317
+ b = residualize(y[i], [c[i] for c in covs])
318
+ if a.std() > 0 and b.std() > 0:
319
+ bs.append(pearson(a, b))
320
+ k = len(covs)
321
+ rss0 = (ry**2).sum()
322
+ rss1 = (residualize(y, [*covs, x]) ** 2).sum()
323
+ F = (rss0 - rss1) / (rss1 / (n - k - 2))
324
+ return {
325
+ "r": r,
326
+ "p_perm": float(p_perm),
327
+ "ci": (float(np.percentile(bs, 2.5)), float(np.percentile(bs, 97.5))),
328
+ "p_F": float(stats.f.sf(F, 1, n - k - 2)),
329
+ "dR2": float((rss0 - rss1) / ((y - y.mean()) ** 2).sum()),
330
+ "n": n,
331
+ }
332
+
333
+
334
+ def compare_r(x1, x2, y, nboot: int = 9999, rng=None) -> dict:
335
+ """Is x1 a better linear predictor of y than x2? d = |r(x1, y)| -
336
+ |r(x2, y)| with a paired bootstrap (the same resampled rows for both, so
337
+ their correlation is respected): 95 % interval and p = share of
338
+ resamples with d <= 0 (one-sided, x1 not better)."""
339
+ rng = np.random.default_rng(rng)
340
+ x1, x2, y = (np.asarray(a, float) for a in (x1, x2, y))
341
+ d0 = abs(pearson(x1, y)) - abs(pearson(x2, y))
342
+ n = y.size
343
+ ds = np.empty(nboot)
344
+ for b in range(nboot):
345
+ i = rng.integers(0, n, n)
346
+ ds[b] = abs(pearson(x1[i], y[i])) - abs(pearson(x2[i], y[i]))
347
+ ds = ds[np.isfinite(ds)]
348
+ return {
349
+ "diff_abs_r": float(d0),
350
+ "ci": (float(np.percentile(ds, 2.5)), float(np.percentile(ds, 97.5))),
351
+ "p_not_better": float(np.mean(ds <= 0)),
352
+ }
353
+
354
+
355
+ def auc(score, label) -> float:
356
+ """Area under the ROC curve of `score` for binary `label` (1 = event):
357
+ the Mann-Whitney probability that an event outscores a non-event, ties
358
+ counted half (average ranks)."""
359
+ from scipy.stats import rankdata
360
+
361
+ score = np.asarray(score, float)
362
+ label = np.asarray(label).astype(bool)
363
+ n1 = label.sum()
364
+ n0 = label.size - n1
365
+ if n1 == 0 or n0 == 0:
366
+ return float("nan")
367
+ r = rankdata(score)
368
+ return float((r[label].sum() - n1 * (n1 + 1) / 2) / (n1 * n0))
quantmet/tc.py ADDED
@@ -0,0 +1,320 @@
1
+ """Tropical cyclones in ensemble forecasts: ECMWF track decoding, recurvature, the outflow-jet interaction metric, and
2
+ Hart's cyclone phase space.
3
+
4
+ Tracks. ECMWF disseminates its tropical-cyclone tracker output as BUFR (open data: ``...-enfo-tf.bufr`` for the
5
+ ensembles, ``...-oper-tf.bufr`` for the deterministic runs). One message per storm, one subset per ensemble member;
6
+ the ``ensembleForecastType`` is 1 for the unperturbed control, 4 for perturbed members, 0 for the high-resolution
7
+ deterministic run carried in the same file (excluded here). The analysis position is ``#2#latitude`` (falling back to
8
+ the observed ``#1#latitude``); step i of the delayed replication is ``#i#timePeriod`` with the centre at
9
+ ``#(2i+2)#latitude`` and MSLP / max wind at ``#(i+1)#``. The tracker also repeats named storms under placeholder
10
+ genesis ids (70-99 + basin letter); de-duplicate by position if you merge them.
11
+
12
+ Recurvature: the lead where the storm's 24-h zonal motion turns from westward (<= -1 m/s) to eastward (>= +2 m/s).
13
+ A storm heading north or east from the start has none (it never had a westward phase in the forecast).
14
+
15
+ Outflow metric (Archambault, Bosart, Keyser & Cordeira 2013, MWR 141; Archambault et al. 2015, MWR 143): negative PV
16
+ advection by the irrotational wind, -v_chi . grad(PV), in the 300-200 hPa layer. The divergent outflow of a recurving
17
+ storm pushes low-PV air poleward against the PV gradient at the jet, building the downstream ridge. ``outflow_index``
18
+ is the area mean of the negative part within 500 km of the centre, sign flipped (PVU/day; 0 = no interaction).
19
+ Numerics: v_chi from a spherical-harmonic inversion (quantmet.helmholtz) is smooth by construction, while the PV
20
+ gradient on a 0.25 deg grid is grid-noisy -- smooth PV (~1 deg) before differentiating, or the contours speckle.
21
+
22
+ Hart (2003, MWR 131) phase space, within 500 km of the centre:
23
+ B = (Z_top - Z_bottom) thickness right of the storm motion minus left (m, NH sign; negate in the SH).
24
+ |B| ~ 0 thermally symmetric (tropical); B > 10 m marks extratropical-transition ONSET (Evans & Hart 2003).
25
+ -VT_L = d(dZ)/d(ln p) over 900-600 hPa, dZ = Zmax - Zmin in the disc; > 0 warm core, < 0 cold core.
26
+ -VT_U = the same over 600-300 hPa. ET is COMPLETE when -VT_L turns negative after onset.
27
+ Hart used 25 hPa levels; with fewer levels (open data: 925/850/700/600/500/400/300) the fit is coarser -- say so.
28
+ Parameters are conventionally smoothed with a 24-h running mean.
29
+
30
+ Powers the tropical-cyclone cards at https://scorvec.com/circulation.html.
31
+
32
+ Requires: numpy; eccodes for ``decode_tracks``; pyshtools + xarray for ``irrotational_on_grid``.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import numpy as np
38
+
39
+ from .constants import A_EARTH
40
+
41
+ MISSING = -1e99
42
+ __all__ = [
43
+ "decode_tracks",
44
+ "at",
45
+ "great_circle_km",
46
+ "within",
47
+ "recurvature",
48
+ "irrotational_on_grid",
49
+ "pv_advection",
50
+ "outflow_index",
51
+ "hart_parameters",
52
+ "phase_series",
53
+ "et_times",
54
+ ]
55
+
56
+
57
+ # ── tracks ────────────────────────────────────────────────────────────────────────────────────────────────────────
58
+
59
+
60
+ def decode_tracks(path) -> list[dict]:
61
+ """All storms in an ECMWF tf BUFR file: [{id, name, named, tracks: {member: track}}]; member 0 = control,
62
+ 1..N perturbed; track = dict(steps (h), lat, lon (-180..180), pmsl (hPa), wind (m/s)). The deterministic subset
63
+ (ensembleForecastType 0) is excluded."""
64
+ import eccodes as ec
65
+
66
+ out = []
67
+ with open(path, "rb") as f:
68
+ while True:
69
+ h = ec.codes_bufr_new_from_file(f)
70
+ if h is None:
71
+ break
72
+ try:
73
+ ec.codes_set(h, "unpack", 1)
74
+ sid = ec.codes_get(h, "#1#stormIdentifier").strip()
75
+ name = ec.codes_get(h, "#1#longStormName").strip()
76
+ ft = np.atleast_1d(ec.codes_get_array(h, "#1#ensembleForecastType"))
77
+ mem = np.atleast_1d(ec.codes_get_array(h, "#1#ensembleMemberNumber"))
78
+ n = len(ft)
79
+
80
+ def arr(key, h=h, n=n):
81
+ a = np.atleast_1d(ec.codes_get_array(h, key)).astype(float)
82
+ return np.repeat(a, n) if a.size == 1 else a
83
+
84
+ rep = ec.codes_get(h, "#1#delayedDescriptorReplicationFactor")
85
+ a_la, a_lo = arr("#2#latitude"), arr("#2#longitude")
86
+ o_la, o_lo = arr("#1#latitude"), arr("#1#longitude")
87
+ a_la = np.where(a_la > MISSING, a_la, o_la)
88
+ a_lo = np.where(a_lo > MISSING, a_lo, o_lo)
89
+ cols = [(np.zeros(n), a_la, a_lo, arr("#1#pressureReducedToMeanSeaLevel"), arr("#1#windSpeedAt10M"))]
90
+ cols.extend(
91
+ (
92
+ arr(f"#{i}#timePeriod"),
93
+ arr(f"#{2 * i + 2}#latitude"),
94
+ arr(f"#{2 * i + 2}#longitude"),
95
+ arr(f"#{i + 1}#pressureReducedToMeanSeaLevel"),
96
+ arr(f"#{i + 1}#windSpeedAt10M"),
97
+ )
98
+ for i in range(1, rep + 1)
99
+ )
100
+ tracks = {}
101
+ for j in range(n):
102
+ if ft[j] not in (1, 4):
103
+ continue
104
+ rows = [
105
+ (c[0][j], c[1][j], c[2][j], c[3][j], c[4][j])
106
+ for c in cols
107
+ if c[1][j] > MISSING and c[0][j] > MISSING
108
+ ]
109
+ if not rows:
110
+ continue
111
+ st, la, lo, p, w = (np.array(x, float) for x in zip(*rows))
112
+ m = 0 if ft[j] == 1 else int(mem[j])
113
+ tracks[m] = dict(
114
+ steps=st.astype(int),
115
+ lat=la,
116
+ lon=(lo + 180) % 360 - 180,
117
+ pmsl=np.where(p > MISSING, p / 100.0, np.nan),
118
+ wind=np.where(w > MISSING, w, np.nan),
119
+ )
120
+ if tracks:
121
+ named = not (sid[:2].isdigit() and int(sid[:2]) >= 70)
122
+ out.append(dict(id=sid, name=name, named=named, tracks=tracks))
123
+ finally:
124
+ ec.codes_release(h)
125
+ return out
126
+
127
+
128
+ def at(track: dict, h: int):
129
+ """(lat, lon) of the track at lead h, or None."""
130
+ k = np.where(np.asarray(track["steps"]) == h)[0]
131
+ return (float(track["lat"][k[0]]), float(track["lon"][k[0]])) if len(k) else None
132
+
133
+
134
+ def great_circle_km(la1, lo1, la2, lo2):
135
+ p1, p2, dl = np.deg2rad(la1), np.deg2rad(la2), np.deg2rad(np.asarray(lo2) - np.asarray(lo1))
136
+ c = np.sin(p1) * np.sin(p2) + np.cos(p1) * np.cos(p2) * np.cos(dl)
137
+ return A_EARTH / 1000 * np.arccos(np.clip(c, -1, 1))
138
+
139
+
140
+ def within(lat, lon, clat, clon, r_km) -> np.ndarray:
141
+ """(lat, lon) boolean mask of grid points within r_km of (clat, clon)."""
142
+ return great_circle_km(np.asarray(lat, float)[:, None], np.asarray(lon, float)[None, :], clat, clon) <= r_km
143
+
144
+
145
+ def recurvature(track: dict, west: float = -1.0, east: float = 2.0):
146
+ """Lead (h) of the turn: the last point with no eastward motion (24-h zonal motion <= 0) before the motion first
147
+ reaches >= east m/s, for a track that moved westward (<= west m/s) earlier; None if it never did or never turns
148
+ east. Needs steps every 12 h or finer."""
149
+ st = np.asarray(track["steps"], float)
150
+ la = np.asarray(track["lat"], float)
151
+ lo = np.rad2deg(np.unwrap(np.deg2rad(np.asarray(track["lon"], float))))
152
+ if len(st) < 5:
153
+ return None
154
+ u = np.full(len(st), np.nan)
155
+ for k in range(len(st)):
156
+ a = np.where(np.abs(st - (st[k] - 12)) < 1e-6)[0]
157
+ b = np.where(np.abs(st - (st[k] + 12)) < 1e-6)[0]
158
+ if len(a) and len(b):
159
+ u[k] = (lo[b[0]] - lo[a[0]]) * 111.2e3 * np.cos(np.deg2rad(la[k])) / 86400.0
160
+ w = np.where(u <= west)[0]
161
+ if not len(w):
162
+ return None
163
+ e = np.where((u >= east) & (np.arange(len(u)) > w[0]))[0]
164
+ if not len(e):
165
+ return None
166
+ k0 = int(np.where((u <= 0) & (np.arange(len(u)) < e[0]))[0][-1])
167
+ return int(st[k0])
168
+
169
+
170
+ # ── outflow metric ────────────────────────────────────────────────────────────────────────────────────────────────
171
+
172
+
173
+ def irrotational_on_grid(u2d, v2d, lat, lon, lmax: int = 106):
174
+ """(u_chi, v_chi) from global (lat, lon) xarray DataArrays u2d, v2d (dims latitude/longitude), interpolated to the
175
+ target (lat, lon) grid (degrees; lon in 0..360). Uses quantmet.helmholtz (spherical-harmonic inversion, T lmax)."""
176
+ import xarray as xr
177
+ from .helmholtz import irrotational_wind, velocity_potential
178
+
179
+ chi, dlat, dlon_ = velocity_potential(u2d, v2d, lmax=lmax)
180
+ uc, vc = irrotational_wind(chi, dlat, dlon_)
181
+ dl = np.asarray(dlon_, float)
182
+ if dl[-1] < 360.0 - 1e-6:
183
+ dl = np.r_[dl, dl[0] + 360.0]
184
+ uc = np.concatenate([uc, uc[:, :1]], axis=1)
185
+ vc = np.concatenate([vc, vc[:, :1]], axis=1)
186
+ dla = np.asarray(dlat, float)
187
+ keep = np.r_[True, np.diff(dla) != 0]
188
+ U = xr.DataArray(uc[keep], dims=("lat", "lon"), coords={"lat": dla[keep], "lon": dl}).sortby("lat")
189
+ V = xr.DataArray(vc[keep], dims=("lat", "lon"), coords={"lat": dla[keep], "lon": dl}).sortby("lat")
190
+ tgt = dict(
191
+ lat=xr.DataArray(np.asarray(lat, float), dims="y"), lon=xr.DataArray(np.asarray(lon, float) % 360, dims="x")
192
+ )
193
+ return U.interp(**tgt).values, V.interp(**tgt).values
194
+
195
+
196
+ def pv_advection(uchi, vchi, pv_layer, lat, lon, sigma_deg: float = 1.0) -> np.ndarray:
197
+ """-v_chi . grad(PV) in PVU/day on a regular (lat, lon) grid (degrees); PV in PVU, Gaussian-smoothed by sigma_deg
198
+ first (0 = no smoothing). Longitude periodic."""
199
+ from scipy.ndimage import gaussian_filter
200
+
201
+ lat = np.asarray(lat, float)
202
+ lon = np.asarray(lon, float)
203
+ pv = np.nan_to_num(np.asarray(pv_layer, float), nan=0.0)
204
+ if sigma_deg > 0:
205
+ n = sigma_deg / abs(lat[1] - lat[0])
206
+ pv = gaussian_filter(pv, sigma=(n, n), mode=("nearest", "wrap"))
207
+ phi = np.deg2rad(lat)
208
+ lam = np.deg2rad(lon)
209
+ cosphi = np.clip(np.cos(phi), 1e-3, None)[:, None]
210
+ dpdx = (np.roll(pv, -1, -1) - np.roll(pv, 1, -1)) / (2 * (lam[1] - lam[0])) / (A_EARTH * cosphi)
211
+ dpdy = np.gradient(pv, phi, axis=0) / A_EARTH
212
+ return -(uchi * dpdx + vchi * dpdy) * 86400.0
213
+
214
+
215
+ def outflow_index(adv, lat, lon, clat, clon, r_km: float = 500.0) -> float:
216
+ """Area (cos lat) mean of max(-adv, 0) within r_km of (clat, clon), PVU/day."""
217
+ lat = np.asarray(lat, float)
218
+ lon = np.asarray(lon, float)
219
+ j = np.where(np.abs(lat - clat) <= r_km / 111.0 + 0.5)[0]
220
+ if not len(j):
221
+ return np.nan
222
+ m = within(lat[j], lon, clat, clon, r_km)
223
+ if not m.any():
224
+ return np.nan
225
+ w = np.broadcast_to(np.cos(np.deg2rad(lat[j]))[:, None], m.shape)[m]
226
+ return float(np.sum(np.maximum(-np.asarray(adv)[j][m], 0) * w) / np.sum(w))
227
+
228
+
229
+ # ── Hart phase space ──────────────────────────────────────────────────────────────────────────────────────────────
230
+
231
+
232
+ def _slope(dz, levs):
233
+ return float(np.polyfit(np.log(np.asarray(levs, float)), np.asarray(dz, float), 1)[0])
234
+
235
+
236
+ def hart_parameters(
237
+ z,
238
+ levs,
239
+ lat,
240
+ lon,
241
+ clat,
242
+ clon,
243
+ motion_deg,
244
+ r_km: float = 500.0,
245
+ low=(900, 850, 800, 750, 700, 650, 600),
246
+ upp=(600, 550, 500, 450, 400, 350, 300),
247
+ thick=(600, 900),
248
+ ) -> dict:
249
+ """B, -VT_L, -VT_U (m) for one time.
250
+
251
+ z : geopotential HEIGHT (m), (lev, lat, lon); levs : hPa for axis 0; motion_deg : storm heading, degrees clockwise
252
+ from north. ``low``/``upp``/``thick`` are matched to the available levels (missing ones are dropped; at least two
253
+ per layer are needed). Southern-hemisphere storms: negate B (Hart's convention keeps B > 0 = frontal)."""
254
+ levs = [int(np.rint(x)) for x in levs]
255
+ low = [p for p in low if p in levs]
256
+ upp = [p for p in upp if p in levs]
257
+ if len(low) < 2 or len(upp) < 2:
258
+ raise ValueError("need at least two levels in each layer")
259
+ top, bot = thick
260
+ if top not in levs or bot not in levs:
261
+ bot = max(low)
262
+ top = min(low)
263
+ lat = np.asarray(lat, float)
264
+ lon = np.asarray(lon, float)
265
+ dl = r_km / 111.0 + 0.5
266
+ j = np.where(np.abs(lat - clat) <= dl)[0]
267
+ dlon = ((lon - clon + 180) % 360) - 180
268
+ i = np.where(np.abs(dlon) <= dl / max(np.cos(np.deg2rad(min(abs(clat) + dl, 80))), 0.2))[0]
269
+ la = np.deg2rad(lat[j])[:, None]
270
+ lo = np.deg2rad(lon[i])[None, :]
271
+ c0, l0 = np.deg2rad(clat), np.deg2rad(clon)
272
+ m = great_circle_km(lat[j][:, None], lon[i][None, :], clat, clon) <= r_km
273
+ brg = np.arctan2(np.sin(lo - l0) * np.cos(la), np.cos(c0) * np.sin(la) - np.sin(c0) * np.cos(la) * np.cos(lo - l0))
274
+ rel = ((brg - np.deg2rad(motion_deg) + np.pi) % (2 * np.pi)) - np.pi
275
+ right, left = m & (rel > 0) & (rel < np.pi), m & (rel < 0) & (rel > -np.pi)
276
+ Z = np.asarray(z, float)[:, j][:, :, i]
277
+ th = Z[levs.index(top)] - Z[levs.index(bot)]
278
+ B = float(th[right].mean() - th[left].mean()) if right.any() and left.any() else np.nan
279
+ dz = {p: float(Z[levs.index(p)][m].max() - Z[levs.index(p)][m].min()) for p in set(low) | set(upp)}
280
+ return dict(B=B, VTL=_slope([dz[p] for p in low], low), VTU=_slope([dz[p] for p in upp], upp))
281
+
282
+
283
+ def phase_series(z_steps, levs, lat, lon, track: dict, steps, smooth_h: float = 24.0, **kw) -> dict:
284
+ """Hart parameters along a track: z_steps (step, lev, lat, lon) at the given steps (h). Storm motion from the
285
+ neighbouring track points; a ``smooth_h`` running mean (Hart: 24 h). Returns lists h, lat, lon, B, VTL, VTU."""
286
+ out = {"h": [], "lat": [], "lon": [], "B": [], "VTL": [], "VTU": []}
287
+ ts = np.asarray(track["steps"])
288
+ for k, h in enumerate(steps):
289
+ pos = at(track, int(h))
290
+ if pos is None:
291
+ continue
292
+ kk = np.where(ts == h)[0][0]
293
+ a, b = max(kk - 1, 0), min(kk + 1, len(ts) - 1)
294
+ if a == b:
295
+ continue
296
+ dlo = ((track["lon"][b] - track["lon"][a] + 180) % 360) - 180
297
+ mot = np.rad2deg(
298
+ np.arctan2(np.deg2rad(dlo) * np.cos(np.deg2rad(pos[0])), np.deg2rad(track["lat"][b] - track["lat"][a]))
299
+ )
300
+ r = hart_parameters(z_steps[k], levs, lat, lon, pos[0], pos[1], mot, **kw)
301
+ if pos[0] < 0:
302
+ r["B"] = -r["B"]
303
+ out["h"].append(int(h))
304
+ out["lat"].append(pos[0])
305
+ out["lon"].append(pos[1])
306
+ for key in ("B", "VTL", "VTU"):
307
+ out[key].append(r[key])
308
+ if smooth_h:
309
+ hh = np.array(out["h"], float)
310
+ for key in ("B", "VTL", "VTU"):
311
+ v = np.array(out[key], float)
312
+ out[key] = [float(np.nanmean(v[np.abs(hh - x) <= smooth_h / 2])) for x in hh]
313
+ return out
314
+
315
+
316
+ def et_times(series: dict, b_onset: float = 10.0):
317
+ """Evans & Hart (2003): onset = first B > b_onset; completion = first -VT_L < 0 at or after onset."""
318
+ onset = next((h for h, b in zip(series["h"], series["B"]) if b > b_onset), None)
319
+ comp = next((h for h, v in zip(series["h"], series["VTL"]) if v < 0 and (onset is None or h >= onset)), None)
320
+ return onset, comp