mannks 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- MannKS/__init__.py +26 -0
- MannKS/_ats.py +493 -0
- MannKS/_datetime.py +129 -0
- MannKS/_helpers.py +255 -0
- MannKS/_stats.py +569 -0
- MannKS/analysis_notes.py +152 -0
- MannKS/check_seasonality.py +147 -0
- MannKS/classification.py +54 -0
- MannKS/inspection.py +210 -0
- MannKS/plotting.py +388 -0
- MannKS/preprocessing.py +88 -0
- MannKS/regional_test.py +184 -0
- MannKS/seasonal_trend_test.py +517 -0
- MannKS/trend_test.py +475 -0
- mannks-0.1.0.dist-info/LICENSE +21 -0
- mannks-0.1.0.dist-info/METADATA +231 -0
- mannks-0.1.0.dist-info/RECORD +18 -0
- mannks-0.1.0.dist-info/WHEEL +4 -0
MannKS/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""
|
|
2
|
+
MannKS: A Python package for non-parametric trend analysis on unequally spaced time series data.
|
|
3
|
+
|
|
4
|
+
This package provides implementations of the Mann-Kendall test and Sen's slope
|
|
5
|
+
estimator, with additional support for seasonal analysis, seasonality testing,
|
|
6
|
+
and plotting utilities.
|
|
7
|
+
"""
|
|
8
|
+
from .trend_test import trend_test
|
|
9
|
+
from .seasonal_trend_test import seasonal_trend_test
|
|
10
|
+
from .check_seasonality import check_seasonality
|
|
11
|
+
from .plotting import plot_seasonal_distribution
|
|
12
|
+
from .inspection import inspect_trend_data
|
|
13
|
+
from .regional_test import regional_test
|
|
14
|
+
from .classification import classify_trend
|
|
15
|
+
from .preprocessing import prepare_censored_data
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
'trend_test',
|
|
19
|
+
'seasonal_trend_test',
|
|
20
|
+
'check_seasonality',
|
|
21
|
+
'plot_seasonal_distribution',
|
|
22
|
+
'inspect_trend_data',
|
|
23
|
+
'regional_test',
|
|
24
|
+
'classify_trend',
|
|
25
|
+
'prepare_censored_data'
|
|
26
|
+
]
|
MannKS/_ats.py
ADDED
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
# ats.py -- Akritas-Theil-Sen implementation (single-file, dependency-light)
|
|
2
|
+
import numpy as np
|
|
3
|
+
import pandas as pd
|
|
4
|
+
from math import inf
|
|
5
|
+
from typing import Tuple, List, Optional, Callable
|
|
6
|
+
from random import randint
|
|
7
|
+
|
|
8
|
+
# ---------- Utilities: interval representation ----------
|
|
9
|
+
def make_intervals(y: np.ndarray,
|
|
10
|
+
censored: np.ndarray,
|
|
11
|
+
cen_type: Optional[np.ndarray] = None,
|
|
12
|
+
lod: Optional[np.ndarray] = None) -> Tuple[np.ndarray, np.ndarray]:
|
|
13
|
+
"""
|
|
14
|
+
Build intervals [a_i, b_i] for each observation.
|
|
15
|
+
Arguments:
|
|
16
|
+
y: numeric face values (for censored rows this should be the numeric reporting value)
|
|
17
|
+
censored: boolean array (True when censored)
|
|
18
|
+
cen_type: None or array of 'lt' / 'gt' / 'none' strings (if None, assume all censored are 'lt')
|
|
19
|
+
lod: numeric detection limits associated with censored observations (len same as y)
|
|
20
|
+
Returns:
|
|
21
|
+
(lower, upper) arrays where lower[i] <= upper[i]; may include +-inf for left/right censor.
|
|
22
|
+
"""
|
|
23
|
+
n = len(y)
|
|
24
|
+
lower = np.empty(n, dtype=float)
|
|
25
|
+
upper = np.empty(n, dtype=float)
|
|
26
|
+
if cen_type is None:
|
|
27
|
+
cen_type = np.array(['lt' if c else 'none' for c in censored])
|
|
28
|
+
if lod is None:
|
|
29
|
+
lod = np.array([y_i for y_i in y])
|
|
30
|
+
|
|
31
|
+
for i in range(n):
|
|
32
|
+
if not censored[i]:
|
|
33
|
+
lower[i] = upper[i] = y[i]
|
|
34
|
+
else:
|
|
35
|
+
t = cen_type[i]
|
|
36
|
+
d = lod[i]
|
|
37
|
+
if t == 'lt' or t == '<' or t.lower() == 'left':
|
|
38
|
+
lower[i] = -inf
|
|
39
|
+
upper[i] = d
|
|
40
|
+
elif t == 'gt' or t == '>' or t.lower() == 'right':
|
|
41
|
+
lower[i] = d
|
|
42
|
+
upper[i] = +inf
|
|
43
|
+
else:
|
|
44
|
+
# default to left-censor
|
|
45
|
+
lower[i] = -inf
|
|
46
|
+
upper[i] = d
|
|
47
|
+
return lower, upper
|
|
48
|
+
|
|
49
|
+
# ---------- Pairwise interval comparison on residuals ----------
|
|
50
|
+
def S_of_beta(beta: float, x: np.ndarray, lower: np.ndarray, upper: np.ndarray) -> int:
|
|
51
|
+
"""
|
|
52
|
+
Compute S(beta) = (#concordant) - (#discordant) using residual intervals:
|
|
53
|
+
R_i(beta) = [lower_i - beta*x_i, upper_i - beta*x_i]
|
|
54
|
+
Definitive comparisons only when intervals do not overlap.
|
|
55
|
+
"""
|
|
56
|
+
n = len(x)
|
|
57
|
+
lower_r = lower - beta * x
|
|
58
|
+
upper_r = upper - beta * x
|
|
59
|
+
|
|
60
|
+
concordant = 0
|
|
61
|
+
discordant = 0
|
|
62
|
+
# O(n^2) loop
|
|
63
|
+
for i in range(n):
|
|
64
|
+
for j in range(i+1, n):
|
|
65
|
+
# check if r_i > r_j (lower_i > upper_j)
|
|
66
|
+
if lower_r[i] > upper_r[j]:
|
|
67
|
+
concordant += 1
|
|
68
|
+
elif upper_r[i] < lower_r[j]:
|
|
69
|
+
discordant += 1
|
|
70
|
+
else:
|
|
71
|
+
# tie/overlap -> do not count
|
|
72
|
+
pass
|
|
73
|
+
return concordant - discordant
|
|
74
|
+
|
|
75
|
+
# ---------- Root-finding to solve S(beta) = 0 ----------
|
|
76
|
+
def bracket_and_bisect_generic(score_func: Callable[[float], float],
|
|
77
|
+
slopes_hint: List[float],
|
|
78
|
+
max_expand=50, tol=1e-8,
|
|
79
|
+
maxiter=60) -> float:
|
|
80
|
+
"""
|
|
81
|
+
Find beta* such that score_func(beta*) = 0 using bisection.
|
|
82
|
+
Generic version that takes a score function and a list of slopes for initial bracket.
|
|
83
|
+
"""
|
|
84
|
+
# Define the initial search bracket.
|
|
85
|
+
if slopes_hint:
|
|
86
|
+
low = np.percentile(slopes_hint, 5)
|
|
87
|
+
high = np.percentile(slopes_hint, 95)
|
|
88
|
+
if np.isclose(low, high):
|
|
89
|
+
bracket_width = max(1.0, abs(low) * 0.5)
|
|
90
|
+
low -= bracket_width
|
|
91
|
+
high += bracket_width
|
|
92
|
+
else:
|
|
93
|
+
# Fallback if no slopes are available.
|
|
94
|
+
low, high = -1e-5, 1e-5
|
|
95
|
+
|
|
96
|
+
s_low = score_func(low)
|
|
97
|
+
s_high = score_func(high)
|
|
98
|
+
|
|
99
|
+
# Expand the bracket until the signs differ
|
|
100
|
+
expand_factor = 1.6
|
|
101
|
+
it = 0
|
|
102
|
+
while s_low * s_high > 0 and it < max_expand:
|
|
103
|
+
width = high - low
|
|
104
|
+
low -= width * expand_factor
|
|
105
|
+
high += width * expand_factor
|
|
106
|
+
s_low = score_func(low)
|
|
107
|
+
s_high = score_func(high)
|
|
108
|
+
it += 1
|
|
109
|
+
|
|
110
|
+
# Grid search fallback
|
|
111
|
+
if s_low * s_high > 0:
|
|
112
|
+
grid = np.linspace(low, high, num=201)
|
|
113
|
+
s_vals = np.array([abs(score_func(g)) for g in grid])
|
|
114
|
+
best_idx = np.argmin(s_vals)
|
|
115
|
+
return float(grid[best_idx])
|
|
116
|
+
|
|
117
|
+
# Bisection
|
|
118
|
+
a, b = low, high
|
|
119
|
+
sa, sb = s_low, s_high
|
|
120
|
+
for _ in range(maxiter):
|
|
121
|
+
m = (a + b) / 2.0
|
|
122
|
+
sm = score_func(m)
|
|
123
|
+
if sm == 0 or (b - a) / 2.0 < tol:
|
|
124
|
+
return float(m)
|
|
125
|
+
|
|
126
|
+
if sa * sm <= 0:
|
|
127
|
+
b, sb = m, sm
|
|
128
|
+
else:
|
|
129
|
+
a, sa = m, sm
|
|
130
|
+
|
|
131
|
+
return float((a + b) / 2.0)
|
|
132
|
+
|
|
133
|
+
def bracket_and_bisect(x: np.ndarray, lower: np.ndarray, upper: np.ndarray,
|
|
134
|
+
beta0: Optional[float]=None, max_expand=50, tol=1e-8,
|
|
135
|
+
maxiter=60) -> float:
|
|
136
|
+
"""
|
|
137
|
+
Backward compatible wrapper for the single-season case.
|
|
138
|
+
"""
|
|
139
|
+
# Normalize x to improve numerical stability in slope calculation
|
|
140
|
+
x_min = np.min(x)
|
|
141
|
+
x_range = np.ptp(x) if np.ptp(x) > 0 else 1.0
|
|
142
|
+
x_norm = (x - x_min) / x_range
|
|
143
|
+
|
|
144
|
+
# Calculate slopes from all uncensored pairs to define the initial search space (on normalized data)
|
|
145
|
+
detected_idx = np.where(np.isfinite(lower) & np.isfinite(upper) & (lower == upper))[0]
|
|
146
|
+
slopes = []
|
|
147
|
+
if len(detected_idx) >= 2:
|
|
148
|
+
for i in range(len(detected_idx)):
|
|
149
|
+
for j in range(i + 1, len(detected_idx)):
|
|
150
|
+
xi, xj = x_norm[detected_idx[i]], x_norm[detected_idx[j]]
|
|
151
|
+
if not np.isclose(xj, xi):
|
|
152
|
+
yi, yj = lower[detected_idx[i]], lower[detected_idx[j]]
|
|
153
|
+
slopes.append((yj - yi) / (xj - xi))
|
|
154
|
+
|
|
155
|
+
def score_func(b):
|
|
156
|
+
return S_of_beta(b, x_norm, lower, upper)
|
|
157
|
+
|
|
158
|
+
beta_hat_norm = bracket_and_bisect_generic(score_func, slopes, max_expand, tol, maxiter)
|
|
159
|
+
|
|
160
|
+
# De-normalize slope
|
|
161
|
+
return beta_hat_norm / x_range
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# ---------- Turnbull-style intercept (practical approach) ----------
|
|
165
|
+
def estimate_intercept_turnbull(residual_lower: np.ndarray, residual_upper: np.ndarray, tol=1e-6, max_iter=100) -> float:
|
|
166
|
+
"""
|
|
167
|
+
Estimates the median of interval-censored data using a Turnbull-style approach.
|
|
168
|
+
Correctly handles point masses (uncensored data) and infinite intervals (censored data).
|
|
169
|
+
"""
|
|
170
|
+
# 1. Identify Unique Endpoints
|
|
171
|
+
# We include all boundaries, including -inf and +inf if present.
|
|
172
|
+
endpoints = np.unique(np.concatenate([residual_lower, residual_upper]))
|
|
173
|
+
|
|
174
|
+
# 2. Construct Candidate Sets
|
|
175
|
+
# We partition the real line into disjoint sets based on endpoints:
|
|
176
|
+
# - Point sets [e_i, e_i] (for finite e_i)
|
|
177
|
+
# - Open intervals (e_i, e_{i+1})
|
|
178
|
+
candidate_sets = []
|
|
179
|
+
|
|
180
|
+
for i in range(len(endpoints)):
|
|
181
|
+
# Point set [e_i, e_i] - only for finite values
|
|
182
|
+
if np.isfinite(endpoints[i]):
|
|
183
|
+
candidate_sets.append((endpoints[i], endpoints[i]))
|
|
184
|
+
|
|
185
|
+
# Open interval (e_i, e_{i+1})
|
|
186
|
+
if i < len(endpoints) - 1:
|
|
187
|
+
candidate_sets.append((endpoints[i], endpoints[i+1]))
|
|
188
|
+
|
|
189
|
+
candidates = np.array(candidate_sets)
|
|
190
|
+
if len(candidates) == 0:
|
|
191
|
+
return 0.0
|
|
192
|
+
|
|
193
|
+
# 3. Initialize P
|
|
194
|
+
n_obs = len(residual_lower)
|
|
195
|
+
n_sets = len(candidates)
|
|
196
|
+
p = np.full(n_sets, 1.0 / n_sets)
|
|
197
|
+
|
|
198
|
+
# 4. Pre-calculate Containment Matrix
|
|
199
|
+
# A candidate set [c_L, c_R] is contained in observation [o_L, o_R] if:
|
|
200
|
+
# o_L <= c_L AND c_R <= o_R
|
|
201
|
+
|
|
202
|
+
L_col = residual_lower[:, np.newaxis]
|
|
203
|
+
R_col = residual_upper[:, np.newaxis]
|
|
204
|
+
C_start = candidates[:, 0]
|
|
205
|
+
C_end = candidates[:, 1]
|
|
206
|
+
|
|
207
|
+
is_contained = (C_start >= L_col) & (C_end <= R_col)
|
|
208
|
+
|
|
209
|
+
# Filter out candidates that are never contained in any observation to save computation
|
|
210
|
+
# (Though typically with endpoints derived from obs, most are relevant)
|
|
211
|
+
valid_candidates_idx = np.where(is_contained.any(axis=0))[0]
|
|
212
|
+
if len(valid_candidates_idx) == 0:
|
|
213
|
+
return 0.0 # Should not happen
|
|
214
|
+
|
|
215
|
+
p = p[valid_candidates_idx]
|
|
216
|
+
p /= np.sum(p) # Renormalize
|
|
217
|
+
is_contained = is_contained[:, valid_candidates_idx]
|
|
218
|
+
candidates = candidates[valid_candidates_idx]
|
|
219
|
+
n_sets = len(p)
|
|
220
|
+
|
|
221
|
+
# 5. EM Loop
|
|
222
|
+
for _ in range(max_iter):
|
|
223
|
+
p_old = p.copy()
|
|
224
|
+
|
|
225
|
+
# E-step
|
|
226
|
+
numer = p * is_contained
|
|
227
|
+
denoms = np.sum(numer, axis=1, keepdims=True)
|
|
228
|
+
denoms[denoms == 0] = 1.0 # Avoid division by zero
|
|
229
|
+
alpha = numer / denoms
|
|
230
|
+
|
|
231
|
+
# M-step
|
|
232
|
+
p = np.sum(alpha, axis=0) / n_obs
|
|
233
|
+
|
|
234
|
+
if np.sum(np.abs(p - p_old)) < tol:
|
|
235
|
+
break
|
|
236
|
+
|
|
237
|
+
# 6. Find Median
|
|
238
|
+
ecdf = np.cumsum(p)
|
|
239
|
+
# Find first index where cumulative probability >= 0.5
|
|
240
|
+
median_indices = np.where(ecdf >= 0.5)[0]
|
|
241
|
+
if len(median_indices) == 0:
|
|
242
|
+
median_idx = len(p) - 1
|
|
243
|
+
else:
|
|
244
|
+
median_idx = median_indices[0]
|
|
245
|
+
|
|
246
|
+
chosen = candidates[median_idx]
|
|
247
|
+
|
|
248
|
+
# Return representative value
|
|
249
|
+
if chosen[0] == chosen[1]:
|
|
250
|
+
return chosen[0] # Point mass
|
|
251
|
+
else:
|
|
252
|
+
# Interval (a, b)
|
|
253
|
+
a, b = chosen[0], chosen[1]
|
|
254
|
+
# Handle infinite bounds heuristically for plot/intercept purposes
|
|
255
|
+
if np.isneginf(a): return b # Left tail -> return upper bound
|
|
256
|
+
if np.isposinf(b): return a # Right tail -> return lower bound
|
|
257
|
+
return (a + b) / 2.0
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# ---------- Public wrapper (Non-Seasonal) ----------
|
|
261
|
+
def ats_slope(x: np.ndarray, y: np.ndarray, censored: np.ndarray,
|
|
262
|
+
cen_type: Optional[np.ndarray] = None, lod: Optional[np.ndarray] = None,
|
|
263
|
+
bootstrap_ci: bool = True, n_boot: int = 500,
|
|
264
|
+
ci_alpha: float = 0.05) -> dict:
|
|
265
|
+
"""
|
|
266
|
+
Compute ATS slope estimate and bootstrap CI.
|
|
267
|
+
Returns dict with keys: beta, intercept, ci_lower, ci_upper, prop_censored, notes
|
|
268
|
+
"""
|
|
269
|
+
lower, upper = make_intervals(y, censored, cen_type=cen_type, lod=lod)
|
|
270
|
+
|
|
271
|
+
# Calculate beta using the (now internally normalized) bisection wrapper
|
|
272
|
+
beta_hat = bracket_and_bisect(x, lower, upper, beta0=None)
|
|
273
|
+
|
|
274
|
+
# Calculate residuals and estimate intercept using Turnbull method
|
|
275
|
+
# r = y - beta * x. Use x - x_min for numerical stability in residual calculation
|
|
276
|
+
x_min = np.min(x)
|
|
277
|
+
x_shifted = x - x_min
|
|
278
|
+
|
|
279
|
+
r_lower = lower - beta_hat * x_shifted
|
|
280
|
+
r_upper = upper - beta_hat * x_shifted
|
|
281
|
+
intercept_shifted = estimate_intercept_turnbull(r_lower, r_upper)
|
|
282
|
+
|
|
283
|
+
# The intercept we want is for the model y = I + beta * x
|
|
284
|
+
# We found I_shifted such that y = I_shifted + beta * (x - x_min)
|
|
285
|
+
# y = (I_shifted - beta * x_min) + beta * x
|
|
286
|
+
intercept = intercept_shifted - beta_hat * x_min
|
|
287
|
+
|
|
288
|
+
prop_cen = float(np.mean(censored))
|
|
289
|
+
|
|
290
|
+
result = {'beta': beta_hat, 'intercept': intercept,
|
|
291
|
+
'prop_censored': prop_cen, 'notes': []}
|
|
292
|
+
|
|
293
|
+
# simple diagnostics: fraction of pairwise comparisons that were ties at final beta
|
|
294
|
+
n = len(x)
|
|
295
|
+
|
|
296
|
+
# Use normalized x for S_of_beta to be consistent with internal checks,
|
|
297
|
+
# but strictly S_of_beta is scale-invariant if beta is scaled.
|
|
298
|
+
# Here we just use original x and beta, as tie check is relative.
|
|
299
|
+
lower_r = lower - beta_hat * x
|
|
300
|
+
upper_r = upper - beta_hat * x
|
|
301
|
+
ties = 0
|
|
302
|
+
total_pairs = 0
|
|
303
|
+
for i in range(n):
|
|
304
|
+
for j in range(i+1, n):
|
|
305
|
+
total_pairs += 1
|
|
306
|
+
if (lower_r[i] <= upper_r[j]) and (upper_r[i] >= lower_r[j]):
|
|
307
|
+
# intervals overlap -> tie
|
|
308
|
+
ties += 1
|
|
309
|
+
result['pairwise_ties_frac'] = ties / total_pairs if total_pairs > 0 else np.nan
|
|
310
|
+
|
|
311
|
+
# Bootstrap CI using residual resampling
|
|
312
|
+
if bootstrap_ci and n >= 10:
|
|
313
|
+
boot_betas = []
|
|
314
|
+
rng = np.random.default_rng()
|
|
315
|
+
|
|
316
|
+
# Calculate fitted values and residuals for the WHOLE dataset
|
|
317
|
+
# fitted = intercept + beta * x
|
|
318
|
+
fitted = intercept_shifted + beta_hat * x_shifted
|
|
319
|
+
resid_lower = lower - fitted
|
|
320
|
+
resid_upper = upper - fitted
|
|
321
|
+
|
|
322
|
+
for b in range(n_boot):
|
|
323
|
+
try:
|
|
324
|
+
resid_idx = rng.integers(0, n, n)
|
|
325
|
+
y_boot_lower = fitted + resid_lower[resid_idx]
|
|
326
|
+
y_boot_upper = fitted + resid_upper[resid_idx]
|
|
327
|
+
|
|
328
|
+
# bracket_and_bisect handles normalization internally
|
|
329
|
+
beta_b = bracket_and_bisect(x, y_boot_lower, y_boot_upper)
|
|
330
|
+
|
|
331
|
+
if np.isfinite(beta_b) and abs(beta_b) < (abs(beta_hat) + 1) * 100:
|
|
332
|
+
boot_betas.append(beta_b)
|
|
333
|
+
except Exception:
|
|
334
|
+
pass
|
|
335
|
+
|
|
336
|
+
if len(boot_betas) >= max(20, int(0.1 * n_boot)):
|
|
337
|
+
lo = np.quantile(boot_betas, ci_alpha/2)
|
|
338
|
+
hi = np.quantile(boot_betas, 1 - ci_alpha/2)
|
|
339
|
+
result['ci_lower'] = float(lo)
|
|
340
|
+
result['ci_upper'] = float(hi)
|
|
341
|
+
result['bootstrap_samples'] = len(boot_betas)
|
|
342
|
+
else:
|
|
343
|
+
result['ci_lower'] = None
|
|
344
|
+
result['ci_upper'] = None
|
|
345
|
+
result['notes'].append(
|
|
346
|
+
f'bootstrap produced only {len(boot_betas)} valid samples '
|
|
347
|
+
f'(need >= {max(20, int(0.1 * n_boot))})'
|
|
348
|
+
)
|
|
349
|
+
return result
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
# ---------- Public wrapper (Seasonal) ----------
|
|
353
|
+
def seasonal_ats_slope(x: np.ndarray, y: np.ndarray, censored: np.ndarray, seasons: np.ndarray,
|
|
354
|
+
cen_type: Optional[np.ndarray] = None, lod: Optional[np.ndarray] = None,
|
|
355
|
+
bootstrap_ci: bool = True, n_boot: int = 500,
|
|
356
|
+
ci_alpha: float = 0.05) -> dict:
|
|
357
|
+
"""
|
|
358
|
+
Compute Stratified ATS slope estimate for seasonal data.
|
|
359
|
+
The overall slope is the root of the sum of the individual seasonal score functions.
|
|
360
|
+
Returns dict with keys: beta, intercept, ci_lower, ci_upper, prop_censored, notes
|
|
361
|
+
"""
|
|
362
|
+
x_min = np.min(x)
|
|
363
|
+
x_range = np.ptp(x) if np.ptp(x) > 0 else 1.0
|
|
364
|
+
x_norm = (x - x_min) / x_range
|
|
365
|
+
|
|
366
|
+
unique_seasons = np.unique(seasons)
|
|
367
|
+
|
|
368
|
+
# Pre-process data for each season:
|
|
369
|
+
# We store the normalized x, lower, upper arrays for each season.
|
|
370
|
+
season_data = {}
|
|
371
|
+
slopes_hint_norm = []
|
|
372
|
+
|
|
373
|
+
# Process full dataset to get interval bounds
|
|
374
|
+
full_lower, full_upper = make_intervals(y, censored, cen_type=cen_type, lod=lod)
|
|
375
|
+
|
|
376
|
+
for s in unique_seasons:
|
|
377
|
+
idx = np.where(seasons == s)[0]
|
|
378
|
+
if len(idx) < 2:
|
|
379
|
+
continue
|
|
380
|
+
|
|
381
|
+
x_s = x_norm[idx]
|
|
382
|
+
lower_s = full_lower[idx]
|
|
383
|
+
upper_s = full_upper[idx]
|
|
384
|
+
season_data[s] = (x_s, lower_s, upper_s)
|
|
385
|
+
|
|
386
|
+
# Collect slopes for initial bracket hint (using normalized x)
|
|
387
|
+
detected_idx = np.where(np.isfinite(lower_s) & np.isfinite(upper_s) & (lower_s == upper_s))[0]
|
|
388
|
+
if len(detected_idx) >= 2:
|
|
389
|
+
for i in range(len(detected_idx)):
|
|
390
|
+
for j in range(i + 1, len(detected_idx)):
|
|
391
|
+
xi, xj = x_s[detected_idx[i]], x_s[detected_idx[j]]
|
|
392
|
+
if not np.isclose(xj, xi):
|
|
393
|
+
yi, yj = lower_s[detected_idx[i]], lower_s[detected_idx[j]]
|
|
394
|
+
slopes_hint_norm.append((yj - yi) / (xj - xi))
|
|
395
|
+
|
|
396
|
+
# Define the global score function (on normalized beta)
|
|
397
|
+
def global_score_func(beta):
|
|
398
|
+
total_score = 0
|
|
399
|
+
for s in season_data:
|
|
400
|
+
x_s, lower_s, upper_s = season_data[s]
|
|
401
|
+
total_score += S_of_beta(beta, x_s, lower_s, upper_s)
|
|
402
|
+
return total_score
|
|
403
|
+
|
|
404
|
+
# Find the stratified ATS slope (normalized)
|
|
405
|
+
beta_hat_norm = bracket_and_bisect_generic(global_score_func, slopes_hint_norm)
|
|
406
|
+
|
|
407
|
+
# De-normalize beta
|
|
408
|
+
beta_hat = beta_hat_norm / x_range
|
|
409
|
+
|
|
410
|
+
# Estimate Intercept: Median of residuals across the ENTIRE dataset
|
|
411
|
+
# r = y - beta * x. Use x_shifted for stability.
|
|
412
|
+
x_shifted = x - x_min
|
|
413
|
+
r_lower = full_lower - beta_hat * x_shifted
|
|
414
|
+
r_upper = full_upper - beta_hat * x_shifted
|
|
415
|
+
intercept_shifted = estimate_intercept_turnbull(r_lower, r_upper)
|
|
416
|
+
intercept = intercept_shifted - beta_hat * x_min
|
|
417
|
+
|
|
418
|
+
prop_cen = float(np.mean(censored))
|
|
419
|
+
result = {'beta': beta_hat, 'intercept': intercept,
|
|
420
|
+
'prop_censored': prop_cen, 'notes': []}
|
|
421
|
+
|
|
422
|
+
# Stratified Bootstrap
|
|
423
|
+
n = len(x)
|
|
424
|
+
if bootstrap_ci and n >= 10:
|
|
425
|
+
boot_betas = []
|
|
426
|
+
rng = np.random.default_rng()
|
|
427
|
+
|
|
428
|
+
# Calculate fitted values and residuals for the WHOLE dataset
|
|
429
|
+
fitted = intercept_shifted + beta_hat * x_shifted
|
|
430
|
+
resid_lower = full_lower - fitted
|
|
431
|
+
resid_upper = full_upper - fitted
|
|
432
|
+
|
|
433
|
+
for b in range(n_boot):
|
|
434
|
+
try:
|
|
435
|
+
# Stratified resampling of residuals
|
|
436
|
+
boot_season_data = {}
|
|
437
|
+
|
|
438
|
+
for s in season_data:
|
|
439
|
+
x_s_norm, _, _ = season_data[s]
|
|
440
|
+
|
|
441
|
+
# Original indices for this season
|
|
442
|
+
idx_s = np.where(seasons == s)[0]
|
|
443
|
+
n_s = len(idx_s)
|
|
444
|
+
|
|
445
|
+
# Resample residuals within this season
|
|
446
|
+
resampled_indices = rng.choice(idx_s, size=n_s, replace=True)
|
|
447
|
+
|
|
448
|
+
# Construct bootstrapped intervals for this season
|
|
449
|
+
# Note: x_s_norm is already normalized
|
|
450
|
+
# fitted values must be grabbed corresponding to idx_s
|
|
451
|
+
fitted_s = fitted[idx_s]
|
|
452
|
+
resid_lower_s_resampled = resid_lower[resampled_indices]
|
|
453
|
+
resid_upper_s_resampled = resid_upper[resampled_indices]
|
|
454
|
+
|
|
455
|
+
y_boot_lower_s = fitted_s + resid_lower_s_resampled
|
|
456
|
+
y_boot_upper_s = fitted_s + resid_upper_s_resampled
|
|
457
|
+
|
|
458
|
+
boot_season_data[s] = (x_s_norm, y_boot_lower_s, y_boot_upper_s)
|
|
459
|
+
|
|
460
|
+
def boot_score_func(beta):
|
|
461
|
+
total_score = 0
|
|
462
|
+
for s in boot_season_data:
|
|
463
|
+
x_b, l_b, u_b = boot_season_data[s]
|
|
464
|
+
total_score += S_of_beta(beta, x_b, l_b, u_b)
|
|
465
|
+
return total_score
|
|
466
|
+
|
|
467
|
+
# Solve for beta_boot (normalized)
|
|
468
|
+
beta_b_norm = bracket_and_bisect_generic(boot_score_func, slopes_hint_norm)
|
|
469
|
+
|
|
470
|
+
# De-normalize
|
|
471
|
+
beta_b = beta_b_norm / x_range
|
|
472
|
+
|
|
473
|
+
if np.isfinite(beta_b) and abs(beta_b) < (abs(beta_hat) + 1) * 100:
|
|
474
|
+
boot_betas.append(beta_b)
|
|
475
|
+
|
|
476
|
+
except Exception:
|
|
477
|
+
pass
|
|
478
|
+
|
|
479
|
+
if len(boot_betas) >= max(20, int(0.1 * n_boot)):
|
|
480
|
+
lo = np.quantile(boot_betas, ci_alpha/2)
|
|
481
|
+
hi = np.quantile(boot_betas, 1 - ci_alpha/2)
|
|
482
|
+
result['ci_lower'] = float(lo)
|
|
483
|
+
result['ci_upper'] = float(hi)
|
|
484
|
+
result['bootstrap_samples'] = len(boot_betas)
|
|
485
|
+
else:
|
|
486
|
+
result['ci_lower'] = None
|
|
487
|
+
result['ci_upper'] = None
|
|
488
|
+
result['notes'].append(
|
|
489
|
+
f'bootstrap produced only {len(boot_betas)} valid samples '
|
|
490
|
+
f'(need >= {max(20, int(0.1 * n_boot))})'
|
|
491
|
+
)
|
|
492
|
+
|
|
493
|
+
return result
|
MannKS/_datetime.py
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import pandas as pd
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def _is_datetime_like(x):
|
|
6
|
+
"""Checks if an array is datetime-like."""
|
|
7
|
+
return np.issubdtype(x.dtype, np.datetime64) or \
|
|
8
|
+
(x.dtype == 'O' and len(x) > 0 and hasattr(x[0], 'year'))
|
|
9
|
+
|
|
10
|
+
def _get_season_func(season_type, period):
|
|
11
|
+
"""
|
|
12
|
+
Returns a function to extract seasonal data based on the season_type,
|
|
13
|
+
and validates the period.
|
|
14
|
+
"""
|
|
15
|
+
def get_dt_prop(dt, prop):
|
|
16
|
+
return getattr(dt.dt, prop) if isinstance(dt, pd.Series) else getattr(dt, prop)
|
|
17
|
+
|
|
18
|
+
season_map = {
|
|
19
|
+
'year': (1, lambda dt: get_dt_prop(dt, 'year')),
|
|
20
|
+
'month': (12, lambda dt: get_dt_prop(dt, 'month')),
|
|
21
|
+
'day_of_week': (7, lambda dt: get_dt_prop(dt, 'dayofweek')),
|
|
22
|
+
'quarter': (4, lambda dt: get_dt_prop(dt, 'quarter')),
|
|
23
|
+
'hour': (24, lambda dt: get_dt_prop(dt, 'hour')),
|
|
24
|
+
'week_of_year': ([52, 53], lambda dt: get_dt_prop(dt, 'isocalendar')().week),
|
|
25
|
+
'biweekly': ([26, 27], lambda dt: (get_dt_prop(dt, 'isocalendar')().week - 1) // 2),
|
|
26
|
+
'day_of_year': (None, lambda dt: get_dt_prop(dt, 'dayofyear')),
|
|
27
|
+
'minute': (60, lambda dt: get_dt_prop(dt, 'minute')),
|
|
28
|
+
'second': (60, lambda dt: get_dt_prop(dt, 'second')),
|
|
29
|
+
}
|
|
30
|
+
if season_type not in season_map:
|
|
31
|
+
raise ValueError(f"Unknown season_type: '{season_type}'. Must be one of {list(season_map.keys())}")
|
|
32
|
+
|
|
33
|
+
expected_period, season_func = season_map[season_type]
|
|
34
|
+
|
|
35
|
+
if expected_period is not None:
|
|
36
|
+
if isinstance(expected_period, list):
|
|
37
|
+
if period not in expected_period:
|
|
38
|
+
raise ValueError(f"For season_type='{season_type}', period must be one of {expected_period}.")
|
|
39
|
+
elif period != expected_period:
|
|
40
|
+
raise ValueError(f"For season_type='{season_type}', period must be {expected_period}.")
|
|
41
|
+
|
|
42
|
+
return season_func
|
|
43
|
+
|
|
44
|
+
def _get_cycle_identifier(dt_series, season_type):
|
|
45
|
+
"""
|
|
46
|
+
Returns a numeric series that uniquely identifies the larger time cycle
|
|
47
|
+
for each timestamp, used for aggregation.
|
|
48
|
+
"""
|
|
49
|
+
dt_accessor = dt_series.dt if isinstance(dt_series, pd.Series) else dt_series
|
|
50
|
+
|
|
51
|
+
if season_type in ['month', 'quarter', 'year', 'day_of_year', 'week_of_year', 'biweekly']:
|
|
52
|
+
# The cycle is the year
|
|
53
|
+
return dt_accessor.year.to_numpy()
|
|
54
|
+
|
|
55
|
+
elif season_type == 'day_of_week':
|
|
56
|
+
# The cycle is the week, identified by year and week number
|
|
57
|
+
iso_cal = dt_accessor.isocalendar()
|
|
58
|
+
return (iso_cal.year * 100 + iso_cal.week).to_numpy()
|
|
59
|
+
|
|
60
|
+
elif season_type in ['hour', 'minute', 'second']:
|
|
61
|
+
# The cycle is the day, identified by the Unix timestamp of the day's start
|
|
62
|
+
# Convert to int64 (nanoseconds) and then to float seconds
|
|
63
|
+
return (dt_accessor.normalize().astype(np.int64) / 10**9)
|
|
64
|
+
|
|
65
|
+
else:
|
|
66
|
+
# Default to year if the concept of a cycle is not obvious
|
|
67
|
+
return dt_accessor.year.to_numpy()
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _get_time_ranks(t_values, cycles):
|
|
71
|
+
"""Convert timestamps to cycle-based ranks matching R implementation."""
|
|
72
|
+
# Create unique cycle identifiers and sort them to ensure rank order
|
|
73
|
+
unique_cycles = np.unique(cycles)
|
|
74
|
+
ranks = np.zeros_like(t_values, dtype=float)
|
|
75
|
+
|
|
76
|
+
# Assign sequential ranks to each cycle
|
|
77
|
+
for i, cycle in enumerate(unique_cycles, start=1):
|
|
78
|
+
mask = cycles == cycle
|
|
79
|
+
ranks[mask] = i
|
|
80
|
+
|
|
81
|
+
return ranks
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _get_theoretical_midpoint(datetime_series):
|
|
85
|
+
"""
|
|
86
|
+
Calculates the theoretical midpoint of a time period for a series of datetimes.
|
|
87
|
+
This is used for the 'middle_lwp' aggregation method to replicate R's logic.
|
|
88
|
+
"""
|
|
89
|
+
if not isinstance(datetime_series, pd.Series):
|
|
90
|
+
datetime_series = pd.Series(datetime_series)
|
|
91
|
+
|
|
92
|
+
# All dates in the group should be in the same period (e.g., month, week)
|
|
93
|
+
first_date = datetime_series.iloc[0]
|
|
94
|
+
|
|
95
|
+
# Determine the period (month, week, etc.)
|
|
96
|
+
# This is a simplification; a more robust version would take period as an arg
|
|
97
|
+
if first_date.day == 1 and first_date.month != 2:
|
|
98
|
+
# Likely start of a month, find end of month
|
|
99
|
+
start_of_period = first_date.replace(day=1)
|
|
100
|
+
end_of_period = (start_of_period + pd.DateOffset(months=1)) - pd.Timedelta(nanoseconds=1)
|
|
101
|
+
else:
|
|
102
|
+
# Assume weekly or other, and just use the range of the data in the group
|
|
103
|
+
start_of_period = datetime_series.min()
|
|
104
|
+
end_of_period = datetime_series.max()
|
|
105
|
+
|
|
106
|
+
midpoint = start_of_period + (end_of_period - start_of_period) / 2
|
|
107
|
+
return midpoint
|
|
108
|
+
|
|
109
|
+
def _get_agg_func(agg_period: str):
|
|
110
|
+
"""
|
|
111
|
+
Returns a function to extract aggregation period identifiers from a
|
|
112
|
+
datetime series.
|
|
113
|
+
"""
|
|
114
|
+
def get_dt_prop(dt, prop):
|
|
115
|
+
return getattr(dt.dt, prop) if isinstance(dt, pd.Series) else getattr(dt, prop)
|
|
116
|
+
|
|
117
|
+
agg_map = {
|
|
118
|
+
'year': lambda dt: get_dt_prop(dt, 'year'),
|
|
119
|
+
'month': lambda dt: get_dt_prop(dt, 'year') * 100 + get_dt_prop(dt, 'month'),
|
|
120
|
+
'day': lambda dt: get_dt_prop(dt, 'date'),
|
|
121
|
+
'week': lambda dt: get_dt_prop(dt, 'isocalendar')().year * 100 + get_dt_prop(dt, 'isocalendar')().week,
|
|
122
|
+
'quarter': lambda dt: get_dt_prop(dt, 'year') * 10 + get_dt_prop(dt, 'quarter'),
|
|
123
|
+
}
|
|
124
|
+
agg_period_lower = agg_period.lower()
|
|
125
|
+
|
|
126
|
+
if agg_period_lower not in agg_map:
|
|
127
|
+
raise ValueError(f"Unknown agg_period: '{agg_period}'. Must be one of {list(agg_map.keys())}")
|
|
128
|
+
|
|
129
|
+
return agg_map[agg_period_lower]
|