gomd 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gomd/__init__.py +380 -0
- gomd/__main__.py +7 -0
- gomd/cli.py +76 -0
- gomd/client.py +1372 -0
- gomd/dynamics.py +629 -0
- gomd/maritime.py +952 -0
- gomd/ocean.py +1646 -0
- gomd/spatial.py +222 -0
- gomd/stats.py +1086 -0
- gomd-2.1.0.dist-info/METADATA +277 -0
- gomd-2.1.0.dist-info/RECORD +15 -0
- gomd-2.1.0.dist-info/WHEEL +5 -0
- gomd-2.1.0.dist-info/entry_points.txt +2 -0
- gomd-2.1.0.dist-info/licenses/LICENSE +21 -0
- gomd-2.1.0.dist-info/top_level.txt +1 -0
gomd/dynamics.py
ADDED
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
"""
|
|
2
|
+
gomd.dynamics — Port, Fisheries and Climate Dynamics Toolkit (new in gomd 2.1.0)
|
|
3
|
+
===============================================================================
|
|
4
|
+
Self-contained (pure Python, no NumPy) implementations of the analysis methods
|
|
5
|
+
used in the GOMD research papers on port activity, national marine catch and
|
|
6
|
+
coral heat stress (https://gomd.pirireis.edu.tr/publications.php):
|
|
7
|
+
|
|
8
|
+
1. symmetric_growth — symmetric (Davis-Haltiwanger-Schuh) growth rate
|
|
9
|
+
2. common_factor_decomposition — static principal-component global factor and variance shares
|
|
10
|
+
3. dispersion_index — variance-to-mean ratio of counts (Poisson benchmark = 1)
|
|
11
|
+
4. cargo_bunching — cargo per call, daily-cargo CV and its Poisson benchmark
|
|
12
|
+
5. weekday_profile — day-of-week profile and weekend ratio of a daily series
|
|
13
|
+
6. size_growth_scaling — Gibrat's-law test and size scaling of growth volatility
|
|
14
|
+
7. portfolio_effect — portfolio effect and synchrony of an aggregate
|
|
15
|
+
8. exceedance_return_interval — empirical return interval of threshold exceedances
|
|
16
|
+
9. fdr_benjamini_hochberg — Benjamini-Hochberg false-discovery-rate control
|
|
17
|
+
10. field_significance — binomial field-significance test for many local tests
|
|
18
|
+
11. local_projection_event — event-study local projections with unit and time fixed effects
|
|
19
|
+
|
|
20
|
+
Every function returns a plain ``dict`` and reports an ``"error"`` key instead of
|
|
21
|
+
raising when the input is too short. The functions describe data; they do not
|
|
22
|
+
establish causality on their own.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import datetime as _dt
|
|
28
|
+
import math
|
|
29
|
+
from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
|
|
30
|
+
|
|
31
|
+
Number = Union[int, float]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _is_num(v: Any) -> bool:
|
|
35
|
+
if v is None:
|
|
36
|
+
return False
|
|
37
|
+
try:
|
|
38
|
+
f = float(v)
|
|
39
|
+
except (TypeError, ValueError):
|
|
40
|
+
return False
|
|
41
|
+
return not (math.isnan(f) or math.isinf(f))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _clean(values: Sequence[Any]) -> List[float]:
|
|
45
|
+
return [float(v) for v in values if _is_num(v)]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _mean(x: Sequence[float]) -> float:
|
|
49
|
+
return sum(x) / len(x)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _var(x: Sequence[float], ddof: int = 1) -> float:
|
|
53
|
+
m = _mean(x)
|
|
54
|
+
return sum((v - m) ** 2 for v in x) / (len(x) - ddof)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _normal_sf(z: float) -> float:
|
|
58
|
+
return 0.5 * math.erfc(z / math.sqrt(2.0))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _ols_simple(x: Sequence[float], y: Sequence[float]) -> Dict[str, float]:
|
|
62
|
+
"""OLS of y on a constant and x with the conventional standard error."""
|
|
63
|
+
n = len(x)
|
|
64
|
+
mx, my = _mean(x), _mean(y)
|
|
65
|
+
sxx = sum((a - mx) ** 2 for a in x)
|
|
66
|
+
sxy = sum((a - mx) * (b - my) for a, b in zip(x, y))
|
|
67
|
+
slope = sxy / sxx
|
|
68
|
+
intercept = my - slope * mx
|
|
69
|
+
rss = sum((b - intercept - slope * a) ** 2 for a, b in zip(x, y))
|
|
70
|
+
tss = sum((b - my) ** 2 for b in y)
|
|
71
|
+
se = math.sqrt(rss / (n - 2) / sxx) if n > 2 and sxx > 0 else float("nan")
|
|
72
|
+
return {
|
|
73
|
+
"slope": slope,
|
|
74
|
+
"intercept": intercept,
|
|
75
|
+
"se": se,
|
|
76
|
+
"t": slope / se if se and se > 0 else float("nan"),
|
|
77
|
+
"r2": 1.0 - rss / tss if tss > 0 else float("nan"),
|
|
78
|
+
"n": n,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# --------------------------------------------------------------------------- 1
|
|
83
|
+
def symmetric_growth(values: Sequence[Any], lag: int = 12, percent: bool = True) -> Dict[str, Any]:
|
|
84
|
+
"""
|
|
85
|
+
Symmetric growth rate g_t = (x_t - x_{t-lag}) / ((x_t + x_{t-lag}) / 2)
|
|
86
|
+
(Davis, Haltiwanger & Schuh, 1996). It is bounded in [-2, 2] (or [-200, 200] %),
|
|
87
|
+
is defined when one of the two values is zero and treats rises and falls
|
|
88
|
+
symmetrically. With monthly data ``lag=12`` gives year-on-year growth.
|
|
89
|
+
|
|
90
|
+
Returns ``growth`` aligned with the input (``None`` for the first ``lag``
|
|
91
|
+
positions and where either value is missing or both are zero).
|
|
92
|
+
"""
|
|
93
|
+
if lag < 1:
|
|
94
|
+
return {"error": "lag must be at least 1."}
|
|
95
|
+
n = len(values)
|
|
96
|
+
out: List[Optional[float]] = [None] * n
|
|
97
|
+
k = 100.0 if percent else 1.0
|
|
98
|
+
for t in range(lag, n):
|
|
99
|
+
a, b = values[t], values[t - lag]
|
|
100
|
+
if _is_num(a) and _is_num(b):
|
|
101
|
+
den = (float(a) + float(b)) / 2.0
|
|
102
|
+
if den != 0:
|
|
103
|
+
out[t] = k * (float(a) - float(b)) / den
|
|
104
|
+
valid = [v for v in out if v is not None]
|
|
105
|
+
return {"growth": out, "n_valid": len(valid), "lag": lag, "unit": "%" if percent else "ratio",
|
|
106
|
+
"mean": _mean(valid) if valid else None}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# --------------------------------------------------------------------------- 2
|
|
110
|
+
def common_factor_decomposition(
|
|
111
|
+
panel: Mapping[str, Sequence[Any]],
|
|
112
|
+
max_iter: int = 500,
|
|
113
|
+
tol: float = 1e-12,
|
|
114
|
+
) -> Dict[str, Any]:
|
|
115
|
+
"""
|
|
116
|
+
Static principal-component estimate of one common ("global") factor in a panel
|
|
117
|
+
of series (Stock & Watson, 2002; Bai & Ng, 2002).
|
|
118
|
+
|
|
119
|
+
``panel`` maps a series name (for example a port) to observations on a common
|
|
120
|
+
time index (for example year-on-year growth rates). Series with a missing value
|
|
121
|
+
or zero variance are dropped. Each series is standardised; the factor is the
|
|
122
|
+
first principal component, scaled to unit variance and oriented to correlate
|
|
123
|
+
positively with the cross-sectional mean.
|
|
124
|
+
|
|
125
|
+
Returns the factor, each series' loading and the share of its variance explained
|
|
126
|
+
by the factor (``share_explained``, the squared correlation), the unweighted mean
|
|
127
|
+
and median of these shares, and the share of the variance of the cross-sectional
|
|
128
|
+
aggregate (the simple mean of the standardised series) explained by the factor.
|
|
129
|
+
The gap between the median series share and the aggregate share shows how much
|
|
130
|
+
of a "common" movement comes from averaging out series-specific shocks.
|
|
131
|
+
"""
|
|
132
|
+
names: List[str] = []
|
|
133
|
+
z: List[List[float]] = []
|
|
134
|
+
t_len: Optional[int] = None
|
|
135
|
+
dropped: List[str] = []
|
|
136
|
+
for name, vals in panel.items():
|
|
137
|
+
vals = list(vals)
|
|
138
|
+
if t_len is None:
|
|
139
|
+
t_len = len(vals)
|
|
140
|
+
if len(vals) != t_len or not all(_is_num(v) for v in vals):
|
|
141
|
+
dropped.append(str(name)); continue
|
|
142
|
+
x = [float(v) for v in vals]
|
|
143
|
+
m = _mean(x)
|
|
144
|
+
sd = math.sqrt(sum((v - m) ** 2 for v in x) / len(x))
|
|
145
|
+
if sd <= 0:
|
|
146
|
+
dropped.append(str(name)); continue
|
|
147
|
+
names.append(str(name)); z.append([(v - m) / sd for v in x])
|
|
148
|
+
n = len(names)
|
|
149
|
+
if t_len is None or n < 3 or t_len < 8:
|
|
150
|
+
return {"error": "At least 3 complete series with 8 or more periods are required.", "n_series": n, "dropped": dropped}
|
|
151
|
+
T = t_len
|
|
152
|
+
# Dual (T x T) Gram matrix: G = Z'Z / N with Z stored series-by-time.
|
|
153
|
+
g = [[0.0] * T for _ in range(T)]
|
|
154
|
+
for row in z:
|
|
155
|
+
for a in range(T):
|
|
156
|
+
ra = row[a]
|
|
157
|
+
if ra == 0.0:
|
|
158
|
+
continue
|
|
159
|
+
ga = g[a]
|
|
160
|
+
for b in range(a, T):
|
|
161
|
+
ga[b] += ra * row[b]
|
|
162
|
+
for a in range(T):
|
|
163
|
+
for b in range(a, T):
|
|
164
|
+
g[a][b] /= n
|
|
165
|
+
g[b][a] = g[a][b]
|
|
166
|
+
f = [sum(row[t] for row in z) / n for t in range(T)] # start from the cross-sectional mean
|
|
167
|
+
norm = math.sqrt(sum(v * v for v in f)) or 1.0
|
|
168
|
+
f = [v / norm for v in f]
|
|
169
|
+
eig = 0.0
|
|
170
|
+
converged = False
|
|
171
|
+
for _ in range(max_iter):
|
|
172
|
+
nf = [sum(g[a][b] * f[b] for b in range(T)) for a in range(T)]
|
|
173
|
+
new_eig = math.sqrt(sum(v * v for v in nf))
|
|
174
|
+
if new_eig == 0:
|
|
175
|
+
break
|
|
176
|
+
nf = [v / new_eig for v in nf]
|
|
177
|
+
delta = max(abs(a - b) for a, b in zip(nf, f))
|
|
178
|
+
f, eig = nf, new_eig
|
|
179
|
+
if delta < tol:
|
|
180
|
+
converged = True
|
|
181
|
+
break
|
|
182
|
+
# unit-variance factor (series are mean zero, so the factor is mean zero as well)
|
|
183
|
+
fm = _mean(f)
|
|
184
|
+
fsd = math.sqrt(sum((v - fm) ** 2 for v in f) / T)
|
|
185
|
+
factor = [(v - fm) / fsd for v in f]
|
|
186
|
+
agg = [sum(row[t] for row in z) / n for t in range(T)]
|
|
187
|
+
if sum(a * b for a, b in zip(factor, agg)) < 0:
|
|
188
|
+
factor = [-v for v in factor]
|
|
189
|
+
loadings = {nm: sum(a * b for a, b in zip(row, factor)) / T for nm, row in zip(names, z)}
|
|
190
|
+
shares = {nm: l * l for nm, l in loadings.items()}
|
|
191
|
+
sv = sorted(shares.values())
|
|
192
|
+
med = sv[n // 2] if n % 2 else 0.5 * (sv[n // 2 - 1] + sv[n // 2])
|
|
193
|
+
am = _mean(agg)
|
|
194
|
+
avar = sum((v - am) ** 2 for v in agg) / T
|
|
195
|
+
acorr = (sum((a - am) * b for a, b in zip(agg, factor)) / T) / math.sqrt(avar) if avar > 0 else float("nan")
|
|
196
|
+
return {
|
|
197
|
+
"factor": factor,
|
|
198
|
+
"loadings": loadings,
|
|
199
|
+
"share_explained": shares,
|
|
200
|
+
"mean_share_explained": _mean(sv),
|
|
201
|
+
"median_share_explained": med,
|
|
202
|
+
"aggregate_share_explained": acorr * acorr,
|
|
203
|
+
"eigenvalue_share": eig / T,
|
|
204
|
+
"n_series": n,
|
|
205
|
+
"n_periods": T,
|
|
206
|
+
"dropped": dropped,
|
|
207
|
+
"converged": converged,
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
# --------------------------------------------------------------------------- 3
|
|
212
|
+
def dispersion_index(counts: Sequence[Any]) -> Dict[str, Any]:
|
|
213
|
+
"""
|
|
214
|
+
Index of dispersion (variance-to-mean ratio) of a series of counts, for example
|
|
215
|
+
daily port calls. It equals 1 for Poisson arrivals; values below 1 indicate more
|
|
216
|
+
regular (scheduled) arrivals and values above 1 indicate clustered arrivals.
|
|
217
|
+
|
|
218
|
+
Also returns the dispersion test statistic (n - 1) * index, which is approximately
|
|
219
|
+
chi-square with n - 1 degrees of freedom under the Poisson hypothesis, and its
|
|
220
|
+
normal (Wilson-Hilferty) approximation z-score and two-sided p-value.
|
|
221
|
+
"""
|
|
222
|
+
x = _clean(counts)
|
|
223
|
+
n = len(x)
|
|
224
|
+
if n < 10:
|
|
225
|
+
return {"error": "At least 10 observations are required.", "n": n}
|
|
226
|
+
m = _mean(x)
|
|
227
|
+
if m <= 0:
|
|
228
|
+
return {"error": "The mean of the counts must be positive.", "n": n}
|
|
229
|
+
idx = _var(x) / m
|
|
230
|
+
df = n - 1
|
|
231
|
+
chi2 = df * idx
|
|
232
|
+
z = ((chi2 / df) ** (1.0 / 3.0) - (1.0 - 2.0 / (9.0 * df))) / math.sqrt(2.0 / (9.0 * df))
|
|
233
|
+
p = 2.0 * _normal_sf(abs(z))
|
|
234
|
+
return {"dispersion_index": idx, "mean": m, "variance": _var(x), "n": n, "chi2": chi2, "df": df, "z": z, "p_value": p,
|
|
235
|
+
"pattern": "regular" if idx < 1 else "clustered" if idx > 1 else "poisson"}
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
# --------------------------------------------------------------------------- 4
|
|
239
|
+
def cargo_bunching(daily_calls: Sequence[Any], daily_cargo: Sequence[Any]) -> Dict[str, Any]:
|
|
240
|
+
"""
|
|
241
|
+
Cargo bunching at a port from daily calls and daily cargo volume (same days).
|
|
242
|
+
|
|
243
|
+
Returns cargo per call (total cargo / total calls), the coefficient of variation
|
|
244
|
+
(CV) of daily cargo, the dispersion index of daily calls and the compound-Poisson
|
|
245
|
+
benchmark. If ships arrive as a Poisson process with rate lambda per day and
|
|
246
|
+
carry independent loads with coefficient of variation c, daily cargo has
|
|
247
|
+
CV^2 = (1 + c^2) / lambda. ``cv_benchmark_equal_loads`` is the lower bound
|
|
248
|
+
1 / sqrt(lambda) (all ships the same size), and ``implied_load_cv`` is the load
|
|
249
|
+
variability c that would reconcile the observed CV with Poisson arrivals
|
|
250
|
+
(``None`` when the observed CV is below the lower bound, i.e. arrivals are more
|
|
251
|
+
regular than Poisson). Holding cargo constant, fewer and larger ships lower
|
|
252
|
+
lambda and raise the benchmark CV.
|
|
253
|
+
"""
|
|
254
|
+
pairs = [(float(a), float(b)) for a, b in zip(daily_calls, daily_cargo) if _is_num(a) and _is_num(b)]
|
|
255
|
+
n = len(pairs)
|
|
256
|
+
if n < 30:
|
|
257
|
+
return {"error": "At least 30 days with both calls and cargo are required.", "n": n}
|
|
258
|
+
calls = [p[0] for p in pairs]
|
|
259
|
+
cargo = [p[1] for p in pairs]
|
|
260
|
+
tc, tq = sum(calls), sum(cargo)
|
|
261
|
+
if tc <= 0 or tq <= 0:
|
|
262
|
+
return {"error": "Total calls and total cargo must be positive.", "n": n}
|
|
263
|
+
lam = tc / n
|
|
264
|
+
mq = tq / n
|
|
265
|
+
cv = math.sqrt(_var(cargo)) / mq
|
|
266
|
+
bench = 1.0 / math.sqrt(lam)
|
|
267
|
+
implied = math.sqrt(cv * cv * lam - 1.0) if cv * cv * lam >= 1.0 else None
|
|
268
|
+
return {
|
|
269
|
+
"n_days": n,
|
|
270
|
+
"total_calls": tc,
|
|
271
|
+
"total_cargo": tq,
|
|
272
|
+
"cargo_per_call": tq / tc,
|
|
273
|
+
"calls_per_day": lam,
|
|
274
|
+
"daily_cargo_cv": cv,
|
|
275
|
+
"calls_dispersion_index": _var(calls) / lam,
|
|
276
|
+
"cv_benchmark_equal_loads": bench,
|
|
277
|
+
"cv_ratio_to_benchmark": cv / bench,
|
|
278
|
+
"implied_load_cv": implied,
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
# --------------------------------------------------------------------------- 5
|
|
283
|
+
def weekday_profile(dates: Sequence[Any], values: Sequence[Any]) -> Dict[str, Any]:
|
|
284
|
+
"""
|
|
285
|
+
Day-of-week profile of a daily series. ``dates`` are ISO strings (YYYY-MM-DD) or
|
|
286
|
+
``datetime.date`` objects. Returns the mean by weekday (Monday first), each
|
|
287
|
+
weekday's index relative to the overall daily mean (1 = average day), the
|
|
288
|
+
Saturday-Sunday mean relative to the Monday-Friday mean and the same ratio for
|
|
289
|
+
Friday-Saturday (the weekend in several countries), and the share of the variance
|
|
290
|
+
of the series explained by weekday means.
|
|
291
|
+
"""
|
|
292
|
+
sums = [0.0] * 7
|
|
293
|
+
cnts = [0] * 7
|
|
294
|
+
xs: List[Tuple[int, float]] = []
|
|
295
|
+
for d, v in zip(dates, values):
|
|
296
|
+
if not _is_num(v):
|
|
297
|
+
continue
|
|
298
|
+
if isinstance(d, _dt.datetime):
|
|
299
|
+
d = d.date()
|
|
300
|
+
if not isinstance(d, _dt.date):
|
|
301
|
+
try:
|
|
302
|
+
d = _dt.date.fromisoformat(str(d)[:10])
|
|
303
|
+
except ValueError:
|
|
304
|
+
continue
|
|
305
|
+
w = d.weekday()
|
|
306
|
+
sums[w] += float(v); cnts[w] += 1; xs.append((w, float(v)))
|
|
307
|
+
n = len(xs)
|
|
308
|
+
if n < 28 or min(cnts) < 2:
|
|
309
|
+
return {"error": "At least 28 dated observations covering every weekday are required.", "n": n}
|
|
310
|
+
means = [s / c for s, c in zip(sums, cnts)]
|
|
311
|
+
overall = sum(v for _, v in xs) / n
|
|
312
|
+
if overall == 0:
|
|
313
|
+
return {"error": "The mean of the series is zero.", "n": n}
|
|
314
|
+
tss = sum((v - overall) ** 2 for _, v in xs)
|
|
315
|
+
bss = sum(c * (m - overall) ** 2 for c, m in zip(cnts, means))
|
|
316
|
+
names = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]
|
|
317
|
+
|
|
318
|
+
def ratio(weekend: Sequence[int]) -> float:
|
|
319
|
+
we = sum(sums[i] for i in weekend) / sum(cnts[i] for i in weekend)
|
|
320
|
+
rest = [i for i in range(7) if i not in weekend]
|
|
321
|
+
wd = sum(sums[i] for i in rest) / sum(cnts[i] for i in rest)
|
|
322
|
+
return we / wd if wd != 0 else float("nan")
|
|
323
|
+
|
|
324
|
+
return {
|
|
325
|
+
"weekday_mean": dict(zip(names, means)),
|
|
326
|
+
"weekday_index": {k: m / overall for k, m in zip(names, means)},
|
|
327
|
+
"sat_sun_ratio": ratio([5, 6]),
|
|
328
|
+
"fri_sat_ratio": ratio([4, 5]),
|
|
329
|
+
"variance_share_weekday": bss / tss if tss > 0 else float("nan"),
|
|
330
|
+
"n": n,
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
# --------------------------------------------------------------------------- 6
|
|
335
|
+
def size_growth_scaling(size: Sequence[Any], growth: Sequence[Any], n_bins: int = 10) -> Dict[str, Any]:
|
|
336
|
+
"""
|
|
337
|
+
Gibrat's-law diagnostics for units such as ports or firms.
|
|
338
|
+
|
|
339
|
+
``size`` is the initial size (positive) and ``growth`` the subsequent growth rate
|
|
340
|
+
(for example log growth) of the same unit. Two regressions are reported:
|
|
341
|
+
* mean test: growth on log size. Gibrat's law of proportionate growth implies
|
|
342
|
+
a zero slope.
|
|
343
|
+
* volatility scaling: units are sorted by size into ``n_bins`` equal-count bins;
|
|
344
|
+
the log standard deviation of growth in each bin is regressed on the mean log
|
|
345
|
+
size of the bin. The slope is -beta in sigma(S) ~ S^(-beta). Gibrat's law
|
|
346
|
+
implies beta = 0; independent sub-units would give beta = 0.5; values between
|
|
347
|
+
0.1 and 0.3 are typical of firms and countries (Stanley et al., 1996).
|
|
348
|
+
"""
|
|
349
|
+
pairs = [(math.log(float(s)), float(g)) for s, g in zip(size, growth) if _is_num(s) and _is_num(g) and float(s) > 0]
|
|
350
|
+
n = len(pairs)
|
|
351
|
+
if n_bins < 3:
|
|
352
|
+
return {"error": "n_bins must be at least 3."}
|
|
353
|
+
if n < max(30, 5 * n_bins):
|
|
354
|
+
return {"error": "At least max(30, 5 * n_bins) units with positive size are required.", "n": n}
|
|
355
|
+
pairs.sort(key=lambda p: p[0])
|
|
356
|
+
ls = [p[0] for p in pairs]
|
|
357
|
+
gr = [p[1] for p in pairs]
|
|
358
|
+
mean_fit = _ols_simple(ls, gr)
|
|
359
|
+
bins = []
|
|
360
|
+
for b in range(n_bins):
|
|
361
|
+
lo = b * n // n_bins
|
|
362
|
+
hi = (b + 1) * n // n_bins
|
|
363
|
+
gs = gr[lo:hi]
|
|
364
|
+
sd = math.sqrt(_var(gs))
|
|
365
|
+
bins.append({"mean_log_size": _mean(ls[lo:hi]), "sd_growth": sd, "mean_growth": _mean(gs), "n": hi - lo})
|
|
366
|
+
good = [b for b in bins if b["sd_growth"] > 0]
|
|
367
|
+
if len(good) < 3:
|
|
368
|
+
return {"error": "Growth has no dispersion within the size bins.", "n": n}
|
|
369
|
+
vol_fit = _ols_simple([b["mean_log_size"] for b in good], [math.log(b["sd_growth"]) for b in good])
|
|
370
|
+
return {
|
|
371
|
+
"n": n,
|
|
372
|
+
"mean_slope": mean_fit["slope"], "mean_slope_se": mean_fit["se"], "mean_slope_t": mean_fit["t"],
|
|
373
|
+
"volatility_beta": -vol_fit["slope"], "volatility_beta_se": vol_fit["se"], "volatility_r2": vol_fit["r2"],
|
|
374
|
+
"bins": bins,
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
# --------------------------------------------------------------------------- 7
|
|
379
|
+
def portfolio_effect(panel: Mapping[str, Sequence[Any]]) -> Dict[str, Any]:
|
|
380
|
+
"""
|
|
381
|
+
Portfolio effect of an aggregate made of several components (for example the
|
|
382
|
+
national catch as the sum of species or stock catches; Doak et al., 1998;
|
|
383
|
+
Schindler et al., 2010; Thibaut & Connolly, 2013).
|
|
384
|
+
|
|
385
|
+
``panel`` maps a component name to its values on a common time index (levels,
|
|
386
|
+
not growth rates). Periods in which any component is missing are dropped.
|
|
387
|
+
|
|
388
|
+
Returns the CV of the aggregate, the weighted-average CV of the components
|
|
389
|
+
(weights = mean share of the aggregate), the portfolio effect
|
|
390
|
+
1 - CV_aggregate / CV_weighted (0 = no stabilisation), the synchrony index of
|
|
391
|
+
Loreau & de Mazancourt (2008), phi = var(sum) / (sum of sd)^2 (1 = perfectly
|
|
392
|
+
synchronous, 1/n = independent equal components), and the effective number of
|
|
393
|
+
components 1 / sum(share^2).
|
|
394
|
+
"""
|
|
395
|
+
names = [str(k) for k in panel.keys()]
|
|
396
|
+
series = [list(v) for v in panel.values()]
|
|
397
|
+
if len(series) < 2:
|
|
398
|
+
return {"error": "At least two components are required."}
|
|
399
|
+
T = min(len(s) for s in series)
|
|
400
|
+
rows = [t for t in range(T) if all(_is_num(s[t]) for s in series)]
|
|
401
|
+
if len(rows) < 8:
|
|
402
|
+
return {"error": "At least 8 periods with all components observed are required.", "n_periods": len(rows)}
|
|
403
|
+
x = [[float(s[t]) for t in rows] for s in series]
|
|
404
|
+
total = [sum(col[i] for col in x) for i in range(len(rows))]
|
|
405
|
+
mt = _mean(total)
|
|
406
|
+
if mt <= 0:
|
|
407
|
+
return {"error": "The aggregate must have a positive mean."}
|
|
408
|
+
means = [_mean(c) for c in x]
|
|
409
|
+
sds = [math.sqrt(_var(c)) for c in x]
|
|
410
|
+
keep = [i for i, m in enumerate(means) if m > 0]
|
|
411
|
+
shares = [means[i] / mt for i in keep]
|
|
412
|
+
cv_w = sum(sh * (sds[i] / means[i]) for sh, i in zip(shares, keep))
|
|
413
|
+
cv_tot = math.sqrt(_var(total)) / mt
|
|
414
|
+
sum_sd = sum(sds)
|
|
415
|
+
return {
|
|
416
|
+
"cv_aggregate": cv_tot,
|
|
417
|
+
"cv_weighted_components": cv_w,
|
|
418
|
+
"portfolio_effect": 1.0 - cv_tot / cv_w if cv_w > 0 else float("nan"),
|
|
419
|
+
"synchrony": _var(total) / (sum_sd * sum_sd) if sum_sd > 0 else float("nan"),
|
|
420
|
+
"effective_components": 1.0 / sum(s * s for s in shares) if shares else float("nan"),
|
|
421
|
+
"n_components": len(names),
|
|
422
|
+
"n_periods": len(rows),
|
|
423
|
+
"component_cv": {names[i]: sds[i] / means[i] for i in keep},
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
# --------------------------------------------------------------------------- 8
|
|
428
|
+
def exceedance_return_interval(
|
|
429
|
+
years: Sequence[Any],
|
|
430
|
+
values: Sequence[Any],
|
|
431
|
+
threshold: float,
|
|
432
|
+
split_year: Optional[Number] = None,
|
|
433
|
+
) -> Dict[str, Any]:
|
|
434
|
+
"""
|
|
435
|
+
Empirical return interval of years in which an annual series reaches a threshold,
|
|
436
|
+
for example annual maximum degree heating weeks (DHW) >= 4 or >= 8 degC-weeks.
|
|
437
|
+
|
|
438
|
+
The return interval is (number of years) / (number of exceedance years). With
|
|
439
|
+
``split_year`` the record is divided into years before ``split_year`` and years
|
|
440
|
+
from ``split_year`` onwards, and the two exceedance probabilities are compared
|
|
441
|
+
with a two-proportion z-test. Also returns the gaps between successive
|
|
442
|
+
exceedance years and the longest run of consecutive exceedance years.
|
|
443
|
+
"""
|
|
444
|
+
pairs = sorted((float(y), float(v)) for y, v in zip(years, values) if _is_num(y) and _is_num(v))
|
|
445
|
+
n = len(pairs)
|
|
446
|
+
if n < 10:
|
|
447
|
+
return {"error": "At least 10 annual values are required.", "n": n}
|
|
448
|
+
hit_years = [y for y, v in pairs if v >= threshold]
|
|
449
|
+
k = len(hit_years)
|
|
450
|
+
gaps = [b - a for a, b in zip(hit_years[:-1], hit_years[1:])]
|
|
451
|
+
run = best = 0
|
|
452
|
+
for _, v in pairs:
|
|
453
|
+
run = run + 1 if v >= threshold else 0
|
|
454
|
+
best = max(best, run)
|
|
455
|
+
out: Dict[str, Any] = {
|
|
456
|
+
"n_years": n, "n_exceedance_years": k, "threshold": threshold,
|
|
457
|
+
"exceedance_probability": k / n,
|
|
458
|
+
"return_interval_years": n / k if k else None,
|
|
459
|
+
"exceedance_years": hit_years,
|
|
460
|
+
"mean_gap_years": _mean(gaps) if gaps else None,
|
|
461
|
+
"longest_run_years": best,
|
|
462
|
+
}
|
|
463
|
+
if split_year is not None:
|
|
464
|
+
a = [v for y, v in pairs if y < float(split_year)]
|
|
465
|
+
b = [v for y, v in pairs if y >= float(split_year)]
|
|
466
|
+
if len(a) >= 5 and len(b) >= 5:
|
|
467
|
+
ka = sum(1 for v in a if v >= threshold)
|
|
468
|
+
kb = sum(1 for v in b if v >= threshold)
|
|
469
|
+
pa, pb = ka / len(a), kb / len(b)
|
|
470
|
+
pp = (ka + kb) / (len(a) + len(b))
|
|
471
|
+
se = math.sqrt(pp * (1 - pp) * (1 / len(a) + 1 / len(b)))
|
|
472
|
+
z = (pb - pa) / se if se > 0 else float("nan")
|
|
473
|
+
out["split"] = {
|
|
474
|
+
"split_year": split_year, "n_before": len(a), "n_after": len(b),
|
|
475
|
+
"probability_before": pa, "probability_after": pb,
|
|
476
|
+
"return_interval_before": len(a) / ka if ka else None,
|
|
477
|
+
"return_interval_after": len(b) / kb if kb else None,
|
|
478
|
+
"z": z, "p_value": 2.0 * _normal_sf(abs(z)) if not math.isnan(z) else None,
|
|
479
|
+
}
|
|
480
|
+
else:
|
|
481
|
+
out["split"] = {"error": "Each sub-period needs at least 5 years."}
|
|
482
|
+
return out
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
# --------------------------------------------------------------------------- 9
|
|
486
|
+
def fdr_benjamini_hochberg(pvalues: Sequence[Any], q: float = 0.05) -> Dict[str, Any]:
|
|
487
|
+
"""
|
|
488
|
+
Benjamini & Hochberg (1995) control of the false discovery rate. Returns the
|
|
489
|
+
adjusted p-values (q-values) in the input order, a rejection flag for each test
|
|
490
|
+
at level ``q`` and the number of discoveries. Missing p-values stay ``None``.
|
|
491
|
+
"""
|
|
492
|
+
idx = [i for i, p in enumerate(pvalues) if _is_num(p)]
|
|
493
|
+
m = len(idx)
|
|
494
|
+
if m == 0:
|
|
495
|
+
return {"error": "No valid p-values."}
|
|
496
|
+
order = sorted(idx, key=lambda i: float(pvalues[i]))
|
|
497
|
+
adj: List[Optional[float]] = [None] * len(pvalues)
|
|
498
|
+
prev = 1.0
|
|
499
|
+
for rank in range(m, 0, -1):
|
|
500
|
+
i = order[rank - 1]
|
|
501
|
+
val = min(prev, float(pvalues[i]) * m / rank)
|
|
502
|
+
adj[i] = val
|
|
503
|
+
prev = val
|
|
504
|
+
rejected = [(a is not None and a <= q) for a in adj]
|
|
505
|
+
return {"adjusted_p": adj, "rejected": rejected, "n_tests": m, "n_discoveries": sum(rejected), "q": q,
|
|
506
|
+
"n_nominal_below_q": sum(1 for i in idx if float(pvalues[i]) <= q)}
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
# -------------------------------------------------------------------------- 10
|
|
510
|
+
def field_significance(pvalues: Sequence[Any], alpha: float = 0.05) -> Dict[str, Any]:
|
|
511
|
+
"""
|
|
512
|
+
Field significance of a collection of local tests (Livezey & Chen, 1983): is the
|
|
513
|
+
number of locally significant tests larger than expected by chance?
|
|
514
|
+
|
|
515
|
+
Under the global null and *independent* tests the number of p-values <= alpha is
|
|
516
|
+
Binomial(m, alpha); the exact upper-tail probability is returned. Tests on
|
|
517
|
+
spatially or temporally correlated series are not independent, so this p-value
|
|
518
|
+
is too small for them; in that case use a permutation or Monte Carlo reference
|
|
519
|
+
distribution and treat this result as a lower bound.
|
|
520
|
+
"""
|
|
521
|
+
p = _clean(pvalues)
|
|
522
|
+
m = len(p)
|
|
523
|
+
if m == 0:
|
|
524
|
+
return {"error": "No valid p-values."}
|
|
525
|
+
if not 0 < alpha < 1:
|
|
526
|
+
return {"error": "alpha must be between 0 and 1."}
|
|
527
|
+
k = sum(1 for v in p if v <= alpha)
|
|
528
|
+
la, lb = math.log(alpha), math.log(1.0 - alpha)
|
|
529
|
+
tail = 0.0
|
|
530
|
+
for j in range(k, m + 1):
|
|
531
|
+
tail += math.exp(math.lgamma(m + 1) - math.lgamma(j + 1) - math.lgamma(m - j + 1) + j * la + (m - j) * lb)
|
|
532
|
+
return {"n_tests": m, "n_significant": k, "expected_by_chance": m * alpha, "share_significant": k / m,
|
|
533
|
+
"p_value_independent": min(1.0, tail), "alpha": alpha}
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
# -------------------------------------------------------------------------- 11
|
|
537
|
+
def _two_way_demean(vals: List[float], unit: List[int], time: List[int], n_u: int, n_t: int,
|
|
538
|
+
max_iter: int = 2000, tol: float = 1e-10) -> List[float]:
|
|
539
|
+
x = list(vals)
|
|
540
|
+
cu = [0] * n_u
|
|
541
|
+
ct = [0] * n_t
|
|
542
|
+
for u, t in zip(unit, time):
|
|
543
|
+
cu[u] += 1; ct[t] += 1
|
|
544
|
+
for _ in range(max_iter):
|
|
545
|
+
su = [0.0] * n_u
|
|
546
|
+
for v, u in zip(x, unit):
|
|
547
|
+
su[u] += v
|
|
548
|
+
mu = [s / c if c else 0.0 for s, c in zip(su, cu)]
|
|
549
|
+
x = [v - mu[u] for v, u in zip(x, unit)]
|
|
550
|
+
st = [0.0] * n_t
|
|
551
|
+
for v, t in zip(x, time):
|
|
552
|
+
st[t] += v
|
|
553
|
+
mt = [s / c if c else 0.0 for s, c in zip(st, ct)]
|
|
554
|
+
x = [v - mt[t] for v, t in zip(x, time)]
|
|
555
|
+
if max(max(abs(m) for m in mu), max(abs(m) for m in mt)) < tol:
|
|
556
|
+
break
|
|
557
|
+
return x
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def local_projection_event(
|
|
561
|
+
outcome: Mapping[str, Sequence[Any]],
|
|
562
|
+
event: Mapping[str, Sequence[Any]],
|
|
563
|
+
horizons: Sequence[int] = (0, 1, 2, 3, 4, 5),
|
|
564
|
+
leads: Sequence[int] = (),
|
|
565
|
+
) -> Dict[str, Any]:
|
|
566
|
+
"""
|
|
567
|
+
Event-study local projections (Jordà, 2005) in a panel with unit and time fixed
|
|
568
|
+
effects, for example the response of log reef-fish catch to a mass-bleaching year.
|
|
569
|
+
|
|
570
|
+
``outcome`` and ``event`` map a unit (for example a country) to values on a common
|
|
571
|
+
time index; ``event`` is 0/1 (or an intensity). For each horizon h the regression is
|
|
572
|
+
y[i, t+h] - y[i, t-1] = beta_h * event[i, t] + unit effect + time effect + error,
|
|
573
|
+
estimated by ordinary least squares after two-way demeaning. Standard errors are
|
|
574
|
+
clustered by unit (with the G / (G - 1) correction). ``leads`` (for example
|
|
575
|
+
``(2, 3)``) adds placebo regressions of y[i, t-1-k] - y[i, t-1] on the event, which
|
|
576
|
+
should be close to zero if units with and without an event followed similar paths
|
|
577
|
+
beforehand.
|
|
578
|
+
|
|
579
|
+
The estimate is a conditional association. It has a causal reading only if the
|
|
580
|
+
timing of events is unrelated to other shocks to the outcome; with repeated events
|
|
581
|
+
the windows of successive events overlap.
|
|
582
|
+
"""
|
|
583
|
+
units = [u for u in outcome.keys() if u in event]
|
|
584
|
+
if len(units) < 5:
|
|
585
|
+
return {"error": "At least 5 units present in both outcome and event are required.", "n_units": len(units)}
|
|
586
|
+
|
|
587
|
+
def run(shift_fn) -> Optional[Dict[str, Any]]:
|
|
588
|
+
dep: List[float] = []; x: List[float] = []; ui: List[int] = []; ti: List[int] = []
|
|
589
|
+
tmap: Dict[int, int] = {}
|
|
590
|
+
umap: Dict[Any, int] = {}
|
|
591
|
+
for u in units:
|
|
592
|
+
y = list(outcome[u]); e = list(event[u])
|
|
593
|
+
T = min(len(y), len(e))
|
|
594
|
+
for t in range(1, T):
|
|
595
|
+
target = shift_fn(t)
|
|
596
|
+
if target < 0 or target >= T:
|
|
597
|
+
continue
|
|
598
|
+
if not (_is_num(y[target]) and _is_num(y[t - 1]) and _is_num(e[t])):
|
|
599
|
+
continue
|
|
600
|
+
dep.append(float(y[target]) - float(y[t - 1])); x.append(float(e[t]))
|
|
601
|
+
ui.append(umap.setdefault(u, len(umap))); ti.append(tmap.setdefault(t, len(tmap)))
|
|
602
|
+
n = len(dep)
|
|
603
|
+
g = len(umap)
|
|
604
|
+
if n < 30 or g < 5:
|
|
605
|
+
return None
|
|
606
|
+
yd = _two_way_demean(dep, ui, ti, g, len(tmap))
|
|
607
|
+
xd = _two_way_demean(x, ui, ti, g, len(tmap))
|
|
608
|
+
sxx = sum(v * v for v in xd)
|
|
609
|
+
if sxx <= 1e-12:
|
|
610
|
+
return None
|
|
611
|
+
beta = sum(a * b for a, b in zip(xd, yd)) / sxx
|
|
612
|
+
score = [0.0] * g
|
|
613
|
+
for a, b, u in zip(xd, yd, ui):
|
|
614
|
+
score[u] += a * (b - beta * a)
|
|
615
|
+
var = sum(s * s for s in score) / (sxx * sxx) * g / (g - 1)
|
|
616
|
+
se = math.sqrt(var)
|
|
617
|
+
t_stat = beta / se if se > 0 else float("nan")
|
|
618
|
+
return {"beta": beta, "se": se, "t": t_stat, "p_value": 2.0 * _normal_sf(abs(t_stat)) if se > 0 else None,
|
|
619
|
+
"ci95_low": beta - 1.96 * se, "ci95_high": beta + 1.96 * se, "n_obs": n, "n_units": g,
|
|
620
|
+
"n_events": sum(1 for v in x if v != 0)}
|
|
621
|
+
|
|
622
|
+
res: Dict[str, Any] = {"horizons": {}, "placebo_leads": {}, "n_units_input": len(units)}
|
|
623
|
+
for h in horizons:
|
|
624
|
+
r = run(lambda t, h=h: t + int(h))
|
|
625
|
+
res["horizons"][int(h)] = r if r is not None else {"error": "Too few observations or no variation in the event."}
|
|
626
|
+
for k in leads:
|
|
627
|
+
r = run(lambda t, k=k: t - 1 - int(k))
|
|
628
|
+
res["placebo_leads"][int(k)] = r if r is not None else {"error": "Too few observations or no variation in the event."}
|
|
629
|
+
return res
|