stmmlib 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stmmlib/__init__.py +30 -0
- stmmlib/_backend.py +153 -0
- stmmlib/_utils.py +175 -0
- stmmlib/beta/__init__.py +17 -0
- stmmlib/beta/config.py +108 -0
- stmmlib/beta/criteria.py +247 -0
- stmmlib/beta/design.py +222 -0
- stmmlib/beta/diagnose.py +168 -0
- stmmlib/beta/diagnostics.py +94 -0
- stmmlib/beta/fit.py +598 -0
- stmmlib/beta/gpu_solvers.py +169 -0
- stmmlib/beta/inference.py +403 -0
- stmmlib/beta/lrt_fixed.py +551 -0
- stmmlib/beta/moran_panel.py +733 -0
- stmmlib/beta/plot_data.py +252 -0
- stmmlib/beta/precision.py +98 -0
- stmmlib/beta/report.py +373 -0
- stmmlib/beta/result.py +696 -0
- stmmlib/beta/simulation.py +343 -0
- stmmlib/beta/solvers.py +2021 -0
- stmmlib/beta/structure.py +56 -0
- stmmlib/diagnostics_suite.py +663 -0
- stmmlib/gaussian/__init__.py +1 -0
- stmmlib/gaussian/blup.py +252 -0
- stmmlib/gaussian/covariance.py +599 -0
- stmmlib/gaussian/design.py +312 -0
- stmmlib/gaussian/diagnostics.py +308 -0
- stmmlib/gaussian/gls.py +285 -0
- stmmlib/gaussian/hessian.py +197 -0
- stmmlib/gaussian/lrt_fixed.py +289 -0
- stmmlib/gaussian/models.py +528 -0
- stmmlib/gaussian/optimization.py +361 -0
- stmmlib/gaussian/simulation.py +186 -0
- stmmlib/gaussian/summary.py +277 -0
- stmmlib/gaussian/transforms.py +419 -0
- stmmlib/gaussian/variance_decomp.py +191 -0
- stmmlib/lrt.py +74 -0
- stmmlib/output_format.py +148 -0
- stmmlib/output_format_beta.py +299 -0
- stmmlib/output_format_normal.py +118 -0
- stmmlib/registry.py +326 -0
- stmmlib/stmm.py +265 -0
- stmmlib-0.2.0.dist-info/METADATA +215 -0
- stmmlib-0.2.0.dist-info/RECORD +47 -0
- stmmlib-0.2.0.dist-info/WHEEL +5 -0
- stmmlib-0.2.0.dist-info/licenses/LICENSE +21 -0
- stmmlib-0.2.0.dist-info/top_level.txt +1 -0
stmmlib/beta/criteria.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"""Conditional selection criteria for Beta STMM: cAIC + conditional R2.
|
|
2
|
+
|
|
3
|
+
Single source of truth for the `beta_criteria` helper previously inlined
|
|
4
|
+
in research notebooks. Import it instead of copying::
|
|
5
|
+
|
|
6
|
+
from stmmlib.beta.criteria import beta_criteria
|
|
7
|
+
|
|
8
|
+
Formulas (validated):
|
|
9
|
+
cAIC (Huber et al. 2016): -2 * ll_cond + 2 * df_hat
|
|
10
|
+
df_hat = tr(H_pen^-1 @ H_unpen), the Liang-Wu-Zou (2008) bias
|
|
11
|
+
correction. H_pen/_unpen use the Fisher weight w = phi*mu*(1-mu).
|
|
12
|
+
cR2 (Nakagawa & Schielzeth 2013, latent-scale working version):
|
|
13
|
+
vf = var(X beta); vre = mean(diag(Z G Z')), G = Q^-1;
|
|
14
|
+
vresid = mean(mu(1-mu)/(1+phi)).
|
|
15
|
+
|
|
16
|
+
Fixes vs the old notebook copy:
|
|
17
|
+
- beta log-density SUBTRACTS ln B(a,b) (the old copy added it, which
|
|
18
|
+
produced a cAIC off by 4.8e6 from the correct value).
|
|
19
|
+
- mu is always recomputed from the fitted (beta, b_hat); a stored
|
|
20
|
+
mu_hat is never trusted.
|
|
21
|
+
- area SAR uses des["rho_bound"] (spectral bound), not the gamma bound.
|
|
22
|
+
|
|
23
|
+
Guards (refuse garbage instead of printing it):
|
|
24
|
+
- df_hat must lie in [0, p+q] (theorem: H_pen = H_unpen + blkdiag(0,Q)).
|
|
25
|
+
- ll_cond sanity vs the Laplace marginal (loose heuristic tolerance).
|
|
26
|
+
- finite vre.
|
|
27
|
+
|
|
28
|
+
Scope: flat (non-nested) models M1-M5, M8-M12. Nested fits (M6/M7/M13/M14)
|
|
29
|
+
return all-NaN fields -- their [prov, kab, cell] precision needs its own
|
|
30
|
+
builder (documented follow-up, not silent garbage).
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import numpy as np
|
|
36
|
+
from scipy.special import betaln
|
|
37
|
+
|
|
38
|
+
from stmmlib.beta.precision import PrecisionShapeError
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _as_opt(fit):
|
|
42
|
+
return fit._opt if hasattr(fit, "_opt") else fit
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def beta_criteria(fit, W=None):
|
|
46
|
+
"""Compute cAIC, df_hat, ll_cond, cR2 for a fitted Beta STMM.
|
|
47
|
+
|
|
48
|
+
Args:
|
|
49
|
+
fit: BetastmmResult (or raw opt dict from beta_laplace_structural).
|
|
50
|
+
W: spatial weight matrix override (default: opt/des stored W).
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
dict with cAIC, df_hat, ll_cond, cR2_marginal, cR2_conditional,
|
|
54
|
+
var_fixed, var_random_latent, var_resid_beta; NaN fields when the
|
|
55
|
+
inputs are inconsistent (never a garbage number). None only when
|
|
56
|
+
required arrays are missing or an unexpected error occurs.
|
|
57
|
+
"""
|
|
58
|
+
if fit is None:
|
|
59
|
+
return None
|
|
60
|
+
try:
|
|
61
|
+
opt = _as_opt(fit)
|
|
62
|
+
X = opt.get("X")
|
|
63
|
+
Z = opt.get("Z")
|
|
64
|
+
y = opt.get("y")
|
|
65
|
+
des = opt.get("des", {}) or {}
|
|
66
|
+
model = opt.get("model", "")
|
|
67
|
+
if X is None or Z is None or y is None:
|
|
68
|
+
return None
|
|
69
|
+
|
|
70
|
+
beta = fit.beta if hasattr(fit, "_opt") else opt.get("beta_hat")
|
|
71
|
+
try:
|
|
72
|
+
phi = float(fit.phi) if hasattr(fit, "_opt") else float(opt.get("phi", 20.0))
|
|
73
|
+
except Exception:
|
|
74
|
+
phi = float(np.exp(np.asarray(opt.get("logphi", [np.log(20.0)]))).ravel()[0])
|
|
75
|
+
|
|
76
|
+
logs = np.asarray(opt.get("logs", np.zeros(2)), dtype=float).ravel()
|
|
77
|
+
raw_dep = np.asarray(opt.get("raw_dep", np.array([])), dtype=float).ravel()
|
|
78
|
+
bh = opt.get("b_hat")
|
|
79
|
+
if beta is None or bh is None:
|
|
80
|
+
return None
|
|
81
|
+
beta = np.asarray(beta, dtype=float).ravel()
|
|
82
|
+
bh = np.asarray(bh, dtype=float).ravel()
|
|
83
|
+
|
|
84
|
+
from stmmlib.beta.structure import stmm_beta_structure
|
|
85
|
+
s = stmm_beta_structure(model)
|
|
86
|
+
if s["area"] in ("nested_iid", "nested_sar"):
|
|
87
|
+
return _nan_result() # nested precision: dedicated builder needed
|
|
88
|
+
|
|
89
|
+
blocks = des.get("blocks", {}) or {}
|
|
90
|
+
n_area = len(blocks.get("area", []))
|
|
91
|
+
n_time = len(blocks.get("area_time", blocks.get("time", [])))
|
|
92
|
+
# Area spectral bound (NOT the gamma bound) for the SAR display mapping
|
|
93
|
+
rb = des.get("rho_bound", des.get("rho_bound_g", 0.95))
|
|
94
|
+
|
|
95
|
+
blocks_Q = []
|
|
96
|
+
if s["area"] in ("iid",):
|
|
97
|
+
Q_a = np.exp(-2.0 * logs[0]) * np.eye(n_area)
|
|
98
|
+
elif s["area"] in ("sar", "nested_sar", "seblup_rao_yu"):
|
|
99
|
+
rho = rb * np.tanh(raw_dep[0]) if len(raw_dep) > 0 else 0.0
|
|
100
|
+
# BUG-B4 fix: explicit W argument wins; opt["W"] may be None even
|
|
101
|
+
# when the key exists (old expression silently ignored use_W).
|
|
102
|
+
W_arr = W if W is not None else opt.get("W")
|
|
103
|
+
if W_arr is None:
|
|
104
|
+
W_arr = des.get("W_gamma")
|
|
105
|
+
if W_arr is None:
|
|
106
|
+
W_arr = np.eye(n_area)
|
|
107
|
+
elif hasattr(W_arr, "toarray"):
|
|
108
|
+
W_arr = W_arr.toarray()
|
|
109
|
+
W_arr = np.asarray(W_arr, dtype=float)[:n_area, :n_area]
|
|
110
|
+
A = np.eye(n_area) - rho * W_arr
|
|
111
|
+
Q_a = np.exp(-2.0 * logs[0]) * (A.T @ A)
|
|
112
|
+
else:
|
|
113
|
+
Q_a = np.eye(n_area)
|
|
114
|
+
blocks_Q.append(Q_a)
|
|
115
|
+
|
|
116
|
+
area_specific = s.get("time_scope") == "area_specific"
|
|
117
|
+
if s["time"] == "ar1" and n_time > 0:
|
|
118
|
+
# Frozen model mapping: ar1_precision squashes phi_t again
|
|
119
|
+
# (pb*tanh(pb*tanh(raw))); keep the tridiagonal form here.
|
|
120
|
+
pb = des.get("phi_bound", 0.95)
|
|
121
|
+
phi_t = pb * np.tanh(raw_dep[1]) if len(raw_dep) > 1 else 0.0
|
|
122
|
+
sigma2_inv = np.exp(-2.0 * (logs[1] if len(logs) > 1 else 0.0))
|
|
123
|
+
if area_specific:
|
|
124
|
+
from stmmlib.beta.precision import ar1_precision as _ar1p
|
|
125
|
+
from stmmlib._backend import Backend as _B
|
|
126
|
+
from scipy.sparse import block_diag as _spbd
|
|
127
|
+
T_loc = int(des.get("T", len(blocks.get("time", []))))
|
|
128
|
+
TQ = _ar1p(max(T_loc, 1), logs[1] if len(logs) > 1 else 0.0, phi_t, _B(gpu=False),
|
|
129
|
+
phi_bound=pb)
|
|
130
|
+
if hasattr(TQ, "toarray"):
|
|
131
|
+
TQ = TQ.toarray()
|
|
132
|
+
Q_t = _spbd([TQ] * n_area, format="csr").toarray()
|
|
133
|
+
else:
|
|
134
|
+
di = np.full(n_time, 1.0 + phi_t ** 2)
|
|
135
|
+
di[0] = di[-1] = 1.0
|
|
136
|
+
Q_t = sigma2_inv * (np.diag(di)
|
|
137
|
+
+ np.diag(np.full(n_time - 1, -phi_t), 1)
|
|
138
|
+
+ np.diag(np.full(n_time - 1, -phi_t), -1))
|
|
139
|
+
blocks_Q.append(Q_t)
|
|
140
|
+
elif n_time > 0:
|
|
141
|
+
Q_t = np.exp(-2.0 * logs[1]) * np.eye(n_time) if len(logs) > 1 else np.eye(n_time)
|
|
142
|
+
blocks_Q.append(Q_t)
|
|
143
|
+
|
|
144
|
+
from scipy.sparse import block_diag as _spblockdiag
|
|
145
|
+
Q_base = _spblockdiag(blocks_Q)
|
|
146
|
+
if hasattr(Q_base, "toarray"):
|
|
147
|
+
Q_base = Q_base.toarray()
|
|
148
|
+
|
|
149
|
+
n_gamma = len(blocks.get("interaction", []))
|
|
150
|
+
if n_gamma > 0:
|
|
151
|
+
# Same Q_g the fit uses (shared helper): the diagonal shortcut here
|
|
152
|
+
# was wrong whenever rho_gamma/phi_gamma were nonzero (M9-M12).
|
|
153
|
+
from stmmlib._backend import Backend
|
|
154
|
+
from stmmlib.beta.solvers import beta_Q_gamma
|
|
155
|
+
Q_g = beta_Q_gamma(
|
|
156
|
+
s, des, int(des.get("m", n_area)), int(des.get("T", n_time)),
|
|
157
|
+
n_gamma, logs, raw_dep, len(logs), len(raw_dep),
|
|
158
|
+
s["area"] in ("sar", "seblup_rao_yu"),
|
|
159
|
+
s["time"] == "ar1", Backend(gpu=False))
|
|
160
|
+
if Q_g is not None and Q_g.size > 0:
|
|
161
|
+
Q_full = np.block([
|
|
162
|
+
[Q_base, np.zeros((Q_base.shape[0], n_gamma))],
|
|
163
|
+
[np.zeros((n_gamma, Q_base.shape[0])), Q_g]])
|
|
164
|
+
else:
|
|
165
|
+
Q_full = Q_base
|
|
166
|
+
else:
|
|
167
|
+
Q_full = Q_base
|
|
168
|
+
|
|
169
|
+
q = Z.shape[1]
|
|
170
|
+
if Q_full.shape[0] != q:
|
|
171
|
+
raise PrecisionShapeError(
|
|
172
|
+
f"Precision/design size mismatch: Q is {Q_full.shape[0]}x"
|
|
173
|
+
f"{Q_full.shape[1]} but Z has {q} columns."
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
# Conditional mean from the fitted (beta, b_hat) -- never stored mu_hat
|
|
177
|
+
mu = np.clip(1 / (1 + np.exp(-(X @ beta + Z @ bh))), 1e-10, 1 - 1e-10)
|
|
178
|
+
w = phi * mu * (1 - mu)
|
|
179
|
+
A11 = X.T @ (X * w[:, None])
|
|
180
|
+
A12 = X.T @ (Z * w[:, None])
|
|
181
|
+
A22 = Z.T @ (Z * w[:, None]) + Q_full
|
|
182
|
+
Hm = np.block([[A11, A12], [A12.T, A22]])
|
|
183
|
+
M22 = Z.T @ (Z * w[:, None])
|
|
184
|
+
Mm = np.block([[A11, A12], [A12.T, M22]])
|
|
185
|
+
|
|
186
|
+
# df guard: nested cells-in-areas make Z'WZ rank-deficient; without
|
|
187
|
+
# enough Q shrinkage the raw solve explodes. df in [0, p+q] or NaN.
|
|
188
|
+
p_dof = X.shape[1] + Z.shape[1]
|
|
189
|
+
try:
|
|
190
|
+
df_hat = float(np.trace(np.linalg.solve(Hm, Mm)))
|
|
191
|
+
if not np.isfinite(df_hat) or df_hat < 0 or df_hat > p_dof:
|
|
192
|
+
df_hat = np.nan
|
|
193
|
+
except np.linalg.LinAlgError:
|
|
194
|
+
df_hat = np.nan
|
|
195
|
+
|
|
196
|
+
# Conditional log-lik: (a-1)ln y + (b-1)ln(1-y) MINUS ln B(a,b)
|
|
197
|
+
a = mu * phi
|
|
198
|
+
b = (1 - mu) * phi
|
|
199
|
+
ll_cond = float(np.sum((a - 1) * np.log(np.clip(y, 1e-15, 1.0)) +
|
|
200
|
+
(b - 1) * np.log(np.clip(1 - y, 1e-15, 1.0)) -
|
|
201
|
+
betaln(a, b)))
|
|
202
|
+
# ll guard (loose heuristic): far below the Laplace marginal means
|
|
203
|
+
# stale inputs -- refuse garbage instead of printing it. Threshold
|
|
204
|
+
# max(10, 1%|ll|): tight enough to catch a miscalibrated mu, loose
|
|
205
|
+
# enough for Laplace-vs-mode wiggle on heavy-shrink fits. The
|
|
206
|
+
# absolute floor exists because at small |ll| a purely relative
|
|
207
|
+
# test has no power; it is a deliberate slack, not a measurement.
|
|
208
|
+
ll_marg = float(opt.get("log_lik", np.nan))
|
|
209
|
+
if (not np.isfinite(ll_cond) or not np.isfinite(df_hat)
|
|
210
|
+
or (np.isfinite(ll_marg)
|
|
211
|
+
and ll_cond < ll_marg - max(10.0, 0.01 * abs(ll_marg)))):
|
|
212
|
+
out = _nan_result()
|
|
213
|
+
out["df_hat"], out["ll_cond"] = df_hat, ll_cond
|
|
214
|
+
return out
|
|
215
|
+
caic = -2 * ll_cond + 2 * df_hat
|
|
216
|
+
|
|
217
|
+
# cR2 (Nakagawa working version)
|
|
218
|
+
try:
|
|
219
|
+
G = np.linalg.inv(Q_full)
|
|
220
|
+
vre = np.sum((Z @ G) * Z) / len(y)
|
|
221
|
+
if not np.isfinite(vre):
|
|
222
|
+
vre = np.nan
|
|
223
|
+
except np.linalg.LinAlgError:
|
|
224
|
+
vre = np.nan
|
|
225
|
+
vf = float(np.var(X @ beta))
|
|
226
|
+
vresid = float(np.mean(mu * (1 - mu) / (1 + phi)))
|
|
227
|
+
tot = vf + vre + vresid
|
|
228
|
+
r2m = vf / tot if tot > 0 else np.nan
|
|
229
|
+
r2c = (vf + vre) / tot if tot > 0 else np.nan
|
|
230
|
+
|
|
231
|
+
return {
|
|
232
|
+
"cAIC": caic, "df_hat": df_hat, "ll_cond": ll_cond,
|
|
233
|
+
"cR2_marginal": r2m, "cR2_conditional": r2c,
|
|
234
|
+
"var_fixed": vf, "var_random_latent": vre, "var_resid_beta": vresid,
|
|
235
|
+
}
|
|
236
|
+
except PrecisionShapeError:
|
|
237
|
+
raise
|
|
238
|
+
except Exception:
|
|
239
|
+
return None
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _nan_result():
|
|
243
|
+
nan = float("nan")
|
|
244
|
+
return {"cAIC": nan, "df_hat": nan, "ll_cond": nan,
|
|
245
|
+
"cR2_marginal": nan, "cR2_conditional": nan,
|
|
246
|
+
"var_fixed": nan, "var_random_latent": nan,
|
|
247
|
+
"var_resid_beta": nan}
|
stmmlib/beta/design.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Design matrix construction for Beta mixed models.
|
|
2
|
+
|
|
3
|
+
Builds Z (area/time/cell/interaction), block indices, W_gamma,
|
|
4
|
+
and metadata for the Beta STMM Laplace engine.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
from stmmlib.beta.structure import stmm_beta_structure
|
|
15
|
+
|
|
16
|
+
if TYPE_CHECKING:
|
|
17
|
+
from stmmlib._backend import Backend
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def stmm_beta_make_design(d: pd.DataFrame, model: str, W=None, backend: Backend = None,
|
|
21
|
+
rho_bound: float = 0.95) -> dict:
|
|
22
|
+
"""Construct design matrix Z and metadata for a Beta STMM.
|
|
23
|
+
|
|
24
|
+
Z has area-major cell order: (area1:time1..timeT, area2:time1..timeT, ...)
|
|
25
|
+
|
|
26
|
+
Args:
|
|
27
|
+
d: pandas DataFrame with area, time columns (internal names).
|
|
28
|
+
model: Model ID string.
|
|
29
|
+
W: Spatial weight matrix.
|
|
30
|
+
backend: Array backend.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
dict with Z, blocks, W_gamma, B_area, B_time, metadata.
|
|
34
|
+
"""
|
|
35
|
+
if backend is None:
|
|
36
|
+
from stmmlib._backend import Backend
|
|
37
|
+
backend = Backend(gpu=False)
|
|
38
|
+
|
|
39
|
+
xp = backend.xp
|
|
40
|
+
s = stmm_beta_structure(model)
|
|
41
|
+
area = s["area"]
|
|
42
|
+
time = s["time"]
|
|
43
|
+
time_scope = s["time_scope"]
|
|
44
|
+
interaction = s["interaction"]
|
|
45
|
+
|
|
46
|
+
# Extract indices
|
|
47
|
+
area_col = ".stmm_area"
|
|
48
|
+
time_col = ".stmm_time"
|
|
49
|
+
area_idx = d[area_col].values.astype(np.int32)
|
|
50
|
+
time_idx = d[time_col].values.astype(np.int32)
|
|
51
|
+
n = len(d)
|
|
52
|
+
m = int(np.max(area_idx)) + 1
|
|
53
|
+
T = int(np.max(time_idx)) + 1
|
|
54
|
+
n_cell = m * T
|
|
55
|
+
|
|
56
|
+
# Build Z
|
|
57
|
+
from scipy.sparse import csr_matrix, eye as sp_eye, kron as sp_kron
|
|
58
|
+
|
|
59
|
+
def _make_Z(idx, n_rows, n_cols):
|
|
60
|
+
row = np.arange(n_rows, dtype=np.int32)
|
|
61
|
+
return csr_matrix((np.ones(n_rows, dtype=np.float64), (row, idx)), shape=(n_rows, n_cols))
|
|
62
|
+
|
|
63
|
+
# Nested hierarchy (M6/M7/M13/M14): Z = [Z_prov, Z_kab, Z_cell]
|
|
64
|
+
if area in ("nested_iid", "nested_sar"):
|
|
65
|
+
if ".stmm_prov" not in d.columns or ".stmm_kab" not in d.columns:
|
|
66
|
+
raise ValueError(
|
|
67
|
+
"Nested models require province and district columns; "
|
|
68
|
+
"pass prov_col=/kab_col= (defaults 'prov'/'kab').")
|
|
69
|
+
prov_idx = d[".stmm_prov"].values.astype(np.int32)
|
|
70
|
+
kab_idx = area_idx # .stmm_area IS the kab level for nested models
|
|
71
|
+
n_prov = int(np.max(prov_idx)) + 1
|
|
72
|
+
n_kab = m
|
|
73
|
+
Z_prov = _make_Z(prov_idx, n, n_prov)
|
|
74
|
+
Z_kab = _make_Z(kab_idx, n, n_kab)
|
|
75
|
+
Zc = _make_Z(area_idx * T + time_idx, n, n_cell)
|
|
76
|
+
Z = np.hstack([Z_prov.toarray(), Z_kab.toarray(), Zc.toarray()])
|
|
77
|
+
blocks = {
|
|
78
|
+
"prov": list(range(n_prov)),
|
|
79
|
+
"kab": list(range(n_prov, n_prov + n_kab)),
|
|
80
|
+
"cell": list(range(n_prov + n_kab, n_prov + n_kab + n_cell)),
|
|
81
|
+
}
|
|
82
|
+
des = {
|
|
83
|
+
"Z": Z, "blocks": blocks, "W_gamma": None, "rho_bound_g": rho_bound,
|
|
84
|
+
"B_area": None, "B_time": None, "n": n, "m": m, "T": T,
|
|
85
|
+
"n_cell": n_cell, "n_gamma": 0,
|
|
86
|
+
"n_prov": n_prov, "n_kab": n_kab,
|
|
87
|
+
"area_labels": d[area_col].unique().tolist(),
|
|
88
|
+
"time_labels": sorted(d[time_col].unique().tolist()),
|
|
89
|
+
"area_idx": area_idx, "time_idx": time_idx,
|
|
90
|
+
}
|
|
91
|
+
# Interaction at kab x time level (M13 temporal / M14 iid)
|
|
92
|
+
if interaction != "none":
|
|
93
|
+
B_area = _contr_sum(m)
|
|
94
|
+
B_time = _contr_sum(T)
|
|
95
|
+
Z_gamma = _build_Z_gamma(area_idx, time_idx, n, m, T, interaction,
|
|
96
|
+
B_area, B_time)
|
|
97
|
+
n_gamma = Z_gamma.shape[1]
|
|
98
|
+
blocks["interaction"] = list(range(Z.shape[1], Z.shape[1] + n_gamma))
|
|
99
|
+
des["Z"] = np.hstack([Z, Z_gamma])
|
|
100
|
+
des["B_area"] = B_area
|
|
101
|
+
des["B_time"] = B_time
|
|
102
|
+
des["n_gamma"] = n_gamma
|
|
103
|
+
return des
|
|
104
|
+
else:
|
|
105
|
+
des = _design_flat(d, area, time, time_scope, interaction, W, backend,
|
|
106
|
+
area_idx, time_idx, n, m, T, n_cell, _make_Z,
|
|
107
|
+
rho_bound=rho_bound)
|
|
108
|
+
return des
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _design_flat(d, area, time, time_scope, interaction, W, backend,
|
|
112
|
+
area_idx, time_idx, n, m, T, n_cell, _make_Z,
|
|
113
|
+
rho_bound: float = 0.95):
|
|
114
|
+
"""Flat (non-nested) design: Z = [Z_area, Z_cell|Z_time] (+ Z_gamma)."""
|
|
115
|
+
# Area dummy
|
|
116
|
+
Z_area = _make_Z(area_idx, n, m)
|
|
117
|
+
|
|
118
|
+
# Cell dummy (area x time)
|
|
119
|
+
cell_idx = area_idx * T + time_idx
|
|
120
|
+
Z_cell = _make_Z(cell_idx, n, n_cell)
|
|
121
|
+
|
|
122
|
+
if time_scope == "global":
|
|
123
|
+
# Global: Z = [Z_area, Z_time]
|
|
124
|
+
Z_time = _make_Z(time_idx, n, T)
|
|
125
|
+
Z = np.hstack([Z_area.toarray(), Z_time.toarray()])
|
|
126
|
+
blocks = {
|
|
127
|
+
"area": list(range(m)),
|
|
128
|
+
"time": list(range(m, m + T)),
|
|
129
|
+
}
|
|
130
|
+
else:
|
|
131
|
+
# Area-specific: Z = [Z_area, Z_cell]
|
|
132
|
+
Z = np.hstack([Z_area.toarray(), Z_cell.toarray()])
|
|
133
|
+
blocks = {
|
|
134
|
+
"area": list(range(m)),
|
|
135
|
+
"area_time": list(range(m, m + n_cell)),
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
# Contrast matrices for interaction
|
|
139
|
+
B_area = None
|
|
140
|
+
B_time = None
|
|
141
|
+
W_gamma = None
|
|
142
|
+
rho_bound_g = rho_bound
|
|
143
|
+
|
|
144
|
+
if interaction != "none":
|
|
145
|
+
B_area = _contr_sum(m)
|
|
146
|
+
B_time = _contr_sum(T)
|
|
147
|
+
|
|
148
|
+
if interaction in ("spatial", "st") and W is not None:
|
|
149
|
+
if backend.is_sparse(W):
|
|
150
|
+
W_arr = W.toarray()
|
|
151
|
+
else:
|
|
152
|
+
W_arr = np.asarray(W, dtype=np.float64)
|
|
153
|
+
W_gamma = W_arr
|
|
154
|
+
# BUG-B1 fix: eigvals (spectral radius), NOT eigvalsh -- W is
|
|
155
|
+
# asymmetric; eigvalsh silently reads one triangle, so the
|
|
156
|
+
# spectral radius can come out wrong on a row-standardized
|
|
157
|
+
# weights matrix.
|
|
158
|
+
eigenvalues = np.linalg.eigvals(W_arr)
|
|
159
|
+
max_abs_eig = np.max(np.abs(eigenvalues))
|
|
160
|
+
rho_bound_g = rho_bound / max_abs_eig if max_abs_eig > 1e-10 else rho_bound
|
|
161
|
+
|
|
162
|
+
# Interaction Z
|
|
163
|
+
Z_gamma = _build_Z_gamma(area_idx, time_idx, n, m, T, interaction, B_area, B_time)
|
|
164
|
+
n_gamma = Z_gamma.shape[1]
|
|
165
|
+
blocks["interaction"] = list(range(Z.shape[1], Z.shape[1] + n_gamma))
|
|
166
|
+
Z = np.hstack([Z, Z_gamma])
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
"Z": Z,
|
|
170
|
+
"blocks": blocks,
|
|
171
|
+
"W_gamma": W_gamma,
|
|
172
|
+
"rho_bound_g": rho_bound_g,
|
|
173
|
+
"B_area": B_area,
|
|
174
|
+
"B_time": B_time,
|
|
175
|
+
"n": n,
|
|
176
|
+
"m": m,
|
|
177
|
+
"T": T,
|
|
178
|
+
"n_cell": n_cell,
|
|
179
|
+
"n_gamma": Z_gamma.shape[1] if interaction != "none" else 0,
|
|
180
|
+
"area_labels": d[".stmm_area"].unique().tolist() if ".stmm_area" in d.columns else [],
|
|
181
|
+
"time_labels": sorted(d[".stmm_time"].unique().tolist()) if ".stmm_time" in d.columns else [],
|
|
182
|
+
"area_idx": area_idx,
|
|
183
|
+
"time_idx": time_idx,
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _contr_sum(n: int):
|
|
188
|
+
"""Sum-to-zero contrast matrix (R-style contr.sum). Returns n x (n-1)."""
|
|
189
|
+
if n <= 1:
|
|
190
|
+
return np.ones((1, 1), dtype=np.float64)
|
|
191
|
+
B = np.eye(n, dtype=np.float64)[:, :-1]
|
|
192
|
+
B[-1, :] = -1.0
|
|
193
|
+
return B
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _build_Z_gamma(area_idx, time_idx, n, m, T, interaction, B_area, B_time):
|
|
197
|
+
"""Build Z_gamma from contrast matrices and full cell Z."""
|
|
198
|
+
from scipy.sparse import eye as sp_eye, kron as sp_kron, csr_matrix
|
|
199
|
+
|
|
200
|
+
# Full cell Z
|
|
201
|
+
cell_idx = area_idx * T + time_idx
|
|
202
|
+
Z_full = csr_matrix(
|
|
203
|
+
(np.ones(n, dtype=np.float64), (np.arange(n, dtype=np.int32), cell_idx)),
|
|
204
|
+
shape=(n, m * T),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
m1 = B_area.shape[1]
|
|
208
|
+
T1 = B_time.shape[1]
|
|
209
|
+
|
|
210
|
+
if interaction == "iid":
|
|
211
|
+
K = sp_kron(csr_matrix(B_area), sp_eye(T, dtype=np.float64), format="csr")
|
|
212
|
+
elif interaction == "spatial":
|
|
213
|
+
K = sp_kron(csr_matrix(B_area), sp_eye(T, dtype=np.float64), format="csr")
|
|
214
|
+
elif interaction == "temporal":
|
|
215
|
+
K = sp_kron(sp_eye(m, dtype=np.float64), csr_matrix(B_time), format="csr")
|
|
216
|
+
elif interaction == "st":
|
|
217
|
+
K = sp_kron(csr_matrix(B_area), csr_matrix(B_time), format="csr")
|
|
218
|
+
else:
|
|
219
|
+
return np.zeros((n, 0), dtype=np.float64)
|
|
220
|
+
|
|
221
|
+
Z_gamma = (Z_full @ K).toarray()
|
|
222
|
+
return Z_gamma
|
stmmlib/beta/diagnose.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Boundary diagnosis for AR/precision parameters (pure decision logic).
|
|
2
|
+
|
|
3
|
+
Maps the manual decision tree in docs/notes/PENANGANAN_BOUNDARY_AR_PHI.md into a
|
|
4
|
+
pure function: no fitting, no simulation, no data access. It only classifies
|
|
5
|
+
evidence the caller already collected (seed endpoints, bound sensitivity,
|
|
6
|
+
gradient, panel length T).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class BoundaryDiagnosis:
|
|
16
|
+
"""Structured boundary verdict (not free text)."""
|
|
17
|
+
|
|
18
|
+
classification: str
|
|
19
|
+
evidence: list = field(default_factory=list)
|
|
20
|
+
recommended_action: str = ""
|
|
21
|
+
citations: list = field(default_factory=list)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
_CITATIONS = [
|
|
25
|
+
"Nickell (1981) panel pile-up on short T",
|
|
26
|
+
"Chung et al. (2013) penalized likelihood for variance parameters",
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
_SEED_TOL = 1e-6
|
|
30
|
+
_SHIFT_TOL = 0.01
|
|
31
|
+
_GRAD_TOL = 1.0
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _as_floats(values) -> list:
|
|
35
|
+
out = []
|
|
36
|
+
for v in values:
|
|
37
|
+
try:
|
|
38
|
+
if isinstance(v, (tuple, list)) and len(v) > 0:
|
|
39
|
+
v = v[0]
|
|
40
|
+
out.append(float(v))
|
|
41
|
+
except (TypeError, ValueError):
|
|
42
|
+
continue
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def diagnose_boundary(
|
|
47
|
+
fit_result=None,
|
|
48
|
+
*,
|
|
49
|
+
n_seeds_tested: int = 0,
|
|
50
|
+
seed_endpoints=(),
|
|
51
|
+
bound_sensitivity=None,
|
|
52
|
+
gradient_at_solution=None,
|
|
53
|
+
T=None,
|
|
54
|
+
t_warn: int = 15,
|
|
55
|
+
) -> BoundaryDiagnosis:
|
|
56
|
+
"""Classify a boundary solution from collected evidence (pure function).
|
|
57
|
+
|
|
58
|
+
Rules: non-identical seed endpoints -> MULTI_BASIN; identical endpoints
|
|
59
|
+
with a significant bound shift and a small gradient -> ARTIFICIAL_BOUND;
|
|
60
|
+
identical endpoints with no shift and a large gradient ->
|
|
61
|
+
STRUCTURAL_BOUNDARY (dual ML + penalized report required, no cosmetic
|
|
62
|
+
fix). T < t_warn appends a Nickell/pile-up warning to evidence.
|
|
63
|
+
"""
|
|
64
|
+
evidence: list = []
|
|
65
|
+
grad = gradient_at_solution
|
|
66
|
+
tt = T
|
|
67
|
+
if grad is None and fit_result is not None:
|
|
68
|
+
try:
|
|
69
|
+
om = (fit_result._opt or {}).get("optim", {}) or {}
|
|
70
|
+
g = om.get("grad_norm")
|
|
71
|
+
grad = float(g) if g is not None else None
|
|
72
|
+
except (AttributeError, TypeError, ValueError):
|
|
73
|
+
grad = None
|
|
74
|
+
if tt is None and fit_result is not None:
|
|
75
|
+
try:
|
|
76
|
+
des = (fit_result._opt or {}).get("des", {}) or {}
|
|
77
|
+
tt = des.get("T")
|
|
78
|
+
except (AttributeError, TypeError):
|
|
79
|
+
tt = None
|
|
80
|
+
|
|
81
|
+
endpoints = _as_floats(seed_endpoints or ())
|
|
82
|
+
n_seed_info = len(endpoints)
|
|
83
|
+
if n_seed_info >= 2:
|
|
84
|
+
spread = max(endpoints) - min(endpoints)
|
|
85
|
+
identical = spread <= _SEED_TOL
|
|
86
|
+
evidence.append(
|
|
87
|
+
f"{n_seed_info} seed endpoints, spread {spread:.3g} "
|
|
88
|
+
f"({'identical' if identical else 'not identical'})."
|
|
89
|
+
)
|
|
90
|
+
else:
|
|
91
|
+
identical = True
|
|
92
|
+
evidence.append(
|
|
93
|
+
f"Only {n_seed_info} seed endpoint(s) "
|
|
94
|
+
f"(n_seeds_tested={n_seeds_tested}); multi-basin not ruled out."
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
shift = None
|
|
98
|
+
shifted = False
|
|
99
|
+
if isinstance(bound_sensitivity, dict) and len(bound_sensitivity) >= 2:
|
|
100
|
+
vals = _as_floats(bound_sensitivity.values())
|
|
101
|
+
if vals:
|
|
102
|
+
shift = max(vals) - min(vals)
|
|
103
|
+
shifted = shift > _SHIFT_TOL
|
|
104
|
+
evidence.append(
|
|
105
|
+
f"Bound sensitivity over {len(vals)} settings, "
|
|
106
|
+
f"shift {shift:.3g} "
|
|
107
|
+
f"({'significant' if shifted else 'small'})."
|
|
108
|
+
)
|
|
109
|
+
elif bound_sensitivity is not None:
|
|
110
|
+
evidence.append("Bound sensitivity incomplete (need 2+ settings).")
|
|
111
|
+
|
|
112
|
+
if grad is not None:
|
|
113
|
+
try:
|
|
114
|
+
evidence.append(f"Gradient at solution {float(grad):.3g}.")
|
|
115
|
+
except (TypeError, ValueError):
|
|
116
|
+
pass
|
|
117
|
+
grad_large = grad is not None and float(grad) >= _GRAD_TOL
|
|
118
|
+
grad_small = grad is not None and float(grad) < _GRAD_TOL
|
|
119
|
+
|
|
120
|
+
if tt is not None:
|
|
121
|
+
try:
|
|
122
|
+
evidence.append(f"Panel length T={tt}.")
|
|
123
|
+
except (TypeError, ValueError):
|
|
124
|
+
pass
|
|
125
|
+
|
|
126
|
+
if not identical:
|
|
127
|
+
classification = "MULTI_BASIN"
|
|
128
|
+
action = "Perbanyak seed dan perbaiki starting value."
|
|
129
|
+
elif shifted and grad_small:
|
|
130
|
+
classification = "ARTIFICIAL_BOUND"
|
|
131
|
+
action = "Longgarkan bound berdasar bukti sensitivitas."
|
|
132
|
+
elif (not shifted) and grad_large and shift is not None:
|
|
133
|
+
classification = "STRUCTURAL_BOUNDARY"
|
|
134
|
+
action = (
|
|
135
|
+
"Wajib lapor ganda ML dan penalized/Bayes modal; "
|
|
136
|
+
"larang cosmetic fix."
|
|
137
|
+
)
|
|
138
|
+
elif shift is None and grad_large:
|
|
139
|
+
classification = "STRUCTURAL_BOUNDARY"
|
|
140
|
+
action = (
|
|
141
|
+
"Wajib lapor ganda ML dan penalized/Bayes modal; "
|
|
142
|
+
"larang cosmetic fix (sensitivitas bound disarankan)."
|
|
143
|
+
)
|
|
144
|
+
elif grad_large and shifted:
|
|
145
|
+
classification = "STRUCTURAL_BOUNDARY"
|
|
146
|
+
action = (
|
|
147
|
+
"Wajib lapor ganda ML dan penalized/Bayes modal; "
|
|
148
|
+
"larang cosmetic fix."
|
|
149
|
+
)
|
|
150
|
+
else:
|
|
151
|
+
classification = "INDETERMINATE"
|
|
152
|
+
action = "Lengkapi bukti (seed, sensitivitas bound, gradien)."
|
|
153
|
+
|
|
154
|
+
try:
|
|
155
|
+
if tt is not None and float(tt) < float(t_warn):
|
|
156
|
+
evidence.append(
|
|
157
|
+
f"T={tt} < {t_warn}: warning Nickell bias and pile-up "
|
|
158
|
+
"pada panel pendek."
|
|
159
|
+
)
|
|
160
|
+
except (TypeError, ValueError):
|
|
161
|
+
pass
|
|
162
|
+
|
|
163
|
+
return BoundaryDiagnosis(
|
|
164
|
+
classification=classification,
|
|
165
|
+
evidence=evidence,
|
|
166
|
+
recommended_action=action,
|
|
167
|
+
citations=list(_CITATIONS),
|
|
168
|
+
)
|