stmmlib 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stmmlib/__init__.py +30 -0
- stmmlib/_backend.py +153 -0
- stmmlib/_utils.py +175 -0
- stmmlib/beta/__init__.py +17 -0
- stmmlib/beta/config.py +108 -0
- stmmlib/beta/criteria.py +247 -0
- stmmlib/beta/design.py +222 -0
- stmmlib/beta/diagnose.py +168 -0
- stmmlib/beta/diagnostics.py +94 -0
- stmmlib/beta/fit.py +598 -0
- stmmlib/beta/gpu_solvers.py +169 -0
- stmmlib/beta/inference.py +403 -0
- stmmlib/beta/lrt_fixed.py +551 -0
- stmmlib/beta/moran_panel.py +733 -0
- stmmlib/beta/plot_data.py +252 -0
- stmmlib/beta/precision.py +98 -0
- stmmlib/beta/report.py +373 -0
- stmmlib/beta/result.py +696 -0
- stmmlib/beta/simulation.py +343 -0
- stmmlib/beta/solvers.py +2021 -0
- stmmlib/beta/structure.py +56 -0
- stmmlib/diagnostics_suite.py +663 -0
- stmmlib/gaussian/__init__.py +1 -0
- stmmlib/gaussian/blup.py +252 -0
- stmmlib/gaussian/covariance.py +599 -0
- stmmlib/gaussian/design.py +312 -0
- stmmlib/gaussian/diagnostics.py +308 -0
- stmmlib/gaussian/gls.py +285 -0
- stmmlib/gaussian/hessian.py +197 -0
- stmmlib/gaussian/lrt_fixed.py +289 -0
- stmmlib/gaussian/models.py +528 -0
- stmmlib/gaussian/optimization.py +361 -0
- stmmlib/gaussian/simulation.py +186 -0
- stmmlib/gaussian/summary.py +277 -0
- stmmlib/gaussian/transforms.py +419 -0
- stmmlib/gaussian/variance_decomp.py +191 -0
- stmmlib/lrt.py +74 -0
- stmmlib/output_format.py +148 -0
- stmmlib/output_format_beta.py +299 -0
- stmmlib/output_format_normal.py +118 -0
- stmmlib/registry.py +326 -0
- stmmlib/stmm.py +265 -0
- stmmlib-0.2.0.dist-info/METADATA +215 -0
- stmmlib-0.2.0.dist-info/RECORD +47 -0
- stmmlib-0.2.0.dist-info/WHEEL +5 -0
- stmmlib-0.2.0.dist-info/licenses/LICENSE +21 -0
- stmmlib-0.2.0.dist-info/top_level.txt +1 -0
stmmlib/beta/fit.py
ADDED
|
@@ -0,0 +1,598 @@
|
|
|
1
|
+
"""Top-level fitting functions: stmm_beta(), betaml().
|
|
2
|
+
|
|
3
|
+
Orchestrates data prep, internal name mapping, optimization,
|
|
4
|
+
and returns appropriate result object.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from typing import TYPE_CHECKING
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
if TYPE_CHECKING:
|
|
15
|
+
from stmmlib._backend import Backend
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _sanitize_data(data: pd.DataFrame,
|
|
19
|
+
response_col: str = "y",
|
|
20
|
+
area_col: str = "area",
|
|
21
|
+
time_col: str = "time",
|
|
22
|
+
prov_col: str | None = "prov",
|
|
23
|
+
kab_col: str | None = "kab",
|
|
24
|
+
) -> tuple[pd.DataFrame, dict, dict]:
|
|
25
|
+
"""Rename user columns to internal names and copy data.
|
|
26
|
+
|
|
27
|
+
Arbitrary source column names are mapped to the internal convention:
|
|
28
|
+
response→``y``, area→``.stmm_area``, time→``.stmm_time``,
|
|
29
|
+
prov→``.stmm_prov``, kab→``.stmm_kab``. Any dataset works as long as the
|
|
30
|
+
caller supplies the column names (or uses the conventional defaults).
|
|
31
|
+
"""
|
|
32
|
+
col_map = {}
|
|
33
|
+
for internal, original in ((".stmm_area", area_col), (".stmm_time", time_col),
|
|
34
|
+
(".stmm_prov", prov_col), (".stmm_kab", kab_col)):
|
|
35
|
+
if original and original in data.columns and original != internal:
|
|
36
|
+
col_map[original] = internal
|
|
37
|
+
|
|
38
|
+
# Labels must match the codes assigned below. pd.Categorical sorts the
|
|
39
|
+
# categories, so code k = k-th name in sorted order (not file order).
|
|
40
|
+
area_labels = {}
|
|
41
|
+
if area_col and area_col in data.columns:
|
|
42
|
+
area_labels = dict(enumerate(pd.Categorical(data[area_col]).categories.tolist()))
|
|
43
|
+
time_labels = {}
|
|
44
|
+
if time_col and time_col in data.columns:
|
|
45
|
+
time_labels = dict(enumerate(sorted(data[time_col].unique().tolist())))
|
|
46
|
+
|
|
47
|
+
if response_col and response_col in data.columns:
|
|
48
|
+
if response_col != "y" and "y" in data.columns:
|
|
49
|
+
raise ValueError(
|
|
50
|
+
f"kolom prediktor 'y' bertabrakan dengan nama internal "
|
|
51
|
+
f"respons '{response_col}'. Ganti nama kolom prediktor "
|
|
52
|
+
"'y' menjadi nama lain, atau pilih response_col lain.")
|
|
53
|
+
col_map[response_col] = "y"
|
|
54
|
+
|
|
55
|
+
d = data.copy()
|
|
56
|
+
if col_map:
|
|
57
|
+
d = d.rename(columns=col_map)
|
|
58
|
+
|
|
59
|
+
# Create categorical indices for area/time/prov/kab
|
|
60
|
+
for c in (".stmm_area", ".stmm_time", ".stmm_prov", ".stmm_kab"):
|
|
61
|
+
if c in d.columns:
|
|
62
|
+
d[c] = pd.Categorical(d[c]).codes.astype(np.int32)
|
|
63
|
+
|
|
64
|
+
return d, area_labels, time_labels
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _validate_contract(d: pd.DataFrame, W=None, area_labels=None) -> None:
|
|
68
|
+
"""Validate input contract before optimization.
|
|
69
|
+
|
|
70
|
+
W ordering contract: row/col i of W corresponds to the i-th area in
|
|
71
|
+
SORTED order (pd.Categorical sorts categories before assigning codes).
|
|
72
|
+
Keep W rows/cols aligned to that order; reordering areas without
|
|
73
|
+
reordering W silently misassigns spatial neighbours. The order cannot be
|
|
74
|
+
checked from the codes alone, so the user gets a warning with the
|
|
75
|
+
expected order.
|
|
76
|
+
|
|
77
|
+
Checks:
|
|
78
|
+
- W (when given) is square, finite, and matches n unique areas.
|
|
79
|
+
- time codes are contiguous 0..T-1.
|
|
80
|
+
- warns (UserWarning) when n < 10*p (p = # X cols incl. intercept).
|
|
81
|
+
"""
|
|
82
|
+
import warnings
|
|
83
|
+
if ".stmm_area" in d.columns:
|
|
84
|
+
n_area = int(d[".stmm_area"].nunique())
|
|
85
|
+
if W is not None:
|
|
86
|
+
W_arr = W.toarray() if hasattr(W, "toarray") else np.asarray(W, dtype=np.float64)
|
|
87
|
+
W_arr = np.asarray(W_arr, dtype=float)
|
|
88
|
+
if W_arr.ndim != 2 or W_arr.shape[0] != W_arr.shape[1]:
|
|
89
|
+
raise ValueError(f"W must be square, got shape {W_arr.shape}.")
|
|
90
|
+
if not np.all(np.isfinite(W_arr)):
|
|
91
|
+
raise ValueError("W must be finite (no NaN/inf).")
|
|
92
|
+
if W_arr.shape[0] != n_area:
|
|
93
|
+
raise ValueError(
|
|
94
|
+
f"W shape {W_arr.shape} mismatches n areas {n_area}.")
|
|
95
|
+
_first = ""
|
|
96
|
+
if area_labels:
|
|
97
|
+
_labs = [area_labels[k] for k in sorted(area_labels)]
|
|
98
|
+
_first = " (urutan yang diharapkan: " + \
|
|
99
|
+
", ".join(str(x) for x in _labs[:5]) + \
|
|
100
|
+
(", ..." if len(_labs) > 5 else "") + ")"
|
|
101
|
+
warnings.warn(
|
|
102
|
+
"W harus diurutkan sesuai sorted(area): baris/kolom i "
|
|
103
|
+
"adalah area ke-i menurut label terurut" + _first +
|
|
104
|
+
". W yang disusun dalam urutan file akan menggeser "
|
|
105
|
+
"tetangga secara diam-diam.",
|
|
106
|
+
UserWarning, stacklevel=3)
|
|
107
|
+
if ".stmm_area" in d.columns:
|
|
108
|
+
a = np.asarray(d[".stmm_area"].values)
|
|
109
|
+
if len(a):
|
|
110
|
+
if int(a.max()) - int(a.min()) + 1 != int(np.unique(a).size):
|
|
111
|
+
raise ValueError("area codes must be contiguous 0..m-1.")
|
|
112
|
+
if ".stmm_time" in d.columns:
|
|
113
|
+
t = np.asarray(d[".stmm_time"].values)
|
|
114
|
+
if len(t):
|
|
115
|
+
if int(t.max()) - int(t.min()) + 1 != int(np.unique(t).size):
|
|
116
|
+
raise ValueError("time codes must be contiguous 0..T-1.")
|
|
117
|
+
skip_x = {".stmm_area", ".stmm_time", ".stmm_prov", ".stmm_kab", "y"}
|
|
118
|
+
x_cols = [c for c in d.columns if c not in skip_x]
|
|
119
|
+
p = len(x_cols) + 1
|
|
120
|
+
if len(d) < 10 * p:
|
|
121
|
+
warnings.warn(f"n={len(d)} < 10*p={10 * p}: small-sample fit.",
|
|
122
|
+
UserWarning, stacklevel=3)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _auto_prior_enabled(config) -> bool:
|
|
126
|
+
"""Whether the automatic weak-prior boundary refit may run.
|
|
127
|
+
|
|
128
|
+
Historical rule: the refit runs only when no explicit config was
|
|
129
|
+
passed. An explicit config decides on its own through `auto_prior`
|
|
130
|
+
(True always, False never, None keeps the historical rule).
|
|
131
|
+
"""
|
|
132
|
+
if config is None:
|
|
133
|
+
return True
|
|
134
|
+
return getattr(config, "auto_prior", None) is True
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def stmm_beta(
|
|
138
|
+
formula: str = None,
|
|
139
|
+
data: pd.DataFrame = None,
|
|
140
|
+
model: str = "m1_iid_iid",
|
|
141
|
+
W=None,
|
|
142
|
+
method: str = "Laplace",
|
|
143
|
+
gpu: str = "auto",
|
|
144
|
+
n_starts: int = 3,
|
|
145
|
+
max_iter: int = 200,
|
|
146
|
+
inner_max_iter: int = 50,
|
|
147
|
+
start_theta: "np.ndarray | None" = None,
|
|
148
|
+
seed: int | None = None,
|
|
149
|
+
compute_hessian: bool = True,
|
|
150
|
+
preset: str | None = None,
|
|
151
|
+
response_col: str = "y",
|
|
152
|
+
area_col: str = "area",
|
|
153
|
+
time_col: str = "time",
|
|
154
|
+
prov_col: str | None = "prov",
|
|
155
|
+
kab_col: str | None = "kab",
|
|
156
|
+
config=None,
|
|
157
|
+
compute_posterior: bool = False,
|
|
158
|
+
inference: str = "wald",
|
|
159
|
+
lrt_terms: "list | None" = None,
|
|
160
|
+
lrt_deadline_s: "float | None" = None,
|
|
161
|
+
**kwargs,
|
|
162
|
+
) -> "BetastmmResult":
|
|
163
|
+
"""Fit a spatio-temporal Beta mixed model.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
formula: Formula string (currently unused; X from data columns
|
|
167
|
+
minus the grouping/response columns).
|
|
168
|
+
data: pandas DataFrame. Response must be a proportion in (0,1).
|
|
169
|
+
model: Model ID (e.g. "m1_iid_iid", "m3_sar_iid", "m5_seblup_rao_yu_it").
|
|
170
|
+
W: Spatial weight matrix (scipy.sparse or numpy array).
|
|
171
|
+
method: "Laplace" (default) or "ML".
|
|
172
|
+
gpu: "auto", "yes", or "no".
|
|
173
|
+
n_starts: Number of random restarts.
|
|
174
|
+
max_iter: Outer optimization iterations.
|
|
175
|
+
inner_max_iter: Inner Newton iterations.
|
|
176
|
+
compute_hessian: finite-difference Hessian at the optimum (for the
|
|
177
|
+
certificate + SEs). Set False for a fast fit (skips ~2k+1 solves).
|
|
178
|
+
response_col: name of the (0,1)-bounded response column.
|
|
179
|
+
area_col: name of the area/grouping column.
|
|
180
|
+
time_col: name of the time/period column.
|
|
181
|
+
prov_col: optional name of the province column (nested models).
|
|
182
|
+
kab_col: optional name of the district column (nested models).
|
|
183
|
+
preset: "fast" (low: ftol=1e-5, gtol=1e-5, no Hessian, max_iter
|
|
184
|
+
cap 500), "balanced" (medium tol: ftol=1e-8, gtol=1e-6, Hessian,
|
|
185
|
+
max_iter cap 500; cheap outer + honest SEs), "medium" (standard:
|
|
186
|
+
ftol=1e-8, gtol=1e-6, no Hessian, no cap) or "full"
|
|
187
|
+
(high/scientific: ftol=1e-12, gtol=1e-6, Hessian; default).
|
|
188
|
+
See `stmm()` docstring for journal anchors.
|
|
189
|
+
config: BetaFitConfig. Opt-in weak-prior MAP penalty (default off):
|
|
190
|
+
phi_prior_mean/sd on logphi, dep_prior_sd on raw_dep. MAP with
|
|
191
|
+
weak Gaussian prior; precedent Chung et al. 2013 blme and brglm2
|
|
192
|
+
MPL Jeffreys. Always report penalized alongside unpenalized.
|
|
193
|
+
Auto MAP refit on boundary: when config is None (no explicit
|
|
194
|
+
config sent) and the unpenalized fit is invalid with log_phi or
|
|
195
|
+
raw_dep parameters at their box bounds, stmm_beta refits ONCE
|
|
196
|
+
with an automatic weak prior (phi_prior_mean=6.9, sd=2.0;
|
|
197
|
+
dep off; blme-style, Chung et al. 2013). The
|
|
198
|
+
unpenalized result is kept in optim["unpenalized"]; disable the
|
|
199
|
+
auto by passing an explicit config (even BetaFitConfig()).
|
|
200
|
+
compute_posterior: store the Laplace posterior curvature of the random
|
|
201
|
+
effects (H_b = Z'diag(w)Z + Q) as a Cholesky factor, needed by
|
|
202
|
+
BetastmmResult.ebp(). Orthogonal to `config` (no effect on the
|
|
203
|
+
auto MAP refit); adds q^2 memory per fit, no extra refit.
|
|
204
|
+
inference: fixed-effect test method. "wald" (default) reports the
|
|
205
|
+
Hessian-based Wald z test. "lrt" additionally re-fits the model
|
|
206
|
+
with each tested predictor dropped and reports the likelihood
|
|
207
|
+
ratio test (D = 2*(ll_full - ll_red), df = 1). The Wald
|
|
208
|
+
p-value is always kept in inference["pvalue_wald"].
|
|
209
|
+
lrt_terms: predictors to test by LRT. None tests every predictor,
|
|
210
|
+
which costs one refit per predictor.
|
|
211
|
+
lrt_deadline_s: wall-clock budget for the whole LRT block. Terms
|
|
212
|
+
left over when it runs out are reported as "skipped_deadline".
|
|
213
|
+
**kwargs: Additional arguments.
|
|
214
|
+
|
|
215
|
+
Returns:
|
|
216
|
+
BetastmmResult with coefficients, inference, logLik, AIC, BIC.
|
|
217
|
+
"""
|
|
218
|
+
# Accuracy presets (L-BFGS-B tolerances; journal anchors in docstring of
|
|
219
|
+
# `stmm()`): fast/low (ftol=1e-5, gtol=1e-5, no Hessian, cap 500),
|
|
220
|
+
# balanced/medium-tol (ftol=1e-8, gtol=1e-6, Hessian, cap 500),
|
|
221
|
+
# medium/standard (ftol=1e-8, gtol=1e-6, no Hessian, no cap),
|
|
222
|
+
# full/high (ftol=1e-12, gtol=1e-6, Hessian, no cap -- original behavior).
|
|
223
|
+
# Only the Hessian flag changes cost structure: the Hessian is used for
|
|
224
|
+
# SEs/lambda_min, never for the optimization path (benchmark: identical
|
|
225
|
+
# logLik/theta/cAIC, delta <= 1e-12). No Hessian -> no SEs, hessian_pd
|
|
226
|
+
# None in the certificate, and predict_interval loses var_fixed.
|
|
227
|
+
from stmmlib.beta.config import resolve_preset
|
|
228
|
+
_ftol, _gtol, _hess, _cap = resolve_preset(preset)
|
|
229
|
+
compute_hessian = compute_hessian and _hess
|
|
230
|
+
if _cap is not None:
|
|
231
|
+
max_iter = min(max_iter, _cap)
|
|
232
|
+
if inference not in ("wald", "lrt"):
|
|
233
|
+
raise ValueError(
|
|
234
|
+
f"inference must be 'wald' or 'lrt', got {inference!r}.")
|
|
235
|
+
from stmmlib._backend import Backend
|
|
236
|
+
backend = Backend(gpu=gpu)
|
|
237
|
+
|
|
238
|
+
if data is None:
|
|
239
|
+
raise ValueError("data must be provided.")
|
|
240
|
+
if response_col not in data.columns and "y" not in data.columns:
|
|
241
|
+
raise ValueError(
|
|
242
|
+
f"response column {response_col!r} not found; pass response_col=<name>.")
|
|
243
|
+
|
|
244
|
+
d, area_labels, time_labels = _sanitize_data(
|
|
245
|
+
data, response_col=response_col, area_col=area_col, time_col=time_col,
|
|
246
|
+
prov_col=prov_col, kab_col=kab_col)
|
|
247
|
+
if "y" not in d.columns:
|
|
248
|
+
raise ValueError(f"response column {response_col!r} not found in data.")
|
|
249
|
+
_validate_contract(d, W, area_labels)
|
|
250
|
+
|
|
251
|
+
# Input guards. A new dataset must fail here, with a message that names
|
|
252
|
+
# the problem, not inside the optimizer as "semua start gagal".
|
|
253
|
+
_y = d["y"].values.astype(np.float64)
|
|
254
|
+
if not np.all(np.isfinite(_y)):
|
|
255
|
+
raise ValueError(
|
|
256
|
+
f"respon {response_col!r} mengandung NaN/Inf; bersihkan data "
|
|
257
|
+
"sebelum fitting.")
|
|
258
|
+
if not np.all((_y > 0.0) & (_y < 1.0)):
|
|
259
|
+
raise ValueError(
|
|
260
|
+
f"respon Beta harus proporsi di (0,1); ditemukan nilai di luar "
|
|
261
|
+
"interval. Data persen 0-100 harus dibagi 100 lebih dulu.")
|
|
262
|
+
_skip_x = {"y", ".stmm_area", ".stmm_time", ".stmm_prov", ".stmm_kab"}
|
|
263
|
+
_pred_cols = [c for c in d.columns if c not in _skip_x]
|
|
264
|
+
for _c in _pred_cols:
|
|
265
|
+
try:
|
|
266
|
+
_col = d[_c].to_numpy(dtype=float)
|
|
267
|
+
except (TypeError, ValueError):
|
|
268
|
+
raise ValueError(
|
|
269
|
+
f"prediktor {_c!r} bukan numerik; ubah ke numerik atau "
|
|
270
|
+
"keluarkan dari data sebelum fitting.") from None
|
|
271
|
+
if not np.all(np.isfinite(_col)):
|
|
272
|
+
raise ValueError(
|
|
273
|
+
f"prediktor {_c!r} mengandung NaN/Inf; bersihkan atau "
|
|
274
|
+
"hilangkan kolom bermasalah sebelum fitting.")
|
|
275
|
+
|
|
276
|
+
# T-aware warning for AR(1) temporal structures (Nickell 1981: AR(1)
|
|
277
|
+
# persistence is weakly identified on short panels; IID time is the
|
|
278
|
+
# safer comparator). Fires at most once per fit.
|
|
279
|
+
import warnings
|
|
280
|
+
from stmmlib.beta.structure import stmm_beta_structure
|
|
281
|
+
try:
|
|
282
|
+
_time_struct = stmm_beta_structure(model)["time"]
|
|
283
|
+
except KeyError:
|
|
284
|
+
_time_struct = None
|
|
285
|
+
# Balance note: the design builds one cell dummy per area x period. A
|
|
286
|
+
# replicated panel (several rows per cell) and an unbalanced panel (empty
|
|
287
|
+
# cells) are both supported: an empty cell carries no information about
|
|
288
|
+
# its own effect. The state is reported, not blocked, and
|
|
289
|
+
# diagnostics_suite.validate_assumptions gives the full picture.
|
|
290
|
+
if (".stmm_area" in d.columns and ".stmm_time" in d.columns
|
|
291
|
+
and _time_struct is not None):
|
|
292
|
+
try:
|
|
293
|
+
_spec = stmm_beta_structure(model)
|
|
294
|
+
except KeyError:
|
|
295
|
+
_spec = None
|
|
296
|
+
if _spec is not None:
|
|
297
|
+
_m = int(d[".stmm_area"].nunique())
|
|
298
|
+
_T = int(d[".stmm_time"].nunique())
|
|
299
|
+
_cells = _m * _T
|
|
300
|
+
if len(d) > _cells:
|
|
301
|
+
warnings.warn(
|
|
302
|
+
f"panel terduplikasi: {len(d)} baris untuk {_cells} sel "
|
|
303
|
+
f"({_m} area x {_T} waktu).", UserWarning, stacklevel=3)
|
|
304
|
+
elif len(d) < _cells:
|
|
305
|
+
warnings.warn(
|
|
306
|
+
f"panel tidak seimbang: {len(d)} baris untuk {_cells} sel "
|
|
307
|
+
f"({_m} area x {_T} waktu); sel kosong tidak memberi "
|
|
308
|
+
"informasi tentang efek selnya.", UserWarning, stacklevel=3)
|
|
309
|
+
if _time_struct == "ar1" and ".stmm_time" in d.columns:
|
|
310
|
+
_T = int(d[".stmm_time"].nunique())
|
|
311
|
+
if _T < 15:
|
|
312
|
+
warnings.warn(
|
|
313
|
+
"AR(1) temporal pada panel T pendek (<15): persistensi lemah "
|
|
314
|
+
"teridentifikasi (Nickell 1981, Econometrica 49(6)); "
|
|
315
|
+
"pertimbangkan pembanding varian waktu IID",
|
|
316
|
+
UserWarning, stacklevel=2)
|
|
317
|
+
y = d["y"].values.astype(np.float64)
|
|
318
|
+
|
|
319
|
+
# Drop y and grouping columns from X columns
|
|
320
|
+
drop_cols = {"y", ".stmm_area", ".stmm_time", ".stmm_prov",
|
|
321
|
+
".stmm_kab"}
|
|
322
|
+
|
|
323
|
+
if model == "beta_regression":
|
|
324
|
+
if inference != "wald":
|
|
325
|
+
raise ValueError(
|
|
326
|
+
"inference='lrt' is not available for beta_regression: the "
|
|
327
|
+
"fixed-effects-only model has no random-effect structure to "
|
|
328
|
+
"refit. Use family='beta' with a model id from M1 to M14.")
|
|
329
|
+
from stmmlib.beta.solvers import beta_regression_base
|
|
330
|
+
opt = beta_regression_base(formula, d, y, method)
|
|
331
|
+
from stmmlib.beta.result import BetaResult
|
|
332
|
+
return BetaResult(opt)
|
|
333
|
+
|
|
334
|
+
from stmmlib.beta.solvers import beta_laplace_structural
|
|
335
|
+
opt = beta_laplace_structural(
|
|
336
|
+
formula, d, y, method, model, W,
|
|
337
|
+
n_starts=n_starts, max_iter=max_iter,
|
|
338
|
+
inner_max_iter=inner_max_iter, backend=backend,
|
|
339
|
+
use_gpu=(str(gpu).lower() == "yes"),
|
|
340
|
+
start_theta=start_theta, seed=seed,
|
|
341
|
+
compute_hessian=compute_hessian,
|
|
342
|
+
ftol=_ftol, gtol=_gtol, config=config,
|
|
343
|
+
compute_posterior=compute_posterior,
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
# Auto weak-prior MAP refit (blme-style, Chung et al. 2013): when the
|
|
347
|
+
# unpenalized fit hits the log_phi/raw_dep box boundary and the
|
|
348
|
+
# certificate goes invalid, refit ONCE with a weak Gaussian prior.
|
|
349
|
+
# Trigger requires config=None (an explicit config, even
|
|
350
|
+
# BetaFitConfig(), disables the auto). The unpenalized snapshot is
|
|
351
|
+
# kept in optim["unpenalized"]; no recursion (the refit passes a
|
|
352
|
+
# config, so this block cannot fire again).
|
|
353
|
+
# Isolasi hasil utama vs refit: copy final SEBELUM refit kedua jalan.
|
|
354
|
+
# _opt_unpen = opt adalah alias; copy putus rantai view/reference.
|
|
355
|
+
import copy as _copy
|
|
356
|
+
|
|
357
|
+
def _snap(a):
|
|
358
|
+
return np.array(a, dtype=np.float64, copy=True) if a is not None else None
|
|
359
|
+
_main_X = _snap(opt.get("X"))
|
|
360
|
+
_main_Z = _snap(opt.get("Z"))
|
|
361
|
+
_main_beta = _snap(opt.get("beta_hat"))
|
|
362
|
+
_main_b = _snap(opt.get("b_hat"))
|
|
363
|
+
_main_logs = _snap(opt.get("logs"))
|
|
364
|
+
_main_raw = _snap(opt.get("raw_dep"))
|
|
365
|
+
_main_boot = _snap(opt.get("_boot_theta"))
|
|
366
|
+
_main_hb = _snap(opt.get("H_b_chol"))
|
|
367
|
+
_main_des = _copy.deepcopy(opt.get("des")) if opt.get("des") is not None else None
|
|
368
|
+
|
|
369
|
+
def _restore_main(o):
|
|
370
|
+
if _main_X is not None:
|
|
371
|
+
o["X"] = _main_X
|
|
372
|
+
if _main_Z is not None:
|
|
373
|
+
o["Z"] = _main_Z
|
|
374
|
+
if _main_beta is not None:
|
|
375
|
+
o["beta_hat"] = _main_beta
|
|
376
|
+
if _main_b is not None:
|
|
377
|
+
o["b_hat"] = _main_b
|
|
378
|
+
if _main_logs is not None:
|
|
379
|
+
o["logs"] = _main_logs
|
|
380
|
+
if _main_raw is not None:
|
|
381
|
+
o["raw_dep"] = _main_raw
|
|
382
|
+
if _main_boot is not None:
|
|
383
|
+
o["_boot_theta"] = _main_boot
|
|
384
|
+
if _main_hb is not None:
|
|
385
|
+
o["H_b_chol"] = _main_hb
|
|
386
|
+
if _main_des is not None:
|
|
387
|
+
o["des"] = _main_des
|
|
388
|
+
try:
|
|
389
|
+
o["des"]["Z"] = o["Z"]
|
|
390
|
+
except Exception:
|
|
391
|
+
pass
|
|
392
|
+
return o
|
|
393
|
+
from stmmlib.beta.config import (BetaFitConfig, DEP_PRIOR_AUTO_SD,
|
|
394
|
+
PHI_PRIOR_AUTO_MEAN, PHI_PRIOR_AUTO_SD)
|
|
395
|
+
_cert = opt.get("certificate") or {}
|
|
396
|
+
if (_auto_prior_enabled(config) and _cert.get("valid") is False
|
|
397
|
+
and _cert.get("at_bound") is True
|
|
398
|
+
and any(b == "log_phi" or str(b).startswith("raw_dep")
|
|
399
|
+
for b in (_cert.get("bound_params") or []))
|
|
400
|
+
and "penalty_auto" not in opt):
|
|
401
|
+
_raw = opt.get("raw_dep")
|
|
402
|
+
_unpen = {
|
|
403
|
+
"log_lik": opt.get("log_lik"),
|
|
404
|
+
"logphi": opt.get("logphi"),
|
|
405
|
+
"raw_dep": list(np.asarray(_raw).ravel()) if _raw is not None else [],
|
|
406
|
+
"status": opt.get("status"),
|
|
407
|
+
"cert_valid": _cert.get("valid"),
|
|
408
|
+
}
|
|
409
|
+
_auto_cfg = BetaFitConfig(phi_prior_mean=PHI_PRIOR_AUTO_MEAN,
|
|
410
|
+
phi_prior_sd=PHI_PRIOR_AUTO_SD,
|
|
411
|
+
dep_prior_sd=DEP_PRIOR_AUTO_SD,
|
|
412
|
+
compute_posterior=compute_posterior)
|
|
413
|
+
_opt_unpen = opt
|
|
414
|
+
_opt_pen = beta_laplace_structural(
|
|
415
|
+
formula, d, y, method, model, W,
|
|
416
|
+
n_starts=n_starts, max_iter=max_iter,
|
|
417
|
+
inner_max_iter=inner_max_iter, backend=backend,
|
|
418
|
+
use_gpu=(str(gpu).lower() == "yes"),
|
|
419
|
+
start_theta=start_theta, seed=seed,
|
|
420
|
+
compute_hessian=compute_hessian,
|
|
421
|
+
ftol=_ftol, gtol=_gtol, config=_auto_cfg,
|
|
422
|
+
)
|
|
423
|
+
# Seleksi validity-first (prioritas):
|
|
424
|
+
# 1. VALID-penalized menang atas unpenalized INVALID apapun ll-nya.
|
|
425
|
+
# 2. Sesama VALID menang ll UNPENALIZED tertinggi (eval tanpa penalti).
|
|
426
|
+
# 3. Penalized INVALID/abort/it=0/degenerate -> pakai unpenalized awal.
|
|
427
|
+
# Satu sumber angka: X = ll unpenalized di titik awal,
|
|
428
|
+
# Y = ll unpenalized di titik penalized (eval tanpa penalti).
|
|
429
|
+
# Bila Y fallback ke objective penalized, label eksplisit obj penalized.
|
|
430
|
+
_ll_unpen = float(_opt_unpen.get("log_lik", float("nan")))
|
|
431
|
+
_pen_theta = _opt_pen.get("_boot_theta")
|
|
432
|
+
_ll_pen_unpen = float("nan")
|
|
433
|
+
_y_from_obj = False
|
|
434
|
+
if _pen_theta is not None:
|
|
435
|
+
try:
|
|
436
|
+
_ev = beta_laplace_structural(
|
|
437
|
+
formula, d, y, method, model, W, config=BetaFitConfig(),
|
|
438
|
+
compute_hessian=False, eval_param=np.asarray(_pen_theta, float))
|
|
439
|
+
_ll_pen_unpen = float(_ev.get("log_lik", float("nan")))
|
|
440
|
+
except Exception:
|
|
441
|
+
_ll_pen_unpen = float("nan")
|
|
442
|
+
if not np.isfinite(_ll_pen_unpen):
|
|
443
|
+
try:
|
|
444
|
+
_ll_pen_unpen = float(_opt_pen.get("log_lik", float("nan")))
|
|
445
|
+
_y_from_obj = bool(np.isfinite(_ll_pen_unpen))
|
|
446
|
+
except Exception:
|
|
447
|
+
_ll_pen_unpen = float("nan")
|
|
448
|
+
def _fll(v):
|
|
449
|
+
return f"{v:.4f}" if np.isfinite(v) else "NA"
|
|
450
|
+
_y_lab = ("obj penalized di titik refit" if _y_from_obj
|
|
451
|
+
else "ll tanpa penalti di titik refit")
|
|
452
|
+
_pen_cert = _opt_pen.get("certificate") or {}
|
|
453
|
+
_pen_optim = _opt_pen.get("optim") or {}
|
|
454
|
+
try:
|
|
455
|
+
_pen_nit = int(_pen_optim.get("nit", 1) or 0)
|
|
456
|
+
except (TypeError, ValueError):
|
|
457
|
+
_pen_nit = 1
|
|
458
|
+
_pen_valid = (
|
|
459
|
+
_pen_cert.get("valid") is True
|
|
460
|
+
and not _opt_pen.get("degenerate", False)
|
|
461
|
+
and not _pen_optim.get("all_starts_abort", False)
|
|
462
|
+
and _opt_pen.get("status") != "invalid"
|
|
463
|
+
and _pen_nit != 0
|
|
464
|
+
and np.isfinite(float(_opt_pen.get("log_lik", float("nan"))))
|
|
465
|
+
)
|
|
466
|
+
_unpen_valid = (_cert.get("valid") is True)
|
|
467
|
+
if _pen_valid and not _unpen_valid:
|
|
468
|
+
opt = _opt_pen
|
|
469
|
+
_cert = opt.get("certificate") or {}
|
|
470
|
+
_optim = opt.setdefault("optim", {})
|
|
471
|
+
_optim["penalty_auto"] = True
|
|
472
|
+
_optim["unpenalized"] = _unpen
|
|
473
|
+
_msg = (f"auto-prior: ll fit utama {_fll(_ll_unpen)} -> "
|
|
474
|
+
f"{_y_lab} {_fll(_ll_pen_unpen)} "
|
|
475
|
+
f"(titik penalized dipakai; unpenalized INVALID)")
|
|
476
|
+
_optim["penalty_refit_msg"] = _msg
|
|
477
|
+
_cert.setdefault("warnings", []).append(_msg)
|
|
478
|
+
elif _pen_valid and _unpen_valid:
|
|
479
|
+
_pen_wins = (np.isfinite(_ll_pen_unpen) and np.isfinite(_ll_unpen)
|
|
480
|
+
and _ll_pen_unpen >= _ll_unpen)
|
|
481
|
+
if _pen_wins:
|
|
482
|
+
opt = _opt_pen
|
|
483
|
+
_cert = opt.get("certificate") or {}
|
|
484
|
+
_optim = opt.setdefault("optim", {})
|
|
485
|
+
_optim["penalty_auto"] = True
|
|
486
|
+
_optim["unpenalized"] = _unpen
|
|
487
|
+
_msg = (f"auto-prior: ll fit utama {_fll(_ll_unpen)} -> "
|
|
488
|
+
f"{_y_lab} {_fll(_ll_pen_unpen)} "
|
|
489
|
+
f"(titik penalized dipakai)")
|
|
490
|
+
_optim["penalty_refit_msg"] = _msg
|
|
491
|
+
_cert.setdefault("warnings", []).append(_msg)
|
|
492
|
+
else:
|
|
493
|
+
opt = _restore_main(_opt_unpen)
|
|
494
|
+
_cert = opt.get("certificate") or {}
|
|
495
|
+
_optim = opt.setdefault("optim", {})
|
|
496
|
+
_optim["penalty_auto"] = True
|
|
497
|
+
_optim["unpenalized"] = _unpen
|
|
498
|
+
_msg = (f"auto-prior: ll fit utama {_fll(_ll_unpen)} -> "
|
|
499
|
+
f"{_y_lab} {_fll(_ll_pen_unpen)}; "
|
|
500
|
+
f"hasil unpenalized dipakai")
|
|
501
|
+
_optim["penalty_refit_msg"] = _msg
|
|
502
|
+
_cert.setdefault("warnings", []).append(_msg)
|
|
503
|
+
else:
|
|
504
|
+
opt = _restore_main(_opt_unpen)
|
|
505
|
+
_cert = opt.get("certificate") or {}
|
|
506
|
+
_optim = opt.setdefault("optim", {})
|
|
507
|
+
_optim["penalty_auto"] = True
|
|
508
|
+
_optim["unpenalized"] = _unpen
|
|
509
|
+
_optim["penalty_refit_failed"] = True
|
|
510
|
+
_msg = (f"refit penalized ditolak: penalized INVALID "
|
|
511
|
+
f"(ll fit utama {_fll(_ll_unpen)} -> "
|
|
512
|
+
f"{_y_lab} {_fll(_ll_pen_unpen)}); "
|
|
513
|
+
f"hasil unpenalized dipakai")
|
|
514
|
+
_optim["penalty_refit_msg"] = _msg
|
|
515
|
+
_cert.setdefault("warnings", []).append(_msg)
|
|
516
|
+
|
|
517
|
+
from stmmlib.beta.inference import compute_stbl_inference
|
|
518
|
+
inf = compute_stbl_inference(opt, skip_hessian=not _hess)
|
|
519
|
+
|
|
520
|
+
# Fixed-effect test method. Wald (Hessian-based z) is the default and
|
|
521
|
+
# always computed. With inference="lrt" each requested predictor is also
|
|
522
|
+
# tested by a restricted refit: drop the column, re-optimise the same
|
|
523
|
+
# model, compare marginal likelihoods (D = 2*(ll_full - ll_red), df=1).
|
|
524
|
+
# The Wald p-value is kept in inf["pvalue_wald"] either way.
|
|
525
|
+
inf["method"] = inference
|
|
526
|
+
inf["test"] = "wald_z" if inference == "wald" else "lrt"
|
|
527
|
+
inf["pvalue_wald"] = np.asarray(inf["pvalue"], dtype=float).copy()
|
|
528
|
+
inf["statistic"] = np.asarray(inf["tvalue"], dtype=float).copy()
|
|
529
|
+
if inference == "lrt":
|
|
530
|
+
from stmmlib.beta.lrt_fixed import lrt_fixed_terms
|
|
531
|
+
if lrt_terms is None:
|
|
532
|
+
import logging
|
|
533
|
+
logging.getLogger(__name__).warning(
|
|
534
|
+
"inference='lrt' without lrt_terms: one restricted refit per "
|
|
535
|
+
"predictor, which is expensive on large panels.")
|
|
536
|
+
_lrt = lrt_fixed_terms(opt, d, y, model, W, lrt_terms, config=config,
|
|
537
|
+
max_iter=max_iter, inner_max_iter=inner_max_iter,
|
|
538
|
+
n_starts=n_starts,
|
|
539
|
+
ftol=_ftol, gtol=_gtol,
|
|
540
|
+
deadline_s=lrt_deadline_s)
|
|
541
|
+
opt["lrt"] = _lrt
|
|
542
|
+
_labels = list(opt["des"]["x_cols"])
|
|
543
|
+
_p = int(opt["p"])
|
|
544
|
+
_pv = np.asarray(inf["pvalue"], dtype=float).copy()
|
|
545
|
+
_st = np.asarray(inf["tvalue"], dtype=float).copy()
|
|
546
|
+
# Only the tested fixed effects carry a p-value under an LRT header.
|
|
547
|
+
# A predictor that was not tested, and the intercept, get NaN; their
|
|
548
|
+
# Wald p-value stays available in pvalue_wald.
|
|
549
|
+
_pv[:_p] = np.nan
|
|
550
|
+
_st[:_p] = np.nan
|
|
551
|
+
for _j, _t in enumerate(_labels):
|
|
552
|
+
_row = _lrt["terms"].get(_t)
|
|
553
|
+
if _row is None or _row["status"] != "ok" or _row["p"] is None:
|
|
554
|
+
continue
|
|
555
|
+
_pv[_j + 1] = _row["p"]
|
|
556
|
+
_st[_j + 1] = np.sign(inf["estimate"][_j + 1]) * np.sqrt(_row["D"])
|
|
557
|
+
inf["pvalue"] = _pv
|
|
558
|
+
inf["statistic"] = _st
|
|
559
|
+
|
|
560
|
+
# Conditional mean (fixed + random effects) for the fitted units.
|
|
561
|
+
X = opt.get("X")
|
|
562
|
+
Z = opt.get("Z")
|
|
563
|
+
b = opt.get("b_hat")
|
|
564
|
+
beta = opt.get("beta_hat")
|
|
565
|
+
if X is not None and beta is not None:
|
|
566
|
+
eta = X @ beta
|
|
567
|
+
if Z is not None and b is not None:
|
|
568
|
+
eta = eta + Z @ b
|
|
569
|
+
opt["mu_hat"] = np.clip(1.0 / (1.0 + np.exp(-eta)), 1e-15, 1.0 - 1e-15)
|
|
570
|
+
|
|
571
|
+
from stmmlib.beta.result import BetastmmResult
|
|
572
|
+
res = BetastmmResult(opt, inf)
|
|
573
|
+
res._area_labels = area_labels
|
|
574
|
+
res._time_labels = time_labels
|
|
575
|
+
res._W = W
|
|
576
|
+
return res
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def betaml(formula=None, data=None, method="ML", gpu="auto"):
|
|
580
|
+
"""Fit a fixed-effects Beta regression via ML (L-BFGS-B).
|
|
581
|
+
|
|
582
|
+
Args:
|
|
583
|
+
formula: Formula (unused; intercept-only for now).
|
|
584
|
+
data: pandas DataFrame with 'y' column in (0,1).
|
|
585
|
+
method: "ML".
|
|
586
|
+
gpu: "auto", "yes", or "no".
|
|
587
|
+
|
|
588
|
+
Returns:
|
|
589
|
+
BetaResult.
|
|
590
|
+
"""
|
|
591
|
+
if data is None or "y" not in data.columns:
|
|
592
|
+
raise ValueError("data must contain a 'y' column.")
|
|
593
|
+
|
|
594
|
+
y = data["y"].values.astype(np.float64)
|
|
595
|
+
from stmmlib.beta.solvers import beta_regression_base
|
|
596
|
+
opt = beta_regression_base(formula, data, y, method, "beta_regression")
|
|
597
|
+
from stmmlib.beta.result import BetaResult
|
|
598
|
+
return BetaResult(opt)
|