SearchLibrium 0.0.268__tar.gz → 0.0.271__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {searchlibrium-0.0.268/src/SearchLibrium.egg-info → searchlibrium-0.0.271}/PKG-INFO +1 -1
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/pyproject.toml +1 -1
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/__init__.py +1 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/larch_models.py +9 -3
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/latent_class.py +86 -0
- searchlibrium-0.0.271/src/SearchLibrium/lc_rail_guard.py +139 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mh_choice_estimator.py +208 -34
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/numba_rrm.py +32 -1
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/rrm.py +129 -2
- searchlibrium-0.0.271/src/SearchLibrium/version.txt +1 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271/src/SearchLibrium.egg-info}/PKG-INFO +1 -1
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/SOURCES.txt +1 -0
- searchlibrium-0.0.268/src/SearchLibrium/version.txt +0 -1
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/LICENSE +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/MANIFEST.in +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/README.md +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/setup.cfg +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/BEHier.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/Halton.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/MixedLogit.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/MixedLogitGSE.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/Mode_Activity_Nested.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/RandomP.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/SEARCH_SM_MARIO.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/Two_Level_Nest.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/__main__.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/_choice_model.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/_device.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/abm.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/accessibility.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/banditsa.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/boxcox_functions.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/call_meta.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/constraints_builder.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/coupled_solver.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/data/Swissmetro_final.csv +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/data/TravelMode.csv +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/data/electricity.csv +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/destination_distances.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/harmony.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/hspbil.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/jax_utils.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/larch_compat.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/latent_spec.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/logistic_regression.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/main.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/main_debug.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mdcev.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/misc.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mixed_logit.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mixed_nested.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mixedrrm.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/multinomial_logit.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/multinomial_nested.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/multinomial_probit.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/numba_engine.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/numba_mxl.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/ordered_logit.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/ordered_logit_mixed.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/predict.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/regularization.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/sample_data.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/sapbil.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/search.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/selection_models.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/setup.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/siman.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/skims.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/sparseea_agds.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_lc_de.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_mario_searches.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_sapbil_vs_banditsa.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/threshold.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/zero_inflated_ordered_probit.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/dependency_links.txt +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/entry_points.txt +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/requires.txt +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/top_level.txt +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_destination_distances.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_jax_mask_equivalence.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_larch_integration.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_lc_smart_init.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_mh_choice_estimator.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_mxl_random_starts.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_per_alternative_random.py +0 -0
- {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_zero_inflated_ordered_probit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SearchLibrium
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.271
|
|
4
4
|
Summary: Automated specification search for discrete choice models (logit, mixed logit, nested logit, RRM and more) using metaheuristic optimisation
|
|
5
5
|
Author: Zeke Ahern, Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
|
|
6
6
|
Maintainer-email: Zeke Ahern <z.ahern@qut.edu.au>
|
|
@@ -132,7 +132,7 @@ Documentation = "https://pypi.org/project/SearchLibrium/"
|
|
|
132
132
|
realpython = "SearchLibrium.__main__:main"
|
|
133
133
|
|
|
134
134
|
[tool.bumpver]
|
|
135
|
-
current_version = "0.0.
|
|
135
|
+
current_version = "0.0.271"
|
|
136
136
|
version_pattern = "MAJOR.MINOR.PATCH"
|
|
137
137
|
commit_message = "[skip ci] Bump version {old_version} -> {new_version}"
|
|
138
138
|
commit = true
|
|
@@ -170,6 +170,7 @@ _IMPORT_GROUPS = [
|
|
|
170
170
|
('zero_inflated_ordered_probit', ['ZeroInflatedOrderedProbit',
|
|
171
171
|
'ZeroInflatedProbit']),
|
|
172
172
|
('latent_class', ['LatentClassMixedLogit']),
|
|
173
|
+
('lc_rail_guard', ['lc_result_rail']),
|
|
173
174
|
('latent_spec', ['vip_table', 'near_unidentified_report', 'align_classes',
|
|
174
175
|
'propose_class_params_spec']),
|
|
175
176
|
('mdcev', ['MDCEVFitResult', 'MDCEVModel']),
|
|
@@ -410,7 +410,11 @@ class _LarchBase(DiscreteChoiceModel):
|
|
|
410
410
|
else:
|
|
411
411
|
optimizer_success = bool(
|
|
412
412
|
optimizer_result.get("success", False))
|
|
413
|
-
optimizer_nit = int(
|
|
413
|
+
optimizer_nit = int(
|
|
414
|
+
optimizer_result.get(
|
|
415
|
+
"nit", optimizer_result.get("iterations", 0)
|
|
416
|
+
) or 0
|
|
417
|
+
)
|
|
414
418
|
ok = bool(np.isfinite(ll) and optimizer_success)
|
|
415
419
|
except Exception as e: # noqa: BLE001
|
|
416
420
|
logger.debug("larch finalize failed: %r", e)
|
|
@@ -719,9 +723,10 @@ class LarchLatentClass(_LarchBase):
|
|
|
719
723
|
"""
|
|
720
724
|
|
|
721
725
|
def setup(self, *args, class_params_spec=None, member_params_spec=None,
|
|
722
|
-
num_classes=2, **kwargs):
|
|
726
|
+
num_classes=2, membership_intercept=True, **kwargs):
|
|
723
727
|
super().setup(*args, **kwargs)
|
|
724
728
|
self.num_classes = int(num_classes)
|
|
729
|
+
self.membership_intercept = bool(membership_intercept)
|
|
725
730
|
if self.num_classes < 2:
|
|
726
731
|
raise ValueError("Larch latent class models require at least two classes")
|
|
727
732
|
self.class_params_spec = [
|
|
@@ -812,7 +817,8 @@ class LarchLatentClass(_LarchBase):
|
|
|
812
817
|
else lx.Model(compute_engine="jax",
|
|
813
818
|
float_dtype=self.float_dtype))
|
|
814
819
|
for class_number in range(1, K):
|
|
815
|
-
expr = P(f"LCM{class_number}_intercept")
|
|
820
|
+
expr = (P(f"LCM{class_number}_intercept")
|
|
821
|
+
if self.membership_intercept else 0)
|
|
816
822
|
for variable in member_vars:
|
|
817
823
|
expr = expr + P(f"LCM{class_number}_{variable}") * LX(variable)
|
|
818
824
|
membership.utility_co[class_number] = expr
|
|
@@ -1236,6 +1236,90 @@ class LatentClassMixedLogit(DiscreteChoiceModel):
|
|
|
1236
1236
|
)
|
|
1237
1237
|
return collapsed
|
|
1238
1238
|
|
|
1239
|
+
def _compute_rail_flags(self):
|
|
1240
|
+
"""Flag bound-pinned, degenerate-SE and collapsed-class fits as railed.
|
|
1241
|
+
|
|
1242
|
+
Rail beats the optimizer's own success flag: a fit pressed against a
|
|
1243
|
+
bound has an unusable estimate even when ``converged`` is True.
|
|
1244
|
+
"""
|
|
1245
|
+
reasons, warns = [], []
|
|
1246
|
+
try:
|
|
1247
|
+
shares = np.asarray(getattr(self, 'class_probs', []),
|
|
1248
|
+
dtype=float).ravel()
|
|
1249
|
+
min_share = float(getattr(self, 'min_share', 0.05))
|
|
1250
|
+
if shares.size and float(np.nanmin(shares)) < min_share:
|
|
1251
|
+
reasons.append(
|
|
1252
|
+
f"class collapsed (min share={float(np.nanmin(shares)):.4g}"
|
|
1253
|
+
f" < {min_share:g})")
|
|
1254
|
+
except Exception:
|
|
1255
|
+
pass
|
|
1256
|
+
try:
|
|
1257
|
+
betas = getattr(self, 'class_betas', None)
|
|
1258
|
+
if isinstance(betas, list) and len(betas) > 1:
|
|
1259
|
+
arrs = [np.asarray(b, dtype=float).ravel() for b in betas]
|
|
1260
|
+
if all(a.size == arrs[0].size for a in arrs):
|
|
1261
|
+
maxdiff = max(float(np.max(np.abs(a - arrs[0])))
|
|
1262
|
+
for a in arrs[1:])
|
|
1263
|
+
if np.isfinite(maxdiff) and maxdiff < 1e-6:
|
|
1264
|
+
reasons.append("classes identical on betas "
|
|
1265
|
+
f"(max |diff|={maxdiff:.1e})")
|
|
1266
|
+
except Exception:
|
|
1267
|
+
pass
|
|
1268
|
+
try:
|
|
1269
|
+
from .lc_rail_guard import rail_flags
|
|
1270
|
+
except Exception:
|
|
1271
|
+
rail_flags = None
|
|
1272
|
+
if rail_flags is not None:
|
|
1273
|
+
try:
|
|
1274
|
+
params = np.asarray(getattr(self, 'coeff_est', []),
|
|
1275
|
+
dtype=float).ravel()
|
|
1276
|
+
names = [str(n) for n in getattr(self, 'coeff_names', [])]
|
|
1277
|
+
se_obj = getattr(self, 'stderr', None)
|
|
1278
|
+
ses = (np.asarray(se_obj, dtype=float).ravel()
|
|
1279
|
+
if se_obj is not None else np.array([]))
|
|
1280
|
+
if params.size and params.size == ses.size == len(names):
|
|
1281
|
+
rf = rail_flags(dict(zip(names, params)),
|
|
1282
|
+
dict(zip(names, ses)))
|
|
1283
|
+
reasons.extend(rf['rail'])
|
|
1284
|
+
warns.extend(rf['warnings'])
|
|
1285
|
+
except Exception as exc:
|
|
1286
|
+
warns.append(
|
|
1287
|
+
f"rail scan skipped: {type(exc).__name__}: {exc}")
|
|
1288
|
+
try:
|
|
1289
|
+
g = getattr(self, 'class_gammas', None)
|
|
1290
|
+
gse_obj = getattr(self, 'gamma_se', None)
|
|
1291
|
+
gse = (np.asarray(gse_obj, dtype=float).ravel()
|
|
1292
|
+
if gse_obj is not None else np.array([]))
|
|
1293
|
+
if g is not None:
|
|
1294
|
+
ga = np.asarray(g, dtype=float).ravel()
|
|
1295
|
+
bound = getattr(self, 'gamma_max_abs', None)
|
|
1296
|
+
gmax = float(np.max(np.abs(ga))) if ga.size else 0.0
|
|
1297
|
+
if (bound is not None and ga.size
|
|
1298
|
+
and gmax >= float(bound) - 1e-3):
|
|
1299
|
+
reasons.append(
|
|
1300
|
+
f"membership gamma at bound {float(bound):g} "
|
|
1301
|
+
f"(max |gamma|={gmax:.4g})")
|
|
1302
|
+
elif ga.size and gmax >= 50.0:
|
|
1303
|
+
reasons.append(
|
|
1304
|
+
f"membership gamma saturated "
|
|
1305
|
+
f"(max |gamma|={gmax:.3g})")
|
|
1306
|
+
if gse.size == ga.size and ga.size:
|
|
1307
|
+
rf = rail_flags(
|
|
1308
|
+
{f"gamma_{i}": float(v)
|
|
1309
|
+
for i, v in enumerate(ga)},
|
|
1310
|
+
{f"gamma_{i}": float(s)
|
|
1311
|
+
for i, s in enumerate(gse)})
|
|
1312
|
+
reasons.extend(rf['rail'])
|
|
1313
|
+
warns.extend(rf['warnings'])
|
|
1314
|
+
except Exception:
|
|
1315
|
+
pass
|
|
1316
|
+
self.rail_reasons = reasons
|
|
1317
|
+
self.rail_warnings = warns
|
|
1318
|
+
self.railed = bool(reasons)
|
|
1319
|
+
if self.railed:
|
|
1320
|
+
self.converged = False
|
|
1321
|
+
return self.railed
|
|
1322
|
+
|
|
1239
1323
|
def _panel_sum(self, arr_task):
|
|
1240
1324
|
"""Sum a per-task array ``(N_task, C)`` into per-panel ``(n_panels, C)``.
|
|
1241
1325
|
|
|
@@ -1556,6 +1640,7 @@ class LatentClassMixedLogit(DiscreteChoiceModel):
|
|
|
1556
1640
|
self.estim_time_sec = time.time() - start_time
|
|
1557
1641
|
self.post_process()
|
|
1558
1642
|
self._detect_collapse()
|
|
1643
|
+
self._compute_rail_flags()
|
|
1559
1644
|
|
|
1560
1645
|
return self
|
|
1561
1646
|
|
|
@@ -1722,6 +1807,7 @@ class LatentClassMixedLogit(DiscreteChoiceModel):
|
|
|
1722
1807
|
else:
|
|
1723
1808
|
self.sort_perm_ = np.arange(C, dtype=int)
|
|
1724
1809
|
self._detect_collapse()
|
|
1810
|
+
self._compute_rail_flags()
|
|
1725
1811
|
|
|
1726
1812
|
return self
|
|
1727
1813
|
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Rail ("parameter pinned at a bound") detection for latent-class fits.
|
|
2
|
+
|
|
3
|
+
A rail is a parameter pressed against a hard bound or a coefficient whose
|
|
4
|
+
standard error is numerically degenerate. Either makes the estimate unusable
|
|
5
|
+
even when the optimizer reports success, so railed fits must not be treated as
|
|
6
|
+
usable solutions.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def rail_flags(values, ses=None, bounds=None, tol=1e-3, se_degenerate=1e5):
|
|
14
|
+
"""Flag bound-pinned parameters and degenerate standard errors.
|
|
15
|
+
|
|
16
|
+
Parameters
|
|
17
|
+
----------
|
|
18
|
+
values, ses : dict name -> float. ``ses`` is optional.
|
|
19
|
+
bounds : dict name -> (lo, hi); None disables a side. A key also matches
|
|
20
|
+
every parameter whose name starts with it.
|
|
21
|
+
tol : absolute tolerance for "sitting on the bound".
|
|
22
|
+
se_degenerate : |SE| at/above this is a rail.
|
|
23
|
+
"""
|
|
24
|
+
rail = []
|
|
25
|
+
warnings = []
|
|
26
|
+
ses = {str(k): v for k, v in (ses or {}).items()}
|
|
27
|
+
bounds = dict(bounds or {})
|
|
28
|
+
for name, val in (values or {}).items():
|
|
29
|
+
name = str(name)
|
|
30
|
+
try:
|
|
31
|
+
v = float(val)
|
|
32
|
+
except Exception:
|
|
33
|
+
continue
|
|
34
|
+
if not np.isfinite(v):
|
|
35
|
+
continue
|
|
36
|
+
lo = hi = None
|
|
37
|
+
for key, bd in bounds.items():
|
|
38
|
+
if name == str(key) or name.startswith(str(key)):
|
|
39
|
+
if isinstance(bd, (tuple, list)) and len(bd) == 2:
|
|
40
|
+
lo, hi = bd
|
|
41
|
+
break
|
|
42
|
+
if lo is not None and np.isfinite(lo) and v <= float(lo) + tol:
|
|
43
|
+
rail.append(f"{name} at/below lower bound {float(lo):g} "
|
|
44
|
+
f"(value={v:.4g})")
|
|
45
|
+
if hi is not None and np.isfinite(hi) and v >= float(hi) - tol:
|
|
46
|
+
rail.append(f"{name} at/above upper bound {float(hi):g} "
|
|
47
|
+
f"(value={v:.4g})")
|
|
48
|
+
s = ses.get(name)
|
|
49
|
+
if s is not None:
|
|
50
|
+
try:
|
|
51
|
+
s = float(s)
|
|
52
|
+
except Exception:
|
|
53
|
+
s = None
|
|
54
|
+
if s is not None and np.isfinite(s):
|
|
55
|
+
if s >= se_degenerate:
|
|
56
|
+
rail.append(f"{name} has degenerate SE {s:.1e} "
|
|
57
|
+
f"(flat/unidentified direction)")
|
|
58
|
+
elif s <= 0.0:
|
|
59
|
+
warnings.append(f"{name} SE=0 (parameter locked/fixed)")
|
|
60
|
+
return {"rail": rail, "railed": bool(rail), "warnings": warnings}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def lc_result_rail(names, betas, ses, shares, membership_coefs=None,
|
|
64
|
+
bounds=15.0, share_floor=0.05, method="lc-em",
|
|
65
|
+
tol=1e-3, se_degenerate=1e5):
|
|
66
|
+
"""Rail scan for ``estimate_dest_choice_lc`` style result arrays."""
|
|
67
|
+
rail = []
|
|
68
|
+
warnings = []
|
|
69
|
+
names = [str(n) for n in names]
|
|
70
|
+
for c, b in enumerate(betas):
|
|
71
|
+
try:
|
|
72
|
+
arr = np.asarray(b, dtype=float).ravel()
|
|
73
|
+
except Exception:
|
|
74
|
+
continue
|
|
75
|
+
se_arr = None
|
|
76
|
+
if ses is not None and c < len(ses):
|
|
77
|
+
try:
|
|
78
|
+
candidate = np.asarray(ses[c], dtype=float).ravel()
|
|
79
|
+
if candidate.size == arr.size:
|
|
80
|
+
se_arr = candidate
|
|
81
|
+
except Exception:
|
|
82
|
+
se_arr = None
|
|
83
|
+
for j, nm in enumerate(names):
|
|
84
|
+
if j >= arr.size:
|
|
85
|
+
break
|
|
86
|
+
v = float(arr[j])
|
|
87
|
+
if np.isfinite(v) and bounds is not None:
|
|
88
|
+
bound = float(bounds)
|
|
89
|
+
if v >= bound - tol:
|
|
90
|
+
rail.append(f"class {c + 1} {nm} at/above beta bound "
|
|
91
|
+
f"{bound:g} (value={v:.4g})")
|
|
92
|
+
elif v <= -bound + tol:
|
|
93
|
+
rail.append(f"class {c + 1} {nm} at/below beta bound "
|
|
94
|
+
f"{-bound:g} (value={v:.4g})")
|
|
95
|
+
if se_arr is not None:
|
|
96
|
+
s = float(se_arr[j])
|
|
97
|
+
if np.isfinite(s):
|
|
98
|
+
if s >= se_degenerate:
|
|
99
|
+
rail.append(f"class {c + 1} {nm} has degenerate SE "
|
|
100
|
+
f"{s:.1e} (flat/unidentified direction)")
|
|
101
|
+
elif s <= 0.0:
|
|
102
|
+
warnings.append(f"class {c + 1} {nm} SE=0 "
|
|
103
|
+
f"(parameter locked/fixed)")
|
|
104
|
+
if method == "lc-em":
|
|
105
|
+
try:
|
|
106
|
+
sh = np.asarray(shares, dtype=float).ravel()
|
|
107
|
+
floor = float(share_floor)
|
|
108
|
+
for c, s in enumerate(sh):
|
|
109
|
+
if not np.isfinite(s):
|
|
110
|
+
continue
|
|
111
|
+
if s < 0.5 * floor:
|
|
112
|
+
rail.append(f"class {c + 1} collapsed: share {s:.4g} < "
|
|
113
|
+
f"{0.5 * floor:g}")
|
|
114
|
+
elif s < floor:
|
|
115
|
+
warnings.append(f"class {c + 1} share {s:.4g} below "
|
|
116
|
+
f"floor {floor:g}")
|
|
117
|
+
except Exception:
|
|
118
|
+
pass
|
|
119
|
+
if len(betas) > 1:
|
|
120
|
+
try:
|
|
121
|
+
arrs = [np.asarray(b, dtype=float).ravel() for b in betas]
|
|
122
|
+
if all(a.size == arrs[0].size for a in arrs):
|
|
123
|
+
maxdiff = max(float(np.max(np.abs(a - arrs[0])))
|
|
124
|
+
for a in arrs[1:])
|
|
125
|
+
if np.isfinite(maxdiff) and maxdiff < 1e-6:
|
|
126
|
+
rail.append("classes identical on betas "
|
|
127
|
+
f"(max |diff|={maxdiff:.1e})")
|
|
128
|
+
except Exception:
|
|
129
|
+
pass
|
|
130
|
+
if membership_coefs is not None:
|
|
131
|
+
try:
|
|
132
|
+
G = np.asarray(membership_coefs, dtype=float)
|
|
133
|
+
gmax = float(np.max(np.abs(G))) if G.size else 0.0
|
|
134
|
+
if np.isfinite(gmax) and gmax >= 50.0:
|
|
135
|
+
rail.append("membership gamma saturated "
|
|
136
|
+
f"(max |gamma|={gmax:.3g})")
|
|
137
|
+
except Exception:
|
|
138
|
+
pass
|
|
139
|
+
return {"rail": rail, "railed": bool(rail), "warnings": warnings}
|
|
@@ -212,12 +212,18 @@ class IdentificationReport:
|
|
|
212
212
|
eigenvalues: np.ndarray
|
|
213
213
|
suggested_drop: tuple[str, ...] = ()
|
|
214
214
|
weak_directions: tuple[dict, ...] = ()
|
|
215
|
+
# Features that do not vary within a case. In a conditional logit such a
|
|
216
|
+
# column is a case-level constant and cancels from the probability, so it
|
|
217
|
+
# is unidentified whatever the information matrix looks like -- see the
|
|
218
|
+
# constant-column pass in identify_weak_features.
|
|
219
|
+
constant_features: tuple[str, ...] = ()
|
|
215
220
|
|
|
216
221
|
def as_dict(self) -> dict:
|
|
217
222
|
return {
|
|
218
223
|
"condition_number": float(self.condition_number),
|
|
219
224
|
"eigenvalues": [float(v) for v in np.asarray(self.eigenvalues).ravel()],
|
|
220
225
|
"suggested_drop": list(self.suggested_drop),
|
|
226
|
+
"constant_features": list(self.constant_features),
|
|
221
227
|
"weak_directions": [
|
|
222
228
|
{
|
|
223
229
|
"eigenvalue": float(d["eigenvalue"]),
|
|
@@ -241,6 +247,7 @@ def identify_weak_features(
|
|
|
241
247
|
offset_col: str = "log_correction",
|
|
242
248
|
max_drop: int | None = None,
|
|
243
249
|
protected: Sequence[str] = (),
|
|
250
|
+
constant_tol: float = 1e-10,
|
|
244
251
|
) -> IdentificationReport:
|
|
245
252
|
"""Find structurally weak directions in a long choice-set frame.
|
|
246
253
|
|
|
@@ -248,6 +255,16 @@ def identify_weak_features(
|
|
|
248
255
|
``possible_overspec.py``) for the sampled MNL used here: eigendecompose the
|
|
249
256
|
scaled exact information matrix at ``beta`` (zeros when omitted) and return
|
|
250
257
|
one feature per near-null direction as ``suggested_drop``.
|
|
258
|
+
|
|
259
|
+
A separate pass catches features that are CONSTANT WITHIN A CASE. Those are
|
|
260
|
+
case-level constants in a conditional logit -- they cancel from the
|
|
261
|
+
probability -- so the estimator leaves them at the ridge (coefficient ~0)
|
|
262
|
+
however much the column varies across cases. The eigen test cannot see
|
|
263
|
+
them: it rescales by ``sqrt(diag(info))``, and for such a column that
|
|
264
|
+
diagonal is its own numerical residue, so the column reports a unit
|
|
265
|
+
"self-correlation" and no weak direction. The share of a feature's
|
|
266
|
+
variation that lies within cases is the direct test, and any feature below
|
|
267
|
+
``constant_tol`` of it is reported in ``constant_features``.
|
|
251
268
|
"""
|
|
252
269
|
frame = data if isinstance(data, ChoiceSetFrame) else ChoiceSetFrame.from_long(
|
|
253
270
|
data, feature_cols, case_col=case_col, alt_col=alt_col,
|
|
@@ -260,35 +277,75 @@ def identify_weak_features(
|
|
|
260
277
|
else:
|
|
261
278
|
beta_vec = np.asarray(beta, dtype=float).reshape(-1)
|
|
262
279
|
info, _, _ = frame.information(beta_vec)
|
|
263
|
-
diag = np.clip(np.diag(info), 1e-300, None)
|
|
264
|
-
scale = np.sqrt(diag)
|
|
265
|
-
scaled = info / np.outer(scale, scale)
|
|
266
|
-
evals, evecs = np.linalg.eigh(0.5 * (scaled + scaled.T))
|
|
267
|
-
condition = float(evals[-1] / max(evals[0], 1e-300))
|
|
268
|
-
weak = np.flatnonzero(evals < float(tol))
|
|
269
280
|
protected_set = {str(p) for p in (protected or ())}
|
|
270
281
|
drops: list[str] = []
|
|
271
282
|
directions: list[dict] = []
|
|
283
|
+
|
|
284
|
+
# ── Case-constant columns (before the eigen pass) ──────────────────────
|
|
285
|
+
# Within-case variation is the only variation a conditional logit can use.
|
|
286
|
+
# A column without it is unidentified whatever the information matrix says
|
|
287
|
+
# -- and what it says is numerical noise: the row is a difference of two
|
|
288
|
+
# ~equal quantities, so its diagonal is a residue and the scaled matrix
|
|
289
|
+
# picks up huge NEGATIVE eigenvalues that then masquerade as weak
|
|
290
|
+
# directions. Identify the constant columns first and run the eigen pass on
|
|
291
|
+
# the remaining submatrix, so those artifacts cannot produce a drop list
|
|
292
|
+
# (and cannot displace the real loaders by already-claimed names).
|
|
293
|
+
d = frame.dimension
|
|
294
|
+
s1 = np.zeros(d)
|
|
295
|
+
s2 = np.zeros(d)
|
|
296
|
+
within_ss = np.zeros(d)
|
|
297
|
+
n_rows = 0
|
|
298
|
+
for x in frame.x:
|
|
299
|
+
xi = np.asarray(x, dtype=float)
|
|
300
|
+
if xi.ndim != 2 or xi.shape[0] < 2:
|
|
301
|
+
continue
|
|
302
|
+
s1 += xi.sum(axis=0)
|
|
303
|
+
s2 += (xi * xi).sum(axis=0)
|
|
304
|
+
n_rows += xi.shape[0]
|
|
305
|
+
within_ss += ((xi - xi.mean(axis=0)) ** 2).sum(axis=0)
|
|
306
|
+
total_ss = np.clip(s2 - (s1 * s1) / max(n_rows, 1), 0.0, None)
|
|
307
|
+
share = np.divide(within_ss, total_ss,
|
|
308
|
+
out=np.zeros(d), where=total_ss > 0)
|
|
309
|
+
# ``where=total_ss > 0`` leaves the share at 0 for a column with no variance
|
|
310
|
+
# at all, which is equally unidentified, so one test covers both.
|
|
311
|
+
constant_mask = share <= float(constant_tol)
|
|
312
|
+
constant = [names[j] for j in np.flatnonzero(constant_mask)
|
|
313
|
+
if names[j] not in protected_set]
|
|
314
|
+
for name in constant:
|
|
315
|
+
directions.append({"eigenvalue": 0.0, "loadings": {name: 1.0}})
|
|
316
|
+
drops.extend(constant)
|
|
317
|
+
|
|
318
|
+
keep = np.flatnonzero(~constant_mask)
|
|
319
|
+
keep_names = [names[j] for j in keep]
|
|
320
|
+
sub = info[np.ix_(keep, keep)]
|
|
321
|
+
diag = np.clip(np.diag(sub), 1e-300, None)
|
|
322
|
+
scale = np.sqrt(diag)
|
|
323
|
+
scaled = sub / np.outer(scale, scale)
|
|
324
|
+
evals, evecs = np.linalg.eigh(0.5 * (scaled + scaled.T))
|
|
325
|
+
condition = float(evals[-1] / max(evals[0], 1e-300)) if evals.size else float("nan")
|
|
326
|
+
weak = np.flatnonzero(evals < float(tol))
|
|
327
|
+
|
|
272
328
|
for idx in weak:
|
|
273
329
|
# Unscale the direction: v_j = evec_j / sqrt(info_jj).
|
|
274
330
|
v = evecs[:, idx] / scale
|
|
275
331
|
v = v / (np.linalg.norm(v) + 1e-300)
|
|
276
332
|
order = np.argsort(-np.abs(v))
|
|
277
|
-
top = {
|
|
333
|
+
top = {keep_names[j]: float(v[j]) for j in order[: min(8, len(order))]}
|
|
278
334
|
directions.append({"eigenvalue": float(evals[idx]), "loadings": top})
|
|
279
335
|
# Never drop user-protected features (e.g. explicitly requested
|
|
280
336
|
# behavioural interactions); fall through to the next-largest loader.
|
|
281
337
|
j = next((int(k) for k in order
|
|
282
|
-
if
|
|
338
|
+
if keep_names[int(k)] not in protected_set), None)
|
|
283
339
|
if j is None:
|
|
284
340
|
continue
|
|
285
|
-
if abs(v[j]) >= float(loading_threshold) and
|
|
286
|
-
drops.append(
|
|
341
|
+
if abs(v[j]) >= float(loading_threshold) and keep_names[j] not in drops:
|
|
342
|
+
drops.append(keep_names[j])
|
|
287
343
|
if max_drop is not None and len(drops) > int(max_drop):
|
|
288
344
|
drops = drops[: int(max_drop)]
|
|
289
345
|
return IdentificationReport(
|
|
290
346
|
condition_number=condition, eigenvalues=evals,
|
|
291
|
-
suggested_drop=tuple(drops), weak_directions=tuple(directions)
|
|
347
|
+
suggested_drop=tuple(drops), weak_directions=tuple(directions),
|
|
348
|
+
constant_features=tuple(constant))
|
|
292
349
|
|
|
293
350
|
|
|
294
351
|
class GaussianRandomWalkProposal:
|
|
@@ -1338,12 +1395,23 @@ def _fit_membership_gammas(case_X, resp, l2: float = 1.0, maxiter: int = 200,
|
|
|
1338
1395
|
|
|
1339
1396
|
|
|
1340
1397
|
def _fit_frame_mnl(frame: "ChoiceSetFrame", x0, *, prior_scale: float = 5.0,
|
|
1341
|
-
bounds: float = 15.0, maxiter: int = 500, tol: float = 1e-9
|
|
1398
|
+
bounds: float = 15.0, maxiter: int = 500, tol: float = 1e-9,
|
|
1399
|
+
grad_tol: float = 1e-6, retries: int = 2):
|
|
1342
1400
|
"""Weighted MNL fit on a (copied) frame: L-BFGS-B on the analytic gradient.
|
|
1343
1401
|
|
|
1344
1402
|
Mirrors larch's ``minimize(logloss, jac=d_logloss, bounds=pbounds)`` with
|
|
1345
1403
|
a weak N(0, prior_scale^2) ridge so unidentified directions are bounded.
|
|
1346
1404
|
The Manski-Lerman offset travels inside the frame, so it is exact here.
|
|
1405
|
+
|
|
1406
|
+
``ftol`` alone is a relative decrease test, so L-BFGS-B can stop with a
|
|
1407
|
+
large gradient and still report success -- and can stop short of the
|
|
1408
|
+
optimum when it runs out of iterations. The fit is therefore re-started
|
|
1409
|
+
from its own solution while the gradient is too large, keeping the better
|
|
1410
|
+
objective value, and the returned result carries the evidence
|
|
1411
|
+
(``grad_inf``, ``grad_tol``, ``fit_message``) alongside ``success``, which
|
|
1412
|
+
is true when either scipy succeeded or the gradient shows a stationary
|
|
1413
|
+
point. The threshold is per observation, since the likelihood scales with
|
|
1414
|
+
the sample.
|
|
1347
1415
|
"""
|
|
1348
1416
|
from scipy.optimize import minimize
|
|
1349
1417
|
d = frame.dimension
|
|
@@ -1355,10 +1423,49 @@ def _fit_frame_mnl(frame: "ChoiceSetFrame", x0, *, prior_scale: float = 5.0,
|
|
|
1355
1423
|
g = grad - prior_prec * np.asarray(beta, dtype=float)
|
|
1356
1424
|
return -total, -g
|
|
1357
1425
|
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1426
|
+
try:
|
|
1427
|
+
g_tol = float(grad_tol) * max(int(frame.n_cases), 1)
|
|
1428
|
+
except Exception:
|
|
1429
|
+
g_tol = float(grad_tol)
|
|
1430
|
+
|
|
1431
|
+
def _run(start, n_iter):
|
|
1432
|
+
# ftol is a relative-decrease test on an objective of order 1e3, so on
|
|
1433
|
+
# its own it stops while the gradient is still ~1e-2. Stop on the
|
|
1434
|
+
# gradient; ftol is only kept as a floating-point floor.
|
|
1435
|
+
return minimize(nll, np.asarray(start, dtype=float), jac=True,
|
|
1436
|
+
method="L-BFGS-B",
|
|
1437
|
+
bounds=[(-float(bounds), float(bounds))] * d,
|
|
1438
|
+
options={"maxiter": int(n_iter),
|
|
1439
|
+
"ftol": min(float(tol), 1e-15),
|
|
1440
|
+
"gtol": g_tol})
|
|
1441
|
+
|
|
1442
|
+
res = _run(x0, maxiter)
|
|
1443
|
+
left = int(retries)
|
|
1444
|
+
while left > 0:
|
|
1445
|
+
g = np.asarray(nll(np.asarray(res.x, dtype=float))[1], dtype=float)
|
|
1446
|
+
g_inf = float(np.max(np.abs(g))) if g.size else float("nan")
|
|
1447
|
+
if not np.isfinite(g_inf) or g_inf <= g_tol:
|
|
1448
|
+
break
|
|
1449
|
+
left -= 1
|
|
1450
|
+
try:
|
|
1451
|
+
retry = _run(res.x, max(2 * int(maxiter), int(maxiter)))
|
|
1452
|
+
except Exception:
|
|
1453
|
+
break
|
|
1454
|
+
if not np.isfinite(float(getattr(retry, "fun", np.inf))) \
|
|
1455
|
+
or float(retry.fun) > float(res.fun) - 1e-9:
|
|
1456
|
+
break # no progress: keep what we had
|
|
1457
|
+
res = retry
|
|
1458
|
+
|
|
1459
|
+
try:
|
|
1460
|
+
g = np.asarray(nll(np.asarray(res.x, dtype=float))[1], dtype=float)
|
|
1461
|
+
g_inf = float(np.max(np.abs(g))) if g.size else float("nan")
|
|
1462
|
+
except Exception:
|
|
1463
|
+
g_inf = float("nan")
|
|
1464
|
+
res.grad_inf = g_inf
|
|
1465
|
+
res.grad_tol = g_tol
|
|
1466
|
+
res.fit_message = str(getattr(res, "message", "") or "")
|
|
1467
|
+
res.converged_evidence = bool(np.isfinite(g_inf) and g_inf <= g_tol)
|
|
1468
|
+
res.success = bool(getattr(res, "success", False) or res.converged_evidence)
|
|
1362
1469
|
return np.asarray(res.x, dtype=float), float(-res.fun), res
|
|
1363
1470
|
|
|
1364
1471
|
|
|
@@ -1501,7 +1608,8 @@ def estimate_dest_choice_lc(
|
|
|
1501
1608
|
fit_ll = float(sum(_weighted_frame(frame, (labels == g).astype(float))
|
|
1502
1609
|
.loglike(b) for g, b in zip(groups, betas)))
|
|
1503
1610
|
return _lc_result("grouped", names, betas, ses, shares, fit_ll,
|
|
1504
|
-
float("nan"), True, 0, frame, labels=groups
|
|
1611
|
+
float("nan"), True, 0, frame, labels=groups,
|
|
1612
|
+
bounds=float(bounds), share_floor=float(share_floor))
|
|
1505
1613
|
|
|
1506
1614
|
# ── Latent classes: EM with weighted-MNL M-steps ──────────────────────
|
|
1507
1615
|
# Class priors are either fixed shares or a person-type membership
|
|
@@ -1540,6 +1648,14 @@ def estimate_dest_choice_lc(
|
|
|
1540
1648
|
shares = np.full(n_classes, 1.0 / n_classes)
|
|
1541
1649
|
gammas = (np.zeros((n_classes - 1, case_X.shape[1] + 1))
|
|
1542
1650
|
if case_X is not None else None)
|
|
1651
|
+
if _start and case_X is not None:
|
|
1652
|
+
# The header promises a jitter of "BOTH the class-specific betas AND
|
|
1653
|
+
# the membership gammas". Only the betas were ever jittered: every
|
|
1654
|
+
# restart began from a ZERO membership equation, i.e. uniform class
|
|
1655
|
+
# priors, so restarts differed solely in the choice coefficients and
|
|
1656
|
+
# the membership surface was effectively estimated once. Perturb it
|
|
1657
|
+
# too so a 3-class fit can leave the symmetric starting point.
|
|
1658
|
+
gammas = rng.normal(0.0, float(perturb_sigma), gammas.shape)
|
|
1543
1659
|
prev_ll = -np.inf
|
|
1544
1660
|
converged = False
|
|
1545
1661
|
iterations = 0
|
|
@@ -1563,20 +1679,38 @@ def estimate_dest_choice_lc(
|
|
|
1563
1679
|
p = np.clip(p, float(share_floor) / n_classes, None)
|
|
1564
1680
|
p /= p.sum(axis=1, keepdims=True)
|
|
1565
1681
|
new_shares = p.mean(axis=0)
|
|
1566
|
-
# M-step: weighted MNL per class + membership gammas
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1682
|
+
# M-step: weighted MNL per class + membership gammas.
|
|
1683
|
+
# Keep each fit's first-order optimality: the optimizer flag only
|
|
1684
|
+
# says the M-step stopped, the gradient says it stopped at an
|
|
1685
|
+
# optimum, and without it the reported class solution carries no
|
|
1686
|
+
# evidence about the M-steps at all.
|
|
1687
|
+
# NB: warm start from the previous iteration's betas, so build the
|
|
1688
|
+
# new vectors in a separate list rather than clearing `betas` first.
|
|
1689
|
+
_prev = betas
|
|
1690
|
+
betas = []
|
|
1691
|
+
mstep_grad = []
|
|
1692
|
+
for c in range(n_classes):
|
|
1693
|
+
_b, _, _r = _fit_frame_mnl(_weighted_frame(frame, p[:, c]),
|
|
1694
|
+
_prev[c], prior_scale=prior_scale,
|
|
1695
|
+
bounds=bounds)
|
|
1696
|
+
betas.append(_b)
|
|
1697
|
+
mstep_grad.append(float(getattr(_r, "grad_inf", float("nan"))))
|
|
1570
1698
|
if case_X is not None:
|
|
1571
1699
|
gammas = _fit_membership_gammas(case_X, p, l2=membership_l2, standardize=True)
|
|
1572
1700
|
shares = new_shares
|
|
1573
1701
|
if share_floor and float(share_floor) > 0:
|
|
1574
1702
|
_coll = int(np.argmin(shares))
|
|
1575
1703
|
if float(shares[_coll]) < 0.5 * float(share_floor):
|
|
1576
|
-
# Re-seed the collapsed class
|
|
1577
|
-
#
|
|
1578
|
-
#
|
|
1579
|
-
|
|
1704
|
+
# Re-seed the collapsed class so the next E-step can
|
|
1705
|
+
# differentiate it. This used to overwrite the class with
|
|
1706
|
+
# `beta0 + jitter`, where beta0 is the POOLED fit -- so a
|
|
1707
|
+
# class that was genuinely small got reset to the common
|
|
1708
|
+
# pooled surface on EVERY iteration it stayed small, could
|
|
1709
|
+
# never differentiate itself, and EM stalled. Perturb the
|
|
1710
|
+
# class's own current estimate instead: that is the
|
|
1711
|
+
# "perturbation move, not a failure" the comment claims, and
|
|
1712
|
+
# it preserves whatever separation the M-step had built.
|
|
1713
|
+
_b = betas[_coll] + rng.normal(0.0, float(perturb_sigma), d)
|
|
1580
1714
|
if crash_cols:
|
|
1581
1715
|
_b[names.index(crash_cols[0])] += crash_anchors[
|
|
1582
1716
|
_coll % len(crash_anchors)]
|
|
@@ -1594,19 +1728,32 @@ def estimate_dest_choice_lc(
|
|
|
1594
1728
|
p = np.exp(llw - m)
|
|
1595
1729
|
p /= p.sum(axis=1, keepdims=True)
|
|
1596
1730
|
restart_lls.append(fit_ll)
|
|
1597
|
-
|
|
1731
|
+
# Rank restarts by (converged, fit_ll), NOT by fit_ll alone. Picking
|
|
1732
|
+
# purely on log-likelihood let a restart that wandered without settling
|
|
1733
|
+
# beat one that had genuinely converged; the caller then rejected the
|
|
1734
|
+
# whole genome ("not converged"), discarding the only usable restart.
|
|
1735
|
+
# That is the mechanism behind the paper-arm runs reporting arm D as
|
|
1736
|
+
# NOT converged at full sample while the n=200 smoke did converge.
|
|
1737
|
+
_rank = (bool(converged), fit_ll)
|
|
1738
|
+
if best is None or _rank > (bool(best[4]), best[0]):
|
|
1598
1739
|
best = (fit_ll, betas, shares, p, converged, iterations,
|
|
1599
|
-
None if gammas is None else np.asarray(gammas).copy()
|
|
1600
|
-
|
|
1740
|
+
None if gammas is None else np.asarray(gammas).copy(),
|
|
1741
|
+
mstep_grad)
|
|
1742
|
+
fit_ll, betas, shares, resp, converged, iterations, gammas, mstep_grad = best
|
|
1601
1743
|
if case_X is None:
|
|
1602
1744
|
# label switching only reorders the fixed-share solution; with a
|
|
1603
1745
|
# membership equation the reference class structure must be kept.
|
|
1604
1746
|
order = np.argsort(-np.asarray(shares))
|
|
1605
1747
|
betas = [betas[i] for i in order]
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1748
|
+
resp = np.asarray(resp)[:, order]
|
|
1749
|
+
mstep_grad = [mstep_grad[i] for i in order]
|
|
1750
|
+
# Derive the reported shares from the SAME responsibilities that ship with
|
|
1751
|
+
# the result. `shares` carried out of the EM loop was the mean of the
|
|
1752
|
+
# share_floor-CLIPPED responsibilities, while the returned `resp` is the
|
|
1753
|
+
# unclipped posterior, so resp.mean(axis=0) did not reproduce shares -- and
|
|
1754
|
+
# stage5_lc_genes' collapse gate reads `shares` while a consumer reading
|
|
1755
|
+
# `responsibilities` would compute something different.
|
|
1756
|
+
shares = np.asarray(resp).mean(axis=0)
|
|
1610
1757
|
ses = []
|
|
1611
1758
|
for c in range(n_classes):
|
|
1612
1759
|
fw = _weighted_frame(frame, resp[:, c])
|
|
@@ -1616,13 +1763,17 @@ def estimate_dest_choice_lc(
|
|
|
1616
1763
|
return _lc_result("lc-em", names, betas, ses, shares, fit_ll, float("nan"),
|
|
1617
1764
|
converged, iterations, frame, responsibilities=resp,
|
|
1618
1765
|
membership_coefs=gammas, membership_vars=memb_cols,
|
|
1619
|
-
crash_cols=crash_cols, restart_lls=restart_lls
|
|
1766
|
+
crash_cols=crash_cols, restart_lls=restart_lls,
|
|
1767
|
+
bounds=float(bounds), share_floor=float(share_floor),
|
|
1768
|
+
mstep_grad_inf=mstep_grad,
|
|
1769
|
+
mstep_grad_tol=1e-6 * frame.n_cases)
|
|
1620
1770
|
|
|
1621
1771
|
|
|
1622
1772
|
def _lc_result(method, names, betas, ses, shares, fit_ll, _unused, converged,
|
|
1623
1773
|
iterations, frame, labels=None, responsibilities=None,
|
|
1624
1774
|
membership_coefs=None, membership_vars=None, crash_cols=None,
|
|
1625
|
-
restart_lls=None
|
|
1775
|
+
restart_lls=None, bounds=15.0, share_floor=0.05,
|
|
1776
|
+
mstep_grad_inf=None, mstep_grad_tol=None) -> dict:
|
|
1626
1777
|
n_classes = len(betas)
|
|
1627
1778
|
k_params = n_classes * frame.dimension + (n_classes - 1)
|
|
1628
1779
|
if membership_vars:
|
|
@@ -1644,6 +1795,29 @@ def _lc_result(method, names, betas, ses, shares, fit_ll, _unused, converged,
|
|
|
1644
1795
|
"converged": bool(converged),
|
|
1645
1796
|
"iterations": int(iterations),
|
|
1646
1797
|
}
|
|
1798
|
+
if mstep_grad_inf is not None:
|
|
1799
|
+
_mg = [float(v) for v in mstep_grad_inf]
|
|
1800
|
+
_mt = None if mstep_grad_tol is None else float(mstep_grad_tol)
|
|
1801
|
+
out["mstep_grad_inf"] = _mg
|
|
1802
|
+
out["mstep_grad_tol"] = _mt
|
|
1803
|
+
out["mstep_stationary"] = [bool(_mt is not None and np.isfinite(v)
|
|
1804
|
+
and v <= _mt) for v in _mg]
|
|
1805
|
+
try:
|
|
1806
|
+
from .lc_rail_guard import lc_result_rail as _lc_rail
|
|
1807
|
+
except Exception:
|
|
1808
|
+
_lc_rail = None
|
|
1809
|
+
if _lc_rail is not None:
|
|
1810
|
+
_rf = _lc_rail(names, betas, ses, shares,
|
|
1811
|
+
membership_coefs=membership_coefs,
|
|
1812
|
+
bounds=float(bounds),
|
|
1813
|
+
share_floor=float(share_floor),
|
|
1814
|
+
method=method)
|
|
1815
|
+
out["rail"] = list(_rf["rail"])
|
|
1816
|
+
out["railed"] = bool(_rf["railed"])
|
|
1817
|
+
out["rail_warnings"] = list(_rf["warnings"])
|
|
1818
|
+
if _rf["railed"]:
|
|
1819
|
+
out["converged_by_optimizer"] = bool(out["converged"])
|
|
1820
|
+
out["converged"] = False
|
|
1647
1821
|
if labels is not None:
|
|
1648
1822
|
out["group_labels"] = list(labels)
|
|
1649
1823
|
if responsibilities is not None:
|
|
@@ -319,8 +319,39 @@ def fit_rrm_numba(model, start=None, compute_inference=True):
|
|
|
319
319
|
result = _sp_min(_obj, start, jac=True, method='BFGS',
|
|
320
320
|
options={'maxiter': int(getattr(model, 'maxiter', 2000) or 2000),
|
|
321
321
|
'gtol': 1e-6, 'disp': False})
|
|
322
|
+
# Same contract as the pure-scipy path: re-polish while the gradient is
|
|
323
|
+
# still large, then judge convergence on first-order optimality instead of
|
|
324
|
+
# scipy's success flag (BFGS reports precision loss after it has already
|
|
325
|
+
# converged, which used to mark every RRM fit as failed).
|
|
326
|
+
#
|
|
327
|
+
# The gradient here is a central difference of the njit likelihood, which
|
|
328
|
+
# is noisy on the near-collinear period design, so the polish and the final
|
|
329
|
+
# verdict use the analytic numpy gradient: both engines then land on the
|
|
330
|
+
# same optimum and agree on whether it is one.
|
|
331
|
+
def _analytic_grad(x):
|
|
332
|
+
try:
|
|
333
|
+
return model.get_loglike_gradient(
|
|
334
|
+
np.asarray(x, dtype=float), np.ones(M, dtype=float) * 1e-10)[1]
|
|
335
|
+
except Exception:
|
|
336
|
+
return None
|
|
337
|
+
|
|
338
|
+
def _obj_exact(b):
|
|
339
|
+
b = np.ascontiguousarray(np.asarray(b, dtype=np.float64))
|
|
340
|
+
f, g = model.get_loglike_gradient(b, np.ones(M, dtype=float) * 1e-10)
|
|
341
|
+
return float(f), np.asarray(g, dtype=np.float64)
|
|
342
|
+
|
|
343
|
+
try:
|
|
344
|
+
try:
|
|
345
|
+
from rrm import assess_convergence as _assess, polish_result as _polish
|
|
346
|
+
except ImportError:
|
|
347
|
+
from .rrm import assess_convergence as _assess, polish_result as _polish
|
|
348
|
+
n_obs = int(len(y_idx))
|
|
349
|
+
result = _polish(_obj_exact, result, grad_fun=_analytic_grad, n_obs=n_obs)
|
|
350
|
+
model.converged, model.grad_norm_inf, model.fit_message = _assess(
|
|
351
|
+
result, n_obs=n_obs, grad=_analytic_grad(result.x))
|
|
352
|
+
except Exception:
|
|
353
|
+
model.converged = bool(getattr(result, 'success', False))
|
|
322
354
|
model.coeff_est = result.x
|
|
323
|
-
model.converged = result.success
|
|
324
355
|
model.beta = result.x
|
|
325
356
|
try:
|
|
326
357
|
model.post_process(compute_inference=compute_inference)
|
|
@@ -46,6 +46,124 @@ import numpy as np
|
|
|
46
46
|
from numpy.typing import NDArray
|
|
47
47
|
from scipy import stats
|
|
48
48
|
from scipy.optimize import minimize
|
|
49
|
+
|
|
50
|
+
# ── Convergence assessment ─────────────────────────────────────────────────
|
|
51
|
+
# scipy's BFGS reports success=False with "Desired error not necessarily
|
|
52
|
+
# achieved due to precision loss" once the line search can no longer improve
|
|
53
|
+
# at float64 precision. That happens AFTER the optimum has been reached, so
|
|
54
|
+
# treating result.success as the convergence verdict marks good fits as
|
|
55
|
+
# failures. Downstream that is expensive: the ABM pipeline (stage4 Schedule
|
|
56
|
+
# Placement) labels the model 'optimizer_stopped' and down-weights its period
|
|
57
|
+
# probabilities (blend_alpha 0.08-0.35), so a converged RRM contributes almost
|
|
58
|
+
# nothing to the simulated schedule. All 8 RRM timing models in runs
|
|
59
|
+
# 27284748/27293944 hit this.
|
|
60
|
+
#
|
|
61
|
+
# The verdict below is first-order optimality (scaled gradient at the
|
|
62
|
+
# solution), with a short re-polish when the gradient is still large, which is
|
|
63
|
+
# what actually recovers fits that stalled in the line search.
|
|
64
|
+
RRM_GRAD_TOL = 1e-6 # on the MEAN log-likelihood gradient, per obs
|
|
65
|
+
RRM_POLISH_ATTEMPTS = 2
|
|
66
|
+
RRM_POLISH_MAXITER = 2000
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def grad_tolerance(n_obs=None, gtol=None):
|
|
70
|
+
"""Absolute gradient threshold for a sample of ``n_obs`` observations.
|
|
71
|
+
|
|
72
|
+
The likelihood is a sum, so its gradient grows with the sample size: a
|
|
73
|
+
fixed absolute threshold is strict on small samples and meaningless on
|
|
74
|
+
large ones. The criterion used here is that the gradient of the *mean* log
|
|
75
|
+
likelihood is below ``RRM_GRAD_TOL``. At that point the remaining
|
|
76
|
+
log-likelihood gain is far below what any reported digit could show.
|
|
77
|
+
"""
|
|
78
|
+
if gtol is not None:
|
|
79
|
+
return float(gtol)
|
|
80
|
+
try:
|
|
81
|
+
n = int(n_obs or 0)
|
|
82
|
+
except Exception:
|
|
83
|
+
n = 0
|
|
84
|
+
return float(RRM_GRAD_TOL * max(n, 1))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _grad_inf_norm(result, n_params=None, grad=None):
|
|
88
|
+
"""Largest absolute gradient component at the reported solution.
|
|
89
|
+
|
|
90
|
+
``grad`` overrides ``result.jac`` so callers holding a more accurate
|
|
91
|
+
gradient (the analytic numpy one, or the numba engine's finite-difference
|
|
92
|
+
estimate at a nearly-collinear solution) can supply it.
|
|
93
|
+
"""
|
|
94
|
+
try:
|
|
95
|
+
jac = (np.asarray(grad, dtype=float) if grad is not None
|
|
96
|
+
else np.asarray(result.jac, dtype=float)).ravel()
|
|
97
|
+
if jac.size == 0 or not np.all(np.isfinite(jac)):
|
|
98
|
+
return float('nan')
|
|
99
|
+
return float(np.max(np.abs(jac)))
|
|
100
|
+
except Exception:
|
|
101
|
+
return float('nan')
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def assess_convergence(result, gtol=None, n_obs=None, grad=None):
|
|
105
|
+
"""Decide convergence from the evidence, not only ``result.success``.
|
|
106
|
+
|
|
107
|
+
Returns ``(converged, grad_inf, message)``. A fit counts as converged when
|
|
108
|
+
scipy says so, or when the gradient is small enough to be at a stationary
|
|
109
|
+
point even though the line search gave up on precision. Unknown gradients
|
|
110
|
+
fall back to scipy's verdict rather than claiming success.
|
|
111
|
+
"""
|
|
112
|
+
tol = grad_tolerance(n_obs, gtol)
|
|
113
|
+
grad = _grad_inf_norm(result, grad=grad)
|
|
114
|
+
msg = str(getattr(result, 'message', '') or '')
|
|
115
|
+
success = bool(getattr(result, 'success', False))
|
|
116
|
+
if success:
|
|
117
|
+
return True, grad, msg
|
|
118
|
+
if np.isfinite(grad) and grad <= tol:
|
|
119
|
+
return True, grad, msg + ' [accepted: scaled gradient below tolerance]'
|
|
120
|
+
return False, grad, msg
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def polish_result(fun, result, gtol=None, attempts=None, maxiter=None,
|
|
124
|
+
grad_fun=None, n_obs=None):
|
|
125
|
+
"""Re-run BFGS from the reported solution while the gradient is too large.
|
|
126
|
+
|
|
127
|
+
The first BFGS call already did the work; this only rescues fits that
|
|
128
|
+
stopped early with a usable point. The best objective value wins, so a
|
|
129
|
+
polish pass can never return a worse fit than the one it started from.
|
|
130
|
+
|
|
131
|
+
``grad_fun(x)`` supplies a more accurate gradient than ``result.jac`` for
|
|
132
|
+
the stopping test (the numba engine's finite differences are noisy where
|
|
133
|
+
the specification is nearly collinear, which would otherwise keep the
|
|
134
|
+
retry loop running at a point that is already optimal).
|
|
135
|
+
"""
|
|
136
|
+
tol = grad_tolerance(n_obs, gtol)
|
|
137
|
+
n_left = int(RRM_POLISH_ATTEMPTS if attempts is None else attempts)
|
|
138
|
+
n_iter = int(RRM_POLISH_MAXITER if maxiter is None else maxiter)
|
|
139
|
+
best = result
|
|
140
|
+
best_f = float(getattr(result, 'fun', np.inf))
|
|
141
|
+
|
|
142
|
+
def _stopping_grad(res):
|
|
143
|
+
if grad_fun is None:
|
|
144
|
+
return None
|
|
145
|
+
try:
|
|
146
|
+
return grad_fun(res.x)
|
|
147
|
+
except Exception:
|
|
148
|
+
return None
|
|
149
|
+
|
|
150
|
+
while n_left > 0:
|
|
151
|
+
ok, grad, _msg = assess_convergence(best, tol, grad=_stopping_grad(best))
|
|
152
|
+
if ok:
|
|
153
|
+
break
|
|
154
|
+
n_left -= 1
|
|
155
|
+
try:
|
|
156
|
+
retry = minimize(fun, np.asarray(best.x, dtype=float), jac=True,
|
|
157
|
+
method='BFGS', tol=1e-8,
|
|
158
|
+
options={'maxiter': n_iter,
|
|
159
|
+
'gtol': max(tol, 1e-6), 'disp': False})
|
|
160
|
+
except Exception:
|
|
161
|
+
break
|
|
162
|
+
f_new = float(getattr(retry, 'fun', np.inf))
|
|
163
|
+
if not np.isfinite(f_new) or f_new > best_f + 1e-9:
|
|
164
|
+
break # no progress: keep what we had
|
|
165
|
+
best, best_f = retry, f_new
|
|
166
|
+
return best
|
|
49
167
|
import pandas as pd
|
|
50
168
|
from time import time
|
|
51
169
|
try:
|
|
@@ -635,8 +753,11 @@ class RandomRegret(DiscreteChoiceModel):
|
|
|
635
753
|
self.fit_start_time = _time()
|
|
636
754
|
result = sp_min(_obj, start, jac=True, method='BFGS',
|
|
637
755
|
options={'maxiter': 2000, 'gtol': 1e-6, 'disp': False})
|
|
756
|
+
result = polish_result(_obj, result, maxiter=2000,
|
|
757
|
+
n_obs=self.nb_samples)
|
|
638
758
|
self.coeff_est = result.x
|
|
639
|
-
self.converged =
|
|
759
|
+
self.converged, self.grad_norm_inf, self.fit_message = \
|
|
760
|
+
assess_convergence(result, n_obs=self.nb_samples)
|
|
640
761
|
self.beta = result.x
|
|
641
762
|
self.post_process()
|
|
642
763
|
return
|
|
@@ -688,8 +809,14 @@ class RandomRegret(DiscreteChoiceModel):
|
|
|
688
809
|
jac=True,
|
|
689
810
|
options={'maxiter': getattr(self, 'maxiter', 2000)},
|
|
690
811
|
)
|
|
812
|
+
# Re-polish while the gradient is still large, then judge convergence
|
|
813
|
+
# on first-order optimality rather than scipy's precision-loss flag
|
|
814
|
+
# (see assess_convergence above).
|
|
815
|
+
_polish_fun = lambda _x: self.get_loglike_gradient(_x, delta)
|
|
816
|
+
result = polish_result(_polish_fun, result, n_obs=self.nb_samples)
|
|
691
817
|
self.coeff_est = result.x
|
|
692
|
-
self.converged =
|
|
818
|
+
self.converged, self.grad_norm_inf, self.fit_message = \
|
|
819
|
+
assess_convergence(result, n_obs=self.nb_samples)
|
|
693
820
|
self.beta = result.x # Extract results
|
|
694
821
|
self.post_process(compute_inference=compute_inference)
|
|
695
822
|
# }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.0.271
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: SearchLibrium
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.271
|
|
4
4
|
Summary: Automated specification search for discrete choice models (logit, mixed logit, nested logit, RRM and more) using metaheuristic optimisation
|
|
5
5
|
Author: Zeke Ahern, Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
|
|
6
6
|
Maintainer-email: Zeke Ahern <z.ahern@qut.edu.au>
|
|
@@ -29,6 +29,7 @@ src/SearchLibrium/larch_compat.py
|
|
|
29
29
|
src/SearchLibrium/larch_models.py
|
|
30
30
|
src/SearchLibrium/latent_class.py
|
|
31
31
|
src/SearchLibrium/latent_spec.py
|
|
32
|
+
src/SearchLibrium/lc_rail_guard.py
|
|
32
33
|
src/SearchLibrium/logistic_regression.py
|
|
33
34
|
src/SearchLibrium/main.py
|
|
34
35
|
src/SearchLibrium/main_debug.py
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0.0.268
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_sapbil_vs_banditsa.py
RENAMED
|
File without changes
|
|
File without changes
|
{searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/zero_inflated_ordered_probit.py
RENAMED
|
File without changes
|
{searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|