SearchLibrium 0.0.268__tar.gz → 0.0.271__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {searchlibrium-0.0.268/src/SearchLibrium.egg-info → searchlibrium-0.0.271}/PKG-INFO +1 -1
  2. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/pyproject.toml +1 -1
  3. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/__init__.py +1 -0
  4. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/larch_models.py +9 -3
  5. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/latent_class.py +86 -0
  6. searchlibrium-0.0.271/src/SearchLibrium/lc_rail_guard.py +139 -0
  7. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mh_choice_estimator.py +208 -34
  8. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/numba_rrm.py +32 -1
  9. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/rrm.py +129 -2
  10. searchlibrium-0.0.271/src/SearchLibrium/version.txt +1 -0
  11. {searchlibrium-0.0.268 → searchlibrium-0.0.271/src/SearchLibrium.egg-info}/PKG-INFO +1 -1
  12. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/SOURCES.txt +1 -0
  13. searchlibrium-0.0.268/src/SearchLibrium/version.txt +0 -1
  14. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/LICENSE +0 -0
  15. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/MANIFEST.in +0 -0
  16. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/README.md +0 -0
  17. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/setup.cfg +0 -0
  18. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/BEHier.py +0 -0
  19. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/Halton.py +0 -0
  20. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/MixedLogit.py +0 -0
  21. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/MixedLogitGSE.py +0 -0
  22. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/Mode_Activity_Nested.py +0 -0
  23. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/RandomP.py +0 -0
  24. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/SEARCH_SM_MARIO.py +0 -0
  25. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/Two_Level_Nest.py +0 -0
  26. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/__main__.py +0 -0
  27. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/_choice_model.py +0 -0
  28. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/_device.py +0 -0
  29. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/abm.py +0 -0
  30. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/accessibility.py +0 -0
  31. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/banditsa.py +0 -0
  32. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/boxcox_functions.py +0 -0
  33. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/call_meta.py +0 -0
  34. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/constraints_builder.py +0 -0
  35. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/coupled_solver.py +0 -0
  36. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/data/Swissmetro_final.csv +0 -0
  37. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/data/TravelMode.csv +0 -0
  38. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/data/electricity.csv +0 -0
  39. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/destination_distances.py +0 -0
  40. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/harmony.py +0 -0
  41. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/hspbil.py +0 -0
  42. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/jax_utils.py +0 -0
  43. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/larch_compat.py +0 -0
  44. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/latent_spec.py +0 -0
  45. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/logistic_regression.py +0 -0
  46. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/main.py +0 -0
  47. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/main_debug.py +0 -0
  48. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mdcev.py +0 -0
  49. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/misc.py +0 -0
  50. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mixed_logit.py +0 -0
  51. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mixed_nested.py +0 -0
  52. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/mixedrrm.py +0 -0
  53. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/multinomial_logit.py +0 -0
  54. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/multinomial_nested.py +0 -0
  55. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/multinomial_probit.py +0 -0
  56. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/numba_engine.py +0 -0
  57. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/numba_mxl.py +0 -0
  58. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/ordered_logit.py +0 -0
  59. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/ordered_logit_mixed.py +0 -0
  60. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/predict.py +0 -0
  61. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/regularization.py +0 -0
  62. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/sample_data.py +0 -0
  63. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/sapbil.py +0 -0
  64. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/search.py +0 -0
  65. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/selection_models.py +0 -0
  66. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/setup.py +0 -0
  67. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/siman.py +0 -0
  68. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/skims.py +0 -0
  69. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/sparseea_agds.py +0 -0
  70. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_lc_de.py +0 -0
  71. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_mario_searches.py +0 -0
  72. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/test_sapbil_vs_banditsa.py +0 -0
  73. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/threshold.py +0 -0
  74. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium/zero_inflated_ordered_probit.py +0 -0
  75. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/dependency_links.txt +0 -0
  76. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/entry_points.txt +0 -0
  77. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/requires.txt +0 -0
  78. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/src/SearchLibrium.egg-info/top_level.txt +0 -0
  79. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_destination_distances.py +0 -0
  80. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_jax_mask_equivalence.py +0 -0
  81. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_larch_integration.py +0 -0
  82. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_lc_smart_init.py +0 -0
  83. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_mh_choice_estimator.py +0 -0
  84. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_mxl_random_starts.py +0 -0
  85. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_per_alternative_random.py +0 -0
  86. {searchlibrium-0.0.268 → searchlibrium-0.0.271}/tests/test_zero_inflated_ordered_probit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SearchLibrium
3
- Version: 0.0.268
3
+ Version: 0.0.271
4
4
  Summary: Automated specification search for discrete choice models (logit, mixed logit, nested logit, RRM and more) using metaheuristic optimisation
5
5
  Author: Zeke Ahern, Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
6
6
  Maintainer-email: Zeke Ahern <z.ahern@qut.edu.au>
@@ -132,7 +132,7 @@ Documentation = "https://pypi.org/project/SearchLibrium/"
132
132
  realpython = "SearchLibrium.__main__:main"
133
133
 
134
134
  [tool.bumpver]
135
- current_version = "0.0.268"
135
+ current_version = "0.0.271"
136
136
  version_pattern = "MAJOR.MINOR.PATCH"
137
137
  commit_message = "[skip ci] Bump version {old_version} -> {new_version}"
138
138
  commit = true
@@ -170,6 +170,7 @@ _IMPORT_GROUPS = [
170
170
  ('zero_inflated_ordered_probit', ['ZeroInflatedOrderedProbit',
171
171
  'ZeroInflatedProbit']),
172
172
  ('latent_class', ['LatentClassMixedLogit']),
173
+ ('lc_rail_guard', ['lc_result_rail']),
173
174
  ('latent_spec', ['vip_table', 'near_unidentified_report', 'align_classes',
174
175
  'propose_class_params_spec']),
175
176
  ('mdcev', ['MDCEVFitResult', 'MDCEVModel']),
@@ -410,7 +410,11 @@ class _LarchBase(DiscreteChoiceModel):
410
410
  else:
411
411
  optimizer_success = bool(
412
412
  optimizer_result.get("success", False))
413
- optimizer_nit = int(optimizer_result.get("nit", 0) or 0)
413
+ optimizer_nit = int(
414
+ optimizer_result.get(
415
+ "nit", optimizer_result.get("iterations", 0)
416
+ ) or 0
417
+ )
414
418
  ok = bool(np.isfinite(ll) and optimizer_success)
415
419
  except Exception as e: # noqa: BLE001
416
420
  logger.debug("larch finalize failed: %r", e)
@@ -719,9 +723,10 @@ class LarchLatentClass(_LarchBase):
719
723
  """
720
724
 
721
725
  def setup(self, *args, class_params_spec=None, member_params_spec=None,
722
- num_classes=2, **kwargs):
726
+ num_classes=2, membership_intercept=True, **kwargs):
723
727
  super().setup(*args, **kwargs)
724
728
  self.num_classes = int(num_classes)
729
+ self.membership_intercept = bool(membership_intercept)
725
730
  if self.num_classes < 2:
726
731
  raise ValueError("Larch latent class models require at least two classes")
727
732
  self.class_params_spec = [
@@ -812,7 +817,8 @@ class LarchLatentClass(_LarchBase):
812
817
  else lx.Model(compute_engine="jax",
813
818
  float_dtype=self.float_dtype))
814
819
  for class_number in range(1, K):
815
- expr = P(f"LCM{class_number}_intercept")
820
+ expr = (P(f"LCM{class_number}_intercept")
821
+ if self.membership_intercept else 0)
816
822
  for variable in member_vars:
817
823
  expr = expr + P(f"LCM{class_number}_{variable}") * LX(variable)
818
824
  membership.utility_co[class_number] = expr
@@ -1236,6 +1236,90 @@ class LatentClassMixedLogit(DiscreteChoiceModel):
1236
1236
  )
1237
1237
  return collapsed
1238
1238
 
1239
+ def _compute_rail_flags(self):
1240
+ """Flag bound-pinned, degenerate-SE and collapsed-class fits as railed.
1241
+
1242
+ Rail beats the optimizer's own success flag: a fit pressed against a
1243
+ bound has an unusable estimate even when ``converged`` is True.
1244
+ """
1245
+ reasons, warns = [], []
1246
+ try:
1247
+ shares = np.asarray(getattr(self, 'class_probs', []),
1248
+ dtype=float).ravel()
1249
+ min_share = float(getattr(self, 'min_share', 0.05))
1250
+ if shares.size and float(np.nanmin(shares)) < min_share:
1251
+ reasons.append(
1252
+ f"class collapsed (min share={float(np.nanmin(shares)):.4g}"
1253
+ f" < {min_share:g})")
1254
+ except Exception:
1255
+ pass
1256
+ try:
1257
+ betas = getattr(self, 'class_betas', None)
1258
+ if isinstance(betas, list) and len(betas) > 1:
1259
+ arrs = [np.asarray(b, dtype=float).ravel() for b in betas]
1260
+ if all(a.size == arrs[0].size for a in arrs):
1261
+ maxdiff = max(float(np.max(np.abs(a - arrs[0])))
1262
+ for a in arrs[1:])
1263
+ if np.isfinite(maxdiff) and maxdiff < 1e-6:
1264
+ reasons.append("classes identical on betas "
1265
+ f"(max |diff|={maxdiff:.1e})")
1266
+ except Exception:
1267
+ pass
1268
+ try:
1269
+ from .lc_rail_guard import rail_flags
1270
+ except Exception:
1271
+ rail_flags = None
1272
+ if rail_flags is not None:
1273
+ try:
1274
+ params = np.asarray(getattr(self, 'coeff_est', []),
1275
+ dtype=float).ravel()
1276
+ names = [str(n) for n in getattr(self, 'coeff_names', [])]
1277
+ se_obj = getattr(self, 'stderr', None)
1278
+ ses = (np.asarray(se_obj, dtype=float).ravel()
1279
+ if se_obj is not None else np.array([]))
1280
+ if params.size and params.size == ses.size == len(names):
1281
+ rf = rail_flags(dict(zip(names, params)),
1282
+ dict(zip(names, ses)))
1283
+ reasons.extend(rf['rail'])
1284
+ warns.extend(rf['warnings'])
1285
+ except Exception as exc:
1286
+ warns.append(
1287
+ f"rail scan skipped: {type(exc).__name__}: {exc}")
1288
+ try:
1289
+ g = getattr(self, 'class_gammas', None)
1290
+ gse_obj = getattr(self, 'gamma_se', None)
1291
+ gse = (np.asarray(gse_obj, dtype=float).ravel()
1292
+ if gse_obj is not None else np.array([]))
1293
+ if g is not None:
1294
+ ga = np.asarray(g, dtype=float).ravel()
1295
+ bound = getattr(self, 'gamma_max_abs', None)
1296
+ gmax = float(np.max(np.abs(ga))) if ga.size else 0.0
1297
+ if (bound is not None and ga.size
1298
+ and gmax >= float(bound) - 1e-3):
1299
+ reasons.append(
1300
+ f"membership gamma at bound {float(bound):g} "
1301
+ f"(max |gamma|={gmax:.4g})")
1302
+ elif ga.size and gmax >= 50.0:
1303
+ reasons.append(
1304
+ f"membership gamma saturated "
1305
+ f"(max |gamma|={gmax:.3g})")
1306
+ if gse.size == ga.size and ga.size:
1307
+ rf = rail_flags(
1308
+ {f"gamma_{i}": float(v)
1309
+ for i, v in enumerate(ga)},
1310
+ {f"gamma_{i}": float(s)
1311
+ for i, s in enumerate(gse)})
1312
+ reasons.extend(rf['rail'])
1313
+ warns.extend(rf['warnings'])
1314
+ except Exception:
1315
+ pass
1316
+ self.rail_reasons = reasons
1317
+ self.rail_warnings = warns
1318
+ self.railed = bool(reasons)
1319
+ if self.railed:
1320
+ self.converged = False
1321
+ return self.railed
1322
+
1239
1323
  def _panel_sum(self, arr_task):
1240
1324
  """Sum a per-task array ``(N_task, C)`` into per-panel ``(n_panels, C)``.
1241
1325
 
@@ -1556,6 +1640,7 @@ class LatentClassMixedLogit(DiscreteChoiceModel):
1556
1640
  self.estim_time_sec = time.time() - start_time
1557
1641
  self.post_process()
1558
1642
  self._detect_collapse()
1643
+ self._compute_rail_flags()
1559
1644
 
1560
1645
  return self
1561
1646
 
@@ -1722,6 +1807,7 @@ class LatentClassMixedLogit(DiscreteChoiceModel):
1722
1807
  else:
1723
1808
  self.sort_perm_ = np.arange(C, dtype=int)
1724
1809
  self._detect_collapse()
1810
+ self._compute_rail_flags()
1725
1811
 
1726
1812
  return self
1727
1813
 
@@ -0,0 +1,139 @@
1
+ """Rail ("parameter pinned at a bound") detection for latent-class fits.
2
+
3
+ A rail is a parameter pressed against a hard bound or a coefficient whose
4
+ standard error is numerically degenerate. Either makes the estimate unusable
5
+ even when the optimizer reports success, so railed fits must not be treated as
6
+ usable solutions.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import numpy as np
11
+
12
+
13
+ def rail_flags(values, ses=None, bounds=None, tol=1e-3, se_degenerate=1e5):
14
+ """Flag bound-pinned parameters and degenerate standard errors.
15
+
16
+ Parameters
17
+ ----------
18
+ values, ses : dict name -> float. ``ses`` is optional.
19
+ bounds : dict name -> (lo, hi); None disables a side. A key also matches
20
+ every parameter whose name starts with it.
21
+ tol : absolute tolerance for "sitting on the bound".
22
+ se_degenerate : |SE| at/above this is a rail.
23
+ """
24
+ rail = []
25
+ warnings = []
26
+ ses = {str(k): v for k, v in (ses or {}).items()}
27
+ bounds = dict(bounds or {})
28
+ for name, val in (values or {}).items():
29
+ name = str(name)
30
+ try:
31
+ v = float(val)
32
+ except Exception:
33
+ continue
34
+ if not np.isfinite(v):
35
+ continue
36
+ lo = hi = None
37
+ for key, bd in bounds.items():
38
+ if name == str(key) or name.startswith(str(key)):
39
+ if isinstance(bd, (tuple, list)) and len(bd) == 2:
40
+ lo, hi = bd
41
+ break
42
+ if lo is not None and np.isfinite(lo) and v <= float(lo) + tol:
43
+ rail.append(f"{name} at/below lower bound {float(lo):g} "
44
+ f"(value={v:.4g})")
45
+ if hi is not None and np.isfinite(hi) and v >= float(hi) - tol:
46
+ rail.append(f"{name} at/above upper bound {float(hi):g} "
47
+ f"(value={v:.4g})")
48
+ s = ses.get(name)
49
+ if s is not None:
50
+ try:
51
+ s = float(s)
52
+ except Exception:
53
+ s = None
54
+ if s is not None and np.isfinite(s):
55
+ if s >= se_degenerate:
56
+ rail.append(f"{name} has degenerate SE {s:.1e} "
57
+ f"(flat/unidentified direction)")
58
+ elif s <= 0.0:
59
+ warnings.append(f"{name} SE=0 (parameter locked/fixed)")
60
+ return {"rail": rail, "railed": bool(rail), "warnings": warnings}
61
+
62
+
63
+ def lc_result_rail(names, betas, ses, shares, membership_coefs=None,
64
+ bounds=15.0, share_floor=0.05, method="lc-em",
65
+ tol=1e-3, se_degenerate=1e5):
66
+ """Rail scan for ``estimate_dest_choice_lc`` style result arrays."""
67
+ rail = []
68
+ warnings = []
69
+ names = [str(n) for n in names]
70
+ for c, b in enumerate(betas):
71
+ try:
72
+ arr = np.asarray(b, dtype=float).ravel()
73
+ except Exception:
74
+ continue
75
+ se_arr = None
76
+ if ses is not None and c < len(ses):
77
+ try:
78
+ candidate = np.asarray(ses[c], dtype=float).ravel()
79
+ if candidate.size == arr.size:
80
+ se_arr = candidate
81
+ except Exception:
82
+ se_arr = None
83
+ for j, nm in enumerate(names):
84
+ if j >= arr.size:
85
+ break
86
+ v = float(arr[j])
87
+ if np.isfinite(v) and bounds is not None:
88
+ bound = float(bounds)
89
+ if v >= bound - tol:
90
+ rail.append(f"class {c + 1} {nm} at/above beta bound "
91
+ f"{bound:g} (value={v:.4g})")
92
+ elif v <= -bound + tol:
93
+ rail.append(f"class {c + 1} {nm} at/below beta bound "
94
+ f"{-bound:g} (value={v:.4g})")
95
+ if se_arr is not None:
96
+ s = float(se_arr[j])
97
+ if np.isfinite(s):
98
+ if s >= se_degenerate:
99
+ rail.append(f"class {c + 1} {nm} has degenerate SE "
100
+ f"{s:.1e} (flat/unidentified direction)")
101
+ elif s <= 0.0:
102
+ warnings.append(f"class {c + 1} {nm} SE=0 "
103
+ f"(parameter locked/fixed)")
104
+ if method == "lc-em":
105
+ try:
106
+ sh = np.asarray(shares, dtype=float).ravel()
107
+ floor = float(share_floor)
108
+ for c, s in enumerate(sh):
109
+ if not np.isfinite(s):
110
+ continue
111
+ if s < 0.5 * floor:
112
+ rail.append(f"class {c + 1} collapsed: share {s:.4g} < "
113
+ f"{0.5 * floor:g}")
114
+ elif s < floor:
115
+ warnings.append(f"class {c + 1} share {s:.4g} below "
116
+ f"floor {floor:g}")
117
+ except Exception:
118
+ pass
119
+ if len(betas) > 1:
120
+ try:
121
+ arrs = [np.asarray(b, dtype=float).ravel() for b in betas]
122
+ if all(a.size == arrs[0].size for a in arrs):
123
+ maxdiff = max(float(np.max(np.abs(a - arrs[0])))
124
+ for a in arrs[1:])
125
+ if np.isfinite(maxdiff) and maxdiff < 1e-6:
126
+ rail.append("classes identical on betas "
127
+ f"(max |diff|={maxdiff:.1e})")
128
+ except Exception:
129
+ pass
130
+ if membership_coefs is not None:
131
+ try:
132
+ G = np.asarray(membership_coefs, dtype=float)
133
+ gmax = float(np.max(np.abs(G))) if G.size else 0.0
134
+ if np.isfinite(gmax) and gmax >= 50.0:
135
+ rail.append("membership gamma saturated "
136
+ f"(max |gamma|={gmax:.3g})")
137
+ except Exception:
138
+ pass
139
+ return {"rail": rail, "railed": bool(rail), "warnings": warnings}
@@ -212,12 +212,18 @@ class IdentificationReport:
212
212
  eigenvalues: np.ndarray
213
213
  suggested_drop: tuple[str, ...] = ()
214
214
  weak_directions: tuple[dict, ...] = ()
215
+ # Features that do not vary within a case. In a conditional logit such a
216
+ # column is a case-level constant and cancels from the probability, so it
217
+ # is unidentified whatever the information matrix looks like -- see the
218
+ # constant-column pass in identify_weak_features.
219
+ constant_features: tuple[str, ...] = ()
215
220
 
216
221
  def as_dict(self) -> dict:
217
222
  return {
218
223
  "condition_number": float(self.condition_number),
219
224
  "eigenvalues": [float(v) for v in np.asarray(self.eigenvalues).ravel()],
220
225
  "suggested_drop": list(self.suggested_drop),
226
+ "constant_features": list(self.constant_features),
221
227
  "weak_directions": [
222
228
  {
223
229
  "eigenvalue": float(d["eigenvalue"]),
@@ -241,6 +247,7 @@ def identify_weak_features(
241
247
  offset_col: str = "log_correction",
242
248
  max_drop: int | None = None,
243
249
  protected: Sequence[str] = (),
250
+ constant_tol: float = 1e-10,
244
251
  ) -> IdentificationReport:
245
252
  """Find structurally weak directions in a long choice-set frame.
246
253
 
@@ -248,6 +255,16 @@ def identify_weak_features(
248
255
  ``possible_overspec.py``) for the sampled MNL used here: eigendecompose the
249
256
  scaled exact information matrix at ``beta`` (zeros when omitted) and return
250
257
  one feature per near-null direction as ``suggested_drop``.
258
+
259
+ A separate pass catches features that are CONSTANT WITHIN A CASE. Those are
260
+ case-level constants in a conditional logit -- they cancel from the
261
+ probability -- so the estimator leaves them at the ridge (coefficient ~0)
262
+ however much the column varies across cases. The eigen test cannot see
263
+ them: it rescales by ``sqrt(diag(info))``, and for such a column that
264
+ diagonal is its own numerical residue, so the column reports a unit
265
+ "self-correlation" and no weak direction. The share of a feature's
266
+ variation that lies within cases is the direct test, and any feature below
267
+ ``constant_tol`` of it is reported in ``constant_features``.
251
268
  """
252
269
  frame = data if isinstance(data, ChoiceSetFrame) else ChoiceSetFrame.from_long(
253
270
  data, feature_cols, case_col=case_col, alt_col=alt_col,
@@ -260,35 +277,75 @@ def identify_weak_features(
260
277
  else:
261
278
  beta_vec = np.asarray(beta, dtype=float).reshape(-1)
262
279
  info, _, _ = frame.information(beta_vec)
263
- diag = np.clip(np.diag(info), 1e-300, None)
264
- scale = np.sqrt(diag)
265
- scaled = info / np.outer(scale, scale)
266
- evals, evecs = np.linalg.eigh(0.5 * (scaled + scaled.T))
267
- condition = float(evals[-1] / max(evals[0], 1e-300))
268
- weak = np.flatnonzero(evals < float(tol))
269
280
  protected_set = {str(p) for p in (protected or ())}
270
281
  drops: list[str] = []
271
282
  directions: list[dict] = []
283
+
284
+ # ── Case-constant columns (before the eigen pass) ──────────────────────
285
+ # Within-case variation is the only variation a conditional logit can use.
286
+ # A column without it is unidentified whatever the information matrix says
287
+ # -- and what it says is numerical noise: the row is a difference of two
288
+ # ~equal quantities, so its diagonal is a residue and the scaled matrix
289
+ # picks up huge NEGATIVE eigenvalues that then masquerade as weak
290
+ # directions. Identify the constant columns first and run the eigen pass on
291
+ # the remaining submatrix, so those artifacts cannot produce a drop list
292
+ # (and cannot displace the real loaders by already-claimed names).
293
+ d = frame.dimension
294
+ s1 = np.zeros(d)
295
+ s2 = np.zeros(d)
296
+ within_ss = np.zeros(d)
297
+ n_rows = 0
298
+ for x in frame.x:
299
+ xi = np.asarray(x, dtype=float)
300
+ if xi.ndim != 2 or xi.shape[0] < 2:
301
+ continue
302
+ s1 += xi.sum(axis=0)
303
+ s2 += (xi * xi).sum(axis=0)
304
+ n_rows += xi.shape[0]
305
+ within_ss += ((xi - xi.mean(axis=0)) ** 2).sum(axis=0)
306
+ total_ss = np.clip(s2 - (s1 * s1) / max(n_rows, 1), 0.0, None)
307
+ share = np.divide(within_ss, total_ss,
308
+ out=np.zeros(d), where=total_ss > 0)
309
+ # ``where=total_ss > 0`` leaves the share at 0 for a column with no variance
310
+ # at all, which is equally unidentified, so one test covers both.
311
+ constant_mask = share <= float(constant_tol)
312
+ constant = [names[j] for j in np.flatnonzero(constant_mask)
313
+ if names[j] not in protected_set]
314
+ for name in constant:
315
+ directions.append({"eigenvalue": 0.0, "loadings": {name: 1.0}})
316
+ drops.extend(constant)
317
+
318
+ keep = np.flatnonzero(~constant_mask)
319
+ keep_names = [names[j] for j in keep]
320
+ sub = info[np.ix_(keep, keep)]
321
+ diag = np.clip(np.diag(sub), 1e-300, None)
322
+ scale = np.sqrt(diag)
323
+ scaled = sub / np.outer(scale, scale)
324
+ evals, evecs = np.linalg.eigh(0.5 * (scaled + scaled.T))
325
+ condition = float(evals[-1] / max(evals[0], 1e-300)) if evals.size else float("nan")
326
+ weak = np.flatnonzero(evals < float(tol))
327
+
272
328
  for idx in weak:
273
329
  # Unscale the direction: v_j = evec_j / sqrt(info_jj).
274
330
  v = evecs[:, idx] / scale
275
331
  v = v / (np.linalg.norm(v) + 1e-300)
276
332
  order = np.argsort(-np.abs(v))
277
- top = {names[j]: float(v[j]) for j in order[: min(8, len(order))]}
333
+ top = {keep_names[j]: float(v[j]) for j in order[: min(8, len(order))]}
278
334
  directions.append({"eigenvalue": float(evals[idx]), "loadings": top})
279
335
  # Never drop user-protected features (e.g. explicitly requested
280
336
  # behavioural interactions); fall through to the next-largest loader.
281
337
  j = next((int(k) for k in order
282
- if names[int(k)] not in protected_set), None)
338
+ if keep_names[int(k)] not in protected_set), None)
283
339
  if j is None:
284
340
  continue
285
- if abs(v[j]) >= float(loading_threshold) and names[j] not in drops:
286
- drops.append(names[j])
341
+ if abs(v[j]) >= float(loading_threshold) and keep_names[j] not in drops:
342
+ drops.append(keep_names[j])
287
343
  if max_drop is not None and len(drops) > int(max_drop):
288
344
  drops = drops[: int(max_drop)]
289
345
  return IdentificationReport(
290
346
  condition_number=condition, eigenvalues=evals,
291
- suggested_drop=tuple(drops), weak_directions=tuple(directions))
347
+ suggested_drop=tuple(drops), weak_directions=tuple(directions),
348
+ constant_features=tuple(constant))
292
349
 
293
350
 
294
351
  class GaussianRandomWalkProposal:
@@ -1338,12 +1395,23 @@ def _fit_membership_gammas(case_X, resp, l2: float = 1.0, maxiter: int = 200,
1338
1395
 
1339
1396
 
1340
1397
  def _fit_frame_mnl(frame: "ChoiceSetFrame", x0, *, prior_scale: float = 5.0,
1341
- bounds: float = 15.0, maxiter: int = 500, tol: float = 1e-9):
1398
+ bounds: float = 15.0, maxiter: int = 500, tol: float = 1e-9,
1399
+ grad_tol: float = 1e-6, retries: int = 2):
1342
1400
  """Weighted MNL fit on a (copied) frame: L-BFGS-B on the analytic gradient.
1343
1401
 
1344
1402
  Mirrors larch's ``minimize(logloss, jac=d_logloss, bounds=pbounds)`` with
1345
1403
  a weak N(0, prior_scale^2) ridge so unidentified directions are bounded.
1346
1404
  The Manski-Lerman offset travels inside the frame, so it is exact here.
1405
+
1406
+ ``ftol`` alone is a relative decrease test, so L-BFGS-B can stop with a
1407
+ large gradient and still report success -- and can stop short of the
1408
+ optimum when it runs out of iterations. The fit is therefore re-started
1409
+ from its own solution while the gradient is too large, keeping the better
1410
+ objective value, and the returned result carries the evidence
1411
+ (``grad_inf``, ``grad_tol``, ``fit_message``) alongside ``success``, which
1412
+ is true when either scipy succeeded or the gradient shows a stationary
1413
+ point. The threshold is per observation, since the likelihood scales with
1414
+ the sample.
1347
1415
  """
1348
1416
  from scipy.optimize import minimize
1349
1417
  d = frame.dimension
@@ -1355,10 +1423,49 @@ def _fit_frame_mnl(frame: "ChoiceSetFrame", x0, *, prior_scale: float = 5.0,
1355
1423
  g = grad - prior_prec * np.asarray(beta, dtype=float)
1356
1424
  return -total, -g
1357
1425
 
1358
- res = minimize(nll, np.asarray(x0, dtype=float), jac=True,
1359
- method="L-BFGS-B",
1360
- bounds=[(-float(bounds), float(bounds))] * d,
1361
- options={"maxiter": int(maxiter), "ftol": float(tol)})
1426
+ try:
1427
+ g_tol = float(grad_tol) * max(int(frame.n_cases), 1)
1428
+ except Exception:
1429
+ g_tol = float(grad_tol)
1430
+
1431
+ def _run(start, n_iter):
1432
+ # ftol is a relative-decrease test on an objective of order 1e3, so on
1433
+ # its own it stops while the gradient is still ~1e-2. Stop on the
1434
+ # gradient; ftol is only kept as a floating-point floor.
1435
+ return minimize(nll, np.asarray(start, dtype=float), jac=True,
1436
+ method="L-BFGS-B",
1437
+ bounds=[(-float(bounds), float(bounds))] * d,
1438
+ options={"maxiter": int(n_iter),
1439
+ "ftol": min(float(tol), 1e-15),
1440
+ "gtol": g_tol})
1441
+
1442
+ res = _run(x0, maxiter)
1443
+ left = int(retries)
1444
+ while left > 0:
1445
+ g = np.asarray(nll(np.asarray(res.x, dtype=float))[1], dtype=float)
1446
+ g_inf = float(np.max(np.abs(g))) if g.size else float("nan")
1447
+ if not np.isfinite(g_inf) or g_inf <= g_tol:
1448
+ break
1449
+ left -= 1
1450
+ try:
1451
+ retry = _run(res.x, max(2 * int(maxiter), int(maxiter)))
1452
+ except Exception:
1453
+ break
1454
+ if not np.isfinite(float(getattr(retry, "fun", np.inf))) \
1455
+ or float(retry.fun) > float(res.fun) - 1e-9:
1456
+ break # no progress: keep what we had
1457
+ res = retry
1458
+
1459
+ try:
1460
+ g = np.asarray(nll(np.asarray(res.x, dtype=float))[1], dtype=float)
1461
+ g_inf = float(np.max(np.abs(g))) if g.size else float("nan")
1462
+ except Exception:
1463
+ g_inf = float("nan")
1464
+ res.grad_inf = g_inf
1465
+ res.grad_tol = g_tol
1466
+ res.fit_message = str(getattr(res, "message", "") or "")
1467
+ res.converged_evidence = bool(np.isfinite(g_inf) and g_inf <= g_tol)
1468
+ res.success = bool(getattr(res, "success", False) or res.converged_evidence)
1362
1469
  return np.asarray(res.x, dtype=float), float(-res.fun), res
1363
1470
 
1364
1471
 
@@ -1501,7 +1608,8 @@ def estimate_dest_choice_lc(
1501
1608
  fit_ll = float(sum(_weighted_frame(frame, (labels == g).astype(float))
1502
1609
  .loglike(b) for g, b in zip(groups, betas)))
1503
1610
  return _lc_result("grouped", names, betas, ses, shares, fit_ll,
1504
- float("nan"), True, 0, frame, labels=groups)
1611
+ float("nan"), True, 0, frame, labels=groups,
1612
+ bounds=float(bounds), share_floor=float(share_floor))
1505
1613
 
1506
1614
  # ── Latent classes: EM with weighted-MNL M-steps ──────────────────────
1507
1615
  # Class priors are either fixed shares or a person-type membership
@@ -1540,6 +1648,14 @@ def estimate_dest_choice_lc(
1540
1648
  shares = np.full(n_classes, 1.0 / n_classes)
1541
1649
  gammas = (np.zeros((n_classes - 1, case_X.shape[1] + 1))
1542
1650
  if case_X is not None else None)
1651
+ if _start and case_X is not None:
1652
+ # The header promises a jitter of "BOTH the class-specific betas AND
1653
+ # the membership gammas". Only the betas were ever jittered: every
1654
+ # restart began from a ZERO membership equation, i.e. uniform class
1655
+ # priors, so restarts differed solely in the choice coefficients and
1656
+ # the membership surface was effectively estimated once. Perturb it
1657
+ # too so a 3-class fit can leave the symmetric starting point.
1658
+ gammas = rng.normal(0.0, float(perturb_sigma), gammas.shape)
1543
1659
  prev_ll = -np.inf
1544
1660
  converged = False
1545
1661
  iterations = 0
@@ -1563,20 +1679,38 @@ def estimate_dest_choice_lc(
1563
1679
  p = np.clip(p, float(share_floor) / n_classes, None)
1564
1680
  p /= p.sum(axis=1, keepdims=True)
1565
1681
  new_shares = p.mean(axis=0)
1566
- # M-step: weighted MNL per class + membership gammas
1567
- betas = [_fit_frame_mnl(_weighted_frame(frame, p[:, c]), betas[c],
1568
- prior_scale=prior_scale, bounds=bounds)[0]
1569
- for c in range(n_classes)]
1682
+ # M-step: weighted MNL per class + membership gammas.
1683
+ # Keep each fit's first-order optimality: the optimizer flag only
1684
+ # says the M-step stopped, the gradient says it stopped at an
1685
+ # optimum, and without it the reported class solution carries no
1686
+ # evidence about the M-steps at all.
1687
+ # NB: warm start from the previous iteration's betas, so build the
1688
+ # new vectors in a separate list rather than clearing `betas` first.
1689
+ _prev = betas
1690
+ betas = []
1691
+ mstep_grad = []
1692
+ for c in range(n_classes):
1693
+ _b, _, _r = _fit_frame_mnl(_weighted_frame(frame, p[:, c]),
1694
+ _prev[c], prior_scale=prior_scale,
1695
+ bounds=bounds)
1696
+ betas.append(_b)
1697
+ mstep_grad.append(float(getattr(_r, "grad_inf", float("nan"))))
1570
1698
  if case_X is not None:
1571
1699
  gammas = _fit_membership_gammas(case_X, p, l2=membership_l2, standardize=True)
1572
1700
  shares = new_shares
1573
1701
  if share_floor and float(share_floor) > 0:
1574
1702
  _coll = int(np.argmin(shares))
1575
1703
  if float(shares[_coll]) < 0.5 * float(share_floor):
1576
- # Re-seed the collapsed class from the pooled fit with a
1577
- # fresh jitter + crash anchor (a perturbation move, not a
1578
- # failure) so the next E-step can differentiate it.
1579
- _b = beta0 + rng.normal(0.0, float(perturb_sigma), d)
1704
+ # Re-seed the collapsed class so the next E-step can
1705
+ # differentiate it. This used to overwrite the class with
1706
+ # `beta0 + jitter`, where beta0 is the POOLED fit -- so a
1707
+ # class that was genuinely small got reset to the common
1708
+ # pooled surface on EVERY iteration it stayed small, could
1709
+ # never differentiate itself, and EM stalled. Perturb the
1710
+ # class's own current estimate instead: that is the
1711
+ # "perturbation move, not a failure" the comment claims, and
1712
+ # it preserves whatever separation the M-step had built.
1713
+ _b = betas[_coll] + rng.normal(0.0, float(perturb_sigma), d)
1580
1714
  if crash_cols:
1581
1715
  _b[names.index(crash_cols[0])] += crash_anchors[
1582
1716
  _coll % len(crash_anchors)]
@@ -1594,19 +1728,32 @@ def estimate_dest_choice_lc(
1594
1728
  p = np.exp(llw - m)
1595
1729
  p /= p.sum(axis=1, keepdims=True)
1596
1730
  restart_lls.append(fit_ll)
1597
- if best is None or fit_ll > best[0]:
1731
+ # Rank restarts by (converged, fit_ll), NOT by fit_ll alone. Picking
1732
+ # purely on log-likelihood let a restart that wandered without settling
1733
+ # beat one that had genuinely converged; the caller then rejected the
1734
+ # whole genome ("not converged"), discarding the only usable restart.
1735
+ # That is the mechanism behind the paper-arm runs reporting arm D as
1736
+ # NOT converged at full sample while the n=200 smoke did converge.
1737
+ _rank = (bool(converged), fit_ll)
1738
+ if best is None or _rank > (bool(best[4]), best[0]):
1598
1739
  best = (fit_ll, betas, shares, p, converged, iterations,
1599
- None if gammas is None else np.asarray(gammas).copy())
1600
- fit_ll, betas, shares, resp, converged, iterations, gammas = best
1740
+ None if gammas is None else np.asarray(gammas).copy(),
1741
+ mstep_grad)
1742
+ fit_ll, betas, shares, resp, converged, iterations, gammas, mstep_grad = best
1601
1743
  if case_X is None:
1602
1744
  # label switching only reorders the fixed-share solution; with a
1603
1745
  # membership equation the reference class structure must be kept.
1604
1746
  order = np.argsort(-np.asarray(shares))
1605
1747
  betas = [betas[i] for i in order]
1606
- shares = np.asarray(shares)[order]
1607
- resp = resp[:, order]
1608
- else:
1609
- shares = np.asarray(shares)
1748
+ resp = np.asarray(resp)[:, order]
1749
+ mstep_grad = [mstep_grad[i] for i in order]
1750
+ # Derive the reported shares from the SAME responsibilities that ship with
1751
+ # the result. `shares` carried out of the EM loop was the mean of the
1752
+ # share_floor-CLIPPED responsibilities, while the returned `resp` is the
1753
+ # unclipped posterior, so resp.mean(axis=0) did not reproduce shares -- and
1754
+ # stage5_lc_genes' collapse gate reads `shares` while a consumer reading
1755
+ # `responsibilities` would compute something different.
1756
+ shares = np.asarray(resp).mean(axis=0)
1610
1757
  ses = []
1611
1758
  for c in range(n_classes):
1612
1759
  fw = _weighted_frame(frame, resp[:, c])
@@ -1616,13 +1763,17 @@ def estimate_dest_choice_lc(
1616
1763
  return _lc_result("lc-em", names, betas, ses, shares, fit_ll, float("nan"),
1617
1764
  converged, iterations, frame, responsibilities=resp,
1618
1765
  membership_coefs=gammas, membership_vars=memb_cols,
1619
- crash_cols=crash_cols, restart_lls=restart_lls)
1766
+ crash_cols=crash_cols, restart_lls=restart_lls,
1767
+ bounds=float(bounds), share_floor=float(share_floor),
1768
+ mstep_grad_inf=mstep_grad,
1769
+ mstep_grad_tol=1e-6 * frame.n_cases)
1620
1770
 
1621
1771
 
1622
1772
  def _lc_result(method, names, betas, ses, shares, fit_ll, _unused, converged,
1623
1773
  iterations, frame, labels=None, responsibilities=None,
1624
1774
  membership_coefs=None, membership_vars=None, crash_cols=None,
1625
- restart_lls=None) -> dict:
1775
+ restart_lls=None, bounds=15.0, share_floor=0.05,
1776
+ mstep_grad_inf=None, mstep_grad_tol=None) -> dict:
1626
1777
  n_classes = len(betas)
1627
1778
  k_params = n_classes * frame.dimension + (n_classes - 1)
1628
1779
  if membership_vars:
@@ -1644,6 +1795,29 @@ def _lc_result(method, names, betas, ses, shares, fit_ll, _unused, converged,
1644
1795
  "converged": bool(converged),
1645
1796
  "iterations": int(iterations),
1646
1797
  }
1798
+ if mstep_grad_inf is not None:
1799
+ _mg = [float(v) for v in mstep_grad_inf]
1800
+ _mt = None if mstep_grad_tol is None else float(mstep_grad_tol)
1801
+ out["mstep_grad_inf"] = _mg
1802
+ out["mstep_grad_tol"] = _mt
1803
+ out["mstep_stationary"] = [bool(_mt is not None and np.isfinite(v)
1804
+ and v <= _mt) for v in _mg]
1805
+ try:
1806
+ from .lc_rail_guard import lc_result_rail as _lc_rail
1807
+ except Exception:
1808
+ _lc_rail = None
1809
+ if _lc_rail is not None:
1810
+ _rf = _lc_rail(names, betas, ses, shares,
1811
+ membership_coefs=membership_coefs,
1812
+ bounds=float(bounds),
1813
+ share_floor=float(share_floor),
1814
+ method=method)
1815
+ out["rail"] = list(_rf["rail"])
1816
+ out["railed"] = bool(_rf["railed"])
1817
+ out["rail_warnings"] = list(_rf["warnings"])
1818
+ if _rf["railed"]:
1819
+ out["converged_by_optimizer"] = bool(out["converged"])
1820
+ out["converged"] = False
1647
1821
  if labels is not None:
1648
1822
  out["group_labels"] = list(labels)
1649
1823
  if responsibilities is not None:
@@ -319,8 +319,39 @@ def fit_rrm_numba(model, start=None, compute_inference=True):
319
319
  result = _sp_min(_obj, start, jac=True, method='BFGS',
320
320
  options={'maxiter': int(getattr(model, 'maxiter', 2000) or 2000),
321
321
  'gtol': 1e-6, 'disp': False})
322
+ # Same contract as the pure-scipy path: re-polish while the gradient is
323
+ # still large, then judge convergence on first-order optimality instead of
324
+ # scipy's success flag (BFGS reports precision loss after it has already
325
+ # converged, which used to mark every RRM fit as failed).
326
+ #
327
+ # The gradient here is a central difference of the njit likelihood, which
328
+ # is noisy on the near-collinear period design, so the polish and the final
329
+ # verdict use the analytic numpy gradient: both engines then land on the
330
+ # same optimum and agree on whether it is one.
331
+ def _analytic_grad(x):
332
+ try:
333
+ return model.get_loglike_gradient(
334
+ np.asarray(x, dtype=float), np.ones(M, dtype=float) * 1e-10)[1]
335
+ except Exception:
336
+ return None
337
+
338
+ def _obj_exact(b):
339
+ b = np.ascontiguousarray(np.asarray(b, dtype=np.float64))
340
+ f, g = model.get_loglike_gradient(b, np.ones(M, dtype=float) * 1e-10)
341
+ return float(f), np.asarray(g, dtype=np.float64)
342
+
343
+ try:
344
+ try:
345
+ from rrm import assess_convergence as _assess, polish_result as _polish
346
+ except ImportError:
347
+ from .rrm import assess_convergence as _assess, polish_result as _polish
348
+ n_obs = int(len(y_idx))
349
+ result = _polish(_obj_exact, result, grad_fun=_analytic_grad, n_obs=n_obs)
350
+ model.converged, model.grad_norm_inf, model.fit_message = _assess(
351
+ result, n_obs=n_obs, grad=_analytic_grad(result.x))
352
+ except Exception:
353
+ model.converged = bool(getattr(result, 'success', False))
322
354
  model.coeff_est = result.x
323
- model.converged = result.success
324
355
  model.beta = result.x
325
356
  try:
326
357
  model.post_process(compute_inference=compute_inference)
@@ -46,6 +46,124 @@ import numpy as np
46
46
  from numpy.typing import NDArray
47
47
  from scipy import stats
48
48
  from scipy.optimize import minimize
49
+
50
+ # ── Convergence assessment ─────────────────────────────────────────────────
51
+ # scipy's BFGS reports success=False with "Desired error not necessarily
52
+ # achieved due to precision loss" once the line search can no longer improve
53
+ # at float64 precision. That happens AFTER the optimum has been reached, so
54
+ # treating result.success as the convergence verdict marks good fits as
55
+ # failures. Downstream that is expensive: the ABM pipeline (stage4 Schedule
56
+ # Placement) labels the model 'optimizer_stopped' and down-weights its period
57
+ # probabilities (blend_alpha 0.08-0.35), so a converged RRM contributes almost
58
+ # nothing to the simulated schedule. All 8 RRM timing models in runs
59
+ # 27284748/27293944 hit this.
60
+ #
61
+ # The verdict below is first-order optimality (scaled gradient at the
62
+ # solution), with a short re-polish when the gradient is still large, which is
63
+ # what actually recovers fits that stalled in the line search.
64
+ RRM_GRAD_TOL = 1e-6 # on the MEAN log-likelihood gradient, per obs
65
+ RRM_POLISH_ATTEMPTS = 2
66
+ RRM_POLISH_MAXITER = 2000
67
+
68
+
69
+ def grad_tolerance(n_obs=None, gtol=None):
70
+ """Absolute gradient threshold for a sample of ``n_obs`` observations.
71
+
72
+ The likelihood is a sum, so its gradient grows with the sample size: a
73
+ fixed absolute threshold is strict on small samples and meaningless on
74
+ large ones. The criterion used here is that the gradient of the *mean* log
75
+ likelihood is below ``RRM_GRAD_TOL``. At that point the remaining
76
+ log-likelihood gain is far below what any reported digit could show.
77
+ """
78
+ if gtol is not None:
79
+ return float(gtol)
80
+ try:
81
+ n = int(n_obs or 0)
82
+ except Exception:
83
+ n = 0
84
+ return float(RRM_GRAD_TOL * max(n, 1))
85
+
86
+
87
+ def _grad_inf_norm(result, n_params=None, grad=None):
88
+ """Largest absolute gradient component at the reported solution.
89
+
90
+ ``grad`` overrides ``result.jac`` so callers holding a more accurate
91
+ gradient (the analytic numpy one, or the numba engine's finite-difference
92
+ estimate at a nearly-collinear solution) can supply it.
93
+ """
94
+ try:
95
+ jac = (np.asarray(grad, dtype=float) if grad is not None
96
+ else np.asarray(result.jac, dtype=float)).ravel()
97
+ if jac.size == 0 or not np.all(np.isfinite(jac)):
98
+ return float('nan')
99
+ return float(np.max(np.abs(jac)))
100
+ except Exception:
101
+ return float('nan')
102
+
103
+
104
+ def assess_convergence(result, gtol=None, n_obs=None, grad=None):
105
+ """Decide convergence from the evidence, not only ``result.success``.
106
+
107
+ Returns ``(converged, grad_inf, message)``. A fit counts as converged when
108
+ scipy says so, or when the gradient is small enough to be at a stationary
109
+ point even though the line search gave up on precision. Unknown gradients
110
+ fall back to scipy's verdict rather than claiming success.
111
+ """
112
+ tol = grad_tolerance(n_obs, gtol)
113
+ grad = _grad_inf_norm(result, grad=grad)
114
+ msg = str(getattr(result, 'message', '') or '')
115
+ success = bool(getattr(result, 'success', False))
116
+ if success:
117
+ return True, grad, msg
118
+ if np.isfinite(grad) and grad <= tol:
119
+ return True, grad, msg + ' [accepted: scaled gradient below tolerance]'
120
+ return False, grad, msg
121
+
122
+
123
+ def polish_result(fun, result, gtol=None, attempts=None, maxiter=None,
124
+ grad_fun=None, n_obs=None):
125
+ """Re-run BFGS from the reported solution while the gradient is too large.
126
+
127
+ The first BFGS call already did the work; this only rescues fits that
128
+ stopped early with a usable point. The best objective value wins, so a
129
+ polish pass can never return a worse fit than the one it started from.
130
+
131
+ ``grad_fun(x)`` supplies a more accurate gradient than ``result.jac`` for
132
+ the stopping test (the numba engine's finite differences are noisy where
133
+ the specification is nearly collinear, which would otherwise keep the
134
+ retry loop running at a point that is already optimal).
135
+ """
136
+ tol = grad_tolerance(n_obs, gtol)
137
+ n_left = int(RRM_POLISH_ATTEMPTS if attempts is None else attempts)
138
+ n_iter = int(RRM_POLISH_MAXITER if maxiter is None else maxiter)
139
+ best = result
140
+ best_f = float(getattr(result, 'fun', np.inf))
141
+
142
+ def _stopping_grad(res):
143
+ if grad_fun is None:
144
+ return None
145
+ try:
146
+ return grad_fun(res.x)
147
+ except Exception:
148
+ return None
149
+
150
+ while n_left > 0:
151
+ ok, grad, _msg = assess_convergence(best, tol, grad=_stopping_grad(best))
152
+ if ok:
153
+ break
154
+ n_left -= 1
155
+ try:
156
+ retry = minimize(fun, np.asarray(best.x, dtype=float), jac=True,
157
+ method='BFGS', tol=1e-8,
158
+ options={'maxiter': n_iter,
159
+ 'gtol': max(tol, 1e-6), 'disp': False})
160
+ except Exception:
161
+ break
162
+ f_new = float(getattr(retry, 'fun', np.inf))
163
+ if not np.isfinite(f_new) or f_new > best_f + 1e-9:
164
+ break # no progress: keep what we had
165
+ best, best_f = retry, f_new
166
+ return best
49
167
  import pandas as pd
50
168
  from time import time
51
169
  try:
@@ -635,8 +753,11 @@ class RandomRegret(DiscreteChoiceModel):
635
753
  self.fit_start_time = _time()
636
754
  result = sp_min(_obj, start, jac=True, method='BFGS',
637
755
  options={'maxiter': 2000, 'gtol': 1e-6, 'disp': False})
756
+ result = polish_result(_obj, result, maxiter=2000,
757
+ n_obs=self.nb_samples)
638
758
  self.coeff_est = result.x
639
- self.converged = result.success
759
+ self.converged, self.grad_norm_inf, self.fit_message = \
760
+ assess_convergence(result, n_obs=self.nb_samples)
640
761
  self.beta = result.x
641
762
  self.post_process()
642
763
  return
@@ -688,8 +809,14 @@ class RandomRegret(DiscreteChoiceModel):
688
809
  jac=True,
689
810
  options={'maxiter': getattr(self, 'maxiter', 2000)},
690
811
  )
812
+ # Re-polish while the gradient is still large, then judge convergence
813
+ # on first-order optimality rather than scipy's precision-loss flag
814
+ # (see assess_convergence above).
815
+ _polish_fun = lambda _x: self.get_loglike_gradient(_x, delta)
816
+ result = polish_result(_polish_fun, result, n_obs=self.nb_samples)
691
817
  self.coeff_est = result.x
692
- self.converged =result.success
818
+ self.converged, self.grad_norm_inf, self.fit_message = \
819
+ assess_convergence(result, n_obs=self.nb_samples)
693
820
  self.beta = result.x # Extract results
694
821
  self.post_process(compute_inference=compute_inference)
695
822
  # }
@@ -0,0 +1 @@
1
+ 0.0.271
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: SearchLibrium
3
- Version: 0.0.268
3
+ Version: 0.0.271
4
4
  Summary: Automated specification search for discrete choice models (logit, mixed logit, nested logit, RRM and more) using metaheuristic optimisation
5
5
  Author: Zeke Ahern, Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
6
6
  Maintainer-email: Zeke Ahern <z.ahern@qut.edu.au>
@@ -29,6 +29,7 @@ src/SearchLibrium/larch_compat.py
29
29
  src/SearchLibrium/larch_models.py
30
30
  src/SearchLibrium/latent_class.py
31
31
  src/SearchLibrium/latent_spec.py
32
+ src/SearchLibrium/lc_rail_guard.py
32
33
  src/SearchLibrium/logistic_regression.py
33
34
  src/SearchLibrium/main.py
34
35
  src/SearchLibrium/main_debug.py
@@ -1 +0,0 @@
1
- 0.0.268
File without changes