SearchLibrium 0.0.254__tar.gz → 0.0.261__tar.gz

{searchlibrium-0.0.254/src/SearchLibrium.egg-info → searchlibrium-0.0.261}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: SearchLibrium
-Version: 0.0.254
+Version: 0.0.261
 Summary: Automated specification search for discrete choice models (logit, mixed logit, nested logit, RRM and more) using metaheuristic optimisation
-Author: Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
-Author-email: Zeke Ahern <z.ahern@qut.edu.au>
+Author: Zeke Ahern, Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
+Maintainer-email: Zeke Ahern <z.ahern@qut.edu.au>
 License-Expression: MIT
 Project-URL: Homepage, https://pypi.org/project/SearchLibrium/
 Project-URL: Repository, https://github.com/zahern/HypothesisX

{searchlibrium-0.0.254 → searchlibrium-0.0.261}/pyproject.toml

@@ -25,12 +25,15 @@ requires-python = ">=3.9"
 license = "MIT"
 license-files = ["LICENSE"]
 authors = [
-    { name = "Zeke Ahern", email = "z.ahern@qut.edu.au" },
+    { name = "Zeke Ahern" },
     { name = "Mario Fernando Taco Morales" },
     { name = "Alexander Paz" },
     { name = "Prithvi Beeramole" },
     { name = "Robert Burdett" },
 ]
+maintainers = [
+    { name = "Zeke Ahern", email = "z.ahern@qut.edu.au" },
+]
 keywords = [
     "econometrics",
     "discrete choice",
@@ -129,7 +132,7 @@ Documentation = "https://pypi.org/project/SearchLibrium/"
 realpython = "SearchLibrium.__main__:main"
 
 [tool.bumpver]
-current_version = "0.0.254"
+current_version = "0.0.261"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit_message = "[skip ci] Bump version {old_version} -> {new_version}"
 commit = true
@@ -137,4 +140,4 @@ tag = true
 push = false
 
 [tool.bumpver.file_patterns]
-"src/SearchLibrium/version.txt" = ["{version}"]
+"src/SearchLibrium/version.txt" = ["{version}"]

{searchlibrium-0.0.254 → searchlibrium-0.0.261}/src/SearchLibrium/harmony.py

@@ -1252,7 +1252,7 @@ class HarmonySearch(Search):
     # }
 
     ''' ---------------------------------------------------------- '''
-    ''' Function.  Discard non-convergent solutions                '''
+    ''' Function.  Discard non-convergent solutions                 '''
     ''' ---------------------------------------------------------- '''
     def screen_solutions(self, solutions):
     # {

{searchlibrium-0.0.254 → searchlibrium-0.0.261}/src/SearchLibrium/mh_choice_estimator.py

@@ -481,6 +481,13 @@ class BlockRandomWalkProposal:
 # ---------------------------------------------------------------------------
 
 
+def _env_int(name: str, default: int) -> int:
+    try:
+        return int(str(os.environ.get(name, "")).strip())
+    except (TypeError, ValueError):
+        return default
+
+
 class HHTSCompetingPrior:
     """Dirichlet-smoothed HHTS shares blended with current utilities."""
 
@@ -496,6 +503,8 @@ class HHTSCompetingPrior:
         max_alpha: float = 0.95,
         min_temperature: float = 0.15,
         max_temperature: float = 8.0,
+        max_history: int | None = None,
+        record_history: bool = True,
     ) -> None:
         raw = shares or {}
         if raw and all(not isinstance(value, Mapping) for value in raw.values()):
@@ -512,8 +521,13 @@ class HHTSCompetingPrior:
         # adapt_to_denominator_ess runs once per CHOICE CASE per Gibbs
         # iteration, so an unbounded list here grew by n_cases x outer
         # iterations (millions of dicts on the 13k-case datasets). Only the
-        # most recent entry is ever read, so keep a bounded window.
-        self.history_maxlen: int = 512
+        # most recent entry is ever read, so keep a bounded window
+        # (SL_HHTS_MAX_HISTORY, default 4096; 0 disables recording).
+        if max_history is None:
+            max_history = _env_int("SL_HHTS_MAX_HISTORY", 4096)
+        self.max_history = max(0, int(max_history))
+        self.record_history = bool(record_history) and self.max_history > 0
+        self.history_maxlen: int = self.max_history or 1
         self.history: deque = deque(maxlen=self.history_maxlen)
 
     @classmethod
@@ -578,7 +592,8 @@ class HHTSCompetingPrior:
         self.alpha = float(np.clip(self.alpha + self.adaptation_rate * error, self.min_alpha, self.max_alpha))
         self.temperature = float(np.clip(self.temperature * np.exp(self.adaptation_rate * error),
                                          self.min_temperature, self.max_temperature))
-        self.history.append({"denominator_ess": ess, "alpha": self.alpha, "temperature": self.temperature})
+        if self.record_history:
+            self.history.append({"denominator_ess": ess, "alpha": self.alpha, "temperature": self.temperature})
         return ess
 
     def sample_choice_sets(self, observations: pd.DataFrame, zone_ids: Sequence[object],
@@ -1265,17 +1280,29 @@ def _weighted_frame(frame: "ChoiceSetFrame",
         frame.feature_names, tuple(float(w) for w in np.asarray(weights).ravel()))
 
 
-def _fit_membership_gammas(case_X, resp, l2: float = 1.0, maxiter: int = 200):
+def _fit_membership_gammas(case_X, resp, l2: float = 1.0, maxiter: int = 200,
+                           standardize: bool = True):
     """Weighted multinomial-logit M-step for the class-membership equation.
 
     ``gammas`` has shape (C-1, K+1) with class 0 as the reference and an
     intercept column.  Rows are weighted by the EM responsibilities (soft
     labels), so the step maximises the expected membership log-likelihood;
     slopes carry an L2 penalty, intercepts do not.
+
+    If ``standardize=True`` (default), covariates are standardized to mean=0,
+    std=1 before fitting, and coefficients are transformed back to the original
+    scale for the returned ``gammas``.
     """
     from scipy.optimize import minimize
     N = np.asarray(case_X).shape[0]
-    Xa = np.column_stack([np.ones(N), np.asarray(case_X, dtype=float)])
+    X = np.asarray(case_X, dtype=float)
+    if standardize:
+        X_mean = X.mean(axis=0)
+        X_std = X.std(axis=0, ddof=0)
+        X_std = np.where(X_std > 1e-12, X_std, 1.0)
+        Xa = np.column_stack([np.ones(N), (X - X_mean) / X_std])
+    else:
+        Xa = np.column_stack([np.ones(N), X])
     C = np.asarray(resp).shape[1]
     Ka = Xa.shape[1]
     if C < 2:
@@ -1296,7 +1323,18 @@ def _fit_membership_gammas(case_X, resp, l2: float = 1.0, maxiter: int = 200):
 
     res = minimize(nll, g0.ravel(), method="L-BFGS-B",
                    options={"maxiter": int(maxiter)})
-    return res.x.reshape(C - 1, Ka)
+    G = res.x.reshape(C - 1, Ka)
+    # Transform back to original scale if standardized
+    if standardize:
+        # G has shape (C-1, Ka) where Ka = n_features + 1 (intercept)
+        # G[:, 0] are intercepts, G[:, 1:] are slopes on standardized features
+        G_out = G.copy()
+        G_out[:, 1:] = G[:, 1:] / X_std
+        # Intercept adjustment: intercept = intercept - sum(mean_k * slope_k / std_k)
+        # For each class (row), compute dot product of (mean/std) with slopes
+        G_out[:, 0] = G[:, 0] - np.sum((X_mean / X_std) * G[:, 1:], axis=1)
+        return G_out
+    return G
 
 
 def _fit_frame_mnl(frame: "ChoiceSetFrame", x0, *, prior_scale: float = 5.0,
@@ -1530,7 +1568,7 @@ def estimate_dest_choice_lc(
                                     prior_scale=prior_scale, bounds=bounds)[0]
                      for c in range(n_classes)]
             if case_X is not None:
-                gammas = _fit_membership_gammas(case_X, p, l2=membership_l2)
+                gammas = _fit_membership_gammas(case_X, p, l2=membership_l2, standardize=True)
             shares = new_shares
             if share_floor and float(share_floor) > 0:
                 _coll = int(np.argmin(shares))

{searchlibrium-0.0.254 → searchlibrium-0.0.261}/src/SearchLibrium/search.py

@@ -60,6 +60,7 @@ from collections import UserDict
 
 from enum import Enum
 import copy
+import itertools
 import numpy as np
 import pandas as pd
 import random
@@ -215,6 +216,16 @@ def process_criterions(criterions):
 # }
 
 
+def _search_env_int(name, default):
+    ''' Read an integer tuning knob from the environment, falling back to
+    `default` when it is unset or not a valid integer. '''
+    import os
+    try:
+        return int(str(os.environ.get(name, '')).strip())
+    except (TypeError, ValueError):
+        return default
+
+
 def count_insig_groups(coeff_names, pvalues, p_val=0.05):
     """Count variable *groups* where *every* coefficient fails the p-value
     threshold.  A group is the set of coefficients belonging to the same
@@ -1555,6 +1566,18 @@ class Search():
         self.explored_specs      = set() # Set of hashes for all explored specifications (evaluated or not)
         self.cache_hits           = 0    # Count of how many times a solution was found in the cache
 
+        # Cache growth bound (SEARCHLIBRIUM_CACHE_LIMIT, default 250000).
+        # evaluated_solutions / explored_specs / _banlist are keyed by a
+        # 64-char signature and grow by one entry per evaluated individual.
+        # Over a long evolutionary search (tens of thousands of fits) that is
+        # tens of MB of dead weight, and a specification-search job can then
+        # trip a cgroup memory limit that the live population never came
+        # close to.  Cap them and drop the oldest entries under pressure.
+        # Both caches are pure speed/heuristic aids: evicting an entry costs a
+        # refit (or one retry of an already-failed spec), never correctness.
+        self._cache_limit = _search_env_int('SEARCHLIBRIUM_CACHE_LIMIT', 250000)
+        self._n_explored = 0             # monotonic count; survives cache trimming
+
         self.last_printed_solution = None  # Track the last printed solution
         self.best_solution = None  # T
 
@@ -4859,12 +4882,37 @@ class Search():
     ''' and cache the result if it converged.                              '''
     ''' ------------------------------------------------------------------ '''
 
+    def _trim_search_caches(self):
+        ''' Drop the oldest entries from the specification caches once they
+        exceed `self._cache_limit`.
+
+        `evaluated_solutions` is insertion-ordered, so the front is the
+        oldest.  `explored_specs` and `_banlist` are unordered sets whose
+        only role is bookkeeping / avoiding a retry of a known-bad spec, so
+        they are cleared wholesale when oversized.  `self._n_explored` keeps
+        the true running total for the exploration summary. '''
+        limit = getattr(self, '_cache_limit', 0)
+        if not limit:
+            return
+        if len(self.evaluated_solutions) > limit:
+            # free ~25% in one pass rather than trimming on every single insert
+            for _sig in list(itertools.islice(self.evaluated_solutions,
+                                              len(self.evaluated_solutions) - limit // 4)):
+                self.evaluated_solutions.pop(_sig, None)
+            logging.debug('trimmed evaluated_solutions to %d (limit %d)',
+                          len(self.evaluated_solutions), limit)
+        if len(self.explored_specs) > limit:
+            self.explored_specs.clear()
+        if len(self._banlist) > limit:
+            self._banlist.clear()
+
     def evaluate_solution(self, sol, track_best=True):
 
         sig = self.setup_signature(sol)
 
         if sig not in self.explored_specs:
             self.explored_specs.add(sig)
+            self._n_explored += 1
 
         if sig in self.evaluated_solutions:
             self.cache_hits += 1
@@ -4877,6 +4925,7 @@ class Search():
 
         if converged:
             self.evaluated_solutions[sig] = [sol.obj(i) for i in range(self.nb_crit)]
+            self._trim_search_caches()
 
         return sol, converged
 
@@ -5124,7 +5173,7 @@ class Search():
         served from cache instead of re-fitting the model. """
 
         sep = '═' * 60
-        total   = len(self.explored_specs)
+        total   = getattr(self, '_n_explored', len(self.explored_specs))
         hits    = self.cache_hits
         hit_pct = (hits / total * 100) if total else 0.0
 

{searchlibrium-0.0.254 → searchlibrium-0.0.261}/src/SearchLibrium/sparseea_agds.py

@@ -37,7 +37,9 @@ object and call ``run()`` (which dispatches to ``run_search``).
 
 import copy
 import datetime
+import gc as _gc
 import logging
+import os
 
 import numpy as np
 
@@ -55,6 +57,84 @@ logger = logging.getLogger(__name__)
 _agds_spec_key = spec_key
 
 
+# ---------------------------------------------------------------------- #
+#  Memory reclamation                                                      #
+# ---------------------------------------------------------------------- #
+# A long AGDS run re-fits one model per evaluated individual - tens of
+# thousands over a 500-generation search.  Each fit materialises a full
+# case x alternative design frame (and, under a JAX backend, a freshly
+# compiled executable), so the transient peak per fit is orders of magnitude
+# larger than the live population.  Those objects *are* released, but glibc
+# retains the freed spans in its arenas, so RSS ratchets steadily upward and a
+# process holding only a few hundred MB of live data still walks into a
+# cgroup memory limit and gets OOM-killed.
+#
+# Calling collect() + malloc_trim() once per generation returns those spans to
+# the OS so RSS tracks the live set instead of the high-water mark.  Every
+# CLEAR_JAX_EVERY generations (and every force=True release) the JAX
+# compilation cache is dropped as well: one executable per distinct spec is
+# retained there beyond the reach of gc/malloc_trim (+24.3 MB/fit measured
+# Oct-2026 on a 1016-trip x 233-destination larch workload, identical across
+# three runs, OOM-killing two 128 GB jobs in the refit that follows the
+# search).  Recompile cost is bounded: only re-visited specs recompile, and
+# an evolutionary search rarely revisits.
+#
+#   SEARCHLIBRIUM_RECLAIM_EVERY=<n>  reclaim every <n> generations (default 1)
+#   SEARCHLIBRIUM_RECLAIM=0          disable entirely
+#   SEARCHLIBRIUM_CLEAR_JAX=1        also drop JAX's cache on EVERY call
+#   SEARCHLIBRIUM_CLEAR_JAX_EVERY=<n>
+#                                    ... otherwise drop it every <n>
+#                                    generations (default 5) and on every
+#                                    force=True release
+
+def _env_int(name, default):
+    try:
+        return int(str(os.environ.get(name, "")).strip())
+    except (TypeError, ValueError):
+        return default
+
+
+RECLAIM_EVERY = max(1, _env_int("SEARCHLIBRIUM_RECLAIM_EVERY", 1))
+RECLAIM_ENABLED = str(os.environ.get("SEARCHLIBRIUM_RECLAIM", "1")).strip().lower() \
+    not in {"0", "false", "no", "off"}
+CLEAR_JAX = str(os.environ.get("SEARCHLIBRIUM_CLEAR_JAX", "0")).strip().lower() \
+    in {"1", "true", "yes", "on"}
+CLEAR_JAX_EVERY = max(1, _env_int("SEARCHLIBRIUM_CLEAR_JAX_EVERY", 5))
+
+
+def _malloc_trim():
+    """Return free glibc heap arenas to the OS.  No-op on non-glibc platforms."""
+    try:
+        import ctypes
+        libc = ctypes.CDLL("libc.so.6")
+        malloc_trim = libc.malloc_trim
+        malloc_trim.argtypes = [ctypes.c_size_t]
+        malloc_trim.restype = ctypes.c_int
+        malloc_trim(0)
+    except Exception:
+        pass
+
+
+def reclaim_memory(force=False, clear_jax=False):
+    """Collect search-local garbage and hand freed heap back to the OS.
+
+    ``force=True`` releases (one-off phase transitions) always also drop
+    JAX's compiled-executable cache; in-loop callers pass ``clear_jax=True``
+    every ``CLEAR_JAX_EVERY`` generations; ``SEARCHLIBRIUM_CLEAR_JAX=1``
+    clears on every call.
+    """
+    if not (force or RECLAIM_ENABLED):
+        return
+    _gc.collect()
+    if clear_jax or force or CLEAR_JAX:
+        try:
+            import jax
+            jax.clear_caches()
+        except Exception:
+            pass
+    _malloc_trim()
+
+
 # ---------------------------------------------------------------------- #
 #  NSGA-III reference-point helpers (numpy, self-contained)               #
 # ---------------------------------------------------------------------- #
@@ -435,6 +515,15 @@ class SparseEAAGDS(Search):
                     pf.flush()
                 except Exception:
                     pass
+
+                # Return the per-generation fit garbage to the OS.  Without
+                # this RSS follows the high-water mark of the transient fit
+                # buffers and a long search is OOM-killed despite a small live
+                # set.  Every CLEAR_JAX_EVERY generations also drop JAX's
+                # global compilation cache (see reclaim_memory).
+                if (gen + 1) % RECLAIM_EVERY == 0:
+                    reclaim_memory(
+                        clear_jax=((gen + 1) % (RECLAIM_EVERY * CLEAR_JAX_EVERY) == 0))
         finally:
             pf.close()
 

searchlibrium-0.0.261/src/SearchLibrium/version.txt

@@ -0,0 +1 @@
+0.0.261

{searchlibrium-0.0.254 → searchlibrium-0.0.261/src/SearchLibrium.egg-info}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: SearchLibrium
-Version: 0.0.254
+Version: 0.0.261
 Summary: Automated specification search for discrete choice models (logit, mixed logit, nested logit, RRM and more) using metaheuristic optimisation
-Author: Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
-Author-email: Zeke Ahern <z.ahern@qut.edu.au>
+Author: Zeke Ahern, Mario Fernando Taco Morales, Alexander Paz, Prithvi Beeramole, Robert Burdett
+Maintainer-email: Zeke Ahern <z.ahern@qut.edu.au>
 License-Expression: MIT
 Project-URL: Homepage, https://pypi.org/project/SearchLibrium/
 Project-URL: Repository, https://github.com/zahern/HypothesisX

searchlibrium-0.0.254/src/SearchLibrium/version.txt

@@ -1 +0,0 @@
-0.0.254