PyAntiGen 1.0.11__tar.gz → 1.0.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PKG-INFO +1 -1
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/PKG-INFO +1 -1
- pyantigen-1.0.13/PyAntiGen.egg-info/scm_version.json +8 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/_version.py +3 -3
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Evaluator.py +133 -35
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Fast_profile.py +22 -3
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Identifiability.py +92 -44
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Model_optimize.py +20 -206
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Optimize.py +665 -898
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Simulate.py +18 -1
- pyantigen-1.0.11/PyAntiGen.egg-info/scm_version.json +0 -8
- {pyantigen-1.0.11 → pyantigen-1.0.13}/.github/workflows/publish.yml +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/.gitignore +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/.vscode/launch.json +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/LICENSE +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/MANIFEST.in +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/SOURCES.txt +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/dependency_links.txt +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/entry_points.txt +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/requires.txt +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/scm_file_list.json +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/PyAntiGen.egg-info/top_level.txt +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/README.md +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/.agents/skills/module_generation/SKILL.md +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/.agents/skills/ode_conversion_antimony/SKILL.md +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/.agents/skills/ode_extraction_pipeline/SKILL.md +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/.gitignore +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/AntimonyGen.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/RxnDict_to_antimony.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/TelluriumGen.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/__init__.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/antimony_utils.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/cli.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/data_interpolation.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/isotopomer_tools.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/model_generation.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/models.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/module_base.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/pyantigen.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/rate_laws.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/reaction_creation.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/AntiGen_paths.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Anchor_cache.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Deadline.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Event_times.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Fit_cache.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Model_simulate.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Nuisance_sensitivity.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Petab_export.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Preequil_cache.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Profile_checkpoint.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Results.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Engine/Sensitivity_analysis.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Flipflop_reference.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Model_generate.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Model_run.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Data.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Events.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Experiment.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Loss_config.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Observed_species.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Optimizer_settings.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Plots.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Solver_settings.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Update_opt_parameters.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/Example/Modules/Update_parameters.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/data/ADneg.csv +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/data/ADpos.csv +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/data/Flipflop.csv +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/framework/template/data/make_flipflop_data.py +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/pyproject.toml +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/setup.cfg +0 -0
- {pyantigen-1.0.11 → pyantigen-1.0.13}/setup.py +0 -0
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (1, 0,
|
|
21
|
+
__version__ = version = '1.0.13'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 13)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'ga5c2cee6b'
|
|
@@ -131,6 +131,17 @@ class EvalSpec:
|
|
|
131
131
|
# in the parent, which runs the invariance check once; a worker must never
|
|
132
132
|
# make that call on its own, or 40 of them would each re-derive it.
|
|
133
133
|
preequil_cache: bool = False
|
|
134
|
+
# Modules.utils.noise_floor.export_cache() snapshot, taken in the parent
|
|
135
|
+
# AFTER its own calibration (see Engine.Optimize.run_optimization_from_groups,
|
|
136
|
+
# clear_cache() + the post-optimum re-evaluation). Workers seed their own
|
|
137
|
+
# (otherwise empty, since spawn shares no memory) floor cache from this in
|
|
138
|
+
# _init_worker, so every worker scores every floored observable against
|
|
139
|
+
# the SAME calibrated sigma the parent settled on, rather than each one
|
|
140
|
+
# independently calibrating against whatever parameter vector it happens
|
|
141
|
+
# to be handed first -- an arbitrary profile-grid point or Sobol sample,
|
|
142
|
+
# not the converged optimum. Same reasoning as fixed_sigmas above, one
|
|
143
|
+
# mechanism down: compute once where it's meaningful, ship the answer.
|
|
144
|
+
floor_cache: dict = field(default_factory=dict)
|
|
134
145
|
# Reserved for future use by the profile grid (Stage 2).
|
|
135
146
|
meta: dict = field(default_factory=dict)
|
|
136
147
|
|
|
@@ -149,8 +160,16 @@ def _init_worker(spec_blob):
|
|
|
149
160
|
from framework.TelluriumGen import TelluriumGen
|
|
150
161
|
from Engine.Event_times import attach_event_times
|
|
151
162
|
from Engine.Optimize import OptRoadRunnerProxy
|
|
163
|
+
from Modules.utils.noise_floor import seed_cache
|
|
152
164
|
|
|
153
165
|
spec = _serializer.loads(spec_blob)
|
|
166
|
+
# Before any task runs: this worker's own Modules.utils.noise_floor
|
|
167
|
+
# module was just re-imported fresh (spawn shares no memory with the
|
|
168
|
+
# parent), so its floor cache starts empty. Seed it from the parent's
|
|
169
|
+
# already-calibrated snapshot so every worker agrees with the parent --
|
|
170
|
+
# and with each other -- on every floored observable's sigma, instead of
|
|
171
|
+
# each recalibrating independently against whichever task it draws first.
|
|
172
|
+
seed_cache(spec.floor_cache)
|
|
154
173
|
models = {}
|
|
155
174
|
t0 = time.time()
|
|
156
175
|
for sim_name, replicate in spec.replicates.items():
|
|
@@ -189,7 +208,7 @@ def _init_worker(spec_blob):
|
|
|
189
208
|
f"{time.time() - t0:.1f}s", flush=True)
|
|
190
209
|
|
|
191
210
|
|
|
192
|
-
def _worker_nll(x):
|
|
211
|
+
def _worker_nll(x, frozen_sigmas=None):
|
|
193
212
|
"""The joint NLL, evaluated with this worker's own compiled models."""
|
|
194
213
|
from Engine.Optimize import evaluate_nll_fixed
|
|
195
214
|
|
|
@@ -202,17 +221,18 @@ def _worker_nll(x):
|
|
|
202
221
|
events_dynamic=spec.events_dynamic, failure_value=FAILURE_VALUE,
|
|
203
222
|
for_inference=getattr(spec, "for_inference", True),
|
|
204
223
|
concentrated=getattr(spec, "concentrated", True),
|
|
224
|
+
frozen_sigmas=frozen_sigmas,
|
|
205
225
|
)
|
|
206
226
|
|
|
207
227
|
|
|
208
|
-
def _eval_task(x):
|
|
228
|
+
def _eval_task(x, frozen_sigmas=None):
|
|
209
229
|
"""Evaluate one parameter vector. Never raises across the pool boundary."""
|
|
210
230
|
if _WORKER["spec"] is None:
|
|
211
231
|
return (FAILURE_VALUE, "worker-not-initialized", 0.0)
|
|
212
232
|
|
|
213
233
|
t0 = time.time()
|
|
214
234
|
try:
|
|
215
|
-
val = _worker_nll(x)
|
|
235
|
+
val = _worker_nll(x, frozen_sigmas=frozen_sigmas)
|
|
216
236
|
_WORKER["n_evals"] += 1
|
|
217
237
|
status = "ok" if np.isfinite(val) and val < FAILURE_VALUE else "sentinel"
|
|
218
238
|
return (float(val), status, time.time() - t0)
|
|
@@ -503,7 +523,17 @@ def _profile_task(job):
|
|
|
503
523
|
cache = {}
|
|
504
524
|
cache_order = []
|
|
505
525
|
|
|
506
|
-
|
|
526
|
+
# Every profile point pins each floored block at its own sigma_used
|
|
527
|
+
# from the fit rather than letting it re-concentrate (see
|
|
528
|
+
# Engine.Optimize._freeze_floor) -- stamped onto the job by
|
|
529
|
+
# profile_batch's frozen_sigmas, not decided here, so the caller
|
|
530
|
+
# controls it per batch.
|
|
531
|
+
frozen = job.get("frozen_sigmas")
|
|
532
|
+
|
|
533
|
+
def _pinned_nll(x_full):
|
|
534
|
+
return _worker_nll(x_full, frozen_sigmas=frozen)
|
|
535
|
+
|
|
536
|
+
raw_objective = _make_nuisance_objective(_pinned_nll, param_idx, n_params)
|
|
507
537
|
|
|
508
538
|
def nuisance_objective(x_nuisance, fixed_val):
|
|
509
539
|
x_arr = np.asarray(x_nuisance, dtype=float)
|
|
@@ -858,32 +888,50 @@ class ParallelEvaluator:
|
|
|
858
888
|
|
|
859
889
|
# -- evaluation --------------------------------------------------------
|
|
860
890
|
|
|
861
|
-
def evaluate_batch(self, xs, label=None
|
|
862
|
-
|
|
891
|
+
def evaluate_batch(self, xs, label=None, heartbeat_s=_HEARTBEAT_SECONDS,
|
|
892
|
+
frozen_sigmas=None):
|
|
893
|
+
"""Evaluate every parameter vector in *xs*; return losses in input order.
|
|
894
|
+
|
|
895
|
+
Uses submit/wait, not map -- see profile_batch's docstring for the
|
|
896
|
+
general reasoning. map() (the previous implementation here) returns
|
|
897
|
+
nothing until the WHOLE batch is done, so one slow straggler among
|
|
898
|
+
many fast points -- a slice-screen point far from the optimum landing
|
|
899
|
+
in a stiff numerical regime, say -- makes the entire batch silent for
|
|
900
|
+
as long as that one point takes, indistinguishable from a hang. A
|
|
901
|
+
heartbeat every heartbeat_s while nothing has landed answers that
|
|
902
|
+
directly: it says how many are done, how many are still in flight, and
|
|
903
|
+
an ETA once at least one has finished.
|
|
904
|
+
|
|
905
|
+
chunk_size no longer applies to this method: submitting one task per
|
|
906
|
+
vector is what makes the heartbeat and per-point completion visibility
|
|
907
|
+
possible at all, and no caller in this codebase sets chunk_size to
|
|
908
|
+
anything but the default anyway.
|
|
909
|
+
|
|
910
|
+
Still returns losses in INPUT order, not completion order -- unlike
|
|
911
|
+
profile_batch, whose callers key off fields in each job/result dict,
|
|
912
|
+
callers here (the slice screen especially) index into the return value
|
|
913
|
+
positionally.
|
|
914
|
+
"""
|
|
863
915
|
xs = [np.asarray(x, dtype=float) for x in xs]
|
|
864
|
-
|
|
916
|
+
n = len(xs)
|
|
917
|
+
if n == 0:
|
|
865
918
|
return []
|
|
866
919
|
if self._pool is None:
|
|
867
920
|
self.start()
|
|
868
921
|
|
|
869
|
-
|
|
870
|
-
if chunk is None:
|
|
871
|
-
# Enough chunks to keep every worker fed, few enough to avoid
|
|
872
|
-
# per-task overhead dominating.
|
|
873
|
-
chunk = max(1, len(xs) // (self.n_workers * 4) or 1)
|
|
922
|
+
from concurrent.futures import wait, FIRST_COMPLETED
|
|
874
923
|
|
|
875
924
|
t0 = time.time()
|
|
925
|
+
tag = f" [{label}]" if label else ""
|
|
876
926
|
if self.verbose:
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
print(f"[pool]{tag} {len(xs)} evaluation(s) submitted to "
|
|
883
|
-
f"{self.n_workers} worker(s) in chunks of {chunk}; "
|
|
884
|
-
f"no output until the batch completes.", flush=True)
|
|
927
|
+
print(f"[pool]{tag} {n} evaluation(s) submitted to "
|
|
928
|
+
f"{self.n_workers} worker(s); progress every "
|
|
929
|
+
f"{_fmt_dur(heartbeat_s)} until results start landing.",
|
|
930
|
+
flush=True)
|
|
931
|
+
|
|
885
932
|
try:
|
|
886
|
-
|
|
933
|
+
futures = {self._pool.submit(_eval_task, x, frozen_sigmas): i
|
|
934
|
+
for i, x in enumerate(xs)}
|
|
887
935
|
except RuntimeError as exc:
|
|
888
936
|
if "bootstrapping phase" in str(exc):
|
|
889
937
|
# spawn re-imports the __main__ module in every worker. If the
|
|
@@ -903,15 +951,46 @@ class ParallelEvaluator:
|
|
|
903
951
|
) from exc
|
|
904
952
|
raise
|
|
905
953
|
|
|
906
|
-
|
|
954
|
+
pending = set(futures)
|
|
955
|
+
out = [None] * n
|
|
956
|
+
work = 0.0
|
|
957
|
+
done = 0
|
|
907
958
|
failures = []
|
|
908
|
-
for i, (val, status, secs) in enumerate(out):
|
|
909
|
-
losses.append(val)
|
|
910
|
-
self.total_worker_seconds += secs
|
|
911
|
-
if status != "ok":
|
|
912
|
-
failures.append((i, status))
|
|
913
959
|
|
|
914
|
-
|
|
960
|
+
while pending:
|
|
961
|
+
finished, pending = wait(pending, timeout=heartbeat_s,
|
|
962
|
+
return_when=FIRST_COMPLETED)
|
|
963
|
+
|
|
964
|
+
if not finished:
|
|
965
|
+
if self.verbose:
|
|
966
|
+
now = time.time()
|
|
967
|
+
msg = (f" [pool{tag}] {done}/{n} done, "
|
|
968
|
+
f"{len(pending)} in flight, "
|
|
969
|
+
f"{_fmt_dur(now - t0)} elapsed")
|
|
970
|
+
if done:
|
|
971
|
+
rate = done / max(now - t0, 1e-9)
|
|
972
|
+
msg += f", ~{_fmt_dur((n - done) / rate)} remaining"
|
|
973
|
+
else:
|
|
974
|
+
msg += " (no point has finished yet, so no estimate)"
|
|
975
|
+
print(msg, flush=True)
|
|
976
|
+
continue
|
|
977
|
+
|
|
978
|
+
for fut in finished:
|
|
979
|
+
i = futures[fut]
|
|
980
|
+
try:
|
|
981
|
+
val, status, secs = fut.result()
|
|
982
|
+
except Exception as exc:
|
|
983
|
+
val = FAILURE_VALUE
|
|
984
|
+
status = f"error: {type(exc).__name__}: {exc}"
|
|
985
|
+
secs = 0.0
|
|
986
|
+
out[i] = val
|
|
987
|
+
work += secs
|
|
988
|
+
self.total_worker_seconds += secs
|
|
989
|
+
if status != "ok":
|
|
990
|
+
failures.append((i, status))
|
|
991
|
+
done += 1
|
|
992
|
+
|
|
993
|
+
self.n_evals += n
|
|
915
994
|
self.n_failures += len(failures)
|
|
916
995
|
|
|
917
996
|
if self.verbose:
|
|
@@ -920,22 +999,31 @@ class ParallelEvaluator:
|
|
|
920
999
|
# wall time is the speedup actually realized. On the first batch it
|
|
921
1000
|
# includes worker startup, so it understates steady-state throughput
|
|
922
1001
|
# -- report both numbers rather than one flattering one.
|
|
923
|
-
|
|
924
|
-
tag = f" [{label}]" if label else ""
|
|
925
|
-
print(f"[pool]{tag} {len(xs)} evals in {elapsed:.1f}s wall "
|
|
1002
|
+
print(f"[pool]{tag} {n} evals in {elapsed:.1f}s wall "
|
|
926
1003
|
f"({work:.1f}s of work, {work / elapsed:.1f}x, "
|
|
927
|
-
f"{
|
|
1004
|
+
f"{n / elapsed:.1f} eval/s)", flush=True)
|
|
928
1005
|
if failures:
|
|
929
1006
|
shown = "; ".join(f"#{i}: {s}" for i, s in failures[:3])
|
|
930
1007
|
more = f" (+{len(failures) - 3} more)" if len(failures) > 3 else ""
|
|
931
1008
|
print(f"[pool]{tag} {len(failures)} failed — {shown}{more}", flush=True)
|
|
932
1009
|
|
|
933
|
-
return
|
|
1010
|
+
return out
|
|
934
1011
|
|
|
935
1012
|
def profile_batch(self, jobs, on_result=None, label=None,
|
|
936
|
-
heartbeat_s=_HEARTBEAT_SECONDS, budget=None
|
|
1013
|
+
heartbeat_s=_HEARTBEAT_SECONDS, budget=None,
|
|
1014
|
+
frozen_sigmas=None):
|
|
937
1015
|
"""Run profile-likelihood points in parallel, within a wall budget.
|
|
938
1016
|
|
|
1017
|
+
``frozen_sigmas``, stamped onto every job here rather than left to
|
|
1018
|
+
each caller's job-building code, is a ``{(block_key_or_exp_id,
|
|
1019
|
+
obs_label): sigma_used}`` lookup (see ``Engine.Optimize.
|
|
1020
|
+
block_sigmas``) pinning each data-floored block found in it at its own
|
|
1021
|
+
resolved sigma for the point's whole nuisance re-optimization, instead
|
|
1022
|
+
of letting it re-concentrate as the nuisance vector moves -- see
|
|
1023
|
+
``Engine.Optimize._freeze_floor``. Every profile pass submitted
|
|
1024
|
+
through one ``batch()`` closure gets it uniformly this way, with no
|
|
1025
|
+
change needed at the individual job-building sites.
|
|
1026
|
+
|
|
939
1027
|
Unlike ``evaluate_batch`` this uses submit/wait rather than map, because
|
|
940
1028
|
each job is minutes to hours long and results must be checkpointed *as
|
|
941
1029
|
they land* -- the whole point of checkpointing is that killing the run
|
|
@@ -975,7 +1063,10 @@ class ParallelEvaluator:
|
|
|
975
1063
|
|
|
976
1064
|
t0 = time.time()
|
|
977
1065
|
results = []
|
|
978
|
-
|
|
1066
|
+
# Stamped once here, not per admitted job: unlike deadline/sec_per_eval
|
|
1067
|
+
# this does not depend on the clock, so every job in the batch gets it
|
|
1068
|
+
# up front.
|
|
1069
|
+
backlog = [dict(j, frozen_sigmas=frozen_sigmas) for j in jobs]
|
|
979
1070
|
futures = {}
|
|
980
1071
|
pending = set()
|
|
981
1072
|
n_jobs = len(jobs)
|
|
@@ -1125,6 +1216,7 @@ def build_eval_spec(
|
|
|
1125
1216
|
):
|
|
1126
1217
|
"""Convenience constructor mirroring the spec-route local variables."""
|
|
1127
1218
|
from Engine.Event_times import without_event_times
|
|
1219
|
+
from Modules.utils.noise_floor import export_cache
|
|
1128
1220
|
|
|
1129
1221
|
return EvalSpec(
|
|
1130
1222
|
model_text=model_text,
|
|
@@ -1150,6 +1242,12 @@ def build_eval_spec(
|
|
|
1150
1242
|
for_inference=bool(for_inference),
|
|
1151
1243
|
concentrated=bool(concentrated),
|
|
1152
1244
|
preequil_cache=bool(preequil_cache),
|
|
1245
|
+
# Captured HERE, at spec-build time -- called in the parent after its
|
|
1246
|
+
# own clear_cache()-and-recalibrate pass (see run_optimization_from_
|
|
1247
|
+
# groups), so this snapshot is the same calibration the parent's own
|
|
1248
|
+
# subsequent diagnostics use, not whatever was cached earlier in the
|
|
1249
|
+
# run (e.g. during the live optimize()).
|
|
1250
|
+
floor_cache=export_cache(),
|
|
1153
1251
|
)
|
|
1154
1252
|
|
|
1155
1253
|
|
|
@@ -281,13 +281,27 @@ def run_fast_profile(batch, nll_batch, res_x, nll_at_optimum, param_names,
|
|
|
281
281
|
n_rounds=DEFAULT_ROUNDS,
|
|
282
282
|
near_zero_frac=DEFAULT_NEAR_ZERO_FRAC,
|
|
283
283
|
span_decades=SPAN_DECADES,
|
|
284
|
-
min_reach_decades=MIN_REACH_DECADES, verbose=True
|
|
284
|
+
min_reach_decades=MIN_REACH_DECADES, verbose=True,
|
|
285
|
+
nll_at_optimum_profile=None):
|
|
285
286
|
"""Screen, then one capped profile point per crossed side, in rounds.
|
|
286
287
|
|
|
287
288
|
*batch* has the profile_batch signature: ``batch(jobs, on_result, label)``.
|
|
288
289
|
*nll_batch* evaluates a list of full parameter vectors, for the screen.
|
|
289
290
|
Returns the report dict; :func:`print_fast_profile_report` renders it and
|
|
290
291
|
:func:`fast_profile_summary` shrinks it for the results snapshot.
|
|
292
|
+
|
|
293
|
+
*nll_at_optimum* anchors the screen, which goes through *nll_batch* and so
|
|
294
|
+
is never sigma-frozen (see ``Engine.Optimize._freeze_floor``).
|
|
295
|
+
*nll_at_optimum_profile* anchors the profile points that come back through
|
|
296
|
+
*batch*, which the caller submits with each floored block's sigma pinned
|
|
297
|
+
at its own ``sigma_used`` from the fit (``frozen_sigmas=sigma_by_block``,
|
|
298
|
+
not the raw floor -- see ``Engine.Optimize._freeze_floor``). Reusing the
|
|
299
|
+
unfrozen anchor for a frozen point's dNLL would compare two different
|
|
300
|
+
objectives, and if freezing were done at the raw floor instead of the
|
|
301
|
+
block's own resolved sigma that mismatch could run to hundreds of nats
|
|
302
|
+
even a fraction of a Wald SE from the optimum, since most floored blocks
|
|
303
|
+
are not binding at a good fit. Defaults to *nll_at_optimum* when not
|
|
304
|
+
given, which is correct whenever the caller submits unfrozen points too.
|
|
291
305
|
"""
|
|
292
306
|
from Engine.Optimize import _cold_simplex, _param_bounds
|
|
293
307
|
|
|
@@ -295,6 +309,8 @@ def run_fast_profile(batch, nll_batch, res_x, nll_at_optimum, param_names,
|
|
|
295
309
|
n = len(param_names)
|
|
296
310
|
scales = list(scales) if scales is not None else ["lin"] * n
|
|
297
311
|
t_start = time.time()
|
|
312
|
+
if nll_at_optimum_profile is None:
|
|
313
|
+
nll_at_optimum_profile = nll_at_optimum
|
|
298
314
|
|
|
299
315
|
# ── The screen, reused when this fit already has one ──────────────────
|
|
300
316
|
screen = load_screen(ckpt_dir, param_names, res_x, threshold,
|
|
@@ -411,7 +427,7 @@ def run_fast_profile(batch, nll_batch, res_x, nll_at_optimum, param_names,
|
|
|
411
427
|
nll = res.get("nll")
|
|
412
428
|
ok = (res.get("status") == "ok" and nll is not None
|
|
413
429
|
and np.isfinite(nll) and nll < FAILURE_VALUE)
|
|
414
|
-
d = float(nll) - float(
|
|
430
|
+
d = float(nll) - float(nll_at_optimum_profile) if ok else None
|
|
415
431
|
prev = side["dnll"]
|
|
416
432
|
side["status"] = res.get("status")
|
|
417
433
|
side["rounds"].append({
|
|
@@ -525,7 +541,10 @@ def run_fast_profile(batch, nll_batch, res_x, nll_at_optimum, param_names,
|
|
|
525
541
|
|
|
526
542
|
report = {
|
|
527
543
|
"threshold": float(threshold),
|
|
528
|
-
|
|
544
|
+
# The anchor every dNLL in this report was actually measured against
|
|
545
|
+
# -- the frozen one, since every round's points come back through
|
|
546
|
+
# batch(). Not the same number as the screen's own anchor above.
|
|
547
|
+
"anchor": float(nll_at_optimum_profile),
|
|
529
548
|
"res_x": [float(v) for v in res_x],
|
|
530
549
|
"param_names": list(param_names),
|
|
531
550
|
"round_evals": int(round_evals),
|
|
@@ -133,6 +133,17 @@ class UnidentifiableParameters(RuntimeError):
|
|
|
133
133
|
)
|
|
134
134
|
|
|
135
135
|
|
|
136
|
+
def _is_usable(p):
|
|
137
|
+
"""Whether a slice point carries real information rather than a failure
|
|
138
|
+
sentinel. Shared by :func:`_verdict` (to find the true furthest usable
|
|
139
|
+
point) and :func:`run_slice_screen` (to decide whether a ladder's
|
|
140
|
+
outermost rung already settles a side) so the two never disagree about
|
|
141
|
+
what counts as a real crossing. ``nll < FAILURE_VALUE`` matters because
|
|
142
|
+
the sentinel is a large *finite* number, not inf or nan -- a failed
|
|
143
|
+
integration must not read as "crossed"."""
|
|
144
|
+
return np.isfinite(p["dnll"]) and p["nll"] < FAILURE_VALUE
|
|
145
|
+
|
|
146
|
+
|
|
136
147
|
def _se_for(wald_se, param_idx):
|
|
137
148
|
"""The Wald SE for one parameter in optimizer space, or None.
|
|
138
149
|
|
|
@@ -313,14 +324,11 @@ def _verdict(points, bound, threshold, p_opt, sign, is_log,
|
|
|
313
324
|
to be spent there. It is recorded rather than consumed: wiring it into the
|
|
314
325
|
profile's opening grid is a separate change.
|
|
315
326
|
"""
|
|
316
|
-
|
|
317
|
-
return np.isfinite(p["dnll"]) and p["nll"] < FAILURE_VALUE
|
|
318
|
-
|
|
319
|
-
finite = [p for p in points if _usable(p)]
|
|
327
|
+
finite = [p for p in points if _is_usable(p)]
|
|
320
328
|
|
|
321
329
|
inner = None
|
|
322
330
|
for p in points: # points are ordered outward
|
|
323
|
-
if not
|
|
331
|
+
if not _is_usable(p) or p["dnll"] > threshold:
|
|
324
332
|
break
|
|
325
333
|
inner = p["x_linear"]
|
|
326
334
|
|
|
@@ -389,25 +397,46 @@ def run_slice_screen(nll_batch, res_x, nll_at_optimum, param_names, bounds,
|
|
|
389
397
|
max_points=6, growth=2.0, range_factor=2.0,
|
|
390
398
|
span_decades=SPAN_DECADES,
|
|
391
399
|
min_reach_decades=MIN_REACH_DECADES, verbose=True):
|
|
392
|
-
"""Evaluate every parameter's slice out across decades,
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
400
|
+
"""Evaluate every parameter's slice out across decades, stopping each side
|
|
401
|
+
the moment it crosses.
|
|
402
|
+
|
|
403
|
+
The screen's job is to hand the profile a starting point, not to walk
|
|
404
|
+
every side all the way out. So the candidate ladder for each side --
|
|
405
|
+
:func:`screen_values`, unchanged -- is submitted one round at a time: round
|
|
406
|
+
*r* asks the pool for every side's *r*-th candidate, all still-active
|
|
407
|
+
sides together in one batch, which keeps this as parallel as the old
|
|
408
|
+
single-batch version was. A side is retired the instant one of its points
|
|
409
|
+
reads ``dnll > threshold`` and never contributes another evaluation --
|
|
410
|
+
:func:`_verdict` reads a side's state from whichever point is furthest out
|
|
411
|
+
among those actually evaluated, so nothing is lost by stopping there. A
|
|
412
|
+
side that never crosses keeps going, round after round, all the way to its
|
|
413
|
+
candidate list's last point -- the declared bound or ``span_decades`` out,
|
|
414
|
+
whichever is further, see :func:`screen_target` -- exactly as before,
|
|
415
|
+
because that is the one thing this screen has to be able to prove.
|
|
416
|
+
|
|
417
|
+
Why stopping matters now and did not always: once a block's sigma is
|
|
418
|
+
capped at a data-derived floor (``sigma_floor_from_data``), a point far
|
|
419
|
+
outside the fitted region can return a genuinely enormous NLL rather than
|
|
420
|
+
a merely large one -- the self-forgiving log(sigma) term that used to
|
|
421
|
+
compress a bad far-out fit is exactly what the floor removes -- and that
|
|
422
|
+
region is also where the integrator is slowest. A declared bound is a
|
|
423
|
+
user-supplied number with no guaranteed relationship to anything physical,
|
|
424
|
+
so nothing about it -- not "it is inside the box", not "it is only one
|
|
425
|
+
point" -- is safe to evaluate unconditionally. The only bound that is safe
|
|
426
|
+
is not asking for a point once its side has already answered the question.
|
|
427
|
+
|
|
428
|
+
The one thing round-by-round stopping gives up: a slice that crosses early
|
|
429
|
+
and then dips back below threshold at a point it never reaches would have
|
|
430
|
+
been read as "open" by a full evaluation, and reads "crossed" here
|
|
431
|
+
instead. That trades a slower profile on a rare, specific slice shape for
|
|
432
|
+
never paying an unbounded evaluation on the common one.
|
|
404
433
|
"""
|
|
405
434
|
from Engine.Optimize import _param_bounds
|
|
406
435
|
|
|
407
436
|
res_x = np.asarray(res_x, dtype=float)
|
|
408
437
|
scales = list(scales) if scales is not None else ["lin"] * len(param_names)
|
|
409
438
|
|
|
410
|
-
plan
|
|
439
|
+
plan = []
|
|
411
440
|
for i, name in enumerate(param_names):
|
|
412
441
|
is_log = scales[i] == "log10"
|
|
413
442
|
lb, ub = _param_bounds(bounds, i)
|
|
@@ -418,21 +447,48 @@ def run_slice_screen(nll_batch, res_x, nll_at_optimum, param_names, bounds,
|
|
|
418
447
|
range_factor=range_factor,
|
|
419
448
|
span_decades=span_decades)
|
|
420
449
|
plan.append({"index": i, "name": name, "side": side, "sign": sign,
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
x = res_x.copy()
|
|
426
|
-
x[i] = v
|
|
427
|
-
xs.append(x)
|
|
450
|
+
"is_log": is_log, "values": vals, "points": [],
|
|
451
|
+
"settled": False,
|
|
452
|
+
"p_opt": float(res_x[i]),
|
|
453
|
+
"bound": (lb if sign < 0 else ub)})
|
|
428
454
|
|
|
455
|
+
n_candidates = sum(len(e["values"]) for e in plan)
|
|
456
|
+
max_rounds = max((len(e["values"]) for e in plan), default=0)
|
|
429
457
|
if verbose:
|
|
430
458
|
print(f"\n[screen] slice screen: {len(param_names)} parameter(s) x 2 "
|
|
431
|
-
f"side(s)
|
|
432
|
-
f"
|
|
433
|
-
f"
|
|
434
|
-
|
|
435
|
-
|
|
459
|
+
f"side(s); up to {n_candidates} evaluation(s) across at most "
|
|
460
|
+
f"{max_rounds} round(s), one round per pool submission. A side "
|
|
461
|
+
f"stops the round it first crosses dNLL={threshold:g}; only "
|
|
462
|
+
f"sides still undecided pay for the next, further-out point. "
|
|
463
|
+
f"No nuisance optimization: each point is an upper bound on "
|
|
464
|
+
f"the profile, which is all the screen needs.", flush=True)
|
|
465
|
+
|
|
466
|
+
n_evaluations = 0
|
|
467
|
+
for r in range(max_rounds):
|
|
468
|
+
round_xs, round_owner = [], []
|
|
469
|
+
for entry in plan:
|
|
470
|
+
if entry["settled"] or r >= len(entry["values"]):
|
|
471
|
+
continue
|
|
472
|
+
v = entry["values"][r]
|
|
473
|
+
x = res_x.copy()
|
|
474
|
+
x[entry["index"]] = v
|
|
475
|
+
round_xs.append(x)
|
|
476
|
+
round_owner.append((entry, v))
|
|
477
|
+
if not round_xs:
|
|
478
|
+
break
|
|
479
|
+
if verbose:
|
|
480
|
+
print(f"[screen] round {r + 1}: {len(round_xs)} side(s) still "
|
|
481
|
+
f"undecided.", flush=True)
|
|
482
|
+
n_evaluations += len(round_xs)
|
|
483
|
+
nlls = nll_batch(round_xs, label=f"slice-screen-r{r + 1}")
|
|
484
|
+
for (entry, v), nll in zip(round_owner, nlls):
|
|
485
|
+
nll = float(nll)
|
|
486
|
+
point = {"x": float(v),
|
|
487
|
+
"x_linear": float(10.0 ** v if entry["is_log"] else v),
|
|
488
|
+
"nll": nll, "dnll": float(nll - nll_at_optimum)}
|
|
489
|
+
entry["points"].append(point)
|
|
490
|
+
if _is_usable(point) and point["dnll"] > threshold:
|
|
491
|
+
entry["settled"] = True
|
|
436
492
|
|
|
437
493
|
report = {"threshold": float(threshold),
|
|
438
494
|
"anchor": float(nll_at_optimum),
|
|
@@ -440,25 +496,17 @@ def run_slice_screen(nll_batch, res_x, nll_at_optimum, param_names, bounds,
|
|
|
440
496
|
"param_names": list(param_names),
|
|
441
497
|
"span_decades": float(span_decades),
|
|
442
498
|
"min_reach_decades": float(min_reach_decades),
|
|
443
|
-
"n_evaluations":
|
|
499
|
+
"n_evaluations": n_evaluations,
|
|
500
|
+
"n_candidates": n_candidates,
|
|
444
501
|
"parameters": {}}
|
|
445
502
|
|
|
446
|
-
pos = 0
|
|
447
503
|
for entry in plan:
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
nll = float(values[pos])
|
|
451
|
-
pos += 1
|
|
452
|
-
pts.append({
|
|
453
|
-
"x": float(v),
|
|
454
|
-
"x_linear": float(10.0 ** v if entry["is_log"] else v),
|
|
455
|
-
"nll": nll,
|
|
456
|
-
"dnll": float(nll - nll_at_optimum),
|
|
457
|
-
})
|
|
458
|
-
side = _verdict(pts, entry["bound"], threshold, entry["p_opt"],
|
|
459
|
-
entry["sign"], entry["is_log"],
|
|
504
|
+
side = _verdict(entry["points"], entry["bound"], threshold,
|
|
505
|
+
entry["p_opt"], entry["sign"], entry["is_log"],
|
|
460
506
|
min_reach_decades=min_reach_decades)
|
|
461
507
|
side["is_log"] = entry["is_log"]
|
|
508
|
+
side["stopped_early"] = bool(entry["settled"]
|
|
509
|
+
and len(entry["points"]) < len(entry["values"]))
|
|
462
510
|
report["parameters"].setdefault(entry["name"], {})[entry["side"]] = side
|
|
463
511
|
|
|
464
512
|
states = [s["state"] for sides in report["parameters"].values()
|