PyAntiGen 1.0.13__tar.gz → 1.0.15__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PKG-INFO +1 -1
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/PKG-INFO +1 -1
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/SOURCES.txt +4 -1
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/scm_file_list.json +4 -1
- pyantigen-1.0.15/PyAntiGen.egg-info/scm_version.json +8 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/_version.py +3 -3
- pyantigen-1.0.15/framework/template/Example/Engine/Deadline.py +262 -0
- pyantigen-1.0.15/framework/template/Example/Engine/Differential_evolution.py +348 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Evaluator.py +238 -323
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Fit_cache.py +119 -10
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Model_optimize.py +15 -1
- pyantigen-1.0.15/framework/template/Example/Engine/Nelder_mead.py +443 -0
- pyantigen-1.0.15/framework/template/Example/Engine/Noise_floor.py +328 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Optimize.py +731 -81
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Profile_checkpoint.py +162 -7
- pyantigen-1.0.15/framework/template/Example/Engine/README.md +86 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Simulate.py +71 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Plots.py +2 -2
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Update_parameters.py +0 -3
- pyantigen-1.0.13/PyAntiGen.egg-info/scm_version.json +0 -8
- pyantigen-1.0.13/framework/template/Example/Engine/Deadline.py +0 -535
- pyantigen-1.0.13/framework/template/Example/Engine/Nuisance_sensitivity.py +0 -298
- {pyantigen-1.0.13 → pyantigen-1.0.15}/.github/workflows/publish.yml +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/.gitignore +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/.vscode/launch.json +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/LICENSE +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/MANIFEST.in +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/dependency_links.txt +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/entry_points.txt +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/requires.txt +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/PyAntiGen.egg-info/top_level.txt +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/README.md +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/.agents/skills/module_generation/SKILL.md +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/.agents/skills/ode_conversion_antimony/SKILL.md +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/.agents/skills/ode_extraction_pipeline/SKILL.md +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/.gitignore +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/AntimonyGen.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/RxnDict_to_antimony.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/TelluriumGen.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/__init__.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/antimony_utils.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/cli.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/data_interpolation.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/isotopomer_tools.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/model_generation.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/models.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/module_base.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/pyantigen.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/rate_laws.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/reaction_creation.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/AntiGen_paths.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Anchor_cache.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Event_times.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Fast_profile.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Identifiability.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Model_simulate.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Petab_export.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Preequil_cache.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Results.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Engine/Sensitivity_analysis.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Flipflop_reference.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Model_generate.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Model_run.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Data.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Events.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Experiment.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Loss_config.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Observed_species.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Optimizer_settings.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Solver_settings.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/Example/Modules/Update_opt_parameters.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/data/ADneg.csv +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/data/ADpos.csv +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/data/Flipflop.csv +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/framework/template/data/make_flipflop_data.py +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/pyproject.toml +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/setup.cfg +0 -0
- {pyantigen-1.0.13 → pyantigen-1.0.15}/setup.py +0 -0
|
@@ -39,6 +39,7 @@ framework/template/Example/Model_generate.py
|
|
|
39
39
|
framework/template/Example/Model_run.py
|
|
40
40
|
framework/template/Example/Engine/Anchor_cache.py
|
|
41
41
|
framework/template/Example/Engine/Deadline.py
|
|
42
|
+
framework/template/Example/Engine/Differential_evolution.py
|
|
42
43
|
framework/template/Example/Engine/Evaluator.py
|
|
43
44
|
framework/template/Example/Engine/Event_times.py
|
|
44
45
|
framework/template/Example/Engine/Fast_profile.py
|
|
@@ -46,11 +47,13 @@ framework/template/Example/Engine/Fit_cache.py
|
|
|
46
47
|
framework/template/Example/Engine/Identifiability.py
|
|
47
48
|
framework/template/Example/Engine/Model_optimize.py
|
|
48
49
|
framework/template/Example/Engine/Model_simulate.py
|
|
49
|
-
framework/template/Example/Engine/
|
|
50
|
+
framework/template/Example/Engine/Nelder_mead.py
|
|
51
|
+
framework/template/Example/Engine/Noise_floor.py
|
|
50
52
|
framework/template/Example/Engine/Optimize.py
|
|
51
53
|
framework/template/Example/Engine/Petab_export.py
|
|
52
54
|
framework/template/Example/Engine/Preequil_cache.py
|
|
53
55
|
framework/template/Example/Engine/Profile_checkpoint.py
|
|
56
|
+
framework/template/Example/Engine/README.md
|
|
54
57
|
framework/template/Example/Engine/Results.py
|
|
55
58
|
framework/template/Example/Engine/Sensitivity_analysis.py
|
|
56
59
|
framework/template/Example/Engine/Simulate.py
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
"framework/template/Example/AntiGen_paths.py",
|
|
28
28
|
"framework/template/Example/Engine/Anchor_cache.py",
|
|
29
29
|
"framework/template/Example/Engine/Deadline.py",
|
|
30
|
+
"framework/template/Example/Engine/Differential_evolution.py",
|
|
30
31
|
"framework/template/Example/Engine/Evaluator.py",
|
|
31
32
|
"framework/template/Example/Engine/Event_times.py",
|
|
32
33
|
"framework/template/Example/Engine/Fast_profile.py",
|
|
@@ -34,11 +35,13 @@
|
|
|
34
35
|
"framework/template/Example/Engine/Identifiability.py",
|
|
35
36
|
"framework/template/Example/Engine/Model_optimize.py",
|
|
36
37
|
"framework/template/Example/Engine/Model_simulate.py",
|
|
37
|
-
"framework/template/Example/Engine/
|
|
38
|
+
"framework/template/Example/Engine/Nelder_mead.py",
|
|
39
|
+
"framework/template/Example/Engine/Noise_floor.py",
|
|
38
40
|
"framework/template/Example/Engine/Optimize.py",
|
|
39
41
|
"framework/template/Example/Engine/Petab_export.py",
|
|
40
42
|
"framework/template/Example/Engine/Preequil_cache.py",
|
|
41
43
|
"framework/template/Example/Engine/Profile_checkpoint.py",
|
|
44
|
+
"framework/template/Example/Engine/README.md",
|
|
42
45
|
"framework/template/Example/Engine/Results.py",
|
|
43
46
|
"framework/template/Example/Engine/Sensitivity_analysis.py",
|
|
44
47
|
"framework/template/Example/Engine/Simulate.py",
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '1.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (1, 0,
|
|
21
|
+
__version__ = version = '1.0.15'
|
|
22
|
+
__version_tuple__ = version_tuple = (1, 0, 15)
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'gd95e4a4f4'
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"""An optional wall-clock stop time, and nothing that tries to predict the future.
|
|
2
|
+
|
|
3
|
+
What this is *not* for. It is not what keeps a run alive on a preemptible
|
|
4
|
+
partition. Hyak's ``ckpt`` partitions stop a job without notice and with no
|
|
5
|
+
grace period, and the time limit Slurm reports (``SLURM_JOB_END_TIME``) is not
|
|
6
|
+
when that happens, so no amount of watching the clock can say when the job will
|
|
7
|
+
die. What makes a kill survivable is that the optimizers write their state as
|
|
8
|
+
they go -- see :mod:`Engine.Nelder_mead` -- so at any instant the copy on disk
|
|
9
|
+
is at most a couple of evaluations old. Given that, there is nothing left for a
|
|
10
|
+
deadline to protect against on ckpt, and this module deliberately does nothing
|
|
11
|
+
there.
|
|
12
|
+
|
|
13
|
+
What it is for. Two cases where the end *is* known and stopping tidily is worth
|
|
14
|
+
a little:
|
|
15
|
+
|
|
16
|
+
* ``--wall-time`` (or ``PROFILE_WALL_TIME``), an explicit budget, which is how
|
|
17
|
+
one asks for a shorter run than the allocation on a laptop or in a test.
|
|
18
|
+
* A partition that runs to its ``--time`` limit and is never preempted, where
|
|
19
|
+
``SLURM_JOB_END_TIME`` really is the end.
|
|
20
|
+
|
|
21
|
+
In both, the point of stopping a little early is the write-up: the trace
|
|
22
|
+
assembly, the plots and the results file take real time, and a run that is
|
|
23
|
+
killed in the middle of them has computed everything and reported nothing.
|
|
24
|
+
So the run stops handing out work ``margin_s`` before the end, lets every point
|
|
25
|
+
in flight save its state at its next evaluation and return, and reports what it
|
|
26
|
+
has as ``INCOMPLETE``.
|
|
27
|
+
|
|
28
|
+
A wrong answer here is cheap by construction. A stop time that comes out too
|
|
29
|
+
early makes the run end early and say so; one that comes out too late means the
|
|
30
|
+
job is killed while working, which the state on disk already survives. Neither
|
|
31
|
+
loses a computed point, which is why none of the source-ranking, staleness
|
|
32
|
+
detection or fallback logic this file used to carry is needed any more.
|
|
33
|
+
|
|
34
|
+
Off a scheduler with no flag, :func:`resolve_deadline` returns ``None``, every
|
|
35
|
+
check passes, and the run behaves exactly as it always did.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
import os
|
|
39
|
+
import time
|
|
40
|
+
|
|
41
|
+
# How long before the end the run stops handing out work. It has to cover what
|
|
42
|
+
# happens *after* the last point lands: the pool shutdown, trace assembly,
|
|
43
|
+
# plotting and the results write, none of which are instant on a large spec.
|
|
44
|
+
DEFAULT_MARGIN_S = 600.0
|
|
45
|
+
|
|
46
|
+
# Set by Model_run.py from --wall-time. An environment variable rather than a
|
|
47
|
+
# threaded argument because the only consumer sits at the bottom of a call chain
|
|
48
|
+
# that would otherwise need the flag added to a dozen signatures for it.
|
|
49
|
+
WALL_TIME_ENV = "PROFILE_WALL_TIME"
|
|
50
|
+
MARGIN_ENV = "PROFILE_DEADLINE_MARGIN"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class DeadlineReached(RuntimeError):
|
|
54
|
+
"""Raised when the wall budget stopped a batch before its jobs were done.
|
|
55
|
+
|
|
56
|
+
Carries how much work was left so the caller can say so plainly. This is a
|
|
57
|
+
normal, successful outcome of a run with a stop time -- not a failure -- and
|
|
58
|
+
callers are expected to catch it, finish reporting on what they have, and
|
|
59
|
+
exit 0.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, n_remaining, label=None):
|
|
63
|
+
self.n_remaining = int(n_remaining)
|
|
64
|
+
self.label = label
|
|
65
|
+
where = f" in {label}" if label else ""
|
|
66
|
+
super().__init__(
|
|
67
|
+
f"wall-clock budget reached with {self.n_remaining} point(s) "
|
|
68
|
+
f"not started{where}"
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# ---------------------------------------------------------------------------
|
|
73
|
+
# Parsing and discovery
|
|
74
|
+
# ---------------------------------------------------------------------------
|
|
75
|
+
|
|
76
|
+
def parse_duration(text):
|
|
77
|
+
"""Seconds from ``4h``, ``3.5h``, ``90m``, ``45s``, ``4:00:00`` or ``1-12:00:00``.
|
|
78
|
+
|
|
79
|
+
A bare number is seconds. That differs from ``sbatch --time``, where a bare
|
|
80
|
+
number is minutes, so prefer an explicit suffix when writing one by hand.
|
|
81
|
+
Returns None for anything unparseable, which the callers treat as "no
|
|
82
|
+
budget" rather than as an error -- a malformed value must not take down a
|
|
83
|
+
run that would otherwise have completed.
|
|
84
|
+
"""
|
|
85
|
+
if text is None:
|
|
86
|
+
return None
|
|
87
|
+
if isinstance(text, (int, float)):
|
|
88
|
+
return float(text) if text > 0 else None
|
|
89
|
+
|
|
90
|
+
s = str(text).strip().lower()
|
|
91
|
+
if not s:
|
|
92
|
+
return None
|
|
93
|
+
|
|
94
|
+
days = 0.0
|
|
95
|
+
if "-" in s:
|
|
96
|
+
head, _, s = s.partition("-")
|
|
97
|
+
try:
|
|
98
|
+
days = float(head)
|
|
99
|
+
except ValueError:
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
if ":" in s:
|
|
104
|
+
parts = [float(p) for p in s.split(":")]
|
|
105
|
+
if len(parts) > 3:
|
|
106
|
+
return None
|
|
107
|
+
while len(parts) < 3:
|
|
108
|
+
parts.insert(0, 0.0)
|
|
109
|
+
h, m, sec = parts
|
|
110
|
+
total = h * 3600.0 + m * 60.0 + sec
|
|
111
|
+
elif s[-1] in "smhd":
|
|
112
|
+
mult = {"s": 1.0, "m": 60.0, "h": 3600.0, "d": 86400.0}[s[-1]]
|
|
113
|
+
total = float(s[:-1]) * mult
|
|
114
|
+
else:
|
|
115
|
+
total = float(s)
|
|
116
|
+
except (ValueError, IndexError):
|
|
117
|
+
return None
|
|
118
|
+
|
|
119
|
+
total += days * 86400.0
|
|
120
|
+
return total if total > 0 else None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def partition_is_preemptible():
|
|
124
|
+
"""Whether this job can be evicted before its time limit.
|
|
125
|
+
|
|
126
|
+
On Hyak the checkpoint partitions -- ``ckpt``, ``ckpt-g2``, ``ckpt-all`` --
|
|
127
|
+
run on other groups' idle nodes and are stopped without notice when an owner
|
|
128
|
+
reclaims them. Every other partition runs to its ``--time`` and is not
|
|
129
|
+
preempted. Read from the partition name because that is the only thing Slurm
|
|
130
|
+
exposes that distinguishes the two; off a scheduler there is nothing to be
|
|
131
|
+
preempted by, so the answer is False.
|
|
132
|
+
"""
|
|
133
|
+
partition = (os.environ.get("SLURM_JOB_PARTITION") or "").lower()
|
|
134
|
+
return "ckpt" in partition
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def resolve_deadline(wall_time=None, now=None, verbose=True):
|
|
138
|
+
"""Absolute epoch time this process should be finished by, or None.
|
|
139
|
+
|
|
140
|
+
In order:
|
|
141
|
+
|
|
142
|
+
1. *wall_time*, else ``PROFILE_WALL_TIME`` -- an explicit budget measured
|
|
143
|
+
from now. It wins over the scheduler because it is how someone asks for a
|
|
144
|
+
shorter run than the allocation allows, and because it is the only source
|
|
145
|
+
that exists on a laptop.
|
|
146
|
+
2. ``SLURM_JOB_END_TIME``, but only on a partition that is not preempted.
|
|
147
|
+
On ckpt the job's real end is an eviction that this cannot see, the
|
|
148
|
+
reported limit is days away, and on a requeued job the variable has been
|
|
149
|
+
seen to carry a value left over from an earlier attempt. Nothing is lost
|
|
150
|
+
by ignoring it there -- state is written as the run goes -- and a
|
|
151
|
+
plausible-looking wrong end time is the one thing here that could make
|
|
152
|
+
a healthy run stop early.
|
|
153
|
+
|
|
154
|
+
None means no stop time, which turns every downstream check into a no-op.
|
|
155
|
+
"""
|
|
156
|
+
now = time.time() if now is None else now
|
|
157
|
+
|
|
158
|
+
budget = parse_duration(wall_time)
|
|
159
|
+
label = None
|
|
160
|
+
if budget is None:
|
|
161
|
+
raw_env = os.environ.get(WALL_TIME_ENV)
|
|
162
|
+
budget = parse_duration(raw_env)
|
|
163
|
+
if raw_env:
|
|
164
|
+
label = f"{WALL_TIME_ENV}={raw_env!r}"
|
|
165
|
+
elif wall_time:
|
|
166
|
+
label = f"--wall-time {wall_time!r}"
|
|
167
|
+
if budget is not None:
|
|
168
|
+
end = now + budget
|
|
169
|
+
if verbose:
|
|
170
|
+
print(f"[deadline] stop time from {label}: {_fmt_epoch(end)} "
|
|
171
|
+
f"({budget / 60.0:.0f} min from now)", flush=True)
|
|
172
|
+
return end
|
|
173
|
+
|
|
174
|
+
if partition_is_preemptible():
|
|
175
|
+
if verbose:
|
|
176
|
+
print("[deadline] preemptible partition: no stop time. Nothing "
|
|
177
|
+
"here can predict an eviction, and every optimizer saves "
|
|
178
|
+
"its state as it goes, so a kill costs a couple of "
|
|
179
|
+
"evaluations either way.", flush=True)
|
|
180
|
+
return None
|
|
181
|
+
|
|
182
|
+
raw = os.environ.get("SLURM_JOB_END_TIME")
|
|
183
|
+
try:
|
|
184
|
+
end = float(raw) if raw else None
|
|
185
|
+
except ValueError:
|
|
186
|
+
end = None
|
|
187
|
+
if end is not None and end > now:
|
|
188
|
+
if verbose:
|
|
189
|
+
print(f"[deadline] stop time from SLURM_JOB_END_TIME: "
|
|
190
|
+
f"{_fmt_epoch(end)} ({(end - now) / 60.0:.0f} min from now)",
|
|
191
|
+
flush=True)
|
|
192
|
+
return end
|
|
193
|
+
return None
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _fmt_epoch(t):
|
|
197
|
+
"""Local-time rendering of an epoch float, for a human reading a log."""
|
|
198
|
+
try:
|
|
199
|
+
return time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(t))
|
|
200
|
+
except (OSError, OverflowError, ValueError):
|
|
201
|
+
return repr(t)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _default_margin():
|
|
205
|
+
return parse_duration(os.environ.get(MARGIN_ENV)) or DEFAULT_MARGIN_S
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class RunBudget:
|
|
209
|
+
"""How much of this run's wall clock is left before it should stop.
|
|
210
|
+
|
|
211
|
+
Deliberately small: a stop time and a margin, and the two questions worth
|
|
212
|
+
asking of them -- "may another point start?" and "when must a running point
|
|
213
|
+
stop?". An unset deadline makes both permissive, so the same object can be
|
|
214
|
+
constructed unconditionally and passed everywhere.
|
|
215
|
+
|
|
216
|
+
There is no estimate of what a point costs. The old admission rule refused a
|
|
217
|
+
point that would not fit, which is exactly what made it wrong: a point can
|
|
218
|
+
be resumed from wherever it is stopped, so starting one with ten minutes to
|
|
219
|
+
spare is not a loss, and refusing it on a guess is.
|
|
220
|
+
"""
|
|
221
|
+
|
|
222
|
+
def __init__(self, deadline=None, margin_s=None):
|
|
223
|
+
self.deadline = deadline
|
|
224
|
+
self.margin_s = _default_margin() if margin_s is None else float(margin_s)
|
|
225
|
+
# Set once a batch has been cut short, so the run can report honestly
|
|
226
|
+
# that it stopped for time rather than because the work was done.
|
|
227
|
+
self.stopped_early = False
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def is_limited(self):
|
|
231
|
+
return self.deadline is not None
|
|
232
|
+
|
|
233
|
+
def remaining(self, now=None):
|
|
234
|
+
"""Seconds left before the deadline; ``inf`` when there is none."""
|
|
235
|
+
if self.deadline is None:
|
|
236
|
+
return float("inf")
|
|
237
|
+
return self.deadline - (time.time() if now is None else now)
|
|
238
|
+
|
|
239
|
+
def work_deadline(self):
|
|
240
|
+
"""The last moment this run may still be computing, or None.
|
|
241
|
+
|
|
242
|
+
Leaves the margin for trace assembly, plotting and the results write.
|
|
243
|
+
Running points are told to stop here; None means run to convergence.
|
|
244
|
+
"""
|
|
245
|
+
if self.deadline is None:
|
|
246
|
+
return None
|
|
247
|
+
return self.deadline - self.margin_s
|
|
248
|
+
|
|
249
|
+
def admits(self, now=None):
|
|
250
|
+
"""Whether another point may be started."""
|
|
251
|
+
if self.deadline is None:
|
|
252
|
+
return True
|
|
253
|
+
return self.remaining(now) > self.margin_s
|
|
254
|
+
|
|
255
|
+
def describe(self):
|
|
256
|
+
"""One line for the log, saying what the budget will actually do."""
|
|
257
|
+
if self.deadline is None:
|
|
258
|
+
return "no wall-clock stop time; running until the work is done"
|
|
259
|
+
return (f"{self.remaining() / 60.0:.0f} min left, "
|
|
260
|
+
f"{self.margin_s / 60.0:.0f} min of it reserved for the "
|
|
261
|
+
f"write-up; no new point starts after that, and running points "
|
|
262
|
+
f"save their state and stop at it")
|