cfdbender 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cfdbender/__init__.py +7 -0
- cfdbender/_pyfs.py +92 -0
- cfdbender/core/__init__.py +13 -0
- cfdbender/core/adapter.py +148 -0
- cfdbender/core/executables.py +157 -0
- cfdbender/core/flight.py +346 -0
- cfdbender/core/matrix.py +354 -0
- cfdbender/core/naming.py +172 -0
- cfdbender/core/plan.py +463 -0
- cfdbender/core/registry.py +735 -0
- cfdbender/core/results.py +117 -0
- cfdbender/core/run.py +150 -0
- cfdbender/core/settings.py +92 -0
- cfdbender/core/workspace.py +93 -0
- cfdbender/geometry/__init__.py +1 -0
- cfdbender/geometry/blade_slicing.py +355 -0
- cfdbender/hpc/__init__.py +21 -0
- cfdbender/hpc/_plan.py +266 -0
- cfdbender/hpc/_profiles.py +305 -0
- cfdbender/hpc/_render.py +271 -0
- cfdbender/hpc/_submit.py +96 -0
- cfdbender/hpc/_walltime.py +301 -0
- cfdbender/matrix/__init__.py +1 -0
- cfdbender/matrix/cli.py +228 -0
- cfdbender/matrix/collection.py +251 -0
- cfdbender/matrix/exports.py +122 -0
- cfdbender/matrix/rerun.py +126 -0
- cfdbender/matrix/selection.py +62 -0
- cfdbender/matrix/submission.py +272 -0
- cfdbender/solvers/__init__.py +47 -0
- cfdbender/solvers/_host/__init__.py +1 -0
- cfdbender/solvers/_host/_config.py +10 -0
- cfdbender/solvers/_host/_execute.py +52 -0
- cfdbender/solvers/_host/_mesh.py +62 -0
- cfdbender/solvers/_host/_ram.py +69 -0
- cfdbender/solvers/_host/_single.py +63 -0
- cfdbender/solvers/flightstream/__init__.py +114 -0
- cfdbender/solvers/flightstream/reader.py +191 -0
- cfdbender/solvers/mock/__init__.py +273 -0
- cfdbender/solvers/openfoam/__init__.py +201 -0
- cfdbender/solvers/openfoam/_config.py +410 -0
- cfdbender/solvers/openfoam/_execute.py +162 -0
- cfdbender/solvers/openfoam/_history.py +158 -0
- cfdbender/solvers/pyfr/__init__.py +118 -0
- cfdbender/solvers/pyfr/_config.py +196 -0
- cfdbender/solvers/pyfr/_execute.py +127 -0
- cfdbender/solvers/pyfr/_history.py +88 -0
- cfdbender/solvers/qprop/__init__.py +303 -0
- cfdbender/solvers/qprop/_config.py +259 -0
- cfdbender/solvers/qprop/_execute.py +269 -0
- cfdbender/solvers/qprop/_output.py +64 -0
- cfdbender/solvers/qprop/_polar.py +98 -0
- cfdbender/solvers/qprop/_prop.py +81 -0
- cfdbender/solvers/qprop/_sections.py +193 -0
- cfdbender/solvers/su2/__init__.py +210 -0
- cfdbender/solvers/su2/_config.py +290 -0
- cfdbender/solvers/su2/_execute.py +196 -0
- cfdbender/solvers/su2/_history.py +117 -0
- cfdbender/solvers/su2/_pysu2_worker.py +25 -0
- cfdbender/solvers/su2/_ram.py +13 -0
- cfdbender/solvers/su2/_reference.py +79 -0
- cfdbender/solvers/su2/_single.py +3 -0
- cfdbender/solvers/su2/routes.py +61 -0
- cfdbender/solvers/xfoil/__init__.py +171 -0
- cfdbender/solvers/xfoil/_config.py +144 -0
- cfdbender/solvers/xfoil/_execute.py +227 -0
- cfdbender/solvers/xfoil/_polar.py +88 -0
- cfdbender/testing/__init__.py +301 -0
- cfdbender/workspace/__init__.py +1 -0
- cfdbender/workspace/cli.py +99 -0
- cfdbender/workspace/planning.py +38 -0
- cfdbender-0.4.2.dist-info/METADATA +57 -0
- cfdbender-0.4.2.dist-info/RECORD +77 -0
- cfdbender-0.4.2.dist-info/WHEEL +5 -0
- cfdbender-0.4.2.dist-info/entry_points.txt +3 -0
- cfdbender-0.4.2.dist-info/licenses/LICENSE +21 -0
- cfdbender-0.4.2.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
"""Read workspace HPC profiles and refuse ineffective or misplaced settings."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
import tomllib
|
|
7
|
+
import warnings
|
|
8
|
+
from collections.abc import Collection, Mapping
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
HPC_FORMATS = ("yaml", "json", "toml", "text")
|
|
13
|
+
HPC_VERSION_ALIAS = "solver_version_alias"
|
|
14
|
+
_PROFILE_KEYS = frozenset(
|
|
15
|
+
{
|
|
16
|
+
"application_id",
|
|
17
|
+
"descriptor",
|
|
18
|
+
"submit",
|
|
19
|
+
"defaults",
|
|
20
|
+
"builds",
|
|
21
|
+
"application_ids",
|
|
22
|
+
"max_walltime",
|
|
23
|
+
"walltime_arithmetic",
|
|
24
|
+
}
|
|
25
|
+
)
|
|
26
|
+
_CLOCK = re.compile(r"(\d+):([0-5]\d):([0-5]\d)")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _descriptor_name(name: str, source: object) -> None:
|
|
30
|
+
alias = (
|
|
31
|
+
name.endswith((".", " "))
|
|
32
|
+
or ":" in name
|
|
33
|
+
or re.search(r"~\d", name)
|
|
34
|
+
or name.split(".")[0].upper()
|
|
35
|
+
in {
|
|
36
|
+
"CON",
|
|
37
|
+
"PRN",
|
|
38
|
+
"AUX",
|
|
39
|
+
"NUL",
|
|
40
|
+
*[f"COM{i}" for i in range(1, 10)],
|
|
41
|
+
*[f"LPT{i}" for i in range(1, 10)],
|
|
42
|
+
}
|
|
43
|
+
)
|
|
44
|
+
if (
|
|
45
|
+
name.strip()
|
|
46
|
+
and not any(sep in name for sep in "/\\")
|
|
47
|
+
and name not in (".", "..")
|
|
48
|
+
and not alias
|
|
49
|
+
):
|
|
50
|
+
return
|
|
51
|
+
raise ValueError(
|
|
52
|
+
f"the HPC profile {source} names its descriptor {name!r}; the "
|
|
53
|
+
"descriptor is written in the folder each point runs in, so its name "
|
|
54
|
+
"is a plain file name, with no folder, no parent folder and no form "
|
|
55
|
+
"Windows reads as another file's."
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True)
|
|
60
|
+
class HpcProfile:
|
|
61
|
+
"""The application, descriptor fields and argv declared by one cluster profile.
|
|
62
|
+
|
|
63
|
+
``builds`` and ``application_ids`` are keyed by exact registered solver
|
|
64
|
+
versions. ``walltime_arithmetic`` selects the row's written clock (``wall``)
|
|
65
|
+
or integer seconds. ``path`` identifies the source in every refusal.
|
|
66
|
+
Descriptor fields keep declaration order, and booleans keep their type.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
application_id: str
|
|
70
|
+
descriptor_format: str
|
|
71
|
+
descriptor_name: str
|
|
72
|
+
fields: dict[str, str | bool]
|
|
73
|
+
submit: tuple[str, ...]
|
|
74
|
+
path: Path
|
|
75
|
+
builds: dict[str, str] = field(default_factory=dict)
|
|
76
|
+
application_ids: dict[str, str] = field(default_factory=dict)
|
|
77
|
+
walltime_arithmetic: str = "wall"
|
|
78
|
+
max_walltime_s: float | None = None
|
|
79
|
+
|
|
80
|
+
def __post_init__(self) -> None:
|
|
81
|
+
"""Hold Python-created profiles to the same descriptor filename rule."""
|
|
82
|
+
_descriptor_name(self.descriptor_name, self.path)
|
|
83
|
+
|
|
84
|
+
def application_id_for(self, solver_version: object) -> str:
|
|
85
|
+
"""Return the version's application override, otherwise the required default."""
|
|
86
|
+
return self.application_ids.get(str(solver_version), self.application_id)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _unknown(
|
|
90
|
+
target: Path, table: Mapping[str, object], allowed: Collection[str], where: str
|
|
91
|
+
) -> None:
|
|
92
|
+
unknown = sorted(key for key in table if key not in allowed)
|
|
93
|
+
if unknown:
|
|
94
|
+
raise ValueError(
|
|
95
|
+
f"the HPC profile {target} states {', '.join(unknown)} {where}, and this package "
|
|
96
|
+
f"reads {', '.join(sorted(allowed))} there. A key nothing reads is a key that looks "
|
|
97
|
+
"like it works. Correct the spelling, or move the key to the table that takes it."
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _aliases(target: Path, table: object, name: str, versions: Collection[str]) -> dict[str, str]:
|
|
102
|
+
if table is None:
|
|
103
|
+
return {}
|
|
104
|
+
if not isinstance(table, dict):
|
|
105
|
+
raise ValueError(
|
|
106
|
+
f"the HPC profile {target} has a {name} entry that is not a table. Write it as "
|
|
107
|
+
f"[{name}] with one line per canonical solver version."
|
|
108
|
+
)
|
|
109
|
+
aliases: dict[str, str] = {}
|
|
110
|
+
for key, value in table.items():
|
|
111
|
+
version = str(key).strip()
|
|
112
|
+
if version not in versions or version != key:
|
|
113
|
+
raise ValueError(
|
|
114
|
+
f"the HPC profile {target} maps {key!r} in its [{name}] table, and a key "
|
|
115
|
+
"there must name ONE registered solver version, because several versions can "
|
|
116
|
+
"share one scheduler name and only the solver version says which. "
|
|
117
|
+
"Supply the registered solver_versions and use their exact spelling."
|
|
118
|
+
)
|
|
119
|
+
alias = value.strip() if isinstance(value, str) else ""
|
|
120
|
+
if not alias:
|
|
121
|
+
raise ValueError(
|
|
122
|
+
f"the HPC profile {target} maps solver version {version!r} to {value!r} in its "
|
|
123
|
+
f"[{name}] table; the value is the scheduler's name, as text, "
|
|
124
|
+
"and it cannot be empty."
|
|
125
|
+
)
|
|
126
|
+
aliases[version] = alias
|
|
127
|
+
return aliases
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _max_walltime(target: Path, stated: object) -> float | None:
|
|
131
|
+
if stated is None:
|
|
132
|
+
return None
|
|
133
|
+
found = _CLOCK.fullmatch(stated.strip()) if isinstance(stated, str) else None
|
|
134
|
+
if found is None or not any(int(part) for part in found.groups()):
|
|
135
|
+
raise ValueError(
|
|
136
|
+
f"the HPC profile {target} states max_walltime = {stated!r}. It is the longest "
|
|
137
|
+
"wall clock the cluster grants a job, written HH:MM:SS with hours beyond 24 "
|
|
138
|
+
'allowed, for example "48:00:00"; 2d and 48h are not read, and a clock of zero '
|
|
139
|
+
"grants nothing."
|
|
140
|
+
)
|
|
141
|
+
hours, minutes, seconds = (int(part) for part in found.groups())
|
|
142
|
+
return float(hours * 3600 + minutes * 60 + seconds)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def read_hpc_profile(path: str | Path, *, solver_versions: Collection[str] = ()) -> HpcProfile:
|
|
146
|
+
"""Read a profile, refusing keys or values this planner cannot act on.
|
|
147
|
+
|
|
148
|
+
Parameters
|
|
149
|
+
----------
|
|
150
|
+
path : str or Path
|
|
151
|
+
A TOML file under the workspace's ``inputs/hpc`` directory.
|
|
152
|
+
solver_versions : collection of str, optional
|
|
153
|
+
Exact registered solver versions supplied by the caller. Translation
|
|
154
|
+
tables require this registry; no solver adapter is imported here.
|
|
155
|
+
|
|
156
|
+
Returns
|
|
157
|
+
-------
|
|
158
|
+
HpcProfile
|
|
159
|
+
The validated scheduler declaration.
|
|
160
|
+
|
|
161
|
+
Raises
|
|
162
|
+
------
|
|
163
|
+
ValueError
|
|
164
|
+
A missing or unreadable file, invalid TOML, ineffective key, unsupported
|
|
165
|
+
format, invalid clock, missing fields or command, or unregistered version.
|
|
166
|
+
"""
|
|
167
|
+
target = Path(path)
|
|
168
|
+
try:
|
|
169
|
+
with target.open("rb") as stream:
|
|
170
|
+
table = tomllib.load(stream)
|
|
171
|
+
except (OSError, ValueError) as error:
|
|
172
|
+
raise ValueError(f"cannot read hpc profile {target}: {error}") from None
|
|
173
|
+
if not str(table.get("application_id") or "").strip():
|
|
174
|
+
raise ValueError(
|
|
175
|
+
f"the HPC profile {target} states no 'application_id'. It is the scheduler's own "
|
|
176
|
+
"name for the application and the one field nothing else in the "
|
|
177
|
+
"workspace can supply, which is why it is the only mandatory one."
|
|
178
|
+
)
|
|
179
|
+
for name in ("descriptor", "submit"):
|
|
180
|
+
if name in table and not isinstance(table[name], dict):
|
|
181
|
+
raise ValueError(
|
|
182
|
+
f"the HPC profile {target} has a {name} entry that is not a table. Write it "
|
|
183
|
+
f"as [{name}], with its keys under it."
|
|
184
|
+
)
|
|
185
|
+
if "defaults" in table:
|
|
186
|
+
warnings.warn(
|
|
187
|
+
f"the HPC profile {target} carries a defaults block, which has no effect; "
|
|
188
|
+
"it is ignored. Delete it.",
|
|
189
|
+
UserWarning,
|
|
190
|
+
stacklevel=2,
|
|
191
|
+
)
|
|
192
|
+
# A scalar owned by the profile cannot silently disappear inside another table.
|
|
193
|
+
for name, value in table.items():
|
|
194
|
+
if name == "defaults" or not isinstance(value, dict):
|
|
195
|
+
continue
|
|
196
|
+
for owned in ("max_walltime", "walltime_arithmetic"):
|
|
197
|
+
if owned in value:
|
|
198
|
+
raise ValueError(
|
|
199
|
+
f"the HPC profile {target} states {owned} under [{name}], and this "
|
|
200
|
+
"package reads it at the top level of the file. A key nothing reads is "
|
|
201
|
+
"a key that looks like it works: a limit written there is dropped and "
|
|
202
|
+
f"the profile loads with none. Move {owned} above the first table."
|
|
203
|
+
)
|
|
204
|
+
descriptor = table.get("descriptor") or {}
|
|
205
|
+
_unknown(target, descriptor, {"format", "name", "fields"}, "under [descriptor]")
|
|
206
|
+
fmt = str(descriptor.get("format") or "yaml").strip().lower()
|
|
207
|
+
if fmt not in HPC_FORMATS:
|
|
208
|
+
raise ValueError(
|
|
209
|
+
f"the HPC profile {target} asks for a {fmt!r} descriptor; this package "
|
|
210
|
+
f"writes {', '.join(HPC_FORMATS)}."
|
|
211
|
+
)
|
|
212
|
+
if "name" in descriptor:
|
|
213
|
+
_descriptor_name(str(descriptor["name"]), target)
|
|
214
|
+
fields = descriptor.get("fields") or {}
|
|
215
|
+
if not isinstance(fields, dict) or not fields:
|
|
216
|
+
raise ValueError(
|
|
217
|
+
f"the HPC profile {target} lists no descriptor fields, so the file it "
|
|
218
|
+
"writes would be empty. The fields table is what this cluster expects and "
|
|
219
|
+
"what makes the profile portable."
|
|
220
|
+
)
|
|
221
|
+
submit_table = table.get("submit") or {}
|
|
222
|
+
_unknown(target, submit_table, {"command"}, "under [submit]")
|
|
223
|
+
submit = submit_table.get("command") or []
|
|
224
|
+
if not isinstance(submit, list) or not submit:
|
|
225
|
+
raise ValueError(
|
|
226
|
+
f"the HPC profile {target} states no submit command. Write it argument by "
|
|
227
|
+
'argument, for example command = ["submit-tool", "{descriptor_path}"]: one string '
|
|
228
|
+
"through a shell splits a path that has a space in it."
|
|
229
|
+
)
|
|
230
|
+
if any(not isinstance(part, str) or not part.strip() for part in submit):
|
|
231
|
+
raise ValueError(f"the HPC profile {target} submit command needs non-empty strings")
|
|
232
|
+
builds = _aliases(target, table.get("builds"), "builds", solver_versions)
|
|
233
|
+
applications = _aliases(
|
|
234
|
+
target, table.get("application_ids"), "application_ids", solver_versions
|
|
235
|
+
)
|
|
236
|
+
arithmetic = str(table.get("walltime_arithmetic") or "wall").strip().lower()
|
|
237
|
+
if arithmetic not in {"wall", "seconds"}:
|
|
238
|
+
raise ValueError(
|
|
239
|
+
f"the HPC profile {target} states walltime_arithmetic = {arithmetic!r}, and "
|
|
240
|
+
"this package writes seconds, wall. 'wall' puts the row's cell in the descriptor "
|
|
241
|
+
"as written (4h stays 4h); 'seconds' puts the whole clock in integer seconds."
|
|
242
|
+
)
|
|
243
|
+
_unknown(target, table, _PROFILE_KEYS, "at its top level")
|
|
244
|
+
return HpcProfile(
|
|
245
|
+
application_id=str(table["application_id"]),
|
|
246
|
+
descriptor_format=fmt,
|
|
247
|
+
descriptor_name=str(descriptor.get("name") or f"submit.{'txt' if fmt == 'text' else fmt}"),
|
|
248
|
+
fields={str(k): v if isinstance(v, bool) else str(v) for k, v in fields.items()},
|
|
249
|
+
submit=tuple(str(part) for part in submit),
|
|
250
|
+
path=target,
|
|
251
|
+
builds=builds,
|
|
252
|
+
application_ids=applications,
|
|
253
|
+
walltime_arithmetic=arithmetic,
|
|
254
|
+
max_walltime_s=_max_walltime(target, table.get("max_walltime")),
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def resolve_hpc_profile(
|
|
259
|
+
inputs_dir: str | Path, name: str | None = None, *, solver_versions: Collection[str] = ()
|
|
260
|
+
) -> HpcProfile | None:
|
|
261
|
+
"""Select a named profile, the only profile, or no profile when the folder is empty.
|
|
262
|
+
|
|
263
|
+
Parameters
|
|
264
|
+
----------
|
|
265
|
+
inputs_dir : str or Path
|
|
266
|
+
The workspace's ``inputs`` directory.
|
|
267
|
+
name : str, optional
|
|
268
|
+
File stem or filename, such as ``h001`` or ``h001.toml``.
|
|
269
|
+
solver_versions : collection of str, optional
|
|
270
|
+
Registered versions for the profile's translation tables.
|
|
271
|
+
|
|
272
|
+
Returns
|
|
273
|
+
-------
|
|
274
|
+
HpcProfile or None
|
|
275
|
+
The selected profile or None when none exists and no name was requested.
|
|
276
|
+
|
|
277
|
+
Raises
|
|
278
|
+
------
|
|
279
|
+
ValueError
|
|
280
|
+
The selector names no file, several files need a selector, or loading fails.
|
|
281
|
+
"""
|
|
282
|
+
directory = Path(inputs_dir) / "hpc"
|
|
283
|
+
found = sorted(directory.glob("*.toml")) if directory.is_dir() else []
|
|
284
|
+
selected = Path(name).name if name is not None else None
|
|
285
|
+
if selected is not None and selected.lower().endswith(".toml"):
|
|
286
|
+
selected = selected[: -len(".toml")]
|
|
287
|
+
if selected is not None:
|
|
288
|
+
for path in found:
|
|
289
|
+
if path.stem == selected:
|
|
290
|
+
return read_hpc_profile(path, solver_versions=solver_versions)
|
|
291
|
+
raise ValueError(
|
|
292
|
+
f"{directory} holds no profile named {selected!r}; "
|
|
293
|
+
f"available: {', '.join(path.stem for path in found) or 'none'}."
|
|
294
|
+
)
|
|
295
|
+
if not found:
|
|
296
|
+
return None
|
|
297
|
+
if len(found) > 1:
|
|
298
|
+
raise ValueError(
|
|
299
|
+
f"{directory} holds {len(found)} profiles "
|
|
300
|
+
f"({', '.join(path.name for path in found)}) and nothing says which cluster "
|
|
301
|
+
"this machine is. One profile needs no selector; several need one: select "
|
|
302
|
+
"the profile by its file stem, hpc (CLI: --hpc) taking <name>, or "
|
|
303
|
+
"resolve_hpc_profile(inputs_dir, name); guessing spends a queue."
|
|
304
|
+
)
|
|
305
|
+
return read_hpc_profile(found[0], solver_versions=solver_versions)
|
cfdbender/hpc/_render.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Render profile-owned descriptors and submit arguments without side effects."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import string
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from ._profiles import HPC_FORMATS, HPC_VERSION_ALIAS, HpcProfile
|
|
12
|
+
|
|
13
|
+
_SUBMIT_VALUES = frozenset(
|
|
14
|
+
{
|
|
15
|
+
"application_id",
|
|
16
|
+
"config_path",
|
|
17
|
+
"config_paths",
|
|
18
|
+
"script_path",
|
|
19
|
+
"workdir",
|
|
20
|
+
"work_dir",
|
|
21
|
+
"descriptor_path",
|
|
22
|
+
HPC_VERSION_ALIAS,
|
|
23
|
+
"solver_version",
|
|
24
|
+
"solver_type",
|
|
25
|
+
"cluster",
|
|
26
|
+
"sim",
|
|
27
|
+
"point",
|
|
28
|
+
"ncpus",
|
|
29
|
+
"walltime",
|
|
30
|
+
"walltime_s",
|
|
31
|
+
"walltime_written",
|
|
32
|
+
"job_name",
|
|
33
|
+
"sims",
|
|
34
|
+
"first_sim",
|
|
35
|
+
"last_sim",
|
|
36
|
+
"batch",
|
|
37
|
+
"batch_id",
|
|
38
|
+
}
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _bound_values(profile: HpcProfile, values: Mapping[str, object]) -> dict[str, object]:
|
|
43
|
+
bound = {key: value for key, value in values.items() if value is not None and value != ""}
|
|
44
|
+
version = bound.get("solver_version")
|
|
45
|
+
bound["application_id"] = profile.application_ids.get(str(version), profile.application_id)
|
|
46
|
+
token = "{" + HPC_VERSION_ALIAS + "}"
|
|
47
|
+
names_alias = any(token in str(part) for part in (*profile.fields.values(), *profile.submit))
|
|
48
|
+
if names_alias and str(version) not in profile.builds:
|
|
49
|
+
mapped = ", ".join(f"{k} -> {v}" for k, v in sorted(profile.builds.items())) or "nothing"
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f"the HPC profile {profile.path} writes {{{HPC_VERSION_ALIAS}}} and its [builds] "
|
|
52
|
+
f"table maps no alias for solver version(s) {version or 'solver_version (missing)'} "
|
|
53
|
+
f"(it maps {mapped}). Add a line per solver version under [builds]. The table is a "
|
|
54
|
+
"declaration: it does not check that the scheduler starts that solver version, "
|
|
55
|
+
"which only the version in a collected log shows."
|
|
56
|
+
)
|
|
57
|
+
if str(version) in profile.builds:
|
|
58
|
+
bound[HPC_VERSION_ALIAS] = profile.builds[str(version)]
|
|
59
|
+
return bound
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def render_descriptor(profile: HpcProfile, values: Mapping[str, object]) -> str:
|
|
63
|
+
"""Render a descriptor using the profile's fields and typed substitutions.
|
|
64
|
+
|
|
65
|
+
Parameters
|
|
66
|
+
----------
|
|
67
|
+
profile : HpcProfile
|
|
68
|
+
The descriptor format, fields, and version translation tables.
|
|
69
|
+
values : mapping of str to object
|
|
70
|
+
Resolved point or group values. None and empty strings are missing.
|
|
71
|
+
|
|
72
|
+
Returns
|
|
73
|
+
-------
|
|
74
|
+
str
|
|
75
|
+
Newline-terminated YAML, JSON, TOML or text, without writing any file.
|
|
76
|
+
|
|
77
|
+
Raises
|
|
78
|
+
------
|
|
79
|
+
ValueError
|
|
80
|
+
Unsupported format, unmapped version, invalid template or missing value.
|
|
81
|
+
"""
|
|
82
|
+
if profile.descriptor_format not in HPC_FORMATS:
|
|
83
|
+
raise ValueError(f"the HPC profile {profile.path} asks for {profile.descriptor_format!r}")
|
|
84
|
+
return _render_descriptor(profile, _bound_values(profile, values))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def build_submit_command(
|
|
88
|
+
profile: HpcProfile, values: Mapping[str, object], descriptor_path: str | Path
|
|
89
|
+
) -> list[str]:
|
|
90
|
+
"""Format the profile's submit argv without invoking it or a shell.
|
|
91
|
+
|
|
92
|
+
Parameters
|
|
93
|
+
----------
|
|
94
|
+
profile : HpcProfile
|
|
95
|
+
Source of every command argument; no scheduler command is supplied here.
|
|
96
|
+
values : mapping of str to object
|
|
97
|
+
Point or group substitutions, including the resolved solver_version.
|
|
98
|
+
descriptor_path : str or Path
|
|
99
|
+
The descriptor's planned path; a space remains inside one argument.
|
|
100
|
+
|
|
101
|
+
Returns
|
|
102
|
+
-------
|
|
103
|
+
list of str
|
|
104
|
+
The command and arguments, ready for a separately authorized executor.
|
|
105
|
+
|
|
106
|
+
Raises
|
|
107
|
+
------
|
|
108
|
+
ValueError
|
|
109
|
+
A template asks for an unknown or unresolved value or an unmapped version.
|
|
110
|
+
"""
|
|
111
|
+
bound = _bound_values(profile, values)
|
|
112
|
+
bound["descriptor_path"] = Path(descriptor_path).as_posix()
|
|
113
|
+
known = _SUBMIT_VALUES | set(bound)
|
|
114
|
+
pending = list(profile.submit)
|
|
115
|
+
unknown: list[str] = []
|
|
116
|
+
while pending:
|
|
117
|
+
part = pending.pop()
|
|
118
|
+
try:
|
|
119
|
+
fields = list(string.Formatter().parse(part))
|
|
120
|
+
except ValueError:
|
|
121
|
+
unknown.append(f"an unreadable brace in {part!r}")
|
|
122
|
+
continue
|
|
123
|
+
for _, name, spec, _ in fields:
|
|
124
|
+
if name is None:
|
|
125
|
+
continue
|
|
126
|
+
if spec:
|
|
127
|
+
pending.append(spec)
|
|
128
|
+
root = re.split(r"[.\[]", name, maxsplit=1)[0]
|
|
129
|
+
if root not in known:
|
|
130
|
+
unknown.append("{" + name + "}")
|
|
131
|
+
if unknown:
|
|
132
|
+
raise ValueError(
|
|
133
|
+
f"the HPC profile {profile.path} writes {', '.join(unknown)} in its submit "
|
|
134
|
+
"command, and no run supplies it. What a run can supply: "
|
|
135
|
+
f"{', '.join(sorted(known))}."
|
|
136
|
+
)
|
|
137
|
+
try:
|
|
138
|
+
return [part.format(**bound) for part in profile.submit]
|
|
139
|
+
except (KeyError, IndexError, ValueError, AttributeError, TypeError) as error:
|
|
140
|
+
said = f"{{{error.args[0]}}}" if isinstance(error, KeyError) else str(error)
|
|
141
|
+
raise ValueError(
|
|
142
|
+
f"the HPC profile {profile.path} writes {said} in its submit command, "
|
|
143
|
+
f"and this point cannot supply it. What it can: {', '.join(sorted(bound))}."
|
|
144
|
+
) from None
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _numeric_field(template: object, values: Mapping[str, object]) -> bool:
|
|
148
|
+
"""Whether a descriptor field is a bare number rather than a quoted string.
|
|
149
|
+
|
|
150
|
+
True only when the template is exactly one substitution and the value
|
|
151
|
+
behind it is an int or a float. `26.123` is a build identifier that
|
|
152
|
+
reads as a float, and quoting it is the difference between a scheduler
|
|
153
|
+
receiving that build and receiving 26.12.
|
|
154
|
+
"""
|
|
155
|
+
if isinstance(template, bool):
|
|
156
|
+
# A TOML boolean in the profile (``driveg = true``) is a boolean the
|
|
157
|
+
# scheduler reads, written bare and lower case: quoted, the cluster
|
|
158
|
+
# reads the string "true" or "True" and not the switch.
|
|
159
|
+
return True
|
|
160
|
+
text = str(template).strip()
|
|
161
|
+
if not (text.startswith("{") and text.endswith("}") and text.count("{") == 1):
|
|
162
|
+
return False
|
|
163
|
+
return isinstance(values.get(text[1:-1]), (int, float)) and not isinstance(
|
|
164
|
+
values.get(text[1:-1]), bool
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _render_descriptor(profile: HpcProfile, values: Mapping[str, object]) -> str:
|
|
169
|
+
"""Render one submission descriptor from a profile and this point's values.
|
|
170
|
+
|
|
171
|
+
THE PROFILE OWNS THE KEYS and this owns none: every line comes from
|
|
172
|
+
the profile's own field table, so a cluster that spells ``cpus`` or
|
|
173
|
+
``queue`` or ``account`` is served by editing that table and nothing
|
|
174
|
+
else. A substitution the values cannot supply is refused rather than
|
|
175
|
+
written empty, because a descriptor with a blank where a job name goes
|
|
176
|
+
is a job the scheduler names for you.
|
|
177
|
+
|
|
178
|
+
Parameters
|
|
179
|
+
----------
|
|
180
|
+
profile : HpcProfile
|
|
181
|
+
The cluster profile: its field table, its descriptor format
|
|
182
|
+
(``json``, ``text``, ``toml`` or ``yaml``) and the path it was
|
|
183
|
+
read from.
|
|
184
|
+
values : mapping of str to object
|
|
185
|
+
This point's values, keyed by the placeholder names the profile's
|
|
186
|
+
templates use (for example ``sim``, ``point``, ``fs_build``,
|
|
187
|
+
``ncpus``).
|
|
188
|
+
|
|
189
|
+
Returns
|
|
190
|
+
-------
|
|
191
|
+
str
|
|
192
|
+
The descriptor text, newline terminated, in the profile's format.
|
|
193
|
+
|
|
194
|
+
Raises
|
|
195
|
+
------
|
|
196
|
+
ValueError
|
|
197
|
+
When a field of the profile asks for a placeholder that ``values``
|
|
198
|
+
does not carry.
|
|
199
|
+
"""
|
|
200
|
+
lines: list[str] = []
|
|
201
|
+
rendered: dict[str, str] = {}
|
|
202
|
+
for key, template in profile.fields.items():
|
|
203
|
+
try:
|
|
204
|
+
text = (
|
|
205
|
+
str(template).lower()
|
|
206
|
+
if isinstance(template, bool)
|
|
207
|
+
else str(template).format(**values)
|
|
208
|
+
)
|
|
209
|
+
except (KeyError, IndexError, ValueError, AttributeError, TypeError) as missing:
|
|
210
|
+
raise ValueError(
|
|
211
|
+
f"the HPC profile {profile.path} asks for {str(missing)!r} in its "
|
|
212
|
+
f"{key!r} "
|
|
213
|
+
f"field, and this run cannot supply it. What it can: "
|
|
214
|
+
f"{', '.join(sorted(values))}."
|
|
215
|
+
) from None
|
|
216
|
+
rendered[key] = text
|
|
217
|
+
if profile.descriptor_format == "json":
|
|
218
|
+
# THE YAML RULE, IN JSON: a boolean is a JSON boolean and a
|
|
219
|
+
# single substitution of a number is a JSON number; the rest are strings.
|
|
220
|
+
typed: dict[str, object] = {}
|
|
221
|
+
for key, text in rendered.items():
|
|
222
|
+
template = profile.fields[key]
|
|
223
|
+
if isinstance(template, bool):
|
|
224
|
+
typed[key] = template
|
|
225
|
+
elif _numeric_field(template, values):
|
|
226
|
+
typed[key] = values[str(template).strip()[1:-1]]
|
|
227
|
+
else:
|
|
228
|
+
typed[key] = text
|
|
229
|
+
return json.dumps(typed, indent=2) + "\n"
|
|
230
|
+
for key, text in rendered.items():
|
|
231
|
+
template = profile.fields[key]
|
|
232
|
+
safe_key = bool(re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", key))
|
|
233
|
+
if profile.descriptor_format == "yaml" and key.lower() in {
|
|
234
|
+
"true",
|
|
235
|
+
"false",
|
|
236
|
+
"null",
|
|
237
|
+
"yes",
|
|
238
|
+
"no",
|
|
239
|
+
"on",
|
|
240
|
+
"off",
|
|
241
|
+
}:
|
|
242
|
+
safe_key = False
|
|
243
|
+
structured_key = key if safe_key else json.dumps(key, ensure_ascii=False)
|
|
244
|
+
if profile.descriptor_format == "text":
|
|
245
|
+
lines.append(f"{key}: {text}")
|
|
246
|
+
elif profile.descriptor_format == "toml":
|
|
247
|
+
# A boolean and a single substitution of a number are bare, as in yaml.
|
|
248
|
+
bare = _numeric_field(template, values)
|
|
249
|
+
# A quoted string escapes its quote, backslash and control characters (F-1458).
|
|
250
|
+
lines.append(
|
|
251
|
+
f"{structured_key} = {text}"
|
|
252
|
+
if bare
|
|
253
|
+
else f"{structured_key} = {json.dumps(text, ensure_ascii=False)}"
|
|
254
|
+
)
|
|
255
|
+
else:
|
|
256
|
+
# yaml. A number stays bare and everything else is quoted, which
|
|
257
|
+
# is what the predecessor's own descriptor does.
|
|
258
|
+
#
|
|
259
|
+
# THE SOURCE TYPE DECIDES, NOT THE TEXT. A build identifier is
|
|
260
|
+
# `26.123`, which reads as a float and is not one: unquoted, a
|
|
261
|
+
# scheduler is handed 26.123 and may hand back 26.12. So a field
|
|
262
|
+
# is bare only when it is exactly one substitution AND the value
|
|
263
|
+
# behind it was a number.
|
|
264
|
+
bare = _numeric_field(template, values)
|
|
265
|
+
# A double-quoted yaml scalar escapes as json does (F-1459).
|
|
266
|
+
lines.append(
|
|
267
|
+
f"{structured_key}: {text}"
|
|
268
|
+
if bare
|
|
269
|
+
else f"{structured_key}: {json.dumps(text, ensure_ascii=False)}"
|
|
270
|
+
)
|
|
271
|
+
return "\n".join(lines) + "\n"
|
cfdbender/hpc/_submit.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Execute a planned profile argv once, without a shell or a Windows console."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import subprocess
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
|
|
10
|
+
from ._plan import SubmissionPlan
|
|
11
|
+
|
|
12
|
+
SUBMIT_TIMEOUT_S = 60.0
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class SubmitResult:
|
|
17
|
+
"""Submit-client evidence; its return code says nothing about solver convergence."""
|
|
18
|
+
|
|
19
|
+
return_code: int
|
|
20
|
+
stdout: str
|
|
21
|
+
stderr: str
|
|
22
|
+
started_at: str
|
|
23
|
+
finished_at: str
|
|
24
|
+
timeout_s: float
|
|
25
|
+
timed_out: bool
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def submit(job: SubmissionPlan) -> SubmitResult:
|
|
29
|
+
"""Write a descriptor and capture the profile's bounded submit client to files.
|
|
30
|
+
|
|
31
|
+
Parameters
|
|
32
|
+
----------
|
|
33
|
+
job : SubmissionPlan
|
|
34
|
+
Fully rendered descriptor and argv. The caller has claimed the point
|
|
35
|
+
and created its folder; existing descriptor and capture files refuse.
|
|
36
|
+
|
|
37
|
+
Returns
|
|
38
|
+
-------
|
|
39
|
+
SubmitResult
|
|
40
|
+
Process result with up to 64 KiB from each captured stream. Complete
|
|
41
|
+
bytes stay in submit.stdout.txt and submit.stderr.txt. Code 124 means
|
|
42
|
+
timeout, with scheduler acceptance unknown; 127 means launch failed.
|
|
43
|
+
"""
|
|
44
|
+
folder = job.descriptor_path.parent
|
|
45
|
+
with job.descriptor_path.open("x", encoding="utf-8", newline="") as descriptor:
|
|
46
|
+
descriptor.write(job.descriptor_text)
|
|
47
|
+
started = datetime.now(UTC).isoformat()
|
|
48
|
+
timed_out = False
|
|
49
|
+
with (
|
|
50
|
+
(folder / "submit.stdout.txt").open("xb") as stdout,
|
|
51
|
+
(folder / "submit.stderr.txt").open("xb") as stderr,
|
|
52
|
+
):
|
|
53
|
+
try:
|
|
54
|
+
result = subprocess.run(
|
|
55
|
+
job.command,
|
|
56
|
+
cwd=folder,
|
|
57
|
+
stdout=stdout,
|
|
58
|
+
stderr=stderr,
|
|
59
|
+
shell=False,
|
|
60
|
+
timeout=SUBMIT_TIMEOUT_S,
|
|
61
|
+
check=False,
|
|
62
|
+
env=os.environ.copy(),
|
|
63
|
+
# The constant exists only on Windows; 0 elsewhere (mypy checks both platforms).
|
|
64
|
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
|
65
|
+
)
|
|
66
|
+
code = result.returncode
|
|
67
|
+
except subprocess.TimeoutExpired:
|
|
68
|
+
code, timed_out = 124, True
|
|
69
|
+
stderr.write(
|
|
70
|
+
(
|
|
71
|
+
f"the submit command {job.command[0]!r} did not return within "
|
|
72
|
+
f"{SUBMIT_TIMEOUT_S} s; whether the scheduler took the job is unknown; "
|
|
73
|
+
f"the descriptor is {job.descriptor_path}" + chr(10)
|
|
74
|
+
).encode("utf-8")
|
|
75
|
+
)
|
|
76
|
+
except OSError as error:
|
|
77
|
+
code = 127
|
|
78
|
+
stderr.write(
|
|
79
|
+
(
|
|
80
|
+
f"the submit command {job.command[0]!r} could not be run: {error}; "
|
|
81
|
+
"the descriptor was written and nothing was submitted." + chr(10)
|
|
82
|
+
).encode("utf-8")
|
|
83
|
+
)
|
|
84
|
+
captures = []
|
|
85
|
+
for name in ("submit.stdout.txt", "submit.stderr.txt"):
|
|
86
|
+
with (folder / name).open("rb") as stream:
|
|
87
|
+
captures.append(stream.read(65536).decode("utf-8", errors="replace"))
|
|
88
|
+
return SubmitResult(
|
|
89
|
+
code,
|
|
90
|
+
captures[0],
|
|
91
|
+
captures[1],
|
|
92
|
+
started,
|
|
93
|
+
datetime.now(UTC).isoformat(),
|
|
94
|
+
SUBMIT_TIMEOUT_S,
|
|
95
|
+
timed_out,
|
|
96
|
+
)
|