hpcflow-new2 0.2.0a188__py3-none-any.whl → 0.2.0a190__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/sdk/__init__.py +21 -15
- hpcflow/sdk/app.py +2133 -770
- hpcflow/sdk/cli.py +281 -250
- hpcflow/sdk/cli_common.py +6 -2
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +77 -42
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +578 -311
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +112 -85
- hpcflow/sdk/config/types.py +145 -0
- hpcflow/sdk/core/actions.py +1054 -994
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +81 -63
- hpcflow/sdk/core/command_files.py +275 -185
- hpcflow/sdk/core/commands.py +111 -107
- hpcflow/sdk/core/element.py +724 -503
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +398 -51
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +380 -334
- hpcflow/sdk/core/loop_cache.py +160 -43
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +728 -600
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +33 -22
- hpcflow/sdk/core/task.py +1546 -1325
- hpcflow/sdk/core/task_schema.py +240 -196
- hpcflow/sdk/core/test_utils.py +126 -88
- hpcflow/sdk/core/types.py +387 -0
- hpcflow/sdk/core/utils.py +410 -305
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +1192 -1028
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/demo/cli.py +46 -33
- hpcflow/sdk/helper/cli.py +18 -16
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +83 -59
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +988 -586
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +408 -153
- hpcflow/sdk/persistence/pending.py +158 -123
- hpcflow/sdk/persistence/store_resource.py +37 -22
- hpcflow/sdk/persistence/types.py +307 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +477 -420
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +444 -404
- hpcflow/sdk/submission/schedulers/__init__.py +133 -40
- hpcflow/sdk/submission/schedulers/direct.py +97 -71
- hpcflow/sdk/submission/schedulers/sge.py +132 -126
- hpcflow/sdk/submission/schedulers/slurm.py +263 -268
- hpcflow/sdk/submission/schedulers/utils.py +7 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +102 -29
- hpcflow/sdk/submission/shells/bash.py +72 -55
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +37 -29
- hpcflow/sdk/submission/submission.py +203 -257
- hpcflow/sdk/submission/types.py +143 -0
- hpcflow/sdk/typing.py +163 -12
- hpcflow/tests/conftest.py +8 -6
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_main_scripts.py +60 -30
- hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
- hpcflow/tests/unit/test_action.py +86 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +13 -6
- hpcflow/tests/unit/test_cli.py +1 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +20 -15
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +3 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +65 -58
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +16 -7
- hpcflow/tests/unit/test_persistence.py +48 -35
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +8 -3
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +3 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +39 -19
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/workflows/test_jobscript.py +2 -1
- hpcflow/tests/workflows/test_workflows.py +18 -13
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
- hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a188.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/runtime.py
CHANGED
@@ -2,14 +2,16 @@
|
|
2
2
|
Information about the Python runtime.
|
3
3
|
"""
|
4
4
|
|
5
|
+
from __future__ import annotations
|
5
6
|
from importlib import import_module
|
6
|
-
import
|
7
|
+
from logging import Logger
|
7
8
|
import os
|
8
9
|
import platform
|
10
|
+
import re
|
9
11
|
import socket
|
10
12
|
import sys
|
11
13
|
from pathlib import Path
|
12
|
-
import
|
14
|
+
from typing import Any, ClassVar
|
13
15
|
|
14
16
|
from rich.table import Table
|
15
17
|
from rich.console import Console
|
@@ -31,10 +33,14 @@ class RunTimeInfo:
|
|
31
33
|
Where to write logging versions.
|
32
34
|
"""
|
33
35
|
|
34
|
-
def __init__(
|
35
|
-
|
36
|
+
def __init__(
|
37
|
+
self, name: str, package_name: str, version: str, logger: Logger
|
38
|
+
) -> None:
|
39
|
+
is_frozen: bool = getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS")
|
36
40
|
bundle_dir = (
|
37
|
-
sys._MEIPASS
|
41
|
+
sys._MEIPASS
|
42
|
+
if is_frozen and hasattr(sys, "_MEIPASS")
|
43
|
+
else os.path.dirname(os.path.abspath(__file__))
|
38
44
|
)
|
39
45
|
|
40
46
|
#: Application name.
|
@@ -69,10 +75,11 @@ class RunTimeInfo:
|
|
69
75
|
self.python_executable_path = Path(sys.executable)
|
70
76
|
|
71
77
|
try:
|
72
|
-
get_ipython
|
73
|
-
self.in_ipython = True
|
78
|
+
get_ipython # type: ignore
|
74
79
|
except NameError:
|
75
80
|
pass
|
81
|
+
else:
|
82
|
+
self.in_ipython = True
|
76
83
|
|
77
84
|
if hasattr(sys, "ps1"):
|
78
85
|
self.is_interactive = True
|
@@ -101,7 +108,7 @@ class RunTimeInfo:
|
|
101
108
|
|
102
109
|
try:
|
103
110
|
#: The virtual environment path.
|
104
|
-
self.venv_path = self.
|
111
|
+
self.venv_path: str | list[str] | None = self.__set_venv_path()
|
105
112
|
except ValueError:
|
106
113
|
self.venv_path = None
|
107
114
|
|
@@ -126,7 +133,7 @@ class RunTimeInfo:
|
|
126
133
|
# )
|
127
134
|
# warnings.warn(msg)
|
128
135
|
|
129
|
-
def to_dict(self):
|
136
|
+
def to_dict(self) -> dict[str, Any]:
|
130
137
|
"""
|
131
138
|
Serialize this class as a dictionary.
|
132
139
|
"""
|
@@ -171,16 +178,16 @@ class RunTimeInfo:
|
|
171
178
|
)
|
172
179
|
return out
|
173
180
|
|
174
|
-
def __repr__(self):
|
181
|
+
def __repr__(self) -> str:
|
175
182
|
out = f"{self.__class__.__name__}("
|
176
183
|
out += ", ".join(f"{k}={v!r}" for k, v in self.to_dict().items())
|
177
184
|
return out
|
178
185
|
|
179
|
-
def
|
180
|
-
out = []
|
181
|
-
if self.
|
186
|
+
def __set_venv_path(self) -> str | list[str]:
|
187
|
+
out: list[str] = []
|
188
|
+
if self.sys_prefix is not None:
|
182
189
|
out.append(self.sys_prefix)
|
183
|
-
elif self.
|
190
|
+
elif self.conda_prefix is not None:
|
184
191
|
out.append(self.conda_prefix)
|
185
192
|
if not out:
|
186
193
|
raise ValueError("Not running in a virtual environment!")
|
@@ -201,7 +208,7 @@ class RunTimeInfo:
|
|
201
208
|
"""
|
202
209
|
pass
|
203
210
|
|
204
|
-
def show(self):
|
211
|
+
def show(self) -> None:
|
205
212
|
"""
|
206
213
|
Display the information known by this class as a human-readable table.
|
207
214
|
"""
|
@@ -215,7 +222,7 @@ class RunTimeInfo:
|
|
215
222
|
console.print(tab)
|
216
223
|
|
217
224
|
@property
|
218
|
-
def executable_path(self):
|
225
|
+
def executable_path(self) -> Path | None:
|
219
226
|
"""Get the path that the user invoked to launch the frozen app, if the app is
|
220
227
|
frozen.
|
221
228
|
|
@@ -223,11 +230,10 @@ class RunTimeInfo:
|
|
223
230
|
whereas `executable_path_resolved` returns the actual frozen app path.
|
224
231
|
|
225
232
|
"""
|
226
|
-
if self.is_frozen
|
227
|
-
return Path(sys.argv[0])
|
233
|
+
return Path(sys.argv[0]) if self.is_frozen else None
|
228
234
|
|
229
235
|
@property
|
230
|
-
def resolved_executable_path(self):
|
236
|
+
def resolved_executable_path(self) -> Path | None:
|
231
237
|
"""Get the resolved path to the frozen app that the user launched, if the app is
|
232
238
|
frozen.
|
233
239
|
|
@@ -239,11 +245,10 @@ class RunTimeInfo:
|
|
239
245
|
[1] https://pyinstaller.org/en/stable/runtime-information.html#using-sys-executable-and-sys-argv-0
|
240
246
|
|
241
247
|
"""
|
242
|
-
if self.is_frozen
|
243
|
-
return Path(sys.executable)
|
248
|
+
return Path(sys.executable) if self.is_frozen else None
|
244
249
|
|
245
250
|
@property
|
246
|
-
def executable_name(self):
|
251
|
+
def executable_name(self) -> str | None:
|
247
252
|
"""Get the name of the frozen app executable, if the app is frozen.
|
248
253
|
|
249
254
|
If the user launches the app via a symbolic link, then this returns the name of
|
@@ -251,47 +256,45 @@ class RunTimeInfo:
|
|
251
256
|
name.
|
252
257
|
|
253
258
|
"""
|
254
|
-
if self.
|
255
|
-
return self.executable_path.name
|
259
|
+
return None if (p := self.executable_path) is None else p.name
|
256
260
|
|
257
261
|
@property
|
258
|
-
def resolved_executable_name(self):
|
262
|
+
def resolved_executable_name(self) -> str | None:
|
259
263
|
"""Get the resolved name of the frozen app executable, if the app is frozen."""
|
260
|
-
if self.
|
261
|
-
return self.resolved_executable_path.name
|
264
|
+
return None if (p := self.resolved_executable_path) is None else p.name
|
262
265
|
|
263
266
|
@property
|
264
|
-
def script_path(self) -> Path:
|
267
|
+
def script_path(self) -> Path | None:
|
265
268
|
"""Get the path to the Python script used to invoked this instance of the app, if
|
266
269
|
the app is not frozen."""
|
267
|
-
if
|
268
|
-
return Path(sys.argv[0])
|
270
|
+
return None if self.is_frozen else Path(sys.argv[0])
|
269
271
|
|
270
272
|
@property
|
271
|
-
def resolved_script_path(self) -> Path:
|
273
|
+
def resolved_script_path(self) -> Path | None:
|
272
274
|
"""Get the resolved path to the Python script used to invoked this instance of the
|
273
275
|
app, if the app is not frozen."""
|
274
|
-
if
|
275
|
-
|
276
|
+
return None if (p := self.script_path) is None else p.resolve()
|
277
|
+
|
278
|
+
# For removing a trailing '.cmd' from a filename
|
279
|
+
__CMD_TRIM: ClassVar[re.Pattern[str]] = re.compile(r"\.cmd$")
|
276
280
|
|
277
281
|
@property
|
278
|
-
def invocation_command(self):
|
282
|
+
def invocation_command(self) -> tuple[str, ...]:
|
279
283
|
"""Get the command that was used to invoke this instance of the app."""
|
280
284
|
if self.is_frozen:
|
281
285
|
# (this also works if we are running tests using the frozen app)
|
282
|
-
|
286
|
+
return (str(self.resolved_executable_path),)
|
283
287
|
elif self.from_CLI:
|
284
288
|
script = str(self.resolved_script_path)
|
285
|
-
if os.name == "nt"
|
289
|
+
if os.name == "nt":
|
286
290
|
# cannot reproduce locally, but on Windows GHA runners, if pytest is
|
287
291
|
# invoked via `hpcflow test`, `resolved_script_path` seems to be the
|
288
292
|
# batch script wrapper (ending in .cmd) rather than the Python entry point
|
289
293
|
# itself, so trim if off:
|
290
|
-
script =
|
291
|
-
|
294
|
+
script = self.__CMD_TRIM.sub("", script) # Work with 3.8 too
|
295
|
+
# script = script.removesuffix(".cmd")
|
296
|
+
return (str(self.python_executable_path), script)
|
292
297
|
else:
|
293
298
|
app_module = import_module(self.package_name)
|
294
299
|
CLI_path = Path(*app_module.__path__, "cli.py")
|
295
|
-
|
296
|
-
|
297
|
-
return tuple(command)
|
300
|
+
return (str(self.python_executable_path), str(CLI_path))
|
@@ -1,22 +1,29 @@
|
|
1
1
|
"""
|
2
|
-
|
2
|
+
Submission enumeration types.
|
3
3
|
"""
|
4
|
+
from __future__ import annotations
|
5
|
+
from dataclasses import dataclass
|
6
|
+
from enum import Enum
|
4
7
|
|
5
|
-
import enum
|
6
8
|
|
9
|
+
@dataclass(frozen=True)
|
10
|
+
class _JES:
|
11
|
+
"""
|
12
|
+
Model of the state of a JobscriptElementState
|
13
|
+
"""
|
7
14
|
|
8
|
-
|
15
|
+
_value: int
|
16
|
+
#: The symbol used to render the state.
|
17
|
+
symbol: str
|
18
|
+
#: The colour used to render the state.
|
19
|
+
colour: str
|
20
|
+
__doc__: str = ""
|
21
|
+
|
22
|
+
|
23
|
+
class JobscriptElementState(_JES, Enum):
|
9
24
|
"""Enumeration to convey a particular jobscript element state as reported by the
|
10
25
|
scheduler."""
|
11
26
|
|
12
|
-
def __new__(cls, value, symbol, colour, doc=None):
|
13
|
-
member = object.__new__(cls)
|
14
|
-
member._value_ = value
|
15
|
-
member.symbol = symbol
|
16
|
-
member.colour = colour
|
17
|
-
member.__doc__ = doc
|
18
|
-
return member
|
19
|
-
|
20
27
|
#: Waiting for resource allocation.
|
21
28
|
pending = (
|
22
29
|
0,
|
@@ -61,8 +68,28 @@ class JobscriptElementState(enum.Enum):
|
|
61
68
|
)
|
62
69
|
|
63
70
|
@property
|
64
|
-
def
|
71
|
+
def value(self) -> int:
|
72
|
+
"""
|
73
|
+
The numerical value of this state.
|
74
|
+
"""
|
75
|
+
return self._value
|
76
|
+
|
77
|
+
@property
|
78
|
+
def rich_repr(self) -> str:
|
65
79
|
"""
|
66
80
|
Rich representation of this enumeration element.
|
67
81
|
"""
|
68
82
|
return f"[{self.colour}]{self.symbol}[/{self.colour}]"
|
83
|
+
|
84
|
+
|
85
|
+
class SubmissionStatus(Enum):
|
86
|
+
"""
|
87
|
+
The overall status of a submission.
|
88
|
+
"""
|
89
|
+
|
90
|
+
#: Not yet submitted.
|
91
|
+
PENDING = 0
|
92
|
+
#: All jobscripts submitted successfully.
|
93
|
+
SUBMITTED = 1
|
94
|
+
#: Some jobscripts submitted successfully.
|
95
|
+
PARTIALLY_SUBMITTED = 2
|