hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/data/demo_data_manifest/__init__.py +3 -0
- hpcflow/sdk/__init__.py +4 -1
- hpcflow/sdk/app.py +160 -15
- hpcflow/sdk/cli.py +14 -0
- hpcflow/sdk/cli_common.py +83 -0
- hpcflow/sdk/config/__init__.py +4 -0
- hpcflow/sdk/config/callbacks.py +25 -2
- hpcflow/sdk/config/cli.py +4 -1
- hpcflow/sdk/config/config.py +188 -14
- hpcflow/sdk/config/config_file.py +91 -3
- hpcflow/sdk/config/errors.py +33 -0
- hpcflow/sdk/core/__init__.py +2 -0
- hpcflow/sdk/core/actions.py +492 -35
- hpcflow/sdk/core/cache.py +22 -0
- hpcflow/sdk/core/command_files.py +221 -5
- hpcflow/sdk/core/commands.py +57 -0
- hpcflow/sdk/core/element.py +407 -8
- hpcflow/sdk/core/environment.py +92 -0
- hpcflow/sdk/core/errors.py +245 -61
- hpcflow/sdk/core/json_like.py +72 -14
- hpcflow/sdk/core/loop.py +122 -21
- hpcflow/sdk/core/loop_cache.py +34 -9
- hpcflow/sdk/core/object_list.py +172 -26
- hpcflow/sdk/core/parallel.py +14 -0
- hpcflow/sdk/core/parameters.py +478 -25
- hpcflow/sdk/core/rule.py +31 -1
- hpcflow/sdk/core/run_dir_files.py +12 -2
- hpcflow/sdk/core/task.py +407 -80
- hpcflow/sdk/core/task_schema.py +70 -9
- hpcflow/sdk/core/test_utils.py +35 -0
- hpcflow/sdk/core/utils.py +101 -4
- hpcflow/sdk/core/validation.py +13 -1
- hpcflow/sdk/core/workflow.py +316 -96
- hpcflow/sdk/core/zarr_io.py +23 -0
- hpcflow/sdk/data/__init__.py +13 -0
- hpcflow/sdk/demo/__init__.py +3 -0
- hpcflow/sdk/helper/__init__.py +3 -0
- hpcflow/sdk/helper/cli.py +9 -0
- hpcflow/sdk/helper/helper.py +28 -0
- hpcflow/sdk/helper/watcher.py +33 -0
- hpcflow/sdk/log.py +40 -0
- hpcflow/sdk/persistence/__init__.py +14 -4
- hpcflow/sdk/persistence/base.py +289 -23
- hpcflow/sdk/persistence/json.py +29 -0
- hpcflow/sdk/persistence/pending.py +217 -107
- hpcflow/sdk/persistence/store_resource.py +58 -2
- hpcflow/sdk/persistence/utils.py +8 -0
- hpcflow/sdk/persistence/zarr.py +68 -1
- hpcflow/sdk/runtime.py +52 -10
- hpcflow/sdk/submission/__init__.py +3 -0
- hpcflow/sdk/submission/jobscript.py +198 -9
- hpcflow/sdk/submission/jobscript_info.py +13 -0
- hpcflow/sdk/submission/schedulers/__init__.py +60 -0
- hpcflow/sdk/submission/schedulers/direct.py +53 -0
- hpcflow/sdk/submission/schedulers/sge.py +45 -7
- hpcflow/sdk/submission/schedulers/slurm.py +45 -8
- hpcflow/sdk/submission/schedulers/utils.py +4 -0
- hpcflow/sdk/submission/shells/__init__.py +11 -1
- hpcflow/sdk/submission/shells/base.py +32 -1
- hpcflow/sdk/submission/shells/bash.py +36 -1
- hpcflow/sdk/submission/shells/os_version.py +18 -6
- hpcflow/sdk/submission/shells/powershell.py +22 -0
- hpcflow/sdk/submission/submission.py +88 -3
- hpcflow/sdk/typing.py +10 -1
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/METADATA +3 -3
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/RECORD +70 -70
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a179.dist-info → hpcflow_new2-0.2.0a181.dist-info}/entry_points.txt +0 -0
@@ -1,3 +1,7 @@
|
|
1
|
+
"""
|
2
|
+
Configuration file adapter.
|
3
|
+
"""
|
4
|
+
|
1
5
|
from __future__ import annotations
|
2
6
|
|
3
7
|
import copy
|
@@ -26,10 +30,23 @@ from .errors import (
|
|
26
30
|
|
27
31
|
|
28
32
|
class ConfigFile:
|
29
|
-
"""
|
33
|
+
"""
|
34
|
+
Configuration file.
|
35
|
+
|
36
|
+
Parameters
|
37
|
+
----------
|
38
|
+
directory:
|
39
|
+
The directory containing the configuration file.
|
40
|
+
logger:
|
41
|
+
Where to log messages.
|
42
|
+
config_options:
|
43
|
+
Configuration options.
|
44
|
+
"""
|
30
45
|
|
31
46
|
def __init__(self, directory, logger, config_options):
|
47
|
+
#: Where to log messages.
|
32
48
|
self.logger = logger
|
49
|
+
#: The directory containing the configuration file.
|
33
50
|
self.directory = self._resolve_config_dir(
|
34
51
|
config_opt=config_options,
|
35
52
|
logger=self.logger,
|
@@ -39,9 +56,13 @@ class ConfigFile:
|
|
39
56
|
self._configs = []
|
40
57
|
|
41
58
|
# set by _load_file_data:
|
59
|
+
#: The path to the config file.
|
42
60
|
self.path = None
|
61
|
+
#: The cached contents of the config file.
|
43
62
|
self.contents = None
|
63
|
+
#: The parsed contents of the config file.
|
44
64
|
self.data = None
|
65
|
+
#: The parsed contents of the config file where the alternate parser was used.
|
45
66
|
self.data_rt = None
|
46
67
|
|
47
68
|
self._load_file_data(config_options)
|
@@ -105,12 +126,31 @@ class ConfigFile:
|
|
105
126
|
return file_schema
|
106
127
|
|
107
128
|
def get_invoc_data(self, config_key):
|
129
|
+
"""
|
130
|
+
Get the invocation data for the given configuration.
|
131
|
+
|
132
|
+
Parameters
|
133
|
+
----------
|
134
|
+
config_key: str
|
135
|
+
The name of the configuration within the configuration file.
|
136
|
+
"""
|
108
137
|
return self.data["configs"][config_key]
|
109
138
|
|
110
139
|
def get_invocation(self, config_key):
|
140
|
+
"""
|
141
|
+
Get the invocation for the given configuration.
|
142
|
+
|
143
|
+
Parameters
|
144
|
+
----------
|
145
|
+
config_key: str
|
146
|
+
The name of the configuration within the configuration file.
|
147
|
+
"""
|
111
148
|
return self.get_invoc_data(config_key)["invocation"]
|
112
149
|
|
113
150
|
def save(self):
|
151
|
+
"""
|
152
|
+
Write the (modified) configuration to the configuration file.
|
153
|
+
"""
|
114
154
|
new_data = copy.deepcopy(self.data)
|
115
155
|
new_data_rt = copy.deepcopy(self.data_rt)
|
116
156
|
new_contents = ""
|
@@ -271,6 +311,9 @@ class ConfigFile:
|
|
271
311
|
|
272
312
|
@staticmethod
|
273
313
|
def get_config_file_path(directory):
|
314
|
+
"""
|
315
|
+
Get the path to the configuration file.
|
316
|
+
"""
|
274
317
|
# Try both ".yml" and ".yaml" extensions:
|
275
318
|
path_yaml = directory.joinpath("config.yaml")
|
276
319
|
if path_yaml.is_file():
|
@@ -307,11 +350,36 @@ class ConfigFile:
|
|
307
350
|
def get_config_item(
|
308
351
|
self, config_key, name, raise_on_missing=False, default_value=None
|
309
352
|
):
|
353
|
+
"""
|
354
|
+
Get a configuration item.
|
355
|
+
|
356
|
+
Parameters
|
357
|
+
----------
|
358
|
+
config_key: str
|
359
|
+
The name of the configuration within the configuration file.
|
360
|
+
name: str
|
361
|
+
The name of the configuration item.
|
362
|
+
raise_on_missing: bool
|
363
|
+
Whether to raise an error if the config item is absent.
|
364
|
+
default_value:
|
365
|
+
The default value to use when the config item is absent
|
366
|
+
(and ``raise_on_missing`` is not specified).
|
367
|
+
"""
|
310
368
|
if raise_on_missing and name not in self.get_invoc_data(config_key)["config"]:
|
311
369
|
raise ValueError(f"missing from file: {name!r}")
|
312
370
|
return self.get_invoc_data(config_key)["config"].get(name, default_value)
|
313
371
|
|
314
372
|
def is_item_set(self, config_key, name):
|
373
|
+
"""
|
374
|
+
Determine if a configuration item is set.
|
375
|
+
|
376
|
+
Parameters
|
377
|
+
----------
|
378
|
+
config_key: str
|
379
|
+
The name of the configuration within the configuration file.
|
380
|
+
name: str
|
381
|
+
The name of the configuration item.
|
382
|
+
"""
|
315
383
|
try:
|
316
384
|
self.get_config_item(config_key, name, raise_on_missing=True)
|
317
385
|
except ValueError:
|
@@ -319,7 +387,16 @@ class ConfigFile:
|
|
319
387
|
return True
|
320
388
|
|
321
389
|
def rename_config_key(self, config_key: str, new_config_key: str):
|
322
|
-
"""
|
390
|
+
"""
|
391
|
+
Change the config key of the loaded config.
|
392
|
+
|
393
|
+
Parameters
|
394
|
+
----------
|
395
|
+
config_key: str
|
396
|
+
The old name of the configuration within the configuration file.
|
397
|
+
new_config_key: str
|
398
|
+
The new name of the configuration.
|
399
|
+
"""
|
323
400
|
|
324
401
|
new_data = copy.deepcopy(self.data)
|
325
402
|
new_data_rt = copy.deepcopy(self.data_rt)
|
@@ -342,7 +419,18 @@ class ConfigFile:
|
|
342
419
|
environment_setup: Optional[str] = None,
|
343
420
|
match: Optional[Dict] = None,
|
344
421
|
):
|
345
|
-
"""
|
422
|
+
"""
|
423
|
+
Modify the invocation parameters of the loaded config.
|
424
|
+
|
425
|
+
Parameters
|
426
|
+
----------
|
427
|
+
config_key: str
|
428
|
+
The name of the configuration within the configuration file.
|
429
|
+
environment_setup:
|
430
|
+
The new value of the ``environment_setup`` key.
|
431
|
+
match:
|
432
|
+
The new values to merge into the ``match`` key.
|
433
|
+
"""
|
346
434
|
|
347
435
|
new_data = copy.deepcopy(self.data)
|
348
436
|
new_data_rt = copy.deepcopy(self.data_rt)
|
hpcflow/sdk/config/errors.py
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
"""
|
2
|
+
Miscellaneous configuration-related errors.
|
3
|
+
"""
|
4
|
+
|
5
|
+
|
1
6
|
class ConfigError(Exception):
|
2
7
|
"""Raised when a valid configuration can not be associated with the current
|
3
8
|
invocation."""
|
@@ -6,6 +11,10 @@ class ConfigError(Exception):
|
|
6
11
|
|
7
12
|
|
8
13
|
class ConfigUnknownItemError(ConfigError):
|
14
|
+
"""
|
15
|
+
Raised when the configuration contains an unknown item.
|
16
|
+
"""
|
17
|
+
|
9
18
|
def __init__(self, name, message=None):
|
10
19
|
self.message = message or (
|
11
20
|
f"Specified name {name!r} is not a valid meta-data or configurable "
|
@@ -15,6 +24,10 @@ class ConfigUnknownItemError(ConfigError):
|
|
15
24
|
|
16
25
|
|
17
26
|
class ConfigUnknownOverrideError(ConfigError):
|
27
|
+
"""
|
28
|
+
Raised when the configuration override contains an unknown item.
|
29
|
+
"""
|
30
|
+
|
18
31
|
def __init__(self, name, message=None):
|
19
32
|
self.message = message or (
|
20
33
|
f"Specified configuration override {name!r} is not a valid configurable item."
|
@@ -23,22 +36,38 @@ class ConfigUnknownOverrideError(ConfigError):
|
|
23
36
|
|
24
37
|
|
25
38
|
class ConfigNonConfigurableError(ConfigError):
|
39
|
+
"""
|
40
|
+
Raised when the configuration contains an item that can't be configured.
|
41
|
+
"""
|
42
|
+
|
26
43
|
def __init__(self, name, message=None):
|
27
44
|
self.message = message or (f"Specified name {name!r} is not a configurable item.")
|
28
45
|
super().__init__(self.message)
|
29
46
|
|
30
47
|
|
31
48
|
class ConfigItemAlreadyUnsetError(ConfigError):
|
49
|
+
"""
|
50
|
+
Raised when the configuration tries to unset an unset item.
|
51
|
+
"""
|
52
|
+
|
32
53
|
def __init__(self, name, message=None):
|
33
54
|
self.message = message or f"Configuration item {name!r} is already not set."
|
34
55
|
super().__init__(self.message)
|
35
56
|
|
36
57
|
|
37
58
|
class ConfigFileValidationError(ConfigError):
|
59
|
+
"""
|
60
|
+
Raised when the configuration file fails validation.
|
61
|
+
"""
|
62
|
+
|
38
63
|
pass
|
39
64
|
|
40
65
|
|
41
66
|
class ConfigItemCallbackError(ConfigError):
|
67
|
+
"""
|
68
|
+
Raised when a configuration callback errors.
|
69
|
+
"""
|
70
|
+
|
42
71
|
def __init__(self, name, callback, err, message=None):
|
43
72
|
self.message = message or (
|
44
73
|
f"Callback function {callback.__name__!r} for configuration item {name!r} "
|
@@ -59,6 +88,10 @@ class ConfigFileInvocationIncompatibleError(ConfigError):
|
|
59
88
|
|
60
89
|
|
61
90
|
class ConfigFileInvocationUnknownMatchKey(ConfigError):
|
91
|
+
"""
|
92
|
+
Raised when the configuration contains an invalid match key.
|
93
|
+
"""
|
94
|
+
|
62
95
|
def __init__(self, match_key, message=None):
|
63
96
|
self.message = message or (
|
64
97
|
f"Specified match key ({match_key!r}) is not a valid run time info "
|