hpcflow-new2 0.2.0a178__py3-none-any.whl → 0.2.0a180__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/data/demo_data_manifest/__init__.py +3 -0
- hpcflow/sdk/__init__.py +4 -1
- hpcflow/sdk/app.py +160 -15
- hpcflow/sdk/cli.py +14 -0
- hpcflow/sdk/cli_common.py +83 -0
- hpcflow/sdk/config/__init__.py +4 -0
- hpcflow/sdk/config/callbacks.py +25 -2
- hpcflow/sdk/config/cli.py +4 -1
- hpcflow/sdk/config/config.py +188 -14
- hpcflow/sdk/config/config_file.py +91 -3
- hpcflow/sdk/config/errors.py +33 -0
- hpcflow/sdk/core/__init__.py +2 -0
- hpcflow/sdk/core/actions.py +492 -35
- hpcflow/sdk/core/cache.py +22 -0
- hpcflow/sdk/core/command_files.py +221 -5
- hpcflow/sdk/core/commands.py +57 -0
- hpcflow/sdk/core/element.py +407 -8
- hpcflow/sdk/core/environment.py +92 -0
- hpcflow/sdk/core/errors.py +245 -61
- hpcflow/sdk/core/json_like.py +72 -14
- hpcflow/sdk/core/loop.py +122 -21
- hpcflow/sdk/core/loop_cache.py +34 -9
- hpcflow/sdk/core/object_list.py +172 -26
- hpcflow/sdk/core/parallel.py +14 -0
- hpcflow/sdk/core/parameters.py +478 -25
- hpcflow/sdk/core/rule.py +31 -1
- hpcflow/sdk/core/run_dir_files.py +12 -2
- hpcflow/sdk/core/task.py +407 -80
- hpcflow/sdk/core/task_schema.py +70 -9
- hpcflow/sdk/core/test_utils.py +35 -0
- hpcflow/sdk/core/utils.py +101 -4
- hpcflow/sdk/core/validation.py +13 -1
- hpcflow/sdk/core/workflow.py +316 -96
- hpcflow/sdk/core/zarr_io.py +23 -0
- hpcflow/sdk/data/__init__.py +13 -0
- hpcflow/sdk/demo/__init__.py +3 -0
- hpcflow/sdk/helper/__init__.py +3 -0
- hpcflow/sdk/helper/cli.py +9 -0
- hpcflow/sdk/helper/helper.py +28 -0
- hpcflow/sdk/helper/watcher.py +33 -0
- hpcflow/sdk/log.py +40 -0
- hpcflow/sdk/persistence/__init__.py +14 -4
- hpcflow/sdk/persistence/base.py +289 -23
- hpcflow/sdk/persistence/json.py +29 -0
- hpcflow/sdk/persistence/pending.py +217 -107
- hpcflow/sdk/persistence/store_resource.py +58 -2
- hpcflow/sdk/persistence/utils.py +8 -0
- hpcflow/sdk/persistence/zarr.py +68 -1
- hpcflow/sdk/runtime.py +52 -10
- hpcflow/sdk/submission/__init__.py +3 -0
- hpcflow/sdk/submission/jobscript.py +198 -9
- hpcflow/sdk/submission/jobscript_info.py +13 -0
- hpcflow/sdk/submission/schedulers/__init__.py +60 -0
- hpcflow/sdk/submission/schedulers/direct.py +53 -0
- hpcflow/sdk/submission/schedulers/sge.py +45 -7
- hpcflow/sdk/submission/schedulers/slurm.py +45 -8
- hpcflow/sdk/submission/schedulers/utils.py +4 -0
- hpcflow/sdk/submission/shells/__init__.py +11 -1
- hpcflow/sdk/submission/shells/base.py +32 -1
- hpcflow/sdk/submission/shells/bash.py +36 -1
- hpcflow/sdk/submission/shells/os_version.py +18 -6
- hpcflow/sdk/submission/shells/powershell.py +22 -0
- hpcflow/sdk/submission/submission.py +88 -3
- hpcflow/sdk/typing.py +10 -1
- {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/RECORD +70 -70
- {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a178.dist-info → hpcflow_new2-0.2.0a180.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/persistence/json.py
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
"""
|
2
|
+
Persistence model based on writing JSON documents.
|
3
|
+
"""
|
4
|
+
|
1
5
|
from __future__ import annotations
|
2
6
|
from contextlib import contextmanager
|
3
7
|
import copy
|
@@ -30,6 +34,10 @@ from hpcflow.sdk.persistence.base import update_param_source_dict
|
|
30
34
|
|
31
35
|
|
32
36
|
class JSONPersistentStore(PersistentStore):
|
37
|
+
"""
|
38
|
+
A store that writes JSON files for all its state serialization.
|
39
|
+
"""
|
40
|
+
|
33
41
|
_name = "json"
|
34
42
|
_features = PersistentStoreFeatures(
|
35
43
|
create=True,
|
@@ -90,6 +98,9 @@ class JSONPersistentStore(PersistentStore):
|
|
90
98
|
yield md
|
91
99
|
|
92
100
|
def remove_replaced_dir(self) -> None:
|
101
|
+
"""
|
102
|
+
Remove the directory containing replaced workflow details.
|
103
|
+
"""
|
93
104
|
with self.using_resource("metadata", "update") as md:
|
94
105
|
if "replaced_workflow" in md:
|
95
106
|
self.remove_path(md["replaced_workflow"], self.fs)
|
@@ -97,6 +108,9 @@ class JSONPersistentStore(PersistentStore):
|
|
97
108
|
md["replaced_workflow"] = None
|
98
109
|
|
99
110
|
def reinstate_replaced_dir(self) -> None:
|
111
|
+
"""
|
112
|
+
Reinstate the directory containing replaced workflow details.
|
113
|
+
"""
|
100
114
|
with self.using_resource("metadata", "read") as md:
|
101
115
|
if "replaced_workflow" in md:
|
102
116
|
self.logger.debug(
|
@@ -128,6 +142,9 @@ class JSONPersistentStore(PersistentStore):
|
|
128
142
|
ts_fmt: str,
|
129
143
|
ts_name_fmt: str,
|
130
144
|
) -> None:
|
145
|
+
"""
|
146
|
+
Write an empty persistent workflow.
|
147
|
+
"""
|
131
148
|
fs.mkdir(wk_path)
|
132
149
|
submissions = []
|
133
150
|
parameters = {
|
@@ -526,17 +543,29 @@ class JSONPersistentStore(PersistentStore):
|
|
526
543
|
return list(int(i) for i in params["data"].keys())
|
527
544
|
|
528
545
|
def get_ts_fmt(self):
|
546
|
+
"""
|
547
|
+
Get the format for timestamps.
|
548
|
+
"""
|
529
549
|
with self.using_resource("metadata", action="read") as md:
|
530
550
|
return md["ts_fmt"]
|
531
551
|
|
532
552
|
def get_ts_name_fmt(self):
|
553
|
+
"""
|
554
|
+
Get the format for timestamps to use in names.
|
555
|
+
"""
|
533
556
|
with self.using_resource("metadata", action="read") as md:
|
534
557
|
return md["ts_name_fmt"]
|
535
558
|
|
536
559
|
def get_creation_info(self):
|
560
|
+
"""
|
561
|
+
Get information about the creation of the workflow.
|
562
|
+
"""
|
537
563
|
with self.using_resource("metadata", action="read") as md:
|
538
564
|
return copy.deepcopy(md["creation_info"])
|
539
565
|
|
540
566
|
def get_name(self):
|
567
|
+
"""
|
568
|
+
Get the name of the workflow.
|
569
|
+
"""
|
541
570
|
with self.using_resource("metadata", action="read") as md:
|
542
571
|
return md["name"]
|