hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a190__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/sdk/__init__.py +21 -15
- hpcflow/sdk/app.py +2133 -770
- hpcflow/sdk/cli.py +281 -250
- hpcflow/sdk/cli_common.py +6 -2
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +77 -42
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +578 -311
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +112 -85
- hpcflow/sdk/config/types.py +145 -0
- hpcflow/sdk/core/actions.py +1054 -994
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +81 -63
- hpcflow/sdk/core/command_files.py +275 -185
- hpcflow/sdk/core/commands.py +111 -107
- hpcflow/sdk/core/element.py +724 -503
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +398 -51
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +380 -334
- hpcflow/sdk/core/loop_cache.py +160 -43
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +728 -600
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +33 -22
- hpcflow/sdk/core/task.py +1546 -1325
- hpcflow/sdk/core/task_schema.py +240 -196
- hpcflow/sdk/core/test_utils.py +126 -88
- hpcflow/sdk/core/types.py +387 -0
- hpcflow/sdk/core/utils.py +410 -305
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +1192 -1028
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/demo/cli.py +46 -33
- hpcflow/sdk/helper/cli.py +18 -16
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +83 -59
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +988 -586
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +408 -153
- hpcflow/sdk/persistence/pending.py +158 -123
- hpcflow/sdk/persistence/store_resource.py +37 -22
- hpcflow/sdk/persistence/types.py +307 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +477 -420
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +444 -404
- hpcflow/sdk/submission/schedulers/__init__.py +133 -40
- hpcflow/sdk/submission/schedulers/direct.py +97 -71
- hpcflow/sdk/submission/schedulers/sge.py +132 -126
- hpcflow/sdk/submission/schedulers/slurm.py +263 -268
- hpcflow/sdk/submission/schedulers/utils.py +7 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +102 -29
- hpcflow/sdk/submission/shells/bash.py +72 -55
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +37 -29
- hpcflow/sdk/submission/submission.py +203 -257
- hpcflow/sdk/submission/types.py +143 -0
- hpcflow/sdk/typing.py +163 -12
- hpcflow/tests/conftest.py +8 -6
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_main_scripts.py +60 -30
- hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
- hpcflow/tests/unit/test_action.py +86 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +13 -6
- hpcflow/tests/unit/test_cli.py +1 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +20 -15
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +3 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +65 -58
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +16 -7
- hpcflow/tests/unit/test_persistence.py +48 -35
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +8 -3
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +3 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +39 -19
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/workflows/test_jobscript.py +2 -1
- hpcflow/tests/workflows/test_workflows.py +18 -13
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
- hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
@@ -4,9 +4,8 @@ from hpcflow.sdk import sdk_classes
|
|
4
4
|
|
5
5
|
|
6
6
|
# most of the modules in `sdk_classes` are imported on-demand via the app object:
|
7
|
-
hiddenimports =
|
8
|
-
|
9
|
-
hiddenimports += [
|
7
|
+
hiddenimports = [
|
8
|
+
*sdk_classes.values(),
|
10
9
|
"hpcflow.sdk.data",
|
11
10
|
"hpcflow.data.demo_data_manifest",
|
12
11
|
"hpcflow.data.scripts",
|
@@ -19,13 +18,16 @@ hiddenimports += [
|
|
19
18
|
"fsspec.implementations.github", # for GitHub fsspec file system
|
20
19
|
]
|
21
20
|
|
22
|
-
py_include_kwargs = dict(include_py_files=True, excludes=("**/__pycache__",))
|
23
21
|
datas = (
|
24
22
|
collect_data_files("hpcflow.sdk.data")
|
25
23
|
+ collect_data_files("hpcflow.data.demo_data_manifest")
|
26
|
-
+ collect_data_files(
|
24
|
+
+ collect_data_files(
|
25
|
+
"hpcflow.data.scripts", include_py_files=True, excludes=("**/__pycache__",)
|
26
|
+
)
|
27
27
|
+ collect_data_files("hpcflow.data.template_components")
|
28
28
|
+ collect_data_files("hpcflow.data.workflows")
|
29
|
-
+ collect_data_files(
|
29
|
+
+ collect_data_files(
|
30
|
+
"hpcflow.tests", include_py_files=True, excludes=("**/__pycache__",)
|
31
|
+
)
|
30
32
|
+ collect_data_files("hpcflow.tests.data")
|
31
33
|
)
|
hpcflow/_version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.2.
|
1
|
+
__version__ = "0.2.0a190"
|
hpcflow/app.py
CHANGED
hpcflow/sdk/__init__.py
CHANGED
@@ -1,15 +1,18 @@
|
|
1
1
|
"""Sub-package to define an extensible hpcflow application."""
|
2
2
|
|
3
|
+
from __future__ import annotations
|
4
|
+
from collections.abc import Mapping
|
3
5
|
import logging
|
4
6
|
import os
|
5
7
|
import sys
|
8
|
+
from typing import Final
|
6
9
|
|
7
10
|
#: Classes used in the construction of a workflow.
|
8
11
|
#: :meta hide-value:
|
9
|
-
sdk_classes = {
|
12
|
+
sdk_classes: Final[Mapping[str, str]] = {
|
10
13
|
"Workflow": "hpcflow.sdk.core.workflow",
|
11
14
|
"Task": "hpcflow.sdk.core.task",
|
12
|
-
"ActionScopeType": "hpcflow.sdk.core.
|
15
|
+
"ActionScopeType": "hpcflow.sdk.core.enums",
|
13
16
|
"ActionScope": "hpcflow.sdk.core.actions",
|
14
17
|
"ActionRule": "hpcflow.sdk.core.actions",
|
15
18
|
"Rule": "hpcflow.sdk.core.rule",
|
@@ -68,9 +71,9 @@ sdk_classes = {
|
|
68
71
|
"ValuePerturbation": "hpcflow.sdk.core.parameters",
|
69
72
|
"InputValue": "hpcflow.sdk.core.parameters",
|
70
73
|
"ResourceSpec": "hpcflow.sdk.core.parameters",
|
71
|
-
"TaskSourceType": "hpcflow.sdk.core.
|
72
|
-
"InputSourceType": "hpcflow.sdk.core.
|
73
|
-
"ParameterPropagationMode": "hpcflow.sdk.core.
|
74
|
+
"TaskSourceType": "hpcflow.sdk.core.enums",
|
75
|
+
"InputSourceType": "hpcflow.sdk.core.enums",
|
76
|
+
"ParameterPropagationMode": "hpcflow.sdk.core.enums",
|
74
77
|
"InputSource": "hpcflow.sdk.core.parameters",
|
75
78
|
"TaskObjective": "hpcflow.sdk.core.task_schema",
|
76
79
|
"TaskSchema": "hpcflow.sdk.core.task_schema",
|
@@ -87,6 +90,7 @@ sdk_classes = {
|
|
87
90
|
"WorkflowBlueprint": "hpcflow.sdk.core.workflow",
|
88
91
|
"Jobscript": "hpcflow.sdk.submission.jobscript",
|
89
92
|
"Submission": "hpcflow.sdk.submission.submission",
|
93
|
+
"QueuedScheduler": "hpcflow.sdk.submission.schedulers",
|
90
94
|
"DirectWindows": "hpcflow.sdk.submission.schedulers.direct",
|
91
95
|
"DirectPosix": "hpcflow.sdk.submission.schedulers.direct",
|
92
96
|
"SlurmPosix": "hpcflow.sdk.submission.schedulers.slurm",
|
@@ -98,7 +102,7 @@ sdk_classes = {
|
|
98
102
|
# these are defined as `BaseApp` methods with an underscore prefix:
|
99
103
|
#: Functions exported by the application.
|
100
104
|
#: :meta hide-value:
|
101
|
-
sdk_funcs = (
|
105
|
+
sdk_funcs: Final[tuple[str, ...]] = (
|
102
106
|
"make_workflow",
|
103
107
|
"make_demo_workflow",
|
104
108
|
"make_and_submit_workflow",
|
@@ -114,24 +118,26 @@ sdk_funcs = (
|
|
114
118
|
"cancel",
|
115
119
|
)
|
116
120
|
|
117
|
-
_SDK_CONSOLE_LOG_LEVEL = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
|
118
121
|
|
119
|
-
|
120
|
-
def get_SDK_logger(name=None):
|
122
|
+
def get_SDK_logger(name: str | None = None) -> logging.Logger:
|
121
123
|
"""Get a logger with prefix of "hpcflow_sdk" instead of "hpcflow.sdk" to ensure the
|
122
124
|
handlers of the SDK logger and app logger are distinct."""
|
123
125
|
name = ".".join(["hpcflow_sdk"] + (name or __name__).split(".")[2:])
|
124
126
|
return logging.getLogger(name)
|
125
127
|
|
126
128
|
|
127
|
-
|
128
|
-
|
129
|
+
def _init_logger() -> None:
|
130
|
+
level = os.environ.get("HPCFLOW_SDK_CONSOLE_LOG_LEVEL", "ERROR")
|
131
|
+
SDK_logger = get_SDK_logger()
|
132
|
+
SDK_logger.setLevel("DEBUG")
|
133
|
+
|
134
|
+
sh = logging.StreamHandler()
|
135
|
+
sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
|
136
|
+
sh.setLevel(level)
|
137
|
+
SDK_logger.addHandler(sh)
|
129
138
|
|
130
|
-
_sh = logging.StreamHandler()
|
131
|
-
_sh.setFormatter(logging.Formatter("%(levelname)s %(name)s: %(message)s"))
|
132
|
-
_sh.setLevel(_SDK_CONSOLE_LOG_LEVEL)
|
133
|
-
_SDK_logger.addHandler(_sh)
|
134
139
|
|
140
|
+
_init_logger()
|
135
141
|
if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"):
|
136
142
|
import multiprocessing
|
137
143
|
|