hpcflow-new2 0.2.0a188__py3-none-any.whl → 0.2.0a190__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/sdk/__init__.py +21 -15
- hpcflow/sdk/app.py +2133 -770
- hpcflow/sdk/cli.py +281 -250
- hpcflow/sdk/cli_common.py +6 -2
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +77 -42
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +578 -311
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +112 -85
- hpcflow/sdk/config/types.py +145 -0
- hpcflow/sdk/core/actions.py +1054 -994
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +81 -63
- hpcflow/sdk/core/command_files.py +275 -185
- hpcflow/sdk/core/commands.py +111 -107
- hpcflow/sdk/core/element.py +724 -503
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +398 -51
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +380 -334
- hpcflow/sdk/core/loop_cache.py +160 -43
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +728 -600
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +33 -22
- hpcflow/sdk/core/task.py +1546 -1325
- hpcflow/sdk/core/task_schema.py +240 -196
- hpcflow/sdk/core/test_utils.py +126 -88
- hpcflow/sdk/core/types.py +387 -0
- hpcflow/sdk/core/utils.py +410 -305
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +1192 -1028
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/demo/cli.py +46 -33
- hpcflow/sdk/helper/cli.py +18 -16
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +83 -59
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +988 -586
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +408 -153
- hpcflow/sdk/persistence/pending.py +158 -123
- hpcflow/sdk/persistence/store_resource.py +37 -22
- hpcflow/sdk/persistence/types.py +307 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +477 -420
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +444 -404
- hpcflow/sdk/submission/schedulers/__init__.py +133 -40
- hpcflow/sdk/submission/schedulers/direct.py +97 -71
- hpcflow/sdk/submission/schedulers/sge.py +132 -126
- hpcflow/sdk/submission/schedulers/slurm.py +263 -268
- hpcflow/sdk/submission/schedulers/utils.py +7 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +102 -29
- hpcflow/sdk/submission/shells/bash.py +72 -55
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +37 -29
- hpcflow/sdk/submission/submission.py +203 -257
- hpcflow/sdk/submission/types.py +143 -0
- hpcflow/sdk/typing.py +163 -12
- hpcflow/tests/conftest.py +8 -6
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_main_scripts.py +60 -30
- hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
- hpcflow/tests/unit/test_action.py +86 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +13 -6
- hpcflow/tests/unit/test_cli.py +1 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +20 -15
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +3 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +65 -58
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +16 -7
- hpcflow/tests/unit/test_persistence.py +48 -35
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +8 -3
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +3 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +39 -19
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/workflows/test_jobscript.py +2 -1
- hpcflow/tests/workflows/test_workflows.py +18 -13
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
- hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a188.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a188.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,143 @@
|
|
1
|
+
"""
|
2
|
+
Types for the submission subsystem.
|
3
|
+
"""
|
4
|
+
from __future__ import annotations
|
5
|
+
from typing import Any, TYPE_CHECKING
|
6
|
+
from typing_extensions import NotRequired, TypeAlias, TypedDict
|
7
|
+
|
8
|
+
if TYPE_CHECKING:
|
9
|
+
from collections.abc import Sequence
|
10
|
+
from datetime import datetime
|
11
|
+
from numpy.typing import NDArray
|
12
|
+
from ..core.element import ElementResources
|
13
|
+
|
14
|
+
|
15
|
+
class JobScriptDescriptor(TypedDict):
|
16
|
+
"""
|
17
|
+
Descriptor for a jobscript.
|
18
|
+
"""
|
19
|
+
|
20
|
+
#: Resources required by the jobscript.
|
21
|
+
resources: Any
|
22
|
+
#: Elements handled by the jobscript.
|
23
|
+
elements: dict[int, list[int]]
|
24
|
+
#: Dependencies of the jobscript.
|
25
|
+
dependencies: NotRequired[dict[int, ResolvedDependencies]]
|
26
|
+
#: Hash of resources.
|
27
|
+
resource_hash: NotRequired[str]
|
28
|
+
|
29
|
+
|
30
|
+
class ResolvedDependencies(TypedDict):
|
31
|
+
"""
|
32
|
+
The resolution of a dependency.
|
33
|
+
"""
|
34
|
+
|
35
|
+
#: Mapping of jobscript elements.
|
36
|
+
js_element_mapping: dict[int, list[int]]
|
37
|
+
#: Whether this is an array mapping.
|
38
|
+
is_array: NotRequired[bool]
|
39
|
+
|
40
|
+
|
41
|
+
class JobScriptCreationArguments(TypedDict):
|
42
|
+
"""
|
43
|
+
Arguments to pass to create a :class:`Jobscript`.
|
44
|
+
"""
|
45
|
+
|
46
|
+
#: The task insertion IDs.
|
47
|
+
task_insert_IDs: list[int]
|
48
|
+
#: The actions of the tasks.
|
49
|
+
task_actions: list[tuple[int, int, int]]
|
50
|
+
#: The elements of the tasks.
|
51
|
+
task_elements: dict[int, list[int]]
|
52
|
+
#: Element action run information.
|
53
|
+
EAR_ID: NDArray
|
54
|
+
#: Resources to use.
|
55
|
+
resources: ElementResources
|
56
|
+
#: Description of what loops are in play.
|
57
|
+
task_loop_idx: list[dict[str, int]]
|
58
|
+
#: Description of dependencies.
|
59
|
+
dependencies: dict[int, ResolvedDependencies]
|
60
|
+
#: When the jobscript was submitted, if known.
|
61
|
+
submit_time: NotRequired[datetime]
|
62
|
+
#: Where the jobscript was submitted, if known.
|
63
|
+
submit_hostname: NotRequired[str]
|
64
|
+
#: Description of what the jobscript was submitted to, if known.
|
65
|
+
submit_machine: NotRequired[str]
|
66
|
+
#: The command line used to do the commit, if known.
|
67
|
+
submit_cmdline: NotRequired[list[str]]
|
68
|
+
#: The job ID from the scheduler, if known.
|
69
|
+
scheduler_job_ID: NotRequired[str]
|
70
|
+
#: The process ID of the subprocess, if known.
|
71
|
+
process_ID: NotRequired[int]
|
72
|
+
#: Version info about the target system.
|
73
|
+
version_info: NotRequired[dict[str, str | list[str]]]
|
74
|
+
#: The name of the OS.
|
75
|
+
os_name: NotRequired[str]
|
76
|
+
#: The name of the shell.
|
77
|
+
shell_name: NotRequired[str]
|
78
|
+
#: The scheduler used.
|
79
|
+
scheduler_name: NotRequired[str]
|
80
|
+
#: Whether the jobscript is currently running.
|
81
|
+
running: NotRequired[bool]
|
82
|
+
#: Do not supply!
|
83
|
+
resource_hash: NotRequired[str]
|
84
|
+
#: Do not supply!
|
85
|
+
elements: NotRequired[dict[int, list[int]]]
|
86
|
+
|
87
|
+
|
88
|
+
class SchedulerRef(TypedDict):
|
89
|
+
"""
|
90
|
+
Scheduler reference descriptor.
|
91
|
+
"""
|
92
|
+
|
93
|
+
#: Jobscript references.
|
94
|
+
js_refs: list # Internal type is horrible and variable
|
95
|
+
#: Number of jobscript elements.
|
96
|
+
num_js_elements: int
|
97
|
+
|
98
|
+
|
99
|
+
class SubmissionPart(TypedDict):
|
100
|
+
"""
|
101
|
+
A part of a submission.
|
102
|
+
"""
|
103
|
+
|
104
|
+
#: Timestamp for when this part was submitted.
|
105
|
+
submit_time: datetime
|
106
|
+
#: The jobscripts involved in this submission.
|
107
|
+
jobscripts: list[int]
|
108
|
+
|
109
|
+
|
110
|
+
# This needs PEP 728 for a better type, alas
|
111
|
+
#: Version data.
|
112
|
+
VersionInfo: TypeAlias = "dict[str, str | list[str]]"
|
113
|
+
|
114
|
+
|
115
|
+
# TODO: This really doesn't belong here?!
|
116
|
+
class JobscriptHeaderArgs(TypedDict):
|
117
|
+
"""
|
118
|
+
Keyword arguments to use when creating a job script from a
|
119
|
+
:class:`Jobscript`.
|
120
|
+
"""
|
121
|
+
|
122
|
+
#: Application invocation. (Arguments, etc.)
|
123
|
+
app_invoc: str | Sequence[str]
|
124
|
+
#: Configuration directory.
|
125
|
+
config_dir: NotRequired[str]
|
126
|
+
#: Configuration key.
|
127
|
+
config_invoc_key: NotRequired[Any]
|
128
|
+
#: Name of EAR file.
|
129
|
+
EAR_file_name: NotRequired[str]
|
130
|
+
#: Name of file containing run directories.
|
131
|
+
element_run_dirs_file_path: NotRequired[str]
|
132
|
+
#: Environment setup.
|
133
|
+
env_setup: NotRequired[str]
|
134
|
+
#: Jobscript index.
|
135
|
+
js_idx: NotRequired[int]
|
136
|
+
#: Log file for the run.
|
137
|
+
run_log_file: NotRequired[str]
|
138
|
+
#: Submission index.
|
139
|
+
sub_idx: NotRequired[int]
|
140
|
+
#: Workflow application alias.
|
141
|
+
workflow_app_alias: NotRequired[str]
|
142
|
+
#: Path to workflow.
|
143
|
+
workflow_path: NotRequired[str]
|
hpcflow/sdk/typing.py
CHANGED
@@ -1,18 +1,169 @@
|
|
1
1
|
"""
|
2
2
|
Common type aliases.
|
3
3
|
"""
|
4
|
-
from
|
4
|
+
from __future__ import annotations
|
5
|
+
from dataclasses import InitVar
|
6
|
+
from typing import ClassVar, Final, TypeVar, cast, TYPE_CHECKING
|
7
|
+
from typing_extensions import NotRequired, TypeAlias, TypedDict
|
5
8
|
from pathlib import Path
|
9
|
+
import re
|
10
|
+
|
11
|
+
if TYPE_CHECKING:
|
12
|
+
from collections.abc import Mapping
|
13
|
+
from datetime import datetime
|
14
|
+
from .core.object_list import (
|
15
|
+
CommandFilesList,
|
16
|
+
EnvironmentsList,
|
17
|
+
ParametersList,
|
18
|
+
TaskSchemasList,
|
19
|
+
)
|
20
|
+
from .submission.enums import JobscriptElementState
|
21
|
+
from .submission.submission import Submission
|
6
22
|
|
7
23
|
#: Type of a value that can be treated as a path.
|
8
|
-
PathLike =
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
#:
|
17
|
-
|
18
|
-
|
24
|
+
PathLike: TypeAlias = "str | Path | None"
|
25
|
+
|
26
|
+
|
27
|
+
class ParamSource(TypedDict):
|
28
|
+
"""
|
29
|
+
A parameter source descriptor.
|
30
|
+
"""
|
31
|
+
|
32
|
+
#: Parameter type name.
|
33
|
+
type: NotRequired[str]
|
34
|
+
#: EAR ID.
|
35
|
+
EAR_ID: NotRequired[int]
|
36
|
+
#: Task insertion ID.
|
37
|
+
task_insert_ID: NotRequired[int]
|
38
|
+
#: Action index.
|
39
|
+
action_idx: NotRequired[int]
|
40
|
+
#: Element index.
|
41
|
+
element_idx: NotRequired[int]
|
42
|
+
#: Element set index.
|
43
|
+
element_set_idx: NotRequired[int]
|
44
|
+
#: Element action run index.
|
45
|
+
run_idx: NotRequired[int]
|
46
|
+
#: Sequence index.
|
47
|
+
sequence_idx: NotRequired[int]
|
48
|
+
#: Task index.
|
49
|
+
task_idx: NotRequired[int]
|
50
|
+
#: Name of method used to create the parameter's value(s).
|
51
|
+
value_class_method: NotRequired[str]
|
52
|
+
|
53
|
+
|
54
|
+
class KnownSubmission(TypedDict):
|
55
|
+
"""
|
56
|
+
Describes a known submission.
|
57
|
+
"""
|
58
|
+
|
59
|
+
#: Local ID.
|
60
|
+
local_id: int
|
61
|
+
#: Workflow global ID.
|
62
|
+
workflow_id: str
|
63
|
+
#: Whether the submission is active.
|
64
|
+
is_active: bool
|
65
|
+
#: Submission index.
|
66
|
+
sub_idx: int
|
67
|
+
#: Submission time.
|
68
|
+
submit_time: str
|
69
|
+
#: Path to submission.
|
70
|
+
path: str
|
71
|
+
#: Start time.
|
72
|
+
start_time: str
|
73
|
+
#: Finish time.
|
74
|
+
end_time: str
|
75
|
+
|
76
|
+
|
77
|
+
class KnownSubmissionItem(TypedDict):
|
78
|
+
"""
|
79
|
+
Describes a known submission.
|
80
|
+
"""
|
81
|
+
|
82
|
+
#: Local ID.
|
83
|
+
local_id: int
|
84
|
+
#: Workflow global ID.
|
85
|
+
workflow_id: str
|
86
|
+
#: Path to the workflow.
|
87
|
+
workflow_path: str
|
88
|
+
#: Time of submission.
|
89
|
+
submit_time: str
|
90
|
+
#: Parsed time of submission.
|
91
|
+
submit_time_obj: NotRequired[datetime | None]
|
92
|
+
#: Time of start.
|
93
|
+
start_time: str
|
94
|
+
#: Parsed time of start.
|
95
|
+
start_time_obj: datetime | None
|
96
|
+
#: Time of finish.
|
97
|
+
end_time: str
|
98
|
+
#: Parsed time of finish.
|
99
|
+
end_time_obj: datetime | None
|
100
|
+
#: Submission index.
|
101
|
+
sub_idx: int
|
102
|
+
#: Jobscripts in submission.
|
103
|
+
jobscripts: list[int]
|
104
|
+
#: Active jobscript state.
|
105
|
+
active_jobscripts: Mapping[int, Mapping[int, JobscriptElementState]]
|
106
|
+
#: Whether this is deleted.
|
107
|
+
deleted: bool
|
108
|
+
#: Whether this is unloadable.
|
109
|
+
unloadable: bool
|
110
|
+
#: Expanded submission object.
|
111
|
+
submission: NotRequired[Submission]
|
112
|
+
|
113
|
+
|
114
|
+
class TemplateComponents(TypedDict):
|
115
|
+
"""
|
116
|
+
Components loaded from templates.
|
117
|
+
"""
|
118
|
+
|
119
|
+
#: Parameters loaded from templates.
|
120
|
+
parameters: NotRequired[ParametersList]
|
121
|
+
#: Command files loaded from templates.
|
122
|
+
command_files: NotRequired[CommandFilesList]
|
123
|
+
#: Execution environments loaded from templates.
|
124
|
+
environments: NotRequired[EnvironmentsList]
|
125
|
+
#: Task schemas loaded from templates.
|
126
|
+
task_schemas: NotRequired[TaskSchemasList]
|
127
|
+
#: Scripts discovered by templates.
|
128
|
+
scripts: NotRequired[dict[str, Path]]
|
129
|
+
|
130
|
+
|
131
|
+
#: Simplification of :class:`TemplateComponents` to allow some types of
|
132
|
+
#: internal manipulations.
|
133
|
+
BasicTemplateComponents: TypeAlias = "dict[str, list[dict]]"
|
134
|
+
|
135
|
+
|
136
|
+
DataIndex: TypeAlias = "dict[str, int | list[int]]"
|
137
|
+
"""
|
138
|
+
The type of indices to data. These are *normally* dictionaries of integers,
|
139
|
+
but can have leaves being lists of integers when dealing with element groups
|
140
|
+
(i.e., when a downstream element uses outputs from multiple upstream elements,
|
141
|
+
rather than just a single upstream element).
|
142
|
+
"""
|
143
|
+
|
144
|
+
|
145
|
+
_T = TypeVar("_T")
|
146
|
+
|
147
|
+
_CLASS_VAR_RE: Final = re.compile(r"ClassVar\[(.*)\]")
|
148
|
+
_INIT_VAR_RE: Final = re.compile(r"InitVar\[(.*)\]")
|
149
|
+
|
150
|
+
|
151
|
+
def hydrate(cls: type[_T]) -> type[_T]:
|
152
|
+
"""
|
153
|
+
Partially hydrates the annotations on fields in a class, so that a @dataclass
|
154
|
+
annotation can recognise that ClassVar-annotated fields are class variables.
|
155
|
+
"""
|
156
|
+
anns = {}
|
157
|
+
for f, a in cls.__annotations__.items():
|
158
|
+
if isinstance(a, str):
|
159
|
+
m = _CLASS_VAR_RE.match(a)
|
160
|
+
if m:
|
161
|
+
anns[f] = ClassVar[m[1]]
|
162
|
+
continue
|
163
|
+
m = _INIT_VAR_RE.match(a)
|
164
|
+
if m:
|
165
|
+
anns[f] = InitVar(cast(type, m[1]))
|
166
|
+
continue
|
167
|
+
anns[f] = a
|
168
|
+
cls.__annotations__ = anns
|
169
|
+
return cls
|
hpcflow/tests/conftest.py
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
from pathlib import Path
|
1
3
|
import pytest
|
2
4
|
from hpcflow.app import app as hf
|
3
5
|
|
4
6
|
|
5
|
-
def pytest_addoption(parser):
|
7
|
+
def pytest_addoption(parser: pytest.Parser):
|
6
8
|
parser.addoption(
|
7
9
|
"--slurm",
|
8
10
|
action="store_true",
|
@@ -29,7 +31,7 @@ def pytest_addoption(parser):
|
|
29
31
|
)
|
30
32
|
|
31
33
|
|
32
|
-
def pytest_configure(config):
|
34
|
+
def pytest_configure(config: pytest.Config):
|
33
35
|
config.addinivalue_line("markers", "slurm: mark test as slurm to run")
|
34
36
|
config.addinivalue_line("markers", "wsl: mark test as wsl to run")
|
35
37
|
config.addinivalue_line(
|
@@ -42,7 +44,7 @@ def pytest_configure(config):
|
|
42
44
|
hf.run_time_info.in_pytest = True
|
43
45
|
|
44
46
|
|
45
|
-
def pytest_collection_modifyitems(config, items):
|
47
|
+
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]):
|
46
48
|
if config.getoption("--slurm"):
|
47
49
|
# --slurm given in cli: only run slurm tests
|
48
50
|
for item in items:
|
@@ -84,19 +86,19 @@ def pytest_collection_modifyitems(config, items):
|
|
84
86
|
)
|
85
87
|
|
86
88
|
|
87
|
-
def pytest_unconfigure(config):
|
89
|
+
def pytest_unconfigure(config: pytest.Config):
|
88
90
|
hf.run_time_info.in_pytest = False
|
89
91
|
|
90
92
|
|
91
93
|
@pytest.fixture
|
92
|
-
def null_config(tmp_path):
|
94
|
+
def null_config(tmp_path: Path):
|
93
95
|
if not hf.is_config_loaded:
|
94
96
|
hf.load_config(config_dir=tmp_path)
|
95
97
|
hf.run_time_info.in_pytest = True
|
96
98
|
|
97
99
|
|
98
100
|
@pytest.fixture
|
99
|
-
def new_null_config(tmp_path):
|
101
|
+
def new_null_config(tmp_path: Path):
|
100
102
|
hf.load_config(config_dir=tmp_path, warn=False)
|
101
103
|
hf.load_template_components(warn=False)
|
102
104
|
hf.run_time_info.in_pytest = True
|
@@ -1,11 +1,14 @@
|
|
1
|
+
from pathlib import Path
|
1
2
|
import pytest
|
2
3
|
from hpcflow.app import app as hf
|
3
4
|
from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
|
4
5
|
|
5
6
|
|
6
7
|
@pytest.mark.slurm
|
7
|
-
def test_workflow_1(tmp_path, null_config):
|
8
|
+
def test_workflow_1(tmp_path: Path, null_config):
|
8
9
|
hf.config.add_scheduler("slurm")
|
9
10
|
wk = make_test_data_YAML_workflow("workflow_1_slurm.yaml", path=tmp_path)
|
10
11
|
wk.submit(wait=True, add_to_known=False)
|
11
|
-
|
12
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
13
|
+
assert isinstance(p2, hf.ElementParameter)
|
14
|
+
assert p2.value == "201"
|
@@ -1,3 +1,4 @@
|
|
1
|
+
from pathlib import Path
|
1
2
|
import time
|
2
3
|
import pytest
|
3
4
|
|
@@ -10,7 +11,7 @@ from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
|
|
10
11
|
|
11
12
|
@pytest.mark.integration
|
12
13
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
13
|
-
def test_script_direct_in_direct_out(null_config, tmp_path):
|
14
|
+
def test_script_direct_in_direct_out(null_config, tmp_path: Path):
|
14
15
|
s1 = hf.TaskSchema(
|
15
16
|
objective="t1",
|
16
17
|
inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
|
@@ -34,12 +35,14 @@ def test_script_direct_in_direct_out(null_config, tmp_path):
|
|
34
35
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
35
36
|
# to be later Python versions):
|
36
37
|
time.sleep(10)
|
37
|
-
|
38
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
39
|
+
assert isinstance(p2, hf.ElementParameter)
|
40
|
+
assert p2.value == p1_val + 100
|
38
41
|
|
39
42
|
|
40
43
|
@pytest.mark.integration
|
41
44
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
42
|
-
def test_script_direct_sub_param_in_direct_out(null_config, tmp_path):
|
45
|
+
def test_script_direct_sub_param_in_direct_out(null_config, tmp_path: Path):
|
43
46
|
s1 = hf.TaskSchema(
|
44
47
|
objective="t1",
|
45
48
|
inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
|
@@ -63,12 +66,14 @@ def test_script_direct_sub_param_in_direct_out(null_config, tmp_path):
|
|
63
66
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
64
67
|
# to be later Python versions):
|
65
68
|
time.sleep(10)
|
66
|
-
|
69
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
70
|
+
assert isinstance(p2, hf.ElementParameter)
|
71
|
+
assert p2.value == p1_val["a"] + 100
|
67
72
|
|
68
73
|
|
69
74
|
@pytest.mark.integration
|
70
75
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
71
|
-
def test_script_direct_in_direct_out_single_label(null_config, tmp_path):
|
76
|
+
def test_script_direct_in_direct_out_single_label(null_config, tmp_path: Path):
|
72
77
|
"""This uses the same test script as the `test_script_direct_in_direct_out` test;
|
73
78
|
single labels are trivial and need not be referenced in the script."""
|
74
79
|
p1_label = "one"
|
@@ -95,12 +100,14 @@ def test_script_direct_in_direct_out_single_label(null_config, tmp_path):
|
|
95
100
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
96
101
|
# to be later Python versions):
|
97
102
|
time.sleep(10)
|
98
|
-
|
103
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
104
|
+
assert isinstance(p2, hf.ElementParameter)
|
105
|
+
assert p2.value == p1_val + 100
|
99
106
|
|
100
107
|
|
101
108
|
@pytest.mark.integration
|
102
109
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
103
|
-
def test_script_direct_in_direct_out_labels(null_config, tmp_path):
|
110
|
+
def test_script_direct_in_direct_out_labels(null_config, tmp_path: Path):
|
104
111
|
p1_label_1 = "one"
|
105
112
|
p1_label_2 = "two"
|
106
113
|
s1 = hf.TaskSchema(
|
@@ -139,12 +146,14 @@ def test_script_direct_in_direct_out_labels(null_config, tmp_path):
|
|
139
146
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
140
147
|
# to be later Python versions):
|
141
148
|
time.sleep(10)
|
142
|
-
|
149
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
150
|
+
assert isinstance(p2, hf.ElementParameter)
|
151
|
+
assert p2.value == p1_1_val + p1_2_val
|
143
152
|
|
144
153
|
|
145
154
|
@pytest.mark.integration
|
146
155
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
147
|
-
def test_script_json_in_json_out(null_config, tmp_path):
|
156
|
+
def test_script_json_in_json_out(null_config, tmp_path: Path):
|
148
157
|
s1 = hf.TaskSchema(
|
149
158
|
objective="t1",
|
150
159
|
inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
|
@@ -168,12 +177,14 @@ def test_script_json_in_json_out(null_config, tmp_path):
|
|
168
177
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
169
178
|
# to be later Python versions):
|
170
179
|
time.sleep(10)
|
171
|
-
|
180
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
181
|
+
assert isinstance(p2, hf.ElementParameter)
|
182
|
+
assert p2.value == p1_val + 100
|
172
183
|
|
173
184
|
|
174
185
|
@pytest.mark.integration
|
175
186
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
176
|
-
def test_script_json_in_json_out_labels(null_config, tmp_path):
|
187
|
+
def test_script_json_in_json_out_labels(null_config, tmp_path: Path):
|
177
188
|
p1_label_1 = "one"
|
178
189
|
p1_label_2 = "two"
|
179
190
|
s1 = hf.TaskSchema(
|
@@ -212,12 +223,14 @@ def test_script_json_in_json_out_labels(null_config, tmp_path):
|
|
212
223
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
213
224
|
# to be later Python versions):
|
214
225
|
time.sleep(10)
|
215
|
-
|
226
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
227
|
+
assert isinstance(p2, hf.ElementParameter)
|
228
|
+
assert p2.value == p1_1_val + p1_2_val
|
216
229
|
|
217
230
|
|
218
231
|
@pytest.mark.integration
|
219
232
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
220
|
-
def test_script_json_sub_param_in_json_out_labels(null_config, tmp_path):
|
233
|
+
def test_script_json_sub_param_in_json_out_labels(null_config, tmp_path: Path):
|
221
234
|
p1_label_1 = "one"
|
222
235
|
p1_label_2 = "two"
|
223
236
|
s1 = hf.TaskSchema(
|
@@ -256,12 +269,14 @@ def test_script_json_sub_param_in_json_out_labels(null_config, tmp_path):
|
|
256
269
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
257
270
|
# to be later Python versions):
|
258
271
|
time.sleep(10)
|
259
|
-
|
272
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
273
|
+
assert isinstance(p2, hf.ElementParameter)
|
274
|
+
assert p2.value == a_val + p1_2_val
|
260
275
|
|
261
276
|
|
262
277
|
@pytest.mark.integration
|
263
278
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
264
|
-
def test_script_json_and_direct_in_json_out(null_config, tmp_path):
|
279
|
+
def test_script_json_and_direct_in_json_out(null_config, tmp_path: Path):
|
265
280
|
s1 = hf.TaskSchema(
|
266
281
|
objective="t1",
|
267
282
|
inputs=[
|
@@ -289,12 +304,14 @@ def test_script_json_and_direct_in_json_out(null_config, tmp_path):
|
|
289
304
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
290
305
|
# to be later Python versions):
|
291
306
|
time.sleep(10)
|
292
|
-
|
307
|
+
p3 = wk.tasks[0].elements[0].outputs.p3
|
308
|
+
assert isinstance(p3, hf.ElementParameter)
|
309
|
+
assert p3.value == p1_val + p2_val
|
293
310
|
|
294
311
|
|
295
312
|
@pytest.mark.integration
|
296
313
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
297
|
-
def test_script_json_in_json_and_direct_out(null_config, tmp_path):
|
314
|
+
def test_script_json_in_json_and_direct_out(null_config, tmp_path: Path):
|
298
315
|
s1 = hf.TaskSchema(
|
299
316
|
objective="t1",
|
300
317
|
inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
|
@@ -321,13 +338,17 @@ def test_script_json_in_json_and_direct_out(null_config, tmp_path):
|
|
321
338
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
322
339
|
# to be later Python versions):
|
323
340
|
time.sleep(10)
|
324
|
-
|
325
|
-
assert
|
341
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
342
|
+
assert isinstance(p2, hf.ElementParameter)
|
343
|
+
p3 = wk.tasks[0].elements[0].outputs.p3
|
344
|
+
assert isinstance(p3, hf.ElementParameter)
|
345
|
+
assert p2.value == p1_val + 100
|
346
|
+
assert p3.value == p1_val + 200
|
326
347
|
|
327
348
|
|
328
349
|
@pytest.mark.integration
|
329
350
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
330
|
-
def test_script_json_in_obj(null_config, tmp_path):
|
351
|
+
def test_script_json_in_obj(null_config, tmp_path: Path):
|
331
352
|
"""Use a custom JSON dumper defined in the P1 class."""
|
332
353
|
s1 = hf.TaskSchema(
|
333
354
|
objective="t1",
|
@@ -353,12 +374,14 @@ def test_script_json_in_obj(null_config, tmp_path):
|
|
353
374
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
354
375
|
# to be later Python versions):
|
355
376
|
time.sleep(10)
|
356
|
-
|
377
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
378
|
+
assert isinstance(p2, hf.ElementParameter)
|
379
|
+
assert p2.value == a_val + 100
|
357
380
|
|
358
381
|
|
359
382
|
@pytest.mark.integration
|
360
383
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
361
|
-
def test_script_hdf5_in_obj(null_config, tmp_path):
|
384
|
+
def test_script_hdf5_in_obj(null_config, tmp_path: Path):
|
362
385
|
"""Use a custom HDF5 dumper defined in the P1 class."""
|
363
386
|
s1 = hf.TaskSchema(
|
364
387
|
objective="t1",
|
@@ -384,12 +407,14 @@ def test_script_hdf5_in_obj(null_config, tmp_path):
|
|
384
407
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
385
408
|
# to be later Python versions):
|
386
409
|
time.sleep(10)
|
387
|
-
|
410
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
411
|
+
assert isinstance(p2, hf.ElementParameter)
|
412
|
+
assert p2.value == a_val + 100
|
388
413
|
|
389
414
|
|
390
415
|
@pytest.mark.integration
|
391
416
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
392
|
-
def test_script_json_out_obj(null_config, tmp_path):
|
417
|
+
def test_script_json_out_obj(null_config, tmp_path: Path):
|
393
418
|
"""Use a custom JSON saver defined in the P1 class."""
|
394
419
|
s1 = hf.TaskSchema(
|
395
420
|
objective="t1",
|
@@ -415,12 +440,14 @@ def test_script_json_out_obj(null_config, tmp_path):
|
|
415
440
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
416
441
|
# to be later Python versions):
|
417
442
|
time.sleep(10)
|
418
|
-
|
443
|
+
p1c = wk.tasks[0].elements[0].outputs.p1c
|
444
|
+
assert isinstance(p1c, hf.ElementParameter)
|
445
|
+
assert p1c.value == P1(a=p1_val + 100)
|
419
446
|
|
420
447
|
|
421
448
|
@pytest.mark.integration
|
422
449
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
423
|
-
def test_script_hdf5_out_obj(null_config, tmp_path):
|
450
|
+
def test_script_hdf5_out_obj(null_config, tmp_path: Path):
|
424
451
|
"""Use a custom HDF5 saver defined in the P1 class."""
|
425
452
|
s1 = hf.TaskSchema(
|
426
453
|
objective="t1",
|
@@ -446,13 +473,14 @@ def test_script_hdf5_out_obj(null_config, tmp_path):
|
|
446
473
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
447
474
|
# to be later Python versions):
|
448
475
|
time.sleep(10)
|
449
|
-
|
476
|
+
p1c = wk.tasks[0].elements[0].outputs.p1c
|
477
|
+
assert isinstance(p1c, hf.ElementParameter)
|
478
|
+
assert p1c.value == P1(a=p1_val + 100)
|
450
479
|
|
451
480
|
|
452
481
|
@pytest.mark.integration
|
453
482
|
@pytest.mark.skipif("hf.run_time_info.is_frozen")
|
454
|
-
def test_script_direct_in_pass_env_spec(new_null_config, tmp_path):
|
455
|
-
|
483
|
+
def test_script_direct_in_pass_env_spec(new_null_config, tmp_path: Path):
|
456
484
|
vers_spec = {"version": "1.2"}
|
457
485
|
env = hf.Environment(
|
458
486
|
name="python_env_with_specifiers",
|
@@ -503,7 +531,9 @@ def test_script_direct_in_pass_env_spec(new_null_config, tmp_path):
|
|
503
531
|
# TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
|
504
532
|
# to be later Python versions):
|
505
533
|
time.sleep(10)
|
506
|
-
|
534
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
535
|
+
assert isinstance(p2, hf.ElementParameter)
|
536
|
+
assert p2.value == {
|
507
537
|
"name": "python_env_with_specifiers",
|
508
538
|
**vers_spec,
|
509
539
|
}
|
@@ -1,14 +1,16 @@
|
|
1
1
|
import time
|
2
|
-
|
2
|
+
from pathlib import Path
|
3
3
|
import pytest
|
4
|
-
|
4
|
+
from hpcflow.app import app as hf
|
5
5
|
from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
|
6
6
|
|
7
7
|
|
8
8
|
@pytest.mark.wsl
|
9
|
-
def test_workflow_1(tmp_path, null_config):
|
9
|
+
def test_workflow_1(tmp_path: Path, null_config):
|
10
10
|
wk = make_test_data_YAML_workflow("workflow_1_wsl.yaml", path=tmp_path)
|
11
11
|
wk.submit(wait=True, add_to_known=False)
|
12
12
|
time.sleep(20) # TODO: bug! for some reason the new parameter isn't actually written
|
13
13
|
# to disk when using WSL until several seconds after the workflow has finished!
|
14
|
-
|
14
|
+
p2 = wk.tasks[0].elements[0].outputs.p2
|
15
|
+
assert isinstance(p2, hf.ElementParameter)
|
16
|
+
assert p2.value == "201"
|