hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a190__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +8 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  5. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  6. hpcflow/sdk/__init__.py +21 -15
  7. hpcflow/sdk/app.py +2133 -770
  8. hpcflow/sdk/cli.py +281 -250
  9. hpcflow/sdk/cli_common.py +6 -2
  10. hpcflow/sdk/config/__init__.py +1 -1
  11. hpcflow/sdk/config/callbacks.py +77 -42
  12. hpcflow/sdk/config/cli.py +126 -103
  13. hpcflow/sdk/config/config.py +578 -311
  14. hpcflow/sdk/config/config_file.py +131 -95
  15. hpcflow/sdk/config/errors.py +112 -85
  16. hpcflow/sdk/config/types.py +145 -0
  17. hpcflow/sdk/core/actions.py +1054 -994
  18. hpcflow/sdk/core/app_aware.py +24 -0
  19. hpcflow/sdk/core/cache.py +81 -63
  20. hpcflow/sdk/core/command_files.py +275 -185
  21. hpcflow/sdk/core/commands.py +111 -107
  22. hpcflow/sdk/core/element.py +724 -503
  23. hpcflow/sdk/core/enums.py +192 -0
  24. hpcflow/sdk/core/environment.py +74 -93
  25. hpcflow/sdk/core/errors.py +398 -51
  26. hpcflow/sdk/core/json_like.py +540 -272
  27. hpcflow/sdk/core/loop.py +380 -334
  28. hpcflow/sdk/core/loop_cache.py +160 -43
  29. hpcflow/sdk/core/object_list.py +370 -207
  30. hpcflow/sdk/core/parameters.py +728 -600
  31. hpcflow/sdk/core/rule.py +59 -41
  32. hpcflow/sdk/core/run_dir_files.py +33 -22
  33. hpcflow/sdk/core/task.py +1546 -1325
  34. hpcflow/sdk/core/task_schema.py +240 -196
  35. hpcflow/sdk/core/test_utils.py +126 -88
  36. hpcflow/sdk/core/types.py +387 -0
  37. hpcflow/sdk/core/utils.py +410 -305
  38. hpcflow/sdk/core/validation.py +82 -9
  39. hpcflow/sdk/core/workflow.py +1192 -1028
  40. hpcflow/sdk/core/zarr_io.py +98 -137
  41. hpcflow/sdk/demo/cli.py +46 -33
  42. hpcflow/sdk/helper/cli.py +18 -16
  43. hpcflow/sdk/helper/helper.py +75 -63
  44. hpcflow/sdk/helper/watcher.py +61 -28
  45. hpcflow/sdk/log.py +83 -59
  46. hpcflow/sdk/persistence/__init__.py +8 -31
  47. hpcflow/sdk/persistence/base.py +988 -586
  48. hpcflow/sdk/persistence/defaults.py +6 -0
  49. hpcflow/sdk/persistence/discovery.py +38 -0
  50. hpcflow/sdk/persistence/json.py +408 -153
  51. hpcflow/sdk/persistence/pending.py +158 -123
  52. hpcflow/sdk/persistence/store_resource.py +37 -22
  53. hpcflow/sdk/persistence/types.py +307 -0
  54. hpcflow/sdk/persistence/utils.py +14 -11
  55. hpcflow/sdk/persistence/zarr.py +477 -420
  56. hpcflow/sdk/runtime.py +44 -41
  57. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  58. hpcflow/sdk/submission/jobscript.py +444 -404
  59. hpcflow/sdk/submission/schedulers/__init__.py +133 -40
  60. hpcflow/sdk/submission/schedulers/direct.py +97 -71
  61. hpcflow/sdk/submission/schedulers/sge.py +132 -126
  62. hpcflow/sdk/submission/schedulers/slurm.py +263 -268
  63. hpcflow/sdk/submission/schedulers/utils.py +7 -2
  64. hpcflow/sdk/submission/shells/__init__.py +14 -15
  65. hpcflow/sdk/submission/shells/base.py +102 -29
  66. hpcflow/sdk/submission/shells/bash.py +72 -55
  67. hpcflow/sdk/submission/shells/os_version.py +31 -30
  68. hpcflow/sdk/submission/shells/powershell.py +37 -29
  69. hpcflow/sdk/submission/submission.py +203 -257
  70. hpcflow/sdk/submission/types.py +143 -0
  71. hpcflow/sdk/typing.py +163 -12
  72. hpcflow/tests/conftest.py +8 -6
  73. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  74. hpcflow/tests/scripts/test_main_scripts.py +60 -30
  75. hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -4
  76. hpcflow/tests/unit/test_action.py +86 -75
  77. hpcflow/tests/unit/test_action_rule.py +9 -4
  78. hpcflow/tests/unit/test_app.py +13 -6
  79. hpcflow/tests/unit/test_cli.py +1 -1
  80. hpcflow/tests/unit/test_command.py +71 -54
  81. hpcflow/tests/unit/test_config.py +20 -15
  82. hpcflow/tests/unit/test_config_file.py +21 -18
  83. hpcflow/tests/unit/test_element.py +58 -62
  84. hpcflow/tests/unit/test_element_iteration.py +3 -1
  85. hpcflow/tests/unit/test_element_set.py +29 -19
  86. hpcflow/tests/unit/test_group.py +4 -2
  87. hpcflow/tests/unit/test_input_source.py +116 -93
  88. hpcflow/tests/unit/test_input_value.py +29 -24
  89. hpcflow/tests/unit/test_json_like.py +44 -35
  90. hpcflow/tests/unit/test_loop.py +65 -58
  91. hpcflow/tests/unit/test_object_list.py +17 -12
  92. hpcflow/tests/unit/test_parameter.py +16 -7
  93. hpcflow/tests/unit/test_persistence.py +48 -35
  94. hpcflow/tests/unit/test_resources.py +20 -18
  95. hpcflow/tests/unit/test_run.py +8 -3
  96. hpcflow/tests/unit/test_runtime.py +2 -1
  97. hpcflow/tests/unit/test_schema_input.py +23 -15
  98. hpcflow/tests/unit/test_shell.py +3 -2
  99. hpcflow/tests/unit/test_slurm.py +8 -7
  100. hpcflow/tests/unit/test_submission.py +39 -19
  101. hpcflow/tests/unit/test_task.py +352 -247
  102. hpcflow/tests/unit/test_task_schema.py +33 -20
  103. hpcflow/tests/unit/test_utils.py +9 -11
  104. hpcflow/tests/unit/test_value_sequence.py +15 -12
  105. hpcflow/tests/unit/test_workflow.py +114 -83
  106. hpcflow/tests/unit/test_workflow_template.py +0 -1
  107. hpcflow/tests/workflows/test_jobscript.py +2 -1
  108. hpcflow/tests/workflows/test_workflows.py +18 -13
  109. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/METADATA +2 -1
  110. hpcflow_new2-0.2.0a190.dist-info/RECORD +165 -0
  111. hpcflow/sdk/core/parallel.py +0 -21
  112. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  113. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/LICENSE +0 -0
  114. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/WHEEL +0 -0
  115. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a190.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,143 @@
1
+ """
2
+ Types for the submission subsystem.
3
+ """
4
+ from __future__ import annotations
5
+ from typing import Any, TYPE_CHECKING
6
+ from typing_extensions import NotRequired, TypeAlias, TypedDict
7
+
8
+ if TYPE_CHECKING:
9
+ from collections.abc import Sequence
10
+ from datetime import datetime
11
+ from numpy.typing import NDArray
12
+ from ..core.element import ElementResources
13
+
14
+
15
+ class JobScriptDescriptor(TypedDict):
16
+ """
17
+ Descriptor for a jobscript.
18
+ """
19
+
20
+ #: Resources required by the jobscript.
21
+ resources: Any
22
+ #: Elements handled by the jobscript.
23
+ elements: dict[int, list[int]]
24
+ #: Dependencies of the jobscript.
25
+ dependencies: NotRequired[dict[int, ResolvedDependencies]]
26
+ #: Hash of resources.
27
+ resource_hash: NotRequired[str]
28
+
29
+
30
+ class ResolvedDependencies(TypedDict):
31
+ """
32
+ The resolution of a dependency.
33
+ """
34
+
35
+ #: Mapping of jobscript elements.
36
+ js_element_mapping: dict[int, list[int]]
37
+ #: Whether this is an array mapping.
38
+ is_array: NotRequired[bool]
39
+
40
+
41
+ class JobScriptCreationArguments(TypedDict):
42
+ """
43
+ Arguments to pass to create a :class:`Jobscript`.
44
+ """
45
+
46
+ #: The task insertion IDs.
47
+ task_insert_IDs: list[int]
48
+ #: The actions of the tasks.
49
+ task_actions: list[tuple[int, int, int]]
50
+ #: The elements of the tasks.
51
+ task_elements: dict[int, list[int]]
52
+ #: Element action run information.
53
+ EAR_ID: NDArray
54
+ #: Resources to use.
55
+ resources: ElementResources
56
+ #: Description of what loops are in play.
57
+ task_loop_idx: list[dict[str, int]]
58
+ #: Description of dependencies.
59
+ dependencies: dict[int, ResolvedDependencies]
60
+ #: When the jobscript was submitted, if known.
61
+ submit_time: NotRequired[datetime]
62
+ #: Where the jobscript was submitted, if known.
63
+ submit_hostname: NotRequired[str]
64
+ #: Description of what the jobscript was submitted to, if known.
65
+ submit_machine: NotRequired[str]
66
+ #: The command line used to do the commit, if known.
67
+ submit_cmdline: NotRequired[list[str]]
68
+ #: The job ID from the scheduler, if known.
69
+ scheduler_job_ID: NotRequired[str]
70
+ #: The process ID of the subprocess, if known.
71
+ process_ID: NotRequired[int]
72
+ #: Version info about the target system.
73
+ version_info: NotRequired[dict[str, str | list[str]]]
74
+ #: The name of the OS.
75
+ os_name: NotRequired[str]
76
+ #: The name of the shell.
77
+ shell_name: NotRequired[str]
78
+ #: The scheduler used.
79
+ scheduler_name: NotRequired[str]
80
+ #: Whether the jobscript is currently running.
81
+ running: NotRequired[bool]
82
+ #: Do not supply!
83
+ resource_hash: NotRequired[str]
84
+ #: Do not supply!
85
+ elements: NotRequired[dict[int, list[int]]]
86
+
87
+
88
+ class SchedulerRef(TypedDict):
89
+ """
90
+ Scheduler reference descriptor.
91
+ """
92
+
93
+ #: Jobscript references.
94
+ js_refs: list # Internal type is horrible and variable
95
+ #: Number of jobscript elements.
96
+ num_js_elements: int
97
+
98
+
99
+ class SubmissionPart(TypedDict):
100
+ """
101
+ A part of a submission.
102
+ """
103
+
104
+ #: Timestamp for when this part was submitted.
105
+ submit_time: datetime
106
+ #: The jobscripts involved in this submission.
107
+ jobscripts: list[int]
108
+
109
+
110
+ # This needs PEP 728 for a better type, alas
111
+ #: Version data.
112
+ VersionInfo: TypeAlias = "dict[str, str | list[str]]"
113
+
114
+
115
+ # TODO: This really doesn't belong here?!
116
+ class JobscriptHeaderArgs(TypedDict):
117
+ """
118
+ Keyword arguments to use when creating a job script from a
119
+ :class:`Jobscript`.
120
+ """
121
+
122
+ #: Application invocation. (Arguments, etc.)
123
+ app_invoc: str | Sequence[str]
124
+ #: Configuration directory.
125
+ config_dir: NotRequired[str]
126
+ #: Configuration key.
127
+ config_invoc_key: NotRequired[Any]
128
+ #: Name of EAR file.
129
+ EAR_file_name: NotRequired[str]
130
+ #: Name of file containing run directories.
131
+ element_run_dirs_file_path: NotRequired[str]
132
+ #: Environment setup.
133
+ env_setup: NotRequired[str]
134
+ #: Jobscript index.
135
+ js_idx: NotRequired[int]
136
+ #: Log file for the run.
137
+ run_log_file: NotRequired[str]
138
+ #: Submission index.
139
+ sub_idx: NotRequired[int]
140
+ #: Workflow application alias.
141
+ workflow_app_alias: NotRequired[str]
142
+ #: Path to workflow.
143
+ workflow_path: NotRequired[str]
hpcflow/sdk/typing.py CHANGED
@@ -1,18 +1,169 @@
1
1
  """
2
2
  Common type aliases.
3
3
  """
4
- from typing import Tuple, TypeVar
4
+ from __future__ import annotations
5
+ from dataclasses import InitVar
6
+ from typing import ClassVar, Final, TypeVar, cast, TYPE_CHECKING
7
+ from typing_extensions import NotRequired, TypeAlias, TypedDict
5
8
  from pathlib import Path
9
+ import re
10
+
11
+ if TYPE_CHECKING:
12
+ from collections.abc import Mapping
13
+ from datetime import datetime
14
+ from .core.object_list import (
15
+ CommandFilesList,
16
+ EnvironmentsList,
17
+ ParametersList,
18
+ TaskSchemasList,
19
+ )
20
+ from .submission.enums import JobscriptElementState
21
+ from .submission.submission import Submission
6
22
 
7
23
  #: Type of a value that can be treated as a path.
8
- PathLike = TypeVar("PathLike", str, Path, None) # TODO: maybe don't need TypeVar?
9
-
10
- #: Type of an element index:
11
- #: (task_insert_ID, element_idx)
12
- E_idx_type = Tuple[int, int]
13
- #: Type of an element iteration index:
14
- #: (task_insert_ID, element_idx, iteration_idx)
15
- EI_idx_type = Tuple[int, int, int]
16
- #: Type of an element action run index:
17
- #: (task_insert_ID, element_idx, iteration_idx, action_idx, run_idx)
18
- EAR_idx_type = Tuple[int, int, int, int, int]
24
+ PathLike: TypeAlias = "str | Path | None"
25
+
26
+
27
+ class ParamSource(TypedDict):
28
+ """
29
+ A parameter source descriptor.
30
+ """
31
+
32
+ #: Parameter type name.
33
+ type: NotRequired[str]
34
+ #: EAR ID.
35
+ EAR_ID: NotRequired[int]
36
+ #: Task insertion ID.
37
+ task_insert_ID: NotRequired[int]
38
+ #: Action index.
39
+ action_idx: NotRequired[int]
40
+ #: Element index.
41
+ element_idx: NotRequired[int]
42
+ #: Element set index.
43
+ element_set_idx: NotRequired[int]
44
+ #: Element action run index.
45
+ run_idx: NotRequired[int]
46
+ #: Sequence index.
47
+ sequence_idx: NotRequired[int]
48
+ #: Task index.
49
+ task_idx: NotRequired[int]
50
+ #: Name of method used to create the parameter's value(s).
51
+ value_class_method: NotRequired[str]
52
+
53
+
54
+ class KnownSubmission(TypedDict):
55
+ """
56
+ Describes a known submission.
57
+ """
58
+
59
+ #: Local ID.
60
+ local_id: int
61
+ #: Workflow global ID.
62
+ workflow_id: str
63
+ #: Whether the submission is active.
64
+ is_active: bool
65
+ #: Submission index.
66
+ sub_idx: int
67
+ #: Submission time.
68
+ submit_time: str
69
+ #: Path to submission.
70
+ path: str
71
+ #: Start time.
72
+ start_time: str
73
+ #: Finish time.
74
+ end_time: str
75
+
76
+
77
+ class KnownSubmissionItem(TypedDict):
78
+ """
79
+ Describes a known submission.
80
+ """
81
+
82
+ #: Local ID.
83
+ local_id: int
84
+ #: Workflow global ID.
85
+ workflow_id: str
86
+ #: Path to the workflow.
87
+ workflow_path: str
88
+ #: Time of submission.
89
+ submit_time: str
90
+ #: Parsed time of submission.
91
+ submit_time_obj: NotRequired[datetime | None]
92
+ #: Time of start.
93
+ start_time: str
94
+ #: Parsed time of start.
95
+ start_time_obj: datetime | None
96
+ #: Time of finish.
97
+ end_time: str
98
+ #: Parsed time of finish.
99
+ end_time_obj: datetime | None
100
+ #: Submission index.
101
+ sub_idx: int
102
+ #: Jobscripts in submission.
103
+ jobscripts: list[int]
104
+ #: Active jobscript state.
105
+ active_jobscripts: Mapping[int, Mapping[int, JobscriptElementState]]
106
+ #: Whether this is deleted.
107
+ deleted: bool
108
+ #: Whether this is unloadable.
109
+ unloadable: bool
110
+ #: Expanded submission object.
111
+ submission: NotRequired[Submission]
112
+
113
+
114
+ class TemplateComponents(TypedDict):
115
+ """
116
+ Components loaded from templates.
117
+ """
118
+
119
+ #: Parameters loaded from templates.
120
+ parameters: NotRequired[ParametersList]
121
+ #: Command files loaded from templates.
122
+ command_files: NotRequired[CommandFilesList]
123
+ #: Execution environments loaded from templates.
124
+ environments: NotRequired[EnvironmentsList]
125
+ #: Task schemas loaded from templates.
126
+ task_schemas: NotRequired[TaskSchemasList]
127
+ #: Scripts discovered by templates.
128
+ scripts: NotRequired[dict[str, Path]]
129
+
130
+
131
+ #: Simplification of :class:`TemplateComponents` to allow some types of
132
+ #: internal manipulations.
133
+ BasicTemplateComponents: TypeAlias = "dict[str, list[dict]]"
134
+
135
+
136
+ DataIndex: TypeAlias = "dict[str, int | list[int]]"
137
+ """
138
+ The type of indices to data. These are *normally* dictionaries of integers,
139
+ but can have leaves being lists of integers when dealing with element groups
140
+ (i.e., when a downstream element uses outputs from multiple upstream elements,
141
+ rather than just a single upstream element).
142
+ """
143
+
144
+
145
+ _T = TypeVar("_T")
146
+
147
+ _CLASS_VAR_RE: Final = re.compile(r"ClassVar\[(.*)\]")
148
+ _INIT_VAR_RE: Final = re.compile(r"InitVar\[(.*)\]")
149
+
150
+
151
+ def hydrate(cls: type[_T]) -> type[_T]:
152
+ """
153
+ Partially hydrates the annotations on fields in a class, so that a @dataclass
154
+ annotation can recognise that ClassVar-annotated fields are class variables.
155
+ """
156
+ anns = {}
157
+ for f, a in cls.__annotations__.items():
158
+ if isinstance(a, str):
159
+ m = _CLASS_VAR_RE.match(a)
160
+ if m:
161
+ anns[f] = ClassVar[m[1]]
162
+ continue
163
+ m = _INIT_VAR_RE.match(a)
164
+ if m:
165
+ anns[f] = InitVar(cast(type, m[1]))
166
+ continue
167
+ anns[f] = a
168
+ cls.__annotations__ = anns
169
+ return cls
hpcflow/tests/conftest.py CHANGED
@@ -1,8 +1,10 @@
1
+ from __future__ import annotations
2
+ from pathlib import Path
1
3
  import pytest
2
4
  from hpcflow.app import app as hf
3
5
 
4
6
 
5
- def pytest_addoption(parser):
7
+ def pytest_addoption(parser: pytest.Parser):
6
8
  parser.addoption(
7
9
  "--slurm",
8
10
  action="store_true",
@@ -29,7 +31,7 @@ def pytest_addoption(parser):
29
31
  )
30
32
 
31
33
 
32
- def pytest_configure(config):
34
+ def pytest_configure(config: pytest.Config):
33
35
  config.addinivalue_line("markers", "slurm: mark test as slurm to run")
34
36
  config.addinivalue_line("markers", "wsl: mark test as wsl to run")
35
37
  config.addinivalue_line(
@@ -42,7 +44,7 @@ def pytest_configure(config):
42
44
  hf.run_time_info.in_pytest = True
43
45
 
44
46
 
45
- def pytest_collection_modifyitems(config, items):
47
+ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]):
46
48
  if config.getoption("--slurm"):
47
49
  # --slurm given in cli: only run slurm tests
48
50
  for item in items:
@@ -84,19 +86,19 @@ def pytest_collection_modifyitems(config, items):
84
86
  )
85
87
 
86
88
 
87
- def pytest_unconfigure(config):
89
+ def pytest_unconfigure(config: pytest.Config):
88
90
  hf.run_time_info.in_pytest = False
89
91
 
90
92
 
91
93
  @pytest.fixture
92
- def null_config(tmp_path):
94
+ def null_config(tmp_path: Path):
93
95
  if not hf.is_config_loaded:
94
96
  hf.load_config(config_dir=tmp_path)
95
97
  hf.run_time_info.in_pytest = True
96
98
 
97
99
 
98
100
  @pytest.fixture
99
- def new_null_config(tmp_path):
101
+ def new_null_config(tmp_path: Path):
100
102
  hf.load_config(config_dir=tmp_path, warn=False)
101
103
  hf.load_template_components(warn=False)
102
104
  hf.run_time_info.in_pytest = True
@@ -1,11 +1,14 @@
1
+ from pathlib import Path
1
2
  import pytest
2
3
  from hpcflow.app import app as hf
3
4
  from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
4
5
 
5
6
 
6
7
  @pytest.mark.slurm
7
- def test_workflow_1(tmp_path, null_config):
8
+ def test_workflow_1(tmp_path: Path, null_config):
8
9
  hf.config.add_scheduler("slurm")
9
10
  wk = make_test_data_YAML_workflow("workflow_1_slurm.yaml", path=tmp_path)
10
11
  wk.submit(wait=True, add_to_known=False)
11
- assert wk.tasks[0].elements[0].outputs.p2.value == "201"
12
+ p2 = wk.tasks[0].elements[0].outputs.p2
13
+ assert isinstance(p2, hf.ElementParameter)
14
+ assert p2.value == "201"
@@ -1,3 +1,4 @@
1
+ from pathlib import Path
1
2
  import time
2
3
  import pytest
3
4
 
@@ -10,7 +11,7 @@ from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
10
11
 
11
12
  @pytest.mark.integration
12
13
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
13
- def test_script_direct_in_direct_out(null_config, tmp_path):
14
+ def test_script_direct_in_direct_out(null_config, tmp_path: Path):
14
15
  s1 = hf.TaskSchema(
15
16
  objective="t1",
16
17
  inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
@@ -34,12 +35,14 @@ def test_script_direct_in_direct_out(null_config, tmp_path):
34
35
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
35
36
  # to be later Python versions):
36
37
  time.sleep(10)
37
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_val + 100
38
+ p2 = wk.tasks[0].elements[0].outputs.p2
39
+ assert isinstance(p2, hf.ElementParameter)
40
+ assert p2.value == p1_val + 100
38
41
 
39
42
 
40
43
  @pytest.mark.integration
41
44
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
42
- def test_script_direct_sub_param_in_direct_out(null_config, tmp_path):
45
+ def test_script_direct_sub_param_in_direct_out(null_config, tmp_path: Path):
43
46
  s1 = hf.TaskSchema(
44
47
  objective="t1",
45
48
  inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
@@ -63,12 +66,14 @@ def test_script_direct_sub_param_in_direct_out(null_config, tmp_path):
63
66
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
64
67
  # to be later Python versions):
65
68
  time.sleep(10)
66
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_val["a"] + 100
69
+ p2 = wk.tasks[0].elements[0].outputs.p2
70
+ assert isinstance(p2, hf.ElementParameter)
71
+ assert p2.value == p1_val["a"] + 100
67
72
 
68
73
 
69
74
  @pytest.mark.integration
70
75
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
71
- def test_script_direct_in_direct_out_single_label(null_config, tmp_path):
76
+ def test_script_direct_in_direct_out_single_label(null_config, tmp_path: Path):
72
77
  """This uses the same test script as the `test_script_direct_in_direct_out` test;
73
78
  single labels are trivial and need not be referenced in the script."""
74
79
  p1_label = "one"
@@ -95,12 +100,14 @@ def test_script_direct_in_direct_out_single_label(null_config, tmp_path):
95
100
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
96
101
  # to be later Python versions):
97
102
  time.sleep(10)
98
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_val + 100
103
+ p2 = wk.tasks[0].elements[0].outputs.p2
104
+ assert isinstance(p2, hf.ElementParameter)
105
+ assert p2.value == p1_val + 100
99
106
 
100
107
 
101
108
  @pytest.mark.integration
102
109
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
103
- def test_script_direct_in_direct_out_labels(null_config, tmp_path):
110
+ def test_script_direct_in_direct_out_labels(null_config, tmp_path: Path):
104
111
  p1_label_1 = "one"
105
112
  p1_label_2 = "two"
106
113
  s1 = hf.TaskSchema(
@@ -139,12 +146,14 @@ def test_script_direct_in_direct_out_labels(null_config, tmp_path):
139
146
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
140
147
  # to be later Python versions):
141
148
  time.sleep(10)
142
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_1_val + p1_2_val
149
+ p2 = wk.tasks[0].elements[0].outputs.p2
150
+ assert isinstance(p2, hf.ElementParameter)
151
+ assert p2.value == p1_1_val + p1_2_val
143
152
 
144
153
 
145
154
  @pytest.mark.integration
146
155
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
147
- def test_script_json_in_json_out(null_config, tmp_path):
156
+ def test_script_json_in_json_out(null_config, tmp_path: Path):
148
157
  s1 = hf.TaskSchema(
149
158
  objective="t1",
150
159
  inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
@@ -168,12 +177,14 @@ def test_script_json_in_json_out(null_config, tmp_path):
168
177
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
169
178
  # to be later Python versions):
170
179
  time.sleep(10)
171
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_val + 100
180
+ p2 = wk.tasks[0].elements[0].outputs.p2
181
+ assert isinstance(p2, hf.ElementParameter)
182
+ assert p2.value == p1_val + 100
172
183
 
173
184
 
174
185
  @pytest.mark.integration
175
186
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
176
- def test_script_json_in_json_out_labels(null_config, tmp_path):
187
+ def test_script_json_in_json_out_labels(null_config, tmp_path: Path):
177
188
  p1_label_1 = "one"
178
189
  p1_label_2 = "two"
179
190
  s1 = hf.TaskSchema(
@@ -212,12 +223,14 @@ def test_script_json_in_json_out_labels(null_config, tmp_path):
212
223
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
213
224
  # to be later Python versions):
214
225
  time.sleep(10)
215
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_1_val + p1_2_val
226
+ p2 = wk.tasks[0].elements[0].outputs.p2
227
+ assert isinstance(p2, hf.ElementParameter)
228
+ assert p2.value == p1_1_val + p1_2_val
216
229
 
217
230
 
218
231
  @pytest.mark.integration
219
232
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
220
- def test_script_json_sub_param_in_json_out_labels(null_config, tmp_path):
233
+ def test_script_json_sub_param_in_json_out_labels(null_config, tmp_path: Path):
221
234
  p1_label_1 = "one"
222
235
  p1_label_2 = "two"
223
236
  s1 = hf.TaskSchema(
@@ -256,12 +269,14 @@ def test_script_json_sub_param_in_json_out_labels(null_config, tmp_path):
256
269
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
257
270
  # to be later Python versions):
258
271
  time.sleep(10)
259
- assert wk.tasks[0].elements[0].outputs.p2.value == a_val + p1_2_val
272
+ p2 = wk.tasks[0].elements[0].outputs.p2
273
+ assert isinstance(p2, hf.ElementParameter)
274
+ assert p2.value == a_val + p1_2_val
260
275
 
261
276
 
262
277
  @pytest.mark.integration
263
278
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
264
- def test_script_json_and_direct_in_json_out(null_config, tmp_path):
279
+ def test_script_json_and_direct_in_json_out(null_config, tmp_path: Path):
265
280
  s1 = hf.TaskSchema(
266
281
  objective="t1",
267
282
  inputs=[
@@ -289,12 +304,14 @@ def test_script_json_and_direct_in_json_out(null_config, tmp_path):
289
304
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
290
305
  # to be later Python versions):
291
306
  time.sleep(10)
292
- assert wk.tasks[0].elements[0].outputs.p3.value == p1_val + p2_val
307
+ p3 = wk.tasks[0].elements[0].outputs.p3
308
+ assert isinstance(p3, hf.ElementParameter)
309
+ assert p3.value == p1_val + p2_val
293
310
 
294
311
 
295
312
  @pytest.mark.integration
296
313
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
297
- def test_script_json_in_json_and_direct_out(null_config, tmp_path):
314
+ def test_script_json_in_json_and_direct_out(null_config, tmp_path: Path):
298
315
  s1 = hf.TaskSchema(
299
316
  objective="t1",
300
317
  inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
@@ -321,13 +338,17 @@ def test_script_json_in_json_and_direct_out(null_config, tmp_path):
321
338
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
322
339
  # to be later Python versions):
323
340
  time.sleep(10)
324
- assert wk.tasks[0].elements[0].outputs.p2.value == p1_val + 100
325
- assert wk.tasks[0].elements[0].outputs.p3.value == p1_val + 200
341
+ p2 = wk.tasks[0].elements[0].outputs.p2
342
+ assert isinstance(p2, hf.ElementParameter)
343
+ p3 = wk.tasks[0].elements[0].outputs.p3
344
+ assert isinstance(p3, hf.ElementParameter)
345
+ assert p2.value == p1_val + 100
346
+ assert p3.value == p1_val + 200
326
347
 
327
348
 
328
349
  @pytest.mark.integration
329
350
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
330
- def test_script_json_in_obj(null_config, tmp_path):
351
+ def test_script_json_in_obj(null_config, tmp_path: Path):
331
352
  """Use a custom JSON dumper defined in the P1 class."""
332
353
  s1 = hf.TaskSchema(
333
354
  objective="t1",
@@ -353,12 +374,14 @@ def test_script_json_in_obj(null_config, tmp_path):
353
374
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
354
375
  # to be later Python versions):
355
376
  time.sleep(10)
356
- assert wk.tasks[0].elements[0].outputs.p2.value == a_val + 100
377
+ p2 = wk.tasks[0].elements[0].outputs.p2
378
+ assert isinstance(p2, hf.ElementParameter)
379
+ assert p2.value == a_val + 100
357
380
 
358
381
 
359
382
  @pytest.mark.integration
360
383
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
361
- def test_script_hdf5_in_obj(null_config, tmp_path):
384
+ def test_script_hdf5_in_obj(null_config, tmp_path: Path):
362
385
  """Use a custom HDF5 dumper defined in the P1 class."""
363
386
  s1 = hf.TaskSchema(
364
387
  objective="t1",
@@ -384,12 +407,14 @@ def test_script_hdf5_in_obj(null_config, tmp_path):
384
407
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
385
408
  # to be later Python versions):
386
409
  time.sleep(10)
387
- assert wk.tasks[0].elements[0].outputs.p2.value == a_val + 100
410
+ p2 = wk.tasks[0].elements[0].outputs.p2
411
+ assert isinstance(p2, hf.ElementParameter)
412
+ assert p2.value == a_val + 100
388
413
 
389
414
 
390
415
  @pytest.mark.integration
391
416
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
392
- def test_script_json_out_obj(null_config, tmp_path):
417
+ def test_script_json_out_obj(null_config, tmp_path: Path):
393
418
  """Use a custom JSON saver defined in the P1 class."""
394
419
  s1 = hf.TaskSchema(
395
420
  objective="t1",
@@ -415,12 +440,14 @@ def test_script_json_out_obj(null_config, tmp_path):
415
440
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
416
441
  # to be later Python versions):
417
442
  time.sleep(10)
418
- assert wk.tasks[0].elements[0].outputs.p1c.value == P1(a=p1_val + 100)
443
+ p1c = wk.tasks[0].elements[0].outputs.p1c
444
+ assert isinstance(p1c, hf.ElementParameter)
445
+ assert p1c.value == P1(a=p1_val + 100)
419
446
 
420
447
 
421
448
  @pytest.mark.integration
422
449
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
423
- def test_script_hdf5_out_obj(null_config, tmp_path):
450
+ def test_script_hdf5_out_obj(null_config, tmp_path: Path):
424
451
  """Use a custom HDF5 saver defined in the P1 class."""
425
452
  s1 = hf.TaskSchema(
426
453
  objective="t1",
@@ -446,13 +473,14 @@ def test_script_hdf5_out_obj(null_config, tmp_path):
446
473
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
447
474
  # to be later Python versions):
448
475
  time.sleep(10)
449
- assert wk.tasks[0].elements[0].outputs.p1c.value == P1(a=p1_val + 100)
476
+ p1c = wk.tasks[0].elements[0].outputs.p1c
477
+ assert isinstance(p1c, hf.ElementParameter)
478
+ assert p1c.value == P1(a=p1_val + 100)
450
479
 
451
480
 
452
481
  @pytest.mark.integration
453
482
  @pytest.mark.skipif("hf.run_time_info.is_frozen")
454
- def test_script_direct_in_pass_env_spec(new_null_config, tmp_path):
455
-
483
+ def test_script_direct_in_pass_env_spec(new_null_config, tmp_path: Path):
456
484
  vers_spec = {"version": "1.2"}
457
485
  env = hf.Environment(
458
486
  name="python_env_with_specifiers",
@@ -503,7 +531,9 @@ def test_script_direct_in_pass_env_spec(new_null_config, tmp_path):
503
531
  # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
504
532
  # to be later Python versions):
505
533
  time.sleep(10)
506
- assert wk.tasks[0].elements[0].outputs.p2.value == {
534
+ p2 = wk.tasks[0].elements[0].outputs.p2
535
+ assert isinstance(p2, hf.ElementParameter)
536
+ assert p2.value == {
507
537
  "name": "python_env_with_specifiers",
508
538
  **vers_spec,
509
539
  }
@@ -1,14 +1,16 @@
1
1
  import time
2
-
2
+ from pathlib import Path
3
3
  import pytest
4
-
4
+ from hpcflow.app import app as hf
5
5
  from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
6
6
 
7
7
 
8
8
  @pytest.mark.wsl
9
- def test_workflow_1(tmp_path, null_config):
9
+ def test_workflow_1(tmp_path: Path, null_config):
10
10
  wk = make_test_data_YAML_workflow("workflow_1_wsl.yaml", path=tmp_path)
11
11
  wk.submit(wait=True, add_to_known=False)
12
12
  time.sleep(20) # TODO: bug! for some reason the new parameter isn't actually written
13
13
  # to disk when using WSL until several seconds after the workflow has finished!
14
- assert wk.tasks[0].elements[0].outputs.p2.value == "201"
14
+ p2 = wk.tasks[0].elements[0].outputs.p2
15
+ assert isinstance(p2, hf.ElementParameter)
16
+ assert p2.value == "201"