hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/bad_script.py +2 -0
  5. hpcflow/data/scripts/do_nothing.py +2 -0
  6. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  7. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  8. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  9. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  10. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  11. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  12. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  13. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  14. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  15. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  16. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  17. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  18. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  19. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  20. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  21. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  22. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  23. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  24. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  25. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  26. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  27. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  28. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  29. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  30. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  31. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  32. hpcflow/data/scripts/script_exit_test.py +5 -0
  33. hpcflow/data/template_components/environments.yaml +1 -1
  34. hpcflow/sdk/__init__.py +26 -15
  35. hpcflow/sdk/app.py +2192 -768
  36. hpcflow/sdk/cli.py +506 -296
  37. hpcflow/sdk/cli_common.py +105 -7
  38. hpcflow/sdk/config/__init__.py +1 -1
  39. hpcflow/sdk/config/callbacks.py +115 -43
  40. hpcflow/sdk/config/cli.py +126 -103
  41. hpcflow/sdk/config/config.py +674 -318
  42. hpcflow/sdk/config/config_file.py +131 -95
  43. hpcflow/sdk/config/errors.py +125 -84
  44. hpcflow/sdk/config/types.py +148 -0
  45. hpcflow/sdk/core/__init__.py +25 -1
  46. hpcflow/sdk/core/actions.py +1771 -1059
  47. hpcflow/sdk/core/app_aware.py +24 -0
  48. hpcflow/sdk/core/cache.py +139 -79
  49. hpcflow/sdk/core/command_files.py +263 -287
  50. hpcflow/sdk/core/commands.py +145 -112
  51. hpcflow/sdk/core/element.py +828 -535
  52. hpcflow/sdk/core/enums.py +192 -0
  53. hpcflow/sdk/core/environment.py +74 -93
  54. hpcflow/sdk/core/errors.py +455 -52
  55. hpcflow/sdk/core/execute.py +207 -0
  56. hpcflow/sdk/core/json_like.py +540 -272
  57. hpcflow/sdk/core/loop.py +751 -347
  58. hpcflow/sdk/core/loop_cache.py +164 -47
  59. hpcflow/sdk/core/object_list.py +370 -207
  60. hpcflow/sdk/core/parameters.py +1100 -627
  61. hpcflow/sdk/core/rule.py +59 -41
  62. hpcflow/sdk/core/run_dir_files.py +21 -37
  63. hpcflow/sdk/core/skip_reason.py +7 -0
  64. hpcflow/sdk/core/task.py +1649 -1339
  65. hpcflow/sdk/core/task_schema.py +308 -196
  66. hpcflow/sdk/core/test_utils.py +191 -114
  67. hpcflow/sdk/core/types.py +440 -0
  68. hpcflow/sdk/core/utils.py +485 -309
  69. hpcflow/sdk/core/validation.py +82 -9
  70. hpcflow/sdk/core/workflow.py +2544 -1178
  71. hpcflow/sdk/core/zarr_io.py +98 -137
  72. hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
  73. hpcflow/sdk/demo/cli.py +53 -33
  74. hpcflow/sdk/helper/cli.py +18 -15
  75. hpcflow/sdk/helper/helper.py +75 -63
  76. hpcflow/sdk/helper/watcher.py +61 -28
  77. hpcflow/sdk/log.py +122 -71
  78. hpcflow/sdk/persistence/__init__.py +8 -31
  79. hpcflow/sdk/persistence/base.py +1360 -606
  80. hpcflow/sdk/persistence/defaults.py +6 -0
  81. hpcflow/sdk/persistence/discovery.py +38 -0
  82. hpcflow/sdk/persistence/json.py +568 -188
  83. hpcflow/sdk/persistence/pending.py +382 -179
  84. hpcflow/sdk/persistence/store_resource.py +39 -23
  85. hpcflow/sdk/persistence/types.py +318 -0
  86. hpcflow/sdk/persistence/utils.py +14 -11
  87. hpcflow/sdk/persistence/zarr.py +1337 -433
  88. hpcflow/sdk/runtime.py +44 -41
  89. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  90. hpcflow/sdk/submission/jobscript.py +1651 -692
  91. hpcflow/sdk/submission/schedulers/__init__.py +167 -39
  92. hpcflow/sdk/submission/schedulers/direct.py +121 -81
  93. hpcflow/sdk/submission/schedulers/sge.py +170 -129
  94. hpcflow/sdk/submission/schedulers/slurm.py +291 -268
  95. hpcflow/sdk/submission/schedulers/utils.py +12 -2
  96. hpcflow/sdk/submission/shells/__init__.py +14 -15
  97. hpcflow/sdk/submission/shells/base.py +150 -29
  98. hpcflow/sdk/submission/shells/bash.py +283 -173
  99. hpcflow/sdk/submission/shells/os_version.py +31 -30
  100. hpcflow/sdk/submission/shells/powershell.py +228 -170
  101. hpcflow/sdk/submission/submission.py +1014 -335
  102. hpcflow/sdk/submission/types.py +140 -0
  103. hpcflow/sdk/typing.py +182 -12
  104. hpcflow/sdk/utils/arrays.py +71 -0
  105. hpcflow/sdk/utils/deferred_file.py +55 -0
  106. hpcflow/sdk/utils/hashing.py +16 -0
  107. hpcflow/sdk/utils/patches.py +12 -0
  108. hpcflow/sdk/utils/strings.py +33 -0
  109. hpcflow/tests/api/test_api.py +32 -0
  110. hpcflow/tests/conftest.py +27 -6
  111. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  112. hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
  113. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  114. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  115. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  116. hpcflow/tests/scripts/test_main_scripts.py +866 -85
  117. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  118. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  119. hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
  120. hpcflow/tests/unit/test_action.py +262 -75
  121. hpcflow/tests/unit/test_action_rule.py +9 -4
  122. hpcflow/tests/unit/test_app.py +33 -6
  123. hpcflow/tests/unit/test_cache.py +46 -0
  124. hpcflow/tests/unit/test_cli.py +134 -1
  125. hpcflow/tests/unit/test_command.py +71 -54
  126. hpcflow/tests/unit/test_config.py +142 -16
  127. hpcflow/tests/unit/test_config_file.py +21 -18
  128. hpcflow/tests/unit/test_element.py +58 -62
  129. hpcflow/tests/unit/test_element_iteration.py +50 -1
  130. hpcflow/tests/unit/test_element_set.py +29 -19
  131. hpcflow/tests/unit/test_group.py +4 -2
  132. hpcflow/tests/unit/test_input_source.py +116 -93
  133. hpcflow/tests/unit/test_input_value.py +29 -24
  134. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  135. hpcflow/tests/unit/test_json_like.py +44 -35
  136. hpcflow/tests/unit/test_loop.py +1396 -84
  137. hpcflow/tests/unit/test_meta_task.py +325 -0
  138. hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
  139. hpcflow/tests/unit/test_object_list.py +17 -12
  140. hpcflow/tests/unit/test_parameter.py +29 -7
  141. hpcflow/tests/unit/test_persistence.py +237 -42
  142. hpcflow/tests/unit/test_resources.py +20 -18
  143. hpcflow/tests/unit/test_run.py +117 -6
  144. hpcflow/tests/unit/test_run_directories.py +29 -0
  145. hpcflow/tests/unit/test_runtime.py +2 -1
  146. hpcflow/tests/unit/test_schema_input.py +23 -15
  147. hpcflow/tests/unit/test_shell.py +23 -2
  148. hpcflow/tests/unit/test_slurm.py +8 -7
  149. hpcflow/tests/unit/test_submission.py +38 -89
  150. hpcflow/tests/unit/test_task.py +352 -247
  151. hpcflow/tests/unit/test_task_schema.py +33 -20
  152. hpcflow/tests/unit/test_utils.py +9 -11
  153. hpcflow/tests/unit/test_value_sequence.py +15 -12
  154. hpcflow/tests/unit/test_workflow.py +114 -83
  155. hpcflow/tests/unit/test_workflow_template.py +0 -1
  156. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  157. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  158. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  159. hpcflow/tests/unit/utils/test_patches.py +5 -0
  160. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  161. hpcflow/tests/workflows/__init__.py +0 -0
  162. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  163. hpcflow/tests/workflows/test_jobscript.py +334 -1
  164. hpcflow/tests/workflows/test_run_status.py +198 -0
  165. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  166. hpcflow/tests/workflows/test_submission.py +140 -0
  167. hpcflow/tests/workflows/test_workflows.py +160 -15
  168. hpcflow/tests/workflows/test_zip.py +18 -0
  169. hpcflow/viz_demo.ipynb +6587 -3
  170. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
  171. hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
  172. hpcflow/sdk/core/parallel.py +0 -21
  173. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  174. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
  175. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
  176. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/runtime.py CHANGED
@@ -2,14 +2,16 @@
2
2
  Information about the Python runtime.
3
3
  """
4
4
 
5
+ from __future__ import annotations
5
6
  from importlib import import_module
6
- import logging
7
+ from logging import Logger
7
8
  import os
8
9
  import platform
10
+ import re
9
11
  import socket
10
12
  import sys
11
13
  from pathlib import Path
12
- import warnings
14
+ from typing import Any, ClassVar
13
15
 
14
16
  from rich.table import Table
15
17
  from rich.console import Console
@@ -31,10 +33,14 @@ class RunTimeInfo:
31
33
  Where to write logging versions.
32
34
  """
33
35
 
34
- def __init__(self, name, package_name, version, logger):
35
- is_frozen = getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS")
36
+ def __init__(
37
+ self, name: str, package_name: str, version: str, logger: Logger
38
+ ) -> None:
39
+ is_frozen: bool = getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS")
36
40
  bundle_dir = (
37
- sys._MEIPASS if is_frozen else os.path.dirname(os.path.abspath(__file__))
41
+ sys._MEIPASS
42
+ if is_frozen and hasattr(sys, "_MEIPASS")
43
+ else os.path.dirname(os.path.abspath(__file__))
38
44
  )
39
45
 
40
46
  #: Application name.
@@ -69,10 +75,11 @@ class RunTimeInfo:
69
75
  self.python_executable_path = Path(sys.executable)
70
76
 
71
77
  try:
72
- get_ipython
73
- self.in_ipython = True
78
+ get_ipython # type: ignore
74
79
  except NameError:
75
80
  pass
81
+ else:
82
+ self.in_ipython = True
76
83
 
77
84
  if hasattr(sys, "ps1"):
78
85
  self.is_interactive = True
@@ -101,7 +108,7 @@ class RunTimeInfo:
101
108
 
102
109
  try:
103
110
  #: The virtual environment path.
104
- self.venv_path = self._set_venv_path()
111
+ self.venv_path: str | list[str] | None = self.__set_venv_path()
105
112
  except ValueError:
106
113
  self.venv_path = None
107
114
 
@@ -126,7 +133,7 @@ class RunTimeInfo:
126
133
  # )
127
134
  # warnings.warn(msg)
128
135
 
129
- def to_dict(self):
136
+ def to_dict(self) -> dict[str, Any]:
130
137
  """
131
138
  Serialize this class as a dictionary.
132
139
  """
@@ -171,16 +178,16 @@ class RunTimeInfo:
171
178
  )
172
179
  return out
173
180
 
174
- def __repr__(self):
181
+ def __repr__(self) -> str:
175
182
  out = f"{self.__class__.__name__}("
176
183
  out += ", ".join(f"{k}={v!r}" for k, v in self.to_dict().items())
177
184
  return out
178
185
 
179
- def _set_venv_path(self):
180
- out = []
181
- if self.is_venv:
186
+ def __set_venv_path(self) -> str | list[str]:
187
+ out: list[str] = []
188
+ if self.sys_prefix is not None:
182
189
  out.append(self.sys_prefix)
183
- elif self.is_conda_venv:
190
+ elif self.conda_prefix is not None:
184
191
  out.append(self.conda_prefix)
185
192
  if not out:
186
193
  raise ValueError("Not running in a virtual environment!")
@@ -201,7 +208,7 @@ class RunTimeInfo:
201
208
  """
202
209
  pass
203
210
 
204
- def show(self):
211
+ def show(self) -> None:
205
212
  """
206
213
  Display the information known by this class as a human-readable table.
207
214
  """
@@ -215,7 +222,7 @@ class RunTimeInfo:
215
222
  console.print(tab)
216
223
 
217
224
  @property
218
- def executable_path(self):
225
+ def executable_path(self) -> Path | None:
219
226
  """Get the path that the user invoked to launch the frozen app, if the app is
220
227
  frozen.
221
228
 
@@ -223,11 +230,10 @@ class RunTimeInfo:
223
230
  whereas `executable_path_resolved` returns the actual frozen app path.
224
231
 
225
232
  """
226
- if self.is_frozen:
227
- return Path(sys.argv[0])
233
+ return Path(sys.argv[0]) if self.is_frozen else None
228
234
 
229
235
  @property
230
- def resolved_executable_path(self):
236
+ def resolved_executable_path(self) -> Path | None:
231
237
  """Get the resolved path to the frozen app that the user launched, if the app is
232
238
  frozen.
233
239
 
@@ -239,11 +245,10 @@ class RunTimeInfo:
239
245
  [1] https://pyinstaller.org/en/stable/runtime-information.html#using-sys-executable-and-sys-argv-0
240
246
 
241
247
  """
242
- if self.is_frozen:
243
- return Path(sys.executable)
248
+ return Path(sys.executable) if self.is_frozen else None
244
249
 
245
250
  @property
246
- def executable_name(self):
251
+ def executable_name(self) -> str | None:
247
252
  """Get the name of the frozen app executable, if the app is frozen.
248
253
 
249
254
  If the user launches the app via a symbolic link, then this returns the name of
@@ -251,47 +256,45 @@ class RunTimeInfo:
251
256
  name.
252
257
 
253
258
  """
254
- if self.is_frozen:
255
- return self.executable_path.name
259
+ return None if (p := self.executable_path) is None else p.name
256
260
 
257
261
  @property
258
- def resolved_executable_name(self):
262
+ def resolved_executable_name(self) -> str | None:
259
263
  """Get the resolved name of the frozen app executable, if the app is frozen."""
260
- if self.is_frozen:
261
- return self.resolved_executable_path.name
264
+ return None if (p := self.resolved_executable_path) is None else p.name
262
265
 
263
266
  @property
264
- def script_path(self) -> Path:
267
+ def script_path(self) -> Path | None:
265
268
  """Get the path to the Python script used to invoked this instance of the app, if
266
269
  the app is not frozen."""
267
- if not self.is_frozen:
268
- return Path(sys.argv[0])
270
+ return None if self.is_frozen else Path(sys.argv[0])
269
271
 
270
272
  @property
271
- def resolved_script_path(self) -> Path:
273
+ def resolved_script_path(self) -> Path | None:
272
274
  """Get the resolved path to the Python script used to invoked this instance of the
273
275
  app, if the app is not frozen."""
274
- if not self.is_frozen:
275
- return self.script_path.resolve()
276
+ return None if (p := self.script_path) is None else p.resolve()
277
+
278
+ # For removing a trailing '.cmd' from a filename
279
+ __CMD_TRIM: ClassVar[re.Pattern[str]] = re.compile(r"\.cmd$")
276
280
 
277
281
  @property
278
- def invocation_command(self):
282
+ def invocation_command(self) -> tuple[str, ...]:
279
283
  """Get the command that was used to invoke this instance of the app."""
280
284
  if self.is_frozen:
281
285
  # (this also works if we are running tests using the frozen app)
282
- command = [str(self.resolved_executable_path)]
286
+ return (str(self.resolved_executable_path),)
283
287
  elif self.from_CLI:
284
288
  script = str(self.resolved_script_path)
285
- if os.name == "nt" and script.endswith(".cmd"):
289
+ if os.name == "nt":
286
290
  # cannot reproduce locally, but on Windows GHA runners, if pytest is
287
291
  # invoked via `hpcflow test`, `resolved_script_path` seems to be the
288
292
  # batch script wrapper (ending in .cmd) rather than the Python entry point
289
293
  # itself, so trim if off:
290
- script = script.rstrip(".cmd")
291
- command = [str(self.python_executable_path), script]
294
+ script = self.__CMD_TRIM.sub("", script) # Work with 3.8 too
295
+ # script = script.removesuffix(".cmd")
296
+ return (str(self.python_executable_path), script)
292
297
  else:
293
298
  app_module = import_module(self.package_name)
294
299
  CLI_path = Path(*app_module.__path__, "cli.py")
295
- command = [str(self.python_executable_path), str(CLI_path)]
296
-
297
- return tuple(command)
300
+ return (str(self.python_executable_path), str(CLI_path))
@@ -1,22 +1,29 @@
1
1
  """
2
- Jobscript state enumeration.
2
+ Submission enumeration types.
3
3
  """
4
+ from __future__ import annotations
5
+ from dataclasses import dataclass
6
+ from enum import Enum
4
7
 
5
- import enum
6
8
 
9
+ @dataclass(frozen=True)
10
+ class _JES:
11
+ """
12
+ Model of the state of a JobscriptElementState
13
+ """
7
14
 
8
- class JobscriptElementState(enum.Enum):
15
+ _value: int
16
+ #: The symbol used to render the state.
17
+ symbol: str
18
+ #: The colour used to render the state.
19
+ colour: str
20
+ __doc__: str = ""
21
+
22
+
23
+ class JobscriptElementState(_JES, Enum):
9
24
  """Enumeration to convey a particular jobscript element state as reported by the
10
25
  scheduler."""
11
26
 
12
- def __new__(cls, value, symbol, colour, doc=None):
13
- member = object.__new__(cls)
14
- member._value_ = value
15
- member.symbol = symbol
16
- member.colour = colour
17
- member.__doc__ = doc
18
- return member
19
-
20
27
  #: Waiting for resource allocation.
21
28
  pending = (
22
29
  0,
@@ -61,8 +68,28 @@ class JobscriptElementState(enum.Enum):
61
68
  )
62
69
 
63
70
  @property
64
- def rich_repr(self):
71
+ def value(self) -> int:
72
+ """
73
+ The numerical value of this state.
74
+ """
75
+ return self._value
76
+
77
+ @property
78
+ def rich_repr(self) -> str:
65
79
  """
66
80
  Rich representation of this enumeration element.
67
81
  """
68
82
  return f"[{self.colour}]{self.symbol}[/{self.colour}]"
83
+
84
+
85
+ class SubmissionStatus(Enum):
86
+ """
87
+ The overall status of a submission.
88
+ """
89
+
90
+ #: Not yet submitted.
91
+ PENDING = 0
92
+ #: All jobscripts submitted successfully.
93
+ SUBMITTED = 1
94
+ #: Some jobscripts submitted successfully.
95
+ PARTIALLY_SUBMITTED = 2