hpcflow-new2 0.2.0a211__py3-none-any.whl → 0.2.0a212__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. hpcflow/_version.py +1 -1
  2. hpcflow/sdk/app.py +15 -24
  3. hpcflow/sdk/config/callbacks.py +5 -10
  4. hpcflow/sdk/config/config.py +11 -23
  5. hpcflow/sdk/core/actions.py +25 -40
  6. hpcflow/sdk/core/app_aware.py +1 -0
  7. hpcflow/sdk/core/cache.py +3 -3
  8. hpcflow/sdk/core/command_files.py +2 -4
  9. hpcflow/sdk/core/element.py +40 -72
  10. hpcflow/sdk/core/enums.py +1 -0
  11. hpcflow/sdk/core/json_like.py +18 -30
  12. hpcflow/sdk/core/object_list.py +14 -21
  13. hpcflow/sdk/core/parameters.py +24 -10
  14. hpcflow/sdk/core/task.py +18 -32
  15. hpcflow/sdk/core/task_schema.py +1 -1
  16. hpcflow/sdk/core/types.py +1 -2
  17. hpcflow/sdk/core/utils.py +10 -17
  18. hpcflow/sdk/core/validation.py +11 -22
  19. hpcflow/sdk/core/workflow.py +15 -23
  20. hpcflow/sdk/persistence/base.py +68 -116
  21. hpcflow/sdk/persistence/discovery.py +1 -0
  22. hpcflow/sdk/persistence/zarr.py +12 -12
  23. hpcflow/sdk/submission/enums.py +1 -0
  24. hpcflow/sdk/submission/jobscript.py +8 -10
  25. hpcflow/sdk/submission/schedulers/direct.py +2 -4
  26. hpcflow/sdk/submission/shells/__init__.py +1 -0
  27. hpcflow/sdk/submission/submission.py +11 -13
  28. hpcflow/sdk/utils/arrays.py +2 -4
  29. hpcflow/tests/unit/test_multi_path_sequences.py +23 -0
  30. {hpcflow_new2-0.2.0a211.dist-info → hpcflow_new2-0.2.0a212.dist-info}/METADATA +1 -1
  31. {hpcflow_new2-0.2.0a211.dist-info → hpcflow_new2-0.2.0a212.dist-info}/RECORD +34 -34
  32. {hpcflow_new2-0.2.0a211.dist-info → hpcflow_new2-0.2.0a212.dist-info}/LICENSE +0 -0
  33. {hpcflow_new2-0.2.0a211.dist-info → hpcflow_new2-0.2.0a212.dist-info}/WHEEL +0 -0
  34. {hpcflow_new2-0.2.0a211.dist-info → hpcflow_new2-0.2.0a212.dist-info}/entry_points.txt +0 -0
hpcflow/_version.py CHANGED
@@ -1 +1 @@
1
- __version__ = "0.2.0a211"
1
+ __version__ = "0.2.0a212"
hpcflow/sdk/app.py CHANGED
@@ -178,8 +178,7 @@ if TYPE_CHECKING:
178
178
  variables: dict[str, str] | None = None,
179
179
  status: bool = True,
180
180
  add_submission: bool = False,
181
- ) -> _Workflow | _Submission | None:
182
- ...
181
+ ) -> _Workflow | _Submission | None: ...
183
182
 
184
183
  class _MakeDemoWorkflow(Protocol):
185
184
  """Type of :py:meth:`BaseApp.make_demo_workflow`"""
@@ -198,8 +197,7 @@ if TYPE_CHECKING:
198
197
  variables: dict[str, str] | None = None,
199
198
  status: bool = True,
200
199
  add_submission: bool = False,
201
- ) -> _Workflow | _Submission | None:
202
- ...
200
+ ) -> _Workflow | _Submission | None: ...
203
201
 
204
202
  class _MakeAndSubmitWorkflow(Protocol):
205
203
  """Type of :py:meth:`BaseApp.make_and_submit_workflow`"""
@@ -225,8 +223,7 @@ if TYPE_CHECKING:
225
223
  tasks: list[int] | None = None,
226
224
  cancel: bool = False,
227
225
  status: bool = True,
228
- ) -> tuple[_Workflow, Mapping[int, Sequence[int]]] | _Workflow:
229
- ...
226
+ ) -> tuple[_Workflow, Mapping[int, Sequence[int]]] | _Workflow: ...
230
227
 
231
228
  class _MakeAndSubmitDemoWorkflow(Protocol):
232
229
  """Type of :py:meth:`BaseApp.make_and_submit_demo_workflow`"""
@@ -251,8 +248,7 @@ if TYPE_CHECKING:
251
248
  tasks: list[int] | None = None,
252
249
  cancel: bool = False,
253
250
  status: bool = True,
254
- ) -> tuple[_Workflow, Mapping[int, Sequence[int]]] | _Workflow:
255
- ...
251
+ ) -> tuple[_Workflow, Mapping[int, Sequence[int]]] | _Workflow: ...
256
252
 
257
253
  class _SubmitWorkflow(Protocol):
258
254
  """Type of :py:meth:`BaseApp.submit_workflow`"""
@@ -265,8 +261,7 @@ if TYPE_CHECKING:
265
261
  wait: bool = False,
266
262
  return_idx: bool = False,
267
263
  tasks: list[int] | None = None,
268
- ) -> Mapping[int, Sequence[int]] | None:
269
- ...
264
+ ) -> Mapping[int, Sequence[int]] | None: ...
270
265
 
271
266
  class _GetKnownSubmissions(Protocol):
272
267
  """Type of :py:meth:`BaseApp.get_known_submissions`"""
@@ -278,8 +273,7 @@ if TYPE_CHECKING:
278
273
  no_update: bool = False,
279
274
  as_json: bool = False,
280
275
  status: Status | None = None,
281
- ) -> Sequence[KnownSubmissionItem]:
282
- ...
276
+ ) -> Sequence[KnownSubmissionItem]: ...
283
277
 
284
278
  class _Show(Protocol):
285
279
  """Type of :py:meth:`BaseApp.show`"""
@@ -289,8 +283,7 @@ if TYPE_CHECKING:
289
283
  max_recent: int = 3,
290
284
  full: bool = False,
291
285
  no_update: bool = False,
292
- ) -> None:
293
- ...
286
+ ) -> None: ...
294
287
 
295
288
  class _Cancel(Protocol):
296
289
  """Type of :py:meth:`BaseApp.cancel`"""
@@ -300,14 +293,12 @@ if TYPE_CHECKING:
300
293
  workflow_ref: int | str | Path,
301
294
  ref_is_path: str | None = None,
302
295
  status: bool = False,
303
- ) -> None:
304
- ...
296
+ ) -> None: ...
305
297
 
306
298
  class _RunTests(Protocol):
307
299
  """Type of :py:meth:`BaseApp.run_tests and run_hpcflow_tests`"""
308
300
 
309
- def __call__(self, *args: str) -> int:
310
- ...
301
+ def __call__(self, *args: str) -> int: ...
311
302
 
312
303
 
313
304
  SDK_logger = get_SDK_logger(__name__)
@@ -558,9 +549,9 @@ class BaseApp(metaclass=Singleton):
558
549
  self._config: Config | None = (
559
550
  None # assigned on first access to `config` property
560
551
  )
561
- self._config_files: dict[
562
- str, ConfigFile
563
- ] = {} # assigned on config load, keys are string absolute paths
552
+ self._config_files: dict[str, ConfigFile] = (
553
+ {}
554
+ ) # assigned on config load, keys are string absolute paths
564
555
 
565
556
  # Set by `_load_template_components`:
566
557
  self._template_components: TemplateComponents = {}
@@ -2588,9 +2579,9 @@ class BaseApp(metaclass=Singleton):
2588
2579
  # date-times:
2589
2580
  line_date: dict[int, str] = {}
2590
2581
 
2591
- removed_IDs: list[
2592
- int
2593
- ] = [] # which submissions we completely remove from the file
2582
+ removed_IDs: list[int] = (
2583
+ []
2584
+ ) # which submissions we completely remove from the file
2594
2585
 
2595
2586
  new_lines: list[str] = []
2596
2587
  line_IDs: list[int] = []
@@ -33,13 +33,11 @@ def callback_vars(config: Config, value) -> str:
33
33
 
34
34
 
35
35
  @overload
36
- def callback_file_paths(config: Config, file_path: PathLike) -> PathLike:
37
- ...
36
+ def callback_file_paths(config: Config, file_path: PathLike) -> PathLike: ...
38
37
 
39
38
 
40
39
  @overload
41
- def callback_file_paths(config: Config, file_path: list[PathLike]) -> list[PathLike]:
42
- ...
40
+ def callback_file_paths(config: Config, file_path: list[PathLike]) -> list[PathLike]: ...
43
41
 
44
42
 
45
43
  def callback_file_paths(config: Config, file_path: PathLike | list[PathLike]):
@@ -67,18 +65,15 @@ def callback_bool(config: Config, value: str | bool) -> bool:
67
65
 
68
66
 
69
67
  @overload
70
- def callback_lowercase(config: Config, value: list[str]) -> list[str]:
71
- ...
68
+ def callback_lowercase(config: Config, value: list[str]) -> list[str]: ...
72
69
 
73
70
 
74
71
  @overload
75
- def callback_lowercase(config: Config, value: dict[str, T]) -> dict[str, T]:
76
- ...
72
+ def callback_lowercase(config: Config, value: dict[str, T]) -> dict[str, T]: ...
77
73
 
78
74
 
79
75
  @overload
80
- def callback_lowercase(config: Config, value: str) -> str:
81
- ...
76
+ def callback_lowercase(config: Config, value: str) -> str: ...
82
77
 
83
78
 
84
79
  def callback_lowercase(
@@ -541,9 +541,7 @@ class Config:
541
541
  else:
542
542
  super().__setattr__(name, value)
543
543
 
544
- def _disable_callbacks(
545
- self, callbacks: Sequence[str]
546
- ) -> tuple[
544
+ def _disable_callbacks(self, callbacks: Sequence[str]) -> tuple[
547
545
  dict[str, tuple[GetterCallback, ...]],
548
546
  dict[str, tuple[SetterCallback, ...]],
549
547
  dict[str, tuple[UnsetterCallback, ...]],
@@ -661,14 +659,12 @@ class Config:
661
659
  @overload
662
660
  def get_all(
663
661
  self, *, include_overrides: bool = True, as_str: Literal[True]
664
- ) -> Mapping[str, str]:
665
- ...
662
+ ) -> Mapping[str, str]: ...
666
663
 
667
664
  @overload
668
665
  def get_all(
669
666
  self, *, include_overrides: bool = True, as_str: Literal[False] = False
670
- ) -> Mapping[str, Any]:
671
- ...
667
+ ) -> Mapping[str, Any]: ...
672
668
 
673
669
  def get_all(
674
670
  self, *, include_overrides: bool = True, as_str: bool = False
@@ -740,8 +736,7 @@ class Config:
740
736
  as_str: Literal[False] = False,
741
737
  callback=True,
742
738
  default_value=None,
743
- ) -> Any:
744
- ...
739
+ ) -> Any: ...
745
740
 
746
741
  @overload
747
742
  def _get(
@@ -753,8 +748,7 @@ class Config:
753
748
  as_str: Literal[True],
754
749
  callback=True,
755
750
  default_value=None,
756
- ) -> list[str] | str:
757
- ...
751
+ ) -> list[str] | str: ...
758
752
 
759
753
  def _get(
760
754
  self,
@@ -829,8 +823,7 @@ class Config:
829
823
  @overload
830
824
  def _set(
831
825
  self, name: str, value: str, *, is_json: Literal[True], callback=True, quiet=False
832
- ) -> None:
833
- ...
826
+ ) -> None: ...
834
827
 
835
828
  @overload
836
829
  def _set(
@@ -841,8 +834,7 @@ class Config:
841
834
  is_json: Literal[False] = False,
842
835
  callback=True,
843
836
  quiet=False,
844
- ) -> None:
845
- ...
837
+ ) -> None: ...
846
838
 
847
839
  def _set(
848
840
  self, name: str, value, *, is_json=False, callback=True, quiet=False
@@ -918,14 +910,12 @@ class Config:
918
910
  *,
919
911
  is_json: Literal[False] = False,
920
912
  quiet: bool = False,
921
- ) -> None:
922
- ...
913
+ ) -> None: ...
923
914
 
924
915
  @overload
925
916
  def set(
926
917
  self, path: str, value: str, *, is_json: Literal[True], quiet: bool = False
927
- ) -> None:
928
- ...
918
+ ) -> None: ...
929
919
 
930
920
  def set(
931
921
  self, path: str, value: Any, *, is_json: bool = False, quiet: bool = False
@@ -1003,8 +993,7 @@ class Config:
1003
993
  copy: bool = False,
1004
994
  ret_root_and_parts: Literal[False] = False,
1005
995
  default: Any | None = None,
1006
- ) -> Any:
1007
- ...
996
+ ) -> Any: ...
1008
997
 
1009
998
  @overload
1010
999
  def get(
@@ -1015,8 +1004,7 @@ class Config:
1015
1004
  copy: bool = False,
1016
1005
  ret_root_and_parts: Literal[True],
1017
1006
  default: Any | None = None,
1018
- ) -> tuple[Any, Any, list[str]]:
1019
- ...
1007
+ ) -> tuple[Any, Any, list[str]]: ...
1020
1008
 
1021
1009
  def get(
1022
1010
  self,
@@ -464,8 +464,7 @@ class ElementActionRun(AppAware):
464
464
  typ: str | None = None,
465
465
  as_strings: Literal[False] = False,
466
466
  use_task_index: bool = False,
467
- ) -> Mapping[str, ParamSource | list[ParamSource]]:
468
- ...
467
+ ) -> Mapping[str, ParamSource | list[ParamSource]]: ...
469
468
 
470
469
  @overload
471
470
  def get_parameter_sources(
@@ -475,8 +474,7 @@ class ElementActionRun(AppAware):
475
474
  typ: str | None = None,
476
475
  as_strings: Literal[True],
477
476
  use_task_index: bool = False,
478
- ) -> Mapping[str, str]:
479
- ...
477
+ ) -> Mapping[str, str]: ...
480
478
 
481
479
  @TimeIt.decorator
482
480
  def get_parameter_sources(
@@ -552,12 +550,12 @@ class ElementActionRun(AppAware):
552
550
  )
553
551
 
554
552
  @overload
555
- def get_EAR_dependencies(self, as_objects: Literal[False] = False) -> set[int]:
556
- ...
553
+ def get_EAR_dependencies(self, as_objects: Literal[False] = False) -> set[int]: ...
557
554
 
558
555
  @overload
559
- def get_EAR_dependencies(self, as_objects: Literal[True]) -> list[ElementActionRun]:
560
- ...
556
+ def get_EAR_dependencies(
557
+ self, as_objects: Literal[True]
558
+ ) -> list[ElementActionRun]: ...
561
559
 
562
560
  @TimeIt.decorator
563
561
  def get_EAR_dependencies(self, as_objects=False) -> list[ElementActionRun] | set[int]:
@@ -592,12 +590,10 @@ class ElementActionRun(AppAware):
592
590
  }
593
591
 
594
592
  @overload
595
- def get_dependent_EARs(self, as_objects: Literal[False] = False) -> set[int]:
596
- ...
593
+ def get_dependent_EARs(self, as_objects: Literal[False] = False) -> set[int]: ...
597
594
 
598
595
  @overload
599
- def get_dependent_EARs(self, as_objects: Literal[True]) -> list[ElementActionRun]:
600
- ...
596
+ def get_dependent_EARs(self, as_objects: Literal[True]) -> list[ElementActionRun]: ...
601
597
 
602
598
  def get_dependent_EARs(
603
599
  self, as_objects: bool = False
@@ -865,9 +861,9 @@ class ElementActionRun(AppAware):
865
861
  "Cannot get output file parser inputs from this from EAR because the "
866
862
  "associated action is not expanded, meaning multiple OFPs might exist."
867
863
  )
868
- inputs: dict[
869
- str, str | list[str] | Mapping[str, Any]
870
- ] = {} # not sure this type is correct
864
+ inputs: dict[str, str | list[str] | Mapping[str, Any]] = (
865
+ {}
866
+ ) # not sure this type is correct
871
867
  for inp_typ in self.action.output_file_parsers[0].inputs or []:
872
868
  inputs[inp_typ] = self.get(f"inputs.{inp_typ}", raise_on_unset=raise_on_unset)
873
869
 
@@ -1130,8 +1126,7 @@ class ElementActionRun(AppAware):
1130
1126
  jobscript: Jobscript,
1131
1127
  environments: EnvironmentsList,
1132
1128
  raise_on_unset: Literal[True],
1133
- ) -> Path:
1134
- ...
1129
+ ) -> Path: ...
1135
1130
 
1136
1131
  @overload
1137
1132
  def try_write_commands(
@@ -1139,8 +1134,7 @@ class ElementActionRun(AppAware):
1139
1134
  jobscript: Jobscript,
1140
1135
  environments: EnvironmentsList,
1141
1136
  raise_on_unset: Literal[False] = False,
1142
- ) -> Path | None:
1143
- ...
1137
+ ) -> Path | None: ...
1144
1138
 
1145
1139
  def try_write_commands(
1146
1140
  self,
@@ -1395,8 +1389,7 @@ class ElementAction(AppAware):
1395
1389
  typ: str | None = None,
1396
1390
  as_strings: Literal[False] = False,
1397
1391
  use_task_index: bool = False,
1398
- ) -> Mapping[str, ParamSource | list[ParamSource]]:
1399
- ...
1392
+ ) -> Mapping[str, ParamSource | list[ParamSource]]: ...
1400
1393
 
1401
1394
  @overload
1402
1395
  def get_parameter_sources(
@@ -1407,8 +1400,7 @@ class ElementAction(AppAware):
1407
1400
  typ: str | None = None,
1408
1401
  as_strings: Literal[True],
1409
1402
  use_task_index: bool = False,
1410
- ) -> Mapping[str, str]:
1411
- ...
1403
+ ) -> Mapping[str, str]: ...
1412
1404
 
1413
1405
  def get_parameter_sources(
1414
1406
  self,
@@ -2124,9 +2116,9 @@ class Action(JSONLike):
2124
2116
  for ifg in self.input_file_generators
2125
2117
  }
2126
2118
  OFPs = {
2127
- ofp.output.typ
2128
- if ofp.output
2129
- else f"OFP_{idx}": [out_file.label for out_file in ofp.output_files]
2119
+ ofp.output.typ if ofp.output else f"OFP_{idx}": [
2120
+ out_file.label for out_file in ofp.output_files
2121
+ ]
2130
2122
  for idx, ofp in enumerate(self.output_file_parsers)
2131
2123
  }
2132
2124
 
@@ -2341,8 +2333,7 @@ class Action(JSONLike):
2341
2333
  ret_specifiers: Literal[False] = False,
2342
2334
  include_suffix: bool = True,
2343
2335
  specs_suffix_delim: str = ".",
2344
- ) -> tuple[str, Path]:
2345
- ...
2336
+ ) -> tuple[str, Path]: ...
2346
2337
 
2347
2338
  @overload
2348
2339
  def get_script_artifact_name(
@@ -2352,8 +2343,7 @@ class Action(JSONLike):
2352
2343
  ret_specifiers: Literal[True],
2353
2344
  include_suffix: bool = True,
2354
2345
  specs_suffix_delim: str = ".",
2355
- ) -> tuple[str, Path, dict]:
2356
- ...
2346
+ ) -> tuple[str, Path, dict]: ...
2357
2347
 
2358
2348
  def get_script_artifact_name(
2359
2349
  self,
@@ -2399,8 +2389,7 @@ class Action(JSONLike):
2399
2389
  script: str,
2400
2390
  env_spec: Mapping[str, Any] | None = None,
2401
2391
  ret_specifiers: Literal[False] = False,
2402
- ) -> str:
2403
- ...
2392
+ ) -> str: ...
2404
2393
 
2405
2394
  @overload
2406
2395
  @classmethod
@@ -2410,8 +2399,7 @@ class Action(JSONLike):
2410
2399
  env_spec: Mapping[str, Any] | None = None,
2411
2400
  *,
2412
2401
  ret_specifiers: Literal[True],
2413
- ) -> tuple[str, dict[str, Any]]:
2414
- ...
2402
+ ) -> tuple[str, dict[str, Any]]: ...
2415
2403
 
2416
2404
  @overload
2417
2405
  @classmethod
@@ -2421,8 +2409,7 @@ class Action(JSONLike):
2421
2409
  env_spec: Mapping[str, Any] | None = None,
2422
2410
  *,
2423
2411
  ret_specifiers: bool,
2424
- ) -> str | tuple[str, dict[str, Any]]:
2425
- ...
2412
+ ) -> str | tuple[str, dict[str, Any]]: ...
2426
2413
 
2427
2414
  @classmethod
2428
2415
  def get_snippet_script_str(
@@ -2474,8 +2461,7 @@ class Action(JSONLike):
2474
2461
  env_spec: Mapping[str, Any] | None = None,
2475
2462
  *,
2476
2463
  ret_specifiers: Literal[True],
2477
- ) -> tuple[Path, dict[str, Any]] | None:
2478
- ...
2464
+ ) -> tuple[Path, dict[str, Any]] | None: ...
2479
2465
 
2480
2466
  @classmethod
2481
2467
  @overload
@@ -2485,8 +2471,7 @@ class Action(JSONLike):
2485
2471
  env_spec: Mapping[str, Any] | None = None,
2486
2472
  *,
2487
2473
  ret_specifiers: Literal[False] = False,
2488
- ) -> Path | None:
2489
- ...
2474
+ ) -> Path | None: ...
2490
2475
 
2491
2476
  @classmethod
2492
2477
  def get_snippet_script_path(
@@ -1,6 +1,7 @@
1
1
  """
2
2
  Utility base class for making classes aware of the overall application context.
3
3
  """
4
+
4
5
  from __future__ import annotations
5
6
  from typing import TYPE_CHECKING
6
7
 
hpcflow/sdk/core/cache.py CHANGED
@@ -115,9 +115,9 @@ class ObjectCache:
115
115
  num_runs = workflow.num_EARs
116
116
 
117
117
  all_store_runs: Sequence[StoreEAR] = workflow._store.get_EARs(range(num_runs))
118
- all_store_iters: Sequence[
119
- StoreElementIter
120
- ] = workflow._store.get_element_iterations(range(num_iters))
118
+ all_store_iters: Sequence[StoreElementIter] = (
119
+ workflow._store.get_element_iterations(range(num_iters))
120
+ )
121
121
  all_store_elements: Sequence[StoreElement] = workflow._store.get_elements(
122
122
  range(num_elems)
123
123
  )
@@ -585,12 +585,10 @@ class _FileContentsSpecifier(JSONLike):
585
585
  return (self.normalised_path, [data_ref], is_new)
586
586
 
587
587
  @overload
588
- def _get_value(self, value_name: None = None) -> dict[str, Any]:
589
- ...
588
+ def _get_value(self, value_name: None = None) -> dict[str, Any]: ...
590
589
 
591
590
  @overload
592
- def _get_value(self, value_name: str) -> Any:
593
- ...
591
+ def _get_value(self, value_name: str) -> Any: ...
594
592
 
595
593
  def _get_value(self, value_name: str | None = None) -> Any:
596
594
  # TODO: fix