hpcflow-new2 0.2.0a210__py3-none-any.whl → 0.2.0a212__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/sdk/app.py +15 -24
- hpcflow/sdk/config/callbacks.py +5 -10
- hpcflow/sdk/config/config.py +11 -23
- hpcflow/sdk/core/actions.py +25 -40
- hpcflow/sdk/core/app_aware.py +1 -0
- hpcflow/sdk/core/cache.py +3 -3
- hpcflow/sdk/core/command_files.py +2 -4
- hpcflow/sdk/core/element.py +40 -72
- hpcflow/sdk/core/enums.py +1 -0
- hpcflow/sdk/core/json_like.py +18 -30
- hpcflow/sdk/core/object_list.py +14 -21
- hpcflow/sdk/core/parameters.py +24 -10
- hpcflow/sdk/core/task.py +18 -32
- hpcflow/sdk/core/task_schema.py +1 -1
- hpcflow/sdk/core/types.py +1 -2
- hpcflow/sdk/core/utils.py +10 -17
- hpcflow/sdk/core/validation.py +11 -22
- hpcflow/sdk/core/workflow.py +15 -23
- hpcflow/sdk/persistence/base.py +68 -116
- hpcflow/sdk/persistence/discovery.py +1 -0
- hpcflow/sdk/persistence/zarr.py +12 -12
- hpcflow/sdk/submission/enums.py +1 -0
- hpcflow/sdk/submission/jobscript.py +8 -10
- hpcflow/sdk/submission/schedulers/direct.py +2 -4
- hpcflow/sdk/submission/shells/__init__.py +1 -0
- hpcflow/sdk/submission/submission.py +11 -13
- hpcflow/sdk/utils/arrays.py +2 -4
- hpcflow/tests/unit/test_multi_path_sequences.py +23 -0
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/RECORD +34 -34
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/core/task.py
CHANGED
@@ -629,12 +629,10 @@ class ElementSet(JSONLike):
|
|
629
629
|
return [it.id_ for it in self.element_iterations]
|
630
630
|
|
631
631
|
@overload
|
632
|
-
def get_task_dependencies(self, as_objects: Literal[False] = False) -> set[int]:
|
633
|
-
...
|
632
|
+
def get_task_dependencies(self, as_objects: Literal[False] = False) -> set[int]: ...
|
634
633
|
|
635
634
|
@overload
|
636
|
-
def get_task_dependencies(self, as_objects: Literal[True]) -> list[WorkflowTask]:
|
637
|
-
...
|
635
|
+
def get_task_dependencies(self, as_objects: Literal[True]) -> list[WorkflowTask]: ...
|
638
636
|
|
639
637
|
def get_task_dependencies(
|
640
638
|
self, as_objects: bool = False
|
@@ -2583,8 +2581,7 @@ class WorkflowTask(AppAware):
|
|
2583
2581
|
| None
|
2584
2582
|
) = None,
|
2585
2583
|
return_indices: Literal[True],
|
2586
|
-
) -> list[int]:
|
2587
|
-
...
|
2584
|
+
) -> list[int]: ...
|
2588
2585
|
|
2589
2586
|
@overload
|
2590
2587
|
def add_elements(
|
@@ -2606,8 +2603,7 @@ class WorkflowTask(AppAware):
|
|
2606
2603
|
| None
|
2607
2604
|
) = None,
|
2608
2605
|
return_indices: Literal[False] = False,
|
2609
|
-
) -> None:
|
2610
|
-
...
|
2606
|
+
) -> None: ...
|
2611
2607
|
|
2612
2608
|
def add_elements(
|
2613
2609
|
self,
|
@@ -2763,15 +2759,13 @@ class WorkflowTask(AppAware):
|
|
2763
2759
|
def get_element_dependencies(
|
2764
2760
|
self,
|
2765
2761
|
as_objects: Literal[False] = False,
|
2766
|
-
) -> set[int]:
|
2767
|
-
...
|
2762
|
+
) -> set[int]: ...
|
2768
2763
|
|
2769
2764
|
@overload
|
2770
2765
|
def get_element_dependencies(
|
2771
2766
|
self,
|
2772
2767
|
as_objects: Literal[True],
|
2773
|
-
) -> list[Element]:
|
2774
|
-
...
|
2768
|
+
) -> list[Element]: ...
|
2775
2769
|
|
2776
2770
|
def get_element_dependencies(
|
2777
2771
|
self,
|
@@ -2793,12 +2787,10 @@ class WorkflowTask(AppAware):
|
|
2793
2787
|
return deps
|
2794
2788
|
|
2795
2789
|
@overload
|
2796
|
-
def get_task_dependencies(self, as_objects: Literal[False] = False) -> set[int]:
|
2797
|
-
...
|
2790
|
+
def get_task_dependencies(self, as_objects: Literal[False] = False) -> set[int]: ...
|
2798
2791
|
|
2799
2792
|
@overload
|
2800
|
-
def get_task_dependencies(self, as_objects: Literal[True]) -> list[WorkflowTask]:
|
2801
|
-
...
|
2793
|
+
def get_task_dependencies(self, as_objects: Literal[True]) -> list[WorkflowTask]: ...
|
2802
2794
|
|
2803
2795
|
def get_task_dependencies(
|
2804
2796
|
self,
|
@@ -2833,12 +2825,10 @@ class WorkflowTask(AppAware):
|
|
2833
2825
|
def get_dependent_elements(
|
2834
2826
|
self,
|
2835
2827
|
as_objects: Literal[False] = False,
|
2836
|
-
) -> set[int]:
|
2837
|
-
...
|
2828
|
+
) -> set[int]: ...
|
2838
2829
|
|
2839
2830
|
@overload
|
2840
|
-
def get_dependent_elements(self, as_objects: Literal[True]) -> list[Element]:
|
2841
|
-
...
|
2831
|
+
def get_dependent_elements(self, as_objects: Literal[True]) -> list[Element]: ...
|
2842
2832
|
|
2843
2833
|
def get_dependent_elements(
|
2844
2834
|
self,
|
@@ -2861,12 +2851,10 @@ class WorkflowTask(AppAware):
|
|
2861
2851
|
return deps
|
2862
2852
|
|
2863
2853
|
@overload
|
2864
|
-
def get_dependent_tasks(self, as_objects: Literal[False] = False) -> set[int]:
|
2865
|
-
...
|
2854
|
+
def get_dependent_tasks(self, as_objects: Literal[False] = False) -> set[int]: ...
|
2866
2855
|
|
2867
2856
|
@overload
|
2868
|
-
def get_dependent_tasks(self, as_objects: Literal[True]) -> list[WorkflowTask]:
|
2869
|
-
...
|
2857
|
+
def get_dependent_tasks(self, as_objects: Literal[True]) -> list[WorkflowTask]: ...
|
2870
2858
|
|
2871
2859
|
@TimeIt.decorator
|
2872
2860
|
def get_dependent_tasks(
|
@@ -3436,15 +3424,13 @@ class Elements:
|
|
3436
3424
|
def __getitem__(
|
3437
3425
|
self,
|
3438
3426
|
selection: int,
|
3439
|
-
) -> Element:
|
3440
|
-
...
|
3427
|
+
) -> Element: ...
|
3441
3428
|
|
3442
3429
|
@overload
|
3443
3430
|
def __getitem__(
|
3444
3431
|
self,
|
3445
3432
|
selection: slice | list[int],
|
3446
|
-
) -> list[Element]:
|
3447
|
-
...
|
3433
|
+
) -> list[Element]: ...
|
3448
3434
|
|
3449
3435
|
@TimeIt.decorator
|
3450
3436
|
def __getitem__(
|
@@ -3519,12 +3505,12 @@ class Parameters(AppAware):
|
|
3519
3505
|
yield from self.__getitem__(slice(None))
|
3520
3506
|
|
3521
3507
|
@overload
|
3522
|
-
def __getitem__(self, selection: int) -> Any | ElementParameter:
|
3523
|
-
...
|
3508
|
+
def __getitem__(self, selection: int) -> Any | ElementParameter: ...
|
3524
3509
|
|
3525
3510
|
@overload
|
3526
|
-
def __getitem__(
|
3527
|
-
|
3511
|
+
def __getitem__(
|
3512
|
+
self, selection: slice | list[int]
|
3513
|
+
) -> list[Any | ElementParameter]: ...
|
3528
3514
|
|
3529
3515
|
def __getitem__(
|
3530
3516
|
self,
|
hpcflow/sdk/core/task_schema.py
CHANGED
@@ -408,7 +408,7 @@ class TaskSchema(JSONLike):
|
|
408
408
|
return type_fmt
|
409
409
|
|
410
410
|
def _prepare_script_data_format_table(
|
411
|
-
script_data_grouped: Mapping[str, Mapping[str, Mapping[str, str]]]
|
411
|
+
script_data_grouped: Mapping[str, Mapping[str, Mapping[str, str]]],
|
412
412
|
) -> str:
|
413
413
|
out = ""
|
414
414
|
rows = ""
|
hpcflow/sdk/core/types.py
CHANGED
@@ -422,8 +422,7 @@ class ResourcePersistingWorkflow(Protocol):
|
|
422
422
|
persisting resources.
|
423
423
|
"""
|
424
424
|
|
425
|
-
def _add_parameter_data(self, data: Any, source: ParamSource) -> int:
|
426
|
-
...
|
425
|
+
def _add_parameter_data(self, data: Any, source: ParamSource) -> int: ...
|
427
426
|
|
428
427
|
def check_parameters_exist(self, id_lst: int | list[int]) -> bool:
|
429
428
|
"""
|
hpcflow/sdk/core/utils.py
CHANGED
@@ -132,13 +132,11 @@ def check_valid_py_identifier(name: str) -> str:
|
|
132
132
|
@overload
|
133
133
|
def group_by_dict_key_values( # type: ignore[overload-overlap]
|
134
134
|
lst: list[dict[T, T2]], key: T
|
135
|
-
) -> list[list[dict[T, T2]]]:
|
136
|
-
...
|
135
|
+
) -> list[list[dict[T, T2]]]: ...
|
137
136
|
|
138
137
|
|
139
138
|
@overload
|
140
|
-
def group_by_dict_key_values(lst: list[TD], key: str) -> list[list[TD]]:
|
141
|
-
...
|
139
|
+
def group_by_dict_key_values(lst: list[TD], key: str) -> list[list[TD]]: ...
|
142
140
|
|
143
141
|
|
144
142
|
def group_by_dict_key_values(lst: list, key):
|
@@ -763,22 +761,19 @@ def reshape(lst: Sequence[T], lens: Sequence[Sequence[int]]) -> list[TList[T]]:
|
|
763
761
|
@overload
|
764
762
|
def remap(
|
765
763
|
lst: list[int], mapping_func: Callable[[Sequence[int]], Sequence[T]]
|
766
|
-
) -> list[T]:
|
767
|
-
...
|
764
|
+
) -> list[T]: ...
|
768
765
|
|
769
766
|
|
770
767
|
@overload
|
771
768
|
def remap(
|
772
769
|
lst: list[list[int]], mapping_func: Callable[[Sequence[int]], Sequence[T]]
|
773
|
-
) -> list[list[T]]:
|
774
|
-
...
|
770
|
+
) -> list[list[T]]: ...
|
775
771
|
|
776
772
|
|
777
773
|
@overload
|
778
774
|
def remap(
|
779
775
|
lst: list[list[list[int]]], mapping_func: Callable[[Sequence[int]], Sequence[T]]
|
780
|
-
) -> list[list[list[T]]]:
|
781
|
-
...
|
776
|
+
) -> list[list[list[T]]]: ...
|
782
777
|
|
783
778
|
|
784
779
|
def remap(lst, mapping_func):
|
@@ -890,13 +885,11 @@ def open_file(filename: str | Path):
|
|
890
885
|
|
891
886
|
|
892
887
|
@overload
|
893
|
-
def get_enum_by_name_or_val(enum_cls: type[E], key: None) -> None:
|
894
|
-
...
|
888
|
+
def get_enum_by_name_or_val(enum_cls: type[E], key: None) -> None: ...
|
895
889
|
|
896
890
|
|
897
891
|
@overload
|
898
|
-
def get_enum_by_name_or_val(enum_cls: type[E], key: str | int | float | E) -> E:
|
899
|
-
...
|
892
|
+
def get_enum_by_name_or_val(enum_cls: type[E], key: str | int | float | E) -> E: ...
|
900
893
|
|
901
894
|
|
902
895
|
def get_enum_by_name_or_val(
|
@@ -1028,9 +1021,9 @@ def dict_values_process_flat(
|
|
1028
1021
|
|
1029
1022
|
"""
|
1030
1023
|
flat: list[T2] = [] # values of `d`, flattened
|
1031
|
-
is_multi: list[
|
1032
|
-
|
1033
|
-
|
1024
|
+
is_multi: list[tuple[bool, int]] = (
|
1025
|
+
[]
|
1026
|
+
) # whether a list, and the number of items to process
|
1034
1027
|
for i in d.values():
|
1035
1028
|
if isinstance(i, list):
|
1036
1029
|
flat.extend(cast("list[T2]", i))
|
hpcflow/sdk/core/validation.py
CHANGED
@@ -17,11 +17,9 @@ class ValidatedData(Protocol, Generic[T]):
|
|
17
17
|
"""
|
18
18
|
|
19
19
|
@property
|
20
|
-
def is_valid(self) -> bool:
|
21
|
-
...
|
20
|
+
def is_valid(self) -> bool: ...
|
22
21
|
|
23
|
-
def get_failures_string(self) -> str:
|
24
|
-
...
|
22
|
+
def get_failures_string(self) -> str: ...
|
25
23
|
|
26
24
|
cast_data: T
|
27
25
|
|
@@ -32,12 +30,10 @@ class PreparedConditionCallable(Protocol):
|
|
32
30
|
"""
|
33
31
|
|
34
32
|
@property
|
35
|
-
def name(self) -> str:
|
36
|
-
...
|
33
|
+
def name(self) -> str: ...
|
37
34
|
|
38
35
|
@property
|
39
|
-
def args(self) -> tuple[str, ...]:
|
40
|
-
...
|
36
|
+
def args(self) -> tuple[str, ...]: ...
|
41
37
|
|
42
38
|
|
43
39
|
class Condition(Protocol):
|
@@ -46,8 +42,7 @@ class Condition(Protocol):
|
|
46
42
|
"""
|
47
43
|
|
48
44
|
@property
|
49
|
-
def callable(self) -> PreparedConditionCallable:
|
50
|
-
...
|
45
|
+
def callable(self) -> PreparedConditionCallable: ...
|
51
46
|
|
52
47
|
|
53
48
|
class Rule(Protocol):
|
@@ -56,12 +51,10 @@ class Rule(Protocol):
|
|
56
51
|
"""
|
57
52
|
|
58
53
|
@property
|
59
|
-
def condition(self) -> Condition:
|
60
|
-
...
|
54
|
+
def condition(self) -> Condition: ...
|
61
55
|
|
62
56
|
@property
|
63
|
-
def path(self) -> object:
|
64
|
-
...
|
57
|
+
def path(self) -> object: ...
|
65
58
|
|
66
59
|
|
67
60
|
class Schema(Protocol):
|
@@ -69,18 +62,14 @@ class Schema(Protocol):
|
|
69
62
|
Typed profile of ``valida.Schema``.
|
70
63
|
"""
|
71
64
|
|
72
|
-
def validate(self, data: T) -> ValidatedData[T]:
|
73
|
-
...
|
65
|
+
def validate(self, data: T) -> ValidatedData[T]: ...
|
74
66
|
|
75
67
|
@property
|
76
|
-
def rules(self) -> Sequence[Rule]:
|
77
|
-
...
|
68
|
+
def rules(self) -> Sequence[Rule]: ...
|
78
69
|
|
79
|
-
def add_schema(self, schema: Schema, root_path: Any = None) -> None:
|
80
|
-
...
|
70
|
+
def add_schema(self, schema: Schema, root_path: Any = None) -> None: ...
|
81
71
|
|
82
|
-
def to_tree(self, **kwargs) -> Sequence[Mapping[str, str]]:
|
83
|
-
...
|
72
|
+
def to_tree(self, **kwargs) -> Sequence[Mapping[str, str]]: ...
|
84
73
|
|
85
74
|
|
86
75
|
def get_schema(filename) -> Schema:
|
hpcflow/sdk/core/workflow.py
CHANGED
@@ -1840,12 +1840,10 @@ class Workflow(AppAware):
|
|
1840
1840
|
task: int
|
1841
1841
|
|
1842
1842
|
@overload
|
1843
|
-
def get_EARs_from_IDs(self, ids: Iterable[int]) -> list[ElementActionRun]:
|
1844
|
-
...
|
1843
|
+
def get_EARs_from_IDs(self, ids: Iterable[int]) -> list[ElementActionRun]: ...
|
1845
1844
|
|
1846
1845
|
@overload
|
1847
|
-
def get_EARs_from_IDs(self, ids: int) -> ElementActionRun:
|
1848
|
-
...
|
1846
|
+
def get_EARs_from_IDs(self, ids: int) -> ElementActionRun: ...
|
1849
1847
|
|
1850
1848
|
@TimeIt.decorator
|
1851
1849
|
def get_EARs_from_IDs(
|
@@ -2393,12 +2391,12 @@ class Workflow(AppAware):
|
|
2393
2391
|
@overload
|
2394
2392
|
def get_task_unique_names(
|
2395
2393
|
self, map_to_insert_ID: Literal[False] = False
|
2396
|
-
) -> Sequence[str]:
|
2397
|
-
...
|
2394
|
+
) -> Sequence[str]: ...
|
2398
2395
|
|
2399
2396
|
@overload
|
2400
|
-
def get_task_unique_names(
|
2401
|
-
|
2397
|
+
def get_task_unique_names(
|
2398
|
+
self, map_to_insert_ID: Literal[True]
|
2399
|
+
) -> Mapping[str, int]: ...
|
2402
2400
|
|
2403
2401
|
def get_task_unique_names(
|
2404
2402
|
self, map_to_insert_ID: bool = False
|
@@ -3072,28 +3070,24 @@ class Workflow(AppAware):
|
|
3072
3070
|
*,
|
3073
3071
|
ret_iter_IDs: Literal[False] = False,
|
3074
3072
|
ret_data_idx: Literal[False] = False,
|
3075
|
-
) -> Sequence[tuple[int, LoopIndex[str, int]]]:
|
3076
|
-
...
|
3073
|
+
) -> Sequence[tuple[int, LoopIndex[str, int]]]: ...
|
3077
3074
|
|
3078
3075
|
@overload
|
3079
3076
|
def get_iteration_task_pathway(
|
3080
3077
|
self, *, ret_iter_IDs: Literal[False] = False, ret_data_idx: Literal[True]
|
3081
|
-
) -> Sequence[tuple[int, LoopIndex[str, int], tuple[Mapping[str, int], ...]]]:
|
3082
|
-
...
|
3078
|
+
) -> Sequence[tuple[int, LoopIndex[str, int], tuple[Mapping[str, int], ...]]]: ...
|
3083
3079
|
|
3084
3080
|
@overload
|
3085
3081
|
def get_iteration_task_pathway(
|
3086
3082
|
self, *, ret_iter_IDs: Literal[True], ret_data_idx: Literal[False] = False
|
3087
|
-
) -> Sequence[tuple[int, LoopIndex[str, int], tuple[int, ...]]]:
|
3088
|
-
...
|
3083
|
+
) -> Sequence[tuple[int, LoopIndex[str, int], tuple[int, ...]]]: ...
|
3089
3084
|
|
3090
3085
|
@overload
|
3091
3086
|
def get_iteration_task_pathway(
|
3092
3087
|
self, *, ret_iter_IDs: Literal[True], ret_data_idx: Literal[True]
|
3093
3088
|
) -> Sequence[
|
3094
3089
|
tuple[int, LoopIndex[str, int], tuple[int, ...], tuple[Mapping[str, int], ...]]
|
3095
|
-
]:
|
3096
|
-
...
|
3090
|
+
]: ...
|
3097
3091
|
|
3098
3092
|
@TimeIt.decorator
|
3099
3093
|
def get_iteration_task_pathway(
|
@@ -3242,8 +3236,7 @@ class Workflow(AppAware):
|
|
3242
3236
|
tasks: list[int] | None = None,
|
3243
3237
|
cancel: bool = False,
|
3244
3238
|
status: bool = True,
|
3245
|
-
) -> Mapping[int, Sequence[int]]:
|
3246
|
-
...
|
3239
|
+
) -> Mapping[int, Sequence[int]]: ...
|
3247
3240
|
|
3248
3241
|
@overload
|
3249
3242
|
def submit(
|
@@ -3258,8 +3251,7 @@ class Workflow(AppAware):
|
|
3258
3251
|
tasks: list[int] | None = None,
|
3259
3252
|
cancel: bool = False,
|
3260
3253
|
status: bool = True,
|
3261
|
-
) -> None:
|
3262
|
-
...
|
3254
|
+
) -> None: ...
|
3263
3255
|
|
3264
3256
|
def submit(
|
3265
3257
|
self,
|
@@ -4339,9 +4331,9 @@ class Workflow(AppAware):
|
|
4339
4331
|
# a fill value means no sub directory should be created
|
4340
4332
|
T_FILL, E_FILL, I_FILL, A_FILL, R_FILL, _, _ = RUN_DIR_ARR_FILL
|
4341
4333
|
|
4342
|
-
depth_idx_cache: dict[
|
4343
|
-
|
4344
|
-
|
4334
|
+
depth_idx_cache: dict[tuple[int, int], NDArray] = (
|
4335
|
+
{}
|
4336
|
+
) # keys are (max_avail, tot_elems_per_dir_level)
|
4345
4337
|
|
4346
4338
|
# format run directories:
|
4347
4339
|
dirs = []
|