hpcflow-new2 0.2.0a210__py3-none-any.whl → 0.2.0a212__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/_version.py +1 -1
- hpcflow/sdk/app.py +15 -24
- hpcflow/sdk/config/callbacks.py +5 -10
- hpcflow/sdk/config/config.py +11 -23
- hpcflow/sdk/core/actions.py +25 -40
- hpcflow/sdk/core/app_aware.py +1 -0
- hpcflow/sdk/core/cache.py +3 -3
- hpcflow/sdk/core/command_files.py +2 -4
- hpcflow/sdk/core/element.py +40 -72
- hpcflow/sdk/core/enums.py +1 -0
- hpcflow/sdk/core/json_like.py +18 -30
- hpcflow/sdk/core/object_list.py +14 -21
- hpcflow/sdk/core/parameters.py +24 -10
- hpcflow/sdk/core/task.py +18 -32
- hpcflow/sdk/core/task_schema.py +1 -1
- hpcflow/sdk/core/types.py +1 -2
- hpcflow/sdk/core/utils.py +10 -17
- hpcflow/sdk/core/validation.py +11 -22
- hpcflow/sdk/core/workflow.py +15 -23
- hpcflow/sdk/persistence/base.py +68 -116
- hpcflow/sdk/persistence/discovery.py +1 -0
- hpcflow/sdk/persistence/zarr.py +12 -12
- hpcflow/sdk/submission/enums.py +1 -0
- hpcflow/sdk/submission/jobscript.py +8 -10
- hpcflow/sdk/submission/schedulers/direct.py +2 -4
- hpcflow/sdk/submission/shells/__init__.py +1 -0
- hpcflow/sdk/submission/submission.py +11 -13
- hpcflow/sdk/utils/arrays.py +2 -4
- hpcflow/tests/unit/test_multi_path_sequences.py +23 -0
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/METADATA +1 -1
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/RECORD +34 -34
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a210.dist-info → hpcflow_new2-0.2.0a212.dist-info}/entry_points.txt +0 -0
hpcflow/sdk/persistence/base.py
CHANGED
@@ -954,28 +954,23 @@ class PersistentStore(
|
|
954
954
|
|
955
955
|
@classmethod
|
956
956
|
@abstractmethod
|
957
|
-
def _store_task_cls(cls) -> type[AnySTask]:
|
958
|
-
...
|
957
|
+
def _store_task_cls(cls) -> type[AnySTask]: ...
|
959
958
|
|
960
959
|
@classmethod
|
961
960
|
@abstractmethod
|
962
|
-
def _store_elem_cls(cls) -> type[AnySElement]:
|
963
|
-
...
|
961
|
+
def _store_elem_cls(cls) -> type[AnySElement]: ...
|
964
962
|
|
965
963
|
@classmethod
|
966
964
|
@abstractmethod
|
967
|
-
def _store_iter_cls(cls) -> type[AnySElementIter]:
|
968
|
-
...
|
965
|
+
def _store_iter_cls(cls) -> type[AnySElementIter]: ...
|
969
966
|
|
970
967
|
@classmethod
|
971
968
|
@abstractmethod
|
972
|
-
def _store_EAR_cls(cls) -> type[AnySEAR]:
|
973
|
-
...
|
969
|
+
def _store_EAR_cls(cls) -> type[AnySEAR]: ...
|
974
970
|
|
975
971
|
@classmethod
|
976
972
|
@abstractmethod
|
977
|
-
def _store_param_cls(cls) -> type[AnySParameter]:
|
978
|
-
...
|
973
|
+
def _store_param_cls(cls) -> type[AnySParameter]: ...
|
979
974
|
|
980
975
|
_resources: dict[str, StoreResource]
|
981
976
|
_features: ClassVar[PersistentStoreFeatures]
|
@@ -1072,8 +1067,7 @@ class PersistentStore(
|
|
1072
1067
|
chunk_size: int | None = None,
|
1073
1068
|
backup: bool = True,
|
1074
1069
|
status: bool = True,
|
1075
|
-
) -> Any:
|
1076
|
-
...
|
1070
|
+
) -> Any: ...
|
1077
1071
|
|
1078
1072
|
@abstractmethod
|
1079
1073
|
def rechunk_runs(
|
@@ -1081,8 +1075,7 @@ class PersistentStore(
|
|
1081
1075
|
chunk_size: int | None = None,
|
1082
1076
|
backup: bool = True,
|
1083
1077
|
status: bool = True,
|
1084
|
-
) -> Any:
|
1085
|
-
...
|
1078
|
+
) -> Any: ...
|
1086
1079
|
|
1087
1080
|
@abstractmethod
|
1088
1081
|
def get_dirs_array(self) -> NDArray:
|
@@ -1361,48 +1354,42 @@ class PersistentStore(
|
|
1361
1354
|
return _rename_path(str(replaced), str(original))
|
1362
1355
|
|
1363
1356
|
@abstractmethod
|
1364
|
-
def _get_num_persistent_tasks(self) -> int:
|
1365
|
-
...
|
1357
|
+
def _get_num_persistent_tasks(self) -> int: ...
|
1366
1358
|
|
1367
1359
|
def _get_num_total_tasks(self) -> int:
|
1368
1360
|
"""Get the total number of persistent and pending tasks."""
|
1369
1361
|
return self._get_num_persistent_tasks() + len(self._pending.add_tasks)
|
1370
1362
|
|
1371
1363
|
@abstractmethod
|
1372
|
-
def _get_num_persistent_loops(self) -> int:
|
1373
|
-
...
|
1364
|
+
def _get_num_persistent_loops(self) -> int: ...
|
1374
1365
|
|
1375
1366
|
def _get_num_total_loops(self) -> int:
|
1376
1367
|
"""Get the total number of persistent and pending loops."""
|
1377
1368
|
return self._get_num_persistent_loops() + len(self._pending.add_loops)
|
1378
1369
|
|
1379
1370
|
@abstractmethod
|
1380
|
-
def _get_num_persistent_submissions(self) -> int:
|
1381
|
-
...
|
1371
|
+
def _get_num_persistent_submissions(self) -> int: ...
|
1382
1372
|
|
1383
1373
|
def _get_num_total_submissions(self) -> int:
|
1384
1374
|
"""Get the total number of persistent and pending submissions."""
|
1385
1375
|
return self._get_num_persistent_submissions() + len(self._pending.add_submissions)
|
1386
1376
|
|
1387
1377
|
@abstractmethod
|
1388
|
-
def _get_num_persistent_elements(self) -> int:
|
1389
|
-
...
|
1378
|
+
def _get_num_persistent_elements(self) -> int: ...
|
1390
1379
|
|
1391
1380
|
def _get_num_total_elements(self) -> int:
|
1392
1381
|
"""Get the total number of persistent and pending elements."""
|
1393
1382
|
return self._get_num_persistent_elements() + len(self._pending.add_elements)
|
1394
1383
|
|
1395
1384
|
@abstractmethod
|
1396
|
-
def _get_num_persistent_elem_iters(self) -> int:
|
1397
|
-
...
|
1385
|
+
def _get_num_persistent_elem_iters(self) -> int: ...
|
1398
1386
|
|
1399
1387
|
def _get_num_total_elem_iters(self) -> int:
|
1400
1388
|
"""Get the total number of persistent and pending element iterations."""
|
1401
1389
|
return self._get_num_persistent_elem_iters() + len(self._pending.add_elem_iters)
|
1402
1390
|
|
1403
1391
|
@abstractmethod
|
1404
|
-
def _get_num_persistent_EARs(self) -> int:
|
1405
|
-
...
|
1392
|
+
def _get_num_persistent_EARs(self) -> int: ...
|
1406
1393
|
|
1407
1394
|
@TimeIt.decorator
|
1408
1395
|
def _get_num_total_EARs(self) -> int:
|
@@ -1414,8 +1401,7 @@ class PersistentStore(
|
|
1414
1401
|
return len(self.get_task(task_ID).element_IDs)
|
1415
1402
|
|
1416
1403
|
@abstractmethod
|
1417
|
-
def _get_num_persistent_parameters(self) -> int:
|
1418
|
-
...
|
1404
|
+
def _get_num_persistent_parameters(self) -> int: ...
|
1419
1405
|
|
1420
1406
|
def _get_num_total_parameters(self) -> int:
|
1421
1407
|
"""Get the total number of persistent and pending parameters."""
|
@@ -1428,8 +1414,7 @@ class PersistentStore(
|
|
1428
1414
|
)
|
1429
1415
|
|
1430
1416
|
@abstractmethod
|
1431
|
-
def _get_num_persistent_added_tasks(self) -> int:
|
1432
|
-
...
|
1417
|
+
def _get_num_persistent_added_tasks(self) -> int: ...
|
1433
1418
|
|
1434
1419
|
def _get_num_total_added_tasks(self) -> int:
|
1435
1420
|
"""Get the total number of tasks ever added to the workflow."""
|
@@ -1945,8 +1930,7 @@ class PersistentStore(
|
|
1945
1930
|
return self._add_parameter(data=None, is_set=False, source=source, save=save)
|
1946
1931
|
|
1947
1932
|
@abstractmethod
|
1948
|
-
def _set_parameter_values(self, set_parameters: dict[int, tuple[Any, bool]]):
|
1949
|
-
...
|
1933
|
+
def _set_parameter_values(self, set_parameters: dict[int, tuple[Any, bool]]): ...
|
1950
1934
|
|
1951
1935
|
@writes_parameter_data
|
1952
1936
|
def set_parameter_value(
|
@@ -2045,8 +2029,7 @@ class PersistentStore(
|
|
2045
2029
|
return tc
|
2046
2030
|
|
2047
2031
|
@abstractmethod
|
2048
|
-
def _get_persistent_template_components(self) -> dict[str, Any]:
|
2049
|
-
...
|
2032
|
+
def _get_persistent_template_components(self) -> dict[str, Any]: ...
|
2050
2033
|
|
2051
2034
|
def get_template(self) -> dict[str, JSONed]:
|
2052
2035
|
"""
|
@@ -2055,8 +2038,7 @@ class PersistentStore(
|
|
2055
2038
|
return self._get_persistent_template()
|
2056
2039
|
|
2057
2040
|
@abstractmethod
|
2058
|
-
def _get_persistent_template(self) -> dict[str, JSONed]:
|
2059
|
-
...
|
2041
|
+
def _get_persistent_template(self) -> dict[str, JSONed]: ...
|
2060
2042
|
|
2061
2043
|
def _get_task_id_to_idx_map(self) -> dict[int, int]:
|
2062
2044
|
return {task.id_: task.index for task in self.get_tasks()}
|
@@ -2107,8 +2089,7 @@ class PersistentStore(
|
|
2107
2089
|
return id_all, id_pers, id_pend
|
2108
2090
|
|
2109
2091
|
@abstractmethod
|
2110
|
-
def _get_persistent_tasks(self, id_lst: Iterable[int]) -> dict[int, AnySTask]:
|
2111
|
-
...
|
2092
|
+
def _get_persistent_tasks(self, id_lst: Iterable[int]) -> dict[int, AnySTask]: ...
|
2112
2093
|
|
2113
2094
|
def get_tasks_by_IDs(self, ids: Iterable[int]) -> Sequence[AnySTask]:
|
2114
2095
|
"""
|
@@ -2137,8 +2118,7 @@ class PersistentStore(
|
|
2137
2118
|
@abstractmethod
|
2138
2119
|
def _get_persistent_loops(
|
2139
2120
|
self, id_lst: Iterable[int] | None = None
|
2140
|
-
) -> dict[int, LoopDescriptor]:
|
2141
|
-
...
|
2121
|
+
) -> dict[int, LoopDescriptor]: ...
|
2142
2122
|
|
2143
2123
|
def get_loops_by_IDs(self, ids: Iterable[int]) -> dict[int, LoopDescriptor]:
|
2144
2124
|
"""Retrieve loops by index (ID), including pending."""
|
@@ -2164,8 +2144,7 @@ class PersistentStore(
|
|
2164
2144
|
@abstractmethod
|
2165
2145
|
def _get_persistent_submissions(
|
2166
2146
|
self, id_lst: Iterable[int] | None = None
|
2167
|
-
) -> dict[int, Mapping[str, JSONed]]:
|
2168
|
-
...
|
2147
|
+
) -> dict[int, Mapping[str, JSONed]]: ...
|
2169
2148
|
|
2170
2149
|
@TimeIt.decorator
|
2171
2150
|
def get_submissions(self) -> dict[int, Mapping[str, JSONed]]:
|
@@ -2297,8 +2276,9 @@ class PersistentStore(
|
|
2297
2276
|
return dict(sorted(subs.items()))
|
2298
2277
|
|
2299
2278
|
@abstractmethod
|
2300
|
-
def _get_persistent_elements(
|
2301
|
-
|
2279
|
+
def _get_persistent_elements(
|
2280
|
+
self, id_lst: Iterable[int]
|
2281
|
+
) -> dict[int, AnySElement]: ...
|
2302
2282
|
|
2303
2283
|
@TimeIt.decorator
|
2304
2284
|
def get_elements(self, ids: Iterable[int]) -> Sequence[AnySElement]:
|
@@ -2328,8 +2308,7 @@ class PersistentStore(
|
|
2328
2308
|
@abstractmethod
|
2329
2309
|
def _get_persistent_element_iters(
|
2330
2310
|
self, id_lst: Iterable[int]
|
2331
|
-
) -> dict[int, AnySElementIter]:
|
2332
|
-
...
|
2311
|
+
) -> dict[int, AnySElementIter]: ...
|
2333
2312
|
|
2334
2313
|
@TimeIt.decorator
|
2335
2314
|
def get_element_iterations(self, ids: Iterable[int]) -> Sequence[AnySElementIter]:
|
@@ -2365,8 +2344,7 @@ class PersistentStore(
|
|
2365
2344
|
return iters_new
|
2366
2345
|
|
2367
2346
|
@abstractmethod
|
2368
|
-
def _get_persistent_EARs(self, id_lst: Iterable[int]) -> dict[int, AnySEAR]:
|
2369
|
-
...
|
2347
|
+
def _get_persistent_EARs(self, id_lst: Iterable[int]) -> dict[int, AnySEAR]: ...
|
2370
2348
|
|
2371
2349
|
@TimeIt.decorator
|
2372
2350
|
def get_EARs(self, ids: Iterable[int]) -> Sequence[AnySEAR]:
|
@@ -2491,8 +2469,7 @@ class PersistentStore(
|
|
2491
2469
|
@abstractmethod
|
2492
2470
|
def _get_persistent_parameters(
|
2493
2471
|
self, id_lst: Iterable[int], **kwargs
|
2494
|
-
) -> Mapping[int, AnySParameter]:
|
2495
|
-
...
|
2472
|
+
) -> Mapping[int, AnySParameter]: ...
|
2496
2473
|
|
2497
2474
|
@TimeIt.decorator
|
2498
2475
|
def get_parameter_set_statuses(self, ids: Iterable[int]) -> list[bool]:
|
@@ -2512,8 +2489,7 @@ class PersistentStore(
|
|
2512
2489
|
@abstractmethod
|
2513
2490
|
def _get_persistent_parameter_set_status(
|
2514
2491
|
self, id_lst: Iterable[int]
|
2515
|
-
) -> dict[int, bool]:
|
2516
|
-
...
|
2492
|
+
) -> dict[int, bool]: ...
|
2517
2493
|
|
2518
2494
|
@TimeIt.decorator
|
2519
2495
|
def get_parameter_sources(self, ids: Iterable[int]) -> list[ParamSource]:
|
@@ -2545,8 +2521,7 @@ class PersistentStore(
|
|
2545
2521
|
@abstractmethod
|
2546
2522
|
def _get_persistent_param_sources(
|
2547
2523
|
self, id_lst: Iterable[int]
|
2548
|
-
) -> dict[int, ParamSource]:
|
2549
|
-
...
|
2524
|
+
) -> dict[int, ParamSource]: ...
|
2550
2525
|
|
2551
2526
|
@TimeIt.decorator
|
2552
2527
|
def get_task_elements(
|
@@ -2591,8 +2566,7 @@ class PersistentStore(
|
|
2591
2566
|
yield element.to_dict(iters_rs[idx])
|
2592
2567
|
|
2593
2568
|
@abstractmethod
|
2594
|
-
def _get_persistent_parameter_IDs(self) -> Iterable[int]:
|
2595
|
-
...
|
2569
|
+
def _get_persistent_parameter_IDs(self) -> Iterable[int]: ...
|
2596
2570
|
|
2597
2571
|
def check_parameters_exist(self, ids: Sequence[int]) -> Iterator[bool]:
|
2598
2572
|
"""
|
@@ -2604,133 +2578,110 @@ class PersistentStore(
|
|
2604
2578
|
return (id_ not in id_miss for id_ in ids)
|
2605
2579
|
|
2606
2580
|
@abstractmethod
|
2607
|
-
def _append_tasks(self, tasks: Iterable[AnySTask]) -> None:
|
2608
|
-
...
|
2581
|
+
def _append_tasks(self, tasks: Iterable[AnySTask]) -> None: ...
|
2609
2582
|
|
2610
2583
|
@abstractmethod
|
2611
|
-
def _append_loops(self, loops: dict[int, LoopDescriptor]) -> None:
|
2612
|
-
...
|
2584
|
+
def _append_loops(self, loops: dict[int, LoopDescriptor]) -> None: ...
|
2613
2585
|
|
2614
2586
|
@abstractmethod
|
2615
|
-
def _append_submissions(self, subs: dict[int, Mapping[str, JSONed]]) -> None:
|
2616
|
-
...
|
2587
|
+
def _append_submissions(self, subs: dict[int, Mapping[str, JSONed]]) -> None: ...
|
2617
2588
|
|
2618
2589
|
@abstractmethod
|
2619
2590
|
def _update_at_submit_metadata(
|
2620
2591
|
self, at_submit_metadata: dict[int, dict[str, Any]]
|
2621
|
-
) -> None:
|
2622
|
-
...
|
2592
|
+
) -> None: ...
|
2623
2593
|
|
2624
2594
|
@abstractmethod
|
2625
|
-
def _append_elements(self, elems: Sequence[AnySElement]) -> None:
|
2626
|
-
...
|
2595
|
+
def _append_elements(self, elems: Sequence[AnySElement]) -> None: ...
|
2627
2596
|
|
2628
2597
|
@abstractmethod
|
2629
|
-
def _append_element_sets(self, task_id: int, es_js: Sequence[Mapping]) -> None:
|
2630
|
-
...
|
2598
|
+
def _append_element_sets(self, task_id: int, es_js: Sequence[Mapping]) -> None: ...
|
2631
2599
|
|
2632
2600
|
@abstractmethod
|
2633
|
-
def _append_elem_iter_IDs(self, elem_ID: int, iter_IDs: Iterable[int]) -> None:
|
2634
|
-
...
|
2601
|
+
def _append_elem_iter_IDs(self, elem_ID: int, iter_IDs: Iterable[int]) -> None: ...
|
2635
2602
|
|
2636
2603
|
@abstractmethod
|
2637
|
-
def _append_elem_iters(self, iters: Sequence[AnySElementIter]) -> None:
|
2638
|
-
...
|
2604
|
+
def _append_elem_iters(self, iters: Sequence[AnySElementIter]) -> None: ...
|
2639
2605
|
|
2640
2606
|
@abstractmethod
|
2641
2607
|
def _append_elem_iter_EAR_IDs(
|
2642
2608
|
self, iter_ID: int, act_idx: int, EAR_IDs: Sequence[int]
|
2643
|
-
) -> None:
|
2644
|
-
...
|
2609
|
+
) -> None: ...
|
2645
2610
|
|
2646
2611
|
@abstractmethod
|
2647
|
-
def _append_EARs(self, EARs: Sequence[AnySEAR]) -> None:
|
2648
|
-
...
|
2612
|
+
def _append_EARs(self, EARs: Sequence[AnySEAR]) -> None: ...
|
2649
2613
|
|
2650
2614
|
@abstractmethod
|
2651
|
-
def _update_elem_iter_EARs_initialised(self, iter_ID: int) -> None:
|
2652
|
-
...
|
2615
|
+
def _update_elem_iter_EARs_initialised(self, iter_ID: int) -> None: ...
|
2653
2616
|
|
2654
2617
|
@abstractmethod
|
2655
|
-
def _update_EAR_submission_data(
|
2656
|
-
|
2618
|
+
def _update_EAR_submission_data(
|
2619
|
+
self, sub_data: Mapping[int, tuple[int, int | None]]
|
2620
|
+
): ...
|
2657
2621
|
|
2658
2622
|
@abstractmethod
|
2659
2623
|
def _update_EAR_start(
|
2660
2624
|
self,
|
2661
2625
|
run_starts: dict[int, tuple[datetime, dict[str, Any] | None, str, int | None]],
|
2662
|
-
) -> None:
|
2663
|
-
...
|
2626
|
+
) -> None: ...
|
2664
2627
|
|
2665
2628
|
@abstractmethod
|
2666
2629
|
def _update_EAR_end(
|
2667
2630
|
self, run_ends: dict[int, tuple[datetime, dict[str, Any] | None, int, bool]]
|
2668
|
-
) -> None:
|
2669
|
-
...
|
2631
|
+
) -> None: ...
|
2670
2632
|
|
2671
2633
|
@abstractmethod
|
2672
|
-
def _update_EAR_skip(self, skips: dict[int, int]) -> None:
|
2673
|
-
...
|
2634
|
+
def _update_EAR_skip(self, skips: dict[int, int]) -> None: ...
|
2674
2635
|
|
2675
2636
|
@abstractmethod
|
2676
|
-
def _update_js_metadata(
|
2677
|
-
|
2637
|
+
def _update_js_metadata(
|
2638
|
+
self, js_meta: dict[int, dict[int, dict[str, Any]]]
|
2639
|
+
) -> None: ...
|
2678
2640
|
|
2679
2641
|
@abstractmethod
|
2680
|
-
def _append_parameters(self, params: Sequence[AnySParameter]) -> None:
|
2681
|
-
...
|
2642
|
+
def _append_parameters(self, params: Sequence[AnySParameter]) -> None: ...
|
2682
2643
|
|
2683
2644
|
@abstractmethod
|
2684
|
-
def _update_template_components(self, tc: dict[str, Any]) -> None:
|
2685
|
-
...
|
2645
|
+
def _update_template_components(self, tc: dict[str, Any]) -> None: ...
|
2686
2646
|
|
2687
2647
|
@abstractmethod
|
2688
|
-
def _update_parameter_sources(self, sources: Mapping[int, ParamSource]) -> None:
|
2689
|
-
...
|
2648
|
+
def _update_parameter_sources(self, sources: Mapping[int, ParamSource]) -> None: ...
|
2690
2649
|
|
2691
2650
|
@abstractmethod
|
2692
|
-
def _update_loop_index(self, loop_indices: dict[int, dict[str, int]]) -> None:
|
2693
|
-
...
|
2651
|
+
def _update_loop_index(self, loop_indices: dict[int, dict[str, int]]) -> None: ...
|
2694
2652
|
|
2695
2653
|
@abstractmethod
|
2696
2654
|
def _update_loop_num_iters(
|
2697
2655
|
self, index: int, num_iters: list[list[list[int] | int]]
|
2698
|
-
) -> None:
|
2699
|
-
...
|
2656
|
+
) -> None: ...
|
2700
2657
|
|
2701
2658
|
@abstractmethod
|
2702
|
-
def _update_loop_parents(self, index: int, parents: list[str]) -> None:
|
2703
|
-
...
|
2659
|
+
def _update_loop_parents(self, index: int, parents: list[str]) -> None: ...
|
2704
2660
|
|
2705
2661
|
@overload
|
2706
2662
|
def using_resource(
|
2707
2663
|
self, res_label: Literal["metadata"], action: str
|
2708
|
-
) -> AbstractContextManager[Metadata]:
|
2709
|
-
...
|
2664
|
+
) -> AbstractContextManager[Metadata]: ...
|
2710
2665
|
|
2711
2666
|
@overload
|
2712
2667
|
def using_resource(
|
2713
2668
|
self, res_label: Literal["submissions"], action: str
|
2714
|
-
) -> AbstractContextManager[list[dict[str, JSONed]]]:
|
2715
|
-
...
|
2669
|
+
) -> AbstractContextManager[list[dict[str, JSONed]]]: ...
|
2716
2670
|
|
2717
2671
|
@overload
|
2718
2672
|
def using_resource(
|
2719
2673
|
self, res_label: Literal["parameters"], action: str
|
2720
|
-
) -> AbstractContextManager[dict[str, dict[str, Any]]]:
|
2721
|
-
...
|
2674
|
+
) -> AbstractContextManager[dict[str, dict[str, Any]]]: ...
|
2722
2675
|
|
2723
2676
|
@overload
|
2724
2677
|
def using_resource(
|
2725
2678
|
self, res_label: Literal["runs"], action: str
|
2726
|
-
) -> AbstractContextManager[dict[str, Any]]:
|
2727
|
-
...
|
2679
|
+
) -> AbstractContextManager[dict[str, Any]]: ...
|
2728
2680
|
|
2729
2681
|
@overload
|
2730
2682
|
def using_resource(
|
2731
2683
|
self, res_label: Literal["attrs"], action: str
|
2732
|
-
) -> AbstractContextManager[ZarrAttrsDict]:
|
2733
|
-
...
|
2684
|
+
) -> AbstractContextManager[ZarrAttrsDict]: ...
|
2734
2685
|
|
2735
2686
|
@contextlib.contextmanager
|
2736
2687
|
def using_resource(
|
@@ -2830,13 +2781,14 @@ class PersistentStore(
|
|
2830
2781
|
raise NotImplementedError
|
2831
2782
|
|
2832
2783
|
@abstractmethod
|
2833
|
-
def _set_run_dirs(self, run_dir_arr: np.ndarray, run_idx: np.ndarray) -> None:
|
2834
|
-
...
|
2784
|
+
def _set_run_dirs(self, run_dir_arr: np.ndarray, run_idx: np.ndarray) -> None: ...
|
2835
2785
|
|
2836
2786
|
@abstractmethod
|
2837
|
-
def _update_iter_data_indices(
|
2838
|
-
|
2787
|
+
def _update_iter_data_indices(
|
2788
|
+
self, iter_data_indices: dict[int, DataIndex]
|
2789
|
+
) -> None: ...
|
2839
2790
|
|
2840
2791
|
@abstractmethod
|
2841
|
-
def _update_run_data_indices(
|
2842
|
-
|
2792
|
+
def _update_run_data_indices(
|
2793
|
+
self, run_data_indices: dict[int, DataIndex]
|
2794
|
+
) -> None: ...
|
hpcflow/sdk/persistence/zarr.py
CHANGED
@@ -462,9 +462,9 @@ class ZarrPersistentStore(
|
|
462
462
|
app, name="attrs", open_call=self._get_root_group
|
463
463
|
),
|
464
464
|
}
|
465
|
-
self._jobscript_at_submit_metadata: dict[
|
466
|
-
|
467
|
-
|
465
|
+
self._jobscript_at_submit_metadata: dict[int, dict[str, Any]] = (
|
466
|
+
{}
|
467
|
+
) # this is a cache
|
468
468
|
|
469
469
|
# these are caches; keys are submission index and then tuples of
|
470
470
|
# (jobscript index, jobscript-block index):
|
@@ -472,9 +472,9 @@ class ZarrPersistentStore(
|
|
472
472
|
self._jobscript_task_element_maps: dict[
|
473
473
|
int, dict[tuple[int, int], dict[int, list[int]]]
|
474
474
|
] = {}
|
475
|
-
self._jobscript_task_actions_arrays: dict[
|
476
|
-
|
477
|
-
|
475
|
+
self._jobscript_task_actions_arrays: dict[int, dict[tuple[int, int], NDArray]] = (
|
476
|
+
{}
|
477
|
+
)
|
478
478
|
self._jobscript_dependencies: dict[
|
479
479
|
int,
|
480
480
|
dict[
|
@@ -1904,9 +1904,9 @@ class ZarrPersistentStore(
|
|
1904
1904
|
arr_idx = 0
|
1905
1905
|
for js_idx_i, js_blk_shapes in enumerate(block_shapes):
|
1906
1906
|
for blk_idx_j, blk_shape_j in enumerate(js_blk_shapes):
|
1907
|
-
self._jobscript_run_ID_arrays[sub_idx][
|
1908
|
-
|
1909
|
-
|
1907
|
+
self._jobscript_run_ID_arrays[sub_idx][(js_idx_i, blk_idx_j)] = (
|
1908
|
+
arr_dat[arr_idx, : blk_shape_j[0], : blk_shape_j[1]]
|
1909
|
+
)
|
1910
1910
|
arr_idx += 1
|
1911
1911
|
|
1912
1912
|
else:
|
@@ -2044,9 +2044,9 @@ class ZarrPersistentStore(
|
|
2044
2044
|
# for a given submission, dependencies are stored for all jobscript-blocks in
|
2045
2045
|
# the same array (and chunk), so retrieve all of them and cache:
|
2046
2046
|
arr = self._get_jobscripts_dependencies_arr(sub_idx)
|
2047
|
-
self._jobscript_dependencies[
|
2048
|
-
|
2049
|
-
|
2047
|
+
self._jobscript_dependencies[sub_idx] = (
|
2048
|
+
self._decode_jobscript_block_dependencies(arr)
|
2049
|
+
)
|
2050
2050
|
else:
|
2051
2051
|
self.logger.debug(
|
2052
2052
|
f"retrieving jobscript-block dependencies for submission {sub_idx} from "
|
hpcflow/sdk/submission/enums.py
CHANGED
@@ -387,12 +387,12 @@ def resolve_jobscript_blocks(
|
|
387
387
|
True if the store supports run parallelism
|
388
388
|
|
389
389
|
"""
|
390
|
-
js_new: list[
|
391
|
-
|
392
|
-
|
393
|
-
new_idx: dict[
|
394
|
-
|
395
|
-
|
390
|
+
js_new: list[list[JobScriptCreationArguments]] = (
|
391
|
+
[]
|
392
|
+
) # TODO: not the same type, e.g. dependencies have tuple keys,
|
393
|
+
new_idx: dict[int, tuple[int, int]] = (
|
394
|
+
{}
|
395
|
+
) # track new positions by new jobscript index and block index
|
396
396
|
new_idx_inv: dict[int, list[int]] = defaultdict(list)
|
397
397
|
prev_hash = None
|
398
398
|
blocks: list[JobScriptCreationArguments] = []
|
@@ -1846,14 +1846,12 @@ class Jobscript(JSONLike):
|
|
1846
1846
|
@overload
|
1847
1847
|
def get_active_states(
|
1848
1848
|
self, as_json: Literal[False] = False
|
1849
|
-
) -> Mapping[int, Mapping[int, JobscriptElementState]]:
|
1850
|
-
...
|
1849
|
+
) -> Mapping[int, Mapping[int, JobscriptElementState]]: ...
|
1851
1850
|
|
1852
1851
|
@overload
|
1853
1852
|
def get_active_states(
|
1854
1853
|
self, as_json: Literal[True]
|
1855
|
-
) -> Mapping[int, Mapping[int, str]]:
|
1856
|
-
...
|
1854
|
+
) -> Mapping[int, Mapping[int, str]]: ...
|
1857
1855
|
|
1858
1856
|
@TimeIt.decorator
|
1859
1857
|
def get_active_states(
|
@@ -116,8 +116,7 @@ class DirectScheduler(Scheduler[DirectRef]):
|
|
116
116
|
@overload
|
117
117
|
@override
|
118
118
|
@classmethod
|
119
|
-
def wait_for_jobscripts(cls, js_refs: list[DirectRef]) -> None:
|
120
|
-
...
|
119
|
+
def wait_for_jobscripts(cls, js_refs: list[DirectRef]) -> None: ...
|
121
120
|
|
122
121
|
@overload
|
123
122
|
@classmethod
|
@@ -126,8 +125,7 @@ class DirectScheduler(Scheduler[DirectRef]):
|
|
126
125
|
js_refs: list[DirectRef],
|
127
126
|
*,
|
128
127
|
callback: Callable[[psutil.Process], None],
|
129
|
-
) -> list[psutil.Process]:
|
130
|
-
...
|
128
|
+
) -> list[psutil.Process]: ...
|
131
129
|
|
132
130
|
@classmethod
|
133
131
|
def wait_for_jobscripts(
|
@@ -131,9 +131,9 @@ class Submission(JSONLike):
|
|
131
131
|
self._JS_parallelism = JS_parallelism
|
132
132
|
self._environments = environments # assigned by _set_environments
|
133
133
|
|
134
|
-
self._submission_parts_lst: list[
|
135
|
-
|
136
|
-
|
134
|
+
self._submission_parts_lst: list[SubmissionPart] | None = (
|
135
|
+
None # assigned on first access
|
136
|
+
)
|
137
137
|
|
138
138
|
if workflow:
|
139
139
|
#: The workflow this is part of.
|
@@ -171,9 +171,9 @@ class Submission(JSONLike):
|
|
171
171
|
filterable = self._app.ElementResources.get_env_instance_filterable_attributes()
|
172
172
|
|
173
173
|
# map required environments and executable labels to job script indices:
|
174
|
-
req_envs: dict[
|
175
|
-
|
176
|
-
|
174
|
+
req_envs: dict[tuple[tuple[str, ...], tuple[Any, ...]], dict[str, set[int]]] = (
|
175
|
+
defaultdict(lambda: defaultdict(set))
|
176
|
+
)
|
177
177
|
with self.workflow.cached_merged_parameters():
|
178
178
|
# using the cache (for `run.env_spec_hashable` -> `run.resources`) should
|
179
179
|
# significantly speed up this loop, unless a large resources sequence is used:
|
@@ -635,14 +635,12 @@ class Submission(JSONLike):
|
|
635
635
|
@overload
|
636
636
|
def get_active_jobscripts(
|
637
637
|
self, as_json: Literal[False] = False
|
638
|
-
) -> Mapping[int, Mapping[int, Mapping[int, JobscriptElementState]]]:
|
639
|
-
...
|
638
|
+
) -> Mapping[int, Mapping[int, Mapping[int, JobscriptElementState]]]: ...
|
640
639
|
|
641
640
|
@overload
|
642
641
|
def get_active_jobscripts(
|
643
642
|
self, as_json: Literal[True]
|
644
|
-
) -> Mapping[int, Mapping[int, Mapping[int, str]]]:
|
645
|
-
...
|
643
|
+
) -> Mapping[int, Mapping[int, Mapping[int, str]]]: ...
|
646
644
|
|
647
645
|
@TimeIt.decorator
|
648
646
|
def get_active_jobscripts(
|
@@ -788,9 +786,9 @@ class Submission(JSONLike):
|
|
788
786
|
status.update("Adding new submission: writing scripts...")
|
789
787
|
|
790
788
|
seen: dict[int, Path] = {}
|
791
|
-
combined_script_data: dict[
|
792
|
-
|
793
|
-
|
789
|
+
combined_script_data: dict[int, dict[int, list[tuple[str, Path, bool]]]] = (
|
790
|
+
defaultdict(lambda: defaultdict(list))
|
791
|
+
)
|
794
792
|
for task in self.workflow.tasks:
|
795
793
|
for schema in task.template.schemas:
|
796
794
|
if schema.name in actions_by_schema:
|
hpcflow/sdk/utils/arrays.py
CHANGED
@@ -7,13 +7,11 @@ if TYPE_CHECKING:
|
|
7
7
|
|
8
8
|
|
9
9
|
@overload
|
10
|
-
def get_2D_idx(idx: int, num_cols: int) -> tuple[int, int]:
|
11
|
-
...
|
10
|
+
def get_2D_idx(idx: int, num_cols: int) -> tuple[int, int]: ...
|
12
11
|
|
13
12
|
|
14
13
|
@overload
|
15
|
-
def get_2D_idx(idx: NDArray, num_cols: int) -> tuple[NDArray, NDArray]:
|
16
|
-
...
|
14
|
+
def get_2D_idx(idx: NDArray, num_cols: int) -> tuple[NDArray, NDArray]: ...
|
17
15
|
|
18
16
|
|
19
17
|
def get_2D_idx(idx: int | NDArray, num_cols: int) -> tuple[int | NDArray, int | NDArray]:
|
@@ -170,6 +170,29 @@ def test_MPS_latin_hypercube_sequence_values():
|
|
170
170
|
assert np.array_equal(np.asarray(seq_2.values), mps_values[1])
|
171
171
|
|
172
172
|
|
173
|
+
def test_MPS_latin_hypercube_sequence_bounds():
|
174
|
+
|
175
|
+
bounds = {"inputs.a": [16789.2, 17812.5]}
|
176
|
+
|
177
|
+
mps = hf.MultiPathSequence.from_latin_hypercube(
|
178
|
+
paths=["inputs.a", "inputs.b"],
|
179
|
+
num_samples=10,
|
180
|
+
bounds=bounds,
|
181
|
+
)
|
182
|
+
|
183
|
+
vals_arr = np.array(mps.values)
|
184
|
+
assert vals_arr.shape == (2, 10)
|
185
|
+
|
186
|
+
vals_0 = vals_arr[0]
|
187
|
+
bounds_0 = list(bounds.values())[0]
|
188
|
+
|
189
|
+
vals_1 = vals_arr[1]
|
190
|
+
bounds_1 = [0, 1]
|
191
|
+
|
192
|
+
assert np.logical_and(vals_0 > bounds_0[0], vals_0 < bounds_0[1]).all()
|
193
|
+
assert np.logical_and(vals_1 > bounds_1[0], vals_1 < bounds_1[1]).all()
|
194
|
+
|
195
|
+
|
173
196
|
def test_MPS_move_from_sequences_list():
|
174
197
|
wft_yaml = dedent(
|
175
198
|
"""\
|