hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
- hpcflow/_version.py +1 -1
- hpcflow/app.py +1 -0
- hpcflow/data/scripts/bad_script.py +2 -0
- hpcflow/data/scripts/do_nothing.py +2 -0
- hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
- hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
- hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
- hpcflow/data/scripts/input_file_generator_basic.py +3 -0
- hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
- hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
- hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
- hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
- hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
- hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
- hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
- hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
- hpcflow/data/scripts/output_file_parser_basic.py +3 -0
- hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
- hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
- hpcflow/data/scripts/script_exit_test.py +5 -0
- hpcflow/data/template_components/environments.yaml +1 -1
- hpcflow/sdk/__init__.py +26 -15
- hpcflow/sdk/app.py +2192 -768
- hpcflow/sdk/cli.py +506 -296
- hpcflow/sdk/cli_common.py +105 -7
- hpcflow/sdk/config/__init__.py +1 -1
- hpcflow/sdk/config/callbacks.py +115 -43
- hpcflow/sdk/config/cli.py +126 -103
- hpcflow/sdk/config/config.py +674 -318
- hpcflow/sdk/config/config_file.py +131 -95
- hpcflow/sdk/config/errors.py +125 -84
- hpcflow/sdk/config/types.py +148 -0
- hpcflow/sdk/core/__init__.py +25 -1
- hpcflow/sdk/core/actions.py +1771 -1059
- hpcflow/sdk/core/app_aware.py +24 -0
- hpcflow/sdk/core/cache.py +139 -79
- hpcflow/sdk/core/command_files.py +263 -287
- hpcflow/sdk/core/commands.py +145 -112
- hpcflow/sdk/core/element.py +828 -535
- hpcflow/sdk/core/enums.py +192 -0
- hpcflow/sdk/core/environment.py +74 -93
- hpcflow/sdk/core/errors.py +455 -52
- hpcflow/sdk/core/execute.py +207 -0
- hpcflow/sdk/core/json_like.py +540 -272
- hpcflow/sdk/core/loop.py +751 -347
- hpcflow/sdk/core/loop_cache.py +164 -47
- hpcflow/sdk/core/object_list.py +370 -207
- hpcflow/sdk/core/parameters.py +1100 -627
- hpcflow/sdk/core/rule.py +59 -41
- hpcflow/sdk/core/run_dir_files.py +21 -37
- hpcflow/sdk/core/skip_reason.py +7 -0
- hpcflow/sdk/core/task.py +1649 -1339
- hpcflow/sdk/core/task_schema.py +308 -196
- hpcflow/sdk/core/test_utils.py +191 -114
- hpcflow/sdk/core/types.py +440 -0
- hpcflow/sdk/core/utils.py +485 -309
- hpcflow/sdk/core/validation.py +82 -9
- hpcflow/sdk/core/workflow.py +2544 -1178
- hpcflow/sdk/core/zarr_io.py +98 -137
- hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
- hpcflow/sdk/demo/cli.py +53 -33
- hpcflow/sdk/helper/cli.py +18 -15
- hpcflow/sdk/helper/helper.py +75 -63
- hpcflow/sdk/helper/watcher.py +61 -28
- hpcflow/sdk/log.py +122 -71
- hpcflow/sdk/persistence/__init__.py +8 -31
- hpcflow/sdk/persistence/base.py +1360 -606
- hpcflow/sdk/persistence/defaults.py +6 -0
- hpcflow/sdk/persistence/discovery.py +38 -0
- hpcflow/sdk/persistence/json.py +568 -188
- hpcflow/sdk/persistence/pending.py +382 -179
- hpcflow/sdk/persistence/store_resource.py +39 -23
- hpcflow/sdk/persistence/types.py +318 -0
- hpcflow/sdk/persistence/utils.py +14 -11
- hpcflow/sdk/persistence/zarr.py +1337 -433
- hpcflow/sdk/runtime.py +44 -41
- hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
- hpcflow/sdk/submission/jobscript.py +1651 -692
- hpcflow/sdk/submission/schedulers/__init__.py +167 -39
- hpcflow/sdk/submission/schedulers/direct.py +121 -81
- hpcflow/sdk/submission/schedulers/sge.py +170 -129
- hpcflow/sdk/submission/schedulers/slurm.py +291 -268
- hpcflow/sdk/submission/schedulers/utils.py +12 -2
- hpcflow/sdk/submission/shells/__init__.py +14 -15
- hpcflow/sdk/submission/shells/base.py +150 -29
- hpcflow/sdk/submission/shells/bash.py +283 -173
- hpcflow/sdk/submission/shells/os_version.py +31 -30
- hpcflow/sdk/submission/shells/powershell.py +228 -170
- hpcflow/sdk/submission/submission.py +1014 -335
- hpcflow/sdk/submission/types.py +140 -0
- hpcflow/sdk/typing.py +182 -12
- hpcflow/sdk/utils/arrays.py +71 -0
- hpcflow/sdk/utils/deferred_file.py +55 -0
- hpcflow/sdk/utils/hashing.py +16 -0
- hpcflow/sdk/utils/patches.py +12 -0
- hpcflow/sdk/utils/strings.py +33 -0
- hpcflow/tests/api/test_api.py +32 -0
- hpcflow/tests/conftest.py +27 -6
- hpcflow/tests/data/multi_path_sequences.yaml +29 -0
- hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
- hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
- hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
- hpcflow/tests/scripts/test_input_file_generators.py +282 -0
- hpcflow/tests/scripts/test_main_scripts.py +866 -85
- hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
- hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
- hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
- hpcflow/tests/unit/test_action.py +262 -75
- hpcflow/tests/unit/test_action_rule.py +9 -4
- hpcflow/tests/unit/test_app.py +33 -6
- hpcflow/tests/unit/test_cache.py +46 -0
- hpcflow/tests/unit/test_cli.py +134 -1
- hpcflow/tests/unit/test_command.py +71 -54
- hpcflow/tests/unit/test_config.py +142 -16
- hpcflow/tests/unit/test_config_file.py +21 -18
- hpcflow/tests/unit/test_element.py +58 -62
- hpcflow/tests/unit/test_element_iteration.py +50 -1
- hpcflow/tests/unit/test_element_set.py +29 -19
- hpcflow/tests/unit/test_group.py +4 -2
- hpcflow/tests/unit/test_input_source.py +116 -93
- hpcflow/tests/unit/test_input_value.py +29 -24
- hpcflow/tests/unit/test_jobscript_unit.py +757 -0
- hpcflow/tests/unit/test_json_like.py +44 -35
- hpcflow/tests/unit/test_loop.py +1396 -84
- hpcflow/tests/unit/test_meta_task.py +325 -0
- hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
- hpcflow/tests/unit/test_object_list.py +17 -12
- hpcflow/tests/unit/test_parameter.py +29 -7
- hpcflow/tests/unit/test_persistence.py +237 -42
- hpcflow/tests/unit/test_resources.py +20 -18
- hpcflow/tests/unit/test_run.py +117 -6
- hpcflow/tests/unit/test_run_directories.py +29 -0
- hpcflow/tests/unit/test_runtime.py +2 -1
- hpcflow/tests/unit/test_schema_input.py +23 -15
- hpcflow/tests/unit/test_shell.py +23 -2
- hpcflow/tests/unit/test_slurm.py +8 -7
- hpcflow/tests/unit/test_submission.py +38 -89
- hpcflow/tests/unit/test_task.py +352 -247
- hpcflow/tests/unit/test_task_schema.py +33 -20
- hpcflow/tests/unit/test_utils.py +9 -11
- hpcflow/tests/unit/test_value_sequence.py +15 -12
- hpcflow/tests/unit/test_workflow.py +114 -83
- hpcflow/tests/unit/test_workflow_template.py +0 -1
- hpcflow/tests/unit/utils/test_arrays.py +40 -0
- hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
- hpcflow/tests/unit/utils/test_hashing.py +65 -0
- hpcflow/tests/unit/utils/test_patches.py +5 -0
- hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
- hpcflow/tests/workflows/__init__.py +0 -0
- hpcflow/tests/workflows/test_directory_structure.py +31 -0
- hpcflow/tests/workflows/test_jobscript.py +334 -1
- hpcflow/tests/workflows/test_run_status.py +198 -0
- hpcflow/tests/workflows/test_skip_downstream.py +696 -0
- hpcflow/tests/workflows/test_submission.py +140 -0
- hpcflow/tests/workflows/test_workflows.py +160 -15
- hpcflow/tests/workflows/test_zip.py +18 -0
- hpcflow/viz_demo.ipynb +6587 -3
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
- hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
- hpcflow/sdk/core/parallel.py +0 -21
- hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
- {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -1,7 +1,9 @@
|
|
1
|
+
# mypy: disable-error-code="annotation-unchecked"
|
2
|
+
from __future__ import annotations
|
1
3
|
from dataclasses import dataclass
|
2
4
|
import enum
|
3
5
|
from types import SimpleNamespace
|
4
|
-
from typing import
|
6
|
+
from typing import Any
|
5
7
|
|
6
8
|
import pytest
|
7
9
|
|
@@ -9,6 +11,10 @@ from hpcflow.app import app as hf
|
|
9
11
|
from hpcflow.sdk.core.json_like import BaseJSONLike, ChildObjectSpec
|
10
12
|
from hpcflow.sdk.core.object_list import ObjectList
|
11
13
|
|
14
|
+
# BE AWARE THAT MYPY CANNOT CORRECTLY TYPE-CHECK THIS FILE AT ALL.
|
15
|
+
# It fails massively due to all the classes inside functions being passed to other functions.
|
16
|
+
# Omitting the types makes it ignore them all, which us for the best.
|
17
|
+
|
12
18
|
|
13
19
|
@pytest.fixture
|
14
20
|
def null_config(tmp_path):
|
@@ -87,16 +93,17 @@ def test_BaseJSONLike_child_object_class_namespace_via_obj(null_config):
|
|
87
93
|
|
88
94
|
|
89
95
|
def test_BaseJSONLike_child_object_class_namespace_via_name_and_dict_namespace(
|
90
|
-
BaseJSONLikeSubClass,
|
96
|
+
BaseJSONLikeSubClass: type[BaseJSONLike],
|
91
97
|
):
|
92
98
|
"""Child object class passed as a name and namespace passed as a dict."""
|
99
|
+
T: type = BaseJSONLikeSubClass # Workaround for python/mypy#14458
|
93
100
|
|
94
101
|
@dataclass
|
95
|
-
class ObjB(
|
102
|
+
class ObjB(T):
|
96
103
|
c: int
|
97
104
|
|
98
105
|
@dataclass
|
99
|
-
class ObjA(
|
106
|
+
class ObjA(T):
|
100
107
|
_child_objects = (
|
101
108
|
ChildObjectSpec(
|
102
109
|
name="b",
|
@@ -118,16 +125,17 @@ def test_BaseJSONLike_child_object_class_namespace_via_name_and_dict_namespace(
|
|
118
125
|
|
119
126
|
|
120
127
|
def test_BaseJSONLike_child_object_class_namespace_via_name_and_func_locals(
|
121
|
-
BaseJSONLikeSubClass,
|
128
|
+
BaseJSONLikeSubClass: type[BaseJSONLike],
|
122
129
|
):
|
123
130
|
"""Child object class passed as a name and namespace passed as function locals."""
|
131
|
+
T: type = BaseJSONLikeSubClass # Workaround for python/mypy#14458
|
124
132
|
|
125
133
|
@dataclass
|
126
|
-
class ObjB(
|
134
|
+
class ObjB(T):
|
127
135
|
c: int
|
128
136
|
|
129
137
|
@dataclass
|
130
|
-
class ObjA(
|
138
|
+
class ObjA(T):
|
131
139
|
_child_objects = (
|
132
140
|
ChildObjectSpec(
|
133
141
|
name="b",
|
@@ -149,16 +157,17 @@ def test_BaseJSONLike_child_object_class_namespace_via_name_and_func_locals(
|
|
149
157
|
|
150
158
|
|
151
159
|
def test_BaseJSONLike_child_object_class_namespace_via_name_and_SimpleNamespace(
|
152
|
-
BaseJSONLikeSubClass,
|
160
|
+
BaseJSONLikeSubClass: type[BaseJSONLike],
|
153
161
|
):
|
154
162
|
"""Child object class passed as a name and namespace passed as a SimpleNamespace."""
|
163
|
+
T: type = BaseJSONLikeSubClass # Workaround for python/mypy#14458
|
155
164
|
|
156
165
|
@dataclass
|
157
|
-
class ObjB(
|
166
|
+
class ObjB(T):
|
158
167
|
c: int
|
159
168
|
|
160
169
|
@dataclass
|
161
|
-
class ObjA(
|
170
|
+
class ObjA(T):
|
162
171
|
_child_objects = (
|
163
172
|
ChildObjectSpec(
|
164
173
|
name="b",
|
@@ -549,12 +558,12 @@ def test_from_json_like_with_parent_ref(null_config):
|
|
549
558
|
class ObjB(BaseJSONLike):
|
550
559
|
name: str
|
551
560
|
c: int
|
552
|
-
obj_A:
|
561
|
+
obj_A: Any = None
|
553
562
|
|
554
563
|
def __eq__(self, other):
|
555
|
-
if
|
556
|
-
return
|
557
|
-
return
|
564
|
+
if not isinstance(other, self.__class__):
|
565
|
+
return False
|
566
|
+
return self.name == other.name and self.c == other.c
|
558
567
|
|
559
568
|
@dataclass
|
560
569
|
class ObjA(BaseJSONLike):
|
@@ -572,14 +581,14 @@ def test_from_json_like_with_parent_ref(null_config):
|
|
572
581
|
self._set_parent_refs()
|
573
582
|
|
574
583
|
def __eq__(self, other):
|
575
|
-
if (
|
584
|
+
if not isinstance(other, self.__class__):
|
585
|
+
return False
|
586
|
+
return (
|
576
587
|
self.a == other.a
|
577
588
|
and self.b == other.b
|
578
589
|
and self.b.obj_A is self
|
579
590
|
and other.b.obj_A is other
|
580
|
-
)
|
581
|
-
return True
|
582
|
-
return False
|
591
|
+
)
|
583
592
|
|
584
593
|
js_1 = {
|
585
594
|
"a": 1,
|
@@ -600,12 +609,12 @@ def test_json_like_round_trip_with_parent_ref(null_config):
|
|
600
609
|
class ObjB(BaseJSONLike):
|
601
610
|
name: str
|
602
611
|
c: int
|
603
|
-
obj_A:
|
612
|
+
obj_A: Any = None
|
604
613
|
|
605
614
|
def __eq__(self, other):
|
606
|
-
if
|
607
|
-
return
|
608
|
-
return
|
615
|
+
if not isinstance(other, self.__class__):
|
616
|
+
return False
|
617
|
+
return self.name == other.name and self.c == other.c
|
609
618
|
|
610
619
|
@dataclass
|
611
620
|
class ObjA(BaseJSONLike):
|
@@ -620,14 +629,14 @@ def test_json_like_round_trip_with_parent_ref(null_config):
|
|
620
629
|
b: float
|
621
630
|
|
622
631
|
def __eq__(self, other):
|
623
|
-
if (
|
632
|
+
if not isinstance(other, self.__class__):
|
633
|
+
return False
|
634
|
+
return (
|
624
635
|
self.a == other.a
|
625
636
|
and self.b == other.b
|
626
637
|
and self.b.obj_A is self
|
627
638
|
and other.b.obj_A is other
|
628
|
-
)
|
629
|
-
return True
|
630
|
-
return False
|
639
|
+
)
|
631
640
|
|
632
641
|
js_1 = {
|
633
642
|
"a": 1,
|
@@ -658,7 +667,7 @@ def test_from_json_like_optional_attr(null_config):
|
|
658
667
|
),
|
659
668
|
)
|
660
669
|
a: int
|
661
|
-
b:
|
670
|
+
b: Any = None
|
662
671
|
|
663
672
|
js_in = {"a": 9, "b": None}
|
664
673
|
objA = ObjA.from_json_like(js_in)
|
@@ -683,7 +692,7 @@ def test_from_json_like_optional_attr_with_is_multiple_both_none(null_config):
|
|
683
692
|
),
|
684
693
|
)
|
685
694
|
a: int
|
686
|
-
b:
|
695
|
+
b: Any = None
|
687
696
|
|
688
697
|
js_in = {
|
689
698
|
"a": 9,
|
@@ -711,7 +720,7 @@ def test_from_json_like_optional_attr_with_is_multiple_one_none(null_config):
|
|
711
720
|
),
|
712
721
|
)
|
713
722
|
a: int
|
714
|
-
b:
|
723
|
+
b: Any = None
|
715
724
|
|
716
725
|
js_in = {
|
717
726
|
"a": 9,
|
@@ -743,7 +752,7 @@ def test_from_json_like_optional_attr_with_is_multiple_one_none_and_shared_data_
|
|
743
752
|
),
|
744
753
|
)
|
745
754
|
a: int
|
746
|
-
b:
|
755
|
+
b: Any = None
|
747
756
|
|
748
757
|
dcts = [
|
749
758
|
{"name": "c1", "c": 2},
|
@@ -782,7 +791,7 @@ def test_from_json_like_optional_attr_with_is_multiple_all_none_and_shared_data_
|
|
782
791
|
),
|
783
792
|
)
|
784
793
|
a: int
|
785
|
-
b:
|
794
|
+
b: Any = None
|
786
795
|
|
787
796
|
dcts = [
|
788
797
|
{"name": "c1", "c": 2},
|
@@ -819,7 +828,7 @@ def test_from_json_like_optional_attr_with_shared_data_name(null_config):
|
|
819
828
|
),
|
820
829
|
)
|
821
830
|
a: int
|
822
|
-
b:
|
831
|
+
b: Any = None
|
823
832
|
|
824
833
|
dcts = [
|
825
834
|
{"name": "c1", "c": 2},
|
@@ -848,7 +857,7 @@ def test_from_json_like_optional_attr_with_enum(null_config):
|
|
848
857
|
class ObjA(BaseJSONLikeSubClass):
|
849
858
|
_child_objects = (ChildObjectSpec(name="b", class_obj=MyEnum, is_enum=True),)
|
850
859
|
a: int
|
851
|
-
b:
|
860
|
+
b: Any = None
|
852
861
|
|
853
862
|
js_in = {
|
854
863
|
"a": 9,
|
@@ -1079,7 +1088,7 @@ def test_from_json_like_with_is_multiple_and_shared_data_dict_lookup(null_config
|
|
1079
1088
|
class ObjB(BaseJSONLikeSubClass):
|
1080
1089
|
name: str
|
1081
1090
|
c: int
|
1082
|
-
_hash_value:
|
1091
|
+
_hash_value: str | None = None
|
1083
1092
|
|
1084
1093
|
@dataclass
|
1085
1094
|
class ObjA(BaseJSONLikeSubClass):
|
@@ -1205,7 +1214,7 @@ def test_to_json_like_with_child_ref(null_config):
|
|
1205
1214
|
class ObjB(BaseJSONLike):
|
1206
1215
|
name: str
|
1207
1216
|
c: int
|
1208
|
-
obj_A:
|
1217
|
+
obj_A: Any = None
|
1209
1218
|
|
1210
1219
|
@dataclass
|
1211
1220
|
class ObjA(BaseJSONLike):
|