hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +9 -6
  2. hpcflow/_version.py +1 -1
  3. hpcflow/app.py +1 -0
  4. hpcflow/data/scripts/bad_script.py +2 -0
  5. hpcflow/data/scripts/do_nothing.py +2 -0
  6. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  7. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  8. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  9. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  10. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  11. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  12. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  13. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  14. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  15. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  16. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  17. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  18. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  19. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  20. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  21. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  22. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  23. hpcflow/data/scripts/main_script_test_hdf5_in_obj.py +1 -1
  24. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  25. hpcflow/data/scripts/main_script_test_hdf5_out_obj.py +1 -1
  26. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  27. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  28. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  29. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  30. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  31. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  32. hpcflow/data/scripts/script_exit_test.py +5 -0
  33. hpcflow/data/template_components/environments.yaml +1 -1
  34. hpcflow/sdk/__init__.py +26 -15
  35. hpcflow/sdk/app.py +2192 -768
  36. hpcflow/sdk/cli.py +506 -296
  37. hpcflow/sdk/cli_common.py +105 -7
  38. hpcflow/sdk/config/__init__.py +1 -1
  39. hpcflow/sdk/config/callbacks.py +115 -43
  40. hpcflow/sdk/config/cli.py +126 -103
  41. hpcflow/sdk/config/config.py +674 -318
  42. hpcflow/sdk/config/config_file.py +131 -95
  43. hpcflow/sdk/config/errors.py +125 -84
  44. hpcflow/sdk/config/types.py +148 -0
  45. hpcflow/sdk/core/__init__.py +25 -1
  46. hpcflow/sdk/core/actions.py +1771 -1059
  47. hpcflow/sdk/core/app_aware.py +24 -0
  48. hpcflow/sdk/core/cache.py +139 -79
  49. hpcflow/sdk/core/command_files.py +263 -287
  50. hpcflow/sdk/core/commands.py +145 -112
  51. hpcflow/sdk/core/element.py +828 -535
  52. hpcflow/sdk/core/enums.py +192 -0
  53. hpcflow/sdk/core/environment.py +74 -93
  54. hpcflow/sdk/core/errors.py +455 -52
  55. hpcflow/sdk/core/execute.py +207 -0
  56. hpcflow/sdk/core/json_like.py +540 -272
  57. hpcflow/sdk/core/loop.py +751 -347
  58. hpcflow/sdk/core/loop_cache.py +164 -47
  59. hpcflow/sdk/core/object_list.py +370 -207
  60. hpcflow/sdk/core/parameters.py +1100 -627
  61. hpcflow/sdk/core/rule.py +59 -41
  62. hpcflow/sdk/core/run_dir_files.py +21 -37
  63. hpcflow/sdk/core/skip_reason.py +7 -0
  64. hpcflow/sdk/core/task.py +1649 -1339
  65. hpcflow/sdk/core/task_schema.py +308 -196
  66. hpcflow/sdk/core/test_utils.py +191 -114
  67. hpcflow/sdk/core/types.py +440 -0
  68. hpcflow/sdk/core/utils.py +485 -309
  69. hpcflow/sdk/core/validation.py +82 -9
  70. hpcflow/sdk/core/workflow.py +2544 -1178
  71. hpcflow/sdk/core/zarr_io.py +98 -137
  72. hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
  73. hpcflow/sdk/demo/cli.py +53 -33
  74. hpcflow/sdk/helper/cli.py +18 -15
  75. hpcflow/sdk/helper/helper.py +75 -63
  76. hpcflow/sdk/helper/watcher.py +61 -28
  77. hpcflow/sdk/log.py +122 -71
  78. hpcflow/sdk/persistence/__init__.py +8 -31
  79. hpcflow/sdk/persistence/base.py +1360 -606
  80. hpcflow/sdk/persistence/defaults.py +6 -0
  81. hpcflow/sdk/persistence/discovery.py +38 -0
  82. hpcflow/sdk/persistence/json.py +568 -188
  83. hpcflow/sdk/persistence/pending.py +382 -179
  84. hpcflow/sdk/persistence/store_resource.py +39 -23
  85. hpcflow/sdk/persistence/types.py +318 -0
  86. hpcflow/sdk/persistence/utils.py +14 -11
  87. hpcflow/sdk/persistence/zarr.py +1337 -433
  88. hpcflow/sdk/runtime.py +44 -41
  89. hpcflow/sdk/submission/{jobscript_info.py → enums.py} +39 -12
  90. hpcflow/sdk/submission/jobscript.py +1651 -692
  91. hpcflow/sdk/submission/schedulers/__init__.py +167 -39
  92. hpcflow/sdk/submission/schedulers/direct.py +121 -81
  93. hpcflow/sdk/submission/schedulers/sge.py +170 -129
  94. hpcflow/sdk/submission/schedulers/slurm.py +291 -268
  95. hpcflow/sdk/submission/schedulers/utils.py +12 -2
  96. hpcflow/sdk/submission/shells/__init__.py +14 -15
  97. hpcflow/sdk/submission/shells/base.py +150 -29
  98. hpcflow/sdk/submission/shells/bash.py +283 -173
  99. hpcflow/sdk/submission/shells/os_version.py +31 -30
  100. hpcflow/sdk/submission/shells/powershell.py +228 -170
  101. hpcflow/sdk/submission/submission.py +1014 -335
  102. hpcflow/sdk/submission/types.py +140 -0
  103. hpcflow/sdk/typing.py +182 -12
  104. hpcflow/sdk/utils/arrays.py +71 -0
  105. hpcflow/sdk/utils/deferred_file.py +55 -0
  106. hpcflow/sdk/utils/hashing.py +16 -0
  107. hpcflow/sdk/utils/patches.py +12 -0
  108. hpcflow/sdk/utils/strings.py +33 -0
  109. hpcflow/tests/api/test_api.py +32 -0
  110. hpcflow/tests/conftest.py +27 -6
  111. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  112. hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
  113. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  114. hpcflow/tests/schedulers/slurm/test_slurm_submission.py +5 -2
  115. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  116. hpcflow/tests/scripts/test_main_scripts.py +866 -85
  117. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  118. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  119. hpcflow/tests/shells/wsl/test_wsl_submission.py +12 -4
  120. hpcflow/tests/unit/test_action.py +262 -75
  121. hpcflow/tests/unit/test_action_rule.py +9 -4
  122. hpcflow/tests/unit/test_app.py +33 -6
  123. hpcflow/tests/unit/test_cache.py +46 -0
  124. hpcflow/tests/unit/test_cli.py +134 -1
  125. hpcflow/tests/unit/test_command.py +71 -54
  126. hpcflow/tests/unit/test_config.py +142 -16
  127. hpcflow/tests/unit/test_config_file.py +21 -18
  128. hpcflow/tests/unit/test_element.py +58 -62
  129. hpcflow/tests/unit/test_element_iteration.py +50 -1
  130. hpcflow/tests/unit/test_element_set.py +29 -19
  131. hpcflow/tests/unit/test_group.py +4 -2
  132. hpcflow/tests/unit/test_input_source.py +116 -93
  133. hpcflow/tests/unit/test_input_value.py +29 -24
  134. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  135. hpcflow/tests/unit/test_json_like.py +44 -35
  136. hpcflow/tests/unit/test_loop.py +1396 -84
  137. hpcflow/tests/unit/test_meta_task.py +325 -0
  138. hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
  139. hpcflow/tests/unit/test_object_list.py +17 -12
  140. hpcflow/tests/unit/test_parameter.py +29 -7
  141. hpcflow/tests/unit/test_persistence.py +237 -42
  142. hpcflow/tests/unit/test_resources.py +20 -18
  143. hpcflow/tests/unit/test_run.py +117 -6
  144. hpcflow/tests/unit/test_run_directories.py +29 -0
  145. hpcflow/tests/unit/test_runtime.py +2 -1
  146. hpcflow/tests/unit/test_schema_input.py +23 -15
  147. hpcflow/tests/unit/test_shell.py +23 -2
  148. hpcflow/tests/unit/test_slurm.py +8 -7
  149. hpcflow/tests/unit/test_submission.py +38 -89
  150. hpcflow/tests/unit/test_task.py +352 -247
  151. hpcflow/tests/unit/test_task_schema.py +33 -20
  152. hpcflow/tests/unit/test_utils.py +9 -11
  153. hpcflow/tests/unit/test_value_sequence.py +15 -12
  154. hpcflow/tests/unit/test_workflow.py +114 -83
  155. hpcflow/tests/unit/test_workflow_template.py +0 -1
  156. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  157. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  158. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  159. hpcflow/tests/unit/utils/test_patches.py +5 -0
  160. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  161. hpcflow/tests/workflows/__init__.py +0 -0
  162. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  163. hpcflow/tests/workflows/test_jobscript.py +334 -1
  164. hpcflow/tests/workflows/test_run_status.py +198 -0
  165. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  166. hpcflow/tests/workflows/test_submission.py +140 -0
  167. hpcflow/tests/workflows/test_workflows.py +160 -15
  168. hpcflow/tests/workflows/test_zip.py +18 -0
  169. hpcflow/viz_demo.ipynb +6587 -3
  170. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +8 -4
  171. hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
  172. hpcflow/sdk/core/parallel.py +0 -21
  173. hpcflow_new2-0.2.0a189.dist-info/RECORD +0 -158
  174. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
  175. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
  176. {hpcflow_new2-0.2.0a189.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -1,7 +1,9 @@
1
+ # mypy: disable-error-code="annotation-unchecked"
2
+ from __future__ import annotations
1
3
  from dataclasses import dataclass
2
4
  import enum
3
5
  from types import SimpleNamespace
4
- from typing import Optional, Any
6
+ from typing import Any
5
7
 
6
8
  import pytest
7
9
 
@@ -9,6 +11,10 @@ from hpcflow.app import app as hf
9
11
  from hpcflow.sdk.core.json_like import BaseJSONLike, ChildObjectSpec
10
12
  from hpcflow.sdk.core.object_list import ObjectList
11
13
 
14
+ # BE AWARE THAT MYPY CANNOT CORRECTLY TYPE-CHECK THIS FILE AT ALL.
15
+ # It fails massively due to all the classes inside functions being passed to other functions.
16
+ # Omitting the types makes it ignore them all, which us for the best.
17
+
12
18
 
13
19
  @pytest.fixture
14
20
  def null_config(tmp_path):
@@ -87,16 +93,17 @@ def test_BaseJSONLike_child_object_class_namespace_via_obj(null_config):
87
93
 
88
94
 
89
95
  def test_BaseJSONLike_child_object_class_namespace_via_name_and_dict_namespace(
90
- BaseJSONLikeSubClass,
96
+ BaseJSONLikeSubClass: type[BaseJSONLike],
91
97
  ):
92
98
  """Child object class passed as a name and namespace passed as a dict."""
99
+ T: type = BaseJSONLikeSubClass # Workaround for python/mypy#14458
93
100
 
94
101
  @dataclass
95
- class ObjB(BaseJSONLikeSubClass):
102
+ class ObjB(T):
96
103
  c: int
97
104
 
98
105
  @dataclass
99
- class ObjA(BaseJSONLikeSubClass):
106
+ class ObjA(T):
100
107
  _child_objects = (
101
108
  ChildObjectSpec(
102
109
  name="b",
@@ -118,16 +125,17 @@ def test_BaseJSONLike_child_object_class_namespace_via_name_and_dict_namespace(
118
125
 
119
126
 
120
127
  def test_BaseJSONLike_child_object_class_namespace_via_name_and_func_locals(
121
- BaseJSONLikeSubClass,
128
+ BaseJSONLikeSubClass: type[BaseJSONLike],
122
129
  ):
123
130
  """Child object class passed as a name and namespace passed as function locals."""
131
+ T: type = BaseJSONLikeSubClass # Workaround for python/mypy#14458
124
132
 
125
133
  @dataclass
126
- class ObjB(BaseJSONLikeSubClass):
134
+ class ObjB(T):
127
135
  c: int
128
136
 
129
137
  @dataclass
130
- class ObjA(BaseJSONLikeSubClass):
138
+ class ObjA(T):
131
139
  _child_objects = (
132
140
  ChildObjectSpec(
133
141
  name="b",
@@ -149,16 +157,17 @@ def test_BaseJSONLike_child_object_class_namespace_via_name_and_func_locals(
149
157
 
150
158
 
151
159
  def test_BaseJSONLike_child_object_class_namespace_via_name_and_SimpleNamespace(
152
- BaseJSONLikeSubClass,
160
+ BaseJSONLikeSubClass: type[BaseJSONLike],
153
161
  ):
154
162
  """Child object class passed as a name and namespace passed as a SimpleNamespace."""
163
+ T: type = BaseJSONLikeSubClass # Workaround for python/mypy#14458
155
164
 
156
165
  @dataclass
157
- class ObjB(BaseJSONLikeSubClass):
166
+ class ObjB(T):
158
167
  c: int
159
168
 
160
169
  @dataclass
161
- class ObjA(BaseJSONLikeSubClass):
170
+ class ObjA(T):
162
171
  _child_objects = (
163
172
  ChildObjectSpec(
164
173
  name="b",
@@ -549,12 +558,12 @@ def test_from_json_like_with_parent_ref(null_config):
549
558
  class ObjB(BaseJSONLike):
550
559
  name: str
551
560
  c: int
552
- obj_A: Optional[Any] = None
561
+ obj_A: Any = None
553
562
 
554
563
  def __eq__(self, other):
555
- if self.name == other.name and self.c == other.c:
556
- return True
557
- return False
564
+ if not isinstance(other, self.__class__):
565
+ return False
566
+ return self.name == other.name and self.c == other.c
558
567
 
559
568
  @dataclass
560
569
  class ObjA(BaseJSONLike):
@@ -572,14 +581,14 @@ def test_from_json_like_with_parent_ref(null_config):
572
581
  self._set_parent_refs()
573
582
 
574
583
  def __eq__(self, other):
575
- if (
584
+ if not isinstance(other, self.__class__):
585
+ return False
586
+ return (
576
587
  self.a == other.a
577
588
  and self.b == other.b
578
589
  and self.b.obj_A is self
579
590
  and other.b.obj_A is other
580
- ):
581
- return True
582
- return False
591
+ )
583
592
 
584
593
  js_1 = {
585
594
  "a": 1,
@@ -600,12 +609,12 @@ def test_json_like_round_trip_with_parent_ref(null_config):
600
609
  class ObjB(BaseJSONLike):
601
610
  name: str
602
611
  c: int
603
- obj_A: Optional[Any] = None
612
+ obj_A: Any = None
604
613
 
605
614
  def __eq__(self, other):
606
- if self.name == other.name and self.c == other.c:
607
- return True
608
- return False
615
+ if not isinstance(other, self.__class__):
616
+ return False
617
+ return self.name == other.name and self.c == other.c
609
618
 
610
619
  @dataclass
611
620
  class ObjA(BaseJSONLike):
@@ -620,14 +629,14 @@ def test_json_like_round_trip_with_parent_ref(null_config):
620
629
  b: float
621
630
 
622
631
  def __eq__(self, other):
623
- if (
632
+ if not isinstance(other, self.__class__):
633
+ return False
634
+ return (
624
635
  self.a == other.a
625
636
  and self.b == other.b
626
637
  and self.b.obj_A is self
627
638
  and other.b.obj_A is other
628
- ):
629
- return True
630
- return False
639
+ )
631
640
 
632
641
  js_1 = {
633
642
  "a": 1,
@@ -658,7 +667,7 @@ def test_from_json_like_optional_attr(null_config):
658
667
  ),
659
668
  )
660
669
  a: int
661
- b: Optional[Any] = None
670
+ b: Any = None
662
671
 
663
672
  js_in = {"a": 9, "b": None}
664
673
  objA = ObjA.from_json_like(js_in)
@@ -683,7 +692,7 @@ def test_from_json_like_optional_attr_with_is_multiple_both_none(null_config):
683
692
  ),
684
693
  )
685
694
  a: int
686
- b: Optional[Any] = None
695
+ b: Any = None
687
696
 
688
697
  js_in = {
689
698
  "a": 9,
@@ -711,7 +720,7 @@ def test_from_json_like_optional_attr_with_is_multiple_one_none(null_config):
711
720
  ),
712
721
  )
713
722
  a: int
714
- b: Optional[Any] = None
723
+ b: Any = None
715
724
 
716
725
  js_in = {
717
726
  "a": 9,
@@ -743,7 +752,7 @@ def test_from_json_like_optional_attr_with_is_multiple_one_none_and_shared_data_
743
752
  ),
744
753
  )
745
754
  a: int
746
- b: Optional[Any] = None
755
+ b: Any = None
747
756
 
748
757
  dcts = [
749
758
  {"name": "c1", "c": 2},
@@ -782,7 +791,7 @@ def test_from_json_like_optional_attr_with_is_multiple_all_none_and_shared_data_
782
791
  ),
783
792
  )
784
793
  a: int
785
- b: Optional[Any] = None
794
+ b: Any = None
786
795
 
787
796
  dcts = [
788
797
  {"name": "c1", "c": 2},
@@ -819,7 +828,7 @@ def test_from_json_like_optional_attr_with_shared_data_name(null_config):
819
828
  ),
820
829
  )
821
830
  a: int
822
- b: Optional[Any] = None
831
+ b: Any = None
823
832
 
824
833
  dcts = [
825
834
  {"name": "c1", "c": 2},
@@ -848,7 +857,7 @@ def test_from_json_like_optional_attr_with_enum(null_config):
848
857
  class ObjA(BaseJSONLikeSubClass):
849
858
  _child_objects = (ChildObjectSpec(name="b", class_obj=MyEnum, is_enum=True),)
850
859
  a: int
851
- b: Optional[Any] = None
860
+ b: Any = None
852
861
 
853
862
  js_in = {
854
863
  "a": 9,
@@ -1079,7 +1088,7 @@ def test_from_json_like_with_is_multiple_and_shared_data_dict_lookup(null_config
1079
1088
  class ObjB(BaseJSONLikeSubClass):
1080
1089
  name: str
1081
1090
  c: int
1082
- _hash_value: Optional[str] = None
1091
+ _hash_value: str | None = None
1083
1092
 
1084
1093
  @dataclass
1085
1094
  class ObjA(BaseJSONLikeSubClass):
@@ -1205,7 +1214,7 @@ def test_to_json_like_with_child_ref(null_config):
1205
1214
  class ObjB(BaseJSONLike):
1206
1215
  name: str
1207
1216
  c: int
1208
- obj_A: Optional[Any] = None
1217
+ obj_A: Any = None
1209
1218
 
1210
1219
  @dataclass
1211
1220
  class ObjA(BaseJSONLike):