hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. hpcflow/__pyinstaller/hook-hpcflow.py +1 -0
  2. hpcflow/_version.py +1 -1
  3. hpcflow/data/scripts/bad_script.py +2 -0
  4. hpcflow/data/scripts/do_nothing.py +2 -0
  5. hpcflow/data/scripts/env_specifier_test/input_file_generator_pass_env_spec.py +4 -0
  6. hpcflow/data/scripts/env_specifier_test/main_script_test_pass_env_spec.py +8 -0
  7. hpcflow/data/scripts/env_specifier_test/output_file_parser_pass_env_spec.py +4 -0
  8. hpcflow/data/scripts/env_specifier_test/v1/input_file_generator_basic.py +4 -0
  9. hpcflow/data/scripts/env_specifier_test/v1/main_script_test_direct_in_direct_out.py +7 -0
  10. hpcflow/data/scripts/env_specifier_test/v1/output_file_parser_basic.py +4 -0
  11. hpcflow/data/scripts/env_specifier_test/v2/main_script_test_direct_in_direct_out.py +7 -0
  12. hpcflow/data/scripts/input_file_generator_basic.py +3 -0
  13. hpcflow/data/scripts/input_file_generator_basic_FAIL.py +3 -0
  14. hpcflow/data/scripts/input_file_generator_test_stdout_stderr.py +8 -0
  15. hpcflow/data/scripts/main_script_test_direct_in.py +3 -0
  16. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2.py +6 -0
  17. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed.py +6 -0
  18. hpcflow/data/scripts/main_script_test_direct_in_direct_out_2_fail_allowed_group.py +7 -0
  19. hpcflow/data/scripts/main_script_test_direct_in_direct_out_3.py +6 -0
  20. hpcflow/data/scripts/main_script_test_direct_in_group_direct_out_3.py +6 -0
  21. hpcflow/data/scripts/main_script_test_direct_in_group_one_fail_direct_out_3.py +6 -0
  22. hpcflow/data/scripts/main_script_test_hdf5_in_obj_2.py +12 -0
  23. hpcflow/data/scripts/main_script_test_json_out_FAIL.py +3 -0
  24. hpcflow/data/scripts/main_script_test_shell_env_vars.py +12 -0
  25. hpcflow/data/scripts/main_script_test_std_out_std_err.py +6 -0
  26. hpcflow/data/scripts/output_file_parser_basic.py +3 -0
  27. hpcflow/data/scripts/output_file_parser_basic_FAIL.py +7 -0
  28. hpcflow/data/scripts/output_file_parser_test_stdout_stderr.py +8 -0
  29. hpcflow/data/scripts/script_exit_test.py +5 -0
  30. hpcflow/data/template_components/environments.yaml +1 -1
  31. hpcflow/sdk/__init__.py +5 -0
  32. hpcflow/sdk/app.py +150 -89
  33. hpcflow/sdk/cli.py +263 -84
  34. hpcflow/sdk/cli_common.py +99 -5
  35. hpcflow/sdk/config/callbacks.py +38 -1
  36. hpcflow/sdk/config/config.py +102 -13
  37. hpcflow/sdk/config/errors.py +19 -5
  38. hpcflow/sdk/config/types.py +3 -0
  39. hpcflow/sdk/core/__init__.py +25 -1
  40. hpcflow/sdk/core/actions.py +914 -262
  41. hpcflow/sdk/core/cache.py +76 -34
  42. hpcflow/sdk/core/command_files.py +14 -128
  43. hpcflow/sdk/core/commands.py +35 -6
  44. hpcflow/sdk/core/element.py +122 -50
  45. hpcflow/sdk/core/errors.py +58 -2
  46. hpcflow/sdk/core/execute.py +207 -0
  47. hpcflow/sdk/core/loop.py +408 -50
  48. hpcflow/sdk/core/loop_cache.py +4 -4
  49. hpcflow/sdk/core/parameters.py +382 -37
  50. hpcflow/sdk/core/run_dir_files.py +13 -40
  51. hpcflow/sdk/core/skip_reason.py +7 -0
  52. hpcflow/sdk/core/task.py +119 -30
  53. hpcflow/sdk/core/task_schema.py +68 -0
  54. hpcflow/sdk/core/test_utils.py +66 -27
  55. hpcflow/sdk/core/types.py +54 -1
  56. hpcflow/sdk/core/utils.py +78 -7
  57. hpcflow/sdk/core/workflow.py +1538 -336
  58. hpcflow/sdk/data/workflow_spec_schema.yaml +2 -0
  59. hpcflow/sdk/demo/cli.py +7 -0
  60. hpcflow/sdk/helper/cli.py +1 -0
  61. hpcflow/sdk/log.py +42 -15
  62. hpcflow/sdk/persistence/base.py +405 -53
  63. hpcflow/sdk/persistence/json.py +177 -52
  64. hpcflow/sdk/persistence/pending.py +237 -69
  65. hpcflow/sdk/persistence/store_resource.py +3 -2
  66. hpcflow/sdk/persistence/types.py +15 -4
  67. hpcflow/sdk/persistence/zarr.py +928 -81
  68. hpcflow/sdk/submission/jobscript.py +1408 -489
  69. hpcflow/sdk/submission/schedulers/__init__.py +40 -5
  70. hpcflow/sdk/submission/schedulers/direct.py +33 -19
  71. hpcflow/sdk/submission/schedulers/sge.py +51 -16
  72. hpcflow/sdk/submission/schedulers/slurm.py +44 -16
  73. hpcflow/sdk/submission/schedulers/utils.py +7 -2
  74. hpcflow/sdk/submission/shells/base.py +68 -20
  75. hpcflow/sdk/submission/shells/bash.py +222 -129
  76. hpcflow/sdk/submission/shells/powershell.py +200 -150
  77. hpcflow/sdk/submission/submission.py +852 -119
  78. hpcflow/sdk/submission/types.py +18 -21
  79. hpcflow/sdk/typing.py +24 -5
  80. hpcflow/sdk/utils/arrays.py +71 -0
  81. hpcflow/sdk/utils/deferred_file.py +55 -0
  82. hpcflow/sdk/utils/hashing.py +16 -0
  83. hpcflow/sdk/utils/patches.py +12 -0
  84. hpcflow/sdk/utils/strings.py +33 -0
  85. hpcflow/tests/api/test_api.py +32 -0
  86. hpcflow/tests/conftest.py +19 -0
  87. hpcflow/tests/data/multi_path_sequences.yaml +29 -0
  88. hpcflow/tests/data/workflow_test_run_abort.yaml +34 -35
  89. hpcflow/tests/schedulers/sge/test_sge_submission.py +36 -0
  90. hpcflow/tests/scripts/test_input_file_generators.py +282 -0
  91. hpcflow/tests/scripts/test_main_scripts.py +821 -70
  92. hpcflow/tests/scripts/test_non_snippet_script.py +46 -0
  93. hpcflow/tests/scripts/test_ouput_file_parsers.py +353 -0
  94. hpcflow/tests/shells/wsl/test_wsl_submission.py +6 -0
  95. hpcflow/tests/unit/test_action.py +176 -0
  96. hpcflow/tests/unit/test_app.py +20 -0
  97. hpcflow/tests/unit/test_cache.py +46 -0
  98. hpcflow/tests/unit/test_cli.py +133 -0
  99. hpcflow/tests/unit/test_config.py +122 -1
  100. hpcflow/tests/unit/test_element_iteration.py +47 -0
  101. hpcflow/tests/unit/test_jobscript_unit.py +757 -0
  102. hpcflow/tests/unit/test_loop.py +1332 -27
  103. hpcflow/tests/unit/test_meta_task.py +325 -0
  104. hpcflow/tests/unit/test_multi_path_sequences.py +229 -0
  105. hpcflow/tests/unit/test_parameter.py +13 -0
  106. hpcflow/tests/unit/test_persistence.py +190 -8
  107. hpcflow/tests/unit/test_run.py +109 -3
  108. hpcflow/tests/unit/test_run_directories.py +29 -0
  109. hpcflow/tests/unit/test_shell.py +20 -0
  110. hpcflow/tests/unit/test_submission.py +5 -76
  111. hpcflow/tests/unit/utils/test_arrays.py +40 -0
  112. hpcflow/tests/unit/utils/test_deferred_file_writer.py +34 -0
  113. hpcflow/tests/unit/utils/test_hashing.py +65 -0
  114. hpcflow/tests/unit/utils/test_patches.py +5 -0
  115. hpcflow/tests/unit/utils/test_redirect_std.py +50 -0
  116. hpcflow/tests/workflows/__init__.py +0 -0
  117. hpcflow/tests/workflows/test_directory_structure.py +31 -0
  118. hpcflow/tests/workflows/test_jobscript.py +332 -0
  119. hpcflow/tests/workflows/test_run_status.py +198 -0
  120. hpcflow/tests/workflows/test_skip_downstream.py +696 -0
  121. hpcflow/tests/workflows/test_submission.py +140 -0
  122. hpcflow/tests/workflows/test_workflows.py +142 -2
  123. hpcflow/tests/workflows/test_zip.py +18 -0
  124. hpcflow/viz_demo.ipynb +6587 -3
  125. {hpcflow_new2-0.2.0a190.dist-info → hpcflow_new2-0.2.0a199.dist-info}/METADATA +7 -4
  126. hpcflow_new2-0.2.0a199.dist-info/RECORD +221 -0
  127. hpcflow_new2-0.2.0a190.dist-info/RECORD +0 -165
  128. {hpcflow_new2-0.2.0a190.dist-info → hpcflow_new2-0.2.0a199.dist-info}/LICENSE +0 -0
  129. {hpcflow_new2-0.2.0a190.dist-info → hpcflow_new2-0.2.0a199.dist-info}/WHEEL +0 -0
  130. {hpcflow_new2-0.2.0a190.dist-info → hpcflow_new2-0.2.0a199.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,65 @@
1
+ from hpcflow.sdk.utils.hashing import get_hash
2
+
3
+
4
+ def test_get_hash_simple_types_is_int():
5
+ assert isinstance(get_hash(1), int)
6
+ assert isinstance(get_hash(3.2), int)
7
+ assert isinstance(get_hash("a"), int)
8
+ assert isinstance(get_hash("abc"), int)
9
+
10
+
11
+ def test_get_hash_compound_types_is_int():
12
+ assert isinstance(get_hash([1, 2, 3]), int)
13
+ assert isinstance(get_hash((1, 2, 3)), int)
14
+ assert isinstance(get_hash({1, 2, 3}), int)
15
+ assert isinstance(get_hash({"a": 1, "b": 2, "c": 3}), int)
16
+
17
+
18
+ def test_get_hash_nested_dict_is_int():
19
+ assert isinstance(get_hash({"a": {"b": {"c": [1, 2, 3, ("4", 5, 6)]}}}), int)
20
+
21
+
22
+ def test_get_hash_distinct_simple_types():
23
+ assert get_hash(1) != get_hash(2)
24
+ assert get_hash(2.2) != get_hash(2.3)
25
+ assert get_hash("a") != get_hash("b")
26
+ assert get_hash("abc") != get_hash("ABC")
27
+
28
+
29
+ def test_get_hash_distinct_compound_types():
30
+ assert get_hash([1, 2, 3]) != get_hash([1, 2, 4])
31
+ assert get_hash((1, 2, 3)) != get_hash((1, 2, 4))
32
+ assert get_hash({1, 2, 3}) != get_hash({1, 2, 4})
33
+ assert get_hash({"a": 1, "b": 2, "c": 3}) != get_hash({"a": 1, "b": 2, "c": 4})
34
+ assert get_hash({"a": {"b": {"c": [1, 2, 3, ("4", 5, 7)]}}}) == get_hash(
35
+ {"a": {"b": {"c": [1, 2, 3, ("4", 5, 7)]}}}
36
+ )
37
+ assert get_hash({"a": 1}) != get_hash(1) != get_hash("a")
38
+
39
+
40
+ def test_get_hash_equal_simple_types():
41
+ assert get_hash(1) == get_hash(1)
42
+ assert get_hash(2.2) == get_hash(2.2)
43
+ assert get_hash("a") == get_hash("a")
44
+ assert get_hash("abc") == get_hash("abc")
45
+
46
+
47
+ def test_get_hash_equal_compound_types():
48
+ assert get_hash([1, 2, 3]) == get_hash([1, 2, 3])
49
+ assert get_hash((1, 2, 3)) == get_hash((1, 2, 3))
50
+ assert get_hash({1, 2, 3}) == get_hash({1, 2, 3})
51
+ assert get_hash({"a": 1, "b": 2, "c": 3}) == get_hash({"a": 1, "b": 2, "c": 3})
52
+ assert get_hash({"a": {"b": {"c": [1, 2, 3, ("4", 5, 6)]}}}) == get_hash(
53
+ {"a": {"b": {"c": [1, 2, 3, ("4", 5, 6)]}}}
54
+ )
55
+
56
+
57
+ def test_get_hash_order_insensitivity():
58
+ assert get_hash({"a": 1, "b": 2}) == get_hash({"b": 2, "a": 1})
59
+ assert get_hash({1, 2, 3}) == get_hash({2, 3, 1})
60
+
61
+
62
+ def test_get_hash_order_sensitivity():
63
+ assert get_hash([1, 2, 3]) != get_hash([2, 3, 1])
64
+ assert get_hash((1, 2, 3)) != get_hash((2, 3, 1))
65
+ assert get_hash("abc") != get_hash("cba")
@@ -0,0 +1,5 @@
1
+ from hpcflow.sdk.utils.patches import resolve_path
2
+
3
+
4
+ def test_absolute_path():
5
+ assert resolve_path("my_file_path").is_absolute()
@@ -0,0 +1,50 @@
1
+ import sys
2
+
3
+ import pytest
4
+ from hpcflow.sdk.core.utils import redirect_std_to_file
5
+
6
+
7
+ def test_stdout_redirect(tmp_path):
8
+ file_name = tmp_path / "test.txt"
9
+ expected = "stdout"
10
+ with redirect_std_to_file(file_name, mode="w"):
11
+ print(expected)
12
+ with file_name.open("r") as fp:
13
+ contents = fp.read().strip()
14
+ assert contents == expected
15
+
16
+
17
+ def test_stderr_redirect(tmp_path):
18
+ file_name = tmp_path / "test.txt"
19
+ expected = "stderr"
20
+ with redirect_std_to_file(file_name, mode="w"):
21
+ print(expected, file=sys.stderr)
22
+ with file_name.open("r") as fp:
23
+ contents = fp.read().strip()
24
+ assert contents == expected
25
+
26
+
27
+ def test_exception_exits_with_code(tmp_path):
28
+ file_name = tmp_path / "test.txt"
29
+ with pytest.raises(SystemExit) as exc:
30
+ with redirect_std_to_file(file_name, mode="w"):
31
+ raise ValueError("oh no!")
32
+ assert exc.value.code == 1
33
+
34
+
35
+ def test_exception_prints_to_file(tmp_path):
36
+ file_name = tmp_path / "test.txt"
37
+ with pytest.raises(SystemExit):
38
+ with redirect_std_to_file(file_name, mode="w"):
39
+ raise ValueError("oh no!")
40
+ with file_name.open("r") as fp:
41
+ contents = fp.read().strip()
42
+ assert 'ValueError("oh no!")' in contents
43
+
44
+
45
+ def test_file_not_created(tmp_path):
46
+ file_name = tmp_path / "test.txt"
47
+ assert not file_name.is_file()
48
+ with redirect_std_to_file(file_name, mode="w"):
49
+ pass
50
+ assert not file_name.is_file()
File without changes
@@ -0,0 +1,31 @@
1
+ """Tests concerning the directory structure of a created or submitted workflow"""
2
+
3
+ import os
4
+ from pathlib import Path
5
+ import pytest
6
+
7
+ from hpcflow.sdk.core.test_utils import (
8
+ make_test_data_YAML_workflow,
9
+ make_workflow_to_run_command,
10
+ )
11
+
12
+
13
+ @pytest.mark.integration
14
+ def test_std_stream_file_not_created(tmp_path, new_null_config):
15
+ """Normally, the app standard stream file should not be written."""
16
+ wk = make_test_data_YAML_workflow("workflow_1.yaml", path=tmp_path)
17
+ wk.submit(wait=True, add_to_known=False)
18
+ run = wk.get_all_EARs()[0]
19
+ std_stream_path = run.get_app_std_path()
20
+ assert not std_stream_path.is_file()
21
+
22
+
23
+ @pytest.mark.integration
24
+ def test_std_stream_file_created_on_exception_raised(tmp_path, new_null_config):
25
+ command = 'wkflow_app --std-stream "$HPCFLOW_RUN_STD_PATH" internal noop --raise'
26
+ wk = make_workflow_to_run_command(command=command, path=tmp_path)
27
+ wk.submit(wait=True, add_to_known=False)
28
+ run = wk.get_all_EARs()[0]
29
+ std_stream_path = run.get_app_std_path()
30
+ assert std_stream_path.is_file()
31
+ assert "ValueError: internal noop raised!" in std_stream_path.read_text()
@@ -1,8 +1,11 @@
1
1
  import os
2
+ import sys
2
3
  from pathlib import Path
3
4
  import pytest
4
5
 
5
6
  from hpcflow.app import app as hf
7
+ from hpcflow.sdk.core import SKIPPED_EXIT_CODE
8
+ from hpcflow.sdk.core.skip_reason import SkipReason
6
9
 
7
10
 
8
11
  @pytest.mark.integration
@@ -21,3 +24,332 @@ def test_action_exit_code_parsing(null_config, tmp_path: Path, exit_code: int):
21
24
  # exit code from bash wraps around:
22
25
  exit_code %= 256
23
26
  assert recorded_exit == exit_code
27
+
28
+
29
+ @pytest.mark.integration
30
+ def test_bad_action_py_script_exit_code(null_config, tmp_path):
31
+ s1 = hf.TaskSchema(
32
+ objective="t1",
33
+ actions=[
34
+ hf.Action(
35
+ script="<<script:bad_script.py>>", # raises SyntaxError
36
+ script_exe="python_script",
37
+ environments=[hf.ActionEnvironment(environment="python_env")],
38
+ )
39
+ ],
40
+ )
41
+ t1 = hf.Task(schema=[s1])
42
+ wk = hf.Workflow.from_template_data(
43
+ tasks=[t1], template_name="bad_script_test", path=tmp_path
44
+ )
45
+ wk.submit(wait=True, add_to_known=False)
46
+ recorded_exit = wk.get_EARs_from_IDs([0])[0].exit_code
47
+ assert recorded_exit == 1
48
+
49
+
50
+ @pytest.mark.integration
51
+ @pytest.mark.parametrize("exit_code", [0, 1, 98, -1, -123124])
52
+ def test_action_py_script_specified_exit_code(null_config, tmp_path, exit_code):
53
+ s1 = hf.TaskSchema(
54
+ objective="t1",
55
+ inputs=[hf.SchemaInput("exit_code")],
56
+ actions=[
57
+ hf.Action(
58
+ script="<<script:script_exit_test.py>>",
59
+ script_exe="python_script",
60
+ script_data_in="direct",
61
+ environments=[hf.ActionEnvironment(environment="python_env")],
62
+ )
63
+ ],
64
+ )
65
+ t1 = hf.Task(schema=[s1], inputs={"exit_code": exit_code})
66
+ wk = hf.Workflow.from_template_data(
67
+ tasks=[t1], template_name="script_exit_test", path=tmp_path
68
+ )
69
+ wk.submit(wait=True, add_to_known=False)
70
+ recorded_exit = wk.get_EARs_from_IDs([0])[0].exit_code
71
+ if os.name == "posix":
72
+ # exit code from bash wraps around:
73
+ exit_code %= 256
74
+ assert recorded_exit == exit_code
75
+
76
+
77
+ @pytest.mark.integration
78
+ def test_skipped_action_same_element(null_config, tmp_path):
79
+ s1 = hf.TaskSchema(
80
+ objective="t1",
81
+ inputs=[hf.SchemaInput("p1")],
82
+ outputs=[hf.SchemaOutput("p2"), hf.SchemaOutput("p3")],
83
+ actions=[
84
+ hf.Action(
85
+ commands=[
86
+ hf.Command(
87
+ command=f"echo <<parameter:p1>>", stdout="<<parameter:p2>>"
88
+ ),
89
+ hf.Command(command=f"exit 1"),
90
+ ],
91
+ ),
92
+ hf.Action( # should be skipped
93
+ commands=[
94
+ hf.Command(
95
+ command=f"echo <<parameter:p2>>", stdout="<<parameter:p3>>"
96
+ ),
97
+ hf.Command(command=f"exit 0"), # exit code should be ignored
98
+ ],
99
+ ),
100
+ ],
101
+ )
102
+ t1 = hf.Task(schema=s1, inputs={"p1": 101})
103
+ wk = hf.Workflow.from_template_data(
104
+ tasks=[t1], template_name="test_skip", path=tmp_path
105
+ )
106
+ wk.submit(wait=True, add_to_known=False, status=False)
107
+
108
+ runs = wk.get_EARs_from_IDs([0, 1])
109
+ exit_codes = [i.exit_code for i in runs]
110
+ is_skipped = [i.skip for i in runs]
111
+
112
+ assert exit_codes == [1, SKIPPED_EXIT_CODE]
113
+ assert is_skipped == [0, 1]
114
+
115
+
116
+ @pytest.mark.integration
117
+ def test_two_skipped_actions_same_element(null_config, tmp_path):
118
+ s1 = hf.TaskSchema(
119
+ objective="t1",
120
+ inputs=[hf.SchemaInput("p1")],
121
+ outputs=[hf.SchemaOutput("p2"), hf.SchemaOutput("p3"), hf.SchemaOutput("p4")],
122
+ actions=[
123
+ hf.Action(
124
+ commands=[
125
+ hf.Command(
126
+ command=f"echo <<parameter:p1>>", stdout="<<parameter:p2>>"
127
+ ),
128
+ hf.Command(command=f"exit 1"),
129
+ ],
130
+ ),
131
+ hf.Action( # should be skipped
132
+ commands=[
133
+ hf.Command(
134
+ command=f"echo <<parameter:p2>>", stdout="<<parameter:p3>>"
135
+ ),
136
+ hf.Command(command=f"exit 0"), # exit code should be ignored
137
+ ],
138
+ ),
139
+ hf.Action( # should be skipped
140
+ commands=[
141
+ hf.Command(
142
+ command=f"echo <<parameter:p3>>", stdout="<<parameter:p4>>"
143
+ ),
144
+ hf.Command(command=f"exit 0"), # exit code should be ignored
145
+ ],
146
+ ),
147
+ ],
148
+ )
149
+ t1 = hf.Task(schema=s1, inputs={"p1": 101})
150
+ wk = hf.Workflow.from_template_data(
151
+ tasks=[t1], template_name="test_skip_two_actions", path=tmp_path
152
+ )
153
+ wk.submit(wait=True, add_to_known=False, status=False)
154
+
155
+ runs = wk.get_EARs_from_IDs([0, 1, 2])
156
+ exit_codes = [i.exit_code for i in runs]
157
+ skip_reasons = [i.skip_reason for i in runs]
158
+
159
+ assert exit_codes == [1, SKIPPED_EXIT_CODE, SKIPPED_EXIT_CODE]
160
+ assert skip_reasons == [
161
+ SkipReason.NOT_SKIPPED,
162
+ SkipReason.UPSTREAM_FAILURE,
163
+ SkipReason.UPSTREAM_FAILURE,
164
+ ]
165
+
166
+
167
+ @pytest.mark.integration
168
+ @pytest.mark.skipif(
169
+ condition=sys.platform == "win32",
170
+ reason="`combine_jobscript_std` not implemented on Windows.",
171
+ )
172
+ def test_combine_jobscript_std_true(null_config, tmp_path):
173
+ out_msg = "hello stdout!"
174
+ err_msg = "hello stderr!"
175
+ s1 = hf.TaskSchema(
176
+ objective="t1",
177
+ actions=[
178
+ hf.Action(
179
+ commands=[
180
+ hf.Command(command=f'echo "{out_msg}"'),
181
+ hf.Command(command=f'>&2 echo "{err_msg}"'),
182
+ ],
183
+ )
184
+ ],
185
+ )
186
+ t1 = hf.Task(schema=s1)
187
+ wk = hf.Workflow.from_template_data(
188
+ tasks=[t1],
189
+ template_name="test_combine_jobscript_std",
190
+ path=tmp_path,
191
+ resources={"any": {"combine_jobscript_std": True}},
192
+ )
193
+ wk.submit(wait=True, add_to_known=False, status=False)
194
+
195
+ jobscript = wk.submissions[0].jobscripts[0]
196
+
197
+ assert jobscript.resources.combine_jobscript_std
198
+
199
+ out_err_path = jobscript.get_std_out_err_path()
200
+ out_path = jobscript._get_stdout_path()
201
+ err_path = jobscript._get_stderr_path()
202
+
203
+ assert out_err_path.is_file()
204
+ assert not out_path.is_file()
205
+ assert not err_path.is_file()
206
+
207
+ assert out_err_path.read_text().strip() == f"{out_msg}\n{err_msg}"
208
+
209
+
210
+ @pytest.mark.integration
211
+ def test_combine_jobscript_std_false(null_config, tmp_path):
212
+ out_msg = "hello stdout!"
213
+ err_msg = "hello stderr!"
214
+ s1 = hf.TaskSchema(
215
+ objective="t1",
216
+ actions=[
217
+ hf.Action(
218
+ commands=[
219
+ hf.Command(command=f'echo "{out_msg}"'),
220
+ hf.Command(command=f'>&2 echo "{err_msg}"'),
221
+ ],
222
+ rules=[
223
+ hf.ActionRule(
224
+ rule=hf.Rule(
225
+ path="resources.os_name",
226
+ condition={"value.equal_to": "posix"},
227
+ )
228
+ )
229
+ ],
230
+ ),
231
+ hf.Action(
232
+ commands=[
233
+ hf.Command(command=f'Write-Output "{out_msg}"'),
234
+ hf.Command(command=f'$host.ui.WriteErrorLine("{err_msg}")'),
235
+ ],
236
+ rules=[
237
+ hf.ActionRule(
238
+ rule=hf.Rule(
239
+ path="resources.os_name",
240
+ condition={"value.equal_to": "nt"},
241
+ )
242
+ )
243
+ ],
244
+ ),
245
+ ],
246
+ )
247
+ t1 = hf.Task(schema=s1)
248
+ wk = hf.Workflow.from_template_data(
249
+ tasks=[t1],
250
+ template_name="test_combine_jobscript_std",
251
+ path=tmp_path,
252
+ resources={"any": {"combine_jobscript_std": False}},
253
+ )
254
+ wk.submit(wait=True, add_to_known=False, status=False)
255
+
256
+ jobscript = wk.submissions[0].jobscripts[0]
257
+
258
+ assert not jobscript.resources.combine_jobscript_std
259
+
260
+ out_err_path = jobscript.direct_std_out_err_path
261
+ out_path = jobscript.direct_stdout_path
262
+ err_path = jobscript.direct_stderr_path
263
+
264
+ assert not out_err_path.is_file()
265
+ assert out_path.is_file()
266
+ assert err_path.is_file()
267
+
268
+ assert out_path.read_text().strip() == out_msg
269
+ assert err_path.read_text().strip() == err_msg
270
+
271
+
272
+ @pytest.mark.integration
273
+ def test_write_app_logs_true(null_config, tmp_path):
274
+
275
+ p1_vals = [101, 102]
276
+ t1 = hf.Task(
277
+ schema=hf.task_schemas.test_t1_conditional_OS,
278
+ sequences=[hf.ValueSequence("inputs.p1", values=p1_vals)],
279
+ )
280
+ wk = hf.Workflow.from_template_data(
281
+ tasks=[t1],
282
+ template_name="test_write_app_logs",
283
+ path=tmp_path,
284
+ config={
285
+ "log_file_level": "debug"
286
+ }, # ensure there is something to write to the log
287
+ resources={"any": {"write_app_logs": True}},
288
+ )
289
+ wk.submit(wait=True, add_to_known=False, status=False)
290
+
291
+ run_0 = wk.tasks[0].elements[0].action_runs[0]
292
+ run_1 = wk.tasks[0].elements[1].action_runs[0]
293
+
294
+ run_0_log_path = run_0.get_app_log_path()
295
+ run_1_log_path = run_1.get_app_log_path()
296
+
297
+ assert run_0_log_path.is_file()
298
+ assert run_1_log_path.is_file()
299
+
300
+
301
+ @pytest.mark.integration
302
+ def test_write_app_logs_false(null_config, tmp_path):
303
+
304
+ p1_vals = [101, 102]
305
+ t1 = hf.Task(
306
+ schema=hf.task_schemas.test_t1_conditional_OS,
307
+ sequences=[hf.ValueSequence("inputs.p1", values=p1_vals)],
308
+ )
309
+ wk = hf.Workflow.from_template_data(
310
+ tasks=[t1],
311
+ template_name="test_write_app_logs",
312
+ path=tmp_path,
313
+ config={
314
+ "log_file_level": "debug"
315
+ }, # ensure there is something to write to the log
316
+ resources={"any": {"write_app_logs": False}},
317
+ )
318
+ wk.submit(wait=True, add_to_known=False, status=False)
319
+
320
+ run_0 = wk.tasks[0].elements[0].action_runs[0]
321
+ run_1 = wk.tasks[0].elements[1].action_runs[0]
322
+
323
+ run_0_log_path = run_0.get_app_log_path()
324
+ run_1_log_path = run_1.get_app_log_path()
325
+
326
+ assert not wk.submissions[0].app_log_path.is_dir()
327
+ assert not run_0_log_path.is_file()
328
+ assert not run_1_log_path.is_file()
329
+
330
+
331
+ @pytest.mark.integration
332
+ def test_jobscript_start_end_times_equal_to_first_and_last_run_start_end_times(
333
+ null_config, tmp_path
334
+ ):
335
+
336
+ t1 = hf.Task(
337
+ schema=hf.task_schemas.test_t1_conditional_OS,
338
+ sequences=[hf.ValueSequence(path="inputs.p1", values=list(range(2)))],
339
+ )
340
+ wk = hf.Workflow.from_template_data(
341
+ template_name="test_jobscript_start_end_times",
342
+ path=tmp_path,
343
+ tasks=[t1],
344
+ )
345
+ wk.submit(wait=True, add_to_known=False, status=False)
346
+
347
+ js = wk.submissions[0].jobscripts[0]
348
+ runs = wk.get_all_EARs()
349
+ assert len(runs) == 2
350
+
351
+ # jobsript has two runs, so start time should be start time of first run:
352
+ assert js.start_time == runs[0].start_time
353
+
354
+ # ...and end time should be end time of second run:
355
+ assert js.end_time == runs[1].end_time
@@ -0,0 +1,198 @@
1
+ import os
2
+ from textwrap import dedent
3
+
4
+ import pytest
5
+
6
+ from hpcflow.app import app as hf
7
+ from hpcflow.sdk.core.actions import EARStatus
8
+
9
+
10
+ @pytest.mark.integration
11
+ @pytest.mark.parametrize("combine_scripts", [True, False])
12
+ def test_run_status_fail_when_missing_script_output_data_file(
13
+ null_config, tmp_path, combine_scripts
14
+ ):
15
+
16
+ s1 = hf.TaskSchema(
17
+ objective="t1",
18
+ outputs=[hf.SchemaOutput(parameter=hf.Parameter("p1"))],
19
+ actions=[
20
+ hf.Action(
21
+ script="<<script:main_script_test_json_out_FAIL.py>>",
22
+ script_data_out="json",
23
+ script_exe="python_script",
24
+ environments=[hf.ActionEnvironment(environment="python_env")],
25
+ requires_dir=True,
26
+ )
27
+ ],
28
+ )
29
+
30
+ tasks = [
31
+ hf.Task(s1), # will fail due to not generaing an output data file
32
+ ]
33
+
34
+ wk = hf.Workflow.from_template_data(
35
+ template_name="test_run_status_fail_missing_script_output_file",
36
+ path=tmp_path,
37
+ tasks=tasks,
38
+ resources={
39
+ "any": {
40
+ "write_app_logs": True,
41
+ "skip_downstream_on_failure": False,
42
+ "combine_scripts": combine_scripts,
43
+ }
44
+ },
45
+ )
46
+ wk.submit(wait=True, add_to_known=False, status=False)
47
+ runs = wk.get_all_EARs()
48
+ assert runs[0].status is EARStatus.error
49
+
50
+
51
+ @pytest.mark.integration
52
+ @pytest.mark.parametrize("combine_scripts", [True, False])
53
+ def test_run_status_fail_when_missing_script_output_data_file_OFP_fail(
54
+ null_config, tmp_path, combine_scripts
55
+ ):
56
+
57
+ out_file_name = "my_output_file.txt"
58
+ out_file = hf.FileSpec(label="my_output_file", name=out_file_name)
59
+
60
+ if os.name == "nt":
61
+ cmd = f"Set-Content -Path {out_file_name} -Value (<<parameter:p1>> + 100)"
62
+ else:
63
+ cmd = f"echo $(( <<parameter:p1>> + 100 )) > {out_file_name}"
64
+
65
+ # this script parses the output file but then deletes this file so it can't be saved!
66
+ act = hf.Action(
67
+ commands=[hf.Command(cmd)],
68
+ output_file_parsers=[
69
+ hf.OutputFileParser(
70
+ output_files=[out_file],
71
+ output=hf.Parameter("p2"),
72
+ script="<<script:output_file_parser_basic_FAIL.py>>",
73
+ save_files=True,
74
+ ),
75
+ ],
76
+ environments=[hf.ActionEnvironment(environment="python_env")],
77
+ )
78
+
79
+ s1 = hf.TaskSchema(
80
+ objective="t1",
81
+ inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
82
+ outputs=[hf.SchemaInput(parameter=hf.Parameter("p2"))],
83
+ actions=[act],
84
+ )
85
+ t1 = hf.Task(schema=s1, inputs={"p1": 100})
86
+
87
+ wk = hf.Workflow.from_template_data(
88
+ template_name="test_run_status_fail_missing_OFP_save_file",
89
+ path=tmp_path,
90
+ tasks=[t1],
91
+ resources={
92
+ "any": {
93
+ "write_app_logs": True,
94
+ "skip_downstream_on_failure": False,
95
+ "combine_scripts": combine_scripts,
96
+ }
97
+ },
98
+ )
99
+ wk.submit(wait=True, add_to_known=False, status=False)
100
+ runs = wk.get_all_EARs()
101
+ assert runs[0].status is EARStatus.success
102
+ assert runs[1].status is EARStatus.error
103
+
104
+
105
+ @pytest.mark.integration
106
+ @pytest.mark.parametrize("combine_scripts", [True, False])
107
+ def test_run_status_fail_when_missing_IFG_input_file(
108
+ null_config, tmp_path, combine_scripts
109
+ ):
110
+
111
+ inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
112
+
113
+ if os.name == "nt":
114
+ cmd = dedent(
115
+ """\
116
+ try {
117
+ Get-Content "<<file:my_input_file>>" -ErrorAction Stop
118
+ } catch {
119
+ Write-Host "File does not exist."
120
+ exit 1
121
+ }
122
+ """
123
+ )
124
+ else:
125
+ cmd = "cat <<file:my_input_file>>"
126
+
127
+ # this script silently fails to generate the input file!
128
+ s1 = hf.TaskSchema(
129
+ objective="t1",
130
+ inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
131
+ actions=[
132
+ hf.Action(
133
+ commands=[hf.Command(cmd)],
134
+ input_file_generators=[
135
+ hf.InputFileGenerator(
136
+ input_file=inp_file,
137
+ inputs=[hf.Parameter("p1")],
138
+ script="<<script:input_file_generator_basic_FAIL.py>>",
139
+ ),
140
+ ],
141
+ environments=[hf.ActionEnvironment(environment="python_env")],
142
+ )
143
+ ],
144
+ )
145
+ t1 = hf.Task(schema=s1, inputs={"p1": 100})
146
+ wk = hf.Workflow.from_template_data(
147
+ template_name="test_run_status_fail_missing_IFG_save_file",
148
+ path=tmp_path,
149
+ tasks=[t1],
150
+ resources={
151
+ "any": {
152
+ "write_app_logs": True,
153
+ "skip_downstream_on_failure": False,
154
+ "combine_scripts": combine_scripts,
155
+ }
156
+ },
157
+ )
158
+ wk.submit(wait=True, add_to_known=False, status=False)
159
+ runs = wk.get_all_EARs()
160
+ assert runs[0].status is EARStatus.error # no input file to save
161
+ assert runs[1].status is EARStatus.error # no input file to consume
162
+
163
+
164
+ @pytest.mark.integration
165
+ @pytest.mark.parametrize("combine_scripts", [True, False])
166
+ def test_run_status_fail_when_action_save_file(null_config, tmp_path, combine_scripts):
167
+
168
+ my_file = hf.FileSpec(label="my_file", name="my_file.txt")
169
+
170
+ # this script does not generate a file that can be saved:
171
+ s1 = hf.TaskSchema(
172
+ objective="t1",
173
+ actions=[
174
+ hf.Action(
175
+ script="<<script:do_nothing.py>>",
176
+ script_exe="python_script",
177
+ environments=[hf.ActionEnvironment(environment="python_env")],
178
+ requires_dir=True,
179
+ save_files=[my_file],
180
+ )
181
+ ],
182
+ )
183
+ t1 = hf.Task(schema=s1)
184
+ wk = hf.Workflow.from_template_data(
185
+ template_name="test_run_status_fail_missing_action_save_file",
186
+ path=tmp_path,
187
+ tasks=[t1],
188
+ resources={
189
+ "any": {
190
+ "write_app_logs": True,
191
+ "skip_downstream_on_failure": False,
192
+ "combine_scripts": combine_scripts,
193
+ }
194
+ },
195
+ )
196
+ wk.submit(wait=True, add_to_known=False, status=False)
197
+ runs = wk.get_all_EARs()
198
+ assert runs[0].status is EARStatus.error # no file to save