hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/tests/workflows/test_submission.py

@@ -0,0 +1,140 @@
+import os
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+
+
+@pytest.mark.integration
+def test_zarr_metadata_file_modification_times_many_jobscripts(null_config, tmp_path):
+    """Test that root group attributes are modified first, then individual jobscript
+    at-submit-metadata chunk files, then the submission at-submit-metadata group
+    attributes."""
+
+    num_js = 30
+    t1 = hf.Task(
+        schema=hf.task_schemas.test_t1_conditional_OS,
+        inputs={"p1": 100},
+        sequences=[
+            hf.ValueSequence(
+                path="resources.any.resources_id", values=list(range(num_js))
+            )
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_zarr_metadata_attrs_modified_times",
+        path=tmp_path,
+        tasks=[t1],
+        store="zarr",
+    )
+    wk.submit(add_to_known=False, status=False, cancel=True)
+
+    mtime_meta_group = Path(wk.path).joinpath(".zattrs").stat().st_mtime
+    mtime_mid_jobscript_chunk = (
+        wk._store._get_jobscripts_at_submit_metadata_arr_path(0)
+        .joinpath(str(int(num_js / 2)))
+        .stat()
+        .st_mtime
+    )
+    mtime_submission_group = (
+        wk._store._get_submission_metadata_group_path(0)
+        .joinpath(".zattrs")
+        .stat()
+        .st_mtime
+    )
+    assert mtime_meta_group < mtime_mid_jobscript_chunk < mtime_submission_group
+
+
+@pytest.mark.integration
+def test_json_metadata_file_modification_times_many_jobscripts(null_config, tmp_path):
+    """Test that the metadata.json file is modified first, then the submissions.json
+    file."""
+
+    num_js = 30
+    t1 = hf.Task(
+        schema=hf.task_schemas.test_t1_conditional_OS,
+        inputs={"p1": 100},
+        sequences=[
+            hf.ValueSequence(
+                path="resources.any.resources_id", values=list(range(num_js))
+            )
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_zarr_metadata_attrs_modified_times",
+        path=tmp_path,
+        tasks=[t1],
+        store="json",
+    )
+    wk.submit(add_to_known=False, status=False, cancel=True)
+
+    mtime_meta = Path(wk.path).joinpath("metadata.json").stat().st_mtime
+    mtime_subs = Path(wk.path).joinpath("submissions.json").stat().st_mtime
+    assert mtime_meta < mtime_subs
+
+
+@pytest.mark.integration
+def test_subission_start_end_times_equal_to_first_and_last_jobscript_start_end_times(
+    null_config, tmp_path
+):
+    num_js = 2
+    t1 = hf.Task(
+        schema=hf.task_schemas.test_t1_conditional_OS,
+        inputs={"p1": 100},
+        sequences=[
+            hf.ValueSequence(
+                path="resources.any.resources_id", values=list(range(num_js))
+            )
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_subission_start_end_times",
+        path=tmp_path,
+        tasks=[t1],
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    sub = wk.submissions[0]
+    jobscripts = sub.jobscripts
+
+    assert len(jobscripts) == num_js
+
+    # submission has two jobscripts, so start time should be start time of first jobscript:
+    assert sub.start_time == jobscripts[0].start_time
+
+    # ...and end time should be end time of second jobscript:
+    assert sub.end_time == jobscripts[1].end_time
+
+
+@pytest.mark.integration
+def test_multiple_jobscript_functions_files(null_config, tmp_path):
+    if os.name == "nt":
+        shell_exes = ["powershell.exe", "pwsh.exe", "pwsh.exe"]
+    else:
+        shell_exes = ["/bin/bash", "bash", "bash"]
+    t1 = hf.Task(
+        schema=hf.task_schemas.test_t1_conditional_OS,
+        inputs={"p1": 100},
+        sequences=[
+            hf.ValueSequence(
+                path="resources.any.shell_args.executable",
+                values=shell_exes,
+            )
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_multi_js_funcs_files",
+        path=tmp_path,
+        tasks=[t1],
+        store="json",
+    )
+    wk.submit(add_to_known=True, status=False, cancel=True)
+
+    sub_js = wk.submissions[0].jobscripts
+    assert len(sub_js) == 2
+
+    funcs_0 = sub_js[0].jobscript_functions_path
+    funcs_1 = sub_js[1].jobscript_functions_path
+
+    assert funcs_0.is_file()
+    assert funcs_1.is_file()
+    assert funcs_0 != funcs_1

hpcflow/tests/workflows/test_workflows.py

@@ -1,8 +1,11 @@
 import os
+import sys
+from pathlib import Path
 import time
 import pytest
 from hpcflow.app import app as hf
-from hpcflow.sdk.core.actions import EARStatus
+from hpcflow.sdk.core.enums import EARStatus
+from hpcflow.sdk.core.skip_reason import SkipReason
 from hpcflow.sdk.core.test_utils import (
     P1_parameter_cls as P1,
     P1_sub_parameter_cls as P1_sub,
@@ -11,26 +14,30 @@ from hpcflow.sdk.core.test_utils import (
 
 
 @pytest.mark.integration
-def test_workflow_1(tmp_path, new_null_config):
+def test_workflow_1(tmp_path: Path, new_null_config):
     wk = make_test_data_YAML_workflow("workflow_1.yaml", path=tmp_path)
     wk.submit(wait=True, add_to_known=False)
-    assert wk.tasks[0].elements[0].outputs.p2.value == "201"
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"
 
 
 @pytest.mark.integration
-def test_workflow_1_with_working_dir_with_spaces(tmp_path, new_null_config):
+def test_workflow_1_with_working_dir_with_spaces(tmp_path: Path, new_null_config):
     workflow_dir = tmp_path / "sub path with spaces"
     workflow_dir.mkdir()
     wk = make_test_data_YAML_workflow("workflow_1.yaml", path=workflow_dir)
     wk.submit(wait=True, add_to_known=False)
-    assert wk.tasks[0].elements[0].outputs.p2.value == "201"
+    p2 = wk.tasks[0].elements[0].outputs.p2
+    assert isinstance(p2, hf.ElementParameter)
+    assert p2.value == "201"
 
 
 @pytest.mark.integration
-@pytest.mark.skip(
-    reason="Sometimes fails on MacOS GHAs runner; too slow on Windows + Linux"
+@pytest.mark.skipif(
+    sys.platform == "darwin", reason="fails/too slow; need to investigate"
 )
-def test_run_abort(tmp_path, new_null_config):
+def test_run_abort(tmp_path: Path, new_null_config):
     wk = make_test_data_YAML_workflow("workflow_test_run_abort.yaml", path=tmp_path)
     wk.submit(add_to_known=False)
 
@@ -56,7 +63,7 @@ def test_run_abort(tmp_path, new_null_config):
 
 @pytest.mark.integration
 @pytest.mark.parametrize("store", ["json", "zarr"])
-def test_multi_command_action_stdout_parsing(null_config, tmp_path, store):
+def test_multi_command_action_stdout_parsing(null_config, tmp_path: Path, store: str):
     if os.name == "nt":
         cmds = [
             "Write-Output (<<parameter:p1>> + 100)",
@@ -98,7 +105,7 @@ def test_multi_command_action_stdout_parsing(null_config, tmp_path, store):
 
 @pytest.mark.integration
 @pytest.mark.parametrize("store", ["json", "zarr"])
-def test_element_get_group(null_config, tmp_path, store):
+def test_element_get_group(null_config, tmp_path: Path, store: str):
     if os.name == "nt":
         cmd = "Write-Output (<<parameter:p1c>> + 100)"
     else:
@@ -146,7 +153,7 @@ def test_element_get_group(null_config, tmp_path, store):
 
 
 @pytest.mark.integration
-def test_element_get_sub_object_group(null_config, tmp_path):
+def test_element_get_sub_object_group(null_config, tmp_path: Path):
     if os.name == "nt":
         cmd = "Write-Output (<<parameter:p1c>> + 100)"
     else:
@@ -196,7 +203,7 @@ def test_element_get_sub_object_group(null_config, tmp_path):
 
 
 @pytest.mark.integration
-def test_element_get_sub_data_group(null_config, tmp_path):
+def test_element_get_sub_data_group(null_config, tmp_path: Path):
     if os.name == "nt":
         cmd = "Write-Output (<<parameter:p1c>> + 100)"
     else:
@@ -243,7 +250,7 @@ def test_element_get_sub_data_group(null_config, tmp_path):
 
 
 @pytest.mark.integration
-def test_input_source_labels_and_groups(null_config, tmp_path):
+def test_input_source_labels_and_groups(null_config, tmp_path: Path):
     """This is structurally the same as the `fit_yield_functions` MatFlow workflow."""
     if os.name == "nt":
         cmds = [
@@ -338,7 +345,7 @@ def test_input_source_labels_and_groups(null_config, tmp_path):
 
 
 @pytest.mark.integration
-def test_loop_simple(null_config, tmp_path):
+def test_loop_simple(null_config, tmp_path: Path):
     if os.name == "nt":
         cmd = "Write-Output (<<parameter:p1>> + 100)"
     else:
@@ -362,7 +369,8 @@ def test_loop_simple(null_config, tmp_path):
 
 
 @pytest.mark.integration
-def test_loop_termination_multi_element(null_config, tmp_path):
+@pytest.mark.skip(reason="need to fix loop termination for multiple elements")
+def test_loop_termination_multi_element(null_config, tmp_path: Path):
     if os.name == "nt":
         cmds = [
             "Write-Output (<<parameter:p1>> + 100)",
@@ -417,3 +425,140 @@ def test_loop_termination_multi_element(null_config, tmp_path):
     assert elem_1.iterations[0].action_runs[0].status is EARStatus.success
     assert elem_1.iterations[1].action_runs[0].status is EARStatus.success
     assert elem_1.iterations[2].action_runs[0].status is EARStatus.skipped
+
+
+@pytest.mark.integration
+def test_input_file_generator_no_errors_on_skip(null_config, tmp_path):
+    """i.e. we don't try to save a file that hasn't been created because the run was
+    skipped"""
+
+    inp_file = hf.FileSpec(label="my_input_file", name="my_input_file.txt")
+
+    if os.name == "nt":
+        cmds = (
+            "Write-Output ((<<parameter:p0>> + 1))",
+            "Get-Content <<file:my_input_file>>",
+        )
+    else:
+        cmds = ('echo "$((<<parameter:p0>> + 1))"', "cat <<file:my_input_file>>")
+
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p0"))],
+        outputs=[hf.SchemaOutput(parameter=hf.Parameter("p1"))],
+        actions=[
+            hf.Action(
+                commands=[hf.Command(command=cmds[0], stdout="<<parameter:p1>>")],
+            )
+        ],
+    )
+
+    s2 = hf.TaskSchema(
+        objective="t2",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        outputs=[hf.SchemaOutput(parameter=hf.Parameter("p0"))],
+        actions=[
+            hf.Action(
+                commands=[hf.Command(cmds[1], stdout="<<int(parameter:p0)>>")],
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=inp_file,
+                        inputs=[hf.Parameter("p1")],
+                        script="<<script:input_file_generator_basic.py>>",
+                    ),
+                ],
+                environments=[hf.ActionEnvironment(environment="python_env")],
+            )
+        ],
+    )
+    p0_val = 100
+    t1 = hf.Task(schema=s1, inputs={"p0": p0_val})
+    t2 = hf.Task(schema=s2)
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1, t2],
+        loops=[
+            hf.Loop(
+                tasks=[0, 1],
+                num_iterations=2,
+                termination={"path": "outputs.p0", "condition": {"value.equal_to": 101}},
+            )
+        ],
+        template_name="input_file_generator_skip_test",
+        path=tmp_path,
+    )
+
+    wk.submit(wait=True, add_to_known=False)
+
+    # check correct runs are set to skip due to loop termination:
+    runs = wk.get_all_EARs()
+    assert runs[0].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[1].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[2].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[3].skip_reason is SkipReason.LOOP_TERMINATION
+    assert runs[4].skip_reason is SkipReason.LOOP_TERMINATION
+    assert runs[5].skip_reason is SkipReason.LOOP_TERMINATION
+
+    # run 4 is the input file generator of the second iteration, which should be skipped
+    # check no error from trying to save the input file:
+    std_stream_path = runs[4].get_app_std_path()
+    if std_stream_path.is_file():
+        assert "FileNotFoundError" not in std_stream_path.read_text()
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("store", ["zarr", "json"])
+def test_get_text_file(null_config, tmp_path, store):
+
+    s1 = hf.TaskSchema("t1", actions=[hf.Action(commands=[hf.Command("echo 'hi!'")])])
+    wk = hf.Workflow.from_template_data(
+        tasks=[hf.Task(s1)], template_name="print_stdout", path=tmp_path, store=store
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    combine = wk.submissions[0].jobscripts[0].resources.combine_jobscript_std
+    filename = "js_0_std.log" if combine else "js_0_stdout.log"
+    rel_path = f"artifacts/submissions/0/js_std/0/{filename}"
+    abs_path = f"{wk.url}/{rel_path}"
+
+    assert wk.get_text_file(rel_path) == "hi!\n"
+    assert wk.get_text_file(abs_path) == "hi!\n"
+
+
+@pytest.mark.integration
+def test_get_text_file_zarr_zip(null_config, tmp_path):
+
+    s1 = hf.TaskSchema("t1", actions=[hf.Action(commands=[hf.Command("echo 'hi!'")])])
+    wk = hf.Workflow.from_template_data(
+        tasks=[hf.Task(s1)], template_name="print_stdout", path=tmp_path, store="zarr"
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    wkz = hf.Workflow(wk.zip())
+
+    combine = wkz.submissions[0].jobscripts[0].resources.combine_jobscript_std
+    filename = "js_0_std.log" if combine else "js_0_stdout.log"
+    rel_path = f"artifacts/submissions/0/js_std/0/{filename}"
+    abs_path = f"{wkz.url}/{rel_path}"
+
+    assert wkz.get_text_file(rel_path) == "hi!\n"
+    assert wkz.get_text_file(abs_path) == "hi!\n"
+
+
+@pytest.mark.parametrize("store", ["zarr", "json"])
+def test_get_text_file_file_not_found(null_config, tmp_path, store):
+    s1 = hf.TaskSchema("t1", actions=[hf.Action(commands=[hf.Command("echo 'hi!'")])])
+    wk = hf.Workflow.from_template_data(
+        tasks=[hf.Task(s1)], template_name="print_stdout", path=tmp_path, store=store
+    )
+    with pytest.raises(FileNotFoundError):
+        wk.get_text_file("non_existent_file.txt")
+
+
+def test_get_text_file_file_not_found_zarr_zip(null_config, tmp_path):
+    s1 = hf.TaskSchema("t1", actions=[hf.Action(commands=[hf.Command("echo 'hi!'")])])
+    wk = hf.Workflow.from_template_data(
+        tasks=[hf.Task(s1)], template_name="print_stdout", path=tmp_path, store="zarr"
+    )
+    wkz = hf.Workflow(wk.zip())
+    with pytest.raises(FileNotFoundError):
+        wkz.get_text_file("non_existent_file.txt")

hpcflow/tests/workflows/test_zip.py

@@ -0,0 +1,18 @@
+import sys
+import pytest
+
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.test_utils import make_test_data_YAML_workflow
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(
+    sys.version_info < (3, 9), reason="Python 3.8 support is being removed anyway."
+)
+def test_workflow_1_zip(tmp_path, new_null_config):
+    wk = make_test_data_YAML_workflow("workflow_1.yaml", path=tmp_path)
+    wk.submit(wait=True, add_to_known=False, status=False)
+
+    zip_path = wk.zip(path=tmp_path)
+    wkz = hf.Workflow(zip_path)
+    assert wkz.tasks[0].elements[0].outputs.p2.value == "201"