hpcflow 0.1.15__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/tests/unit/test_resources.py

@@ -0,0 +1,243 @@
+from __future__ import annotations
+import os
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import UnsupportedSchedulerError
+
+
+def test_init_scope_equivalence_simple() -> None:
+    rs1 = hf.ResourceSpec(scope=hf.ActionScope.any(), num_cores=1)
+    rs2 = hf.ResourceSpec(scope="any", num_cores=1)
+    assert rs1 == rs2
+
+
+def test_init_scope_equivalence_with_kwargs() -> None:
+    rs1 = hf.ResourceSpec(
+        scope=hf.ActionScope.input_file_generator(file="my_file"), num_cores=1
+    )
+    rs2 = hf.ResourceSpec(scope="input_file_generator[file=my_file]", num_cores=1)
+    assert rs1 == rs2
+
+
+def test_init_no_args() -> None:
+    rs1 = hf.ResourceSpec()
+    rs2 = hf.ResourceSpec(scope="any")
+    assert rs1 == rs2
+
+
+def test_resource_list_raise_on_identical_scopes() -> None:
+    with pytest.raises(ValueError):
+        hf.ResourceList.normalise([{"scope": "any"}, {"scope": "any"}])
+
+
+def test_merge_other_same_scope() -> None:
+    res_lst_1 = hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
+    res_lst_2 = hf.ResourceList.from_json_like({"any": {}})
+    res_lst_2.merge_other(res_lst_1)
+    assert res_lst_2 == hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
+
+
+def test_merge_other_same_scope_no_overwrite() -> None:
+    res_lst_1 = hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
+    res_lst_2 = hf.ResourceList.from_json_like({"any": {"num_cores": 2}})
+    res_lst_2.merge_other(res_lst_1)
+    assert res_lst_2 == hf.ResourceList.from_json_like({"any": {"num_cores": 2}})
+
+
+def test_merge_other_multi_scope() -> None:
+    res_lst_1 = hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
+    res_lst_2 = hf.ResourceList.from_json_like({"any": {}, "main": {"num_cores": 3}})
+    res_lst_2.merge_other(res_lst_1)
+    assert res_lst_2 == hf.ResourceList.from_json_like(
+        {"any": {"num_cores": 1}, "main": {"num_cores": 3}}
+    )
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_merge_other_persistent_workflow_reload(null_config, tmp_path: Path, store: str):
+    wkt = hf.WorkflowTemplate(
+        name="test_load",
+        resources={"any": {"num_cores": 2}},
+        tasks=[
+            hf.Task(
+                schema=hf.task_schemas.test_t1_ps,
+                inputs={"p1": 101},
+            ),
+        ],
+    )
+    wk = hf.Workflow.from_template(wkt, path=tmp_path, store=store)
+    wk = hf.Workflow(wk.path)
+    assert wk.template.tasks[0].element_sets[0].resources[0].num_cores == 2
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_use_persistent_resource_spec(null_config, tmp_path: Path, store: str):
+    # create a workflow from which we can use a resource spec in a new workflow:
+    num_cores_check = 2
+    wk_base = hf.Workflow.from_template_data(
+        template_name="wk_base",
+        path=tmp_path,
+        store=store,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_ps],
+                inputs=[hf.InputValue("p1", 101)],
+                resources={"any": {"num_cores": num_cores_check}},
+            )
+        ],
+    )
+    resource_spec = wk_base.tasks[0].template.element_sets[0].resources[0]
+
+    wk = hf.Workflow.from_template_data(
+        template_name="wk",
+        path=tmp_path,
+        store=store,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_ps],
+                inputs=[hf.InputValue("p1", 101)],
+            ),
+        ],
+        resources=[resource_spec],
+    )
+
+    assert wk.tasks[0].template.element_sets[0].resources[0].num_cores == num_cores_check
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_use_persistent_resource_list(null_config, tmp_path: Path, store: str):
+    # create a workflow from which we can use the resource list in a new workflow:
+    num_cores_check = 2
+    wk_base = hf.Workflow.from_template_data(
+        template_name="wk_base",
+        path=tmp_path,
+        store=store,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_ps],
+                inputs=[hf.InputValue("p1", 101)],
+                resources={"any": {"num_cores": num_cores_check}},
+            )
+        ],
+    )
+    resource_list = wk_base.tasks[0].template.element_sets[0].resources
+
+    wk = hf.Workflow.from_template_data(
+        template_name="wk",
+        path=tmp_path,
+        store=store,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_ps],
+                inputs=[hf.InputValue("p1", 101)],
+            ),
+        ],
+        resources=resource_list[:],  # must pass a list!
+    )
+
+    assert wk.tasks[0].template.element_sets[0].resources[0].num_cores == num_cores_check
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_default_scheduler_set(new_null_config, tmp_path: Path, store: str):
+    wk = hf.Workflow.from_template_data(
+        template_name="wk",
+        path=tmp_path,
+        store=store,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_bash],
+                inputs=[hf.InputValue("p1", 101)],
+            ),
+        ],
+    )
+    wk.add_submission()
+    assert wk.submissions[0].jobscripts[0].scheduler_name == hf.config.default_scheduler
+
+
+def test_scheduler_case_insensitive(null_config) -> None:
+    rs1 = hf.ResourceSpec(scheduler="direct")
+    rs2 = hf.ResourceSpec(scheduler="dIrEcT")
+    assert rs1 == rs2
+    assert rs1.scheduler == rs2.scheduler == "direct"
+
+
+def test_scheduler_strip(null_config) -> None:
+    rs1 = hf.ResourceSpec(scheduler="  direct ")
+    rs2 = hf.ResourceSpec(scheduler="direct")
+    assert rs1 == rs2
+    assert rs1.scheduler == rs2.scheduler == "direct"
+
+
+def test_shell_case_insensitive(null_config) -> None:
+    shell_name = "bash" if os.name == "posix" else "powershell"
+    shell_name_title = shell_name
+    n = shell_name_title[0]
+    shell_name_title = shell_name_title.replace(n, n.upper())
+    assert shell_name != shell_name_title
+    rs1 = hf.ResourceSpec(shell=shell_name)
+    rs2 = hf.ResourceSpec(shell=shell_name_title)
+    assert rs1 == rs2
+    assert rs1.shell == rs2.shell == shell_name
+
+
+def test_shell_strip(null_config) -> None:
+    shell_name = "bash" if os.name == "posix" else "powershell"
+    rs1 = hf.ResourceSpec(shell=f"  {shell_name} ")
+    rs2 = hf.ResourceSpec(shell=shell_name)
+    assert rs1 == rs2
+    assert rs1.shell == rs2.shell == shell_name
+
+
+def test_os_name_case_insensitive(null_config):
+    rs1 = hf.ResourceSpec(os_name="nt")
+    rs2 = hf.ResourceSpec(os_name="NT")
+    assert rs1 == rs2
+    assert rs1.os_name == rs2.os_name == "nt"
+
+
+def test_os_name_strip(null_config) -> None:
+    rs1 = hf.ResourceSpec(os_name="  nt ")
+    rs2 = hf.ResourceSpec(os_name="nt")
+    assert rs1 == rs2
+    assert rs1.os_name == rs2.os_name == "nt"
+
+
+def test_raise_on_unsupported_scheduler(new_null_config, tmp_path: Path):
+    # slurm not supported by default config file:
+    wk = hf.Workflow.from_template_data(
+        template_name="wk1",
+        path=tmp_path,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_bash],
+                inputs=[hf.InputValue("p1", 101)],
+                resources=[hf.ResourceSpec(scheduler="slurm")],
+            )
+        ],
+    )
+    with pytest.raises(UnsupportedSchedulerError):
+        wk.add_submission()
+
+
+def test_can_use_non_default_scheduler(new_null_config, tmp_path: Path):
+    # for either OS choose a compatible scheduler not set by default:
+    if os.name == "nt":
+        opt_scheduler = "direct_posix"  # i.e for WSL
+    else:
+        opt_scheduler = "slurm"
+    hf.config.add_scheduler(opt_scheduler)
+
+    wk = hf.Workflow.from_template_data(
+        template_name="wk1",
+        path=tmp_path,
+        tasks=[
+            hf.Task(
+                schema=[hf.task_schemas.test_t1_bash],
+                inputs=[hf.InputValue("p1", 101)],
+                resources=[hf.ResourceSpec(scheduler=opt_scheduler)],
+            )
+        ],
+    )
+    wk.add_submission()

hpcflow/tests/unit/test_run.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from pathlib import Path
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.actions import SkipReason
+from hpcflow.sdk.core.test_utils import make_workflow_to_run_command
+
+
+def test_compose_commands_no_shell_var(null_config, tmp_path: Path):
+    ts = hf.TaskSchema(
+        objective="test_compose_commands",
+        actions=[hf.Action(commands=[hf.Command(command="Start-Sleep 10")])],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_compose_commands",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts)],
+    )
+    sub = wk.add_submission()
+    assert sub is not None
+    js = sub.jobscripts[0]
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    _, shell_vars = run.compose_commands(environments=sub.environments, shell=js.shell)
+    assert shell_vars == {0: []}
+
+
+def test_compose_commands_single_shell_var(null_config, tmp_path: Path):
+    ts = hf.TaskSchema(
+        objective="test_compose_commands",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p1")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="Write-Output (<<parameter:p1>> + 100)",
+                        stdout="<<int(parameter:p1)>>",
+                    ),
+                ],
+            ),
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_compose_commands",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 101})],
+    )
+    sub = wk.add_submission()
+    assert sub is not None
+    js = sub.jobscripts[0]
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    _, shell_vars = run.compose_commands(environments=sub.environments, shell=js.shell)
+    assert shell_vars == {0: [("outputs.p1", "parameter_p1", "stdout")]}
+
+
+def test_compose_commands_multi_single_shell_var(null_config, tmp_path: Path):
+    ts = hf.TaskSchema(
+        objective="test_compose_commands",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p1")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(command="Start-Sleep 10"),
+                    hf.Command(
+                        command="Write-Output (<<parameter:p1>> + 100)",
+                        stdout="<<int(parameter:p1)>>",
+                    ),
+                ],
+            ),
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_compose_commands",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 101})],
+    )
+    sub = wk.add_submission()
+    assert sub is not None
+    js = sub.jobscripts[0]
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    _, shell_vars = run.compose_commands(environments=sub.environments, shell=js.shell)
+    assert shell_vars == {0: [], 1: [("outputs.p1", "parameter_p1", "stdout")]}
+
+
+@pytest.mark.integration
+def test_run_dir_diff_new_file(null_config, tmp_path):
+    if os.name == "nt":
+        command = "New-Item -Path 'new_file.txt' -ItemType File"
+    else:
+        command = "touch new_file.txt"
+    wk = make_workflow_to_run_command(
+        command=command,
+        requires_dir=True,
+        path=tmp_path,
+        name="w2",
+        overwrite=True,
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    assert wk.get_all_EARs()[0].dir_diff.files_created == ["new_file.txt"]
+
+
+@pytest.mark.integration
+def test_run_skip_reason_upstream_failure(null_config, tmp_path):
+    ts = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaInput("p2")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p1>> + 100 ))",
+                        stdout="<<parameter:p2>>",
+                    ),
+                    hf.Command(command="exit 1"),
+                ]
+            ),
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p2>> + 100 ))",
+                        stdout="<<parameter:p2>>",
+                    ),
+                ]
+            ),  # should be skipped due to failure of action 0
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test_skip_reason",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 100})],
+    )
+    wk.submit(wait=True, add_to_known=False, status=False)
+    runs = wk.get_all_EARs()
+    assert not runs[0].success
+    assert not runs[1].success
+    assert runs[0].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[1].skip_reason is SkipReason.UPSTREAM_FAILURE
+
+
+@pytest.mark.integration
+def test_run_skip_reason_loop_termination(null_config, tmp_path):
+    ts = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaInput("p1")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p1>> + 100 ))",
+                        stdout="<<int(parameter:p1)>>",
+                    ),
+                ]
+            ),
+        ],
+    )
+    loop_term = hf.Rule(path="outputs.p1", condition={"value.equal_to": 300})
+    wk = hf.Workflow.from_template_data(
+        template_name="test_skip_reason",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 100})],
+        loops=[
+            hf.Loop(name="my_loop", tasks=[0], termination=loop_term, num_iterations=3)
+        ],
+    )
+    # loop should terminate after the second iteration
+    wk.submit(wait=True, add_to_known=False, status=False)
+    runs = wk.get_all_EARs()
+
+    assert runs[0].get("outputs.p1") == 200
+    assert runs[1].get("outputs.p1") == 300
+    assert not runs[2].get("outputs.p1")
+
+    assert runs[0].success
+    assert runs[1].success
+    assert not runs[2].success
+
+    assert runs[0].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[1].skip_reason is SkipReason.NOT_SKIPPED
+    assert runs[2].skip_reason is SkipReason.LOOP_TERMINATION
+
+
+def test_get_data_in_values_input_files(null_config, tmp_path: Path):
+    with (file_name := Path("my_file.txt")).open("wt") as fh:
+        fh.write("hello!\n")
+    file_spec = hf.FileSpec("my_file", str(file_name))
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter="p1"),
+        ],
+        actions=[
+            hf.Action(
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=file_spec,
+                        inputs=[hf.Parameter("p1")],
+                    )
+                ],
+            )
+        ],
+    )
+    t1 = hf.Task(schema=[s1], input_files=[hf.InputFile(file_spec, path="my_file.txt")])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="test_get_data_in_values_input_files",
+    )
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    assert run.get_data_in_values(("input_files.my_file",)) == {
+        "my_file": wk.input_files_path.joinpath("0/my_file.txt").as_posix()
+    }
+
+
+def test_get_data_in_values_user_provided_input_file(null_config, tmp_path: Path):
+
+    # pass an input file so the IFG doesn't need to run
+    with (file_name := Path("my_file.txt")).open("wt") as fh:
+        fh.write("hello!\n")
+
+    file_spec = hf.FileSpec("my_file", str(file_name))
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter="p1"),
+        ],
+        actions=[
+            hf.Action(
+                input_file_generators=[
+                    hf.InputFileGenerator(
+                        input_file=file_spec,
+                        inputs=[hf.Parameter("p1")],
+                    )
+                ],
+            )
+        ],
+    )
+    t1 = hf.Task(schema=[s1], input_files=[hf.InputFile(file_spec, path="my_file.txt")])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="test_get_data_in_values_input_files",
+    )
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    assert run.get_data_in_values(("input_files.my_file",)) == {
+        "my_file": wk.input_files_path.joinpath("0/my_file.txt").as_posix()
+    }
+
+
+def test_get_data_in_values_input_file_to_script(null_config, tmp_path: Path):
+    # pass an input file so the IFG doesn't need to run
+    with (file_name := Path("my_file.txt")).open("wt") as fh:
+        fh.write("hello!\n")
+
+    file_spec = hf.FileSpec("my_file", str(file_name))
+    act = hf.Action(
+        input_file_generators=[
+            hf.InputFileGenerator(
+                input_file=file_spec,
+                inputs=[hf.Parameter("p1")],
+            )
+        ],
+        script_data_in={"input_files.my_file": "direct"},
+    )
+    s1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        actions=[act],
+    )
+    t1 = hf.Task(schema=[s1], input_files=[hf.InputFile(file_spec, path="my_file.txt")])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="test_get_data_in_values_input_files",
+    )
+    act_runs = wk.tasks[0].elements[0].iterations[0].action_runs
+    assert len(act_runs) == 1
+    assert act_runs[0].get_data_in_values(("input_files.my_file",)) == {
+        "my_file": wk.input_files_path.joinpath("0/my_file.txt").as_posix()
+    }

hpcflow/tests/unit/test_run_directories.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.test_utils import make_workflow
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_run_directories(null_config, tmp_path, store):
+    wk = make_workflow(
+        schemas_spec=[
+            [{"p1": None}, ("p1",), "t1"],
+            [{"p2": None}, ("p2",), "t2", {"requires_dir": True}],
+        ],
+        local_inputs={0: ("p1",)},
+        local_sequences={1: [("inputs.p2", 2, 0)]},
+        path=tmp_path,
+        store=store,
+    )
+    lp_0 = hf.Loop(name="my_loop", tasks=[1], num_iterations=2)
+    wk.add_loop(lp_0)
+    sub = wk.add_submission()  # populates run directories
+
+    run_dirs = wk.get_run_directories()
+
+    assert run_dirs[0] is None
+    assert str(run_dirs[1]) == str(Path(wk.path).joinpath("execute/t_1/e_0/i_0"))
+    assert str(run_dirs[2]) == str(Path(wk.path).joinpath("execute/t_1/e_1/i_0"))
+    assert str(run_dirs[3]) == str(Path(wk.path).joinpath("execute/t_1/e_0/i_1"))
+    assert str(run_dirs[4]) == str(Path(wk.path).joinpath("execute/t_1/e_1/i_1"))

hpcflow/tests/unit/test_runtime.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+from hpcflow.app import app as hf
+
+
+def test_in_pytest_if_not_frozen() -> None:
+    """This is to check we can get the correct invocation command when running non-frozen
+    tests (when frozen the invocation command is just the executable file)."""
+    if not hf.run_time_info.is_frozen:
+        assert hf.run_time_info.in_pytest