hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/tests/unit/test_element_iteration.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+
+
+@pytest.mark.parametrize("store", ["json", "zarr"])
+def test_decode(null_config, tmp_path: Path, store: str):
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        outputs=[hf.SchemaInput(parameter=hf.Parameter("p2"))],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="Write-Output (<<parameter:p1>> + 100)",
+                        stdout="<<parameter:p2>>",
+                    )
+                ]
+            )
+        ],
+    )
+    wk = hf.Workflow.from_template_data(
+        tasks=[hf.Task(schema=s1, inputs=[hf.InputValue("p1", value=101)])],
+        loops=[hf.Loop(tasks=[0], num_iterations=1)],
+        path=tmp_path,
+        template_name="wk0",
+        store=store,
+    )
+    iter_i = wk.tasks[0].elements[0].iterations[0]
+    assert iter_i.id_ == 0
+    assert iter_i.index == 0
+    assert iter_i.EARs_initialised == True
+    assert sorted(iter_i.data_idx) == sorted(
+        {"inputs.p1": 2, "resources.any": 1, "outputs.p2": 3}
+    )
+    assert iter_i.loop_idx == {"loop_0": 0}
+    assert sorted(iter_i.schema_parameters) == sorted(
+        ["resources.any", "inputs.p1", "outputs.p2"]
+    )
+
+
+@pytest.mark.integration
+def test_loop_skipped_true_single_action_elements(null_config, tmp_path):
+    ts = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaInput("p1")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command="echo $(( <<parameter:p1>> + 100 ))",
+                        stdout="<<int(parameter:p1)>>",
+                    ),
+                ]
+            ),
+        ],
+    )
+    loop_term = hf.Rule(path="outputs.p1", condition={"value.equal_to": 300})
+    wk = hf.Workflow.from_template_data(
+        template_name="test_loop_skipped",
+        path=tmp_path,
+        tasks=[hf.Task(schema=ts, inputs={"p1": 100})],
+        loops=[
+            hf.Loop(name="my_loop", tasks=[0], termination=loop_term, num_iterations=3)
+        ],
+    )
+    # loop should terminate after the second iteration; third iteration should
+    # be loop-skipped
+    wk.submit(wait=True, add_to_known=False, status=False)
+    iters = wk.get_all_element_iterations()
+
+    assert not iters[0].loop_skipped
+    assert not iters[1].loop_skipped
+    assert iters[2].loop_skipped
+
+    # check latest iteration is the latest non-loop-skipped iteration:
+    assert wk.tasks[0].elements[0].latest_iteration_non_skipped.id_ == iters[1].id_
+
+    # check element inputs are from latest non-loop-skipped iteration:
+    assert wk.tasks[0].elements[0].inputs.p1.value == 200
+    assert wk.tasks[0].elements[0].get("inputs.p1") == 200
+
+    # check element outputs are from latest non-loop-skipped iteration:
+    assert wk.tasks[0].elements[0].outputs.p1.value == 300
+    assert wk.tasks[0].elements[0].get("outputs.p1") == 300

hpcflow/tests/unit/test_element_set.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import MalformedNestingOrderPath
+
+if TYPE_CHECKING:
+    from pathlib import Path
+    from hpcflow.sdk.core.parameters import Parameter
+    from hpcflow.sdk.core.types import ResourceSpecArgs
+    from hpcflow.sdk.core.workflow import Workflow
+
+
+@pytest.fixture
+def null_config(tmp_path: Path):
+    if not hf.is_config_loaded:
+        hf.load_config(config_dir=tmp_path)
+
+
+@pytest.fixture
+def param_p1() -> Parameter:
+    return hf.Parameter("p1")
+
+
+@pytest.fixture
+def param_p2() -> Parameter:
+    return hf.Parameter("p2")
+
+
+@pytest.fixture
+def param_p3() -> Parameter:
+    return hf.Parameter("p3")
+
+
+@pytest.fixture
+def workflow_w1(
+    null_config, tmp_path: Path, param_p1: Parameter, param_p2: Parameter
+) -> Workflow:
+    s1 = hf.TaskSchema("t1", actions=[], inputs=[param_p1], outputs=[param_p2])
+    s2 = hf.TaskSchema("t2", actions=[], inputs=[param_p2])
+
+    t1 = hf.Task(
+        schema=s1,
+        sequences=[hf.ValueSequence("inputs.p1", values=[101, 102], nesting_order=1)],
+    )
+    t2 = hf.Task(schema=s2, nesting_order={"inputs.p2": 1})
+
+    wkt = hf.WorkflowTemplate(name="w1", tasks=[t1, t2])
+    return hf.Workflow.from_template(wkt, path=tmp_path)
+
+
+@pytest.fixture
+def workflow_w2(workflow_w1: Workflow) -> Workflow:
+    """Add another element set to the second task."""
+    workflow_w1.tasks.t2.add_elements(nesting_order={"inputs.p2": 1})
+    return workflow_w1
+
+
+def test_resources_init_equivalence_dict_list_of_obj() -> None:
+    es1 = hf.ElementSet(resources={"any": {"num_cores": 1}})
+    es2 = hf.ElementSet(resources=[hf.ResourceSpec(scope="any", num_cores=1)])
+    assert es1 == es2
+
+
+def test_resources_init_equivalence_list_list_of_obj() -> None:
+    res_1_kwargs: ResourceSpecArgs = {"scope": "any", "num_cores": 1}
+    es1 = hf.ElementSet(resources=[res_1_kwargs])
+    es2 = hf.ElementSet(resources=[hf.ResourceSpec(**res_1_kwargs)])
+    assert es1 == es2
+
+
+def test_resources_init_equivalence_list_of_obj_resource_list_obj() -> None:
+    res_1_kwargs: ResourceSpecArgs = {"scope": "any", "num_cores": 1}
+    es1 = hf.ElementSet(resources=[hf.ResourceSpec(**res_1_kwargs)])
+    es2 = hf.ElementSet(resources=hf.ResourceList([hf.ResourceSpec(**res_1_kwargs)]))
+    assert es1 == es2
+
+
+def test_repeats_single_int_equivalence(null_config) -> None:
+    es1 = hf.ElementSet(repeats=2)
+    es2 = hf.ElementSet(repeats=[{"name": "", "number": 2, "nesting_order": 0}])
+    assert es1 == es2
+
+
+def test_merge_envs(null_config) -> None:
+    envs = {"my_env": {"version": "1.0"}}
+    es = hf.ElementSet(environments=envs)
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == envs
+
+
+def test_merge_envs_existing_any_resources(null_config) -> None:
+    envs = {"my_env": {"version": "1.0"}}
+    num_cores = 2
+    es = hf.ElementSet(resources={"any": {"num_cores": num_cores}}, environments=envs)
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == envs
+    assert es.resources.get(scope=hf.ActionScope.any()).num_cores == num_cores
+
+
+def test_merge_envs_resource_envs_precedence(null_config) -> None:
+    envs = {"my_env": {"version": "1.0"}}
+    res_envs = {"other_env": {"version": "2.0"}}
+    es = hf.ElementSet(resources={"any": {"environments": res_envs}}, environments=envs)
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == res_envs
+
+
+def test_merge_envs_no_envs_with_resource_envs(null_config) -> None:
+    envs = {"my_env": {"version": "1.0"}}
+    es = hf.ElementSet(resources={"any": {"environments": envs}})
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == envs
+
+
+def test_raise_env_and_envs_specified(null_config) -> None:
+    with pytest.raises(ValueError):
+        hf.ElementSet(env_preset="my_preset", environments={"my_env": {"version": 1}})
+
+
+def test_nesting_order_paths_raise(null_config) -> None:
+    with pytest.raises(MalformedNestingOrderPath):
+        hf.ElementSet(nesting_order={"bad_path.p1": 1})
+
+
+def test_nesting_order_paths_no_raise(null_config) -> None:
+    hf.ElementSet(nesting_order={"inputs.p1": 1, "resources.any": 2, "repeats": 3})
+
+
+def test_input_source_str_dict_list_str_list_dict_equivalence(null_config) -> None:
+    inp_source_dict: dict[str, str | int] = {
+        "source_type": "task",
+        "task_source_type": "output",
+        "task_ref": 0,
+    }
+    inp_source_str = "task.0.output"
+    inp_source_list_dict = [inp_source_dict]
+    inp_source_list_str = [inp_source_str]
+    assert (
+        hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_dict}}
+        ).input_sources
+        == hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_list_dict}}
+        ).input_sources
+        == hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_str}}
+        ).input_sources
+        == hf.ElementSet.from_json_like(
+            {"input_sources": {"p1": inp_source_list_str}}
+        ).input_sources
+    )
+
+
+def test_element_set_input_dict_equivalence(null_config):
+    assert hf.ElementSet(
+        inputs=[hf.InputValue("p1", label="A", value=1)]
+    ) == hf.ElementSet(inputs={"p1[A]": 1})
+
+    assert hf.ElementSet(
+        inputs=[hf.InputValue("p1", label="A", path="b", value=1)]
+    ) == hf.ElementSet(inputs={"p1[A].b": 1})

hpcflow/tests/unit/test_group.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+from pathlib import Path
+import pytest
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import MissingElementGroup
+
+
+def test_group_simple(null_config, tmp_path: Path):
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        "echo $(( <<parameter:p1>> + 1 ))", stdout="<<parameter:p2>>"
+                    )
+                ]
+            )
+        ],
+    )
+    s2 = hf.TaskSchema(
+        objective="t2",
+        inputs=[hf.SchemaInput("p2", group="my_group")],
+        outputs=[hf.SchemaOutput("p3")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        "echo $(( <<parameter:p2>> + 2 ))", stdout="<<parameter:p3>>"
+                    )
+                ]
+            )
+        ],
+    )
+
+    t1 = hf.Task(
+        schema=s1,
+        sequences=[hf.ValueSequence("inputs.p1", values=[1, 2, 3])],
+        groups=[hf.ElementGroup(name="my_group")],
+    )
+    t2 = hf.Task(schema=s2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test_groups",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert [task.num_elements for task in wk.tasks] == [3, 1]
+    assert len(wk.tasks.t2.elements[0].get_data_idx("inputs.p2")["inputs.p2"]) == 3
+
+
+def test_group_raise_no_elements(null_config, tmp_path: Path):
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        "echo $(( <<parameter:p1>> + 1 ))", stdout="<<parameter:p2>>"
+                    )
+                ]
+            )
+        ],
+    )
+    s2 = hf.TaskSchema(
+        objective="t2",
+        inputs=[hf.SchemaInput("p2", group="my_group")],
+        outputs=[hf.SchemaOutput("p3")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        "echo $(( <<parameter:p2>> + 2 ))", stdout="<<parameter:p3>>"
+                    )
+                ]
+            )
+        ],
+    )
+
+    t1 = hf.Task(
+        schema=s1,
+        sequences=[hf.ValueSequence("inputs.p1", values=[1, 2, 3])],
+    )
+    t2 = hf.Task(schema=s2)
+    with pytest.raises(MissingElementGroup):
+        hf.Workflow.from_template_data(
+            template_name="test_groups",
+            path=tmp_path,
+            tasks=[t1, t2],
+        )
+
+
+def test_group_on_input_only_task(null_config, tmp_path: Path):
+
+    s1 = hf.TaskSchema(objective="t1", inputs=[hf.SchemaInput("p1")])
+    s2 = hf.TaskSchema(objective="t2", inputs=[hf.SchemaInput("p1", group="all")])
+
+    t1 = hf.Task(
+        schema=s1,
+        inputs={"p1": 100},
+        repeats=2,
+        groups=[hf.ElementGroup(name="all")],  # define a group on a task with no actions
+    )
+    t2 = hf.Task(schema=s2)
+
+    wk = hf.Workflow.from_template_data(
+        template_name="test_input_group",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks.t1.num_elements == 2
+    assert wk.tasks.t2.num_elements == 1