hpcflow-new2 0.2.0a190__py3-none-any.whl → 0.2.0a200__py3-none-any.whl

hpcflow/tests/unit/test_meta_task.py

@@ -0,0 +1,325 @@
+from textwrap import dedent
+
+import pytest
+
+from hpcflow.app import app as hf
+from hpcflow.sdk.config.errors import UnknownMetaTaskConstitutiveSchema
+
+
+def test_basic_meta_task_workflow(new_null_config, tmp_path):
+    wk_yaml = dedent(
+        """\
+        name: test_meta_task
+        template_components:
+          task_schemas:
+            - objective: s0
+              inputs: 
+                - parameter: p1
+              outputs:
+                - parameter: p2
+              actions:
+                - commands:
+                  - command: echo "$((<<parameter:p1>> + 1))"
+                    stdout: <<int(parameter:p2)>>
+  
+            - objective: s1
+              inputs: 
+                - parameter: p2
+                - parameter: p2b
+              outputs:
+                - parameter: p3
+              actions:
+                - commands:
+                  - command: echo "$((<<parameter:p2>> + <<parameter:p2b>>))"
+                    stdout: <<int(parameter:p3)>>
+  
+            - objective: s2
+              inputs: 
+                - parameter: p3
+              outputs:
+                - parameter: p4
+              actions:
+                - commands:
+                  - command: echo "$((<<parameter:p3>> + 1))"
+                    stdout: <<int(parameter:p4)>>
+  
+            - objective: s3
+              inputs: 
+                - parameter: p4
+              outputs:
+                - parameter: p5
+              actions:
+                - commands:
+                  - command: echo "$((<<parameter:p4>> + 1))"
+                    stdout: <<int(parameter:p5)>>
+
+          meta_task_schemas:
+            - objective: system_analysis
+              inputs:
+                - parameter: p2
+              outputs:
+                - parameter: p4
+
+        meta_tasks:
+          system_analysis:
+            - schema: s1
+              inputs:
+                p2b: 220
+            - schema: s2
+
+        tasks:
+          - schema: s0
+            inputs:
+              p1: 100
+          - schema: system_analysis
+          - schema: s3
+        """
+    )
+    wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
+
+    # basic check of param dependendices
+    s0_di = wk.tasks.s0.elements[0].get_data_idx()
+    s1_di = wk.tasks.s1.elements[0].get_data_idx()
+    s2_di = wk.tasks.s2.elements[0].get_data_idx()
+    s3_di = wk.tasks.s3.elements[0].get_data_idx()
+
+    assert s0_di["outputs.p2"] == s1_di["inputs.p2"]
+    assert s1_di["outputs.p3"] == s2_di["inputs.p3"]
+    assert s2_di["outputs.p4"] == s3_di["inputs.p4"]
+
+
+def test_basic_meta_task_workflow_API(new_null_config, tmp_path):
+    """as above but using Python API."""
+    # normal task schemas:
+    s0 = hf.TaskSchema(
+        objective="s0",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command='echo "$((<<parameter:p1>> + 1))"',
+                        stdout="<<int(parameter:p2)>>",
+                    )
+                ]
+            )
+        ],
+    )
+    s1 = hf.TaskSchema(
+        objective="s1",
+        inputs=[hf.SchemaInput("p2")],
+        outputs=[hf.SchemaOutput("p3")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command='echo "$((<<parameter:p2>> + 1))"',
+                        stdout="<<int(parameter:p3)>>",
+                    )
+                ]
+            )
+        ],
+    )
+    s2 = hf.TaskSchema(
+        objective="s2",
+        inputs=[hf.SchemaInput("p3")],
+        outputs=[hf.SchemaOutput("p4")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command='echo "$((<<parameter:p3>> + 1))"',
+                        stdout="<<int(parameter:p4)>>",
+                    )
+                ]
+            )
+        ],
+    )
+    s3 = hf.TaskSchema(
+        objective="s3",
+        inputs=[hf.SchemaInput("p4")],
+        outputs=[hf.SchemaOutput("p5")],
+        actions=[
+            hf.Action(
+                commands=[
+                    hf.Command(
+                        command='echo "$((<<parameter:p4>> + 1))"',
+                        stdout="<<int(parameter:p5)>>",
+                    )
+                ]
+            )
+        ],
+    )
+
+    # meta=task schema:
+    ms = hf.MetaTaskSchema(
+        objective="system_analysis",
+        inputs=[hf.SchemaInput("p2")],
+        outputs=[hf.SchemaOutput("p4")],
+    )
+
+    # meta-task:
+    m1 = hf.MetaTask(
+        schema=ms,
+        tasks=[
+            hf.Task(schema=s1),
+            hf.Task(schema=s2),
+        ],
+    )
+
+    # workflow template tasks list:
+    tasks = [
+        hf.Task(schema=s0, inputs={"p1": 100}),
+        m1,
+        hf.Task(schema=s3),
+    ]
+
+    wk = hf.Workflow.from_template_data(
+        template_name="meta_task_workflow",
+        tasks=tasks,
+        path=tmp_path,
+    )
+
+    # basic check of param dependendices
+    s0_di = wk.tasks.s0.elements[0].get_data_idx()
+    s1_di = wk.tasks.s1.elements[0].get_data_idx()
+    s2_di = wk.tasks.s2.elements[0].get_data_idx()
+    s3_di = wk.tasks.s3.elements[0].get_data_idx()
+
+    assert s0_di["outputs.p2"] == s1_di["inputs.p2"]
+    assert s1_di["outputs.p3"] == s2_di["inputs.p3"]
+    assert s2_di["outputs.p4"] == s3_di["inputs.p4"]
+
+
+def test_meta_task_custom_parametrisation(new_null_config, tmp_path):
+    """test customising the parametrisation of inputs, sequences, and resources within the
+    `tasks` list."""
+    wk_yaml = dedent(
+        """\
+        name: test_metatask_multi_element_sets_custom_parametrisation
+        template_components:
+          task_schemas:
+            - objective: s1
+              inputs: 
+                - parameter: p1
+                - parameter: p2
+              outputs:
+                - parameter: p3
+              actions:
+                - commands:
+                  - command: echo "$((<<parameter:p1>> + <<parameter:p2>>))"
+                    stdout: <<int(parameter:p3)>>
+
+          meta_task_schemas:
+            - objective: system_analysis
+              inputs:
+                - parameter: p1
+                - parameter: p2
+              outputs:
+                - parameter: p3
+
+        meta_tasks:
+          system_analysis:
+            - schema: s1
+              element_sets:
+                - inputs:
+                    p1: 100
+                    p2: 200
+                - inputs:
+                    p1: 100
+                  sequences:
+                    - path: inputs.p2
+                      values: [200, 201]
+        tasks:
+          - schema: system_analysis
+            inputs:
+              s1: # should apply to first element set by default
+                p1: 101
+            resources:
+              s1: # should apply to first element set by default
+                any:
+                  num_cores: 2
+          - schema: system_analysis
+            inputs:
+              s1.0: # applies to first element set of s1
+                p1: 102
+              s1.1: # applies to second element set of s1
+                p1: 103
+            sequences:
+              s1.1: # sequences list in second element set is replaced with this list:
+                - path: inputs.p2
+                  values: [300, 301]
+        """
+    )
+    wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)
+
+    assert wk.tasks.s1_1.template.element_sets[0].resources[0].num_cores == 2  # modified
+    assert (
+        wk.tasks.s1_2.template.element_sets[0].resources[0].num_cores is None
+    )  # unaffected
+
+    assert wk.tasks.s1_1.template.element_sets[0].inputs[0].value == 101  # modified
+    assert wk.tasks.s1_1.template.element_sets[0].inputs[1].value == 200  # unaffected
+    assert wk.tasks.s1_1.template.element_sets[1].sequences[0].values == [
+        200,
+        201,
+    ]  # unaffected
+
+    assert wk.tasks.s1_2.template.element_sets[0].inputs[0].value == 102  # modified
+    assert wk.tasks.s1_2.template.element_sets[1].inputs[0].value == 103  # modified
+    assert wk.tasks.s1_2.template.element_sets[1].sequences[0].values == [
+        300,
+        301,
+    ]  # modified
+
+
+def test_meta_task_custom_parametrisation_raises_on_bad_schema_name(
+    new_null_config, tmp_path
+):
+    wk_yaml = dedent(
+        """\
+      name: test_metatask_raise_on_bad_schema_name
+      template_components:
+        task_schemas:
+          - objective: s1
+            inputs: 
+              - parameter: p1
+              - parameter: p2
+            outputs:
+              - parameter: p3
+            actions:
+              - commands:
+                - command: echo "$((<<parameter:p1>> + <<parameter:p2>>))"
+                  stdout: <<int(parameter:p3)>>
+
+        meta_task_schemas:
+          - objective: system_analysis
+            inputs:
+              - parameter: p1
+              - parameter: p2
+            outputs:
+              - parameter: p3
+
+      meta_tasks:
+        system_analysis:
+          - schema: s1
+            element_sets:
+              - inputs:
+                  p1: 100
+                  p2: 200
+              - inputs:
+                  p1: 100
+                sequences:
+                  - path: inputs.p2
+                    values: [200, 201]
+      tasks:
+        - schema: system_analysis
+          resources:
+            BAD_SCHEMA_NAME: # should raise!
+              any:
+                num_cores: 2
+      """
+    )
+    with pytest.raises(UnknownMetaTaskConstitutiveSchema):
+        wk = hf.Workflow.from_YAML_string(wk_yaml, path=tmp_path)

hpcflow/tests/unit/test_multi_path_sequences.py

@@ -0,0 +1,229 @@
+from __future__ import annotations
+from textwrap import dedent
+
+import numpy as np
+import pytest
+
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.test_utils import make_schemas
+from hpcflow.sdk.core.utils import get_file_context
+
+
+def test_MPS_sequences():
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    assert mps.sequences == [
+        hf.ValueSequence(path="inputs.p1", values=[0, 1]),
+        hf.ValueSequence(path="inputs.p2", values=[2, 3]),
+    ]
+
+
+def test_MPS_sequences_moved_to_element_set():
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    es = hf.ElementSet(multi_path_sequences=[mps])
+    expected_mps_seqs = [
+        hf.ValueSequence(path="inputs.p1", values=[0, 1]),
+        hf.ValueSequence(path="inputs.p2", values=[2, 3]),
+    ]
+    assert mps._sequences is None
+    assert mps.sequence_indices == [0, 2]
+    assert es.sequences == mps.sequences
+    assert es.sequences == expected_mps_seqs
+
+
+def test_MPS_sequences_moved_to_element_set_with_existing_sequences():
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
+    expected_mps_seqs = [
+        hf.ValueSequence(path="inputs.p1", values=[0, 1]),
+        hf.ValueSequence(path="inputs.p2", values=[2, 3]),
+    ]
+    es = hf.ElementSet(
+        sequences=[seq],
+        multi_path_sequences=[mps],
+    )
+    assert mps._sequences is None
+    assert mps.sequence_indices == [1, 3]
+    assert mps.sequences == expected_mps_seqs
+    assert es.sequences == [seq, *expected_mps_seqs]
+
+
+def test_MPS_sequences_moved_to_task_element_set():
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    s1 = make_schemas(({"p1": None, "p2": None}, ()))
+    t1 = hf.Task(s1, multi_path_sequences=[mps])
+    expected_mps_seqs = [
+        hf.ValueSequence(path="inputs.p1", values=[0, 1]),
+        hf.ValueSequence(path="inputs.p2", values=[2, 3]),
+    ]
+    es = t1.element_sets[0]
+    assert mps._sequences is None
+    assert mps.sequence_indices == [0, 2]
+    assert es.sequences == mps.sequences
+    assert es.sequences == expected_mps_seqs
+
+
+def test_MPS_sequences_moved_to_task_element_set_with_existing_sequences():
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
+    s1 = make_schemas(({"p0": None, "p1": None, "p2": None}, ()))
+    t1 = hf.Task(s1, sequences=[seq], multi_path_sequences=[mps])
+    expected_mps_seqs = [
+        hf.ValueSequence(path="inputs.p1", values=[0, 1]),
+        hf.ValueSequence(path="inputs.p2", values=[2, 3]),
+    ]
+    es = t1.element_sets[0]
+    assert mps._sequences is None
+    assert mps.sequence_indices == [1, 3]
+    assert mps.sequences == expected_mps_seqs
+    assert es.sequences == [seq, *expected_mps_seqs]
+
+
+def test_MPS_sequence_element_inputs(null_config, tmp_path):
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    s1 = make_schemas(({"p1": None, "p2": None}, ()))
+    t1 = hf.Task(s1, multi_path_sequences=[mps])
+    wf = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="test_multi_path_sequence",
+        path=tmp_path,
+    )
+    assert len(wf.template.tasks[0].element_sets[0].sequences) == 2
+    assert wf.tasks[0].num_elements == 2
+    assert wf.tasks[0].elements[0].get("inputs") == {"p1": 0, "p2": 2}
+    assert wf.tasks[0].elements[1].get("inputs") == {"p1": 1, "p2": 3}
+
+
+def test_MPS_sequence_element_inputs_with_existing_sequence(null_config, tmp_path):
+    mps = hf.MultiPathSequence(paths=("inputs.p1", "inputs.p2"), values=[[0, 1], [2, 3]])
+    seq = hf.ValueSequence(path="inputs.p0", values=[0, 1])
+    s1 = make_schemas(({"p0": None, "p1": None, "p2": None}, ()))
+    t1 = hf.Task(s1, sequences=[seq], multi_path_sequences=[mps])
+    wf = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="test_multi_path_sequence",
+        path=tmp_path,
+    )
+    assert len(wf.template.tasks[0].element_sets[0].sequences) == 3
+    assert wf.tasks[0].num_elements == 2
+    assert wf.tasks[0].elements[0].get("inputs") == {"p0": 0, "p1": 0, "p2": 2}
+    assert wf.tasks[0].elements[1].get("inputs") == {"p0": 1, "p1": 1, "p2": 3}
+
+    # check the same on reload:
+    wf = wf.reload()
+    assert len(wf.template.tasks[0].element_sets[0].sequences) == 3
+    assert wf.tasks[0].num_elements == 2
+    assert wf.tasks[0].elements[0].get("inputs") == {"p0": 0, "p1": 0, "p2": 2}
+    assert wf.tasks[0].elements[1].get("inputs") == {"p0": 1, "p1": 1, "p2": 3}
+
+
+@pytest.mark.integration
+def test_MPS_element_outputs(null_config, tmp_path):
+    with get_file_context("hpcflow.tests.data", "multi_path_sequences.yaml") as file_path:
+        wf = hf.make_and_submit_workflow(
+            file_path,
+            path=tmp_path,
+            status=False,
+            add_to_known=False,
+            wait=True,
+        )
+        assert wf.tasks[0].num_elements == 2
+
+        p2 = wf.tasks[0].elements[0].outputs.p2
+        assert isinstance(p2, hf.ElementParameter)
+        assert p2.value == 302
+
+        p2 = wf.tasks[0].elements[1].outputs.p2
+        assert isinstance(p2, hf.ElementParameter)
+        assert p2.value == 304
+
+
+def test_MPS_latin_hypercube_sequence_values():
+    wft_yaml = dedent(
+        """\
+        name: test_latin_hypercube_sampling
+        template_components:
+          task_schemas:
+            - objective: define_p1
+              inputs:
+                - parameter: p1
+        tasks:
+          - schema: define_p1
+            inputs:
+              p1: {}
+            multi_path_sequences:
+              - paths: [inputs.p1.a, inputs.p1.b]
+                values::from_latin_hypercube:
+                  num_samples: 5
+    """
+    )
+    wft = hf.WorkflowTemplate.from_YAML_string(wft_yaml)
+    es = wft.tasks[0].element_sets[0]
+    assert len(es.multi_path_sequences) == 1
+    mps_values = np.asarray(es.multi_path_sequences[0].values)
+    assert mps_values.shape == (2, 5)
+    assert len(es.sequences) == 2
+    seq_1 = es.sequences[0]
+    seq_2 = es.sequences[1]
+    assert seq_1.path == "inputs.p1.a"
+    assert seq_2.path == "inputs.p1.b"
+    assert np.array_equal(np.asarray(seq_1.values), mps_values[0])
+    assert np.array_equal(np.asarray(seq_2.values), mps_values[1])
+
+
+def test_MPS_move_from_sequences_list():
+    wft_yaml = dedent(
+        """\
+        name: test_latin_hypercube_sampling
+        template_components:
+          task_schemas:
+            - objective: define_p1_p2_p3_p4
+              inputs:
+                - parameter: p1
+                - parameter: p2
+                - parameter: p3
+                - parameter: p4
+        tasks:
+          - schema: define_p1_p2_p3_p4
+            inputs:
+              p1: {}
+              p2: {}
+              p3: {}
+            
+            multi_path_sequences:
+              - paths: [inputs.p1.a, inputs.p1.b]
+                values::from_latin_hypercube:
+                  num_samples: 4             
+            
+            sequences:
+              - paths: [inputs.p2.a, inputs.p2.b] # actually a multi-path sequence
+                values::from_latin_hypercube:
+                  num_samples: 4
+            
+              - path: inputs.p4 # a normal sequence
+                values: [0, 1, 2, 3]
+            
+              - paths: [inputs.p3.a, inputs.p3.b] # actually a multi-path sequence
+                values::from_latin_hypercube:
+                  num_samples: 4
+    """
+    )
+    wft = hf.WorkflowTemplate.from_YAML_string(wft_yaml)
+    es = wft.tasks[0].element_sets[0]
+    mps_lst = es.multi_path_sequences
+    seq_lst = es.sequences
+    assert len(mps_lst) == 3
+    assert len(seq_lst) == 7  # one original plus three multi-path with two paths each
+
+    # check ordering of multi-path sequences is preserved:
+    assert mps_lst[0].paths == ["inputs.p1.a", "inputs.p1.b"]
+    assert mps_lst[1].paths == ["inputs.p2.a", "inputs.p2.b"]
+    assert mps_lst[2].paths == ["inputs.p3.a", "inputs.p3.b"]
+
+    # check sensible ordering of sequences:
+    assert seq_lst[0].path == "inputs.p4"
+    assert seq_lst[1].path == "inputs.p1.a"
+    assert seq_lst[2].path == "inputs.p1.b"
+    assert seq_lst[3].path == "inputs.p2.a"
+    assert seq_lst[4].path == "inputs.p2.b"
+    assert seq_lst[5].path == "inputs.p3.a"
+    assert seq_lst[6].path == "inputs.p3.b"

hpcflow/tests/unit/test_parameter.py

@@ -11,6 +11,7 @@ import pytest
 import requests
 
 from hpcflow.app import app as hf
+from hpcflow.sdk.core.errors import ParametersMetadataReadOnlyError
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.parameters import ParameterValue
 
@@ -228,3 +229,15 @@ def test_demo_data_substitution_value_sequence_class_method(
         "9",
         "10",
     ]
+
+
+def test_json_store_parameters_metadata_cache_raises_on_modify(
+    null_config, tmp_path: Path
+):
+    wk = hf.make_demo_workflow("workflow_1", path=tmp_path, store="json")
+    assert isinstance(wk, hf.Workflow)
+    num_params = len(wk.get_all_parameters())
+    with pytest.raises(ParametersMetadataReadOnlyError):
+        with wk._store.parameters_metadata_cache():
+            wk._add_unset_parameter_data(source={"type": "UNSET-FOR-THIS-TEST"})
+    assert len(wk.get_all_parameters()) == num_params