hpcflow-new2 0.2.0a69__py3-none-any.whl → 0.2.0a71__py3-none-any.whl

hpcflow/sdk/core/task_schema.py

@@ -1,7 +1,7 @@
 from contextlib import contextmanager
 import copy
 from dataclasses import dataclass, field
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from hpcflow.sdk import app
 from hpcflow.sdk.core.parameters import Parameter
@@ -50,7 +50,7 @@ class TaskSchema(JSONLike):
     def __init__(
         self,
         objective: Union[app.TaskObjective, str],
-        actions: List[app.Action],
+        actions: List[app.Action] = None,
         method: Optional[str] = None,
         implementation: Optional[str] = None,
         inputs: Optional[List[Union[app.Parameter, app.SchemaInput]]] = None,
@@ -59,7 +59,7 @@ class TaskSchema(JSONLike):
         _hash_value: Optional[str] = None,
     ):
         self.objective = objective
-        self.actions = actions
+        self.actions = actions or []
         self.method = method
         self.implementation = implementation
         self.inputs = inputs or []
@@ -203,10 +203,11 @@ class TaskSchema(JSONLike):
                     for OFP_j in act_i.output_file_parsers:
                         extra_ins = extra_ins - set(OFP_j.inputs or [])
 
-                if extra_ins:
+                if self.actions and extra_ins:
+                    # allow for no actions (e.g. defining inputs for downstream tasks)
                     raise ValueError(
-                        f"Schema {self.name!r} inputs {tuple(extra_ins)!r} are not used by "
-                        f"any actions."
+                        f"Schema {self.name!r} inputs {tuple(extra_ins)!r} are not used "
+                        f"by any actions."
                     )
 
             missing_outs = set(self.output_types) - set(all_outs)
@@ -225,11 +226,12 @@ class TaskSchema(JSONLike):
     def make_persistent(self, workflow: app.Workflow, source: Dict) -> List[int]:
         new_refs = []
         for input_i in self.inputs:
-            if input_i.default_value is not None:
-                _, dat_ref, is_new = input_i.default_value.make_persistent(
-                    workflow, source
-                )
-                new_refs.extend(dat_ref) if is_new else None
+            for lab_info in input_i.labelled_info():
+                if "default_value" in lab_info:
+                    _, dat_ref, is_new = lab_info["default_value"].make_persistent(
+                        workflow, source
+                    )
+                    new_refs.extend(dat_ref) if is_new else None
         return new_refs
 
     @property
@@ -243,19 +245,26 @@ class TaskSchema(JSONLike):
 
     @property
     def input_types(self):
-        return tuple(i.typ for i in self.inputs)
+        return tuple(j for i in self.inputs for j in i.all_labelled_types)
 
     @property
     def output_types(self):
         return tuple(i.typ for i in self.outputs)
 
     @property
-    def provides_parameters(self):
-        return tuple(
-            i
-            for i in self.inputs + self.outputs
-            if i.propagation_mode != ParameterPropagationMode.NEVER
-        )
+    def provides_parameters(self) -> Tuple[Tuple[str, str]]:
+        out = []
+        for schema_inp in self.inputs:
+            for labelled_info in schema_inp.labelled_info():
+                prop_mode = labelled_info["propagation_mode"]
+                if prop_mode is not ParameterPropagationMode.NEVER:
+                    out.append(
+                        (schema_inp.input_or_output, labelled_info["labelled_type"])
+                    )
+        for schema_out in self.outputs:
+            if schema_out.propagation_mode is not ParameterPropagationMode.NEVER:
+                out.append((schema_out.input_or_output, schema_out.typ))
+        return tuple(out)
 
     @property
     def task_template(self):

hpcflow/sdk/core/test_utils.py

@@ -1,5 +1,7 @@
-from typing import List, Tuple, Union
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
 from hpcflow.app import app as hf
+from hpcflow.sdk.core.parameters import ParameterValue
 
 
 def make_schemas(ins_outs, ret_list=False):
@@ -39,6 +41,7 @@ def make_schemas(ins_outs, ret_list=False):
             output_file_parsers=out_file_parsers,
             environments=[hf.ActionEnvironment(hf.Environment(name="env_1"))],
         )
+        print(f"{ins_i=}")
         out.append(
             hf.TaskSchema(
                 objective=obj,
@@ -149,3 +152,15 @@ def make_workflow(
         store=store,
     )
     return wk
+
+
+@dataclass
+class P1_parameter_cls(ParameterValue):
+    _typ = "p1"
+
+    a: int
+    d: Optional[int] = None
+
+    @classmethod
+    def from_data(cls, b, c):
+        return cls(a=b + c)

hpcflow/sdk/core/utils.py

@@ -1,4 +1,4 @@
-import copy
+import enum
 from functools import wraps
 import contextlib
 import hashlib
@@ -14,7 +14,7 @@ import string
 import subprocess
 from datetime import datetime, timezone
 import sys
-from typing import List, Mapping
+from typing import Type, Union, List, Mapping
 
 from ruamel.yaml import YAML
 import sentry_sdk
@@ -613,3 +613,19 @@ def open_file(filename):
     else:
         opener = "open" if sys.platform == "darwin" else "xdg-open"
         subprocess.call([opener, filename])
+
+
+def get_enum_by_name_or_val(enum_cls: Type, key: Union[str, None]) -> enum.Enum:
+    """Retrieve an enum by name or value, assuming uppercase names and integer values."""
+    err = f"Unknown enum key or value {key!r} for class {enum_cls!r}"
+    if key is None or isinstance(key, enum_cls):
+        return key
+    elif isinstance(key, (int, float)):
+        return enum_cls(int(key))  # retrieve by value
+    elif isinstance(key, str):
+        try:
+            return getattr(enum_cls, key.upper())  # retrieve by name
+        except AttributeError:
+            raise ValueError(err)
+    else:
+        raise ValueError(err)

hpcflow/sdk/core/workflow.py

@@ -127,9 +127,11 @@ class WorkflowTemplate(JSONLike):
             if "element_sets" not in task_dat:
                 # add a single element set:
                 schemas = task_dat.pop("schemas")
+                out_labels = task_dat.pop("output_labels", [])
                 data["tasks"][task_idx] = {
                     "schemas": schemas,
                     "element_sets": [task_dat],
+                    "output_labels": out_labels,
                 }
 
         # extract out any template components:
@@ -1179,15 +1181,15 @@ class Workflow:
                     for loop in self.loops:
                         loop._accept_pending_num_added_iters()
 
-                    if is_workflow_creation:
-                        self._store.remove_replaced_dir()
-
                     # TODO: handle errors in commit pending?
                     self._store._pending.commit_all()
-
                     self._accept_pending()
-                    self.app.persistence_logger.info("exiting batch update")
-                    self._in_batch_mode = False
+
+                if is_workflow_creation:
+                    self._store.remove_replaced_dir()
+
+                self.app.persistence_logger.info("exiting batch update")
+                self._in_batch_mode = False
 
     @classmethod
     def temporary_rename(cls, path: str, fs) -> List[str]:
@@ -1341,10 +1343,11 @@ class Workflow:
         return self.get_parameters([index], **kwargs)[0]
 
     def get_parameter_data(self, index: int, **kwargs: Dict) -> Any:
-        return (
-            self.get_parameter(index, **kwargs).data
-            or self.get_parameter(index, **kwargs).file
-        )
+        param = self.get_parameter(index, **kwargs)
+        if param.data is not None:
+            return param.data
+        else:
+            return param.file
 
     def get_parameter_source(self, index: int) -> Dict:
         return self.get_parameter_sources([index])[0]
@@ -1576,7 +1579,6 @@ class Workflow:
                 if EAR.action.abortable and exit_code == ABORT_EXIT_CODE:
                     # the point of aborting an EAR is to continue with the workflow:
                     success = True
-                self._store.set_EAR_end(EAR_ID, exit_code, success)
 
                 for IFG_i in EAR.action.input_file_generators:
                     inp_file = IFG_i.input_file
@@ -1644,6 +1646,8 @@ class Workflow:
                         )
                         self._store.set_EAR_skip(EAR_dep_ID)
 
+                self._store.set_EAR_end(EAR_ID, exit_code, success)
+
     def set_EAR_skip(self, EAR_ID: int) -> None:
         """Record that an EAR is to be skipped due to an upstream failure."""
         with self._store.cached_load():

hpcflow/tests/unit/test_app.py

@@ -58,14 +58,7 @@ def test_shared_data_from_json_like_with_shared_data_dependency(act_1):
                 "implementation": ts1.implementation,
                 "version": ts1.version,
                 "objective": ts1.objective.name,
-                "inputs": [
-                    {
-                        "group": None,
-                        "where": None,
-                        "parameter": f"hash:{p1_hash}",
-                        "propagation_mode": "IMPLICIT",
-                    }
-                ],
+                "inputs": [{"parameter": f"hash:{p1_hash}", "labels": {"": {}}}],
                 "outputs": [],
                 "actions": [
                     {

hpcflow/tests/unit/test_input_value.py

@@ -88,3 +88,44 @@ def test_value_sequence_from_json_like_class_method_attribute_is_set():
         json_like, shared_data=hf.template_components
     )
     assert val_seq.value_class_method == cls_method
+
+
+def test_path_attributes():
+    inp = hf.InputValue(parameter="p1", value=101, path="a.b")
+    assert inp.labelled_type == "p1"
+    assert inp.normalised_path == "inputs.p1.a.b"
+    assert inp.normalised_inputs_path == "p1.a.b"
+
+
+def test_path_attributes_with_label_arg():
+    inp = hf.InputValue(parameter="p1", value=101, path="a.b", label="1")
+    assert inp.labelled_type == "p1[1]"
+    assert inp.normalised_path == "inputs.p1[1].a.b"
+    assert inp.normalised_inputs_path == "p1[1].a.b"
+
+
+def test_path_attributes_with_label_arg_cast():
+    inp = hf.InputValue(parameter="p1", value=101, path="a.b", label=1)
+    assert inp.labelled_type == "p1[1]"
+    assert inp.normalised_path == "inputs.p1[1].a.b"
+    assert inp.normalised_inputs_path == "p1[1].a.b"
+
+
+def test_from_json_like():
+    inp = hf.InputValue.from_json_like(
+        json_like={"parameter": "p1", "value": 101},
+        shared_data=hf.template_components,
+    )
+    assert inp.parameter.typ == hf.Parameter("p1").typ
+    assert inp.value == 101
+    assert inp.label == ""
+
+
+def test_from_json_like_with_label():
+    inp = hf.InputValue.from_json_like(
+        json_like={"parameter": "p1[1]", "value": 101},
+        shared_data=hf.template_components,
+    )
+    assert inp.parameter.typ == hf.Parameter("p1").typ
+    assert inp.value == 101
+    assert inp.label == "1"

hpcflow/tests/unit/test_schema_input.py

@@ -0,0 +1,191 @@
+import pytest
+
+from hpcflow.app import app as hf
+from hpcflow.sdk.core.test_utils import P1_parameter_cls as P1
+
+
+@pytest.fixture
+def null_config(tmp_path):
+    if not hf.is_config_loaded:
+        hf.load_config(config_dir=tmp_path)
+
+
+def test_null_default_value():
+    p1 = hf.Parameter("p1")
+    p1_inp = hf.SchemaInput(parameter=p1)
+    assert "default_value" not in p1_inp.labels[""]
+
+
+def test_none_default_value():
+    """A `None` default value is set with a value of `None`"""
+    p1 = hf.Parameter("p1")
+    p1_inp = hf.SchemaInput(parameter=p1, default_value=None)
+    def_val_exp = hf.InputValue(parameter=p1, label="", value=None)
+    def_val_exp._schema_input = p1_inp
+    assert p1_inp.labels[""]["default_value"] == def_val_exp
+
+
+def test_from_json_like_labels_and_default():
+    json_like = {
+        "parameter": "p1",
+        "labels": {"0": {}},
+        "default_value": None,
+    }
+    inp = hf.SchemaInput.from_json_like(
+        json_like=json_like,
+        shared_data=hf.template_components,
+    )
+    assert inp.labels["0"]["default_value"].value == None
+
+
+def test_element_get_removes_schema_param_trivial_label(null_config, tmp_path):
+    p1_val = 101
+    label = "my_label"
+    s1 = hf.TaskSchema(
+        objective="t1", inputs=[hf.SchemaInput(parameter="p1", labels={label: {}})]
+    )
+    t1 = hf.Task(schemas=[s1], inputs=[hf.InputValue("p1", p1_val, label=label)])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="temp",
+    )
+    assert f"inputs.p1[{label}]" in wk.tasks[0].elements[0].get_data_idx("inputs")
+    assert wk.tasks[0].elements[0].get("inputs") == {"p1": p1_val}
+
+
+def test_element_inputs_removes_schema_param_trivial_label(null_config, tmp_path):
+    p1_val = 101
+    label = "my_label"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter="p1", labels={label: {}})],
+        actions=[
+            hf.Action(
+                environments=[hf.ActionEnvironment(environment=hf.envs.null_env)],
+                commands=[hf.Command(command=f"echo <<parameter:p1[{label}]>>")],
+            ),
+        ],
+    )
+    t1 = hf.Task(schemas=[s1], inputs=[hf.InputValue("p1", p1_val, label=label)])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="temp",
+    )
+    element = wk.tasks[0].elements[0]
+    # element inputs:
+    assert element.inputs._get_prefixed_names() == ["p1"]
+
+    # element iteration inputs:
+    assert element.iterations[0].inputs._get_prefixed_names() == ["p1"]
+
+    # run inputs:
+    assert element.iterations[0].action_runs[0].inputs._get_prefixed_names() == ["p1"]
+
+
+def test_element_get_does_not_removes_multiple_schema_param_label(null_config, tmp_path):
+    p1_val = 101
+    label = "my_label"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter="p1", labels={label: {}}, multiple=True)],
+    )
+    t1 = hf.Task(schemas=[s1], inputs=[hf.InputValue("p1", p1_val, label=label)])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="temp",
+    )
+    assert f"inputs.p1[{label}]" in wk.tasks[0].elements[0].get_data_idx("inputs")
+    assert wk.tasks[0].elements[0].get("inputs") == {f"p1[{label}]": p1_val}
+
+
+def test_element_inputs_does_not_remove_multiple_schema_param_label(
+    null_config, tmp_path
+):
+    p1_val = 101
+    label = "my_label"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter="p1", labels={label: {}}, multiple=True)],
+        actions=[
+            hf.Action(
+                environments=[hf.ActionEnvironment(environment=hf.envs.null_env)],
+                commands=[hf.Command(command=f"echo <<parameter:p1[{label}]>>")],
+            ),
+        ],
+    )
+    t1 = hf.Task(schemas=[s1], inputs=[hf.InputValue("p1", p1_val, label=label)])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="temp",
+    )
+    element = wk.tasks[0].elements[0]
+    # element inputs:
+    assert element.inputs._get_prefixed_names() == [f"p1[{label}]"]
+
+    # element iteration inputs:
+    assert element.iterations[0].inputs._get_prefixed_names() == [f"p1[{label}]"]
+
+    # run inputs:
+    assert element.iterations[0].action_runs[0].inputs._get_prefixed_names() == [
+        f"p1[{label}]"
+    ]
+
+
+def test_get_input_values_for_multiple_schema_input(null_config, tmp_path):
+    p1_val = 101
+    label = "my_label"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter="p1", labels={label: {}}, multiple=True),
+            hf.SchemaInput(parameter="p2", default_value=201),
+        ],
+        actions=[
+            hf.Action(
+                environments=[hf.ActionEnvironment(environment=hf.envs.null_env)],
+                commands=[
+                    hf.Command(command=f"echo <<parameter:p1[{label}]>> <<parameter:p2>>")
+                ],
+            ),
+        ],
+    )
+    t1 = hf.Task(schemas=[s1], inputs=[hf.InputValue("p1", p1_val, label=label)])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="temp",
+    )
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    assert run.get_input_values() == {"p2": 201, "p1": {label: 101}}
+
+
+def test_get_input_values_for_multiple_schema_input_with_object(null_config, tmp_path):
+    p1_val = P1(a=101)
+    label = "my_label"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[
+            hf.SchemaInput(parameter="p1", labels={label: {}}, multiple=True),
+            hf.SchemaInput(parameter="p2", default_value=201),
+        ],
+        actions=[
+            hf.Action(
+                environments=[hf.ActionEnvironment(environment=hf.envs.null_env)],
+                commands=[
+                    hf.Command(command=f"echo <<parameter:p1[{label}]>> <<parameter:p2>>")
+                ],
+            ),
+        ],
+    )
+    t1 = hf.Task(schemas=[s1], inputs=[hf.InputValue("p1", p1_val, label=label)])
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        path=tmp_path,
+        template_name="temp",
+    )
+    run = wk.tasks[0].elements[0].iterations[0].action_runs[0]
+    assert run.get_input_values() == {"p2": 201, "p1": {label: p1_val}}