hpcflow-new2 0.2.0a162__py3-none-any.whl → 0.2.0a164__py3-none-any.whl

hpcflow/sdk/demo/cli.py

@@ -184,7 +184,7 @@ def get_demo_workflow_CLI(app):
             status=status,
         )
         if print_idx:
-            click.echo(out)
+            click.echo(out[1])
 
     @demo_workflow.command("copy")
     @click.argument("workflow_name")

hpcflow/sdk/persistence/base.py

@@ -310,6 +310,7 @@ class StoreEAR:
     is_pending: bool
     elem_iter_ID: int
     action_idx: int
+    commands_idx: List[int]
     data_idx: Dict[str, int]
     submission_idx: Optional[int] = None
     skip: Optional[bool] = False
@@ -336,6 +337,7 @@ class StoreEAR:
             "id_": self.id_,
             "elem_iter_ID": self.elem_iter_ID,
             "action_idx": self.action_idx,
+            "commands_idx": self.commands_idx,
             "data_idx": self.data_idx,
             "submission_idx": self.submission_idx,
             "success": self.success,
@@ -371,6 +373,7 @@ class StoreEAR:
             "is_pending": self.is_pending,
             "elem_iter_ID": self.elem_iter_ID,
             "action_idx": self.action_idx,
+            "commands_idx": self.commands_idx,
             "data_idx": self.data_idx,
             "submission_idx": self.submission_idx,
             "success": self.success,
@@ -414,6 +417,7 @@ class StoreEAR:
             is_pending=self.is_pending,
             elem_iter_ID=self.elem_iter_ID,
             action_idx=self.action_idx,
+            commands_idx=self.commands_idx,
             data_idx=self.data_idx,
             metadata=self.metadata,
             submission_idx=sub_idx,
@@ -1013,6 +1017,7 @@ class PersistentStore(ABC):
         self,
         elem_iter_ID: int,
         action_idx: int,
+        commands_idx: List[int],
         data_idx: Dict,
         metadata: Dict,
         save: bool = True,
@@ -1025,6 +1030,7 @@ class PersistentStore(ABC):
             is_pending=True,
             elem_iter_ID=elem_iter_ID,
             action_idx=action_idx,
+            commands_idx=commands_idx,
             data_idx=data_idx,
             metadata=metadata,
         )

hpcflow/sdk/persistence/zarr.py

@@ -237,6 +237,7 @@ class ZarrStoreEAR(StoreEAR):
             self.exit_code,
             self.metadata,
             self.run_hostname,
+            self.commands_idx,
         ]
         return EAR_enc
 
@@ -258,6 +259,7 @@ class ZarrStoreEAR(StoreEAR):
             "exit_code": EAR_dat[11],
             "metadata": EAR_dat[12],
             "run_hostname": EAR_dat[13],
+            "commands_idx": EAR_dat[14],
         }
         return cls(is_pending=False, **obj_dat)
 

hpcflow/sdk/submission/submission.py

@@ -15,9 +15,11 @@ from hpcflow.sdk.core.errors import (
     MissingEnvironmentError,
     MissingEnvironmentExecutableError,
     MissingEnvironmentExecutableInstanceError,
+    MultipleEnvironmentsError,
     SubmissionFailure,
 )
 from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
+from hpcflow.sdk.core.object_list import ObjectListMultipleMatchError
 from hpcflow.sdk.log import TimeIt
 
 
@@ -90,31 +92,40 @@ class Submission(JSONLike):
         req_envs = defaultdict(lambda: defaultdict(set))
         for js_idx, js_i in enumerate(self.jobscripts):
             for run in js_i.all_EARs:
-                env_label = run.action.get_environment_label()
+                env_spec_h = tuple(zip(*run.env_spec.items()))  # hashable
                 for exec_label_j in run.action.get_required_executables():
-                    req_envs[env_label][exec_label_j].add(js_idx)
-                if env_label not in req_envs:
-                    req_envs[env_label] = {}
+                    req_envs[env_spec_h][exec_label_j].add(js_idx)
+                if env_spec_h not in req_envs:
+                    req_envs[env_spec_h] = {}
 
         # check these envs/execs exist in app data:
         envs = []
-        for env_lab, exec_js in req_envs.items():
+        for env_spec_h, exec_js in req_envs.items():
+            env_spec = dict(zip(*env_spec_h))
+            non_name_spec = {k: v for k, v in env_spec.items() if k != "name"}
+            spec_str = f" with specifiers {non_name_spec!r}" if non_name_spec else ""
+            env_ref = f"{env_spec['name']!r}{spec_str}"
             try:
-                env_i = self.app.envs.get(env_lab)
+                env_i = self.app.envs.get(**env_spec)
+            except ObjectListMultipleMatchError:
+                raise MultipleEnvironmentsError(
+                    f"Multiple environments {env_ref} are defined on this machine."
+                )
             except ValueError:
                 raise MissingEnvironmentError(
-                    f"The environment {env_lab!r} is not defined on this machine, so the "
+                    f"The environment {env_ref} is not defined on this machine, so the "
                     f"submission cannot be created."
                 ) from None
             else:
-                envs.append(env_i)
+                if env_i not in envs:
+                    envs.append(env_i)
 
             for exec_i_lab, js_idx_set in exec_js.items():
                 try:
                     exec_i = env_i.executables.get(exec_i_lab)
                 except ValueError:
                     raise MissingEnvironmentExecutableError(
-                        f"The environment {env_lab!r} as defined on this machine has no "
+                        f"The environment {env_ref} as defined on this machine has no "
                         f"executable labelled {exec_i_lab!r}, which is required for this "
                         f"submission, so the submission cannot be created."
                     ) from None
@@ -127,7 +138,7 @@ class Submission(JSONLike):
                     if not exec_instances:
                         raise MissingEnvironmentExecutableInstanceError(
                             f"No matching executable instances found for executable "
-                            f"{exec_i_lab!r} of environment {env_lab!r} for jobscript "
+                            f"{exec_i_lab!r} of environment {env_ref} for jobscript "
                             f"index {js_idx_j!r} with requested resources "
                             f"{filter_exec!r}."
                         )

hpcflow/tests/scripts/test_main_scripts.py

@@ -447,3 +447,63 @@ def test_script_hdf5_out_obj(null_config, tmp_path):
     # to be later Python versions):
     time.sleep(10)
     assert wk.tasks[0].elements[0].outputs.p1c.value == P1(a=p1_val + 100)
+
+
+@pytest.mark.integration
+@pytest.mark.skipif("hf.run_time_info.is_frozen")
+def test_script_direct_in_pass_env_spec(new_null_config, tmp_path):
+
+    vers_spec = {"version": "1.2"}
+    env = hf.Environment(
+        name="python_env_with_specifiers",
+        specifiers=vers_spec,
+        executables=[
+            hf.Executable(
+                label="python_script",
+                instances=[
+                    hf.ExecutableInstance(
+                        command="python <<script_name>> <<args>>",
+                        num_cores=1,
+                        parallel_mode=None,
+                    )
+                ],
+            )
+        ],
+    )
+    hf.envs.add_object(env, skip_duplicates=True)
+
+    s1 = hf.TaskSchema(
+        objective="t1",
+        inputs=[hf.SchemaInput(parameter=hf.Parameter("p1"))],
+        outputs=[hf.SchemaOutput(parameter=hf.Parameter("p2"))],
+        actions=[
+            hf.Action(
+                script="<<script:main_script_test_direct_in_direct_out_env_spec.py>>",
+                script_data_in="direct",
+                script_data_out="direct",
+                script_exe="python_script",
+                script_pass_env_spec=True,
+                environments=[
+                    hf.ActionEnvironment(environment="python_env_with_specifiers")
+                ],
+            )
+        ],
+    )
+    t1 = hf.Task(
+        schema=s1,
+        inputs={"p1": 101},
+        environments={"python_env_with_specifiers": vers_spec},
+    )
+    wk = hf.Workflow.from_template_data(
+        tasks=[t1],
+        template_name="main_script_test",
+        path=tmp_path,
+    )
+    wk.submit(wait=True, add_to_known=False)
+    # TODO: investigate why the value is not always populated on GHA Ubuntu runners (tends
+    # to be later Python versions):
+    time.sleep(10)
+    assert wk.tasks[0].elements[0].outputs.p2.value == {
+        "name": "python_env_with_specifiers",
+        **vers_spec,
+    }

hpcflow/tests/unit/test_action.py

@@ -3,6 +3,7 @@ import pytest
 
 from hpcflow.app import app as hf
 from hpcflow.sdk.core.errors import (
+    ActionEnvironmentMissingNameError,
     UnknownScriptDataKey,
     UnknownScriptDataParameter,
     UnsupportedScriptDataFormat,
@@ -602,3 +603,188 @@ def test_process_script_data_in_fmt_dict_mixed(null_config):
         "p1": {"format": "json"},
         "p2": {"format": "hdf5"},
     }
+
+
+def test_ActionEnvironment_env_str(null_config):
+    act_env = hf.ActionEnvironment(environment="my_env")
+    assert act_env.environment == {"name": "my_env"}
+
+
+def test_ActionEnvironment_env_dict(null_config):
+    act_env = hf.ActionEnvironment(environment={"name": "my_env", "key": "value"})
+    assert act_env.environment == {"name": "my_env", "key": "value"}
+
+
+def test_ActionEnvironment_raises_on_missing_name(null_config):
+    with pytest.raises(ActionEnvironmentMissingNameError):
+        hf.ActionEnvironment(environment={"key": "value"})
+
+
+def test_rules_allow_runs_initialised(null_config, tmp_path):
+    """Test rules that do not depend on execution allow for runs to be initialised."""
+    act = hf.Action(
+        script="<<script:path/to/some/script>>",
+        rules=[hf.ActionRule(path="inputs.p1", condition={"value.less_than": 2})],
+    )
+    ts = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        actions=[act],
+    )
+    t1 = hf.Task(
+        schema=ts, sequences=[hf.ValueSequence(path="inputs.p1", values=[1.5, 2.5])]
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert wk.tasks[0].elements[1].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].actions) == 1
+    assert len(wk.tasks[0].elements[1].actions) == 0
+
+
+def test_rules_prevent_runs_initialised(null_config, tmp_path):
+    """Test rules that depend on execution prevent initialising runs."""
+    act1 = hf.Action(script="<<script:path/to/some/script>>")
+    act2 = hf.Action(
+        script="<<script:path/to/some/script>>",
+        rules=[hf.ActionRule(path="inputs.p2", condition={"value.less_than": 2})],
+    )
+    ts1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[act1],
+    )
+    ts2 = hf.TaskSchema(
+        objective="ts2",
+        inputs=[hf.SchemaInput("p2")],
+        actions=[act2],
+    )
+    t1 = hf.Task(schema=ts1, inputs={"p1": 1.2})
+    t2 = hf.Task(schema=ts2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert not wk.tasks[1].elements[0].iterations[0].EARs_initialised
+
+
+def test_command_rules_allow_runs_initialised(null_config, tmp_path):
+    """Test command rules that do not depend on execution allow for runs to be
+    initialised."""
+    act = hf.Action(
+        commands=[
+            hf.Command(
+                command='echo "p1=<<parameter:p1>>"',
+                rules=[hf.ActionRule(path="inputs.p1", condition={"value.less_than": 2})],
+            )
+        ],
+    )
+    ts = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        actions=[act],
+    )
+    t1 = hf.Task(
+        schema=ts, sequences=[hf.ValueSequence(path="inputs.p1", values=[1.5, 2.5])]
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert wk.tasks[0].elements[1].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].actions) == 1
+    assert len(wk.tasks[0].elements[1].actions) == 1
+    assert len(wk.tasks[0].elements[0].action_runs[0].commands_idx) == 1
+    assert len(wk.tasks[0].elements[1].action_runs[0].commands_idx) == 0
+
+
+def test_command_rules_prevent_runs_initialised(null_config, tmp_path):
+    """Test command rules that do depend on execution prevent runs being initialised."""
+    act1 = hf.Action(
+        commands=[
+            hf.Command(command='echo "p1=<<parameter:p1>>"', stdout="<<parameter:p2>>")
+        ]
+    )
+    act2 = hf.Action(
+        commands=[
+            hf.Command(
+                command='echo "p1=<<parameter:p2>>"',
+                rules=[hf.ActionRule(path="inputs.p2", condition={"value.less_than": 2})],
+            )
+        ],
+    )
+    ts1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[act1],
+    )
+    ts2 = hf.TaskSchema(
+        objective="ts2",
+        inputs=[hf.SchemaInput("p2")],
+        actions=[act2],
+    )
+    t1 = hf.Task(schema=ts1, inputs={"p1": 0})
+    t2 = hf.Task(schema=ts2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].action_runs[0].commands_idx) == 1
+    assert not wk.tasks[1].elements[0].iterations[0].EARs_initialised
+
+
+def test_command_rules_prevent_runs_initialised_with_valid_action_rules(
+    null_config, tmp_path
+):
+    """Test command rules that do depend on execution prevent runs being initialised, even
+    when the parent action rules can be tested and are valid."""
+    act1 = hf.Action(
+        commands=[
+            hf.Command(command='echo "p1=<<parameter:p1>>"', stdout="<<parameter:p2>>")
+        ]
+    )
+
+    # action rule is testable and valid, but command rule is not testable, so the action
+    # runs should not be initialised:
+    act2 = hf.Action(
+        commands=[
+            hf.Command(
+                command='echo "p1=<<parameter:p1>>; p2=<<parameter:p2>>"',
+                rules=[hf.ActionRule(path="inputs.p2", condition={"value.less_than": 2})],
+            )
+        ],
+        rules=[hf.ActionRule(path="inputs.p1", condition={"value.less_than": 2})],
+    )
+    ts1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[act1],
+    )
+    ts2 = hf.TaskSchema(
+        objective="ts2",
+        inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")],
+        actions=[act2],
+    )
+    t1 = hf.Task(schema=ts1, inputs={"p1": 0})
+    t2 = hf.Task(schema=ts2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].action_runs[0].commands_idx) == 1
+
+    assert not wk.tasks[1].elements[0].iterations[0].EARs_initialised

hpcflow/tests/unit/test_element.py

@@ -519,11 +519,12 @@ def test_element_get_group_sequence(null_config, tmp_path):
     wkt_yaml = dedent(
         """\
         name: test_list_idx_sequence
-        task_schemas:
-          - objective: test_group_schema
-            inputs:
-              - parameter: p1
-                group: my_group
+        template_components:
+          task_schemas:
+            - objective: test_group_schema
+              inputs:
+                - parameter: p1
+                  group: my_group
         tasks:
           - schema: test_t1_ps
             inputs:
@@ -556,26 +557,27 @@ def test_element_get_group_sequence_obj(new_null_config, tmp_path):
     wkt_yaml = dedent(
         """\
         name: test_list_idx_sequence
-        task_schemas:
-          - objective: test_t1_ps_obj
-            parameter_class_modules: ["hpcflow.sdk.core.test_utils"]
-            inputs:
-              - parameter: p1c
-            outputs:
-              - parameter: p2
-            actions:
-              - environments:
-                  - scope:
-                      type: any
-                    environment: null_env
-                commands:
-                  - command: Write-Output ((<<parameter:p1c>> + 100))
-                    stdout: <<parameter:p2>>
-          - objective: test_group_schema
-            parameter_class_modules: ["hpcflow.sdk.core.test_utils"]
-            inputs:
-              - parameter: p1c
-                group: my_group
+        template_components:
+          task_schemas:
+            - objective: test_t1_ps_obj
+              parameter_class_modules: ["hpcflow.sdk.core.test_utils"]
+              inputs:
+                - parameter: p1c
+              outputs:
+                - parameter: p2
+              actions:
+                - environments:
+                    - scope:
+                        type: any
+                      environment: null_env
+                  commands:
+                    - command: Write-Output ((<<parameter:p1c>> + 100))
+                      stdout: <<parameter:p2>>
+            - objective: test_group_schema
+              parameter_class_modules: ["hpcflow.sdk.core.test_utils"]
+              inputs:
+                - parameter: p1c
+                  group: my_group
 
         tasks:
           - schema: test_t1_ps_obj

hpcflow/tests/unit/test_element_set.py

@@ -69,3 +69,35 @@ def test_repeats_single_int_equivalence(null_config):
     es1 = hf.ElementSet(repeats=2)
     es2 = hf.ElementSet(repeats=[{"name": "", "number": 2, "nesting_order": 0}])
     assert es1 == es2
+
+
+def test_merge_envs(null_config):
+    envs = {"my_env": {"version": "1.0"}}
+    es = hf.ElementSet(environments=envs)
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == envs
+
+
+def test_merge_envs_existing_any_resources(null_config):
+    envs = {"my_env": {"version": "1.0"}}
+    num_cores = 2
+    es = hf.ElementSet(resources={"any": {"num_cores": num_cores}}, environments=envs)
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == envs
+    assert es.resources.get(scope=hf.ActionScope.any()).num_cores == num_cores
+
+
+def test_merge_envs_resource_envs_precedence(null_config):
+    envs = {"my_env": {"version": "1.0"}}
+    res_envs = {"other_env": {"version": "2.0"}}
+    es = hf.ElementSet(resources={"any": {"environments": res_envs}}, environments=envs)
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == res_envs
+
+
+def test_merge_envs_no_envs_with_resource_envs(null_config):
+    envs = {"my_env": {"version": "1.0"}}
+    es = hf.ElementSet(resources={"any": {"environments": envs}})
+    assert es.resources.get(scope=hf.ActionScope.any()).environments == envs
+
+
+def test_raise_env_and_envs_specified(null_config):
+    with pytest.raises(ValueError):
+        hf.ElementSet(env_preset="my_preset", environments={"my_env": {"version": 1}})

hpcflow/tests/unit/test_parameter.py

@@ -152,11 +152,12 @@ def test_demo_data_substitution_param_value_class_method(new_null_config, tmp_pa
     yaml_str = dedent(
         """\
         name: temp
-        task_schemas:
-          - objective: test
-            inputs:
-              - parameter: p1c
-            parameter_class_modules: [hpcflow.sdk.core.test_utils]
+        template_components:
+          task_schemas:
+            - objective: test
+              inputs:
+                - parameter: p1c
+              parameter_class_modules: [hpcflow.sdk.core.test_utils]
         tasks: 
           - schema: test
             inputs:
@@ -174,10 +175,11 @@ def test_demo_data_substitution_value_sequence_class_method(new_null_config, tmp
     yaml_str = dedent(
         """\
         name: temp
-        task_schemas:
-          - objective: test
-            inputs:
-              - parameter: p1
+        template_components:
+          task_schemas:
+            - objective: test
+              inputs:
+                - parameter: p1
         tasks:
           - schema: test
             sequences:

hpcflow/tests/unit/test_persistence.py

@@ -74,6 +74,7 @@ def test_store_pending_add_EAR(tmp_path):
     EAR_ID = store.add_EAR(
         elem_iter_ID=0,
         action_idx=0,
+        commands_idx=[],
         data_idx={},
         metadata={},
     )
@@ -83,6 +84,7 @@ def test_store_pending_add_EAR(tmp_path):
             is_pending=True,
             elem_iter_ID=0,
             action_idx=0,
+            commands_idx=[],
             data_idx={},
             metadata={},
         )
@@ -186,7 +188,7 @@ def test_get_task_elements_single_element_iter_EAR_pending(tmp_path):
     store = JSONPersistentStore.make_test_store_from_spec(
         [{"elements": [{"iterations": [{}]}]}], dir=tmp_path
     )
-    store.add_EAR(elem_iter_ID=0, action_idx=0, data_idx={}, metadata={})
+    store.add_EAR(elem_iter_ID=0, action_idx=0, commands_idx=[], data_idx={}, metadata={})
     assert store.get_task_elements(0, slice(0, None)) == [
         {
             "id": 0,
@@ -209,6 +211,7 @@ def test_get_task_elements_single_element_iter_EAR_pending(tmp_path):
                                 "is_pending": True,
                                 "elem_iter_ID": 0,
                                 "action_idx": 0,
+                                "commands_idx": [],
                                 "data_idx": {},
                                 "metadata": {},
                             }

hpcflow/tests/unit/test_resources.py

@@ -29,33 +29,31 @@ def test_resource_list_raise_on_identical_scopes():
         hf.ResourceList.normalise([{"scope": "any"}, {"scope": "any"}])
 
 
-def test_merge_template_resources_same_scope():
+def test_merge_other_same_scope():
     res_lst_1 = hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
     res_lst_2 = hf.ResourceList.from_json_like({"any": {}})
-    res_lst_2.merge_template_resources(res_lst_1)
+    res_lst_2.merge_other(res_lst_1)
     assert res_lst_2 == hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
 
 
-def test_merge_template_resources_same_scope_no_overwrite():
+def test_merge_other_same_scope_no_overwrite():
     res_lst_1 = hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
     res_lst_2 = hf.ResourceList.from_json_like({"any": {"num_cores": 2}})
-    res_lst_2.merge_template_resources(res_lst_1)
+    res_lst_2.merge_other(res_lst_1)
     assert res_lst_2 == hf.ResourceList.from_json_like({"any": {"num_cores": 2}})
 
 
-def test_merge_template_resources_multi_scope():
+def test_merge_other_multi_scope():
     res_lst_1 = hf.ResourceList.from_json_like({"any": {"num_cores": 1}})
     res_lst_2 = hf.ResourceList.from_json_like({"any": {}, "main": {"num_cores": 3}})
-    res_lst_2.merge_template_resources(res_lst_1)
+    res_lst_2.merge_other(res_lst_1)
     assert res_lst_2 == hf.ResourceList.from_json_like(
         {"any": {"num_cores": 1}, "main": {"num_cores": 3}}
     )
 
 
 @pytest.mark.parametrize("store", ["json", "zarr"])
-def test_merge_template_resources_persistent_workflow_reload(
-    null_config, tmp_path, store
-):
+def test_merge_other_persistent_workflow_reload(null_config, tmp_path, store):
     wkt = hf.WorkflowTemplate(
         name="test_load",
         resources={"any": {"num_cores": 2}},