hpcflow-new2 0.2.0a163__tar.gz → 0.2.0a164__tar.gz

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a163
+Version: 0.2.0a164
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

hpcflow_new2-0.2.0a164/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a164"

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/sdk/core/actions.py

@@ -141,6 +141,7 @@ class ElementActionRun:
         element_action,
         index: int,
         data_idx: Dict,
+        commands_idx: List[int],
         start_time: Union[datetime, None],
         end_time: Union[datetime, None],
         snapshot_start: Union[Dict, None],
@@ -157,6 +158,7 @@ class ElementActionRun:
         self._element_action = element_action
         self._index = index  # local index of this run with the action
         self._data_idx = data_idx
+        self._commands_idx = commands_idx
         self._start_time = start_time
         self._end_time = end_time
         self._submission_idx = submission_idx
@@ -222,6 +224,10 @@ class ElementActionRun:
     def data_idx(self):
         return self._data_idx
 
+    @property
+    def commands_idx(self):
+        return self._commands_idx
+
     @property
     def metadata(self):
         return self._metadata
@@ -715,11 +721,13 @@ class ElementActionRun:
 
         shell_vars = {}  # keys are cmd_idx, each value is a list of tuples
         for cmd_idx, command in enumerate(self.action.commands):
-            cmd_str, shell_vars_i = command.get_command_line(
-                EAR=self, shell=jobscript.shell, env=env
-            )
-            shell_vars[cmd_idx] = shell_vars_i
-            command_lns.append(cmd_str)
+            if cmd_idx in self.commands_idx:
+                # only execute commands that have no rules, or all valid rules:
+                cmd_str, shell_vars_i = command.get_command_line(
+                    EAR=self, shell=jobscript.shell, env=env
+                )
+                shell_vars[cmd_idx] = shell_vars_i
+                command_lns.append(cmd_str)
 
         commands = "\n".join(command_lns) + "\n"
 
@@ -1025,6 +1033,7 @@ class ActionRule(JSONLike):
 
         self.rule = rule
         self.action = None  # assigned by parent action
+        self.command = None  # assigned by parent command
 
     def __eq__(self, other):
         if type(other) is not self.__class__:
@@ -1958,9 +1967,17 @@ class Action(JSONLike):
                 return True
 
     @TimeIt.decorator
-    def test_rules(self, element_iter) -> List[bool]:
+    def test_rules(self, element_iter) -> Tuple[bool, List[int]]:
         """Test all rules against the specified element iteration."""
-        return [i.test(element_iteration=element_iter) for i in self.rules]
+        rules_valid = [rule.test(element_iteration=element_iter) for rule in self.rules]
+        action_valid = all(rules_valid)
+        commands_idx = []
+        if action_valid:
+            for cmd_idx, cmd in enumerate(self.commands):
+                if any(not i.test(element_iteration=element_iter) for i in cmd.rules):
+                    continue
+                commands_idx.append(cmd_idx)
+        return action_valid, commands_idx
 
     def get_required_executables(self) -> Tuple[str]:
         """Return executable labels required by this action."""

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/sdk/core/commands.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from functools import partial
 from pathlib import Path
 import re
@@ -6,15 +6,24 @@ from typing import Dict, Iterable, List, Optional, Tuple, Union
 
 import numpy as np
 
+from hpcflow.sdk import app
 from hpcflow.sdk.core.element import ElementResources
 from hpcflow.sdk.core.errors import NoCLIFormatMethodError
-from hpcflow.sdk.core.json_like import JSONLike
+from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from hpcflow.sdk.core.parameters import ParameterValue
 
 
 @dataclass
 class Command(JSONLike):
     _app_attr = "app"
+    _child_objects = (
+        ChildObjectSpec(
+            name="rules",
+            class_name="ActionRule",
+            is_multiple=True,
+            parent_ref="command",
+        ),
+    )
 
     command: Optional[str] = None
     executable: Optional[str] = None
@@ -23,6 +32,7 @@ class Command(JSONLike):
     stdout: Optional[str] = None
     stderr: Optional[str] = None
     stdin: Optional[str] = None
+    rules: Optional[List[app.ActionRule]] = field(default_factory=lambda: [])
 
     def __repr__(self) -> str:
         out = []
@@ -40,6 +50,8 @@ class Command(JSONLike):
             out.append(f"stderr={self.stderr!r}")
         if self.stdin:
             out.append(f"stdin={self.stdin!r}")
+        if self.rules:
+            out.append(f"rules={self.rules!r}")
 
         return f"{self.__class__.__name__}({', '.join(out)})"
 

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/sdk/core/task.py

@@ -1944,8 +1944,8 @@ class WorkflowTask:
             # run-specific data index to `test_rules` and `generate_data_index`
             # (e.g. if we wanted to increase the memory requirements of a action because
             # it previously failed)
-            rules_valid = action.test_rules(element_iter=element_iter)
-            if all(rules_valid):
+            act_valid, cmds_idx = action.test_rules(element_iter=element_iter)
+            if act_valid:
                 self.app.logger.info(f"All action rules evaluated to true {log_common}")
                 EAR_ID = self.workflow.num_EARs + count
                 param_source = {
@@ -1969,6 +1969,7 @@ class WorkflowTask:
                 run_0 = {
                     "elem_iter_ID": element_iter.id_,
                     "action_idx": act_idx,
+                    "commands_idx": cmds_idx,
                     "metadata": {},
                 }
                 action_runs[(act_idx, EAR_ID)] = run_0
@@ -1982,6 +1983,7 @@ class WorkflowTask:
             self.workflow._store.add_EAR(
                 elem_iter_ID=element_iter.id_,
                 action_idx=act_idx,
+                commands_idx=run["commands_idx"],
                 data_idx=all_data_idx[(act_idx, EAR_ID_i)],
                 metadata={},
             )

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/sdk/core/task_schema.py

@@ -528,7 +528,7 @@ class TaskSchema(JSONLike):
                 f'</tr><tr><td class="action-header-cell">scope:</td>'
                 f"<td><code>{act.get_precise_scope().to_string()}</code></td></tr>"
                 f'<tr><td class="action-header-cell">environment:</td>'
-                f"<td><code>{act.get_environment().name}</code></td></tr>"
+                f"<td><code>{act.get_environment_name()}</code></td></tr>"
                 f"{inp_fg_rows}"
                 f"{out_fp_rows}"
                 f"{act_i_script_rows}"

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/sdk/persistence/base.py

@@ -310,6 +310,7 @@ class StoreEAR:
     is_pending: bool
     elem_iter_ID: int
     action_idx: int
+    commands_idx: List[int]
     data_idx: Dict[str, int]
     submission_idx: Optional[int] = None
     skip: Optional[bool] = False
@@ -336,6 +337,7 @@ class StoreEAR:
             "id_": self.id_,
             "elem_iter_ID": self.elem_iter_ID,
             "action_idx": self.action_idx,
+            "commands_idx": self.commands_idx,
             "data_idx": self.data_idx,
             "submission_idx": self.submission_idx,
             "success": self.success,
@@ -371,6 +373,7 @@ class StoreEAR:
             "is_pending": self.is_pending,
             "elem_iter_ID": self.elem_iter_ID,
             "action_idx": self.action_idx,
+            "commands_idx": self.commands_idx,
             "data_idx": self.data_idx,
             "submission_idx": self.submission_idx,
             "success": self.success,
@@ -414,6 +417,7 @@ class StoreEAR:
             is_pending=self.is_pending,
             elem_iter_ID=self.elem_iter_ID,
             action_idx=self.action_idx,
+            commands_idx=self.commands_idx,
             data_idx=self.data_idx,
             metadata=self.metadata,
             submission_idx=sub_idx,
@@ -1013,6 +1017,7 @@ class PersistentStore(ABC):
         self,
         elem_iter_ID: int,
         action_idx: int,
+        commands_idx: List[int],
         data_idx: Dict,
         metadata: Dict,
         save: bool = True,
@@ -1025,6 +1030,7 @@ class PersistentStore(ABC):
             is_pending=True,
             elem_iter_ID=elem_iter_ID,
             action_idx=action_idx,
+            commands_idx=commands_idx,
             data_idx=data_idx,
             metadata=metadata,
         )

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/sdk/persistence/zarr.py

@@ -237,6 +237,7 @@ class ZarrStoreEAR(StoreEAR):
             self.exit_code,
             self.metadata,
             self.run_hostname,
+            self.commands_idx,
         ]
         return EAR_enc
 
@@ -258,6 +259,7 @@ class ZarrStoreEAR(StoreEAR):
             "exit_code": EAR_dat[11],
             "metadata": EAR_dat[12],
             "run_hostname": EAR_dat[13],
+            "commands_idx": EAR_dat[14],
         }
         return cls(is_pending=False, **obj_dat)
 

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/tests/unit/test_action.py

@@ -618,3 +618,173 @@ def test_ActionEnvironment_env_dict(null_config):
 def test_ActionEnvironment_raises_on_missing_name(null_config):
     with pytest.raises(ActionEnvironmentMissingNameError):
         hf.ActionEnvironment(environment={"key": "value"})
+
+
+def test_rules_allow_runs_initialised(null_config, tmp_path):
+    """Test rules that do not depend on execution allow for runs to be initialised."""
+    act = hf.Action(
+        script="<<script:path/to/some/script>>",
+        rules=[hf.ActionRule(path="inputs.p1", condition={"value.less_than": 2})],
+    )
+    ts = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        actions=[act],
+    )
+    t1 = hf.Task(
+        schema=ts, sequences=[hf.ValueSequence(path="inputs.p1", values=[1.5, 2.5])]
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert wk.tasks[0].elements[1].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].actions) == 1
+    assert len(wk.tasks[0].elements[1].actions) == 0
+
+
+def test_rules_prevent_runs_initialised(null_config, tmp_path):
+    """Test rules that depend on execution prevent initialising runs."""
+    act1 = hf.Action(script="<<script:path/to/some/script>>")
+    act2 = hf.Action(
+        script="<<script:path/to/some/script>>",
+        rules=[hf.ActionRule(path="inputs.p2", condition={"value.less_than": 2})],
+    )
+    ts1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[act1],
+    )
+    ts2 = hf.TaskSchema(
+        objective="ts2",
+        inputs=[hf.SchemaInput("p2")],
+        actions=[act2],
+    )
+    t1 = hf.Task(schema=ts1, inputs={"p1": 1.2})
+    t2 = hf.Task(schema=ts2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert not wk.tasks[1].elements[0].iterations[0].EARs_initialised
+
+
+def test_command_rules_allow_runs_initialised(null_config, tmp_path):
+    """Test command rules that do not depend on execution allow for runs to be
+    initialised."""
+    act = hf.Action(
+        commands=[
+            hf.Command(
+                command='echo "p1=<<parameter:p1>>"',
+                rules=[hf.ActionRule(path="inputs.p1", condition={"value.less_than": 2})],
+            )
+        ],
+    )
+    ts = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        actions=[act],
+    )
+    t1 = hf.Task(
+        schema=ts, sequences=[hf.ValueSequence(path="inputs.p1", values=[1.5, 2.5])]
+    )
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert wk.tasks[0].elements[1].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].actions) == 1
+    assert len(wk.tasks[0].elements[1].actions) == 1
+    assert len(wk.tasks[0].elements[0].action_runs[0].commands_idx) == 1
+    assert len(wk.tasks[0].elements[1].action_runs[0].commands_idx) == 0
+
+
+def test_command_rules_prevent_runs_initialised(null_config, tmp_path):
+    """Test command rules that do depend on execution prevent runs being initialised."""
+    act1 = hf.Action(
+        commands=[
+            hf.Command(command='echo "p1=<<parameter:p1>>"', stdout="<<parameter:p2>>")
+        ]
+    )
+    act2 = hf.Action(
+        commands=[
+            hf.Command(
+                command='echo "p1=<<parameter:p2>>"',
+                rules=[hf.ActionRule(path="inputs.p2", condition={"value.less_than": 2})],
+            )
+        ],
+    )
+    ts1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[act1],
+    )
+    ts2 = hf.TaskSchema(
+        objective="ts2",
+        inputs=[hf.SchemaInput("p2")],
+        actions=[act2],
+    )
+    t1 = hf.Task(schema=ts1, inputs={"p1": 0})
+    t2 = hf.Task(schema=ts2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].action_runs[0].commands_idx) == 1
+    assert not wk.tasks[1].elements[0].iterations[0].EARs_initialised
+
+
+def test_command_rules_prevent_runs_initialised_with_valid_action_rules(
+    null_config, tmp_path
+):
+    """Test command rules that do depend on execution prevent runs being initialised, even
+    when the parent action rules can be tested and are valid."""
+    act1 = hf.Action(
+        commands=[
+            hf.Command(command='echo "p1=<<parameter:p1>>"', stdout="<<parameter:p2>>")
+        ]
+    )
+
+    # action rule is testable and valid, but command rule is not testable, so the action
+    # runs should not be initialised:
+    act2 = hf.Action(
+        commands=[
+            hf.Command(
+                command='echo "p1=<<parameter:p1>>; p2=<<parameter:p2>>"',
+                rules=[hf.ActionRule(path="inputs.p2", condition={"value.less_than": 2})],
+            )
+        ],
+        rules=[hf.ActionRule(path="inputs.p1", condition={"value.less_than": 2})],
+    )
+    ts1 = hf.TaskSchema(
+        objective="ts1",
+        inputs=[hf.SchemaInput("p1")],
+        outputs=[hf.SchemaOutput("p2")],
+        actions=[act1],
+    )
+    ts2 = hf.TaskSchema(
+        objective="ts2",
+        inputs=[hf.SchemaInput("p1"), hf.SchemaInput("p2")],
+        actions=[act2],
+    )
+    t1 = hf.Task(schema=ts1, inputs={"p1": 0})
+    t2 = hf.Task(schema=ts2)
+    wk = hf.Workflow.from_template_data(
+        template_name="test",
+        path=tmp_path,
+        tasks=[t1, t2],
+    )
+    assert wk.tasks[0].elements[0].iterations[0].EARs_initialised
+    assert len(wk.tasks[0].elements[0].action_runs[0].commands_idx) == 1
+
+    assert not wk.tasks[1].elements[0].iterations[0].EARs_initialised

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/hpcflow/tests/unit/test_persistence.py

@@ -74,6 +74,7 @@ def test_store_pending_add_EAR(tmp_path):
     EAR_ID = store.add_EAR(
         elem_iter_ID=0,
         action_idx=0,
+        commands_idx=[],
         data_idx={},
         metadata={},
     )
@@ -83,6 +84,7 @@ def test_store_pending_add_EAR(tmp_path):
             is_pending=True,
             elem_iter_ID=0,
             action_idx=0,
+            commands_idx=[],
             data_idx={},
             metadata={},
         )
@@ -186,7 +188,7 @@ def test_get_task_elements_single_element_iter_EAR_pending(tmp_path):
     store = JSONPersistentStore.make_test_store_from_spec(
         [{"elements": [{"iterations": [{}]}]}], dir=tmp_path
     )
-    store.add_EAR(elem_iter_ID=0, action_idx=0, data_idx={}, metadata={})
+    store.add_EAR(elem_iter_ID=0, action_idx=0, commands_idx=[], data_idx={}, metadata={})
     assert store.get_task_elements(0, slice(0, None)) == [
         {
             "id": 0,
@@ -209,6 +211,7 @@ def test_get_task_elements_single_element_iter_EAR_pending(tmp_path):
                                 "is_pending": True,
                                 "elem_iter_ID": 0,
                                 "action_idx": 0,
+                                "commands_idx": [],
                                 "data_idx": {},
                                 "metadata": {},
                             }

{hpcflow_new2-0.2.0a163 → hpcflow_new2-0.2.0a164}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "hpcflow-new2"
-version = "0.2.0a163"
+version = "0.2.0a164"
 
 description = "Computational workflow management"
 authors = ["aplowman <adam.plowman@manchester.ac.uk>"]
@@ -76,7 +76,7 @@ hook-dirs = "hpcflow.__pyinstaller:get_hook_dirs"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0a163"
+version = "0.2.0a164"
 tag_format = "v$version"
 version_files = [ 
     "pyproject.toml:version",

hpcflow_new2-0.2.0a163/hpcflow/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0a163"