hpcflow-new2 0.2.0a107__tar.gz → 0.2.0a109__tar.gz

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a107
+Version: 0.2.0a109
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

hpcflow_new2-0.2.0a109/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a109"

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/app.py

@@ -1108,6 +1108,7 @@ class BaseApp(metaclass=Singleton):
         JS_parallelism: Optional[bool] = None,
         wait: Optional[bool] = False,
         add_to_known: Optional[bool] = True,
+        return_idx: Optional[bool] = False,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1150,6 +1151,9 @@ class BaseApp(metaclass=Singleton):
         add_to_known
             If True, add the new submission to the known-submissions file, which is
             used by the `show` command to monitor current and recent submissions.
+        return_idx
+            If True, return a dict representing the jobscript indices submitted for each
+            submission.
         """
 
         self.API_logger.info("make_and_submit_workflow called")
@@ -1169,6 +1173,7 @@ class BaseApp(metaclass=Singleton):
             JS_parallelism=JS_parallelism,
             wait=wait,
             add_to_known=add_to_known,
+            return_idx=return_idx,
         )
 
     def _make_demo_workflow(
@@ -1240,6 +1245,7 @@ class BaseApp(metaclass=Singleton):
         JS_parallelism: Optional[bool] = None,
         wait: Optional[bool] = False,
         add_to_known: Optional[bool] = True,
+        return_idx: Optional[bool] = False,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1279,6 +1285,9 @@ class BaseApp(metaclass=Singleton):
         add_to_known
             If True, add the new submission to the known-submissions file, which is
             used by the `show` command to monitor current and recent submissions.
+        return_idx
+            If True, return a dict representing the jobscript indices submitted for each
+            submission.
         """
 
         self.API_logger.info("make_and_submit_demo_workflow called")
@@ -1297,6 +1306,7 @@ class BaseApp(metaclass=Singleton):
             JS_parallelism=JS_parallelism,
             wait=wait,
             add_to_known=add_to_known,
+            return_idx=return_idx,
         )
 
     def _submit_workflow(
@@ -1304,6 +1314,7 @@ class BaseApp(metaclass=Singleton):
         workflow_path: PathLike,
         JS_parallelism: Optional[bool] = None,
         wait: Optional[bool] = False,
+        return_idx: Optional[bool] = False,
     ) -> Dict[int, int]:
         """Submit an existing {app_name} workflow.
 
@@ -1319,7 +1330,7 @@ class BaseApp(metaclass=Singleton):
 
         self.API_logger.info("submit_workflow called")
         wk = self.Workflow(workflow_path)
-        return wk.submit(JS_parallelism=JS_parallelism, wait=wait)
+        return wk.submit(JS_parallelism=JS_parallelism, wait=wait, return_idx=return_idx)
 
     def _run_hpcflow_tests(self, *args):
         """Run hpcflow test suite. This function is only available from derived apps."""

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/cli.py

@@ -22,6 +22,7 @@ from hpcflow.sdk.cli_common import (
     js_parallelism_option,
     wait_option,
     add_to_known_opt,
+    print_idx_opt,
 )
 from hpcflow.sdk.helper.cli import get_helper_CLI
 from hpcflow.sdk.submission.shells import ALL_SHELLS
@@ -104,6 +105,7 @@ def _make_API_CLI(app):
     @js_parallelism_option
     @wait_option
     @add_to_known_opt
+    @print_idx_opt
     def make_and_submit_workflow(
         template_file_or_str,
         string,
@@ -117,6 +119,7 @@ def _make_API_CLI(app):
         js_parallelism=None,
         wait=False,
         add_to_known=True,
+        print_idx=False,
     ):
         """Generate and submit a new {app_name} workflow.
 
@@ -124,7 +127,7 @@ def _make_API_CLI(app):
         format, or a YAML/JSON string.
 
         """
-        app.make_and_submit_workflow(
+        out = app.make_and_submit_workflow(
             template_file_or_str=template_file_or_str,
             is_string=string,
             template_format=format,
@@ -137,7 +140,10 @@ def _make_API_CLI(app):
             JS_parallelism=js_parallelism,
             wait=wait,
             add_to_known=add_to_known,
+            return_idx=print_idx,
         )
+        if print_idx:
+            click.echo(out)
 
     @click.command(context_settings={"ignore_unknown_options": True})
     @click.argument("py_test_args", nargs=-1, type=click.UNPROCESSED)
@@ -288,14 +294,20 @@ def _make_workflow_CLI(app):
     @js_parallelism_option
     @wait_option
     @add_to_known_opt
+    @print_idx_opt
     @click.pass_context
-    def submit_workflow(ctx, js_parallelism=None, wait=False, add_to_known=True):
+    def submit_workflow(
+        ctx, js_parallelism=None, wait=False, add_to_known=True, print_idx=False
+    ):
         """Submit the workflow."""
-        ctx.obj["workflow"].submit(
+        out = ctx.obj["workflow"].submit(
             JS_parallelism=js_parallelism,
             wait=wait,
             add_to_known=add_to_known,
+            return_idx=print_idx,
         )
+        if print_idx:
+            click.echo(out)
 
     @workflow.command(name="wait")
     @click.option(

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/cli_common.py

@@ -79,3 +79,9 @@ add_to_known_opt = click.option(
     default=True,
     help="If True, add this submission to the known-submissions file.",
 )
+print_idx_opt = click.option(
+    "--print-idx",
+    help="If True, print the submitted jobscript indices for each submission index.",
+    is_flag=True,
+    default=False,
+)

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/config/cli.py

@@ -289,7 +289,32 @@ def get_config_CLI(app):
             defaults = json.loads(defaults)
         else:
             defaults = {}
-        ctx.obj["config"].add_scheduler(name, defaults=defaults)
+        ctx.obj["config"].add_scheduler(name, **defaults)
+        ctx.obj["config"].save()
+
+    @config.command()
+    @click.argument("name")
+    @click.option("--defaults")
+    @click.pass_context
+    @CLI_exception_wrapper_gen(ConfigError)
+    def add_shell(ctx, name, defaults):
+        if defaults:
+            defaults = json.loads(defaults)
+        else:
+            defaults = {}
+        ctx.obj["config"].add_shell(name, **defaults)
+        ctx.obj["config"].save()
+
+    @config.command()
+    @click.option("--defaults")
+    @click.pass_context
+    @CLI_exception_wrapper_gen(ConfigError)
+    def add_shell_wsl(ctx, defaults):
+        if defaults:
+            defaults = json.loads(defaults)
+        else:
+            defaults = {}
+        ctx.obj["config"].add_shell_WSL(**defaults)
         ctx.obj["config"].save()
 
     @config.command()

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/config/config.py

@@ -737,11 +737,24 @@ class Config:
         self._logger.info(f"Resetting config file to defaults.")
         self._app.reset_config()
 
-    def add_scheduler(self, scheduler, **kwargs):
+    def add_scheduler(self, scheduler, **defaults):
         if scheduler in self.get("schedulers"):
             print(f"Scheduler {scheduler!r} already exists.")
             return
-        self.update(f"schedulers.{scheduler}", kwargs)
+        self.update(f"schedulers.{scheduler}.defaults", defaults)
+
+    def add_shell(self, shell, **defaults):
+        if shell in self.get("shells"):
+            return
+        if shell.lower() == "wsl":
+            # check direct_posix scheduler is added:
+            self.add_scheduler("direct_posix")
+        self.update(f"shells.{shell}.defaults", defaults)
+
+    def add_shell_WSL(self, **defaults):
+        if "WSL_executable" not in defaults:
+            defaults["WSL_executable"] = "wsl.exe"
+        self.add_shell("wsl", **defaults)
 
     def import_from_file(self, file_path, rename=True, make_new=False):
         """Import config items from a (remote or local) YAML file. Existing config items

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/core/actions.py

@@ -513,16 +513,14 @@ class ElementActionRun:
             outputs[out_typ] = self.get(f"outputs.{out_typ}")
         return outputs
 
-    def compose_source(self) -> str:
+    def compose_source(self, snip_path: Path) -> str:
         """Generate the file contents of this source."""
 
-        script_name = self.action.get_script_name(self.action.script)
-        script_key = self.action.get_app_data_script_path(self.action.script)
-        script_path = self.app.scripts.get(script_key)
-        with script_path.open("rt") as fp:
+        script_name = snip_path.name
+        with snip_path.open("rt") as fp:
             script_str = fp.read()
 
-        is_python = script_path.suffix == ".py"
+        is_python = snip_path.suffix == ".py"
         if is_python:
             py_imports = dedent(
                 """\
@@ -639,10 +637,12 @@ class ElementActionRun:
 
         # write the script if it is specified as a app data script, otherwise we assume
         # the script already exists in the working directory:
-        if self.action.script and self.action.is_app_data_script(self.action.script):
-            script_name = self.action.get_script_name(self.action.script)
+        snip_path = self.action.get_snippet_script_path(self.action.script)
+        if snip_path:
+            script_name = snip_path.name
+            source_str = self.compose_source(snip_path)
             with Path(script_name).open("wt", newline="\n") as fp:
-                fp.write(self.compose_source())
+                fp.write(source_str)
 
     def _param_dump_JSON(self, dump_path: Path):
         with dump_path.open("wt") as fp:
@@ -1358,15 +1358,16 @@ class Action(JSONLike):
         )
 
     @staticmethod
-    def is_app_data_script(script: str) -> bool:
+    def is_snippet_script(script: str) -> bool:
+        """Returns True if the provided script string represents a script snippets that is
+        to be modified before execution (e.g. to receive and provide parameter data)."""
         return script.startswith("<<script:")
 
     @classmethod
     def get_script_name(cls, script: str) -> str:
         """Return the script name."""
-        if cls.is_app_data_script(script):
-            # an app data script:
-            pattern = r"\<\<script:(?:.*\/)*(.*:?)\>\>"
+        if cls.is_snippet_script(script):
+            pattern = r"\<\<script:(?:.*(?:\/|\\))*(.*)\>\>"
             match_obj = re.match(pattern, script)
             return match_obj.group(1)
         else:
@@ -1374,8 +1375,8 @@ class Action(JSONLike):
             return script
 
     @classmethod
-    def get_app_data_script_path(cls, script) -> str:
-        if not cls.is_app_data_script(script):
+    def get_snippet_script_str(cls, script) -> str:
+        if not cls.is_snippet_script(script):
             raise ValueError(
                 f"Must be an app-data script name (e.g. "
                 f"<<script:path/to/app/data/script.py>>), but received {script}"
@@ -1384,6 +1385,17 @@ class Action(JSONLike):
         match_obj = re.match(pattern, script)
         return match_obj.group(1)
 
+    @classmethod
+    def get_snippet_script_path(cls, script_path) -> Path:
+        if not cls.is_snippet_script(script_path):
+            return False
+
+        path = cls.get_snippet_script_str(script_path)
+        if path in cls.app.scripts:
+            path = cls.app.scripts.get(path)
+
+        return Path(path)
+
     @staticmethod
     def get_param_dump_file_stem(js_idx: int, js_act_idx: int):
         return f"js_{js_idx}_act_{js_act_idx}_inputs"

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/core/command_files.py

@@ -134,12 +134,9 @@ class InputFileGenerator(JSONLike):
     def compose_source(self, action) -> str:
         """Generate the file contents of this input file generator source."""
 
-        script_name = self.script
-        script_key = action.get_app_data_script_path(self.script)
-        script_path = self.app.scripts.get(script_key)
-        script_main_func = Path(script_name).stem
-
-        with script_path.open("rt") as fp:
+        snip_path = action.get_snippet_script_path(self.script)
+        script_main_func = snip_path.stem
+        with snip_path.open("rt") as fp:
             script_str = fp.read()
 
         main_block = dedent(
@@ -261,12 +258,9 @@ class OutputFileParser(JSONLike):
     def compose_source(self, action) -> str:
         """Generate the file contents of this output file parser source."""
 
-        script_name = self.script
-        script_key = action.get_app_data_script_path(self.script)
-        script_path = self.app.scripts.get(script_key)
-        script_main_func = Path(script_name).stem
-
-        with script_path.open("rt") as fp:
+        snip_path = action.get_snippet_script_path(self.script)
+        script_main_func = snip_path.stem
+        with snip_path.open("rt") as fp:
             script_str = fp.read()
 
         main_block = dedent(

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/core/task.py

@@ -222,6 +222,17 @@ class ElementSet(JSONLike):
         return tuple(self._element_local_idx_range)
 
     def _validate(self):
+
+        # support inputs passed as a dict:
+        _inputs = []
+        try:
+            for k, v in self.inputs.items():
+                _inputs.append(self.app.InputValue(parameter=k, value=v))
+        except AttributeError:
+            pass
+        else:
+            self.inputs = _inputs
+
         inp_paths = [i.normalised_inputs_path for i in self.inputs]
         dup_inp_paths = get_duplicate_items(inp_paths)
         if dup_inp_paths:

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/core/task_schema.py

@@ -217,6 +217,27 @@ class TaskSchema(JSONLike):
                 tab_cmds_i = Table(show_header=False, box=None)
                 tab_cmds_i.add_column(justify="right")
                 tab_cmds_i.add_column()
+                if act.rules:
+                    seen_rules = []  # bug: some rules seem to be repeated
+                    for act_rule_j in act.rules:
+                        if act_rule_j.rule in seen_rules:
+                            continue
+                        else:
+                            seen_rules.append(act_rule_j.rule)
+                        r_path = ""
+                        if act_rule_j.rule.check_missing:
+                            r_cond = f"check missing: {act_rule_j.rule.check_missing}"
+                        elif act_rule_j.rule.check_exists:
+                            r_cond = f"check exists: {act_rule_j.rule.check_exists}"
+                        elif act_rule_j.rule.condition:
+                            r_path = f"{act_rule_j.rule.path}: "
+                            r_cond = str(act_rule_j.rule.condition.to_json_like())
+                        else:
+                            continue
+                        tab_cmds_i.add_row(
+                            "[italic]rule:[/italic]",
+                            escape(f"{r_path}{r_cond}"),
+                        )
                 tab_cmds_i.add_row(
                     "[italic]scope:[/italic]",
                     escape(act.get_precise_scope().to_string()),

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/core/workflow.py

@@ -127,6 +127,7 @@ class WorkflowTemplate(JSONLike):
     loops: Optional[List[app.Loop]] = field(default_factory=lambda: [])
     workflow: Optional[app.Workflow] = None
     resources: Optional[Dict[str, Dict]] = None
+    source_file: Optional[str] = field(default=None, compare=False)
 
     def __post_init__(self):
         self.resources = self.app.ResourceList.normalise(self.resources)
@@ -230,6 +231,7 @@ class WorkflowTemplate(JSONLike):
         cls.app.logger.debug("parsing workflow template from a YAML file")
         data = read_YAML_file(path)
         cls._check_name(data, path)
+        data["source_file"] = str(path)
         return cls._from_data(data)
 
     @classmethod
@@ -257,6 +259,7 @@ class WorkflowTemplate(JSONLike):
         cls.app.logger.debug("parsing workflow template from a JSON file")
         data = read_JSON_file(path)
         cls._check_name(data, path)
+        data["source_file"] = str(path)
         return cls._from_data(data)
 
     @classmethod
@@ -1359,6 +1362,11 @@ class Workflow:
         # actually make template inputs/resources persistent, now the workflow exists:
         wk_dummy.make_persistent(wk)
 
+        if template.source_file:
+            wk.artifacts_path.mkdir(exist_ok=False)
+            src = Path(template.source_file)
+            wk.artifacts_path.joinpath(src.name).write_text(src.read_text())
+
         return wk
 
     def to_zip(self, log=None) -> str:
@@ -1805,6 +1813,7 @@ class Workflow:
         print_stdout: Optional[bool] = False,
         wait: Optional[bool] = False,
         add_to_known: Optional[bool] = True,
+        return_idx: Optional[bool] = False,
     ) -> Dict[int, int]:
         """Submit the workflow for execution.
 
@@ -1824,6 +1833,9 @@ class Workflow:
         add_to_known
             If True, add the submitted submissions to the known-submissions file, which is
             used by the `show` command to monitor current and recent submissions.
+        return_idx
+            If True, return a dict representing the jobscript indices submitted for each
+            submission.
         """
 
         console = rich.console.Console()
@@ -1858,7 +1870,8 @@ class Workflow:
         if wait:
             self.wait(submitted_js)
 
-        return submitted_js
+        if return_idx:
+            return submitted_js
 
     def wait(self, sub_js: Optional[Dict] = None):
         """Wait for the completion of specified/all submitted jobscripts."""

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/sdk/demo/cli.py

@@ -14,6 +14,7 @@ from hpcflow.sdk.cli_common import (
     js_parallelism_option,
     wait_option,
     add_to_known_opt,
+    print_idx_opt,
 )
 
 
@@ -130,6 +131,7 @@ def get_demo_workflow_CLI(app):
     @js_parallelism_option
     @wait_option
     @add_to_known_opt
+    @print_idx_opt
     def make_and_submit_demo_workflow(
         workflow_name,
         format,
@@ -142,8 +144,9 @@ def get_demo_workflow_CLI(app):
         js_parallelism=None,
         wait=False,
         add_to_known=True,
+        print_idx=False,
     ):
-        app.make_and_submit_demo_workflow(
+        out = app.make_and_submit_demo_workflow(
             workflow_name=workflow_name,
             template_format=format,
             path=path,
@@ -155,7 +158,10 @@ def get_demo_workflow_CLI(app):
             JS_parallelism=js_parallelism,
             wait=wait,
             add_to_known=add_to_known,
+            return_idx=print_idx,
         )
+        if print_idx:
+            click.echo(out)
 
     @demo_workflow.command("copy")
     @click.argument("workflow_name")

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/tests/unit/test_action.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import pytest
 
 from hpcflow.app import app as hf
@@ -215,3 +216,39 @@ def test_get_command_input_types_label_sub_parameters_false_no_sub_param():
 def test_get_command_input_types_label_sub_parameters_false_with_sub_parameter():
     act = hf.Action(commands=[hf.Command("Write-Output (<<parameter:p1[one].a>> + 100)")])
     assert act.get_command_input_types(sub_parameters=False) == ("p1[one]",)
+
+
+def test_get_script_name(null_config):
+    expected = {
+        "<<script:/software/hello.py>>": "hello.py",
+        "<<script:software/hello.py>>": "hello.py",
+        r"<<script:C:\long\path\to\script.py>>": "script.py",
+        "/path/to/script.py": "/path/to/script.py",
+    }
+    for k, v in expected.items():
+        assert hf.Action.get_script_name(k) == v
+
+
+def test_is_snippet_script(null_config):
+    expected = {
+        "<<script:/software/hello.py>>": True,
+        "<<script:software/hello.py>>": True,
+        r"<<script:C:\long\path\to\script.py>>": True,
+        "/path/to/script.py": False,
+    }
+    for k, v in expected.items():
+        assert hf.Action.is_snippet_script(k) == v
+
+
+def test_get_snippet_script_path(null_config):
+    expected = {
+        "<<script:/software/hello.py>>": Path("/software/hello.py"),
+        "<<script:software/hello.py>>": Path("software/hello.py"),
+        r"<<script:C:\long\path\to\script.py>>": Path(r"C:\long\path\to\script.py"),
+    }
+    for k, v in expected.items():
+        assert hf.Action.get_snippet_script_path(k) == v
+
+
+def test_get_snippet_script_path_False(null_config):
+    assert not hf.Action.get_snippet_script_path("/path/to/script.py")

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/hpcflow/tests/unit/test_task.py

@@ -2083,3 +2083,10 @@ def test_path_to_PV_classes_resources_path_ignored(path_to_PV_classes_workflow):
     assert path_to_PV_classes_workflow.tasks.t1._paths_to_PV_classes(
         paths_1
     ) == path_to_PV_classes_workflow.tasks.t1._paths_to_PV_classes(paths_2)
+
+
+def test_input_values_specified_by_dict(null_config):
+    ts = hf.TaskSchema(objective="t1", inputs=[hf.SchemaInput("p1")])
+    t1 = hf.Task(schema=ts, inputs=[hf.InputValue(parameter="p1", value=101)])
+    t2 = hf.Task(schema=ts, inputs={"p1": 101})
+    assert t1 == t2

{hpcflow_new2-0.2.0a107 → hpcflow_new2-0.2.0a109}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "hpcflow-new2"
-version = "0.2.0a107"
+version = "0.2.0a109"
 
 description = "Computational workflow management"
 authors = ["aplowman <adam.plowman@manchester.ac.uk>"]
@@ -68,7 +68,7 @@ hook-dirs = "hpcflow.__pyinstaller:get_hook_dirs"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0a107"
+version = "0.2.0a109"
 tag_format = "v$version"
 version_files = [ 
     "pyproject.toml:version",

hpcflow_new2-0.2.0a107/hpcflow/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0a107"