hpcflow-new2 0.2.0a159__py3-none-any.whl → 0.2.0a160__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a159"
+__version__ = "0.2.0a160"

hpcflow/sdk/__init__.py

@@ -1,4 +1,5 @@
 """Sub-package to define an extensible hpcflow application."""
+
 import logging
 import os
 import sys
@@ -90,6 +91,7 @@ sdk_classes = {
     "SlurmPosix": "hpcflow.sdk.submission.schedulers.slurm",
     "SGEPosix": "hpcflow.sdk.submission.schedulers.sge",
     "OutputLabel": "hpcflow.sdk.core.task",
+    "RunDirAppFiles": "hpcflow.sdk.core.run_dir_files",
 }
 
 # these are defined as `BaseApp` methods with an underscore prefix:

hpcflow/sdk/app.py

@@ -20,6 +20,7 @@ import warnings
 import zipfile
 from platformdirs import user_cache_path, user_data_dir
 from reretry import retry
+import rich
 from rich.console import Console, Group
 from rich.syntax import Syntax
 from rich.table import Table, box
@@ -965,6 +966,7 @@ class BaseApp(metaclass=Singleton):
         }
         return item
 
+    @TimeIt.decorator
     def read_known_submissions_file(self) -> List[Dict]:
         """Retrieve existing workflows that *might* be running."""
         known = []
@@ -1024,6 +1026,7 @@ class BaseApp(metaclass=Singleton):
 
         return next_id
 
+    @TimeIt.decorator
     def set_inactive_in_known_subs_file(self, inactive_IDs: List[int]):
         """Set workflows in the known-submissions file to the non-running state.
 
@@ -1141,6 +1144,7 @@ class BaseApp(metaclass=Singleton):
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
         variables: Optional[Dict[str, str]] = None,
+        status: Optional[bool] = True,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a file or string containing a workflow
         template parametrisation.
@@ -1177,10 +1181,17 @@ class BaseApp(metaclass=Singleton):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables
             String variables to substitute in `template_file_or_str`.
+        status
+            If True, display a live status to track workflow creation progress.
         """
 
         self.API_logger.info("make_workflow called")
 
+        if status:
+            console = rich.console.Console()
+            status = console.status("Making persistent workflow...")
+            status.start()
+
         common = {
             "path": path,
             "name": name,
@@ -1190,6 +1201,7 @@ class BaseApp(metaclass=Singleton):
             "ts_name_fmt": ts_name_fmt,
             "store_kwargs": store_kwargs,
             "variables": variables,
+            "status": status,
         }
 
         if not is_string:
@@ -1200,10 +1212,24 @@ class BaseApp(metaclass=Singleton):
             )
 
         elif template_format == "json":
-            wk = self.Workflow.from_JSON_string(JSON_str=template_file_or_str, **common)
+            try:
+                wk = self.Workflow.from_JSON_string(
+                    JSON_str=template_file_or_str, **common
+                )
+            except Exception:
+                if status:
+                    status.stop()
+                raise
 
         elif template_format == "yaml":
-            wk = self.Workflow.from_YAML_string(YAML_str=template_file_or_str, **common)
+            try:
+                wk = self.Workflow.from_YAML_string(
+                    YAML_str=template_file_or_str, **common
+                )
+            except Exception:
+                if status:
+                    status.stop()
+                raise
 
         elif not template_format:
             raise ValueError(
@@ -1216,6 +1242,10 @@ class BaseApp(metaclass=Singleton):
                 f"Template format {template_format!r} not understood. Available template "
                 f"formats are {ALL_TEMPLATE_FORMATS!r}."
             )
+
+        if status:
+            status.stop()
+
         return wk
 
     def _make_and_submit_workflow(
@@ -1236,6 +1266,8 @@ class BaseApp(metaclass=Singleton):
         add_to_known: Optional[bool] = True,
         return_idx: Optional[bool] = False,
         tasks: Optional[List[int]] = None,
+        cancel: Optional[bool] = False,
+        status: Optional[bool] = True,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1288,6 +1320,11 @@ class BaseApp(metaclass=Singleton):
         tasks
             List of task indices to include in this submission. By default all tasks are
             included.
+        cancel
+            Immediately cancel the submission. Useful for testing and benchmarking.
+        status
+            If True, display a live status to track workflow creation and submission
+            progress.
         """
 
         self.API_logger.info("make_and_submit_workflow called")
@@ -1304,6 +1341,7 @@ class BaseApp(metaclass=Singleton):
             ts_name_fmt=ts_name_fmt,
             store_kwargs=store_kwargs,
             variables=variables,
+            status=status,
         )
         return wk.submit(
             JS_parallelism=JS_parallelism,
@@ -1311,6 +1349,8 @@ class BaseApp(metaclass=Singleton):
             add_to_known=add_to_known,
             return_idx=return_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
 
     def _make_demo_workflow(
@@ -1325,6 +1365,7 @@ class BaseApp(metaclass=Singleton):
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
         variables: Optional[Dict[str, str]] = None,
+        status: Optional[bool] = True,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a builtin demo workflow template.
 
@@ -1358,10 +1399,17 @@ class BaseApp(metaclass=Singleton):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables
             String variables to substitute in the demo workflow template file.
+        status
+            If True, display a live status to track workflow creation progress.
         """
 
         self.API_logger.info("make_demo_workflow called")
 
+        if status:
+            console = rich.console.Console()
+            status = console.status("Making persistent workflow...")
+            status.start()
+
         with self.get_demo_workflow_template_file(workflow_name) as template_path:
             wk = self.Workflow.from_file(
                 template_path=template_path,
@@ -1374,7 +1422,10 @@ class BaseApp(metaclass=Singleton):
                 ts_name_fmt=ts_name_fmt,
                 store_kwargs=store_kwargs,
                 variables=variables,
+                status=status,
             )
+        if status:
+            status.stop()
         return wk
 
     def _make_and_submit_demo_workflow(
@@ -1394,6 +1445,8 @@ class BaseApp(metaclass=Singleton):
         add_to_known: Optional[bool] = True,
         return_idx: Optional[bool] = False,
         tasks: Optional[List[int]] = None,
+        cancel: Optional[bool] = False,
+        status: Optional[bool] = True,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1443,6 +1496,10 @@ class BaseApp(metaclass=Singleton):
         tasks
             List of task indices to include in this submission. By default all tasks are
             included.
+        cancel
+            Immediately cancel the submission. Useful for testing and benchmarking.
+        status
+            If True, display a live status to track submission progress.
         """
 
         self.API_logger.info("make_and_submit_demo_workflow called")
@@ -1465,6 +1522,8 @@ class BaseApp(metaclass=Singleton):
             add_to_known=add_to_known,
             return_idx=return_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
 
     def _submit_workflow(
@@ -1556,8 +1615,13 @@ class BaseApp(metaclass=Singleton):
         )
         return shell.get_version_info(exclude_os)
 
+    @TimeIt.decorator
     def _get_known_submissions(
-        self, max_recent: int = 3, no_update: bool = False, as_json: bool = False
+        self,
+        max_recent: int = 3,
+        no_update: bool = False,
+        as_json: bool = False,
+        status: Optional[Any] = None,
     ):
         """Retrieve information about active and recently inactive finished {app_name}
         workflows.
@@ -1582,6 +1646,8 @@ class BaseApp(metaclass=Singleton):
         inactive_IDs = []
 
         try:
+            if status:
+                status.update("Reading known submissions file...")
             known_subs = self.read_known_submissions_file()
         except FileNotFoundError:
             known_subs = []
@@ -1614,6 +1680,8 @@ class BaseApp(metaclass=Singleton):
                 out_item["deleted"] = not path_exists
                 if path_exists:
                     try:
+                        if status:
+                            status.update(f"Inspecting workflow {file_dat_i['path']!r}.")
                         wk_i = self.Workflow(file_dat_i["path"])
                     except Exception:
                         wk_i = None
@@ -1640,15 +1708,22 @@ class BaseApp(metaclass=Singleton):
                     out_item["deleted"] = True
 
                 else:
-                    sub = wk_i.submissions[file_dat_i["sub_idx"]]
-
-                    all_jobscripts = sub._submission_parts[submit_time_str]
-                    out_item.update(
-                        {
-                            "jobscripts": all_jobscripts,
-                            "submission": sub,
-                        }
-                    )
+                    if status:
+                        status.update(
+                            f"Reading workflow {file_dat_i['path']!r} submission info..."
+                        )
+                    with wk_i._store.cache_ctx():
+                        sub = wk_i.submissions[file_dat_i["sub_idx"]]
+
+                        all_jobscripts = sub._submission_parts[submit_time_str]
+                        out_item.update(
+                            {
+                                "jobscripts": all_jobscripts,
+                                "submission": sub,
+                                "sub_start_time": sub.start_time,
+                                "sub_end_time": sub.end_time,
+                            }
+                        )
                     if file_dat_i["is_active"]:
                         # check it really is active:
                         run_key = (file_dat_i["path"], file_dat_i["sub_idx"])
@@ -1694,9 +1769,7 @@ class BaseApp(metaclass=Singleton):
         out_access = sorted(
             out_access,
             key=lambda i: (
-                i["submission"].end_time
-                or i["submission"].start_time
-                or i["submit_time_obj"]
+                i["sub_end_time"] or i["sub_start_time"] or i["submit_time_obj"]
             ),
             reverse=True,
         )
@@ -1826,6 +1899,7 @@ class BaseApp(metaclass=Singleton):
             run_dat = self._get_known_submissions(
                 max_recent=max_recent,
                 no_update=no_update,
+                status=status,
             )
         except Exception:
             status.stop()

hpcflow/sdk/cli.py

@@ -25,6 +25,9 @@ from hpcflow.sdk.cli_common import (
     add_to_known_opt,
     print_idx_opt,
     tasks_opt,
+    cancel_opt,
+    submit_status_opt,
+    make_status_opt,
     zip_path_opt,
     zip_overwrite_opt,
     zip_log_opt,
@@ -71,6 +74,7 @@ def _make_API_CLI(app):
     @ts_fmt_option
     @ts_name_fmt_option
     @variables_option
+    @make_status_opt
     def make_workflow(
         template_file_or_str,
         string,
@@ -82,6 +86,7 @@ def _make_API_CLI(app):
         ts_fmt=None,
         ts_name_fmt=None,
         variables=None,
+        status=True,
     ):
         """Generate a new {app_name} workflow.
 
@@ -100,6 +105,7 @@ def _make_API_CLI(app):
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
             variables=dict(variables),
+            status=status,
         )
         click.echo(wk.path)
 
@@ -119,6 +125,8 @@ def _make_API_CLI(app):
     @add_to_known_opt
     @print_idx_opt
     @tasks_opt
+    @cancel_opt
+    @submit_status_opt
     def make_and_submit_workflow(
         template_file_or_str,
         string,
@@ -135,6 +143,8 @@ def _make_API_CLI(app):
         add_to_known=True,
         print_idx=False,
         tasks=None,
+        cancel=False,
+        status=True,
     ):
         """Generate and submit a new {app_name} workflow.
 
@@ -159,6 +169,8 @@ def _make_API_CLI(app):
             add_to_known=add_to_known,
             return_idx=print_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
         if print_idx:
             click.echo(out)
@@ -320,6 +332,8 @@ def _make_workflow_CLI(app):
     @add_to_known_opt
     @print_idx_opt
     @tasks_opt
+    @cancel_opt
+    @submit_status_opt
     @click.pass_context
     def submit_workflow(
         ctx,
@@ -328,6 +342,8 @@ def _make_workflow_CLI(app):
         add_to_known=True,
         print_idx=False,
         tasks=None,
+        cancel=False,
+        status=True,
     ):
         """Submit the workflow."""
         out = ctx.obj["workflow"].submit(
@@ -336,6 +352,8 @@ def _make_workflow_CLI(app):
             add_to_known=add_to_known,
             return_idx=print_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
         if print_idx:
             click.echo(out)

hpcflow/sdk/cli_common.py

@@ -109,6 +109,22 @@ tasks_opt = click.option(
     ),
     callback=sub_tasks_callback,
 )
+cancel_opt = click.option(
+    "--cancel",
+    help="Immediately cancel the submission. Useful for testing and benchmarking.",
+    is_flag=True,
+    default=False,
+)
+submit_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track submission progress.",
+    default=True,
+)
+make_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track workflow creation progress.",
+    default=True,
+)
 zip_path_opt = click.option(
     "--path",
     default=".",

hpcflow/sdk/core/actions.py

@@ -29,6 +29,7 @@ from hpcflow.sdk.core.utils import (
     swap_nested_dict_keys,
 )
 from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.core.run_dir_files import RunDirAppFiles
 
 
 ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
@@ -251,13 +252,16 @@ class ElementActionRun:
     @property
     def snapshot_start(self):
         if self._ss_start_obj is None and self._snapshot_start:
-            self._ss_start_obj = JSONLikeDirSnapShot(**self._snapshot_start)
+            self._ss_start_obj = JSONLikeDirSnapShot(
+                root_path=".",
+                **self._snapshot_start,
+            )
         return self._ss_start_obj
 
     @property
     def snapshot_end(self):
         if self._ss_end_obj is None and self._snapshot_end:
-            self._ss_end_obj = JSONLikeDirSnapShot(**self._snapshot_end)
+            self._ss_end_obj = JSONLikeDirSnapShot(root_path=".", **self._snapshot_end)
         return self._ss_end_obj
 
     @property
@@ -331,6 +335,7 @@ class ElementActionRun:
             run_idx=self.index,
         )
 
+    @TimeIt.decorator
     def get_parameter_sources(
         self,
         path: str = None,
@@ -363,6 +368,7 @@ class ElementActionRun:
             raise_on_unset=raise_on_unset,
         )
 
+    @TimeIt.decorator
     def get_EAR_dependencies(self, as_objects=False):
         """Get EARs that this EAR depends on."""
 
@@ -434,6 +440,7 @@ class ElementActionRun:
         return self._outputs
 
     @property
+    @TimeIt.decorator
     def resources(self):
         if not self._resources:
             self._resources = self.app.ElementResources(**self.get_resources())
@@ -451,6 +458,7 @@ class ElementActionRun:
             self._output_files = self.app.ElementOutputFiles(element_action_run=self)
         return self._output_files
 
+    @TimeIt.decorator
     def get_resources(self):
         """Resolve specific resources for this EAR, considering all applicable scopes and
         template-level resources."""
@@ -1457,11 +1465,11 @@ class Action(JSONLike):
 
     @staticmethod
     def get_param_dump_file_stem(js_idx: int, js_act_idx: int):
-        return f"js_{js_idx}_act_{js_act_idx}_inputs"
+        return RunDirAppFiles.get_run_param_dump_file_prefix(js_idx, js_act_idx)
 
     @staticmethod
     def get_param_load_file_stem(js_idx: int, js_act_idx: int):
-        return f"js_{js_idx}_act_{js_act_idx}_outputs"
+        return RunDirAppFiles.get_run_param_load_file_prefix(js_idx, js_act_idx)
 
     def get_param_dump_file_path_JSON(self, js_idx: int, js_act_idx: int):
         return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".json")
@@ -1945,7 +1953,7 @@ class Action(JSONLike):
                 """\
                     import {app_module} as app
                     app.load_config(
-                        log_file_path=Path("{app_package_name}.log").resolve(),
+                        log_file_path=Path("{run_log_file}").resolve(),
                         config_dir=r"{cfg_dir}",
                         config_key=r"{cfg_invoc_key}",
                     )
@@ -1954,7 +1962,7 @@ class Action(JSONLike):
                     EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
                 """
             ).format(
-                app_package_name=self.app.package_name,
+                run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
                 app_module=self.app.module,
                 cfg_dir=self.app.config.config_directory,
                 cfg_invoc_key=self.app.config.config_key,

hpcflow/sdk/core/command_files.py

@@ -159,7 +159,7 @@ class InputFileGenerator(JSONLike):
                 from pathlib import Path
                 import {app_module} as app
                 app.load_config(
-                    log_file_path=Path("{app_package_name}.log").resolve(),
+                    log_file_path=Path("{run_log_file}").resolve(),
                     config_dir=r"{cfg_dir}",
                     config_key=r"{cfg_invoc_key}",
                 )
@@ -171,7 +171,7 @@ class InputFileGenerator(JSONLike):
         """
         )
         main_block = main_block.format(
-            app_package_name=self.app.package_name,
+            run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
             app_module=self.app.module,
             cfg_dir=self.app.config.config_directory,
             cfg_invoc_key=self.app.config.config_key,
@@ -303,7 +303,7 @@ class OutputFileParser(JSONLike):
                 from pathlib import Path
                 import {app_module} as app
                 app.load_config(
-                    log_file_path=Path("{app_package_name}.log").resolve(),
+                    log_file_path=Path("{run_log_file}").resolve(),
                     config_dir=r"{cfg_dir}",
                     config_key=r"{cfg_invoc_key}",
                 )
@@ -321,7 +321,7 @@ class OutputFileParser(JSONLike):
         """
         )
         main_block = main_block.format(
-            app_package_name=self.app.package_name,
+            run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
             app_module=self.app.module,
             cfg_dir=self.app.config.config_directory,
             cfg_invoc_key=self.app.config.config_key,

hpcflow/sdk/core/element.py

@@ -13,9 +13,11 @@ from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from hpcflow.sdk.core.parallel import ParallelMode
 from hpcflow.sdk.core.utils import (
     check_valid_py_identifier,
+    dict_values_process_flat,
     get_enum_by_name_or_val,
     split_param_label,
 )
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.shells import get_shell
 
 
@@ -500,6 +502,7 @@ class ElementIteration:
             if i.startswith(prefix)
         )
 
+    @TimeIt.decorator
     def get_data_idx(
         self,
         path: str = None,
@@ -538,6 +541,7 @@ class ElementIteration:
 
         return copy.deepcopy(data_idx)
 
+    @TimeIt.decorator
     def get_parameter_sources(
         self,
         path: str = None,
@@ -555,24 +559,16 @@ class ElementIteration:
             ID.
         """
         data_idx = self.get_data_idx(path, action_idx, run_idx)
-        out = {}
-        for k, v in data_idx.items():
-            is_multi = False
-            if isinstance(v, list):
-                is_multi = True
-            else:
-                v = [v]
 
-            sources_k = []
-            for dat_idx_i in v:
-                src = self.workflow.get_parameter_source(dat_idx_i)
-                sources_k.append(src)
-
-            if not is_multi:
-                sources_k = src
-
-            out[k] = sources_k
+        # the value associated with `repeats.*` is the repeats index, not a parameter ID:
+        for k in list(data_idx.keys()):
+            if k.startswith("repeats."):
+                data_idx.pop(k)
 
+        out = dict_values_process_flat(
+            data_idx,
+            callable=self.workflow.get_parameter_sources,
+        )
         task_key = "task_insert_ID"
 
         if use_task_index:
@@ -631,6 +627,7 @@ class ElementIteration:
 
         return out
 
+    @TimeIt.decorator
     def get(
         self,
         path: str = None,
@@ -856,6 +853,7 @@ class ElementIteration:
             out[res_i.scope.to_string()] = res_i._get_value()
         return out
 
+    @TimeIt.decorator
     def get_resources(self, action: app.Action, set_defaults: bool = False) -> Dict:
         """Resolve specific resources for the specified action of this iteration,
         considering all applicable scopes.
@@ -998,6 +996,7 @@ class Element:
         return self._iteration_IDs
 
     @property
+    @TimeIt.decorator
     def iterations(self) -> Dict[app.ElementAction]:
         # TODO: fix this
         if self._iteration_objs is None:

hpcflow/sdk/core/run_dir_files.py

@@ -0,0 +1,63 @@
+import re
+from hpcflow.sdk.core.utils import JSONLikeDirSnapShot
+
+
+class RunDirAppFiles:
+    """A class to encapsulate the naming/recognition of app-created files within run
+    directories."""
+
+    _app_attr = "app"
+
+    CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*"
+
+    @classmethod
+    def get_log_file_name(cls):
+        """File name for the app log file."""
+        return f"{cls.app.package_name}.log"
+
+    @classmethod
+    def get_std_file_name(cls):
+        """File name for stdout and stderr streams from the app."""
+        return f"{cls.app.package_name}_std.txt"
+
+    @staticmethod
+    def get_run_file_prefix(js_idx: int, js_action_idx: int):
+        return f"js_{js_idx}_act_{js_action_idx}"
+
+    @classmethod
+    def get_commands_file_name(cls, js_idx: int, js_action_idx: int, shell):
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + shell.JS_EXT
+
+    @classmethod
+    def get_run_param_dump_file_prefix(cls, js_idx: int, js_action_idx: int):
+        """Get the prefix to a file in the run directory that the app will dump parameter
+        data to."""
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + "_inputs"
+
+    @classmethod
+    def get_run_param_load_file_prefix(cls, js_idx: int, js_action_idx: int):
+        """Get the prefix to a file in the run directory that the app will load parameter
+        data from."""
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + "_outputs"
+
+    @classmethod
+    def take_snapshot(cls):
+        """Take a JSONLikeDirSnapShot, and process to ignore files created by the app.
+
+        This includes command files that are invoked by jobscripts, the app log file, and
+        the app standard out/error file.
+
+        """
+        snapshot = JSONLikeDirSnapShot()
+        snapshot.take(".")
+        ss_js = snapshot.to_json_like()
+        ss_js.pop("root_path")  # always the current working directory of the run
+        for k in list(ss_js["data"].keys()):
+            if (
+                k == cls.get_log_file_name()
+                or k == cls.get_std_file_name()
+                or re.match(cls.CMD_FILES_RE_PATTERN, k)
+            ):
+                ss_js["data"].pop(k)
+
+        return ss_js