hpcflow-new2 0.2.0a159__py3-none-any.whl → 0.2.0a161__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a159"
+__version__ = "0.2.0a161"

hpcflow/sdk/__init__.py

@@ -1,4 +1,5 @@
 """Sub-package to define an extensible hpcflow application."""
+
 import logging
 import os
 import sys
@@ -90,6 +91,7 @@ sdk_classes = {
     "SlurmPosix": "hpcflow.sdk.submission.schedulers.slurm",
     "SGEPosix": "hpcflow.sdk.submission.schedulers.sge",
     "OutputLabel": "hpcflow.sdk.core.task",
+    "RunDirAppFiles": "hpcflow.sdk.core.run_dir_files",
 }
 
 # these are defined as `BaseApp` methods with an underscore prefix:

hpcflow/sdk/app.py

@@ -20,6 +20,7 @@ import warnings
 import zipfile
 from platformdirs import user_cache_path, user_data_dir
 from reretry import retry
+import rich
 from rich.console import Console, Group
 from rich.syntax import Syntax
 from rich.table import Table, box
@@ -939,7 +940,15 @@ class BaseApp(metaclass=Singleton):
         return self.user_data_hostname_dir / self.known_subs_file_name
 
     def _format_known_submissions_line(
-        self, local_id, workflow_id, submit_time, sub_idx, is_active, wk_path
+        self,
+        local_id,
+        workflow_id,
+        submit_time,
+        sub_idx,
+        is_active,
+        wk_path,
+        start_time,
+        end_time,
     ):
         line = [
             str(local_id),
@@ -948,23 +957,35 @@ class BaseApp(metaclass=Singleton):
             str(sub_idx),
             submit_time,
             str(wk_path),
+            start_time,
+            end_time,
         ]
         return self._known_subs_file_sep.join(line) + "\n"
 
     def _parse_known_submissions_line(self, line: str) -> Dict:
-        local_id, workflow_id, is_active, sub_idx, submit_time, path_i = line.split(
-            self._known_subs_file_sep, maxsplit=5
-        )
+        (
+            local_id,
+            workflow_id,
+            is_active,
+            sub_idx,
+            submit_time,
+            path_i,
+            start_time,
+            end_time,
+        ) = line.split(self._known_subs_file_sep, maxsplit=7)
         item = {
             "local_id": int(local_id),
             "workflow_id": workflow_id,
             "is_active": bool(int(is_active)),
-            "submit_time": submit_time,
             "sub_idx": int(sub_idx),
-            "path": path_i.strip(),
+            "submit_time": submit_time,
+            "path": path_i,
+            "start_time": start_time,
+            "end_time": end_time.strip(),
         }
         return item
 
+    @TimeIt.decorator
     def read_known_submissions_file(self) -> List[Dict]:
         """Retrieve existing workflows that *might* be running."""
         known = []
@@ -1014,6 +1035,8 @@ class BaseApp(metaclass=Singleton):
             submit_time=sub_time,
             sub_idx=sub_idx,
             wk_path=wk_path,
+            start_time="",
+            end_time="",
         )
         with self.known_subs_file_path.open("at", newline="\n") as fh:
             # TODO: check wk_path is an absolute path? what about if a remote fsspec path?
@@ -1024,16 +1047,22 @@ class BaseApp(metaclass=Singleton):
 
         return next_id
 
-    def set_inactive_in_known_subs_file(self, inactive_IDs: List[int]):
-        """Set workflows in the known-submissions file to the non-running state.
+    @TimeIt.decorator
+    def update_known_subs_file(
+        self,
+        inactive_IDs: List[int],
+        start_times: Dict[int, str],
+        end_times: Dict[int, str],
+    ):
+        """Update submission records in the known-submission file.
 
         Note we aim for atomicity to help with the scenario where a new workflow
         submission is adding itself to the file at the same time as we have decided an
         existing workflow should no longer be part of this file. Ideally, such a scenario
         should not arise because both operations should only ever be interactively
-        initiated by the single user (`Workflow.submit` and `App.get_known_submissions`). If this
-        operation is atomic, then at least the known-submissions file should be left in a
-        usable (but inaccurate) state.
+        initiated by the single user (`Workflow.submit` and `App.get_known_submissions`).
+        If this operation is atomic, then at least the known-submissions file should be
+        left in a usable (but inaccurate) state.
 
         Returns
         -------
@@ -1063,30 +1092,41 @@ class BaseApp(metaclass=Singleton):
                 continue
             item = self._parse_known_submissions_line(line)
             line_IDs.append(item["local_id"])
-            is_active = item["is_active"]
+            shows_as_active = item["is_active"]
+            is_inactive = item["local_id"] in inactive_IDs
+            start_time = item["start_time"] or start_times.get(item["local_id"], "")
+            end_time = item["end_time"] or end_times.get(item["local_id"], "")
 
-            if item["local_id"] in inactive_IDs and is_active:
-                # need to modify to set as inactive:
-                non_run_line = self._format_known_submissions_line(
+            update_inactive = is_inactive and shows_as_active
+            update_start = item["local_id"] in start_times
+            update_end = item["local_id"] in end_times
+
+            if update_inactive or update_start or update_end:
+
+                updated = self._format_known_submissions_line(
                     local_id=item["local_id"],
                     workflow_id=item["workflow_id"],
-                    is_active=False,
+                    is_active=not is_inactive,
                     submit_time=item["submit_time"],
                     sub_idx=item["sub_idx"],
                     wk_path=item["path"],
+                    start_time=start_time,
+                    end_time=end_time,
                 )
-                new_lines.append(non_run_line)
-                is_active = False
+                new_lines.append(updated)
+
                 self.submission_logger.debug(
-                    f"will set the following (workflow, submission) from the "
-                    f"known-submissions file to inactive: "
+                    f"Updating (workflow, submission) from the known-submissions file: "
+                    f"{'set to inactive; ' if update_inactive else ''}"
+                    f"{f'set start_time: {start_time!r}; ' if update_start else ''}"
+                    f"{f'set end_time: {end_time!r}; ' if update_end else ''}"
                     f"({item['path']}, {item['sub_idx']})"
                 )
             else:
                 # leave this one alone:
                 new_lines.append(line + "\n")
 
-            if not is_active:
+            if is_inactive:
                 line_date[ln_idx] = item["submit_time"]
 
         ld_srt_idx = list(dict(sorted(line_date.items(), key=lambda i: i[1])).keys())
@@ -1141,6 +1181,7 @@ class BaseApp(metaclass=Singleton):
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
         variables: Optional[Dict[str, str]] = None,
+        status: Optional[bool] = True,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a file or string containing a workflow
         template parametrisation.
@@ -1177,10 +1218,17 @@ class BaseApp(metaclass=Singleton):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables
             String variables to substitute in `template_file_or_str`.
+        status
+            If True, display a live status to track workflow creation progress.
         """
 
         self.API_logger.info("make_workflow called")
 
+        if status:
+            console = rich.console.Console()
+            status = console.status("Making persistent workflow...")
+            status.start()
+
         common = {
             "path": path,
             "name": name,
@@ -1190,6 +1238,7 @@ class BaseApp(metaclass=Singleton):
             "ts_name_fmt": ts_name_fmt,
             "store_kwargs": store_kwargs,
             "variables": variables,
+            "status": status,
         }
 
         if not is_string:
@@ -1200,10 +1249,24 @@ class BaseApp(metaclass=Singleton):
             )
 
         elif template_format == "json":
-            wk = self.Workflow.from_JSON_string(JSON_str=template_file_or_str, **common)
+            try:
+                wk = self.Workflow.from_JSON_string(
+                    JSON_str=template_file_or_str, **common
+                )
+            except Exception:
+                if status:
+                    status.stop()
+                raise
 
         elif template_format == "yaml":
-            wk = self.Workflow.from_YAML_string(YAML_str=template_file_or_str, **common)
+            try:
+                wk = self.Workflow.from_YAML_string(
+                    YAML_str=template_file_or_str, **common
+                )
+            except Exception:
+                if status:
+                    status.stop()
+                raise
 
         elif not template_format:
             raise ValueError(
@@ -1216,6 +1279,10 @@ class BaseApp(metaclass=Singleton):
                 f"Template format {template_format!r} not understood. Available template "
                 f"formats are {ALL_TEMPLATE_FORMATS!r}."
             )
+
+        if status:
+            status.stop()
+
         return wk
 
     def _make_and_submit_workflow(
@@ -1236,6 +1303,8 @@ class BaseApp(metaclass=Singleton):
         add_to_known: Optional[bool] = True,
         return_idx: Optional[bool] = False,
         tasks: Optional[List[int]] = None,
+        cancel: Optional[bool] = False,
+        status: Optional[bool] = True,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1288,6 +1357,11 @@ class BaseApp(metaclass=Singleton):
         tasks
             List of task indices to include in this submission. By default all tasks are
             included.
+        cancel
+            Immediately cancel the submission. Useful for testing and benchmarking.
+        status
+            If True, display a live status to track workflow creation and submission
+            progress.
         """
 
         self.API_logger.info("make_and_submit_workflow called")
@@ -1304,6 +1378,7 @@ class BaseApp(metaclass=Singleton):
             ts_name_fmt=ts_name_fmt,
             store_kwargs=store_kwargs,
             variables=variables,
+            status=status,
         )
         return wk.submit(
             JS_parallelism=JS_parallelism,
@@ -1311,6 +1386,8 @@ class BaseApp(metaclass=Singleton):
             add_to_known=add_to_known,
             return_idx=return_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
 
     def _make_demo_workflow(
@@ -1325,6 +1402,7 @@ class BaseApp(metaclass=Singleton):
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
         variables: Optional[Dict[str, str]] = None,
+        status: Optional[bool] = True,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a builtin demo workflow template.
 
@@ -1358,10 +1436,17 @@ class BaseApp(metaclass=Singleton):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables
             String variables to substitute in the demo workflow template file.
+        status
+            If True, display a live status to track workflow creation progress.
         """
 
         self.API_logger.info("make_demo_workflow called")
 
+        if status:
+            console = rich.console.Console()
+            status = console.status("Making persistent workflow...")
+            status.start()
+
         with self.get_demo_workflow_template_file(workflow_name) as template_path:
             wk = self.Workflow.from_file(
                 template_path=template_path,
@@ -1374,7 +1459,10 @@ class BaseApp(metaclass=Singleton):
                 ts_name_fmt=ts_name_fmt,
                 store_kwargs=store_kwargs,
                 variables=variables,
+                status=status,
             )
+        if status:
+            status.stop()
         return wk
 
     def _make_and_submit_demo_workflow(
@@ -1394,6 +1482,8 @@ class BaseApp(metaclass=Singleton):
         add_to_known: Optional[bool] = True,
         return_idx: Optional[bool] = False,
         tasks: Optional[List[int]] = None,
+        cancel: Optional[bool] = False,
+        status: Optional[bool] = True,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1443,6 +1533,10 @@ class BaseApp(metaclass=Singleton):
         tasks
             List of task indices to include in this submission. By default all tasks are
             included.
+        cancel
+            Immediately cancel the submission. Useful for testing and benchmarking.
+        status
+            If True, display a live status to track submission progress.
         """
 
         self.API_logger.info("make_and_submit_demo_workflow called")
@@ -1465,6 +1559,8 @@ class BaseApp(metaclass=Singleton):
             add_to_known=add_to_known,
             return_idx=return_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
 
     def _submit_workflow(
@@ -1556,8 +1652,13 @@ class BaseApp(metaclass=Singleton):
         )
         return shell.get_version_info(exclude_os)
 
+    @TimeIt.decorator
     def _get_known_submissions(
-        self, max_recent: int = 3, no_update: bool = False, as_json: bool = False
+        self,
+        max_recent: int = 3,
+        no_update: bool = False,
+        as_json: bool = False,
+        status: Optional[Any] = None,
     ):
         """Retrieve information about active and recently inactive finished {app_name}
         workflows.
@@ -1580,8 +1681,14 @@ class BaseApp(metaclass=Singleton):
 
         out = []
         inactive_IDs = []
+        start_times = {}
+        end_times = {}
+
+        ts_fmt = self._submission_ts_fmt
 
         try:
+            if status:
+                status.update("Reading known submissions file...")
             known_subs = self.read_known_submissions_file()
         except FileNotFoundError:
             known_subs = []
@@ -1592,14 +1699,31 @@ class BaseApp(metaclass=Singleton):
         # loop in reverse so we process more-recent submissions first:
         for file_dat_i in known_subs[::-1]:
             submit_time_str = file_dat_i["submit_time"]
-            submit_time_obj = datetime.strptime(submit_time_str, self._submission_ts_fmt)
+            submit_time_obj = datetime.strptime(submit_time_str, ts_fmt)
             submit_time_obj = submit_time_obj.replace(tzinfo=timezone.utc).astimezone()
+
+            start_time_str = file_dat_i["start_time"]
+            start_time_obj = None
+            if start_time_str:
+                start_time_obj = datetime.strptime(start_time_str, ts_fmt)
+                start_time_obj = start_time_obj.replace(tzinfo=timezone.utc).astimezone()
+
+            end_time_str = file_dat_i["end_time"]
+            end_time_obj = None
+            if end_time_str:
+                end_time_obj = datetime.strptime(end_time_str, ts_fmt)
+                end_time_obj = end_time_obj.replace(tzinfo=timezone.utc).astimezone()
+
             out_item = {
                 "local_id": file_dat_i["local_id"],
                 "workflow_id": file_dat_i["workflow_id"],
                 "workflow_path": file_dat_i["path"],
                 "submit_time": submit_time_str,
                 "submit_time_obj": submit_time_obj,
+                "start_time": start_time_str,
+                "start_time_obj": start_time_obj,
+                "end_time": end_time_str,
+                "end_time_obj": end_time_obj,
                 "sub_idx": file_dat_i["sub_idx"],
                 "jobscripts": [],
                 "active_jobscripts": {},
@@ -1614,7 +1738,11 @@ class BaseApp(metaclass=Singleton):
                 out_item["deleted"] = not path_exists
                 if path_exists:
                     try:
+                        if status:
+                            status.update(f"Inspecting workflow {file_dat_i['path']!r}.")
                         wk_i = self.Workflow(file_dat_i["path"])
+                    except KeyboardInterrupt:
+                        raise
                     except Exception:
                         wk_i = None
                         self.submission_logger.info(
@@ -1640,15 +1768,36 @@ class BaseApp(metaclass=Singleton):
                     out_item["deleted"] = True
 
                 else:
-                    sub = wk_i.submissions[file_dat_i["sub_idx"]]
+                    if status:
+                        status.update(
+                            f"Reading workflow {file_dat_i['path']!r} submission info..."
+                        )
+                    with wk_i._store.cache_ctx():
+                        sub = wk_i.submissions[file_dat_i["sub_idx"]]
+
+                        all_jobscripts = sub._submission_parts[submit_time_str]
+                        out_item.update(
+                            {
+                                "jobscripts": all_jobscripts,
+                                "submission": sub,
+                            }
+                        )
+                        if not out_item["start_time"]:
+                            start_time_obj = sub.start_time
+                            if start_time_obj:
+                                start_time = datetime.strftime(start_time_obj, ts_fmt)
+                                out_item["start_time"] = start_time
+                                start_times[file_dat_i["local_id"]] = start_time
+                            out_item["start_time_obj"] = start_time_obj
+
+                        if not out_item["end_time"]:
+                            end_time_obj = sub.end_time
+                            if end_time_obj:
+                                end_time = datetime.strftime(end_time_obj, ts_fmt)
+                                out_item["end_time"] = end_time
+                                end_times[file_dat_i["local_id"]] = end_time
+                            out_item["end_time_obj"] = end_time_obj
 
-                    all_jobscripts = sub._submission_parts[submit_time_str]
-                    out_item.update(
-                        {
-                            "jobscripts": all_jobscripts,
-                            "submission": sub,
-                        }
-                    )
                     if file_dat_i["is_active"]:
                         # check it really is active:
                         run_key = (file_dat_i["path"], file_dat_i["sub_idx"])
@@ -1657,6 +1806,8 @@ class BaseApp(metaclass=Singleton):
                         else:
                             try:
                                 act_i_js = sub.get_active_jobscripts(as_json=as_json)
+                            except KeyboardInterrupt:
+                                raise
                             except Exception:
                                 self.submission_logger.info(
                                     f"failed to retrieve active jobscripts from workflow "
@@ -1679,8 +1830,10 @@ class BaseApp(metaclass=Singleton):
 
             out.append(out_item)
 
-        if inactive_IDs and not no_update:
-            removed_IDs = self.set_inactive_in_known_subs_file(inactive_IDs)
+        if (inactive_IDs or start_times or end_times) and not no_update:
+            removed_IDs = self.update_known_subs_file(
+                inactive_IDs, start_times, end_times
+            )
             # remove these from the output, to avoid confusion (if kept, they would not
             # appear in the next invocation of this method):
             out = [i for i in out if i["local_id"] not in removed_IDs]
@@ -1694,9 +1847,7 @@ class BaseApp(metaclass=Singleton):
         out_access = sorted(
             out_access,
             key=lambda i: (
-                i["submission"].end_time
-                or i["submission"].start_time
-                or i["submit_time_obj"]
+                i["end_time_obj"] or i["start_time_obj"] or i["submit_time_obj"]
             ),
             reverse=True,
         )
@@ -1756,6 +1907,7 @@ class BaseApp(metaclass=Singleton):
         )
         rich_print(group)
 
+    @TimeIt.decorator
     def _show(
         self,
         max_recent: int = 3,
@@ -1826,8 +1978,9 @@ class BaseApp(metaclass=Singleton):
             run_dat = self._get_known_submissions(
                 max_recent=max_recent,
                 no_update=no_update,
+                status=status,
             )
-        except Exception:
+        except (Exception, KeyboardInterrupt):
             status.stop()
             raise
         else:
@@ -1878,8 +2031,8 @@ class BaseApp(metaclass=Singleton):
 
             start_time, end_time = None, None
             if not no_access:
-                start_time = dat_i["submission"].start_time
-                end_time = dat_i["submission"].end_time if not act_js else None
+                start_time = dat_i["start_time_obj"]
+                end_time = dat_i["end_time_obj"]
 
             if "actions" in columns:
                 if not no_access:

hpcflow/sdk/cli.py

@@ -25,6 +25,9 @@ from hpcflow.sdk.cli_common import (
     add_to_known_opt,
     print_idx_opt,
     tasks_opt,
+    cancel_opt,
+    submit_status_opt,
+    make_status_opt,
     zip_path_opt,
     zip_overwrite_opt,
     zip_log_opt,
@@ -71,6 +74,7 @@ def _make_API_CLI(app):
     @ts_fmt_option
     @ts_name_fmt_option
     @variables_option
+    @make_status_opt
     def make_workflow(
         template_file_or_str,
         string,
@@ -82,6 +86,7 @@ def _make_API_CLI(app):
         ts_fmt=None,
         ts_name_fmt=None,
         variables=None,
+        status=True,
     ):
         """Generate a new {app_name} workflow.
 
@@ -100,6 +105,7 @@ def _make_API_CLI(app):
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
             variables=dict(variables),
+            status=status,
         )
         click.echo(wk.path)
 
@@ -119,6 +125,8 @@ def _make_API_CLI(app):
     @add_to_known_opt
     @print_idx_opt
     @tasks_opt
+    @cancel_opt
+    @submit_status_opt
     def make_and_submit_workflow(
         template_file_or_str,
         string,
@@ -135,6 +143,8 @@ def _make_API_CLI(app):
         add_to_known=True,
         print_idx=False,
         tasks=None,
+        cancel=False,
+        status=True,
     ):
         """Generate and submit a new {app_name} workflow.
 
@@ -159,6 +169,8 @@ def _make_API_CLI(app):
             add_to_known=add_to_known,
             return_idx=print_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
         if print_idx:
             click.echo(out)
@@ -320,6 +332,8 @@ def _make_workflow_CLI(app):
     @add_to_known_opt
     @print_idx_opt
     @tasks_opt
+    @cancel_opt
+    @submit_status_opt
     @click.pass_context
     def submit_workflow(
         ctx,
@@ -328,6 +342,8 @@ def _make_workflow_CLI(app):
         add_to_known=True,
         print_idx=False,
         tasks=None,
+        cancel=False,
+        status=True,
     ):
         """Submit the workflow."""
         out = ctx.obj["workflow"].submit(
@@ -336,6 +352,8 @@ def _make_workflow_CLI(app):
             add_to_known=add_to_known,
             return_idx=print_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
         if print_idx:
             click.echo(out)

hpcflow/sdk/cli_common.py

@@ -109,6 +109,22 @@ tasks_opt = click.option(
     ),
     callback=sub_tasks_callback,
 )
+cancel_opt = click.option(
+    "--cancel",
+    help="Immediately cancel the submission. Useful for testing and benchmarking.",
+    is_flag=True,
+    default=False,
+)
+submit_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track submission progress.",
+    default=True,
+)
+make_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track workflow creation progress.",
+    default=True,
+)
 zip_path_opt = click.option(
     "--path",
     default=".",