hpcflow-new2 0.2.0a160__py3-none-any.whl → 0.2.0a161__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a160"
+__version__ = "0.2.0a161"

hpcflow/sdk/app.py

@@ -940,7 +940,15 @@ class BaseApp(metaclass=Singleton):
         return self.user_data_hostname_dir / self.known_subs_file_name
 
     def _format_known_submissions_line(
-        self, local_id, workflow_id, submit_time, sub_idx, is_active, wk_path
+        self,
+        local_id,
+        workflow_id,
+        submit_time,
+        sub_idx,
+        is_active,
+        wk_path,
+        start_time,
+        end_time,
     ):
         line = [
             str(local_id),
@@ -949,20 +957,31 @@ class BaseApp(metaclass=Singleton):
             str(sub_idx),
             submit_time,
             str(wk_path),
+            start_time,
+            end_time,
         ]
         return self._known_subs_file_sep.join(line) + "\n"
 
     def _parse_known_submissions_line(self, line: str) -> Dict:
-        local_id, workflow_id, is_active, sub_idx, submit_time, path_i = line.split(
-            self._known_subs_file_sep, maxsplit=5
-        )
+        (
+            local_id,
+            workflow_id,
+            is_active,
+            sub_idx,
+            submit_time,
+            path_i,
+            start_time,
+            end_time,
+        ) = line.split(self._known_subs_file_sep, maxsplit=7)
         item = {
             "local_id": int(local_id),
             "workflow_id": workflow_id,
             "is_active": bool(int(is_active)),
-            "submit_time": submit_time,
             "sub_idx": int(sub_idx),
-            "path": path_i.strip(),
+            "submit_time": submit_time,
+            "path": path_i,
+            "start_time": start_time,
+            "end_time": end_time.strip(),
         }
         return item
 
@@ -1016,6 +1035,8 @@ class BaseApp(metaclass=Singleton):
             submit_time=sub_time,
             sub_idx=sub_idx,
             wk_path=wk_path,
+            start_time="",
+            end_time="",
         )
         with self.known_subs_file_path.open("at", newline="\n") as fh:
             # TODO: check wk_path is an absolute path? what about if a remote fsspec path?
@@ -1027,16 +1048,21 @@ class BaseApp(metaclass=Singleton):
         return next_id
 
     @TimeIt.decorator
-    def set_inactive_in_known_subs_file(self, inactive_IDs: List[int]):
-        """Set workflows in the known-submissions file to the non-running state.
+    def update_known_subs_file(
+        self,
+        inactive_IDs: List[int],
+        start_times: Dict[int, str],
+        end_times: Dict[int, str],
+    ):
+        """Update submission records in the known-submission file.
 
         Note we aim for atomicity to help with the scenario where a new workflow
         submission is adding itself to the file at the same time as we have decided an
         existing workflow should no longer be part of this file. Ideally, such a scenario
         should not arise because both operations should only ever be interactively
-        initiated by the single user (`Workflow.submit` and `App.get_known_submissions`). If this
-        operation is atomic, then at least the known-submissions file should be left in a
-        usable (but inaccurate) state.
+        initiated by the single user (`Workflow.submit` and `App.get_known_submissions`).
+        If this operation is atomic, then at least the known-submissions file should be
+        left in a usable (but inaccurate) state.
 
         Returns
         -------
@@ -1066,30 +1092,41 @@ class BaseApp(metaclass=Singleton):
                 continue
             item = self._parse_known_submissions_line(line)
             line_IDs.append(item["local_id"])
-            is_active = item["is_active"]
+            shows_as_active = item["is_active"]
+            is_inactive = item["local_id"] in inactive_IDs
+            start_time = item["start_time"] or start_times.get(item["local_id"], "")
+            end_time = item["end_time"] or end_times.get(item["local_id"], "")
+
+            update_inactive = is_inactive and shows_as_active
+            update_start = item["local_id"] in start_times
+            update_end = item["local_id"] in end_times
 
-            if item["local_id"] in inactive_IDs and is_active:
-                # need to modify to set as inactive:
-                non_run_line = self._format_known_submissions_line(
+            if update_inactive or update_start or update_end:
+
+                updated = self._format_known_submissions_line(
                     local_id=item["local_id"],
                     workflow_id=item["workflow_id"],
-                    is_active=False,
+                    is_active=not is_inactive,
                     submit_time=item["submit_time"],
                     sub_idx=item["sub_idx"],
                     wk_path=item["path"],
+                    start_time=start_time,
+                    end_time=end_time,
                 )
-                new_lines.append(non_run_line)
-                is_active = False
+                new_lines.append(updated)
+
                 self.submission_logger.debug(
-                    f"will set the following (workflow, submission) from the "
-                    f"known-submissions file to inactive: "
+                    f"Updating (workflow, submission) from the known-submissions file: "
+                    f"{'set to inactive; ' if update_inactive else ''}"
+                    f"{f'set start_time: {start_time!r}; ' if update_start else ''}"
+                    f"{f'set end_time: {end_time!r}; ' if update_end else ''}"
                     f"({item['path']}, {item['sub_idx']})"
                 )
             else:
                 # leave this one alone:
                 new_lines.append(line + "\n")
 
-            if not is_active:
+            if is_inactive:
                 line_date[ln_idx] = item["submit_time"]
 
         ld_srt_idx = list(dict(sorted(line_date.items(), key=lambda i: i[1])).keys())
@@ -1644,6 +1681,10 @@ class BaseApp(metaclass=Singleton):
 
         out = []
         inactive_IDs = []
+        start_times = {}
+        end_times = {}
+
+        ts_fmt = self._submission_ts_fmt
 
         try:
             if status:
@@ -1658,14 +1699,31 @@ class BaseApp(metaclass=Singleton):
         # loop in reverse so we process more-recent submissions first:
         for file_dat_i in known_subs[::-1]:
             submit_time_str = file_dat_i["submit_time"]
-            submit_time_obj = datetime.strptime(submit_time_str, self._submission_ts_fmt)
+            submit_time_obj = datetime.strptime(submit_time_str, ts_fmt)
             submit_time_obj = submit_time_obj.replace(tzinfo=timezone.utc).astimezone()
+
+            start_time_str = file_dat_i["start_time"]
+            start_time_obj = None
+            if start_time_str:
+                start_time_obj = datetime.strptime(start_time_str, ts_fmt)
+                start_time_obj = start_time_obj.replace(tzinfo=timezone.utc).astimezone()
+
+            end_time_str = file_dat_i["end_time"]
+            end_time_obj = None
+            if end_time_str:
+                end_time_obj = datetime.strptime(end_time_str, ts_fmt)
+                end_time_obj = end_time_obj.replace(tzinfo=timezone.utc).astimezone()
+
             out_item = {
                 "local_id": file_dat_i["local_id"],
                 "workflow_id": file_dat_i["workflow_id"],
                 "workflow_path": file_dat_i["path"],
                 "submit_time": submit_time_str,
                 "submit_time_obj": submit_time_obj,
+                "start_time": start_time_str,
+                "start_time_obj": start_time_obj,
+                "end_time": end_time_str,
+                "end_time_obj": end_time_obj,
                 "sub_idx": file_dat_i["sub_idx"],
                 "jobscripts": [],
                 "active_jobscripts": {},
@@ -1683,6 +1741,8 @@ class BaseApp(metaclass=Singleton):
                         if status:
                             status.update(f"Inspecting workflow {file_dat_i['path']!r}.")
                         wk_i = self.Workflow(file_dat_i["path"])
+                    except KeyboardInterrupt:
+                        raise
                     except Exception:
                         wk_i = None
                         self.submission_logger.info(
@@ -1720,10 +1780,24 @@ class BaseApp(metaclass=Singleton):
                             {
                                 "jobscripts": all_jobscripts,
                                 "submission": sub,
-                                "sub_start_time": sub.start_time,
-                                "sub_end_time": sub.end_time,
                             }
                         )
+                        if not out_item["start_time"]:
+                            start_time_obj = sub.start_time
+                            if start_time_obj:
+                                start_time = datetime.strftime(start_time_obj, ts_fmt)
+                                out_item["start_time"] = start_time
+                                start_times[file_dat_i["local_id"]] = start_time
+                            out_item["start_time_obj"] = start_time_obj
+
+                        if not out_item["end_time"]:
+                            end_time_obj = sub.end_time
+                            if end_time_obj:
+                                end_time = datetime.strftime(end_time_obj, ts_fmt)
+                                out_item["end_time"] = end_time
+                                end_times[file_dat_i["local_id"]] = end_time
+                            out_item["end_time_obj"] = end_time_obj
+
                     if file_dat_i["is_active"]:
                         # check it really is active:
                         run_key = (file_dat_i["path"], file_dat_i["sub_idx"])
@@ -1732,6 +1806,8 @@ class BaseApp(metaclass=Singleton):
                         else:
                             try:
                                 act_i_js = sub.get_active_jobscripts(as_json=as_json)
+                            except KeyboardInterrupt:
+                                raise
                             except Exception:
                                 self.submission_logger.info(
                                     f"failed to retrieve active jobscripts from workflow "
@@ -1754,8 +1830,10 @@ class BaseApp(metaclass=Singleton):
 
             out.append(out_item)
 
-        if inactive_IDs and not no_update:
-            removed_IDs = self.set_inactive_in_known_subs_file(inactive_IDs)
+        if (inactive_IDs or start_times or end_times) and not no_update:
+            removed_IDs = self.update_known_subs_file(
+                inactive_IDs, start_times, end_times
+            )
             # remove these from the output, to avoid confusion (if kept, they would not
             # appear in the next invocation of this method):
             out = [i for i in out if i["local_id"] not in removed_IDs]
@@ -1769,7 +1847,7 @@ class BaseApp(metaclass=Singleton):
         out_access = sorted(
             out_access,
             key=lambda i: (
-                i["sub_end_time"] or i["sub_start_time"] or i["submit_time_obj"]
+                i["end_time_obj"] or i["start_time_obj"] or i["submit_time_obj"]
             ),
             reverse=True,
         )
@@ -1829,6 +1907,7 @@ class BaseApp(metaclass=Singleton):
         )
         rich_print(group)
 
+    @TimeIt.decorator
     def _show(
         self,
         max_recent: int = 3,
@@ -1901,7 +1980,7 @@ class BaseApp(metaclass=Singleton):
                 no_update=no_update,
                 status=status,
             )
-        except Exception:
+        except (Exception, KeyboardInterrupt):
             status.stop()
             raise
         else:
@@ -1952,8 +2031,8 @@ class BaseApp(metaclass=Singleton):
 
             start_time, end_time = None, None
             if not no_access:
-                start_time = dat_i["submission"].start_time
-                end_time = dat_i["submission"].end_time if not act_js else None
+                start_time = dat_i["start_time_obj"]
+                end_time = dat_i["end_time_obj"]
 
             if "actions" in columns:
                 if not no_access:

hpcflow/sdk/core/command_files.py

@@ -144,10 +144,9 @@ class InputFileGenerator(JSONLike):
             self.app.ActionRule.check_missing(f"input_files.{self.input_file.label}")
         ] + self.rules
 
-    def compose_source(self, action) -> str:
+    def compose_source(self, snip_path) -> str:
         """Generate the file contents of this input file generator source."""
 
-        snip_path = action.get_snippet_script_path(self.script)
         script_main_func = snip_path.stem
         with snip_path.open("rt") as fp:
             script_str = fp.read()
@@ -190,9 +189,14 @@ class InputFileGenerator(JSONLike):
         return out
 
     def write_source(self, action):
-        script_path = action.get_script_name(self.script)
-        with Path(script_path).open("wt", newline="\n") as fp:
-            fp.write(self.compose_source(action))
+
+        # write the script if it is specified as a snippet script, otherwise we assume
+        # the script already exists in the working directory:
+        snip_path = action.get_snippet_script_path(self.script)
+        if snip_path:
+            source_str = self.compose_source(snip_path)
+            with Path(snip_path.name).open("wt", newline="\n") as fp:
+                fp.write(source_str)
 
 
 @dataclass
@@ -284,14 +288,13 @@ class OutputFileParser(JSONLike):
             for i in self.output_files
         ] + self.rules
 
-    def compose_source(self, action) -> str:
+    def compose_source(self, snip_path) -> str:
         """Generate the file contents of this output file parser source."""
 
         if self.output is None:
             # might be used just for saving files:
             return
 
-        snip_path = action.get_snippet_script_path(self.script)
         script_main_func = snip_path.stem
         with snip_path.open("rt") as fp:
             script_str = fp.read()
@@ -343,9 +346,14 @@ class OutputFileParser(JSONLike):
         if self.output is None:
             # might be used just for saving files:
             return
-        script_path = action.get_script_name(self.script)
-        with Path(script_path).open("wt", newline="\n") as fp:
-            fp.write(self.compose_source(action))
+
+        # write the script if it is specified as a snippet script, otherwise we assume
+        # the script already exists in the working directory:
+        snip_path = action.get_snippet_script_path(self.script)
+        if snip_path:
+            source_str = self.compose_source(snip_path)
+            with Path(snip_path.name).open("wt", newline="\n") as fp:
+                fp.write(source_str)
 
 
 class _FileContentsSpecifier(JSONLike):

hpcflow/tests/unit/test_submission.py

@@ -537,6 +537,7 @@ def test_scheduler_config_defaults(new_null_config, tmp_path):
     wkt = hf.WorkflowTemplate(name="temp", tasks=[t1, t2])
     wk = hf.Workflow.from_template(
         template=wkt,
+        path=tmp_path,
     )
     sub = wk.add_submission()
     assert sub.jobscripts[0].resources.scheduler_args == {"options": {"a": "c"}}

{hpcflow_new2-0.2.0a160.dist-info → hpcflow_new2-0.2.0a161.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a160
+Version: 0.2.0a161
 Summary: Computational workflow management
 License: MIT
 Author: aplowman

{hpcflow_new2-0.2.0a160.dist-info → hpcflow_new2-0.2.0a161.dist-info}/RECORD

@@ -1,7 +1,7 @@
 hpcflow/__init__.py,sha256=WIETuRHeOp2SqUqHUzpjQ-lk9acbYv-6aWOhZPRdlhs,64
 hpcflow/__pyinstaller/__init__.py,sha256=YOzBlPSck6slucv6lJM9K80JtsJWxXRL00cv6tRj3oc,98
 hpcflow/__pyinstaller/hook-hpcflow.py,sha256=SeMopsPkhCyd9gqIrzwFNRj3ZlkUlUYl-74QYz61mo4,1089
-hpcflow/_version.py,sha256=SLYvEGJeZwqDNbRaoPhsYZ7OzpYUUDy4We22ieDRFfQ,26
+hpcflow/_version.py,sha256=f61tdocu5S9MzLcsCrrQ4yVYVvCmCf9Z86QwA-46gcI,26
 hpcflow/app.py,sha256=d-kgfnZNlqlCi2H8bK26714brD_u3ibN3FaEZgjF9aA,1332
 hpcflow/cli.py,sha256=G2J3D9v6MnMWOWMMWK6UEKLn_6wnV9lT_qygEBBxg-I,66
 hpcflow/data/demo_data_manifest/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,7 +34,7 @@ hpcflow/data/workflows/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
 hpcflow/data/workflows/workflow_1.yaml,sha256=lF7Re2SVc_5gQk5AwB0gXaq-n-T5ia4su3zNQ9oMRV0,220
 hpcflow/examples.ipynb,sha256=cLKp4QsxwwMXRanDnfWY9kqsV23q6G4raOpu6IZXnMw,28553
 hpcflow/sdk/__init__.py,sha256=SdzVm7dydLv2kmr2tqrH14Gf1GEAEhsEuAuqiGBLHhM,5700
-hpcflow/sdk/app.py,sha256=tDcEaT_YWLnFy4Wi4UIwTxwvHO3F80Gxc1yd1tnmDc0,92880
+hpcflow/sdk/app.py,sha256=7f7jR3eX9iVQ69YwPOoOmeJaAFmwJcj24UIVkshzHpA,95754
 hpcflow/sdk/cli.py,sha256=y6eRg36D_IzIljc9TMvmm69NqX2zZoKUnBufX9UbN1U,35025
 hpcflow/sdk/cli_common.py,sha256=kDSIe90mxD37lQqvIXDyRauLtTRRnNueSljcRPF_w0M,4738
 hpcflow/sdk/config/__init__.py,sha256=qJrrxcAN4f1u_RyTtXgz-xlTLwNafE9v0VEMP1x6-bU,70
@@ -45,7 +45,7 @@ hpcflow/sdk/config/config_file.py,sha256=JlMcprj0aujFVk8552ahP2f8EXB0tglMaHwzbcG
 hpcflow/sdk/config/errors.py,sha256=2D7HJ1dbyeoD3xk4MuaGSsbJsUyQzyw8kaThEBZfP2I,6876
 hpcflow/sdk/core/__init__.py,sha256=GcIklEsXy3M5PWpmxyhd2KoI0u6HjXRIjD_aR1bgRjo,215
 hpcflow/sdk/core/actions.py,sha256=539vu9ts5u5Poah6-ZGOEANrb58NPEbhhoVlS7ajayE,74544
-hpcflow/sdk/core/command_files.py,sha256=yxTQnxVO0q3qfp1nEy-FSSWzHFSdhnbtQ6YvcjJX5UQ,18181
+hpcflow/sdk/core/command_files.py,sha256=mo7JzKko2WQ_DOFbJFyuvnnRN3pj1hiqPe7tRNSNDV4,18515
 hpcflow/sdk/core/commands.py,sha256=-Tiu7zVVwWr1xiTXVB9oH3E4g09ebRRtHSRrMdFDCRY,12060
 hpcflow/sdk/core/element.py,sha256=ksNVQcQX_-IdofAsc4Us3Mw_rEMLO8D5zOM8Cmajaw8,45669
 hpcflow/sdk/core/environment.py,sha256=DGUz1NvliKh6opP0IueGHD69rn_8wFLhDsq6kAmEgM4,4849
@@ -137,7 +137,7 @@ hpcflow/tests/unit/test_runtime.py,sha256=HjHPTS3UkX1LcwheFgpp4px_VlRis8KAE2Hoeq
 hpcflow/tests/unit/test_schema_input.py,sha256=spkTtvNuheh-y29Tsx7YRX6y3dV80vXx0hcg0jVfMp4,12084
 hpcflow/tests/unit/test_shell.py,sha256=FDtQ9fHRhSKiVtxMJ8BRisoeSvvk8zmJndTB4LlhqGc,3442
 hpcflow/tests/unit/test_slurm.py,sha256=ewfNuXXUEEelAxcd7MBbAQ-RCvU8xBenHTAyfXYF-R0,1064
-hpcflow/tests/unit/test_submission.py,sha256=fPemvWs7rMelKW_2ctEUUjnckGQFXgDzlFRYVY19eJs,16659
+hpcflow/tests/unit/test_submission.py,sha256=kQ3ksjGlfp47AYuwTA27RDX2XxRU3YxKlKC1ACTbXw8,16682
 hpcflow/tests/unit/test_task.py,sha256=94TwyjlhKMRRXTQjys2a1PiK7A-rCzhnvrkk4vRz39I,70000
 hpcflow/tests/unit/test_task_schema.py,sha256=7a7o42gQhrZPMXfH0a6sGzFCJnuFrbDEl9u3u_bFsgw,3624
 hpcflow/tests/unit/test_utils.py,sha256=JMhSRZFqmI9ZhREJet9en_y3aRVlQlWE7OKpkdt8SVI,14172
@@ -147,7 +147,7 @@ hpcflow/tests/unit/test_workflow_template.py,sha256=EItRqUyXpU2z_z1rvpRqa848YOkX
 hpcflow/tests/workflows/test_jobscript.py,sha256=9sp1o0g72JZbv2QlOl5v7wCZEFjotxiIKGNUxVaFgaA,724
 hpcflow/tests/workflows/test_workflows.py,sha256=xai6FRtGqG4lStJk6KmsqPUSuvqs9FrsBOxMVALshIs,13400
 hpcflow/viz_demo.ipynb,sha256=1QdnVsk72vihv2L6hOGyk318uEa22ZSgGxQCa7hW2oo,6238
-hpcflow_new2-0.2.0a160.dist-info/METADATA,sha256=PiPRoSuisn-l79v7FIky-jd5I2RpwzBE2eJ9kzPNuR0,2473
-hpcflow_new2-0.2.0a160.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
-hpcflow_new2-0.2.0a160.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
-hpcflow_new2-0.2.0a160.dist-info/RECORD,,
+hpcflow_new2-0.2.0a161.dist-info/METADATA,sha256=Fq-m5tYcROogOvrwnQds7tMF1xN_r_fza4kYOoK9O0Q,2473
+hpcflow_new2-0.2.0a161.dist-info/WHEEL,sha256=kLuE8m1WYU0Ig0_YEGrXyTtiJvKPpLpDEiChiNyei5Y,88
+hpcflow_new2-0.2.0a161.dist-info/entry_points.txt,sha256=aoGtCnFdfPcXfBdu2zZyMOJoz6fPgdR0elqsgrE-USU,106
+hpcflow_new2-0.2.0a161.dist-info/RECORD,,