hpcflow-new2 0.2.0a158__py3-none-any.whl → 0.2.0a160__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a158"
+__version__ = "0.2.0a160"

hpcflow/app.py

@@ -14,9 +14,6 @@ __dir__ = sdk_app.get_app_module_dir()
 config_options = ConfigOptions(
     directory_env_var="HPCFLOW_CONFIG_DIR",
     default_directory="~/.hpcflow-new",
-    sentry_DSN="https://2463b288fd1a40f4bada9f5ff53f6811@o1180430.ingest.sentry.io/6293231",
-    sentry_traces_sample_rate=1.0,
-    sentry_env="main" if "a" in __version__ else "develop",
 )
 
 # load built in template components (in this case, for demonstration purposes):

hpcflow/sdk/__init__.py

@@ -1,4 +1,5 @@
 """Sub-package to define an extensible hpcflow application."""
+
 import logging
 import os
 import sys
@@ -90,6 +91,7 @@ sdk_classes = {
     "SlurmPosix": "hpcflow.sdk.submission.schedulers.slurm",
     "SGEPosix": "hpcflow.sdk.submission.schedulers.sge",
     "OutputLabel": "hpcflow.sdk.core.task",
+    "RunDirAppFiles": "hpcflow.sdk.core.run_dir_files",
 }
 
 # these are defined as `BaseApp` methods with an underscore prefix:

hpcflow/sdk/app.py

@@ -20,6 +20,7 @@ import warnings
 import zipfile
 from platformdirs import user_cache_path, user_data_dir
 from reretry import retry
+import rich
 from rich.console import Console, Group
 from rich.syntax import Syntax
 from rich.table import Table, box
@@ -31,8 +32,6 @@ from fsspec.core import url_to_fs
 from fsspec.implementations.local import LocalFileSystem
 
 
-from setuptools import find_packages
-
 from hpcflow import __version__
 from hpcflow.sdk.core.actions import EARStatus
 from hpcflow.sdk.core.errors import WorkflowNotFoundError
@@ -730,12 +729,13 @@ class BaseApp(metaclass=Singleton):
 
     @TimeIt.decorator
     def _load_scripts(self):
+        from setuptools import find_packages
+
         # TODO: load custom directories / custom functions (via decorator)
 
         app_module = import_module(self.package_name)
         root_scripts_dir = self.scripts_dir
 
-        # TODO: setuptools.find_packages takes a long time to import
         packages = find_packages(
             where=str(Path(app_module.__path__[0], *root_scripts_dir.split(".")))
         )
@@ -966,6 +966,7 @@ class BaseApp(metaclass=Singleton):
         }
         return item
 
+    @TimeIt.decorator
     def read_known_submissions_file(self) -> List[Dict]:
         """Retrieve existing workflows that *might* be running."""
         known = []
@@ -1025,6 +1026,7 @@ class BaseApp(metaclass=Singleton):
 
         return next_id
 
+    @TimeIt.decorator
     def set_inactive_in_known_subs_file(self, inactive_IDs: List[int]):
         """Set workflows in the known-submissions file to the non-running state.
 
@@ -1142,6 +1144,7 @@ class BaseApp(metaclass=Singleton):
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
         variables: Optional[Dict[str, str]] = None,
+        status: Optional[bool] = True,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a file or string containing a workflow
         template parametrisation.
@@ -1178,10 +1181,17 @@ class BaseApp(metaclass=Singleton):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables
             String variables to substitute in `template_file_or_str`.
+        status
+            If True, display a live status to track workflow creation progress.
         """
 
         self.API_logger.info("make_workflow called")
 
+        if status:
+            console = rich.console.Console()
+            status = console.status("Making persistent workflow...")
+            status.start()
+
         common = {
             "path": path,
             "name": name,
@@ -1191,6 +1201,7 @@ class BaseApp(metaclass=Singleton):
             "ts_name_fmt": ts_name_fmt,
             "store_kwargs": store_kwargs,
             "variables": variables,
+            "status": status,
         }
 
         if not is_string:
@@ -1201,10 +1212,24 @@ class BaseApp(metaclass=Singleton):
             )
 
         elif template_format == "json":
-            wk = self.Workflow.from_JSON_string(JSON_str=template_file_or_str, **common)
+            try:
+                wk = self.Workflow.from_JSON_string(
+                    JSON_str=template_file_or_str, **common
+                )
+            except Exception:
+                if status:
+                    status.stop()
+                raise
 
         elif template_format == "yaml":
-            wk = self.Workflow.from_YAML_string(YAML_str=template_file_or_str, **common)
+            try:
+                wk = self.Workflow.from_YAML_string(
+                    YAML_str=template_file_or_str, **common
+                )
+            except Exception:
+                if status:
+                    status.stop()
+                raise
 
         elif not template_format:
             raise ValueError(
@@ -1217,6 +1242,10 @@ class BaseApp(metaclass=Singleton):
                 f"Template format {template_format!r} not understood. Available template "
                 f"formats are {ALL_TEMPLATE_FORMATS!r}."
             )
+
+        if status:
+            status.stop()
+
         return wk
 
     def _make_and_submit_workflow(
@@ -1237,6 +1266,8 @@ class BaseApp(metaclass=Singleton):
         add_to_known: Optional[bool] = True,
         return_idx: Optional[bool] = False,
         tasks: Optional[List[int]] = None,
+        cancel: Optional[bool] = False,
+        status: Optional[bool] = True,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1289,6 +1320,11 @@ class BaseApp(metaclass=Singleton):
         tasks
             List of task indices to include in this submission. By default all tasks are
             included.
+        cancel
+            Immediately cancel the submission. Useful for testing and benchmarking.
+        status
+            If True, display a live status to track workflow creation and submission
+            progress.
         """
 
         self.API_logger.info("make_and_submit_workflow called")
@@ -1305,6 +1341,7 @@ class BaseApp(metaclass=Singleton):
             ts_name_fmt=ts_name_fmt,
             store_kwargs=store_kwargs,
             variables=variables,
+            status=status,
         )
         return wk.submit(
             JS_parallelism=JS_parallelism,
@@ -1312,6 +1349,8 @@ class BaseApp(metaclass=Singleton):
             add_to_known=add_to_known,
             return_idx=return_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
 
     def _make_demo_workflow(
@@ -1326,6 +1365,7 @@ class BaseApp(metaclass=Singleton):
         ts_name_fmt: Optional[str] = None,
         store_kwargs: Optional[Dict] = None,
         variables: Optional[Dict[str, str]] = None,
+        status: Optional[bool] = True,
     ) -> get_app_attribute("Workflow"):
         """Generate a new {app_name} workflow from a builtin demo workflow template.
 
@@ -1359,10 +1399,17 @@ class BaseApp(metaclass=Singleton):
             Keyword arguments to pass to the store's `write_empty_workflow` method.
         variables
             String variables to substitute in the demo workflow template file.
+        status
+            If True, display a live status to track workflow creation progress.
         """
 
         self.API_logger.info("make_demo_workflow called")
 
+        if status:
+            console = rich.console.Console()
+            status = console.status("Making persistent workflow...")
+            status.start()
+
         with self.get_demo_workflow_template_file(workflow_name) as template_path:
             wk = self.Workflow.from_file(
                 template_path=template_path,
@@ -1375,7 +1422,10 @@ class BaseApp(metaclass=Singleton):
                 ts_name_fmt=ts_name_fmt,
                 store_kwargs=store_kwargs,
                 variables=variables,
+                status=status,
             )
+        if status:
+            status.stop()
         return wk
 
     def _make_and_submit_demo_workflow(
@@ -1395,6 +1445,8 @@ class BaseApp(metaclass=Singleton):
         add_to_known: Optional[bool] = True,
         return_idx: Optional[bool] = False,
         tasks: Optional[List[int]] = None,
+        cancel: Optional[bool] = False,
+        status: Optional[bool] = True,
     ) -> Dict[int, int]:
         """Generate and submit a new {app_name} workflow from a file or string containing a
         workflow template parametrisation.
@@ -1444,6 +1496,10 @@ class BaseApp(metaclass=Singleton):
         tasks
             List of task indices to include in this submission. By default all tasks are
             included.
+        cancel
+            Immediately cancel the submission. Useful for testing and benchmarking.
+        status
+            If True, display a live status to track submission progress.
         """
 
         self.API_logger.info("make_and_submit_demo_workflow called")
@@ -1466,6 +1522,8 @@ class BaseApp(metaclass=Singleton):
             add_to_known=add_to_known,
             return_idx=return_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
 
     def _submit_workflow(
@@ -1557,8 +1615,13 @@ class BaseApp(metaclass=Singleton):
         )
         return shell.get_version_info(exclude_os)
 
+    @TimeIt.decorator
     def _get_known_submissions(
-        self, max_recent: int = 3, no_update: bool = False, as_json: bool = False
+        self,
+        max_recent: int = 3,
+        no_update: bool = False,
+        as_json: bool = False,
+        status: Optional[Any] = None,
     ):
         """Retrieve information about active and recently inactive finished {app_name}
         workflows.
@@ -1583,6 +1646,8 @@ class BaseApp(metaclass=Singleton):
         inactive_IDs = []
 
         try:
+            if status:
+                status.update("Reading known submissions file...")
             known_subs = self.read_known_submissions_file()
         except FileNotFoundError:
             known_subs = []
@@ -1615,6 +1680,8 @@ class BaseApp(metaclass=Singleton):
                 out_item["deleted"] = not path_exists
                 if path_exists:
                     try:
+                        if status:
+                            status.update(f"Inspecting workflow {file_dat_i['path']!r}.")
                         wk_i = self.Workflow(file_dat_i["path"])
                     except Exception:
                         wk_i = None
@@ -1641,15 +1708,22 @@ class BaseApp(metaclass=Singleton):
                     out_item["deleted"] = True
 
                 else:
-                    sub = wk_i.submissions[file_dat_i["sub_idx"]]
-
-                    all_jobscripts = sub._submission_parts[submit_time_str]
-                    out_item.update(
-                        {
-                            "jobscripts": all_jobscripts,
-                            "submission": sub,
-                        }
-                    )
+                    if status:
+                        status.update(
+                            f"Reading workflow {file_dat_i['path']!r} submission info..."
+                        )
+                    with wk_i._store.cache_ctx():
+                        sub = wk_i.submissions[file_dat_i["sub_idx"]]
+
+                        all_jobscripts = sub._submission_parts[submit_time_str]
+                        out_item.update(
+                            {
+                                "jobscripts": all_jobscripts,
+                                "submission": sub,
+                                "sub_start_time": sub.start_time,
+                                "sub_end_time": sub.end_time,
+                            }
+                        )
                     if file_dat_i["is_active"]:
                         # check it really is active:
                         run_key = (file_dat_i["path"], file_dat_i["sub_idx"])
@@ -1695,9 +1769,7 @@ class BaseApp(metaclass=Singleton):
         out_access = sorted(
             out_access,
             key=lambda i: (
-                i["submission"].end_time
-                or i["submission"].start_time
-                or i["submit_time_obj"]
+                i["sub_end_time"] or i["sub_start_time"] or i["submit_time_obj"]
             ),
             reverse=True,
         )
@@ -1827,6 +1899,7 @@ class BaseApp(metaclass=Singleton):
             run_dat = self._get_known_submissions(
                 max_recent=max_recent,
                 no_update=no_update,
+                status=status,
             )
         except Exception:
             status.stop()

hpcflow/sdk/cli.py

@@ -25,6 +25,9 @@ from hpcflow.sdk.cli_common import (
     add_to_known_opt,
     print_idx_opt,
     tasks_opt,
+    cancel_opt,
+    submit_status_opt,
+    make_status_opt,
     zip_path_opt,
     zip_overwrite_opt,
     zip_log_opt,
@@ -71,6 +74,7 @@ def _make_API_CLI(app):
     @ts_fmt_option
     @ts_name_fmt_option
     @variables_option
+    @make_status_opt
     def make_workflow(
         template_file_or_str,
         string,
@@ -82,6 +86,7 @@ def _make_API_CLI(app):
         ts_fmt=None,
         ts_name_fmt=None,
         variables=None,
+        status=True,
     ):
         """Generate a new {app_name} workflow.
 
@@ -100,6 +105,7 @@ def _make_API_CLI(app):
             ts_fmt=ts_fmt,
             ts_name_fmt=ts_name_fmt,
             variables=dict(variables),
+            status=status,
         )
         click.echo(wk.path)
 
@@ -119,6 +125,8 @@ def _make_API_CLI(app):
     @add_to_known_opt
     @print_idx_opt
     @tasks_opt
+    @cancel_opt
+    @submit_status_opt
     def make_and_submit_workflow(
         template_file_or_str,
         string,
@@ -135,6 +143,8 @@ def _make_API_CLI(app):
         add_to_known=True,
         print_idx=False,
         tasks=None,
+        cancel=False,
+        status=True,
     ):
         """Generate and submit a new {app_name} workflow.
 
@@ -159,6 +169,8 @@ def _make_API_CLI(app):
             add_to_known=add_to_known,
             return_idx=print_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
         if print_idx:
             click.echo(out)
@@ -320,6 +332,8 @@ def _make_workflow_CLI(app):
     @add_to_known_opt
     @print_idx_opt
     @tasks_opt
+    @cancel_opt
+    @submit_status_opt
     @click.pass_context
     def submit_workflow(
         ctx,
@@ -328,6 +342,8 @@ def _make_workflow_CLI(app):
         add_to_known=True,
         print_idx=False,
         tasks=None,
+        cancel=False,
+        status=True,
     ):
         """Submit the workflow."""
         out = ctx.obj["workflow"].submit(
@@ -336,6 +352,8 @@ def _make_workflow_CLI(app):
             add_to_known=add_to_known,
             return_idx=print_idx,
             tasks=tasks,
+            cancel=cancel,
+            status=status,
         )
         if print_idx:
             click.echo(out)

hpcflow/sdk/cli_common.py

@@ -109,6 +109,22 @@ tasks_opt = click.option(
     ),
     callback=sub_tasks_callback,
 )
+cancel_opt = click.option(
+    "--cancel",
+    help="Immediately cancel the submission. Useful for testing and benchmarking.",
+    is_flag=True,
+    default=False,
+)
+submit_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track submission progress.",
+    default=True,
+)
+make_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track workflow creation progress.",
+    default=True,
+)
 zip_path_opt = click.option(
     "--path",
     default=".",

hpcflow/sdk/config/config.py

@@ -65,7 +65,6 @@ DEFAULT_CONFIG = {
     "invocation": {"environment_setup": None, "match": {}},
     "config": {
         "machine": socket.gethostname(),
-        "telemetry": True,
         "log_file_path": "logs/<<app_name>>_v<<app_version>>.log",
         "environment_sources": [],
         "task_schema_sources": [],
@@ -85,9 +84,6 @@ class ConfigOptions:
 
     default_directory: Union[Path, str]
     directory_env_var: str
-    sentry_DSN: str
-    sentry_traces_sample_rate: float
-    sentry_env: str
     default_config: Optional[Dict] = field(
         default_factory=lambda: deepcopy(DEFAULT_CONFIG)
     )

hpcflow/sdk/core/actions.py

@@ -4,7 +4,6 @@ from dataclasses import dataclass
 from datetime import datetime
 import enum
 import json
-import h5py
 from pathlib import Path
 import re
 from textwrap import indent, dedent
@@ -30,6 +29,7 @@ from hpcflow.sdk.core.utils import (
     swap_nested_dict_keys,
 )
 from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.core.run_dir_files import RunDirAppFiles
 
 
 ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
@@ -252,13 +252,16 @@ class ElementActionRun:
     @property
     def snapshot_start(self):
         if self._ss_start_obj is None and self._snapshot_start:
-            self._ss_start_obj = JSONLikeDirSnapShot(**self._snapshot_start)
+            self._ss_start_obj = JSONLikeDirSnapShot(
+                root_path=".",
+                **self._snapshot_start,
+            )
         return self._ss_start_obj
 
     @property
     def snapshot_end(self):
         if self._ss_end_obj is None and self._snapshot_end:
-            self._ss_end_obj = JSONLikeDirSnapShot(**self._snapshot_end)
+            self._ss_end_obj = JSONLikeDirSnapShot(root_path=".", **self._snapshot_end)
         return self._ss_end_obj
 
     @property
@@ -332,6 +335,7 @@ class ElementActionRun:
             run_idx=self.index,
         )
 
+    @TimeIt.decorator
     def get_parameter_sources(
         self,
         path: str = None,
@@ -364,6 +368,7 @@ class ElementActionRun:
             raise_on_unset=raise_on_unset,
         )
 
+    @TimeIt.decorator
     def get_EAR_dependencies(self, as_objects=False):
         """Get EARs that this EAR depends on."""
 
@@ -435,6 +440,7 @@ class ElementActionRun:
         return self._outputs
 
     @property
+    @TimeIt.decorator
     def resources(self):
         if not self._resources:
             self._resources = self.app.ElementResources(**self.get_resources())
@@ -452,6 +458,7 @@ class ElementActionRun:
             self._output_files = self.app.ElementOutputFiles(element_action_run=self)
         return self._output_files
 
+    @TimeIt.decorator
     def get_resources(self):
         """Resolve specific resources for this EAR, considering all applicable scopes and
         template-level resources."""
@@ -592,6 +599,8 @@ class ElementActionRun:
         return outputs
 
     def write_source(self, js_idx: int, js_act_idx: int):
+        import h5py
+
         for fmt, ins in self.action.script_data_in_grouped.items():
             if fmt == "json":
                 in_vals = self.get_input_values(inputs=ins, label_dict=False)
@@ -627,6 +636,8 @@ class ElementActionRun:
     def _param_save(self, js_idx: int, js_act_idx: int):
         """Save script-generated parameters that are stored within the supported script
         data output formats (HDF5, JSON, etc)."""
+        import h5py
+
         for fmt in self.action.script_data_out_grouped:
             if fmt == "json":
                 load_path = self.action.get_param_load_file_path_JSON(js_idx, js_act_idx)
@@ -994,6 +1005,7 @@ class ActionRule(JSONLike):
             return True
         return False
 
+    @TimeIt.decorator
     def test(self, element_iteration: app.ElementIteration) -> bool:
         return self.rule.test(element_like=element_iteration, action=self.action)
 
@@ -1453,11 +1465,11 @@ class Action(JSONLike):
 
     @staticmethod
     def get_param_dump_file_stem(js_idx: int, js_act_idx: int):
-        return f"js_{js_idx}_act_{js_act_idx}_inputs"
+        return RunDirAppFiles.get_run_param_dump_file_prefix(js_idx, js_act_idx)
 
     @staticmethod
     def get_param_load_file_stem(js_idx: int, js_act_idx: int):
-        return f"js_{js_idx}_act_{js_act_idx}_outputs"
+        return RunDirAppFiles.get_run_param_load_file_prefix(js_idx, js_act_idx)
 
     def get_param_dump_file_path_JSON(self, js_idx: int, js_act_idx: int):
         return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".json")
@@ -1896,6 +1908,7 @@ class Action(JSONLike):
             if typ in (OFP.inputs or []):
                 return True
 
+    @TimeIt.decorator
     def test_rules(self, element_iter) -> List[bool]:
         """Test all rules against the specified element iteration."""
         return [i.test(element_iteration=element_iter) for i in self.rules]
@@ -1940,7 +1953,7 @@ class Action(JSONLike):
                 """\
                     import {app_module} as app
                     app.load_config(
-                        log_file_path=Path("{app_package_name}.log").resolve(),
+                        log_file_path=Path("{run_log_file}").resolve(),
                         config_dir=r"{cfg_dir}",
                         config_key=r"{cfg_invoc_key}",
                     )
@@ -1949,7 +1962,7 @@ class Action(JSONLike):
                     EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
                 """
             ).format(
-                app_package_name=self.app.package_name,
+                run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
                 app_module=self.app.module,
                 cfg_dir=self.app.config.config_directory,
                 cfg_invoc_key=self.app.config.config_key,

hpcflow/sdk/core/command_files.py

@@ -159,7 +159,7 @@ class InputFileGenerator(JSONLike):
                 from pathlib import Path
                 import {app_module} as app
                 app.load_config(
-                    log_file_path=Path("{app_package_name}.log").resolve(),
+                    log_file_path=Path("{run_log_file}").resolve(),
                     config_dir=r"{cfg_dir}",
                     config_key=r"{cfg_invoc_key}",
                 )
@@ -171,7 +171,7 @@ class InputFileGenerator(JSONLike):
         """
         )
         main_block = main_block.format(
-            app_package_name=self.app.package_name,
+            run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
             app_module=self.app.module,
             cfg_dir=self.app.config.config_directory,
             cfg_invoc_key=self.app.config.config_key,
@@ -303,7 +303,7 @@ class OutputFileParser(JSONLike):
                 from pathlib import Path
                 import {app_module} as app
                 app.load_config(
-                    log_file_path=Path("{app_package_name}.log").resolve(),
+                    log_file_path=Path("{run_log_file}").resolve(),
                     config_dir=r"{cfg_dir}",
                     config_key=r"{cfg_invoc_key}",
                 )
@@ -321,7 +321,7 @@ class OutputFileParser(JSONLike):
         """
         )
         main_block = main_block.format(
-            app_package_name=self.app.package_name,
+            run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
             app_module=self.app.module,
             cfg_dir=self.app.config.config_directory,
             cfg_invoc_key=self.app.config.config_key,