hpcflow-new2 0.2.0a159__py3-none-any.whl → 0.2.0a161__py3-none-any.whl

hpcflow/sdk/core/actions.py

@@ -29,6 +29,7 @@ from hpcflow.sdk.core.utils import (
     swap_nested_dict_keys,
 )
 from hpcflow.sdk.log import TimeIt
+from hpcflow.sdk.core.run_dir_files import RunDirAppFiles
 
 
 ACTION_SCOPE_REGEX = r"(\w*)(?:\[(.*)\])?"
@@ -251,13 +252,16 @@ class ElementActionRun:
     @property
     def snapshot_start(self):
         if self._ss_start_obj is None and self._snapshot_start:
-            self._ss_start_obj = JSONLikeDirSnapShot(**self._snapshot_start)
+            self._ss_start_obj = JSONLikeDirSnapShot(
+                root_path=".",
+                **self._snapshot_start,
+            )
         return self._ss_start_obj
 
     @property
     def snapshot_end(self):
         if self._ss_end_obj is None and self._snapshot_end:
-            self._ss_end_obj = JSONLikeDirSnapShot(**self._snapshot_end)
+            self._ss_end_obj = JSONLikeDirSnapShot(root_path=".", **self._snapshot_end)
         return self._ss_end_obj
 
     @property
@@ -331,6 +335,7 @@ class ElementActionRun:
             run_idx=self.index,
         )
 
+    @TimeIt.decorator
     def get_parameter_sources(
         self,
         path: str = None,
@@ -363,6 +368,7 @@ class ElementActionRun:
             raise_on_unset=raise_on_unset,
         )
 
+    @TimeIt.decorator
     def get_EAR_dependencies(self, as_objects=False):
         """Get EARs that this EAR depends on."""
 
@@ -434,6 +440,7 @@ class ElementActionRun:
         return self._outputs
 
     @property
+    @TimeIt.decorator
     def resources(self):
         if not self._resources:
             self._resources = self.app.ElementResources(**self.get_resources())
@@ -451,6 +458,7 @@ class ElementActionRun:
             self._output_files = self.app.ElementOutputFiles(element_action_run=self)
         return self._output_files
 
+    @TimeIt.decorator
     def get_resources(self):
         """Resolve specific resources for this EAR, considering all applicable scopes and
         template-level resources."""
@@ -1457,11 +1465,11 @@ class Action(JSONLike):
 
     @staticmethod
     def get_param_dump_file_stem(js_idx: int, js_act_idx: int):
-        return f"js_{js_idx}_act_{js_act_idx}_inputs"
+        return RunDirAppFiles.get_run_param_dump_file_prefix(js_idx, js_act_idx)
 
     @staticmethod
     def get_param_load_file_stem(js_idx: int, js_act_idx: int):
-        return f"js_{js_idx}_act_{js_act_idx}_outputs"
+        return RunDirAppFiles.get_run_param_load_file_prefix(js_idx, js_act_idx)
 
     def get_param_dump_file_path_JSON(self, js_idx: int, js_act_idx: int):
         return Path(self.get_param_dump_file_stem(js_idx, js_act_idx) + ".json")
@@ -1945,7 +1953,7 @@ class Action(JSONLike):
                 """\
                     import {app_module} as app
                     app.load_config(
-                        log_file_path=Path("{app_package_name}.log").resolve(),
+                        log_file_path=Path("{run_log_file}").resolve(),
                         config_dir=r"{cfg_dir}",
                         config_key=r"{cfg_invoc_key}",
                     )
@@ -1954,7 +1962,7 @@ class Action(JSONLike):
                     EAR = wk.get_EARs_from_IDs([EAR_ID])[0]
                 """
             ).format(
-                app_package_name=self.app.package_name,
+                run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
                 app_module=self.app.module,
                 cfg_dir=self.app.config.config_directory,
                 cfg_invoc_key=self.app.config.config_key,

hpcflow/sdk/core/command_files.py

@@ -144,10 +144,9 @@ class InputFileGenerator(JSONLike):
             self.app.ActionRule.check_missing(f"input_files.{self.input_file.label}")
         ] + self.rules
 
-    def compose_source(self, action) -> str:
+    def compose_source(self, snip_path) -> str:
         """Generate the file contents of this input file generator source."""
 
-        snip_path = action.get_snippet_script_path(self.script)
         script_main_func = snip_path.stem
         with snip_path.open("rt") as fp:
             script_str = fp.read()
@@ -159,7 +158,7 @@ class InputFileGenerator(JSONLike):
                 from pathlib import Path
                 import {app_module} as app
                 app.load_config(
-                    log_file_path=Path("{app_package_name}.log").resolve(),
+                    log_file_path=Path("{run_log_file}").resolve(),
                     config_dir=r"{cfg_dir}",
                     config_key=r"{cfg_invoc_key}",
                 )
@@ -171,7 +170,7 @@ class InputFileGenerator(JSONLike):
         """
         )
         main_block = main_block.format(
-            app_package_name=self.app.package_name,
+            run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
             app_module=self.app.module,
             cfg_dir=self.app.config.config_directory,
             cfg_invoc_key=self.app.config.config_key,
@@ -190,9 +189,14 @@ class InputFileGenerator(JSONLike):
         return out
 
     def write_source(self, action):
-        script_path = action.get_script_name(self.script)
-        with Path(script_path).open("wt", newline="\n") as fp:
-            fp.write(self.compose_source(action))
+
+        # write the script if it is specified as a snippet script, otherwise we assume
+        # the script already exists in the working directory:
+        snip_path = action.get_snippet_script_path(self.script)
+        if snip_path:
+            source_str = self.compose_source(snip_path)
+            with Path(snip_path.name).open("wt", newline="\n") as fp:
+                fp.write(source_str)
 
 
 @dataclass
@@ -284,14 +288,13 @@ class OutputFileParser(JSONLike):
             for i in self.output_files
         ] + self.rules
 
-    def compose_source(self, action) -> str:
+    def compose_source(self, snip_path) -> str:
         """Generate the file contents of this output file parser source."""
 
         if self.output is None:
             # might be used just for saving files:
             return
 
-        snip_path = action.get_snippet_script_path(self.script)
         script_main_func = snip_path.stem
         with snip_path.open("rt") as fp:
             script_str = fp.read()
@@ -303,7 +306,7 @@ class OutputFileParser(JSONLike):
                 from pathlib import Path
                 import {app_module} as app
                 app.load_config(
-                    log_file_path=Path("{app_package_name}.log").resolve(),
+                    log_file_path=Path("{run_log_file}").resolve(),
                     config_dir=r"{cfg_dir}",
                     config_key=r"{cfg_invoc_key}",
                 )
@@ -321,7 +324,7 @@ class OutputFileParser(JSONLike):
         """
         )
         main_block = main_block.format(
-            app_package_name=self.app.package_name,
+            run_log_file=self.app.RunDirAppFiles.get_log_file_name(),
             app_module=self.app.module,
             cfg_dir=self.app.config.config_directory,
             cfg_invoc_key=self.app.config.config_key,
@@ -343,9 +346,14 @@ class OutputFileParser(JSONLike):
         if self.output is None:
             # might be used just for saving files:
             return
-        script_path = action.get_script_name(self.script)
-        with Path(script_path).open("wt", newline="\n") as fp:
-            fp.write(self.compose_source(action))
+
+        # write the script if it is specified as a snippet script, otherwise we assume
+        # the script already exists in the working directory:
+        snip_path = action.get_snippet_script_path(self.script)
+        if snip_path:
+            source_str = self.compose_source(snip_path)
+            with Path(snip_path.name).open("wt", newline="\n") as fp:
+                fp.write(source_str)
 
 
 class _FileContentsSpecifier(JSONLike):

hpcflow/sdk/core/element.py

@@ -13,9 +13,11 @@ from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from hpcflow.sdk.core.parallel import ParallelMode
 from hpcflow.sdk.core.utils import (
     check_valid_py_identifier,
+    dict_values_process_flat,
     get_enum_by_name_or_val,
     split_param_label,
 )
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.shells import get_shell
 
 
@@ -500,6 +502,7 @@ class ElementIteration:
             if i.startswith(prefix)
         )
 
+    @TimeIt.decorator
     def get_data_idx(
         self,
         path: str = None,
@@ -538,6 +541,7 @@ class ElementIteration:
 
         return copy.deepcopy(data_idx)
 
+    @TimeIt.decorator
     def get_parameter_sources(
         self,
         path: str = None,
@@ -555,24 +559,16 @@ class ElementIteration:
             ID.
         """
         data_idx = self.get_data_idx(path, action_idx, run_idx)
-        out = {}
-        for k, v in data_idx.items():
-            is_multi = False
-            if isinstance(v, list):
-                is_multi = True
-            else:
-                v = [v]
 
-            sources_k = []
-            for dat_idx_i in v:
-                src = self.workflow.get_parameter_source(dat_idx_i)
-                sources_k.append(src)
-
-            if not is_multi:
-                sources_k = src
-
-            out[k] = sources_k
+        # the value associated with `repeats.*` is the repeats index, not a parameter ID:
+        for k in list(data_idx.keys()):
+            if k.startswith("repeats."):
+                data_idx.pop(k)
 
+        out = dict_values_process_flat(
+            data_idx,
+            callable=self.workflow.get_parameter_sources,
+        )
         task_key = "task_insert_ID"
 
         if use_task_index:
@@ -631,6 +627,7 @@ class ElementIteration:
 
         return out
 
+    @TimeIt.decorator
     def get(
         self,
         path: str = None,
@@ -856,6 +853,7 @@ class ElementIteration:
             out[res_i.scope.to_string()] = res_i._get_value()
         return out
 
+    @TimeIt.decorator
     def get_resources(self, action: app.Action, set_defaults: bool = False) -> Dict:
         """Resolve specific resources for the specified action of this iteration,
         considering all applicable scopes.
@@ -998,6 +996,7 @@ class Element:
         return self._iteration_IDs
 
     @property
+    @TimeIt.decorator
     def iterations(self) -> Dict[app.ElementAction]:
         # TODO: fix this
         if self._iteration_objs is None:

hpcflow/sdk/core/run_dir_files.py

@@ -0,0 +1,63 @@
+import re
+from hpcflow.sdk.core.utils import JSONLikeDirSnapShot
+
+
+class RunDirAppFiles:
+    """A class to encapsulate the naming/recognition of app-created files within run
+    directories."""
+
+    _app_attr = "app"
+
+    CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*"
+
+    @classmethod
+    def get_log_file_name(cls):
+        """File name for the app log file."""
+        return f"{cls.app.package_name}.log"
+
+    @classmethod
+    def get_std_file_name(cls):
+        """File name for stdout and stderr streams from the app."""
+        return f"{cls.app.package_name}_std.txt"
+
+    @staticmethod
+    def get_run_file_prefix(js_idx: int, js_action_idx: int):
+        return f"js_{js_idx}_act_{js_action_idx}"
+
+    @classmethod
+    def get_commands_file_name(cls, js_idx: int, js_action_idx: int, shell):
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + shell.JS_EXT
+
+    @classmethod
+    def get_run_param_dump_file_prefix(cls, js_idx: int, js_action_idx: int):
+        """Get the prefix to a file in the run directory that the app will dump parameter
+        data to."""
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + "_inputs"
+
+    @classmethod
+    def get_run_param_load_file_prefix(cls, js_idx: int, js_action_idx: int):
+        """Get the prefix to a file in the run directory that the app will load parameter
+        data from."""
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + "_outputs"
+
+    @classmethod
+    def take_snapshot(cls):
+        """Take a JSONLikeDirSnapShot, and process to ignore files created by the app.
+
+        This includes command files that are invoked by jobscripts, the app log file, and
+        the app standard out/error file.
+
+        """
+        snapshot = JSONLikeDirSnapShot()
+        snapshot.take(".")
+        ss_js = snapshot.to_json_like()
+        ss_js.pop("root_path")  # always the current working directory of the run
+        for k in list(ss_js["data"].keys()):
+            if (
+                k == cls.get_log_file_name()
+                or k == cls.get_std_file_name()
+                or re.match(cls.CMD_FILES_RE_PATTERN, k)
+            ):
+                ss_js["data"].pop(k)
+
+        return ss_js

hpcflow/sdk/core/task.py

@@ -1283,6 +1283,7 @@ class WorkflowTask:
         return self.template.num_element_sets
 
     @property
+    @TimeIt.decorator
     def elements(self):
         if self._elements is None:
             self._elements = self.app.Elements(self)
@@ -1828,6 +1829,7 @@ class WorkflowTask:
         param_src_updates = {}
 
         count = 0
+        # TODO: generator is an IO op here, can be pre-calculated/cached?
         for act_idx, action in self.template.all_schema_actions():
             log_common = (
                 f"for action {act_idx} of element iteration {element_iter.index} of "
@@ -2604,41 +2606,38 @@ class Elements:
     def task(self):
         return self._task
 
-    def _get_selection(self, selection):
+    @TimeIt.decorator
+    def _get_selection(self, selection: Union[int, slice, List[int]]) -> List[int]:
+        """Normalise an element selection into a list of element indices."""
         if isinstance(selection, int):
-            start, stop, step = selection, selection + 1, 1
+            lst = [selection]
 
         elif isinstance(selection, slice):
-            start, stop, step = selection.start, selection.stop, selection.step
-            stop = self.task.num_elements if stop is None else stop
-            start = start or 0
-            step = 1 if step is None else step
+            lst = list(range(*selection.indices(self.task.num_elements)))
 
+        elif isinstance(selection, list):
+            lst = selection
         else:
             raise RuntimeError(
-                f"{self.__class__.__name__} selection must be an `int` or a `slice` "
-                f"object, but received type {type(selection)}."
+                f"{self.__class__.__name__} selection must be an `int`, `slice` object, "
+                f"or list of `int`s, but received type {type(selection)}."
             )
-
-        selection = slice(start, stop, step)
-        length = len(range(*selection.indices(self.task.num_elements)))
-
-        return selection, length
+        return lst
 
     def __len__(self):
         return self.task.num_elements
 
     def __iter__(self):
-        all_elems = self.task.workflow.get_task_elements(self.task, slice(None))
-        for i in all_elems:
+        for i in self.task.workflow.get_task_elements(self.task):
             yield i
 
+    @TimeIt.decorator
     def __getitem__(
         self,
-        selection: Union[int, slice],
+        selection: Union[int, slice, List[int]],
     ) -> Union[app.Element, List[app.Element]]:
-        sel_normed, _ = self._get_selection(selection)
-        elements = self.task.workflow.get_task_elements(self.task, sel_normed)
+        idx_lst = self._get_selection(selection)
+        elements = self.task.workflow.get_task_elements(self.task, idx_lst)
 
         if isinstance(selection, int):
             return elements[0]
@@ -2657,34 +2656,34 @@ class Parameters:
     raise_on_unset: Optional[bool] = False
     default: Optional[Any] = None
 
-    def _get_selection(self, selection):
+    @TimeIt.decorator
+    def _get_selection(self, selection: Union[int, slice, List[int]]) -> List[int]:
+        """Normalise an element selection into a list of element indices."""
         if isinstance(selection, int):
-            start, stop, step = selection, selection + 1, 1
+            lst = [selection]
 
         elif isinstance(selection, slice):
-            start, stop, step = selection.start, selection.stop, selection.step
-            stop = self.task.num_elements if stop is None else stop
-            start = start or 0
-            step = 1 if step is None else step
+            lst = list(range(*selection.indices(self.task.num_elements)))
 
+        elif isinstance(selection, list):
+            lst = selection
         else:
             raise RuntimeError(
-                f"{self.__class__.__name__} selection must be an `int` or a `slice` "
-                f"object, but received type {type(selection)!r}."
+                f"{self.__class__.__name__} selection must be an `int`, `slice` object, "
+                f"or list of `int`s, but received type {type(selection)}."
             )
-
-        selection = slice(start, stop, step)
-        length = len(range(*selection.indices(self.task.num_elements)))
-
-        return selection, length
+        return lst
 
     def __iter__(self):
         for i in self.__getitem__(slice(None)):
             yield i
 
-    def __getitem__(self, selection: Union[int, slice]) -> Union[Any, List[Any]]:
-        selection, length = self._get_selection(selection)
-        elements = self.task.workflow.get_task_elements(self.task, selection)
+    def __getitem__(
+        self,
+        selection: Union[int, slice, List[int]],
+    ) -> Union[Any, List[Any]]:
+        idx_lst = self._get_selection(selection)
+        elements = self.task.workflow.get_task_elements(self.task, idx_lst)
         if self.return_element_parameters:
             params = [
                 self._app.ElementParameter(
@@ -2706,7 +2705,7 @@ class Parameters:
                 for i in elements
             ]
 
-        if length == 1:
+        if isinstance(selection, int):
             return params[0]
         else:
             return params

hpcflow/sdk/core/utils.py

@@ -834,3 +834,40 @@ def linspace_rect(
 
     rect = np.hstack(stacked)
     return rect
+
+
+def dict_values_process_flat(d, callable):
+    """
+    Return a copy of a dict, where the values are processed by a callable that is to
+    be called only once, and where the values may be single items or lists of items.
+
+    Examples
+    --------
+    d = {'a': 0, 'b': [1, 2], 'c': 5}
+    >>> dict_values_process_flat(d, callable=lambda x: [i + 1 for i in x])
+    {'a': 1, 'b': [2, 3], 'c': 6}
+
+    """
+    flat = []  # values of `d`, flattened
+    is_multi = []  # whether a list, and the number of items to process
+    for i in d.values():
+        try:
+            flat.extend(i)
+            is_multi.append((True, len(i)))
+        except TypeError:
+            flat.append(i)
+            is_multi.append((False, 1))
+
+    processed = callable(flat)
+
+    out = {}
+    for idx_i, (m, k) in enumerate(zip(is_multi, d.keys())):
+
+        start_idx = sum(i[1] for i in is_multi[:idx_i])
+        end_idx = start_idx + m[1]
+        proc_idx_k = processed[slice(start_idx, end_idx)]
+        if not m[0]:
+            proc_idx_k = proc_idx_k[0]
+        out[k] = proc_idx_k
+
+    return out