hpcflow-new2 0.2.0a158__py3-none-any.whl → 0.2.0a160__py3-none-any.whl

hpcflow/sdk/core/element.py

@@ -13,9 +13,11 @@ from hpcflow.sdk.core.json_like import ChildObjectSpec, JSONLike
 from hpcflow.sdk.core.parallel import ParallelMode
 from hpcflow.sdk.core.utils import (
     check_valid_py_identifier,
+    dict_values_process_flat,
     get_enum_by_name_or_val,
     split_param_label,
 )
+from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.shells import get_shell
 
 
@@ -500,6 +502,7 @@ class ElementIteration:
             if i.startswith(prefix)
         )
 
+    @TimeIt.decorator
     def get_data_idx(
         self,
         path: str = None,
@@ -538,6 +541,7 @@ class ElementIteration:
 
         return copy.deepcopy(data_idx)
 
+    @TimeIt.decorator
     def get_parameter_sources(
         self,
         path: str = None,
@@ -555,24 +559,16 @@ class ElementIteration:
             ID.
         """
         data_idx = self.get_data_idx(path, action_idx, run_idx)
-        out = {}
-        for k, v in data_idx.items():
-            is_multi = False
-            if isinstance(v, list):
-                is_multi = True
-            else:
-                v = [v]
 
-            sources_k = []
-            for dat_idx_i in v:
-                src = self.workflow.get_parameter_source(dat_idx_i)
-                sources_k.append(src)
-
-            if not is_multi:
-                sources_k = src
-
-            out[k] = sources_k
+        # the value associated with `repeats.*` is the repeats index, not a parameter ID:
+        for k in list(data_idx.keys()):
+            if k.startswith("repeats."):
+                data_idx.pop(k)
 
+        out = dict_values_process_flat(
+            data_idx,
+            callable=self.workflow.get_parameter_sources,
+        )
         task_key = "task_insert_ID"
 
         if use_task_index:
@@ -631,6 +627,7 @@ class ElementIteration:
 
         return out
 
+    @TimeIt.decorator
     def get(
         self,
         path: str = None,
@@ -856,6 +853,7 @@ class ElementIteration:
             out[res_i.scope.to_string()] = res_i._get_value()
         return out
 
+    @TimeIt.decorator
     def get_resources(self, action: app.Action, set_defaults: bool = False) -> Dict:
         """Resolve specific resources for the specified action of this iteration,
         considering all applicable scopes.
@@ -998,6 +996,7 @@ class Element:
         return self._iteration_IDs
 
     @property
+    @TimeIt.decorator
     def iterations(self) -> Dict[app.ElementAction]:
         # TODO: fix this
         if self._iteration_objs is None:

hpcflow/sdk/core/rule.py

@@ -7,6 +7,7 @@ from valida.rules import Rule as ValidaRule
 from hpcflow.sdk import app
 from hpcflow.sdk.core.json_like import JSONLike
 from hpcflow.sdk.core.utils import get_in_container
+from hpcflow.sdk.log import TimeIt
 
 
 class Rule(JSONLike):
@@ -68,6 +69,7 @@ class Rule(JSONLike):
         else:
             return False
 
+    @TimeIt.decorator
     def test(
         self,
         element_like: Union[app.ElementIteration, app.ElementActionRun],

hpcflow/sdk/core/run_dir_files.py

@@ -0,0 +1,63 @@
+import re
+from hpcflow.sdk.core.utils import JSONLikeDirSnapShot
+
+
+class RunDirAppFiles:
+    """A class to encapsulate the naming/recognition of app-created files within run
+    directories."""
+
+    _app_attr = "app"
+
+    CMD_FILES_RE_PATTERN = r"js_\d+_act_\d+\.?\w*"
+
+    @classmethod
+    def get_log_file_name(cls):
+        """File name for the app log file."""
+        return f"{cls.app.package_name}.log"
+
+    @classmethod
+    def get_std_file_name(cls):
+        """File name for stdout and stderr streams from the app."""
+        return f"{cls.app.package_name}_std.txt"
+
+    @staticmethod
+    def get_run_file_prefix(js_idx: int, js_action_idx: int):
+        return f"js_{js_idx}_act_{js_action_idx}"
+
+    @classmethod
+    def get_commands_file_name(cls, js_idx: int, js_action_idx: int, shell):
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + shell.JS_EXT
+
+    @classmethod
+    def get_run_param_dump_file_prefix(cls, js_idx: int, js_action_idx: int):
+        """Get the prefix to a file in the run directory that the app will dump parameter
+        data to."""
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + "_inputs"
+
+    @classmethod
+    def get_run_param_load_file_prefix(cls, js_idx: int, js_action_idx: int):
+        """Get the prefix to a file in the run directory that the app will load parameter
+        data from."""
+        return cls.get_run_file_prefix(js_idx, js_action_idx) + "_outputs"
+
+    @classmethod
+    def take_snapshot(cls):
+        """Take a JSONLikeDirSnapShot, and process to ignore files created by the app.
+
+        This includes command files that are invoked by jobscripts, the app log file, and
+        the app standard out/error file.
+
+        """
+        snapshot = JSONLikeDirSnapShot()
+        snapshot.take(".")
+        ss_js = snapshot.to_json_like()
+        ss_js.pop("root_path")  # always the current working directory of the run
+        for k in list(ss_js["data"].keys()):
+            if (
+                k == cls.get_log_file_name()
+                or k == cls.get_std_file_name()
+                or re.match(cls.CMD_FILES_RE_PATTERN, k)
+            ):
+                ss_js["data"].pop(k)
+
+        return ss_js

hpcflow/sdk/core/task.py

@@ -1283,6 +1283,7 @@ class WorkflowTask:
         return self.template.num_element_sets
 
     @property
+    @TimeIt.decorator
     def elements(self):
         if self._elements is None:
             self._elements = self.app.Elements(self)
@@ -1828,6 +1829,7 @@ class WorkflowTask:
         param_src_updates = {}
 
         count = 0
+        # TODO: generator is an IO op here, can be pre-calculated/cached?
         for act_idx, action in self.template.all_schema_actions():
             log_common = (
                 f"for action {act_idx} of element iteration {element_iter.index} of "
@@ -2604,41 +2606,38 @@ class Elements:
     def task(self):
         return self._task
 
-    def _get_selection(self, selection):
+    @TimeIt.decorator
+    def _get_selection(self, selection: Union[int, slice, List[int]]) -> List[int]:
+        """Normalise an element selection into a list of element indices."""
         if isinstance(selection, int):
-            start, stop, step = selection, selection + 1, 1
+            lst = [selection]
 
         elif isinstance(selection, slice):
-            start, stop, step = selection.start, selection.stop, selection.step
-            stop = self.task.num_elements if stop is None else stop
-            start = start or 0
-            step = 1 if step is None else step
+            lst = list(range(*selection.indices(self.task.num_elements)))
 
+        elif isinstance(selection, list):
+            lst = selection
         else:
             raise RuntimeError(
-                f"{self.__class__.__name__} selection must be an `int` or a `slice` "
-                f"object, but received type {type(selection)}."
+                f"{self.__class__.__name__} selection must be an `int`, `slice` object, "
+                f"or list of `int`s, but received type {type(selection)}."
             )
-
-        selection = slice(start, stop, step)
-        length = len(range(*selection.indices(self.task.num_elements)))
-
-        return selection, length
+        return lst
 
     def __len__(self):
         return self.task.num_elements
 
     def __iter__(self):
-        all_elems = self.task.workflow.get_task_elements(self.task, slice(None))
-        for i in all_elems:
+        for i in self.task.workflow.get_task_elements(self.task):
             yield i
 
+    @TimeIt.decorator
     def __getitem__(
         self,
-        selection: Union[int, slice],
+        selection: Union[int, slice, List[int]],
     ) -> Union[app.Element, List[app.Element]]:
-        sel_normed, _ = self._get_selection(selection)
-        elements = self.task.workflow.get_task_elements(self.task, sel_normed)
+        idx_lst = self._get_selection(selection)
+        elements = self.task.workflow.get_task_elements(self.task, idx_lst)
 
         if isinstance(selection, int):
             return elements[0]
@@ -2657,34 +2656,34 @@ class Parameters:
     raise_on_unset: Optional[bool] = False
     default: Optional[Any] = None
 
-    def _get_selection(self, selection):
+    @TimeIt.decorator
+    def _get_selection(self, selection: Union[int, slice, List[int]]) -> List[int]:
+        """Normalise an element selection into a list of element indices."""
         if isinstance(selection, int):
-            start, stop, step = selection, selection + 1, 1
+            lst = [selection]
 
         elif isinstance(selection, slice):
-            start, stop, step = selection.start, selection.stop, selection.step
-            stop = self.task.num_elements if stop is None else stop
-            start = start or 0
-            step = 1 if step is None else step
+            lst = list(range(*selection.indices(self.task.num_elements)))
 
+        elif isinstance(selection, list):
+            lst = selection
         else:
             raise RuntimeError(
-                f"{self.__class__.__name__} selection must be an `int` or a `slice` "
-                f"object, but received type {type(selection)!r}."
+                f"{self.__class__.__name__} selection must be an `int`, `slice` object, "
+                f"or list of `int`s, but received type {type(selection)}."
             )
-
-        selection = slice(start, stop, step)
-        length = len(range(*selection.indices(self.task.num_elements)))
-
-        return selection, length
+        return lst
 
     def __iter__(self):
         for i in self.__getitem__(slice(None)):
             yield i
 
-    def __getitem__(self, selection: Union[int, slice]) -> Union[Any, List[Any]]:
-        selection, length = self._get_selection(selection)
-        elements = self.task.workflow.get_task_elements(self.task, selection)
+    def __getitem__(
+        self,
+        selection: Union[int, slice, List[int]],
+    ) -> Union[Any, List[Any]]:
+        idx_lst = self._get_selection(selection)
+        elements = self.task.workflow.get_task_elements(self.task, idx_lst)
         if self.return_element_parameters:
             params = [
                 self._app.ElementParameter(
@@ -2706,7 +2705,7 @@ class Parameters:
                 for i in elements
             ]
 
-        if length == 1:
+        if isinstance(selection, int):
             return params[0]
         else:
             return params

hpcflow/sdk/core/utils.py

@@ -20,7 +20,6 @@ import fsspec
 import numpy as np
 
 from ruamel.yaml import YAML
-import sentry_sdk
 from watchdog.utils.dirsnapshot import DirectorySnapshot
 
 from hpcflow.sdk.core.errors import (
@@ -347,20 +346,6 @@ class Singleton(type):
         return cls._instances[cls]
 
 
-@contextlib.contextmanager
-def sentry_wrap(name, transaction_op=None, span_op=None):
-    if not transaction_op:
-        transaction_op = name
-    if not span_op:
-        span_op = name
-    try:
-        with sentry_sdk.start_transaction(op=transaction_op, name=name):
-            with sentry_sdk.start_span(op=span_op) as span:
-                yield span
-    finally:
-        sentry_sdk.flush()  # avoid queue message on stdout
-
-
 def capitalise_first_letter(chars):
     return chars[0].upper() + chars[1:]
 
@@ -849,3 +834,40 @@ def linspace_rect(
 
     rect = np.hstack(stacked)
     return rect
+
+
+def dict_values_process_flat(d, callable):
+    """
+    Return a copy of a dict, where the values are processed by a callable that is to
+    be called only once, and where the values may be single items or lists of items.
+
+    Examples
+    --------
+    d = {'a': 0, 'b': [1, 2], 'c': 5}
+    >>> dict_values_process_flat(d, callable=lambda x: [i + 1 for i in x])
+    {'a': 1, 'b': [2, 3], 'c': 6}
+
+    """
+    flat = []  # values of `d`, flattened
+    is_multi = []  # whether a list, and the number of items to process
+    for i in d.values():
+        try:
+            flat.extend(i)
+            is_multi.append((True, len(i)))
+        except TypeError:
+            flat.append(i)
+            is_multi.append((False, 1))
+
+    processed = callable(flat)
+
+    out = {}
+    for idx_i, (m, k) in enumerate(zip(is_multi, d.keys())):
+
+        start_idx = sum(i[1] for i in is_multi[:idx_i])
+        end_idx = start_idx + m[1]
+        proc_idx_k = processed[slice(start_idx, end_idx)]
+        if not m[0]:
+            proc_idx_k = proc_idx_k[0]
+        out[k] = proc_idx_k
+
+    return out