hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl

hpcflow/sdk/core/json_like.py

@@ -1,3 +1,7 @@
+"""
+Serialization and deserialization mechanism intended to map between a complex
+graph of objects and either JSON or YAML.
+"""
 from __future__ import annotations
 
 import copy
@@ -10,7 +14,7 @@ from .utils import classproperty, get_md5_hash
 from .validation import get_schema
 from .errors import ToJSONLikeChildReferenceError
 
-
+#: Primitive types supported by the serialization mechanism.
 PRIMITIVES = (
     int,
     float,
@@ -22,6 +26,10 @@ _SDK_logger = get_SDK_logger(__name__)
 
 
 def to_json_like(obj, shared_data=None, parent_refs=None, path=None):
+    """
+    Convert the object to a JSON-like basic value tree.
+    Such trees are trivial to serialize as JSON or YAML.
+    """
     path = path or []
 
     if len(path) > 50:
@@ -81,32 +89,54 @@ def to_json_like(obj, shared_data=None, parent_refs=None, path=None):
 
 @dataclass
 class ChildObjectSpec:
+    """
+    Used to describe what the child structure of an class is so that the generic
+    deserializer can build the structure.
+    """
+
+    #: The name of the attribute.
     name: str
+    #: The name of the class (or class of members of a list) used to deserialize the
+    #: attribute.
     class_name: Optional[str] = None
+    #: The class (or class of members of a list) used to deserialize the
+    #: attribute.
     class_obj: Optional[
         Type
     ] = None  # TODO: no need for class_obj/class_name if shared data?
+    #: The name of the key used in the JSON document, if different from the attribute
+    #: name.
     json_like_name: Optional[str] = None
+    #: If true, the attribute is really a list of instances,
+    #: or a dictionary if :attr:`dict_key_attr` is set.
     is_multiple: Optional[bool] = False
+    #: If set, the name of an attribute of the object to use as a dictionary key.
+    #: Requires that :attr:`is_multiple` be set as well.
     dict_key_attr: Optional[str] = None
+    #: If set, the name of an attribute of the object to use as a dictionary value.
+    #: If not set but :attr:`dict_key_attr` is set, the whole object is the value.
+    #: Requires that :attr:`dict_key_attr` be set as well.
     dict_val_attr: Optional[str] = None
+    #: If set, the attribute of the child object that contains a reference to its parent.
     parent_ref: Optional[
         str
     ] = None  # TODO: do parent refs make sense when from shared? Prob not.
-    is_single_attribute: Optional[
-        bool
-    ] = False  # if True, obj is not represented as a dict of attr name-values, but just a value.
-    is_enum: Optional[
-        bool
-    ] = False  # if true, we don't invoke to/from_json_like on the data/Enum
-    is_dict_values: Optional[
-        bool
-    ] = False  # if True, the child object is a dict, whose values are of the specified class. The dict structure will remain.
-    is_dict_values_ensure_list: Optional[
-        bool
-    ] = False  # if True, values that are not lists are cast to lists and multiple child objects are instantiated for each dict value
-
+    #: If true, the object is not represented as a dict of attr name-values, but just a value.
+    is_single_attribute: Optional[bool] = False
+    #: If true, the object is an enum member and should use special serialization rules.
+    is_enum: Optional[bool] = False
+    #: If true, the child object is a dict, whose values are of the specified class.
+    #: The dict structure will remain.
+    is_dict_values: Optional[bool] = False
+    #: If true, values that are not lists are cast to lists and multiple child objects
+    #: are instantiated for each dict value.
+    is_dict_values_ensure_list: Optional[bool] = False
+    #: What key to look values up under in the shared data cache.
+    #: If unspecified, the shared data cache is ignored.
     shared_data_name: Optional[str] = None
+    #: What attribute provides the value of the key into the shared data cache.
+    #: If unspecified, a hash of the object dictionary is used.
+    #: Ignored if :py:attr:`~.shared_data_name` is unspecified.
     shared_data_primary_key: Optional[str] = None
     # shared_data_secondary_keys: Optional[Tuple[str]] = None # TODO: what's the point?
 
@@ -155,6 +185,8 @@ class ChildObjectSpec:
 
 class BaseJSONLike:
     """
+    An object that has a serialization as JSON or YAML.
+
     Parameters
     ----------
     _class_namespace : namespace
@@ -200,6 +232,21 @@ class BaseJSONLike:
         json_like: Union[Dict, List],
         shared_data: Optional[Dict[str, ObjectList]] = None,
     ):
+        """
+        Make an instance of this class from JSON (or YAML) data.
+
+        Parameters
+        ----------
+        json_like:
+            The data to deserialise.
+        shared_data:
+            Shared context data.
+
+        Returns
+        -------
+            The deserialised object.
+        """
+
         def _from_json_like_item(child_obj_spec, json_like_i):
             if not (
                 child_obj_spec.class_name
@@ -403,12 +450,20 @@ class BaseJSONLike:
         return get_md5_hash(json_like)
 
     def to_dict(self):
+        """
+        Serialize this object as a dictionary.
+        """
         if hasattr(self, "__dict__"):
             return dict(self.__dict__)
         elif hasattr(self, "__slots__"):
             return {k: getattr(self, k) for k in self.__slots__}
 
     def to_json_like(self, dct=None, shared_data=None, exclude=None, path=None):
+        """
+        Serialize this object as an object structure that can be trivially converted
+        to JSON. Note that YAML can also be produced from the result of this method;
+        it just requires a different final serialization step.
+        """
         if dct is None:
             dct = {k: v for k, v in self.to_dict().items() if k not in (exclude or [])}
 
@@ -475,6 +530,9 @@ class JSONLike(BaseJSONLike):
         return getattr(cls, cls._app_attr)
 
     def to_dict(self):
+        """
+        Serialize this object as a dictionary.
+        """
         out = super().to_dict()
 
         # remove parent references:

hpcflow/sdk/core/loop.py

@@ -1,3 +1,9 @@
+"""
+A looping construct for a workflow.
+There are multiple types of loop,
+notably looping over a set of values or until a condition holds.
+"""
+
 from __future__ import annotations
 
 import copy
@@ -26,11 +32,29 @@ from hpcflow.sdk.log import TimeIt
 # @dataclass
 # class Loop:
 #     parameter: Parameter
-#     stopping_criteria: StoppingCriterion  # TODO: should be a logical combination of these (maybe provide a superclass in valida to re-use some logic there?)
+#     stopping_criteria: StoppingCriterion
+#     # TODO: should be a logical combination of these (maybe provide a superclass in valida to re-use some logic there?)
 #     maximum_iterations: int
 
 
 class Loop(JSONLike):
+    """
+    A loop in a workflow template.
+
+    Parameters
+    ----------
+    tasks: list[int | ~hpcflow.app.WorkflowTask]
+        List of task insert IDs or workflow tasks.
+    num_iterations:
+        Number of iterations to perform.
+    name: str
+        Loop name.
+    non_iterable_parameters: list[str]
+        Specify input parameters that should not iterate.
+    termination: v~hpcflow.app.Rule
+        Stopping criterion, expressed as a rule.
+    """
+
     _app_attr = "app"
     _child_objects = (ChildObjectSpec(name="termination", class_name="Rule"),)
 
@@ -42,21 +66,6 @@ class Loop(JSONLike):
         non_iterable_parameters: Optional[List[str]] = None,
         termination: Optional[app.Rule] = None,
     ) -> None:
-        """
-
-        Parameters
-        ----------
-        name
-            Loop name, optional
-        tasks
-            List of task insert IDs or WorkflowTask objects
-        non_iterable_parameters
-            Specify input parameters that should not iterate.
-        termination
-            Stopping criterion, expressed as a rule.
-
-        """
-
         _task_insert_IDs = []
         for task in tasks:
             if isinstance(task, WorkflowTask):
@@ -98,22 +107,37 @@ class Loop(JSONLike):
 
     @property
     def name(self):
+        """
+        The name of the loop, if one was provided.
+        """
         return self._name
 
     @property
     def num_iterations(self):
+        """
+        The number of loop iterations to do.
+        """
         return self._num_iterations
 
     @property
     def non_iterable_parameters(self):
+        """
+        Which parameters are not iterable.
+        """
         return self._non_iterable_parameters
 
     @property
     def termination(self):
+        """
+        A termination rule for the loop, if one is provided.
+        """
         return self._termination
 
     @property
     def workflow_template(self):
+        """
+        The workflow template that contains this loop.
+        """
         return self._workflow_template
 
     @workflow_template.setter
@@ -123,6 +147,9 @@ class Loop(JSONLike):
 
     @property
     def task_objects(self) -> Tuple[app.WorkflowTask]:
+        """
+        The tasks in the loop.
+        """
         if not self.workflow_template:
             raise RuntimeError(
                 "Workflow template must be assigned to retrieve task objects of the loop."
@@ -169,7 +196,25 @@ class Loop(JSONLike):
 
 
 class WorkflowLoop:
-    """Class to represent a Loop that is bound to a Workflow."""
+    """
+    Class to represent a :py:class:`.Loop` that is bound to a
+    :py:class:`~hpcflow.app.Workflow`.
+
+    Parameters
+    ----------
+    index: int
+        The index of this loop in the workflow.
+    workflow: ~hpcflow.app.Workflow
+        The workflow containing this loop.
+    template: Loop
+        The loop that this was generated from.
+    num_added_iterations:
+        Description of what iterations have been added.
+    iterable_parameters:
+        Description of what parameters are being iterated over.
+    parents: list[str]
+        The paths to the parent entities of this loop.
+    """
 
     _app_attr = "app"
 
@@ -229,7 +274,9 @@ class WorkflowLoop:
 
     @property
     def num_added_iterations(self):
-
+        """
+        The number of added iterations.
+        """
         if self._pending_num_added_iterations:
             return self._pending_num_added_iterations
         else:
@@ -281,53 +328,82 @@ class WorkflowLoop:
 
     @property
     def index(self):
+        """
+        The index of this loop within its workflow.
+        """
         return self._index
 
     @property
     def task_insert_IDs(self):
+        """
+        The insertion IDs of the tasks inside this loop.
+        """
         return self.template.task_insert_IDs
 
     @property
     def task_objects(self):
+        """
+        The tasks in this loop.
+        """
         return self.template.task_objects
 
     @property
     def task_indices(self) -> Tuple[int]:
-        """Get the list of task indices that define the extent of the loop."""
+        """
+        The list of task indices that define the extent of the loop.
+        """
         return tuple(i.index for i in self.task_objects)
 
     @property
     def workflow(self):
+        """
+        The workflow containing this loop.
+        """
         return self._workflow
 
     @property
     def template(self):
+        """
+        The loop template for this loop.
+        """
         return self._template
 
     @property
     def parents(self) -> List[str]:
+        """
+        The parents of this loop.
+        """
         return self._parents + self._pending_parents
 
     @property
     def name(self):
+        """
+        The name of this loop, if one is defined.
+        """
         return self.template.name
 
     @property
     def iterable_parameters(self):
+        """
+        The parameters that are being iterated over.
+        """
         return self._iterable_parameters
 
     @property
     def num_iterations(self):
+        """
+        The number of iterations.
+        """
         return self.template.num_iterations
 
     @property
     def downstream_tasks(self) -> List[app.WorkflowLoop]:
-        """Return tasks that are not part of the loop, and downstream from this loop."""
+        """Tasks that are not part of the loop, and downstream from this loop."""
         return self.workflow.tasks[self.task_objects[-1].index + 1 :]
 
     @property
     def upstream_tasks(self) -> List[app.WorkflowLoop]:
-        """Return tasks that are not part of the loop, and upstream from this loop."""
+        """Tasks that are not part of the loop, and upstream from this loop."""
         return self.workflow.tasks[: self.task_objects[0].index]
 
     @staticmethod
@@ -367,6 +443,20 @@ class WorkflowLoop:
         template: app.Loop,
         iter_loop_idx: List[Dict],
     ) -> Tuple[app.WorkflowLoop, List[Dict[str, int]]]:
+        """
+        Make a new empty loop.
+
+        Parameters
+        ----------
+        index: int
+            The index of the loop to create.
+        workflow: ~hpcflow.app.Workflow
+            The workflow that will contain the loop.
+        template: Loop
+            The template for the loop.
+        iter_loop_idx: list[dict]
+            Iteration information from parent loops.
+        """
         parent_loops = cls._get_parent_loops(index, workflow, template)
         parent_names = [i.name for i in parent_loops]
         num_added_iters = {}
@@ -436,6 +526,17 @@ class WorkflowLoop:
 
     @TimeIt.decorator
     def add_iteration(self, parent_loop_indices=None, cache: Optional[LoopCache] = None):
+        """
+        Add an iteration to this loop.
+
+        Parameters
+        ----------
+        parent_loop_indices:
+            Where have any parent loops got up to?
+        cache:
+            A cache used to make adding the iteration more efficient.
+            One will be created if it is not supplied.
+        """
         if not cache:
             cache = LoopCache.build(self.workflow)
         parent_loops = self.get_parent_loops()

hpcflow/sdk/core/loop_cache.py

@@ -1,3 +1,7 @@
+"""
+Cache of loop statuses.
+"""
+
 from dataclasses import dataclass
 from collections import defaultdict
 from typing import Dict, List, Optional, Tuple
@@ -10,40 +14,55 @@ from hpcflow.sdk.core.cache import DependencyCache
 
 @dataclass
 class LoopCache:
-    """Class to store a cache for use in `Workflow.add_empty_loop` and
-    `WorkflowLoop.add_iterations`.
+    """Class to store a cache for use in :py:meth:`.Workflow.add_empty_loop` and
+    :py:meth:`.WorkflowLoop.add_iterations`. Use :py:meth:`build` to get a new instance.
 
-    Attributes
+    Parameters
     ----------
-    element_dependents
+    element_dependents:
         Keys are element IDs, values are dicts whose keys are element IDs that depend on
         the key element ID (via `Element.get_dependent_elements_recursively`), and whose
         values are dicts with keys: `group_names`, which is a tuple of the string group
         names associated with the dependent element's element set.
-    elements
+    elements:
         Keys are element IDs, values are dicts with keys: `input_statuses`,
         `input_sources`, and `task_insert_ID`.
-    zeroth_iters
+    zeroth_iters:
         Keys are element IDs, values are data associated with the zeroth iteration of that
         element, namely a tuple of iteration ID and `ElementIteration.data_idx`.
-    data_idx
+    data_idx:
         Keys are element IDs, values are data associated with all iterations of that
         element, namely a dict whose keys are the iteration loop index as a tuple, and
         whose values are data indices via `ElementIteration.get_data_idx()`.
-    iterations
+    iterations:
         Keys are iteration IDs, values are tuples of element ID and iteration index within
         that element.
-    task_iterations
+    task_iterations:
         Keys are task insert IDs, values are list of all iteration IDs associated with
         that task.
 
     """
 
+    #: Keys are element IDs, values are dicts whose keys are element IDs that depend on
+    #: the key element ID (via `Element.get_dependent_elements_recursively`), and whose
+    #: values are dicts with keys: `group_names`, which is a tuple of the string group
+    #: names associated with the dependent element's element set.
     element_dependents: Dict[int, Dict]
+    #: Keys are element IDs, values are dicts with keys: `input_statuses`,
+    #: `input_sources`, and `task_insert_ID`.
     elements: Dict[int, Dict]
+    #: Keys are element IDs, values are data associated with the zeroth iteration of that
+    #: element, namely a tuple of iteration ID and `ElementIteration.data_idx`.
     zeroth_iters: Dict[int, Tuple]
+    #: Keys are element IDs, values are data associated with all iterations of that
+    #: element, namely a dict whose keys are the iteration loop index as a tuple, and
+    #: whose values are data indices via `ElementIteration.get_data_idx()`.
     data_idx: Dict[int, Dict]
+    #: Keys are iteration IDs, values are tuples of element ID and iteration index within
+    #: that element.
     iterations: Dict[int, Tuple]
+    #: Keys are task insert IDs, values are list of all iteration IDs associated with
+    #: that task.
     task_iterations: Dict[int, List[int]]
 
     @TimeIt.decorator
@@ -53,6 +72,9 @@ class LoopCache:
 
     @TimeIt.decorator
     def get_iter_loop_indices(self, iter_IDs: List[int]) -> List[Dict[str, int]]:
+        """
+        Retrieve the mapping from element to loop index for each given iteration.
+        """
         iter_loop_idx = []
         for i in iter_IDs:
             elem_id, idx = self.iterations[i]
@@ -61,6 +83,9 @@ class LoopCache:
 
     @TimeIt.decorator
     def update_loop_indices(self, new_loop_name: str, iter_IDs: List[int]):
+        """
+        Set the loop indices for a named loop to the given list of iteration IDs.
+        """
         elem_ids = {v[0] for k, v in self.iterations.items() if k in iter_IDs}
         for i in elem_ids:
             new_item = {}