hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl

hpcflow/sdk/core/task_schema.py

@@ -1,3 +1,7 @@
+"""
+Abstract task, prior to instantiation.
+"""
+
 from contextlib import contextmanager
 import copy
 from dataclasses import dataclass
@@ -22,6 +26,15 @@ from .utils import check_valid_py_identifier
 
 @dataclass
 class TaskObjective(JSONLike):
+    """
+    A thing that a task is attempting to achieve.
+
+    Parameter
+    ---------
+    name: str
+        The name of the objective. A valid Python identifier.
+    """
+
     _child_objects = (
         ChildObjectSpec(
             name="name",
@@ -29,6 +42,7 @@ class TaskObjective(JSONLike):
         ),
     )
 
+    #: The name of the objective. A valid Python identifier.
     name: str
 
     def __post_init__(self):
@@ -41,21 +55,28 @@ class TaskSchema(JSONLike):
 
     Parameters
     ----------
-    objective
+    objective:
         This is a string representing the objective of the task schema.
-    actions
+    actions:
         A list of Action objects whose commands are to be executed by the task.
-    method
+    method:
         An optional string to label the task schema by its method.
-    implementation
+    implementation:
         An optional string to label the task schema by its implementation.
-    inputs
+    inputs:
         A list of SchemaInput objects that define the inputs to the task.
-    outputs
+    outputs:
         A list of SchemaOutput objects that define the outputs of the task.
-    web_doc
-        True if this object should be included in the Sphinx documentation (in the case
-        of built-in task schemas). True by default.
+    version:
+        The version of this task schema.
+    parameter_class_modules:
+        Where to find implementations of parameter value handlers.
+    web_doc:
+        True if this object should be included in the Sphinx documentation
+        (normally only relevant for built-in task schemas). True by default.
+    environment_presets:
+        Information about default execution environments. Can be overridden in specific
+        cases in the concrete tasks.
     """
 
     _validation_schema = "task_schema_spec_schema.yaml"
@@ -93,14 +114,24 @@ class TaskSchema(JSONLike):
         environment_presets: Optional[Dict[str, Dict[str, Dict[str, Any]]]] = None,
         _hash_value: Optional[str] = None,
     ):
+        #: This is a string representing the objective of the task schema.
         self.objective = objective
+        #: A list of Action objects whose commands are to be executed by the task.
         self.actions = actions or []
+        #: An optional string to label the task schema by its method.
         self.method = method
+        #: An optional string to label the task schema by its implementation.
         self.implementation = implementation
+        #: A list of SchemaInput objects that define the inputs to the task.
         self.inputs = inputs or []
+        #: A list of SchemaOutput objects that define the outputs of the task.
         self.outputs = outputs or []
+        #: Where to find implementations of parameter value handlers.
         self.parameter_class_modules = parameter_class_modules or []
+        #: Whether this object should be included in the Sphinx documentation
+        #: (normally only relevant for built-in task schemas).
         self.web_doc = web_doc
+        #: Information about default execution environments.
         self.environment_presets = environment_presets
         self._hash_value = _hash_value
 
@@ -112,6 +143,7 @@ class TaskSchema(JSONLike):
 
         self._validate()
         self.actions = self._expand_actions()
+        #: The version of this task schema.
         self.version = version
         self._task_template = None  # assigned by parent Task
 
@@ -309,6 +341,10 @@ class TaskSchema(JSONLike):
         return self._show_info()
 
     def get_info_html(self) -> str:
+        """
+        Describe the task schema as an HTML document.
+        """
+
         def _format_parameter_type(param):
             param_typ_fmt = param.typ
             if param.typ in param_types:
@@ -586,6 +622,9 @@ class TaskSchema(JSONLike):
     @classmethod
     @contextmanager
     def ignore_invalid_actions(cls):
+        """
+        A context manager within which invalid actions will be ignored.
+        """
         try:
             cls._validate_actions = False
             yield
@@ -709,6 +748,10 @@ class TaskSchema(JSONLike):
             out.parameter._set_value_class()
 
     def make_persistent(self, workflow: app.Workflow, source: Dict) -> List[int]:
+        """
+        Convert this task schema to persistent form within the context of the given
+        workflow.
+        """
         new_refs = []
         for input_i in self.inputs:
             for lab_info in input_i.labelled_info():
@@ -721,6 +764,9 @@ class TaskSchema(JSONLike):
 
     @property
     def name(self):
+        """
+        The name of this schema.
+        """
         out = (
             f"{self.objective.name}"
             f"{f'_{self.method}' if self.method else ''}"
@@ -730,14 +776,23 @@ class TaskSchema(JSONLike):
 
     @property
     def input_types(self):
+        """
+        The input types to the schema.
+        """
         return tuple(j for i in self.inputs for j in i.all_labelled_types)
 
     @property
     def output_types(self):
+        """
+        The output types from the schema.
+        """
         return tuple(i.typ for i in self.outputs)
 
     @property
     def provides_parameters(self) -> Tuple[Tuple[str, str]]:
+        """
+        The parameters that this schema provides.
+        """
         out = []
         for schema_inp in self.inputs:
             for labelled_info in schema_inp.labelled_info():
@@ -753,6 +808,9 @@ class TaskSchema(JSONLike):
 
     @property
     def task_template(self):
+        """
+        The template that this schema is contained in.
+        """
         return self._task_template
 
     @classmethod
@@ -770,6 +828,9 @@ class TaskSchema(JSONLike):
         return out
 
     def get_key(self):
+        """
+        Get the hashable value that represents this schema.
+        """
         return (str(self.objective), self.method, self.implementation)
 
     def _get_single_label_lookup(self, prefix="") -> Dict[str, str]:

hpcflow/sdk/core/test_utils.py

@@ -1,3 +1,7 @@
+"""
+Utilities for making data to use in testing.
+"""
+
 from dataclasses import dataclass
 from importlib import resources
 from pathlib import Path
@@ -7,6 +11,9 @@ from hpcflow.sdk.core.parameters import ParameterValue
 
 
 def make_schemas(ins_outs, ret_list=False):
+    """
+    Construct a collection of schemas.
+    """
     out = []
     for idx, info in enumerate(ins_outs):
         if len(info) == 2:
@@ -57,6 +64,9 @@ def make_schemas(ins_outs, ret_list=False):
 
 
 def make_parameters(num):
+    """
+    Construct a sequence of parameters.
+    """
     return [hf.Parameter(f"p{i + 1}") for i in range(num)]
 
 
@@ -64,6 +74,9 @@ def make_actions(
     ins_outs: List[Tuple[Union[Tuple, str], str]],
     env="env1",
 ) -> List[hf.Action]:
+    """
+    Construct a collection of actions.
+    """
     act_env = hf.ActionEnvironment(environment=env)
     actions = []
     for ins_outs_i in ins_outs:
@@ -98,6 +111,9 @@ def make_tasks(
     input_sources=None,
     groups=None,
 ):
+    """
+    Construct a sequence of tasks.
+    """
     local_inputs = local_inputs or {}
     local_sequences = local_sequences or {}
     local_resources = local_resources or {}
@@ -148,6 +164,9 @@ def make_workflow(
     overwrite=False,
     store="zarr",
 ):
+    """
+    Construct a workflow.
+    """
     tasks = make_tasks(
         schemas_spec,
         local_inputs=local_inputs,
@@ -202,6 +221,10 @@ def make_test_data_YAML_workflow_template(workflow_name, **kwargs):
 
 @dataclass
 class P1_sub_parameter_cls(ParameterValue):
+    """
+    Parameter value handler: ``p1_sub``
+    """
+
     _typ = "p1_sub"
 
     e: int
@@ -222,6 +245,10 @@ class P1_sub_parameter_cls(ParameterValue):
 
 @dataclass
 class P1_sub_parameter_cls_2(ParameterValue):
+    """
+    Parameter value handler: ``p1_sub_2``
+    """
+
     _typ = "p1_sub_2"
 
     f: int
@@ -229,6 +256,14 @@ class P1_sub_parameter_cls_2(ParameterValue):
 
 @dataclass
 class P1_parameter_cls(ParameterValue):
+    """
+    Parameter value handler: ``p1c``
+
+    Note
+    ----
+    This is a composite value handler.
+    """
+
     _typ = "p1c"
     _sub_parameters = {"sub_param": "p1_sub", "sub_param_2": "p1_sub_2"}
 

hpcflow/sdk/core/utils.py

@@ -1,3 +1,7 @@
+"""
+Miscellaneous utilities.
+"""
+
 import copy
 import enum
 from functools import wraps
@@ -45,12 +49,18 @@ def load_config(func):
 
 
 def make_workflow_id():
+    """
+    Generate a random ID for a workflow.
+    """
     length = 12
     chars = string.ascii_letters + "0123456789"
     return "".join(random.choices(chars, k=length))
 
 
 def get_time_stamp():
+    """
+    Get the current time in standard string form.
+    """
     return datetime.now(timezone.utc).astimezone().strftime("%Y.%m.%d_%H:%M:%S_%z")
 
 
@@ -181,6 +191,11 @@ def swap_nested_dict_keys(dct, inner_key):
 
 
 def get_in_container(cont, path, cast_indices=False, allow_getattr=False):
+    """
+    Follow a path (sequence of indices of appropriate type) into a container to obtain
+    a "leaf" value. Containers can be lists, tuples, dicts,
+    or any class (with `getattr()`) if ``allow_getattr`` is True.
+    """
     cur_data = cont
     err_msg = (
         "Data at path {path_comps!r} is not a sequence, but is of type "
@@ -221,6 +236,11 @@ def get_in_container(cont, path, cast_indices=False, allow_getattr=False):
 
 
 def set_in_container(cont, path, value, ensure_path=False, cast_indices=False):
+    """
+    Follow a path (sequence of indices of appropriate type) into a container to update
+    a "leaf" value. Containers can be lists, tuples or dicts.
+    The "branch" holding the leaf to update must be modifiable.
+    """
     if ensure_path:
         num_path = len(path)
         for idx in range(1, num_path):
@@ -315,6 +335,10 @@ def search_dir_files_by_regex(pattern, group=0, directory=".") -> List[str]:
 
 
 class classproperty(object):
+    """
+    Simple class property decorator.
+    """
+
     def __init__(self, f):
         self.f = f
 
@@ -323,6 +347,11 @@ class classproperty(object):
 
 
 class PrettyPrinter(object):
+    """
+    A class that produces a nice readable version of itself with ``str()``.
+    Intended to be subclassed.
+    """
+
     def __str__(self):
         lines = [self.__class__.__name__ + ":"]
         for key, val in vars(self).items():
@@ -331,6 +360,11 @@ class PrettyPrinter(object):
 
 
 class Singleton(type):
+    """
+    Metaclass that enforces that only one instance can exist of the classes to which it
+    is applied.
+    """
+
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
@@ -347,6 +381,10 @@ class Singleton(type):
 
 
 def capitalise_first_letter(chars):
+    """
+    Convert the first character of a string to upper case (if that makes sense).
+    The rest of the string is unchanged.
+    """
     return chars[0].upper() + chars[1:]
 
 
@@ -374,6 +412,18 @@ def check_in_object_list(spec_name, spec_pos=1, obj_list_pos=2):
 
 @TimeIt.decorator
 def substitute_string_vars(string, variables: Dict[str, str] = None):
+    """
+    Scan ``string`` and substitute sequences like ``<<var:ABC>>`` with the value
+    looked up in the supplied dictionary (with ``ABC`` as the key).
+
+    Default values for the substitution can be supplied like:
+    ``<<var:ABC[default=XYZ]>>``
+
+    Examples
+    --------
+    >>> substitute_string_vars("abc <<var:def>> ghi", {"def": "123"})
+    "abc 123 def"
+    """
     variables = variables or {}
 
     def var_repl(match_obj):
@@ -410,7 +460,7 @@ def substitute_string_vars(string, variables: Dict[str, str] = None):
 
 @TimeIt.decorator
 def read_YAML_str(yaml_str, typ="safe", variables: Dict[str, str] = None):
-    """Load a YAML string."""
+    """Load a YAML string. This will produce basic objects."""
     if variables is not False and "<<var:" in yaml_str:
         yaml_str = substitute_string_vars(yaml_str, variables=variables)
     yaml = YAML(typ=typ)
@@ -419,30 +469,35 @@ def read_YAML_str(yaml_str, typ="safe", variables: Dict[str, str] = None):
 
 @TimeIt.decorator
 def read_YAML_file(path: PathLike, typ="safe", variables: Dict[str, str] = None):
+    """Load a YAML file. This will produce basic objects."""
     with fsspec.open(path, "rt") as f:
         yaml_str = f.read()
     return read_YAML_str(yaml_str, typ=typ, variables=variables)
 
 
 def write_YAML_file(obj, path: PathLike, typ="safe"):
+    """Write a basic object to a YAML file."""
     yaml = YAML(typ=typ)
     with Path(path).open("wt") as fp:
         yaml.dump(obj, fp)
 
 
 def read_JSON_string(json_str: str, variables: Dict[str, str] = None):
+    """Load a JSON string. This will produce basic objects."""
     if variables is not False and "<<var:" in json_str:
         json_str = substitute_string_vars(json_str, variables=variables)
     return json.loads(json_str)
 
 
 def read_JSON_file(path, variables: Dict[str, str] = None):
+    """Load a JSON file. This will produce basic objects."""
     with fsspec.open(path, "rt") as f:
         json_str = f.read()
     return read_JSON_string(json_str, variables=variables)
 
 
 def write_JSON_file(obj, path: PathLike):
+    """Write a basic object to a JSON file."""
     with Path(path).open("wt") as fp:
         json.dump(obj, fp)
 
@@ -486,6 +541,13 @@ def get_item_repeat_index(lst, distinguish_singular=False, item_callable=None):
 
 
 def get_process_stamp():
+    """
+    Return a globally unique string identifying this process.
+
+    Note
+    ----
+    This should only be called once per process.
+    """
     return "{} {} {}".format(
         datetime.now(),
         socket.gethostname(),
@@ -494,11 +556,18 @@ def get_process_stamp():
 
 
 def remove_ansi_escape_sequences(string):
+    """
+    Strip ANSI terminal escape codes from a string.
+    """
     ansi_escape = re.compile(r"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]")
     return ansi_escape.sub("", string)
 
 
 def get_md5_hash(obj):
+    """
+    Compute the MD5 hash of an object.
+    This is the hash of the JSON of the object (with sorted keys) as a hex string.
+    """
     json_str = json.dumps(obj, sort_keys=True)
     return hashlib.md5(json_str.encode("utf-8")).hexdigest()
 
@@ -562,6 +631,9 @@ def ensure_in(item, lst) -> int:
 
 
 def list_to_dict(lst, exclude=None):
+    """
+    Convert a list of dicts to a dict of lists.
+    """
     # TODD: test
     exclude = exclude or []
     dct = {k: [] for k in lst[0].keys() if k not in exclude}
@@ -617,7 +689,7 @@ def flatten(lst):
     """Flatten an arbitrarily (but of uniform depth) nested list and return shape
     information to enable un-flattening.
 
-    Un-flattening can be performed with the `reshape` function.
+    Un-flattening can be performed with the :py:func:`reshape` function.
 
     lst
         List to be flattened. Each element must contain all lists or otherwise all items
@@ -657,6 +729,10 @@ def flatten(lst):
 
 
 def reshape(lst, lens):
+    """
+    Reverse the destructuring of the :py:func:`flatten` function.
+    """
+
     def _reshape(lst, lens):
         lens_acc = [0] + list(accumulate(lens))
         lst_rs = [lst[lens_acc[idx] : lens_acc[idx + 1]] for idx in range(len(lens))]
@@ -669,15 +745,30 @@ def reshape(lst, lens):
 
 
 def is_fsspec_url(url: str) -> bool:
+    """
+    Test if a URL appears to be one that can be understood by fsspec.
+    """
     return bool(re.match(r"(?:[a-z0-9]+:{1,2})+\/\/", url))
 
 
 class JSONLikeDirSnapShot(DirectorySnapshot):
-    """Overridden DirectorySnapshot from watchdog to allow saving and loading from JSON."""
+    """
+    Overridden DirectorySnapshot from watchdog to allow saving and loading from JSON.
+    """
 
     def __init__(self, root_path=None, data=None):
-        """Create an empty snapshot or load from JSON-like data."""
+        """Create an empty snapshot or load from JSON-like data.
+
+        Parameters
+        ----------
+        root_path: str
+            Where to take the snapshot based at.
+        data: dict
+            Serialised snapshot to reload from.
+            See :py:meth:`to_json_like`.
+        """
 
+        #: Where to take the snapshot based at.
         self.root_path = root_path
         self._stat_info = {}
         self._inode_to_path = {}
@@ -874,10 +965,16 @@ def dict_values_process_flat(d, callable):
 
 
 def nth_key(dct, n):
+    """
+    Given a dict in some order, get the n'th key of that dict.
+    """
     it = iter(dct)
     next(islice(it, n, n), None)
     return next(it)
 
 
 def nth_value(dct, n):
+    """
+    Given a dict in some order, get the n'th value of that dict.
+    """
     return dct[nth_key(dct, n)]

hpcflow/sdk/core/validation.py

@@ -1,10 +1,22 @@
+"""
+Schema management.
+"""
+
 from importlib import resources
 
 from valida import Schema
 
 
 def get_schema(filename):
-    """Get a valida `Schema` object from the embedded data directory."""
+    """
+    Get a valida `Schema` object from the embedded data directory.
+
+    Parameter
+    ---------
+    schema: str
+        The name of the schema file within the resources package
+        (:py:mod:`hpcflow.sdk.data`).
+    """
     package = "hpcflow.sdk.data"
     try:
         fh = resources.files(package).joinpath(filename).open("rt")