hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

hpcflow/sdk/cli_common.py

@@ -7,10 +7,14 @@ from hpcflow.sdk.persistence import ALL_STORE_FORMATS, DEFAULT_STORE_FORMAT
 
 
 def sub_tasks_callback(ctx, param, value):
+    """
+    Parse subtasks.
+    """
     if value:
         return [int(i) for i in value.split(",")]
 
 
+#: Standard option
 format_option = click.option(
     "--format",
     type=click.Choice(ALL_TEMPLATE_FORMATS),
@@ -20,11 +24,13 @@ format_option = click.option(
         "particular format."
     ),
 )
+#: Standard option
 path_option = click.option(
     "--path",
     type=click.Path(exists=True),
     help="The directory path into which the new workflow will be generated.",
 )
+#: Standard option
 name_option = click.option(
     "--name",
     help=(
@@ -33,6 +39,7 @@ name_option = click.option(
         "will be used, in combination with a date-timestamp."
     ),
 )
+#: Standard option
 overwrite_option = click.option(
     "--overwrite",
     is_flag=True,
@@ -42,12 +49,15 @@ overwrite_option = click.option(
         "the existing directory will be overwritten."
     ),
 )
+#: Standard option
 store_option = click.option(
     "--store",
     type=click.Choice(ALL_STORE_FORMATS),
     help="The persistent store type to use.",
     default=DEFAULT_STORE_FORMAT,
 )
+
+#: Standard option
 ts_fmt_option = click.option(
     "--ts-fmt",
     help=(
@@ -56,6 +66,7 @@ ts_fmt_option = click.option(
         "should not include a time zone name."
     ),
 )
+#: Standard option
 ts_name_fmt_option = click.option(
     "--ts-name-fmt",
     help=(
@@ -63,6 +74,8 @@ ts_name_fmt_option = click.option(
         "includes a timestamp."
     ),
 )
+
+#: Standard option
 variables_option = click.option(
     "-v",
     "--var",
@@ -74,6 +87,7 @@ variables_option = click.option(
         "string. Multiple variable values can be specified."
     ),
 )
+#: Standard option
 js_parallelism_option = click.option(
     "--js-parallelism",
     help=(
@@ -84,23 +98,27 @@ js_parallelism_option = click.option(
     ),
     type=click.BOOL,
 )
+#: Standard option
 wait_option = click.option(
     "--wait",
     help=("If True, this command will block until the workflow execution is complete."),
     is_flag=True,
     default=False,
 )
+#: Standard option
 add_to_known_opt = click.option(
     "--add-to-known/--no-add-to-known",
     default=True,
     help="If True, add this submission to the known-submissions file.",
 )
+#: Standard option
 print_idx_opt = click.option(
     "--print-idx",
     help="If True, print the submitted jobscript indices for each submission index.",
     is_flag=True,
     default=False,
 )
+#: Standard option
 tasks_opt = click.option(
     "--tasks",
     help=(
@@ -109,22 +127,27 @@ tasks_opt = click.option(
     ),
     callback=sub_tasks_callback,
 )
+#: Standard option
 cancel_opt = click.option(
     "--cancel",
     help="Immediately cancel the submission. Useful for testing and benchmarking.",
     is_flag=True,
     default=False,
 )
+#: Standard option
 submit_status_opt = click.option(
     "--status/--no-status",
     help="If True, display a live status to track submission progress.",
     default=True,
 )
+#: Standard option
 make_status_opt = click.option(
     "--status/--no-status",
     help="If True, display a live status to track workflow creation progress.",
     default=True,
 )
+
+#: Standard option
 zip_path_opt = click.option(
     "--path",
     default=".",
@@ -134,15 +157,21 @@ zip_path_opt = click.option(
         "path is assumed to be the full file path to the new zip file."
     ),
 )
+#: Standard option
 zip_overwrite_opt = click.option(
     "--overwrite",
     is_flag=True,
     default=False,
     help="If set, any existing file will be overwritten.",
 )
+#: Standard option
 zip_log_opt = click.option("--log", help="Path to a log file to use during zipping.")
+#: Standard option
 zip_include_execute_opt = click.option("--include-execute", is_flag=True)
+#: Standard option
 zip_include_rechunk_backups_opt = click.option("--include-rechunk-backups", is_flag=True)
+
+#: Standard option
 unzip_path_opt = click.option(
     "--path",
     default=".",
@@ -152,12 +181,16 @@ unzip_path_opt = click.option(
         "Otherwise, this path will represent the new workflow directory path."
     ),
 )
+#: Standard option
 unzip_log_opt = click.option("--log", help="Path to a log file to use during unzipping.")
+
+#: Standard option
 rechunk_backup_opt = click.option(
     "--backup/--no-backup",
     default=True,
     help=("First copy a backup of the array to a directory ending in `.bak`."),
 )
+#: Standard option
 rechunk_chunk_size_opt = click.option(
     "--chunk-size",
     type=click.INT,
@@ -168,8 +201,58 @@ rechunk_chunk_size_opt = click.option(
         "array's shape)."
     ),
 )
+#: Standard option
 rechunk_status_opt = click.option(
     "--status/--no-status",
     default=True,
     help="If True, display a live status to track rechunking progress.",
 )
+
+
+def _add_doc_from_help(*args):
+    """
+    Attach the ``help`` field of each of its arguments as its ``__doc__``.
+    Only necessary because the wrappers in Click don't do this for us.
+
+    :meta private:
+    """
+    # Yes, this is ugly!
+    from types import SimpleNamespace
+
+    for opt in args:
+        ns = SimpleNamespace()
+        params = getattr(opt(ns), "__click_params__", [])
+        if params:
+            help = getattr(params[0], "help", "")
+            if help:
+                opt.__doc__ = f"Click option decorator: {help}"
+
+
+_add_doc_from_help(
+    format_option,
+    path_option,
+    name_option,
+    overwrite_option,
+    store_option,
+    ts_fmt_option,
+    ts_name_fmt_option,
+    variables_option,
+    js_parallelism_option,
+    wait_option,
+    add_to_known_opt,
+    print_idx_opt,
+    tasks_opt,
+    cancel_opt,
+    submit_status_opt,
+    make_status_opt,
+    zip_path_opt,
+    zip_overwrite_opt,
+    zip_log_opt,
+    zip_include_execute_opt,
+    zip_include_rechunk_backups_opt,
+    unzip_path_opt,
+    unzip_log_opt,
+    rechunk_backup_opt,
+    rechunk_chunk_size_opt,
+    rechunk_status_opt,
+)

hpcflow/sdk/config/__init__.py

@@ -1 +1,5 @@
+"""
+Configuration loading and manipulation.
+"""
+
 from .config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG

hpcflow/sdk/config/callbacks.py

@@ -10,7 +10,9 @@ from hpcflow.sdk.submission.shells import get_supported_shells
 
 
 def callback_vars(config, value):
-    """Substitute configuration variables."""
+    """
+    Callback that substitutes configuration variables.
+    """
 
     def vars_repl(match_obj):
         var_name = match_obj.groups()[0]
@@ -27,6 +29,9 @@ def callback_vars(config, value):
 
 
 def callback_file_paths(config, file_path):
+    """
+    Callback that resolves file paths.
+    """
     if isinstance(file_path, list):
         return [config._resolve_path(i) for i in file_path]
     else:
@@ -34,6 +39,9 @@ def callback_file_paths(config, file_path):
 
 
 def callback_bool(config, value):
+    """
+    Callback that coerces values to boolean.
+    """
     if not isinstance(value, bool):
         if value.lower() == "true":
             return True
@@ -45,6 +53,9 @@ def callback_bool(config, value):
 
 
 def callback_lowercase(config, value):
+    """
+    Callback that forces a string to lower case.
+    """
     if isinstance(value, list):
         return [i.lower() for i in value]
     elif isinstance(value, dict):
@@ -54,6 +65,9 @@ def callback_lowercase(config, value):
 
 
 def exists_in_schedulers(config, value):
+    """
+    Callback that tests that a value is a supported scheduler name.
+    """
     if value not in config.schedulers:
         raise ValueError(
             f"Cannot set default scheduler; {value!r} is not a supported scheduler "
@@ -64,6 +78,9 @@ def exists_in_schedulers(config, value):
 
 
 def callback_supported_schedulers(config, schedulers):
+    """
+    Callback that tests that all values are names of supported schedulers.
+    """
     # validate against supported schedulers according to the OS - this won't validate that
     # a particular scheduler actually exists on this system:
     available = config._app.get_OS_supported_schedulers()
@@ -122,6 +139,9 @@ def callback_scheduler_set_up(config, schedulers):
 
 
 def callback_supported_shells(config, shell_name):
+    """
+    Callback that tests if a shell names is supported on this OS.
+    """
     supported = get_supported_shells(os.name)
     if shell_name not in supported:
         raise UnsupportedShellError(shell=shell_name, supported=supported)
@@ -146,11 +166,14 @@ def set_callback_file_paths(config, value):
 
 
 def check_load_data_files(config, value):
-    """Check data files (e.g. task schema files) can be loaded successfully. This is only
+    """Check data files (e.g., task schema files) can be loaded successfully. This is only
     done on `config.set` (and not on `config.get` or `config._validate`) because it could
     be expensive in the case of remote files."""
     config._app.reload_template_components(warn=False)
 
 
 def callback_update_log_console_level(config, value):
+    """
+    Callback to set the logging level.
+    """
     config._app.log.update_console_level(value)

hpcflow/sdk/config/cli.py

@@ -44,7 +44,10 @@ def warning_formatter(func=custom_warning_formatter):
 
 
 def CLI_exception_wrapper_gen(*exception_cls):
-    """Decorator factory"""
+    """
+    Decorator factory that enhances the wrapped function to display a nice message on
+    success or failure.
+    """
 
     def CLI_exception_wrapper(func):
         """Decorator

hpcflow/sdk/config/config.py

@@ -1,3 +1,7 @@
+"""
+Configuration system class.
+"""
+
 from __future__ import annotations
 import contextlib
 
@@ -61,6 +65,7 @@ from .errors import (
 logger = logging.getLogger(__name__)
 
 _DEFAULT_SHELL = DEFAULT_SHELL_NAMES[os.name]
+#: The default configuration descriptor.
 DEFAULT_CONFIG = {
     "invocation": {"environment_setup": None, "match": {}},
     "config": {
@@ -82,12 +87,17 @@ DEFAULT_CONFIG = {
 class ConfigOptions:
     """Application-level options for configuration"""
 
+    #: The default directory.
     default_directory: Union[Path, str]
+    #: The environment variable containing the directory name.
     directory_env_var: str
+    #: The default configuration.
     default_config: Optional[Dict] = field(
         default_factory=lambda: deepcopy(DEFAULT_CONFIG)
     )
+    #: Any extra schemas to apply.
     extra_schemas: Optional[List[Schema]] = field(default_factory=lambda: [])
+    #: Default directory of known configurations.
     default_known_configs_dir: Optional[str] = None
 
     def __post_init__(self):
@@ -96,7 +106,9 @@ class ConfigOptions:
         self._configurable_keys = cfg_keys
 
     def init_schemas(self):
-        # Get allowed configurable keys from config schemas:
+        """
+        Get allowed configurable keys from config schemas.
+        """
         cfg_schemas = [get_schema("config_schema.yaml")] + self.extra_schemas
         cfg_keys = []
         for cfg_schema in cfg_schemas:
@@ -130,18 +142,114 @@ class ConfigOptions:
 class Config:
     """Application configuration as defined in one or more config files.
 
+    This class supports indexing into the collection of properties via Python dot notation.
+
     Notes
     -----
     On modifying/setting existing values, modifications are not automatically copied
-    to the configuration file; use `save()` to save to the file. Items in `overrides`
+    to the configuration file; use :meth:`save()` to save to the file. Items in `overrides`
     are not saved into the file.
 
     `schedulers` is used for specifying the available schedulers on this machine, and the
-    default arguments that should be used when initialising the `Scheduler` object.
+    default arguments that should be used when initialising the
+    :py:class:`Scheduler` object.
 
     `shells` is used for specifying the default arguments that should be used when
-    initialising the `Shell` object.
-
+    initialising the :py:class:`Shell` object.
+
+    Parameters
+    ----------
+    app:
+        The main hpcflow application instance.
+    config_file:
+        The configuration file that contains this config.
+    options:
+        Configuration options to be applied.
+    logger:
+        Where to log messages relating to configuration.
+    config_key:
+        The name of the configuration within the configuration file.
+    uid: int
+        User ID.
+    callbacks: dict
+        Overrides for the callback system.
+    variables: dict[str, str]
+        Variables to substitute when processing the configuration.
+
+    Attributes
+    ----------
+    config_directory:
+        The directory containing the configuration file.
+    config_file_name:
+        The name of the configuration file.
+    config_file_path:
+        The full path to the configuration file.
+    config_file_contents:
+        The cached contents of the configuration file.
+    config_key:
+        The primary key to select the configuration within the configuration file.
+    config_schemas:
+        The schemas that apply to the configuration file.
+    host_user_id:
+        User ID as understood by the script.
+    host_user_id_file_path:
+        Where user ID information is stored.
+    invoking_user_id:
+        User ID that created the workflow.
+    machine:
+        Machine to submit to.
+        Mapped to a field in the configuration file.
+    user_name:
+        User to submit as.
+        Mapped to a field in the configuration file.
+    user_orcid:
+        User's ORCID.
+        Mapped to a field in the configuration file.
+    user_affiliation:
+        User's institutional affiliation.
+        Mapped to a field in the configuration file.
+    linux_release_file:
+        Where to get the description of the Linux release version data.
+        Mapped to a field in the configuration file.
+    log_file_path:
+        Where to log to.
+        Mapped to a field in the configuration file.
+    log_file_level:
+        At what level to do logging to the file.
+        Mapped to a field in the configuration file.
+    log_console_level:
+        At what level to do logging to the console. Usually coarser than to a file.
+        Mapped to a field in the configuration file.
+    task_schema_sources:
+        Where to get task schemas.
+        Mapped to a field in the configuration file.
+    parameter_sources:
+        Where to get parameter descriptors.
+        Mapped to a field in the configuration file.
+    command_file_sources:
+        Where to get command files.
+        Mapped to a field in the configuration file.
+    environment_sources:
+        Where to get execution environment descriptors.
+        Mapped to a field in the configuration file.
+    default_scheduler:
+        The name of the default scheduler.
+        Mapped to a field in the configuration file.
+    default_shell:
+        The name of the default shell.
+        Mapped to a field in the configuration file.
+    schedulers:
+        Settings for supported scheduler(s).
+        Mapped to a field in the configuration file.
+    shells:
+        Settings for supported shell(s).
+        Mapped to a field in the configuration file.
+    demo_data_dir:
+        Location of demo data.
+        Mapped to a field in the configuration file.
+    demo_data_manifest_file:
+        Where the manifest describing the demo data is.
+        Mapped to a field in the configuration file.
     """
 
     def __init__(
@@ -517,7 +625,16 @@ class Config:
                 print(f"value is already: {callback_val!r}")
 
     def set(self, path: str, value, is_json=False, quiet=False):
-        """Set the value of a configuration item."""
+        """
+        Set the value of a configuration item.
+
+        Parameters
+        ----------
+        path:
+            Which configuration item to set.
+        value:
+            What to set it to.
+        """
         self._logger.debug(f"Attempting to set config item {path!r} to {value!r}.")
 
         if is_json:
@@ -542,7 +659,18 @@ class Config:
         self._set(name, root, quiet=quiet)
 
     def unset(self, name):
-        """Unset the value of a configuration item."""
+        """
+        Unset the value of a configuration item.
+
+        Parameters
+        ----------
+        name: str
+            The name of the configuration item.
+
+        Notes
+        -----
+            Only top level configuration items may be unset.
+        """
         if name not in self._configurable_keys:
             raise ConfigNonConfigurableError(name=name)
         if name in self._unset_keys or not self._file.is_item_set(self._config_key, name):
@@ -564,6 +692,14 @@ class Config:
         ret_parts=False,
         default=None,
     ):
+        """
+        Get the value of a configuration item.
+
+        Parameters
+        ----------
+        path: str
+            The name of or path to the configuration item.
+        """
         parts = path.split(".")
         root = deepcopy(self._get(parts[0], callback=callback))
         try:
@@ -582,7 +718,16 @@ class Config:
         return tuple(ret)
 
     def append(self, path, value, is_json=False):
-        """Append a value to a list-like configuration item."""
+        """
+        Append a value to a list-like configuration item.
+
+        Parameters
+        ----------
+        path: str
+            The name of or path to the configuration item.
+        value:
+            The value to append.
+        """
         if is_json:
             value = self._parse_JSON(path, value)
 
@@ -612,7 +757,16 @@ class Config:
         self._set(parts[0], root)
 
     def prepend(self, path, value, is_json=False):
-        """Prepend a value to a list-like configuration item."""
+        """
+        Prepend a value to a list-like configuration item.
+
+        Parameters
+        ----------
+        path: str
+            The name of or path to the configuration item.
+        value:
+            The value to prepend.
+        """
         if is_json:
             value = self._parse_JSON(path, value)
 
@@ -638,7 +792,16 @@ class Config:
         self._set(parts[0], root)
 
     def pop(self, path, index):
-        """Remove a value from a specified index of a list-like configuration item."""
+        """
+        Remove a value from a specified index of a list-like configuration item.
+
+        Parameters
+        ----------
+        path: str
+            The name of or path to the configuration item.
+        index: int
+            Where to remove the value from. 0 for the first item, -1 for the last.
+        """
 
         existing, root, parts = self.get(
             path,
@@ -674,8 +837,10 @@ class Config:
 
         Parameters
         ----------
-        path
+        path: str
             A dot-delimited string of the nested path to update.
+        value: dict
+            A dictionary to merge in.
         """
 
         if is_json:
@@ -739,12 +904,18 @@ class Config:
         self._app.reset_config()
 
     def add_scheduler(self, scheduler, **defaults):
+        """
+        Add a scheduler.
+        """
         if scheduler in self.get("schedulers"):
             print(f"Scheduler {scheduler!r} already exists.")
             return
         self.update(f"schedulers.{scheduler}.defaults", defaults)
 
     def add_shell(self, shell, **defaults):
+        """
+        Add a shell.
+        """
         if shell in self.get("shells"):
             return
         if shell.lower() == "wsl":
@@ -753,6 +924,9 @@ class Config:
         self.update(f"shells.{shell}.defaults", defaults)
 
     def add_shell_WSL(self, **defaults):
+        """
+        Add shell with WSL prefix.
+        """
         if "WSL_executable" not in defaults:
             defaults["WSL_executable"] = "wsl.exe"
         self.add_shell("wsl", **defaults)
@@ -763,13 +937,13 @@ class Config:
 
         Parameters
         ----------
-        file_path
+        file_path:
             Local or remote path to a config import YAML file which may have top-level
             keys "invocation" and "config".
-        rename
+        rename:
             If True, the current config will be renamed to the stem of the file specified
             in `file_path`. Ignored if `make_new` is True.
-        make_new
+        make_new:
             If True, add the config items as a new config, rather than modifying the
             current config. The name of the new config will be the stem of the file
             specified in `file_path`.