hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

hpcflow/_version.py

@@ -1 +1 @@
-__version__ = "0.2.0a179"
+__version__ = "0.2.0a180"

hpcflow/data/demo_data_manifest/__init__.py

@@ -0,0 +1,3 @@
+"""
+Manifest for demonstration data.
+"""

hpcflow/sdk/__init__.py

@@ -4,7 +4,8 @@ import logging
 import os
 import sys
 
-# classes used in the construction of a workflow:
+#: Classes used in the construction of a workflow.
+#: :meta hide-value:
 sdk_classes = {
     "Workflow": "hpcflow.sdk.core.workflow",
     "Task": "hpcflow.sdk.core.task",
@@ -95,6 +96,8 @@ sdk_classes = {
 }
 
 # these are defined as `BaseApp` methods with an underscore prefix:
+#: Functions exported by the application.
+#: :meta hide-value:
 sdk_funcs = (
     "make_workflow",
     "make_demo_workflow",

hpcflow/sdk/app.py

@@ -65,7 +65,7 @@ DEMO_WK_FORMATS = {".yaml": "yaml", ".yml": "yaml", ".json": "json", ".jsonc": "
 
 
 def rate_limit_safe_url_to_fs(app, *args, logger=None, **kwargs):
-    """Call fsspec's `url_to_fs` but retry on `requests.exceptions.HTTPError`s
+    R"""Call fsspec's ``url_to_fs`` but retry on ``requests.exceptions.HTTPError``\ s.
 
     References
     ----------
@@ -125,14 +125,24 @@ def get_app_attribute(name):
 
 
 def get_app_module_all():
+    """
+    The list of all symbols exported by this module.
+    """
     return ["app"] + list(sdk_classes.keys()) + list(sdk_funcs)
 
 
 def get_app_module_dir():
+    """
+    The sorted list of all symbols exported by this module.
+    """
     return lambda: sorted(get_app_module_all())
 
 
 class Singleton(type):
+    """
+    Metaclass that enforces that only one instance of a class can be made.
+    """
+
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
@@ -158,15 +168,42 @@ class BaseApp(metaclass=Singleton):
 
     Parameters
     ----------
+    name:
+        The name of the application.
+    version:
+        The version of the application.
     module:
         The module name in which the app object is defined.
+    description:
+        Description of the application.
+    gh_org:
+        Name of Github organisation responsible for the application.
+    gh_repo:
+        Github repository containing the application source.
+    config_options:
+        Configuration options.
+    scripts_dir:
+        Directory for scripts.
+    workflows_dir:
+        Directory for workflows.
+    demo_data_dir:
+        Directory for demonstration data.
+    data_data_manifest_dir:
+        Directory for demonstration data manifests.
+    template_components:
+        Template components.
+    pytest_args:
+        Arguments for pytest.
+    package_name:
+        Name of package if not the application name.
     docs_import_conv:
         The convention for the app alias used in import statements in the documentation.
         E.g. for the `hpcflow` base app, this is `hf`. This is combined with `module` to
         form the complete import statement. E.g. for the `hpcflow` base app, the complete
         import statement is: `import hpcflow.app as hf`, where `hpcflow.app` is the
         `module` argument and `hf` is the `docs_import_conv` argument.
-
+    docs_url:
+        URL to documentation.
     """
 
     _known_subs_file_name = "known_submissions.txt"
@@ -194,22 +231,38 @@ class BaseApp(metaclass=Singleton):
     ):
         SDK_logger.info(f"Generating {self.__class__.__name__} {name!r}.")
 
+        #: The name of the application.
         self.name = name
+        #: Name of package.
         self.package_name = package_name or name.lower()
+        #: The version of the application.
         self.version = version
+        #: The module name in which the app object is defined.
         self.module = module
+        #: Description of the application.
         self.description = description
+        #: Name of Github organisation responsible for the application.
         self.gh_org = gh_org
+        #: Github repository containing the application source.
         self.gh_repo = gh_repo
+        #: Configuration options.
         self.config_options = config_options
+        #: Arguments for pytest.
         self.pytest_args = pytest_args
+        #: Directory for scripts.
         self.scripts_dir = scripts_dir
+        #: Directory for workflows.
         self.workflows_dir = workflows_dir
+        #: Directory for demonstration data.
         self.demo_data_dir = demo_data_dir
+        #: Directory for demonstration data manifests.
         self.demo_data_manifest_dir = demo_data_manifest_dir
+        #: The convention for the app alias used in import statements in the documentation.
         self.docs_import_conv = docs_import_conv
+        #: URL to documentation.
         self.docs_url = docs_url
 
+        #: Command line interface subsystem.
         self.cli = make_cli(self)
 
         self._log = AppLog(self)
@@ -293,14 +346,23 @@ class BaseApp(metaclass=Singleton):
 
     @property
     def run_time_info(self) -> RunTimeInfo:
+        """
+        Information about the runtime.
+        """
         return self._run_time_info
 
     @property
     def log(self) -> AppLog:
+        """
+        The application log.
+        """
         return self._log
 
     @property
     def timeit(self) -> bool:
+        """
+        Whether the timing analysis system is active.
+        """
         return TimeIt.active
 
     @timeit.setter
@@ -309,6 +371,9 @@ class BaseApp(metaclass=Singleton):
 
     @property
     def template_components(self) -> Dict[str, ObjectList]:
+        """
+        The template component data.
+        """
         if not self.is_template_components_loaded:
             self._load_template_components()
         return self._template_components
@@ -406,6 +471,10 @@ class BaseApp(metaclass=Singleton):
     def load_builtin_template_component_data(
         cls, package
     ) -> Dict[str, Union[List, Dict]]:
+        """
+        Load the template component data built into the package.
+        This is as opposed to the template components defined by users.
+        """
         SDK_logger.info(
             f"Loading built-in template component data for package: {package!r}."
         )
@@ -426,74 +495,119 @@ class BaseApp(metaclass=Singleton):
 
     @property
     def parameters(self) -> get_app_attribute("ParametersList"):
+        """
+        The known template parameters.
+        """
         self._ensure_template_component("parameters")
         return self._parameters
 
     @property
     def command_files(self) -> get_app_attribute("CommandFilesList"):
+        """
+        The known template command files.
+        """
         self._ensure_template_component("command_files")
         return self._command_files
 
     @property
     def envs(self) -> get_app_attribute("EnvironmentsList"):
+        """
+        The known template execution environments.
+        """
         self._ensure_template_component("environments")
         return self._environments
 
     @property
     def scripts(self):
+        """
+        The known template scripts.
+        """
         self._ensure_template_component("scripts")
         return self._scripts
 
     @property
     def task_schemas(self) -> get_app_attribute("TaskSchemasList"):
+        """
+        The known template task schemas.
+        """
         self._ensure_template_component("task_schemas")
         return self._task_schemas
 
     @property
     def logger(self) -> Logger:
+        """
+        The main underlying logger.
+        """
         return self.log.logger
 
     @property
     def API_logger(self) -> Logger:
+        """
+        The logger for API messages.
+        """
         return self.logger.getChild("api")
 
     @property
     def CLI_logger(self) -> Logger:
+        """
+        The logger for CLI messages.
+        """
         return self.logger.getChild("cli")
 
     @property
     def config_logger(self) -> Logger:
+        """
+        The logger for configuration messages.
+        """
         return self.logger.getChild("config")
 
     @property
     def persistence_logger(self) -> Logger:
+        """
+        The logger for persistence engine messages.
+        """
         return self.logger.getChild("persistence")
 
     @property
     def submission_logger(self) -> Logger:
+        """
+        The logger for job submission messages.
+        """
         return self.logger.getChild("submission")
 
     @property
     def runtime_info_logger(self) -> Logger:
+        """
+        The logger for runtime messages.
+        """
         return self.logger.getChild("runtime")
 
     @property
     def is_config_loaded(self) -> bool:
+        """
+        Whether the configuration is loaded.
+        """
         return bool(self._config)
 
     @property
     def is_template_components_loaded(self) -> bool:
-        """Return True if any template component (e.g. parameters) has been loaded."""
+        """Whether any template component (e.g. parameters) has been loaded."""
         return bool(self._template_components)
 
     @property
     def config(self) -> Config:
+        """
+        The configuration.
+        """
         if not self.is_config_loaded:
             self.load_config()
         return self._config
 
     @property
     def scheduler_lookup(self):
+        """
+        The scheduler mapping.
+        """
         return {
             ("direct", "posix"): self.DirectPosix,
             ("direct", "nt"): self.DirectWindows,
@@ -550,35 +664,42 @@ class BaseApp(metaclass=Singleton):
 
     @property
     def user_data_dir(self) -> Path:
+        """
+        The user's data directory.
+        """
         if self._user_data_dir is None:
             self._user_data_dir = Path(user_data_dir(appname=self.package_name))
         return self._user_data_dir
 
     @property
     def user_cache_dir(self) -> Path:
-        """Retrieve the app cache directory."""
+        """The user's cache directory."""
         if self._user_cache_dir is None:
             self._user_cache_dir = Path(user_cache_path(appname=self.package_name))
         return self._user_cache_dir
 
     @property
     def user_runtime_dir(self) -> Path:
-        """Retrieve a temporary directory."""
+        """The user's temporary runtime directory."""
         if self._user_runtime_dir is None:
             self._user_runtime_dir = self.user_data_dir.joinpath("temp")
         return self._user_runtime_dir
 
     @property
     def demo_data_cache_dir(self) -> Path:
-        """Retrieve a directory for example data caching."""
+        """A directory for example data caching."""
         if self._demo_data_cache_dir is None:
             self._demo_data_cache_dir = self.user_cache_dir.joinpath("demo_data")
         return self._demo_data_cache_dir
 
     @property
     def user_data_hostname_dir(self) -> Path:
-        """We segregate by hostname to account for the case where multiple machines might
-        use the same shared file system"""
+        """
+        The directory for holding user data.
+
+        We segregate by hostname to account for the case where multiple machines might
+        use the same shared file system.
+        """
 
         # This might need to cover e.g. multiple login nodes, as described in the
         # config file:
@@ -589,7 +710,7 @@ class BaseApp(metaclass=Singleton):
 
     @property
     def user_cache_hostname_dir(self) -> Path:
-        """Retrieve the hostname-scoped app cache directory."""
+        """The hostname-scoped app cache directory."""
         if self._user_cache_hostname_dir is None:
             machine_name = self.config.get("machine")
             self._user_cache_hostname_dir = self.user_cache_dir.joinpath(machine_name)
@@ -718,11 +839,17 @@ class BaseApp(metaclass=Singleton):
         warn=True,
         **overrides,
     ) -> None:
+        """
+        Load the user's configuration.
+        """
         if warn and self.is_config_loaded:
             warnings.warn("Configuration is already loaded; reloading.")
         self._load_config(config_dir, config_key, **overrides)
 
     def unload_config(self):
+        """
+        Discard any loaded configuration.
+        """
         self._config_files = {}
         self._config = None
 
@@ -762,6 +889,10 @@ class BaseApp(metaclass=Singleton):
         warn=True,
         **overrides,
     ) -> None:
+        """
+        Reload the configuration. Use if a user has updated the configuration file
+        outside the scope of this application.
+        """
         if warn and not self.is_config_loaded:
             warnings.warn("Configuration is not loaded; loading.")
         self.log.remove_file_handlers()
@@ -909,7 +1040,10 @@ class BaseApp(metaclass=Singleton):
         with self.get_demo_workflow_template_file(name) as path:
             return self.WorkflowTemplate.from_file(path)
 
-    def template_components_from_json_like(self, json_like) -> None:
+    def template_components_from_json_like(self, json_like):
+        """
+        Get template components from a (simply parsed) JSOM document.
+        """
         cls_lookup = {
             "parameters": self.ParametersList,
             "command_files": self.CommandFilesList,
@@ -944,6 +1078,9 @@ class BaseApp(metaclass=Singleton):
         return param_map
 
     def get_info(self) -> Dict[str, Any]:
+        """
+        Get miscellaneous runtime system information.
+        """
         return {
             "name": self.name,
             "version": self.version,
@@ -951,13 +1088,12 @@ class BaseApp(metaclass=Singleton):
             "is_frozen": self.run_time_info.is_frozen,
         }
 
-    @property
-    def known_subs_file_name(self):
-        return self._known_subs_file_name
-
     @property
     def known_subs_file_path(self):
-        return self.user_data_hostname_dir / self.known_subs_file_name
+        """
+        The path to the file describing known submissions.
+        """
+        return self.user_data_hostname_dir / self._known_subs_file_name
 
     def _format_known_submissions_line(
         self,
@@ -2242,6 +2378,9 @@ class BaseApp(metaclass=Singleton):
         use_current_env=False,
         env_source_file=None,
     ):
+        """
+        Configure an execution environment.
+        """
         if not setup:
             setup = []
         if not executables:
@@ -2505,9 +2644,15 @@ class BaseApp(metaclass=Singleton):
         return cache_file_path
 
     def cache_demo_data_file(self, file_name) -> Path:
+        """
+        Get the name of a cached demo data file.
+        """
         return self.get_demo_data_file_path(file_name)
 
     def cache_all_demo_data_files(self) -> List[Path]:
+        """
+        Get the name of all cached demo data file.
+        """
         return [self.get_demo_data_file_path(i) for i in self.list_demo_data_files()]
 
     def copy_demo_data(

hpcflow/sdk/cli.py

@@ -1,3 +1,7 @@
+"""
+Command line interface implementation.
+"""
+
 import json
 import os
 from typing import Dict, List
@@ -38,26 +42,36 @@ from hpcflow.sdk.cli_common import (
     rechunk_backup_opt,
     rechunk_chunk_size_opt,
     rechunk_status_opt,
+    _add_doc_from_help,
 )
 from hpcflow.sdk.helper.cli import get_helper_CLI
 from hpcflow.sdk.log import TimeIt
 from hpcflow.sdk.submission.shells import ALL_SHELLS
 
+#: Standard option
 string_option = click.option(
     "--string",
     is_flag=True,
     default=False,
     help="Determines if passing a file path or a string.",
 )
+#: Standard option
 workflow_ref_type_opt = click.option(
     "--ref-type",
     "-r",
     type=click.Choice(["assume-id", "id", "path"]),
     default="assume-id",
+    help="How to interpret a reference, as an ID, a path, or to guess.",
 )
 
 
+_add_doc_from_help(string_option, workflow_ref_type_opt)
+
+
 def parse_jobscript_wait_spec(jobscripts: str) -> Dict[int, List[int]]:
+    """
+    Parse a jobscript wait specification.
+    """
     sub_js_idx_dct = {}
     for sub_i in jobscripts.split(";"):
         sub_idx_str, js_idx_lst_str = sub_i.split(":")