hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a180__py3-none-any.whl

hpcflow/sdk/submission/schedulers/direct.py

@@ -1,3 +1,7 @@
+"""
+A direct job "scheduler" that just runs immediate subprocesses.
+"""
+
 from pathlib import Path
 import shutil
 import signal
@@ -11,6 +15,22 @@ from hpcflow.sdk.submission.shells.base import Shell
 
 
 class DirectScheduler(NullScheduler):
+    """
+    A direct scheduler, that just runs jobs immediately as direct subprocesses.
+
+    The correct subclass (:py:class:`DirectPosix` or :py:class:`DirectWindows`) should
+    be used to create actual instances.
+
+    Keyword Args
+    ------------
+    shell_args: str
+        Arguments to pass to the shell. Pre-quoted.
+    shebang_args: str
+        Arguments to set on the shebang line. Pre-quoted.
+    options: dict
+        Options to the jobscript command.
+    """
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -29,6 +49,9 @@ class DirectScheduler(NullScheduler):
         js_path: str,
         deps: List[Tuple],
     ) -> List[str]:
+        """
+        Get the concrete submission command.
+        """
         return shell.get_direct_submit_command(js_path)
 
     @staticmethod
@@ -104,6 +127,10 @@ class DirectScheduler(NullScheduler):
         js_refs: List[Tuple[int, List[str]]],
         jobscripts: List = None,
     ):
+        """
+        Cancel some jobs.
+        """
+
         def callback(proc):
             try:
                 js = js_proc_id[proc.pid]
@@ -145,7 +172,21 @@ class DirectScheduler(NullScheduler):
 
 
 class DirectPosix(DirectScheduler):
+    """
+    A direct scheduler for POSIX systems.
+
+    Keyword Args
+    ------------
+    shell_args: str
+        Arguments to pass to the shell. Pre-quoted.
+    shebang_args: str
+        Arguments to set on the shebang line. Pre-quoted.
+    options: dict
+        Options to the jobscript command.
+    """
+
     _app_attr = "app"
+    #: Default shell.
     DEFAULT_SHELL_EXECUTABLE = "/bin/bash"
 
     def __init__(self, *args, **kwargs):
@@ -153,7 +194,19 @@ class DirectPosix(DirectScheduler):
 
 
 class DirectWindows(DirectScheduler):
+    """
+    A direct scheduler for Windows.
+
+    Keyword Args
+    ------------
+    shell_args: str
+        Arguments to pass to the shell. Pre-quoted.
+    options: dict
+        Options to the jobscript command.
+    """
+
     _app_attr = "app"
+    #: Default shell.
     DEFAULT_SHELL_EXECUTABLE = "powershell.exe"
 
     def __init__(self, *args, **kwargs):

hpcflow/sdk/submission/schedulers/sge.py

@@ -1,3 +1,7 @@
+"""
+An interface to SGE.
+"""
+
 from pathlib import Path
 import re
 from typing import Dict, List, Tuple
@@ -15,6 +19,18 @@ from hpcflow.sdk.submission.shells.base import Shell
 
 class SGEPosix(Scheduler):
     """
+    A scheduler that uses SGE.
+
+    Keyword Args
+    ------------
+    cwd_switch: str
+        Override of default switch to use to set the current working directory.
+    shell_args: str
+        Arguments to pass to the shell. Pre-quoted.
+    shebang_args: str
+        Arguments to set on the shebang line. Pre-quoted.
+    options: dict
+        Options to the jobscript command.
 
     Notes
     -----
@@ -29,17 +45,26 @@ class SGEPosix(Scheduler):
 
     _app_attr = "app"
 
+    #: Default args for shebang line.
     DEFAULT_SHEBANG_ARGS = ""
+    #: Default submission command.
     DEFAULT_SUBMIT_CMD = "qsub"
+    #: Default command to show the queue state.
     DEFAULT_SHOW_CMD = ["qstat"]
+    #: Default cancel command.
     DEFAULT_DEL_CMD = "qdel"
+    #: Default job control directive prefix.
     DEFAULT_JS_CMD = "#$"
+    #: Default prefix to enable array processing.
     DEFAULT_ARRAY_SWITCH = "-t"
+    #: Default shell variable with array ID.
     DEFAULT_ARRAY_ITEM_VAR = "SGE_TASK_ID"
+    #: Default switch to control CWD.
     DEFAULT_CWD_SWITCH = "-cwd"
+    #: Default command to get the login nodes.
     DEFAULT_LOGIN_NODES_CMD = ["qconf", "-sh"]
 
-    # maps scheduler states:
+    #: Maps scheduler state codes to :py:class:`JobscriptElementState` values.
     state_lookup = {
         "qw": JobscriptElementState.pending,
         "hq": JobscriptElementState.waiting,
@@ -136,7 +161,7 @@ class SGEPosix(Scheduler):
         nodes = stdout.strip().split("\n")
         return nodes
 
-    def format_core_request_lines(self, resources):
+    def _format_core_request_lines(self, resources):
         lns = []
         if resources.num_cores > 1:
             lns.append(
@@ -146,10 +171,10 @@ class SGEPosix(Scheduler):
             lns.append(f"{self.js_cmd} -tc {resources.max_array_items}")
         return lns
 
-    def format_array_request(self, num_elements):
+    def _format_array_request(self, num_elements):
         return f"{self.js_cmd} {self.array_switch} 1-{num_elements}"
 
-    def format_std_stream_file_option_lines(self, is_array, sub_idx):
+    def _format_std_stream_file_option_lines(self, is_array, sub_idx):
         # note: we can't modify the file names
         base = f"./artifacts/submissions/{sub_idx}"
         return [
@@ -158,13 +183,16 @@ class SGEPosix(Scheduler):
         ]
 
     def format_options(self, resources, num_elements, is_array, sub_idx):
+        """
+        Format the options to the jobscript command.
+        """
         opts = []
         opts.append(self.format_switch(self.cwd_switch))
-        opts.extend(self.format_core_request_lines(resources))
+        opts.extend(self._format_core_request_lines(resources))
         if is_array:
-            opts.append(self.format_array_request(num_elements))
+            opts.append(self._format_array_request(num_elements))
 
-        opts.extend(self.format_std_stream_file_option_lines(is_array, sub_idx))
+        opts.extend(self._format_std_stream_file_option_lines(is_array, sub_idx))
 
         for opt_k, opt_v in self.options.items():
             if isinstance(opt_v, list):
@@ -197,6 +225,13 @@ class SGEPosix(Scheduler):
         js_path: str,
         deps: List[Tuple],
     ) -> List[str]:
+        """
+        Get the command to use to submit a job to the scheduler.
+
+        Returns
+        -------
+        List of argument words.
+        """
         cmd = [self.submit_cmd, "-terse"]
 
         dep_job_IDs = []
@@ -285,6 +320,9 @@ class SGEPosix(Scheduler):
         return info
 
     def cancel_jobs(self, js_refs: List[str], jobscripts: List = None):
+        """
+        Cancel submitted jobs.
+        """
         cmd = [self.del_cmd] + js_refs
         self.app.submission_logger.info(
             f"cancelling {self.__class__.__name__} jobscripts with command: {cmd}."

hpcflow/sdk/submission/schedulers/slurm.py

@@ -1,3 +1,7 @@
+"""
+An interface to SLURM.
+"""
+
 from pathlib import Path
 import subprocess
 import time
@@ -18,12 +22,24 @@ from hpcflow.sdk.submission.shells.base import Shell
 
 class SlurmPosix(Scheduler):
     """
+    A scheduler that uses SLURM.
+
+    Keyword Args
+    ------------
+    shell_args: str
+        Arguments to pass to the shell. Pre-quoted.
+    shebang_args: str
+        Arguments to set on the shebang line. Pre-quoted.
+    options: dict
+        Options to the jobscript command.
 
     Notes
     -----
     - runs in current working directory by default [2]
 
-    # TODO: consider getting memory usage like: https://stackoverflow.com/a/44143229/5042280
+    Todo
+    ----
+    - consider getting memory usage like: https://stackoverflow.com/a/44143229/5042280
 
     References
     ----------
@@ -34,16 +50,24 @@ class SlurmPosix(Scheduler):
 
     _app_attr = "app"
 
+    #: Default shell.
     DEFAULT_SHELL_EXECUTABLE = "/bin/bash"
+    #: Default args for shebang line.
     DEFAULT_SHEBANG_ARGS = ""
+    #: Default submission command.
     DEFAULT_SUBMIT_CMD = "sbatch"
+    #: Default command to show the queue state.
     DEFAULT_SHOW_CMD = ["squeue", "--me"]
+    #: Default cancel command.
     DEFAULT_DEL_CMD = "scancel"
+    #: Default job control directive prefix.
     DEFAULT_JS_CMD = "#SBATCH"
+    #: Default prefix to enable array processing.
     DEFAULT_ARRAY_SWITCH = "--array"
+    #: Default shell variable with array ID.
     DEFAULT_ARRAY_ITEM_VAR = "SLURM_ARRAY_TASK_ID"
 
-    # maps scheduler states:
+    #: Maps scheduler state codes to :py:class:`JobscriptElementState` values.
     state_lookup = {
         "PENDING": JobscriptElementState.pending,
         "RUNNING": JobscriptElementState.running,
@@ -301,7 +325,7 @@ class SlurmPosix(Scheduler):
             if part_match:
                 resources.SLURM_partition = part_match
 
-    def format_core_request_lines(self, resources):
+    def _format_core_request_lines(self, resources):
         lns = []
         if resources.SLURM_partition:
             lns.append(f"{self.js_cmd} --partition {resources.SLURM_partition}")
@@ -324,13 +348,13 @@ class SlurmPosix(Scheduler):
 
         return lns
 
-    def format_array_request(self, num_elements, resources):
+    def _format_array_request(self, num_elements, resources):
         # TODO: Slurm docs start indices at zero, why are we starting at one?
         #   https://slurm.schedmd.com/sbatch.html#OPT_array
         max_str = f"%{resources.max_array_items}" if resources.max_array_items else ""
         return f"{self.js_cmd} {self.array_switch} 1-{num_elements}{max_str}"
 
-    def format_std_stream_file_option_lines(self, is_array, sub_idx):
+    def _format_std_stream_file_option_lines(self, is_array, sub_idx):
         base = r"%x_"
         if is_array:
             base += r"%A.%a"
@@ -344,12 +368,15 @@ class SlurmPosix(Scheduler):
         ]
 
     def format_options(self, resources, num_elements, is_array, sub_idx):
+        """
+        Format the options to the scheduler.
+        """
         opts = []
-        opts.extend(self.format_core_request_lines(resources))
+        opts.extend(self._format_core_request_lines(resources))
         if is_array:
-            opts.append(self.format_array_request(num_elements, resources))
+            opts.append(self._format_array_request(num_elements, resources))
 
-        opts.extend(self.format_std_stream_file_option_lines(is_array, sub_idx))
+        opts.extend(self._format_std_stream_file_option_lines(is_array, sub_idx))
 
         for opt_k, opt_v in self.options.items():
             if isinstance(opt_v, list):
@@ -387,6 +414,13 @@ class SlurmPosix(Scheduler):
         js_path: str,
         deps: List[Tuple],
     ) -> List[str]:
+        """
+        Get the command to use to submit a job to the scheduler.
+
+        Returns
+        -------
+        List of argument words.
+        """
         cmd = [self.submit_cmd, "--parsable"]
 
         dep_cmd = []
@@ -528,6 +562,9 @@ class SlurmPosix(Scheduler):
         return info
 
     def cancel_jobs(self, js_refs: List[str], jobscripts: List = None):
+        """
+        Cancel submitted jobs.
+        """
         cmd = [self.del_cmd] + js_refs
         self.app.submission_logger.info(
             f"cancelling {self.__class__.__name__} jobscripts with command: {cmd}."

hpcflow/sdk/submission/schedulers/utils.py

@@ -1,3 +1,7 @@
+"""
+Helper for running a subprocess.
+"""
+
 import subprocess
 
 

hpcflow/sdk/submission/shells/__init__.py

@@ -1,3 +1,6 @@
+"""
+Adapters for various shells.
+"""
 import os
 from typing import Dict, Optional
 
@@ -7,6 +10,7 @@ from .base import Shell
 from .bash import Bash, WSLBash
 from .powershell import WindowsPowerShell
 
+#: All supported shells.
 ALL_SHELLS = {
     "bash": {"posix": Bash},
     "powershell": {"nt": WindowsPowerShell},
@@ -14,7 +18,7 @@ ALL_SHELLS = {
     "wsl": {"nt": WSLBash},  # TODO: cast this to wsl+bash in ResourceSpec?
 }
 
-# used to set the default shell in the default config:
+#: The default shell in the default config.
 DEFAULT_SHELL_NAMES = {
     "posix": "bash",
     "nt": "powershell",
@@ -22,11 +26,17 @@ DEFAULT_SHELL_NAMES = {
 
 
 def get_supported_shells(os_name: Optional[str] = None) -> Dict[str, Shell]:
+    """
+    Get shells supported on the current or given OS.
+    """
     os_name = os_name or os.name
     return {k: v.get(os_name) for k, v in ALL_SHELLS.items() if v.get(os_name)}
 
 
 def get_shell(shell_name, os_name: Optional[str] = None, **kwargs) -> Shell:
+    """
+    Get a shell interface with the given name for a given OS (or the current one).
+    """
     # TODO: apply config default shell args?
 
     os_name = os_name or os.name

hpcflow/sdk/submission/shells/base.py

@@ -1,3 +1,7 @@
+"""
+Base model of a shell.
+"""
+
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -10,6 +14,12 @@ class Shell(ABC):
     bash on a POSIX OS, and provides snippets that are used to compose a jobscript for
     that combination.
 
+    Parameters
+    ----------
+    executable: str
+        Which executable implements the shell.
+    os_args:
+        Arguments to pass to the shell.
     """
 
     def __init__(self, executable=None, os_args=None):
@@ -25,10 +35,16 @@ class Shell(ABC):
 
     @property
     def executable(self) -> List[str]:
+        """
+        The executable to use plus any mandatory arguments.
+        """
         return [self._executable]
 
     @property
-    def shebang_executable(self) -> List[str]:
+    def shebang_executable(self) -> str:
+        """
+        The executable to use in a shebang line.
+        """
         return self.executable
 
     def get_direct_submit_command(self, js_path) -> List[str]:
@@ -40,6 +56,9 @@ class Shell(ABC):
         """Get shell and operating system information."""
 
     def get_wait_command(self, workflow_app_alias: str, sub_idx: int, deps: Dict):
+        """
+        Get the command to wait for a workflow.
+        """
         if deps:
             return (
                 f'{workflow_app_alias} workflow $WK_PATH_ARG wait --jobscripts "{sub_idx}:'
@@ -51,9 +70,15 @@ class Shell(ABC):
 
     @staticmethod
     def process_app_invoc_executable(app_invoc_exe):
+        """
+        Perform any post-processing of an application invocation command name.
+        """
         return app_invoc_exe
 
     def process_JS_header_args(self, header_args: Dict) -> Dict:
+        """
+        Process the application invocation key in the jobscript header arguments.
+        """
         app_invoc = self.process_app_invoc_executable(header_args["app_invoc"][0])
         if len(header_args["app_invoc"]) > 1:
             app_invoc += ' "' + header_args["app_invoc"][1] + '"'
@@ -62,7 +87,13 @@ class Shell(ABC):
         return header_args
 
     def prepare_JS_path(self, js_path: Path) -> str:
+        """
+        Prepare the jobscript path for use.
+        """
         return str(js_path)
 
     def prepare_element_run_dirs(self, run_dirs: List[List[Path]]) -> List[List[str]]:
+        """
+        Prepare the element run directory names for use.
+        """
         return [[str(j) for j in i] for i in run_dirs]

hpcflow/sdk/submission/shells/bash.py

@@ -1,3 +1,7 @@
+"""
+Shell models based on the Bourne-Again Shell.
+"""
+
 from pathlib import Path
 import subprocess
 from textwrap import dedent, indent
@@ -11,14 +15,22 @@ from hpcflow.sdk.submission.shells.os_version import (
 
 
 class Bash(Shell):
-    """Class to represent using bash on a POSIX OS to generate and submit a jobscript."""
+    """
+    Class to represent using bash on a POSIX OS to generate and submit a jobscript.
+    """
 
+    #: Default for executable name.
     DEFAULT_EXE = "/bin/bash"
 
+    #: File extension for jobscripts.
     JS_EXT = ".sh"
+    #: Basic indent.
     JS_INDENT = "  "
+    #: Indent for environment setup.
     JS_ENV_SETUP_INDENT = 2 * JS_INDENT
+    #: Template for the jobscript shebang line.
     JS_SHEBANG = """#!{shebang_executable} {shebang_args}"""
+    #: Template for the common part of the jobscript header.
     JS_HEADER = dedent(
         """\
         {workflow_app_alias} () {{
@@ -39,6 +51,7 @@ class Bash(Shell):
         ELEM_RUN_DIR_FILE="$WK_PATH/artifacts/submissions/${{SUB_IDX}}/{element_run_dirs_file_path}"
     """
     )
+    #: Template for the jobscript header when scheduled.
     JS_SCHEDULER_HEADER = dedent(
         """\
         {shebang}
@@ -47,6 +60,7 @@ class Bash(Shell):
         {header}
     """
     )
+    #: Template for the jobscript header when directly executed.
     JS_DIRECT_HEADER = dedent(
         """\
         {shebang}
@@ -55,6 +69,7 @@ class Bash(Shell):
         {wait_command}
     """
     )
+    #: Template for the jobscript body.
     JS_MAIN = dedent(
         """\
         elem_EAR_IDs=`sed "$((${{JS_elem_idx}} + 1))q;d" "$EAR_ID_FILE"`
@@ -103,6 +118,7 @@ class Bash(Shell):
         done
     """
     )
+    #: Template for the element processing loop in a jobscript.
     JS_ELEMENT_LOOP = dedent(
         """\
         for ((JS_elem_idx=0;JS_elem_idx<{num_elements};JS_elem_idx++))
@@ -112,6 +128,7 @@ class Bash(Shell):
         cd "$WK_PATH"
     """
     )
+    #: Template for the array handling code in a jobscript.
     JS_ELEMENT_ARRAY = dedent(
         """\
         JS_elem_idx=$(({scheduler_array_item_var} - 1))
@@ -125,6 +142,9 @@ class Bash(Shell):
 
     @property
     def linux_release_file(self):
+        """
+        The name of the file describing the Linux version.
+        """
         return self.os_args["linux_release_file"]
 
     def _get_OS_info_POSIX(self):
@@ -169,6 +189,9 @@ class Bash(Shell):
         return app_invoc_exe
 
     def format_stream_assignment(self, shell_var_name, command):
+        """
+        Produce code to assign the output of the command to a shell variable.
+        """
         return f"{shell_var_name}=`{command}`"
 
     def format_save_parameter(
@@ -180,6 +203,9 @@ class Bash(Shell):
         cmd_idx: int,
         stderr: bool,
     ):
+        """
+        Produce code to save a parameter's value into the workflow persistent store.
+        """
         # TODO: quote shell_var_name as well? e.g. if it's a white-space delimited list?
         #   and test.
         stderr_str = " --stderr" if stderr else ""
@@ -192,6 +218,9 @@ class Bash(Shell):
         )
 
     def format_loop_check(self, workflow_app_alias: str, loop_name: str, run_ID: int):
+        """
+        Produce code to check the looping status of part of a workflow.
+        """
         return (
             f"{workflow_app_alias} "
             f'internal workflow "$WK_PATH_ARG" check-loop '
@@ -248,8 +277,14 @@ class Bash(Shell):
 
 
 class WSLBash(Bash):
+    """
+    A variant of bash that handles running under WSL on Windows.
+    """
+
+    #: Default name of the WSL interface executable.
     DEFAULT_WSL_EXE = "wsl"
 
+    #: Template for the common part of the jobscript header.
     JS_HEADER = Bash.JS_HEADER.replace(
         'WK_PATH_ARG="$WK_PATH"',
         'WK_PATH_ARG=`wslpath -m "$WK_PATH"`',

hpcflow/sdk/submission/shells/os_version.py

@@ -1,3 +1,7 @@
+"""
+Operating system information discovery helpers.
+"""
+
 import os
 import platform
 import re
@@ -8,6 +12,9 @@ DEFAULT_LINUX_RELEASE_FILE = "/etc/os-release"
 
 
 def get_OS_info() -> Dict:
+    """
+    Get basic operating system version info.
+    """
     uname = platform.uname()
     return {
         "OS_name": uname.system,
@@ -17,6 +24,9 @@ def get_OS_info() -> Dict:
 
 
 def get_OS_info_windows() -> Dict:
+    """
+    Get operating system version info: Windows version.
+    """
     return get_OS_info()
 
 
@@ -26,15 +36,17 @@ def get_OS_info_POSIX(
     linux_release_file: Optional[str] = None,
 ) -> Dict:
     """
+    Get operating system version info: POSIX version.
+
     Parameters
     ----------
-    WSL_executable
+    WSL_executable:
         Executable to run subprocess calls via WSL on Windows.
-    use_py
-        If True, use the `platform.uname` Python function to get the OS information.
-        Otherwise use subprocess to call `uname`. We set this to False when getting OS
-        info in WSL on Windows, since we need to call the WSL executable.
-    linux_release_file
+    use_py:
+        If True, use the :py:func:`platform.uname` Python function to get the OS
+        information. Otherwise use subprocess to call ``uname``. We set this to False
+        when getting OS info in WSL on Windows, since we need to call the WSL executable.
+    linux_release_file:
         If on Linux, record the name and version fields from this file.
 
     """