hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl

hpcflow/sdk/core/zarr_io.py

@@ -1,3 +1,7 @@
+"""
+Utilities for working with Zarr.
+"""
+
 from typing import Any, Dict, Union
 
 import zarr
@@ -68,6 +72,9 @@ def _zarr_encode(obj, zarr_group, path=None, encoded=None):
 
 
 def zarr_encode(data, zarr_group, is_pending_add, is_set):
+    """
+    Encode data into a zarr group.
+    """
     data, encoded = _zarr_encode(data, zarr_group)
     zarr_group.attrs["encoded"] = encoded
     zarr_group.attrs["data"] = data
@@ -176,6 +183,9 @@ def zarr_decode(
     path=None,
     dataset_copy=False,
 ):
+    """
+    Decode data from a zarr group.
+    """
     if param_data is None:
         return None
 
@@ -204,19 +214,32 @@ def zarr_decode(
 
 
 class ZarrEncodable:
+    """
+    Base class of data that can be converted to and from zarr form.
+    """
+
     _typ = None
 
     def to_dict(self):
+        """
+        Convert this object to a dict.
+        """
         if hasattr(self, "__dict__"):
             return dict(self.__dict__)
         elif hasattr(self, "__slots__"):
             return {k: getattr(self, k) for k in self.__slots__}
 
     def to_zarr(self, zarr_group):
+        """
+        Save this object into the given zarr group.
+        """
         data = self.to_dict()
         zarr_encode(data, zarr_group)
 
     @classmethod
     def from_zarr(cls, zarr_group, dataset_copy=False):
+        """
+        Read an instance of this class from the given zarr group.
+        """
         data = zarr_decode(zarr_group, dataset_copy=dataset_copy)
         return cls(**data)

hpcflow/sdk/data/__init__.py

@@ -0,0 +1,13 @@
+"""
+YAML schemas.
+
+Contents:
+
+* ``config_file_schema.yaml`` – Schema for configuration selection files.
+* ``config_schema.yaml`` – Schema for configuration files.
+* ``environments_spec_schema.yaml`` – Schema for execution environment definition files.
+* ``files_spec_schema.yaml`` – Schema for input/output specification files.
+* ``parameters_spec_schema.yaml`` – Schema for parameter specification files.
+* ``task_schema_spec_schema.yaml`` – Schema for task template specification files.
+* ``workflow_spec_schema.yaml`` – Schema for workflow files.
+"""

hpcflow/sdk/demo/__init__.py

@@ -0,0 +1,3 @@
+"""
+Demonstration code.
+"""

hpcflow/sdk/helper/__init__.py

@@ -0,0 +1,3 @@
+"""
+Helpers for the CLI.
+"""

hpcflow/sdk/helper/cli.py

@@ -1,3 +1,7 @@
+"""
+Common Click command line options related to the helper.
+"""
+
 from datetime import timedelta
 
 import click
@@ -18,7 +22,9 @@ from .helper import (
     get_helper_PID,
     get_helper_uptime,
 )
+from ..cli_common import _add_doc_from_help
 
+#: Helper option: ``--timeout``
 timeout_option = click.option(
     "--timeout",
     type=click.FLOAT,
@@ -26,6 +32,7 @@ timeout_option = click.option(
     show_default=True,
     help="Helper timeout in seconds.",
 )
+#: Helper option: ``--timeout-check-interval``
 timeout_check_interval_option = click.option(
     "--timeout-check-interval",
     type=click.FLOAT,
@@ -33,6 +40,7 @@ timeout_check_interval_option = click.option(
     show_default=True,
     help="Interval between testing if the timeout has been exceeded in seconds.",
 )
+#: Helper option: ``--watch interval``
 watch_interval_option = click.option(
     "--watch-interval",
     type=click.FLOAT,
@@ -43,6 +51,7 @@ watch_interval_option = click.option(
         "seconds."
     ),
 )
+_add_doc_from_help(timeout_option, timeout_check_interval_option, watch_interval_option)
 
 
 def get_helper_CLI(app):

hpcflow/sdk/helper/helper.py

@@ -1,3 +1,7 @@
+"""
+Implementation of a helper process used to monitor jobs.
+"""
+
 from datetime import datetime, timedelta
 import logging
 from logging.handlers import RotatingFileHandler
@@ -71,6 +75,9 @@ def start_helper(
     watch_interval=DEFAULT_WATCH_INTERVAL,
     logger=None,
 ):
+    """
+    Start the helper process.
+    """
     PID_file = get_PID_file_path(app)
     if PID_file.is_file():
         with PID_file.open("rt") as fp:
@@ -135,11 +142,17 @@ def restart_helper(
     timeout_check_interval=DEFAULT_TIMEOUT_CHECK,
     watch_interval=DEFAULT_WATCH_INTERVAL,
 ):
+    """
+    Restart the helper process.
+    """
     logger = stop_helper(app, return_logger=True)
     start_helper(app, timeout, timeout_check_interval, watch_interval, logger=logger)
 
 
 def get_helper_PID(app):
+    """
+    Get the process ID of the helper process.
+    """
     PID_file = get_PID_file_path(app)
     if not PID_file.is_file():
         print("Helper not running!")
@@ -151,6 +164,9 @@ def get_helper_PID(app):
 
 
 def stop_helper(app, return_logger=False):
+    """
+    Stop the helper process.
+    """
     logger = get_helper_logger(app)
     pid_info = get_helper_PID(app)
     if pid_info:
@@ -168,6 +184,9 @@ def stop_helper(app, return_logger=False):
 
 
 def clear_helper(app):
+    """
+    Stop the helper or remove any stale information relating to it.
+    """
     try:
         stop_helper(app)
     except psutil.NoSuchProcess:
@@ -179,6 +198,9 @@ def clear_helper(app):
 
 
 def get_helper_uptime(app):
+    """
+    Get the amount of time that the helper has been running.
+    """
     pid_info = get_helper_PID(app)
     if pid_info:
         proc = psutil.Process(pid_info[0])
@@ -188,6 +210,9 @@ def get_helper_uptime(app):
 
 
 def get_helper_logger(app):
+    """
+    Get the logger for helper-related messages.
+    """
     log_path = get_helper_log_path(app)
     logger = logging.getLogger(__name__)
     logger.setLevel(logging.INFO)
@@ -225,6 +250,9 @@ def run_helper(
     timeout_check_interval=DEFAULT_TIMEOUT_CHECK,
     watch_interval=DEFAULT_WATCH_INTERVAL,
 ):
+    """
+    Run the helper core.
+    """
     # TODO: when writing to watch_workflows from a workflow, copy, modify and then rename
     # this will be atomic - so there will be only one event fired.
     # Also return a local run ID (the position in the file) to be used in jobscript naming

hpcflow/sdk/helper/watcher.py

@@ -1,3 +1,7 @@
+"""
+File-system watcher classes.
+"""
+
 from datetime import timedelta
 from pathlib import Path
 from watchdog.observers.polling import PollingObserver
@@ -5,6 +9,10 @@ from watchdog.events import FileSystemEventHandler, PatternMatchingEventHandler
 
 
 class MonitorController:
+    """
+    Controller for tracking watch files.
+    """
+
     def __init__(self, workflow_dirs_file_path, watch_interval, logger):
 
         if isinstance(watch_interval, timedelta):
@@ -46,6 +54,9 @@ class MonitorController:
 
     @staticmethod
     def parse_watch_workflows_file(path, logger):
+        """
+        Parse the file describing what workflows to watch.
+        """
         # TODO: and parse element IDs as well; and record which are set/unset.
         with Path(path).open("rt") as fp:
             lns = fp.readlines()
@@ -69,20 +80,33 @@ class MonitorController:
         return wks
 
     def on_modified(self, event):
+        """
+        Callback when files are modified.
+        """
         self.logger.info(f"Watch file modified: {event.src_path}")
         wks = self.parse_watch_workflows_file(event.src_path, logger=self.logger)
         self.workflow_monitor.update_workflow_paths(wks)
 
     def join(self):
+        """
+        Join the worker thread.
+        """
         self.observer.join()
 
     def stop(self):
+        """
+        Stop this monitor.
+        """
         self.observer.stop()
         self.observer.join()  # wait for it to stop!
         self.workflow_monitor.stop()
 
 
 class WorkflowMonitor:
+    """
+    Workflow monitor.
+    """
+
     def __init__(self, workflow_paths, watch_interval, logger):
 
         if isinstance(watch_interval, timedelta):
@@ -106,15 +130,24 @@ class WorkflowMonitor:
         self.observer.start()
 
     def on_modified(self, event):
+        """
+        Triggered on a workflow being modified.
+        """
         self.logger.info(f"Workflow modified: {event.src_path}")
 
     def update_workflow_paths(self, new_paths):
+        """
+        Change the set of paths to monitored workflows.
+        """
         self.logger.info(f"Updating watched workflows.")
         self.stop()
         self.workflow_paths = new_paths
         self._monitor_workflow_paths()
 
     def stop(self):
+        """
+        Stop this monitor.
+        """
         if self.observer:
             self.observer.stop()
             self.observer.join()  # wait for it to stop!

hpcflow/sdk/log.py

@@ -1,3 +1,7 @@
+"""
+Interface to the standard logger, and performance logging utility.
+"""
+
 from functools import wraps
 import logging
 from pathlib import Path
@@ -7,17 +11,31 @@ import statistics
 
 
 class TimeIt:
+    """
+    Method execution time instrumentation.
+    """
 
+    #: Whether the instrumentation is active.
     active = False
+    #: Where to log to.
     file_path = None
+    #: The details be tracked.
     timers = defaultdict(list)
+    #: Traces of the stack.
     trace = []
+    #: Trace indices.
     trace_idx = []
+    #: Preceding traces.
     trace_prev = []
+    #: Preceding trace indices.
     trace_idx_prev = []
 
     @classmethod
     def decorator(cls, func):
+        """
+        Decorator for a method that is to have its execution time monitored.
+        """
+
         @wraps(func)
         def wrapper(*args, **kwargs):
 
@@ -51,6 +69,9 @@ class TimeIt:
 
     @classmethod
     def summarise(cls):
+        """
+        Produce a machine-readable summary of method execution time statistics.
+        """
         stats = {
             k: {
                 "number": len(v),
@@ -81,6 +102,10 @@ class TimeIt:
 
     @classmethod
     def summarise_string(cls):
+        """
+        Produce a human-readable summary of method execution time statistics.
+        """
+
         def _format_nodes(node, depth=0, depth_final=None):
             if depth_final is None:
                 depth_final = []
@@ -126,13 +151,22 @@ class TimeIt:
 
 
 class AppLog:
+    """
+    Application log control.
+    """
+
+    #: Default logging level for the console.
     DEFAULT_LOG_CONSOLE_LEVEL = "WARNING"
+    #: Default logging level for log files.
     DEFAULT_LOG_FILE_LEVEL = "INFO"
 
     def __init__(self, app, log_console_level=None):
+        #: The application context.
         self.app = app
+        #: The base logger for the application.
         self.logger = logging.getLogger(app.package_name)
         self.logger.setLevel(logging.DEBUG)
+        #: The handler for directing logging messages to the console.
         self.console_handler = self._add_console_logger(
             level=log_console_level or AppLog.DEFAULT_LOG_CONSOLE_LEVEL
         )
@@ -146,10 +180,16 @@ class AppLog:
         return handler
 
     def update_console_level(self, new_level):
+        """
+        Set the logging level for console messages.
+        """
         if new_level:
             self.console_handler.setLevel(new_level.upper())
 
     def add_file_logger(self, path, level=None, fmt=None, max_bytes=None):
+        """
+        Add a log file.
+        """
         fmt = fmt or f"%(asctime)s %(levelname)s %(name)s: %(message)s"
         level = level or AppLog.DEFAULT_LOG_FILE_LEVEL
         max_bytes = max_bytes or int(10e6)

hpcflow/sdk/persistence/__init__.py

@@ -1,3 +1,7 @@
+"""
+Workflow persistence subsystem.
+"""
+
 import copy
 from pathlib import Path
 import random
@@ -11,21 +15,27 @@ from hpcflow.sdk.persistence.base import PersistentStore
 from hpcflow.sdk.persistence.json import JSONPersistentStore
 from hpcflow.sdk.persistence.zarr import ZarrPersistentStore, ZarrZipPersistentStore
 
-ALL_STORE_CLS = {
+_ALL_STORE_CLS = {
     "zarr": ZarrPersistentStore,
     "zip": ZarrZipPersistentStore,
     "json": JSONPersistentStore,
     # "json-single": JSONPersistentStore,  # TODO
 }
+#: The name of the default persistence store.
 DEFAULT_STORE_FORMAT = "zarr"
-ALL_STORE_FORMATS = tuple(ALL_STORE_CLS.keys())
+#: The persistence formats supported.
+ALL_STORE_FORMATS = tuple(_ALL_STORE_CLS.keys())
+#: The persistence formats supported for creation.
 ALL_CREATE_STORE_FORMATS = tuple(
-    k for k, v in ALL_STORE_CLS.items() if v._features.create
+    k for k, v in _ALL_STORE_CLS.items() if v._features.create
 )
 
 
 def store_cls_from_str(store_format: str) -> Type[PersistentStore]:
+    """
+    Get the class that implements the persistence store from its name.
+    """
     try:
-        return ALL_STORE_CLS[store_format]
+        return _ALL_STORE_CLS[store_format]
     except KeyError:
         raise ValueError(f"Store format {store_format!r} not known.")