hpcflow-new2 0.2.0a179__py3-none-any.whl → 0.2.0a181__py3-none-any.whl

hpcflow/sdk/persistence/store_resource.py

@@ -1,3 +1,7 @@
+"""
+Models of data stores as resources.
+"""
+
 from abc import ABC, abstractmethod
 import copy
 import json
@@ -13,6 +17,12 @@ class StoreResource(ABC):
     A `PersistentStore` maps workflow data across zero or more store resources. Updates to
     persistent workflow data that live in the same store resource are performed together.
 
+    Parameters
+    ----------
+    app: App
+        The main application context.
+    name:
+        The store name.
     """
 
     def __init__(self, app, name: str) -> None:
@@ -26,6 +36,9 @@ class StoreResource(ABC):
 
     @property
     def logger(self):
+        """
+        The logger.
+        """
         return self.app.persistence_logger
 
     @abstractmethod
@@ -37,6 +50,14 @@ class StoreResource(ABC):
         pass
 
     def open(self, action):
+        """
+        Open the store.
+
+        Parameters
+        ----------
+        action: str
+            What we are opening the store for; typically either ``read`` or ``update``.
+        """
         if action == "read":
             # reuse "update" data if set, rather than re-loading from disk -- but copy,
             # so changes made in the "read" scope do not update!
@@ -64,6 +85,15 @@ class StoreResource(ABC):
             pass
 
     def close(self, action):
+        """
+        Close the store for a particular action.
+
+        Parameters
+        ----------
+        action: str
+            What we are closing the store for.
+            Should match a previous call to :py:meth:`close`.
+        """
         if action == "read":
             self.logger.debug(f"{self!r}: closing read.")
         elif action == "update":
@@ -88,7 +118,22 @@ class StoreResource(ABC):
 
 
 class JSONFileStoreResource(StoreResource):
-    """For caching reads and writes to a JSON file."""
+    """
+    For caching reads and writes to a JSON file.
+
+    Parameters
+    ----------
+    app: App
+        The main application context.
+    name:
+        The store name.
+    filename:
+        The name of the JSON file.
+    path:
+        The path to the directory containing the JSON file.
+    fs:
+        The filesystem that the JSON file resides within.
+    """
 
     def __init__(self, app, name: str, filename: str, path: Union[str, Path], fs):
         self.filename = filename
@@ -114,7 +159,18 @@ class JSONFileStoreResource(StoreResource):
 
 
 class ZarrAttrsStoreResource(StoreResource):
-    """For caching reads and writes to Zarr attributes on groups and arrays."""
+    """
+    For caching reads and writes to Zarr attributes on groups and arrays.
+
+    Parameters
+    ----------
+    app: App
+        The main application context.
+    name:
+        The store name.
+    open_call:
+        How to actually perform an open on the underlying resource.
+    """
 
     def __init__(self, app, name: str, open_call: Callable):
         self.open_call = open_call

hpcflow/sdk/persistence/utils.py

@@ -1,9 +1,17 @@
+"""
+Miscellaneous persistence-related helpers.
+"""
+
 from getpass import getpass
 
 from hpcflow.sdk.core.errors import WorkflowNotFoundError
 
 
 def ask_pw_on_auth_exc(f, *args, add_pw_to=None, **kwargs):
+    """
+    Run the given function on the given arguments and add a password if the function
+    fails with an SSHException.
+    """
     from paramiko.ssh_exception import SSHException
 
     try:

hpcflow/sdk/persistence/zarr.py

@@ -1,3 +1,7 @@
+"""
+Persistence model based on writing Zarr arrays.
+"""
+
 from __future__ import annotations
 
 import copy
@@ -132,6 +136,10 @@ def append_items_to_ragged_array(arr, items):
 
 @dataclass
 class ZarrStoreTask(StoreTask):
+    """
+    Represents a task in a Zarr persistent store.
+    """
+
     def encode(self) -> Tuple[int, np.ndarray, Dict]:
         """Prepare store task data for the persistent store."""
         wk_task = {"id_": self.id_, "element_IDs": np.array(self.element_IDs)}
@@ -147,6 +155,10 @@ class ZarrStoreTask(StoreTask):
 
 @dataclass
 class ZarrStoreElement(StoreElement):
+    """
+    Represents an element in a Zarr persistent store.
+    """
+
     def encode(self, attrs: Dict) -> List:
         """Prepare store elements data for the persistent store.
 
@@ -180,6 +192,10 @@ class ZarrStoreElement(StoreElement):
 
 @dataclass
 class ZarrStoreElementIter(StoreElementIter):
+    """
+    Represents an element iteration in a Zarr persistent store.
+    """
+
     def encode(self, attrs: Dict) -> List:
         """Prepare store element iteration data for the persistent store.
 
@@ -216,6 +232,10 @@ class ZarrStoreElementIter(StoreElementIter):
 
 @dataclass
 class ZarrStoreEAR(StoreEAR):
+    """
+    Represents an element action run in a Zarr persistent store.
+    """
+
     def encode(self, attrs: Dict, ts_fmt: str) -> Tuple[List, Tuple[np.datetime64]]:
         """Prepare store EAR data for the persistent store.
 
@@ -268,6 +288,10 @@ class ZarrStoreEAR(StoreEAR):
 
 @dataclass
 class ZarrStoreParameter(StoreParameter):
+    """
+    Represents a parameter in a Zarr persistent store.
+    """
+
     _encoders = {  # keys are types
         np.ndarray: _encode_numpy_array,
         np.ma.core.MaskedArray: _encode_masked_array,
@@ -301,6 +325,10 @@ class ZarrStoreParameter(StoreParameter):
 
 
 class ZarrPersistentStore(PersistentStore):
+    """
+    A persistent store implemented using Zarr.
+    """
+
     _name = "zarr"
     _features = PersistentStoreFeatures(
         create=True,
@@ -346,6 +374,9 @@ class ZarrPersistentStore(PersistentStore):
             yield attrs
 
     def remove_replaced_dir(self) -> None:
+        """
+        Remove the directory containing replaced workflow details.
+        """
         with self.using_resource("attrs", "update") as md:
             if "replaced_workflow" in md:
                 self.logger.debug("removing temporarily renamed pre-existing workflow.")
@@ -353,6 +384,9 @@ class ZarrPersistentStore(PersistentStore):
                 md["replaced_workflow"] = None
 
     def reinstate_replaced_dir(self) -> None:
+        """
+        Reinstate the directory containing replaced workflow details.
+        """
         with self.using_resource("attrs", "read") as md:
             if "replaced_workflow" in md:
                 self.logger.debug(
@@ -380,6 +414,9 @@ class ZarrPersistentStore(PersistentStore):
         compressor: Optional[Union[str, None]] = "blosc",
         compressor_kwargs: Optional[Dict[str, Any]] = None,
     ) -> None:
+        """
+        Write an empty persistent workflow.
+        """
         attrs = {
             "name": name,
             "ts_fmt": ts_fmt,
@@ -796,6 +833,9 @@ class ZarrPersistentStore(PersistentStore):
 
     @property
     def zarr_store(self) -> zarr.storage.Store:
+        """
+        The underlying store object.
+        """
         if self._zarr_store is None:
             self._zarr_store = self._get_zarr_store(self.path, self.fs)
         return self._zarr_store
@@ -1139,18 +1179,30 @@ class ZarrPersistentStore(PersistentStore):
         return list(range(len(base_arr)))
 
     def get_ts_fmt(self):
+        """
+        Get the format for timestamps.
+        """
         with self.using_resource("attrs", action="read") as attrs:
             return attrs["ts_fmt"]
 
     def get_ts_name_fmt(self):
+        """
+        Get the format for timestamps to use in names.
+        """
         with self.using_resource("attrs", action="read") as attrs:
             return attrs["ts_name_fmt"]
 
     def get_creation_info(self):
+        """
+        Get information about the creation of the workflow.
+        """
         with self.using_resource("attrs", action="read") as attrs:
             return copy.deepcopy(attrs["creation_info"])
 
     def get_name(self):
+        """
+        Get the name of the workflow.
+        """
         with self.using_resource("attrs", action="read") as attrs:
             return attrs["name"]
 
@@ -1163,6 +1215,8 @@ class ZarrPersistentStore(PersistentStore):
         include_rechunk_backups=False,
     ):
         """
+        Convert the persistent store to zipped form.
+
         Parameters
         ----------
         path:
@@ -1299,6 +1353,9 @@ class ZarrPersistentStore(PersistentStore):
         backup: Optional[bool] = True,
         status: Optional[bool] = True,
     ):
+        """
+        Rechunk the parameter data to be stored more efficiently.
+        """
         arr = self._get_parameter_base_array()
         return self._rechunk_arr(arr, chunk_size, backup, status)
 
@@ -1308,13 +1365,21 @@ class ZarrPersistentStore(PersistentStore):
         backup: Optional[bool] = True,
         status: Optional[bool] = True,
     ):
+        """
+        Rechunk the run data to be stored more efficiently.
+        """
         arr = self._get_EARs_arr()
         return self._rechunk_arr(arr, chunk_size, backup, status)
 
 
 class ZarrZipPersistentStore(ZarrPersistentStore):
     """A store designed mainly as an archive format that can be uploaded to data
-    repositories such as Zenodo."""
+    repositories such as Zenodo.
+
+    Note
+    ----
+    Archive format persistent stores cannot be updated without being unzipped first.
+    """
 
     _name = "zip"
     _features = PersistentStoreFeatures(
@@ -1333,6 +1398,8 @@ class ZarrZipPersistentStore(ZarrPersistentStore):
 
     def unzip(self, path=".", log=None):
         """
+        Expand the persistent store.
+
         Parameters
         ----------
         path:

hpcflow/sdk/runtime.py

@@ -1,3 +1,7 @@
+"""
+Information about the Python runtime.
+"""
+
 from importlib import import_module
 import logging
 import os
@@ -15,17 +19,16 @@ class RunTimeInfo:
     """Get useful run-time information, including the executable name used to
     invoke the CLI, in the case a PyInstaller-built executable was used.
 
-    Attributes
+    Parameters
     ----------
-    sys_prefix : str
-        From `sys.prefix`. If running in a virtual environment, this will point to the
-        environment directory. If not running in a virtual environment, this will point to
-        the Python installation root.
-    sys_base_prefix : str
-        From `sys.base_prefix`. This will be equal to `sys_prefix` (`sys.prefix`) if not
-        running within a virtual environment. However, if running within a virtual
-        environment, this will be the Python installation directory, and `sys_prefix` will
-        be equal to the virtual environment directory.
+    name:
+        Application name.
+    package_name:
+        Application package name.
+    version:
+        Application version.
+    logger:
+        Where to write logging versions.
     """
 
     def __init__(self, name, package_name, version, logger):
@@ -34,22 +37,35 @@ class RunTimeInfo:
             sys._MEIPASS if is_frozen else os.path.dirname(os.path.abspath(__file__))
         )
 
+        #: Application name.
         self.name = name.split(".")[0]  # if name is given as __name__ # TODO: what?
+        #: Application package name.
         self.package_name = package_name
+        #: Application version.
         self.version = version
+        #: Whether this is a frozen application.
         self.is_frozen = is_frozen
+        #: Working directory.
         self.working_dir = os.getcwd()
+        #: Where to write log messages.
         self.logger = logger
+        #: Host that this is running on.
         self.hostname = socket.gethostname()
 
+        #: Whether this application is inside iPython.
         self.in_ipython = False
+        #: Whether this application is being used interactively.
         self.is_interactive = False
+        #: Whether this application is being used in test mode.
         self.in_pytest = False  # set in `conftest.py`
+        #: Whether this application is being run from the CLI.
         self.from_CLI = False  # set in CLI
 
         if self.is_frozen:
+            #: The bundle directory, if frozen.
             self.bundle_dir = Path(bundle_dir)
         else:
+            #: The path to Python itself.
             self.python_executable_path = Path(sys.executable)
 
             try:
@@ -61,16 +77,30 @@ class RunTimeInfo:
             if hasattr(sys, "ps1"):
                 self.is_interactive = True
 
+        #: The Python version.
         self.python_version = platform.python_version()
+        #: Whether the application is in a virtual environment.
         self.is_venv = hasattr(sys, "real_prefix") or sys.base_prefix != sys.prefix
+        #: Whether the application is in a Conda virtual environment.
         self.is_conda_venv = "CONDA_PREFIX" in os.environ
 
+        #: From `sys.prefix`. If running in a virtual environment, this will point to the
+        #: environment directory. If not running in a virtual environment, this will
+        #: point to the Python installation root.
         self.sys_prefix = getattr(sys, "prefix", None)
+        #: From `sys.base_prefix`. This will be equal to `sys_prefix` (`sys.prefix`) if
+        #: not running within a virtual environment. However, if running within a virtual
+        #: environment, this will be the Python installation directory, and `sys_prefix`
+        #: will be equal to the virtual environment directory.
         self.sys_base_prefix = getattr(sys, "base_prefix", None)
+        #: The old base prefix, from `sys.real_prefix`. Compatibility version of
+        #: :py:attr:`sys_base_prefix`.
         self.sys_real_prefix = getattr(sys, "real_prefix", None)
+        #: The Conda prefix, if defined.
         self.conda_prefix = os.environ.get("CONDA_PREFIX")
 
         try:
+            #: The virtual environment path.
             self.venv_path = self._set_venv_path()
         except ValueError:
             self.venv_path = None
@@ -97,6 +127,9 @@ class RunTimeInfo:
         #     warnings.warn(msg)
 
     def to_dict(self):
+        """
+        Serialize this class as a dictionary.
+        """
         out = {
             "name": self.name,
             "package_name": self.package_name,
@@ -157,12 +190,21 @@ class RunTimeInfo:
             return out
 
     def get_activate_env_command(self):
+        """
+        Get the command to activate the virtual environment.
+        """
         pass
 
     def get_deactivate_env_command(self):
+        """
+        Get the command to deactivate the virtual environment.
+        """
         pass
 
     def show(self):
+        """
+        Display the information known by this class as a human-readable table.
+        """
         tab = Table(show_header=False, box=None)
         tab.add_column()
         tab.add_column()

hpcflow/sdk/submission/__init__.py

@@ -0,0 +1,3 @@
+"""
+Subsystem for submitting work to schedulers for enactment.
+"""