hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/config/errors.py

@@ -2,12 +2,20 @@
 Miscellaneous configuration-related errors.
 """
 
+from __future__ import annotations
+from typing import Any, TYPE_CHECKING
 
-class ConfigError(Exception):
-    """Raised when a valid configuration can not be associated with the current
-    invocation."""
+if TYPE_CHECKING:
+    from collections.abc import Sequence, Iterable
+    from .types import ConfigMetadata
+    from ..typing import PathLike
 
-    pass
+
+class ConfigError(Exception):
+    """
+    Raised when a valid configuration can not be associated with the current
+    invocation.
+    """
 
 
 class ConfigUnknownItemError(ConfigError):
@@ -15,12 +23,14 @@ class ConfigUnknownItemError(ConfigError):
     Raised when the configuration contains an unknown item.
     """
 
-    def __init__(self, name, message=None):
-        self.message = message or (
-            f"Specified name {name!r} is not a valid meta-data or configurable "
-            f"configuration item."
+    def __init__(self, name: str, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"Specified name {name!r} is not a valid meta-data or configurable "
+                f"configuration item."
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigUnknownOverrideError(ConfigError):
@@ -28,11 +38,13 @@ class ConfigUnknownOverrideError(ConfigError):
     Raised when the configuration override contains an unknown item.
     """
 
-    def __init__(self, name, message=None):
-        self.message = message or (
-            f"Specified configuration override {name!r} is not a valid configurable item."
+    def __init__(self, name: str, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"Specified configuration override {name!r} is not a valid configurable item."
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigNonConfigurableError(ConfigError):
@@ -40,8 +52,14 @@ class ConfigNonConfigurableError(ConfigError):
     Raised when the configuration contains an item that can't be configured.
     """
 
-    def __init__(self, name, message=None):
-        self.message = message or (f"Specified name {name!r} is not a configurable item.")
+    def __init__(self, name: str | Iterable[str], message: str | None = None):
+        if not message:
+            if not isinstance(name, str):
+                names_str = ", ".join(f"{i!r}" for i in name)
+                msg = f"Specified names {names_str} are not configurable items."
+            else:
+                msg = f"Specified name {name!r} is not a configurable item."
+        self.message = message or msg
         super().__init__(self.message)
 
 
@@ -50,9 +68,8 @@ class ConfigItemAlreadyUnsetError(ConfigError):
     Raised when the configuration tries to unset an unset item.
     """
 
-    def __init__(self, name, message=None):
-        self.message = message or f"Configuration item {name!r} is already not set."
-        super().__init__(self.message)
+    def __init__(self, name: str, message: str = ""):
+        super().__init__(message or f"Configuration item {name!r} is already not set.")
 
 
 class ConfigFileValidationError(ConfigError):
@@ -60,7 +77,8 @@ class ConfigFileValidationError(ConfigError):
     Raised when the configuration file fails validation.
     """
 
-    pass
+    def __init__(self, msg: str) -> None:
+        super().__init__(msg)
 
 
 class ConfigItemCallbackError(ConfigError):
@@ -68,23 +86,24 @@ class ConfigItemCallbackError(ConfigError):
     Raised when a configuration callback errors.
     """
 
-    def __init__(self, name, callback, err, message=None):
-        self.message = message or (
-            f"Callback function {callback.__name__!r} for configuration item {name!r} "
-            f"failed with exception: \n\n{str(err)}"
+    def __init__(self, name: str, callback: Any, err: Any, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"Callback function {callback.__name__!r} for configuration item {name!r} "
+                f"failed with exception: \n\n{str(err)}"
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigFileInvocationIncompatibleError(ConfigError):
     """Raised when, given the run time information of the invocation, no compatible
     configuration can be found in the config file."""
 
-    def __init__(self, message=None):
-        self.message = message or (
-            "No config could be found that matches the current invocation."
+    def __init__(self, message: str | None = ""):
+        super().__init__(
+            message or ("No config could be found that matches the current invocation.")
         )
-        super().__init__(self.message)
 
 
 class ConfigFileInvocationUnknownMatchKey(ConfigError):
@@ -92,134 +111,156 @@ class ConfigFileInvocationUnknownMatchKey(ConfigError):
     Raised when the configuration contains an invalid match key.
     """
 
-    def __init__(self, match_key, message=None):
-        self.message = message or (
-            f"Specified match key ({match_key!r}) is not a valid run time info "
-            f"attribute."
-        )
+    def __init__(self, match_key: str, message: str = ""):
         self.match_key = match_key
-        super().__init__(self.message)
+        super().__init__(
+            message
+            or (
+                f"Specified match key ({match_key!r}) is not a valid run time info "
+                f"attribute."
+            )
+        )
 
 
 class ConfigInvocationKeyNotFoundError(ConfigError):
     """Raised when a configuration invocation key is passed but it is not a valid key."""
 
-    def __init__(self, invoc_key, file_path, available_keys) -> None:
+    def __init__(
+        self, invoc_key: str, file_path: PathLike, available_keys: Sequence[str]
+    ):
         self.invoc_key = invoc_key
         self.file_path = file_path
         self.available_keys = available_keys
-        self.message = (
+        super().__init__(
             f"Invocation key {self.invoc_key!r} does not exist in configuration file. "
             f"Available keys in configuration file {str(self.file_path)!r} are "
             f"{self.available_keys!r}."
         )
-        super().__init__(self.message)
 
 
 class ConfigValidationError(ConfigError):
     """Raised when the matching config data is invalid."""
 
-    def __init__(self, message, meta_data=None):
+    def __init__(self, message: str, meta_data: ConfigMetadata | None = None):
         self.meta_data = meta_data
-        self.message = message + (f"config {self.meta_data}\n" if meta_data else "")
-        super().__init__(self.message)
+        super().__init__(message + (f"config {self.meta_data}\n" if meta_data else ""))
 
 
 class ConfigDefaultValidationError(ConfigError):
     """Raised when the specified default configuration in the `ConfigOptions` object is
     invalid."""
 
-    def __init__(self, validation_err, message=None):
-        self.message = message or (
-            f"The default configuration specified in the `ConfigOptions` object is "
-            f"invalid.\n\n{validation_err}"
+    def __init__(self, validation_err: Any, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"The default configuration specified in the `ConfigOptions` object is "
+                f"invalid.\n\n{validation_err}"
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigChangeInvalidError(ConfigError):
     """Raised when trying to set an invalid key in the Config."""
 
-    def __init__(self, name, message=None):
-        self.message = message or (
-            f"Cannot modify value for invalid config item {name!r}. Use the `config list`"
-            f" sub-command to list all configurable items."
+    def __init__(self, name: str, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"Cannot modify value for invalid config item {name!r}. Use the `config list`"
+                f" sub-command to list all configurable items."
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigChangeInvalidJSONError(ConfigError):
     """Raised when attempting to set a config key using an invalid JSON string."""
 
-    def __init__(self, name, json_str, err, message=None):
-        self.message = message or (
-            f"The config file has not been modified. Invalid JSON string for config item "
-            f"{name!r}. {json_str!r}\n\n{err!r}"
+    def __init__(self, name: str, json_str: str, err: Any, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"The config file has not been modified. Invalid JSON string for config item "
+                f"{name!r}. {json_str!r}\n\n{err!r}"
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigChangeValidationError(ConfigError):
     """Raised when a change to the configurable data would invalidate the config."""
 
-    def __init__(self, name, validation_err, message=None):
-        self.message = message or (
-            f"The configuration has not been modified. Requested modification to item "
-            f"{name!r} would invalidate the config in the following way."
-            f"\n\n{validation_err}"
+    def __init__(self, name: str, validation_err: Any, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"The configuration has not been modified. Requested modification to item "
+                f"{name!r} would invalidate the config in the following way."
+                f"\n\n{validation_err}"
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigChangeFileUpdateError(ConfigError):
     """Raised when the updating of the config YAML file fails."""
 
-    def __init__(self, names, err, message=None):
-        self.message = message or (
-            f"Failed to update the config file for modification of config items {names!r}."
-            f"\n\n{err!r}"
+    def __init__(self, names: Sequence[str], err, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"Failed to update the config file for modification of config items {names!r}."
+                f"\n\n{err!r}"
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigChangeTypeInvalidError(ConfigError):
     """Raised when trying to modify a config item using a list operation, when the config
     item is not a list."""
 
-    def __init__(self, name, typ, message=None):
-        self.message = message or (
-            f"The configuration has not been modified. The config item {name!r} has type "
-            f"{typ!r} and so cannot be modified in that way."
+    def __init__(self, name: str, typ: type, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"The configuration has not been modified. The config item {name!r} has type "
+                f"{typ!r} and so cannot be modified in that way."
+            )
         )
-        super().__init__(self.message)
 
 
 class ConfigChangePopIndexError(ConfigError):
     """Raised when trying to pop an item from a config item with an invalid index."""
 
-    def __init__(self, name, length, index, message=None):
-        self.message = message or (
-            f"The configuration has not been modified. The config item {name!r} has length "
-            f"{length!r} and so cannot be popped with index {index}."
+    def __init__(self, name: str, length: int, index: int, message: str = ""):
+        super().__init__(
+            message
+            or (
+                f"The configuration has not been modified. The config item {name!r} has length "
+                f"{length!r} and so cannot be popped with index {index}."
+            )
         )
-        super().__init__(self.message)
 
 
 class MissingTaskSchemaFileError(ConfigError):
     """Raised when a task schema file specified in the config file does not exist."""
 
-    def __init__(self, file_name, err, message=None):
-        self.message = message or (
-            f"The task schema file {file_name!r} cannot be found. \n{err!s}"
+    def __init__(self, file_name: str, err: Any, message: str = ""):
+        super().__init__(
+            message or (f"The task schema file {file_name!r} cannot be found. \n{err!s}")
         )
-        super().__init__(self.message)
 
 
 class MissingEnvironmentFileError(ConfigError):
     """Raised when an environment file specified in the config file does not exist."""
 
-    def __init__(self, file_name, err, message=None):
-        self.message = message or (
-            f"The environment file {file_name!r} cannot be found. \n{err!s}"
+    def __init__(self, file_name: str, err: Any, message: str = ""):
+        super().__init__(
+            message or (f"The environment file {file_name!r} cannot be found. \n{err!s}")
         )
-        super().__init__(self.message)
+
+
+class ConfigReadOnlyError(ConfigError):
+    pass
+
+
+class UnknownMetaTaskConstitutiveSchema(ValueError):
+    pass

hpcflow/sdk/config/types.py

@@ -0,0 +1,148 @@
+"""
+Types used in configuration.
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING
+from typing_extensions import TypeAlias, TypedDict, TypeVar
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Sequence
+    from pathlib import Path
+    from typing import Any
+    from typing_extensions import NotRequired
+    from .config import Config
+    from ..core.validation import Schema
+
+
+T = TypeVar("T")
+#: Type of a getter callback.
+GetterCallback: TypeAlias = "Callable[[Config, T], T]"
+#: Type of a setter callback.
+SetterCallback: TypeAlias = "Callable[[Config, T], Any]"
+#: Type of a unsetter callback.
+UnsetterCallback: TypeAlias = "Callable[[Config], None]"
+
+
+class SGEParallelEnvsDescriptor(TypedDict):
+    """
+    SGE parallel environment descriptor.
+    """
+
+    #: Number of supported cores.
+    num_cores: list[int]
+
+
+class SLURMPartitionsDescriptor(TypedDict):
+    """
+    SLURM partition descriptor.
+    """
+
+    #: Number of supported cores.
+    num_cores: NotRequired[list[int]]
+    #: Number of cores per node.
+    num_cores_per_node: NotRequired[list[int]]
+    #: Number of supported nodes.
+    num_nodes: NotRequired[list[int]]
+    #: Possible parallel modes.
+    parallel_modes: NotRequired[list[str]]
+
+
+class SchedulerConfigDescriptor(TypedDict):
+    """
+    Scheduler configuration descriptor.
+    """
+
+    #: Default values.
+    defaults: dict[str, Any]
+    #: SGE parallel environments, by name.
+    parallel_environments: NotRequired[dict[str, SGEParallelEnvsDescriptor]]
+    #: SLURM partitions, by name.
+    partitions: NotRequired[dict[str, SLURMPartitionsDescriptor]]
+
+
+class ShellConfigDescriptor(TypedDict):
+    """
+    Shell configuration descriptor.
+    """
+
+    #: Default values.
+    defaults: dict[str, Any]
+
+
+class ConfigDescriptor(TypedDict):
+    """
+    Configuration descriptor.
+    """
+
+    #: Execution machine.
+    machine: NotRequired[str]
+    #: Where to log.
+    log_file_path: NotRequired[str]
+    #: Where to find execution environments.
+    environment_sources: NotRequired[list[str]]
+    #: Where to find task schemas.
+    task_schema_sources: NotRequired[list[str]]
+    #: Where to find command files.
+    command_file_sources: NotRequired[list[str]]
+    #: Where to find parameter implementations.
+    parameter_sources: NotRequired[list[str]]
+    #: Default scheduler.
+    default_scheduler: NotRequired[str]
+    #: Default shell.
+    default_shell: NotRequired[str]
+    #: Supported schedulers.
+    schedulers: NotRequired[dict[str, SchedulerConfigDescriptor]]
+    #: Supported shells.
+    shells: NotRequired[dict[str, ShellConfigDescriptor]]
+
+
+class InvocationDescriptor(TypedDict):
+    """
+    Invocation descriptor.
+    """
+
+    #: Used to set up the environment.
+    environment_setup: str | None
+    #: setting to apply if matched.
+    match: dict[str, str | list[str]]
+
+
+class DefaultConfiguration(TypedDict):
+    """
+    The default configuration.
+    """
+
+    #: Default invocation.
+    invocation: InvocationDescriptor
+    #: Default configuration.
+    config: ConfigDescriptor
+
+
+#: A configuration dictionary.
+ConfigDict: TypeAlias = "dict[str, dict[str, DefaultConfiguration]]"
+
+
+class ConfigMetadata(TypedDict):
+    """
+    Metadata supported by the :class:`Config` class.
+    """
+
+    #: Location of directory containing the config file.
+    config_directory: Path
+    #: Name of the config file.
+    config_file_name: str
+    #: Full path to the config file.
+    config_file_path: Path
+    #: The contents of the config file.
+    config_file_contents: str
+    #: The key identifying the config section within the config file.
+    config_key: str
+    #: Schemas that apply to the config.
+    config_schemas: Sequence[Schema]
+    #: The user that invoked things.
+    invoking_user_id: str
+    #: The user hosting things.
+    host_user_id: str
+    #: Path to file holding description of :attr:``host_user_id``.
+    host_user_id_file_path: Path

hpcflow/sdk/core/__init__.py

@@ -1,6 +1,6 @@
 """Core programmatic models for hpcflow.
 
-EAR abort exit code is set to 64 [1].
+EAR abort exit code is set to 64. EAR skipped exit code is set to 65.
 
 References
 ----------
@@ -8,7 +8,31 @@ https://tldp.org/LDP/abs/html/exitcodes.html
 
 """
 
+import numpy as np
+
 #: Formats supported for templates.
 ALL_TEMPLATE_FORMATS = ("yaml", "json")
 #: The exit code used by an EAR when it aborts.
 ABORT_EXIT_CODE = 64
+SKIPPED_EXIT_CODE = 65
+NO_COMMANDS_EXIT_CODE = 66
+RUN_DIR_ARR_DTYPE = [
+    ("task_insert_ID", np.uint8),
+    ("element_idx", np.uint32),
+    ("iteration_idx", np.uint32),
+    ("action_idx", np.uint8),
+    ("run_idx", np.uint8),
+    ("element_depth", np.uint8),
+    ("iteration_depth", np.uint8),
+]
+_uint8_max = np.iinfo(np.uint8).max
+_uint32_max = np.iinfo(np.uint32).max
+RUN_DIR_ARR_FILL = (
+    _uint8_max,
+    _uint32_max,
+    _uint32_max,
+    _uint8_max,
+    _uint8_max,
+    _uint8_max,
+    _uint8_max,
+)