hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a190__py3-none-any.whl

hpcflow/sdk/cli_common.py

@@ -1,17 +1,21 @@
 """Click CLI options that are used as decorators in multiple modules."""
 
+from __future__ import annotations
 import click
 
 from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS
-from hpcflow.sdk.persistence import ALL_STORE_FORMATS, DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.defaults import DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.discovery import ALL_STORE_FORMATS
 
 
-def sub_tasks_callback(ctx, param, value):
+def sub_tasks_callback(ctx, param, value: str | None) -> list[int] | None:
     """
     Parse subtasks.
     """
     if value:
         return [int(i) for i in value.split(",")]
+    else:
+        return None
 
 
 #: Standard option

hpcflow/sdk/config/__init__.py

@@ -2,4 +2,4 @@
 Configuration loading and manipulation.
 """
 
-from .config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG
+from hpcflow.sdk.config.config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG

hpcflow/sdk/config/callbacks.py

@@ -1,44 +1,58 @@
 """Module that defines built-in callback functions for configuration item values."""
 
-
+from __future__ import annotations
 import os
 import re
-import fsspec
+import fsspec  # type: ignore
+from typing import overload, TYPE_CHECKING
 from hpcflow.sdk.core.errors import UnsupportedSchedulerError, UnsupportedShellError
-
 from hpcflow.sdk.submission.shells import get_supported_shells
 
+if TYPE_CHECKING:
+    from typing import Any, TypeVar
+    from .config import Config
+    from ..typing import PathLike
+
+    T = TypeVar("T")
 
-def callback_vars(config, value):
+
+def callback_vars(config: Config, value) -> str:
     """
     Callback that substitutes configuration variables.
     """
 
-    def vars_repl(match_obj):
-        var_name = match_obj.groups()[0]
-        return config._variables[var_name]
+    def vars_repl(match_obj: re.Match[str]) -> str:
+        return config._variables[match_obj[1]]
 
-    vars_join = "|".join(list(config._variables.keys()))
-    vars_regex = r"\<\<(" + vars_join + r")\>\>"
-    value = re.sub(
+    vars_regex = rf"\<\<({ '|'.join(config._variables) })\>\>"
+    return re.sub(
         pattern=vars_regex,
         repl=vars_repl,
         string=str(value),
     )
-    return value
 
 
-def callback_file_paths(config, file_path):
+@overload
+def callback_file_paths(config: Config, file_path: PathLike) -> PathLike:
+    ...
+
+
+@overload
+def callback_file_paths(config: Config, file_path: list[PathLike]) -> list[PathLike]:
+    ...
+
+
+def callback_file_paths(config: Config, file_path: PathLike | list[PathLike]):
     """
     Callback that resolves file paths.
     """
     if isinstance(file_path, list):
-        return [config._resolve_path(i) for i in file_path]
+        return [config._resolve_path(path) for path in file_path]
     else:
         return config._resolve_path(file_path)
 
 
-def callback_bool(config, value):
+def callback_bool(config: Config, value: str | bool) -> bool:
     """
     Callback that coerces values to boolean.
     """
@@ -52,19 +66,36 @@ def callback_bool(config, value):
     return value
 
 
-def callback_lowercase(config, value):
+@overload
+def callback_lowercase(config: Config, value: list[str]) -> list[str]:
+    ...
+
+
+@overload
+def callback_lowercase(config: Config, value: dict[str, T]) -> dict[str, T]:
+    ...
+
+
+@overload
+def callback_lowercase(config: Config, value: str) -> str:
+    ...
+
+
+def callback_lowercase(
+    config: Config, value: list[str] | dict[str, T] | str
+) -> list[str] | dict[str, T] | str:
     """
     Callback that forces a string to lower case.
     """
     if isinstance(value, list):
-        return [i.lower() for i in value]
+        return [item.lower() for item in value]
     elif isinstance(value, dict):
         return {k.lower(): v for k, v in value.items()}
     else:
         return value.lower()
 
 
-def exists_in_schedulers(config, value):
+def exists_in_schedulers(config: Config, value: T) -> T:
     """
     Callback that tests that a value is a supported scheduler name.
     """
@@ -77,21 +108,25 @@ def exists_in_schedulers(config, value):
     return value
 
 
-def callback_supported_schedulers(config, schedulers):
+def callback_supported_schedulers(
+    config: Config, schedulers: dict[str, Any]
+) -> dict[str, Any]:
     """
     Callback that tests that all values are names of supported schedulers.
     """
     # validate against supported schedulers according to the OS - this won't validate that
     # a particular scheduler actually exists on this system:
-    available = config._app.get_OS_supported_schedulers()
-    for k in schedulers:
-        if k not in available:
-            raise UnsupportedSchedulerError(scheduler=k, available=available)
-
+    available = set(config._app.get_OS_supported_schedulers())
+    if any((witness := k) not in available for k in schedulers):
+        raise UnsupportedSchedulerError(scheduler=witness, available=available)
     return schedulers
 
 
-def set_scheduler_invocation_match(config, scheduler: str):
+def _hostname_in_invocation(config: Config) -> bool:
+    return "hostname" in config._file.get_invocation(config._config_key)["match"]
+
+
+def set_scheduler_invocation_match(config: Config, scheduler: str) -> None:
     """Invoked on set of `default_scheduler`.
 
     For clusters with "proper" schedulers (SGE, SLURM, etc.), login nodes are typically
@@ -100,25 +135,25 @@ def set_scheduler_invocation_match(config, scheduler: str):
     that on clusters the hostname match is explicitly set.
 
     """
-    default_args = config.get(f"schedulers.{scheduler}").get("defaults", {})
     sched = config._app.get_scheduler(
         scheduler_name=scheduler,
         os_name=os.name,
-        scheduler_args=default_args,
+        scheduler_args=config.get(f"schedulers.{scheduler}").get("defaults", {}),
     )
-    if hasattr(sched, "DEFAULT_LOGIN_NODE_MATCH"):
-        if "hostname" not in config._file.get_invocation(config._config_key)["match"]:
+    if isinstance(sched, config._app.QueuedScheduler):
+        if not _hostname_in_invocation(config):
             config._file.update_invocation(
                 config_key=config._config_key,
                 match={"hostname": sched.DEFAULT_LOGIN_NODE_MATCH},
             )
 
 
-def callback_scheduler_set_up(config, schedulers):
+def callback_scheduler_set_up(
+    config: Config, schedulers: dict[str, Any]
+) -> dict[str, Any]:
     """Invoked on set of `schedulers`.
 
     Runs scheduler-specific config initialisation.
-
     """
     for k, v in schedulers.items():
         sched = config._app.get_scheduler(
@@ -126,19 +161,19 @@ def callback_scheduler_set_up(config, schedulers):
             os_name=os.name,
             scheduler_args=v.get("defaults", {}),
         )
-        if hasattr(sched, "get_login_nodes"):
-            # some `Scheduler` classes have a `get_login_nodes` method which can be used
+
+        if isinstance(sched, config._app.SGEPosix):
+            # some `QueuedScheduler` classes have a `get_login_nodes` method which can be used
             # to populate the names of login nodes explicitly, if not already set:
-            if "hostname" not in config._file.get_invocation(config._config_key)["match"]:
-                login_nodes = sched.get_login_nodes()
+            if not _hostname_in_invocation(config):
                 config._file.update_invocation(
                     config_key=config._config_key,
-                    match={"hostname": login_nodes},
+                    match={"hostname": sched.get_login_nodes()},
                 )
     return schedulers
 
 
-def callback_supported_shells(config, shell_name):
+def callback_supported_shells(config: Config, shell_name: str) -> str:
     """
     Callback that tests if a shell names is supported on this OS.
     """
@@ -148,31 +183,31 @@ def callback_supported_shells(config, shell_name):
     return shell_name
 
 
-def set_callback_file_paths(config, value):
+def set_callback_file_paths(config: Config, value: PathLike | list[PathLike]) -> None:
     """Check the file(s) is/are accessible. This is only done on `config.set` (and not on
     `config.get` or `config._validate`) because it could be expensive in the case of remote
     files."""
     value = callback_file_paths(config, value)
 
-    to_check = value
-    if not isinstance(value, list):
-        to_check = [value]
+    to_check = value if isinstance(value, list) else [value]
 
     for file_path in to_check:
+        if file_path is None:
+            continue
         with fsspec.open(file_path, mode="rt") as fh:
             pass
             # TODO: also check something in it?
         print(f"Checked access to: {file_path}")
 
 
-def check_load_data_files(config, value):
+def check_load_data_files(config: Config, value: Any) -> None:
     """Check data files (e.g., task schema files) can be loaded successfully. This is only
     done on `config.set` (and not on `config.get` or `config._validate`) because it could
     be expensive in the case of remote files."""
     config._app.reload_template_components(warn=False)
 
 
-def callback_update_log_console_level(config, value):
+def callback_update_log_console_level(config: Config, value: str) -> None:
     """
     Callback to set the logging level.
     """