hpcflow-new2 0.2.0a189__py3-none-any.whl → 0.2.0a199__py3-none-any.whl

hpcflow/sdk/cli_common.py

@@ -1,17 +1,59 @@
 """Click CLI options that are used as decorators in multiple modules."""
 
+from __future__ import annotations
 import click
 
 from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS
-from hpcflow.sdk.persistence import ALL_STORE_FORMATS, DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.defaults import DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.discovery import ALL_STORE_FORMATS
 
 
-def sub_tasks_callback(ctx, param, value):
+class BoolOrString(click.ParamType):
+    """Custom Click parameter type to accepts a bool or a choice of strings."""
+
+    name = "bool-or-string"
+
+    def __init__(self, allowed_strings, true_strings=None, false_strings=None):
+        self.allowed_strings = allowed_strings
+        self.true_strings = true_strings if true_strings else ["true", "yes", "on"]
+        self.false_strings = false_strings if false_strings else ["false", "no", "off"]
+
+    def convert(self, value, param, ctx):
+        # Check if the value is a boolean
+        if isinstance(value, bool):
+            return value
+
+        # Normalize value to string
+        value = str(value).lower()
+
+        # Check if the value is one of the true strings
+        if value in self.true_strings:
+            return True
+
+        # Check if the value is one of the false strings
+        if value in self.false_strings:
+            return False
+
+        # If the value matches neither, it must be one of the expected strings
+        if value not in self.allowed_strings:
+            allowed_fmt = ", ".join(f"{i!r}" for i in self.allowed_strings)
+            self.fail(
+                message=f"{value} is not a valid boolean or one of {allowed_fmt}.",
+                param=param,
+                ctx=ctx,
+            )
+
+        return value
+
+
+def sub_tasks_callback(ctx, param, value: str | None) -> list[int] | None:
     """
     Parse subtasks.
     """
     if value:
         return [int(i) for i in value.split(",")]
+    else:
+        return None
 
 
 #: Standard option
@@ -91,12 +133,14 @@ variables_option = click.option(
 js_parallelism_option = click.option(
     "--js-parallelism",
     help=(
-        "If True, allow multiple jobscripts to execute simultaneously. Raises if "
-        "set to True but the store type does not support the "
-        "`jobscript_parallelism` feature. If not set, jobscript parallelism will "
-        "be used if the store type supports it."
+        "If True, allow multiple jobscripts to execute simultaneously. If "
+        "'scheduled'/'direct', only allow simultaneous execution of scheduled/direct "
+        "jobscripts. Raises if set to True, 'scheduled', or 'direct', but the store type "
+        "does not support the `jobscript_parallelism` feature. If not set, jobscript "
+        "parallelism will be used if the store type supports it, for scheduled "
+        "jobscripts only."
     ),
-    type=click.BOOL,
+    type=BoolOrString(["direct", "scheduled"]),
 )
 #: Standard option
 wait_option = click.option(
@@ -140,6 +184,17 @@ submit_status_opt = click.option(
     help="If True, display a live status to track submission progress.",
     default=True,
 )
+#: Standard option
+force_arr_opt = click.option(
+    "--force-array",
+    help=(
+        "Used to force the use of job arrays, even if the scheduler does not support it. "
+        "This is provided for testing purposes only."
+    ),
+    is_flag=True,
+    default=False,
+)
+
 #: Standard option
 make_status_opt = click.option(
     "--status/--no-status",
@@ -147,6 +202,14 @@ make_status_opt = click.option(
     default=True,
 )
 
+#: Standard option
+add_sub_opt = click.option(
+    "--add-submission",
+    help=("If True, add a submission to the workflow (but do not submit)."),
+    is_flag=True,
+    default=False,
+)
+
 #: Standard option
 zip_path_opt = click.option(
     "--path",
@@ -207,6 +270,34 @@ rechunk_status_opt = click.option(
     default=True,
     help="If True, display a live status to track rechunking progress.",
 )
+cancel_status_opt = click.option(
+    "--status/--no-status",
+    default=True,
+    help="If True, display a live status to track cancel progress.",
+)
+
+list_js_max_js_opt = click.option(
+    "--max-js", type=click.INT, help="Display up to this jobscript only."
+)
+list_js_jobscripts_opt = click.option(
+    "--jobscripts", help="Comma-separated list of jobscript indices to show."
+)
+list_task_js_max_js_opt = click.option(
+    "--max-js", type=click.INT, help="Include jobscripts up to this jobscript only."
+)
+list_task_js_task_names_opt = click.option(
+    "--task-names", help="Comma-separated list of task name sub-strings to show."
+)
+list_js_width_opt = click.option(
+    "--width", type=click.INT, help="Width in characters of the table to print."
+)
+jobscript_std_array_idx_opt = click.option(
+    "--array-idx",
+    type=click.INT,
+    help=(
+        "For array jobs only, the job array index whose standard stream is to be printed."
+    ),
+)
 
 
 def _add_doc_from_help(*args):
@@ -255,4 +346,11 @@ _add_doc_from_help(
     rechunk_backup_opt,
     rechunk_chunk_size_opt,
     rechunk_status_opt,
+    cancel_status_opt,
+    list_js_max_js_opt,
+    list_js_jobscripts_opt,
+    list_task_js_max_js_opt,
+    list_task_js_task_names_opt,
+    list_js_width_opt,
+    jobscript_std_array_idx_opt,
 )

hpcflow/sdk/config/__init__.py

@@ -2,4 +2,4 @@
 Configuration loading and manipulation.
 """
 
-from .config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG
+from hpcflow.sdk.config.config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG

hpcflow/sdk/config/callbacks.py

@@ -1,44 +1,58 @@
 """Module that defines built-in callback functions for configuration item values."""
 
-
+from __future__ import annotations
 import os
 import re
-import fsspec
+import fsspec  # type: ignore
+from typing import overload, TYPE_CHECKING
 from hpcflow.sdk.core.errors import UnsupportedSchedulerError, UnsupportedShellError
-
 from hpcflow.sdk.submission.shells import get_supported_shells
 
+if TYPE_CHECKING:
+    from typing import Any, TypeVar
+    from .config import Config
+    from ..typing import PathLike
+
+    T = TypeVar("T")
 
-def callback_vars(config, value):
+
+def callback_vars(config: Config, value) -> str:
     """
     Callback that substitutes configuration variables.
     """
 
-    def vars_repl(match_obj):
-        var_name = match_obj.groups()[0]
-        return config._variables[var_name]
+    def vars_repl(match_obj: re.Match[str]) -> str:
+        return config._variables[match_obj[1]]
 
-    vars_join = "|".join(list(config._variables.keys()))
-    vars_regex = r"\<\<(" + vars_join + r")\>\>"
-    value = re.sub(
+    vars_regex = rf"\<\<({ '|'.join(config._variables) })\>\>"
+    return re.sub(
         pattern=vars_regex,
         repl=vars_repl,
         string=str(value),
     )
-    return value
 
 
-def callback_file_paths(config, file_path):
+@overload
+def callback_file_paths(config: Config, file_path: PathLike) -> PathLike:
+    ...
+
+
+@overload
+def callback_file_paths(config: Config, file_path: list[PathLike]) -> list[PathLike]:
+    ...
+
+
+def callback_file_paths(config: Config, file_path: PathLike | list[PathLike]):
     """
     Callback that resolves file paths.
     """
     if isinstance(file_path, list):
-        return [config._resolve_path(i) for i in file_path]
+        return [config._resolve_path(path) for path in file_path]
     else:
         return config._resolve_path(file_path)
 
 
-def callback_bool(config, value):
+def callback_bool(config: Config, value: str | bool) -> bool:
     """
     Callback that coerces values to boolean.
     """
@@ -52,19 +66,36 @@ def callback_bool(config, value):
     return value
 
 
-def callback_lowercase(config, value):
+@overload
+def callback_lowercase(config: Config, value: list[str]) -> list[str]:
+    ...
+
+
+@overload
+def callback_lowercase(config: Config, value: dict[str, T]) -> dict[str, T]:
+    ...
+
+
+@overload
+def callback_lowercase(config: Config, value: str) -> str:
+    ...
+
+
+def callback_lowercase(
+    config: Config, value: list[str] | dict[str, T] | str
+) -> list[str] | dict[str, T] | str:
     """
     Callback that forces a string to lower case.
     """
     if isinstance(value, list):
-        return [i.lower() for i in value]
+        return [item.lower() for item in value]
     elif isinstance(value, dict):
         return {k.lower(): v for k, v in value.items()}
     else:
         return value.lower()
 
 
-def exists_in_schedulers(config, value):
+def exists_in_schedulers(config: Config, value: T) -> T:
     """
     Callback that tests that a value is a supported scheduler name.
     """
@@ -77,21 +108,25 @@ def exists_in_schedulers(config, value):
     return value
 
 
-def callback_supported_schedulers(config, schedulers):
+def callback_supported_schedulers(
+    config: Config, schedulers: dict[str, Any]
+) -> dict[str, Any]:
     """
     Callback that tests that all values are names of supported schedulers.
     """
     # validate against supported schedulers according to the OS - this won't validate that
     # a particular scheduler actually exists on this system:
-    available = config._app.get_OS_supported_schedulers()
-    for k in schedulers:
-        if k not in available:
-            raise UnsupportedSchedulerError(scheduler=k, available=available)
-
+    available = set(config._app.get_OS_supported_schedulers())
+    if any((witness := k) not in available for k in schedulers):
+        raise UnsupportedSchedulerError(scheduler=witness, available=available)
     return schedulers
 
 
-def set_scheduler_invocation_match(config, scheduler: str):
+def _hostname_in_invocation(config: Config) -> bool:
+    return "hostname" in config._file.get_invocation(config._config_key)["match"]
+
+
+def set_scheduler_invocation_match(config: Config, scheduler: str) -> None:
     """Invoked on set of `default_scheduler`.
 
     For clusters with "proper" schedulers (SGE, SLURM, etc.), login nodes are typically
@@ -100,25 +135,25 @@ def set_scheduler_invocation_match(config, scheduler: str):
     that on clusters the hostname match is explicitly set.
 
     """
-    default_args = config.get(f"schedulers.{scheduler}").get("defaults", {})
     sched = config._app.get_scheduler(
         scheduler_name=scheduler,
         os_name=os.name,
-        scheduler_args=default_args,
+        scheduler_args=config.get(f"schedulers.{scheduler}").get("defaults", {}),
     )
-    if hasattr(sched, "DEFAULT_LOGIN_NODE_MATCH"):
-        if "hostname" not in config._file.get_invocation(config._config_key)["match"]:
+    if isinstance(sched, config._app.QueuedScheduler):
+        if not _hostname_in_invocation(config):
             config._file.update_invocation(
                 config_key=config._config_key,
                 match={"hostname": sched.DEFAULT_LOGIN_NODE_MATCH},
             )
 
 
-def callback_scheduler_set_up(config, schedulers):
+def callback_scheduler_set_up(
+    config: Config, schedulers: dict[str, Any]
+) -> dict[str, Any]:
     """Invoked on set of `schedulers`.
 
     Runs scheduler-specific config initialisation.
-
     """
     for k, v in schedulers.items():
         sched = config._app.get_scheduler(
@@ -126,19 +161,19 @@ def callback_scheduler_set_up(config, schedulers):
             os_name=os.name,
             scheduler_args=v.get("defaults", {}),
         )
-        if hasattr(sched, "get_login_nodes"):
-            # some `Scheduler` classes have a `get_login_nodes` method which can be used
+
+        if isinstance(sched, config._app.SGEPosix):
+            # some `QueuedScheduler` classes have a `get_login_nodes` method which can be used
             # to populate the names of login nodes explicitly, if not already set:
-            if "hostname" not in config._file.get_invocation(config._config_key)["match"]:
-                login_nodes = sched.get_login_nodes()
+            if not _hostname_in_invocation(config):
                 config._file.update_invocation(
                     config_key=config._config_key,
-                    match={"hostname": login_nodes},
+                    match={"hostname": sched.get_login_nodes()},
                 )
     return schedulers
 
 
-def callback_supported_shells(config, shell_name):
+def callback_supported_shells(config: Config, shell_name: str) -> str:
     """
     Callback that tests if a shell names is supported on this OS.
     """
@@ -148,32 +183,69 @@ def callback_supported_shells(config, shell_name):
     return shell_name
 
 
-def set_callback_file_paths(config, value):
+def set_callback_file_paths(config: Config, value: PathLike | list[PathLike]) -> None:
     """Check the file(s) is/are accessible. This is only done on `config.set` (and not on
     `config.get` or `config._validate`) because it could be expensive in the case of remote
     files."""
     value = callback_file_paths(config, value)
 
-    to_check = value
-    if not isinstance(value, list):
-        to_check = [value]
+    to_check = value if isinstance(value, list) else [value]
 
     for file_path in to_check:
+        if file_path is None:
+            continue
         with fsspec.open(file_path, mode="rt") as fh:
             pass
             # TODO: also check something in it?
         print(f"Checked access to: {file_path}")
 
 
-def check_load_data_files(config, value):
+def check_load_data_files(config: Config, value: Any) -> None:
     """Check data files (e.g., task schema files) can be loaded successfully. This is only
     done on `config.set` (and not on `config.get` or `config._validate`) because it could
     be expensive in the case of remote files."""
     config._app.reload_template_components(warn=False)
 
 
-def callback_update_log_console_level(config, value):
+def callback_log_file_path(config, value):
+    value = value.strip()
+    if value:
+        return config._resolve_path(value)
+    else:
+        return value
+
+
+def callback_update_log_console_level(config: Config, value: str) -> None:
     """
     Callback to set the logging level.
     """
-    config._app.log.update_console_level(value)
+    config._app.log.update_console_level(new_level=value)
+
+
+def callback_unset_log_console_level(config: Config) -> None:
+    """Reset the console handler to the default level."""
+    config._app.log.update_console_level()
+
+
+def callback_update_log_file_level(config: Config, value: str) -> None:
+    """Callback to set the level of the log file handler."""
+    config._app.log.update_file_level(new_level=value)
+
+
+def callback_update_log_file_path(config: Config, value: str) -> None:
+    """
+    Callback to update the log file path, or remove the file handler if no path specifed.
+    """
+    config._app.log.remove_file_handler()
+    if value:
+        config._app.log.add_file_logger(path=value, level=config.get("log_file_level"))
+
+
+def callback_unset_log_file_level(config: Config) -> None:
+    """Callback to reset the file handler to the default level."""
+    config._app.log.update_file_level()
+
+
+def callback_unset_log_file_path(config: Config) -> None:
+    """Callback to remove the log file handler."""
+    config._app.log.remove_file_handler()