hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/sdk/cli_common.py

@@ -0,0 +1,385 @@
+"""Click CLI options that are used as decorators in multiple modules."""
+
+from __future__ import annotations
+import click
+
+from hpcflow.sdk.core import ALL_TEMPLATE_FORMATS
+from hpcflow.sdk.persistence.defaults import DEFAULT_STORE_FORMAT
+from hpcflow.sdk.persistence.discovery import ALL_STORE_FORMATS
+
+
+class BoolOrString(click.ParamType):
+    """Custom Click parameter type to accepts a bool or a choice of strings."""
+
+    name = "bool-or-string"
+
+    def __init__(self, allowed_strings, true_strings=None, false_strings=None):
+        self.allowed_strings = allowed_strings
+        self.true_strings = true_strings if true_strings else ["true", "yes", "on"]
+        self.false_strings = false_strings if false_strings else ["false", "no", "off"]
+
+    def convert(self, value, param, ctx):
+        # Check if the value is a boolean
+        if isinstance(value, bool):
+            return value
+
+        # Normalize value to string
+        value = str(value).lower()
+
+        # Check if the value is one of the true strings
+        if value in self.true_strings:
+            return True
+
+        # Check if the value is one of the false strings
+        if value in self.false_strings:
+            return False
+
+        # If the value matches neither, it must be one of the expected strings
+        if value not in self.allowed_strings:
+            allowed_fmt = ", ".join(f"{i!r}" for i in self.allowed_strings)
+            self.fail(
+                message=f"{value} is not a valid boolean or one of {allowed_fmt}.",
+                param=param,
+                ctx=ctx,
+            )
+
+        return value
+
+
+def sub_tasks_callback(ctx, param, value: str | None) -> list[int] | None:
+    """
+    Parse subtasks.
+    """
+    if value:
+        return [int(i) for i in value.split(",")]
+    else:
+        return None
+
+
+#: Standard option
+format_option = click.option(
+    "--format",
+    type=click.Choice(ALL_TEMPLATE_FORMATS),
+    default=None,
+    help=(
+        'If specified, one of "json" or "yaml". This forces parsing from a '
+        "particular format."
+    ),
+)
+#: Standard option
+path_option = click.option(
+    "--path",
+    type=click.Path(exists=True),
+    help=(
+        "The directory in which the workflow will be generated. If not specified, the "
+        "config item `default_workflow_path` will be used; if that is not set, the "
+        "current directory is used."
+    ),
+)
+#: Standard option
+name_option = click.option(
+    "--name",
+    help=(
+        "The name of the workflow. If specified, the workflow directory will be "
+        "`path` joined with `name`. If not specified the workflow template name "
+        "will be used, in combination with a date-timestamp."
+    ),
+)
+#: Standard option
+name_timestamp_option = click.option(
+    "--name-timestamp/--name-no-timestamp",
+    "name_add_timestamp",
+    is_flag=True,
+    default=None,
+    help=(
+        "If True, suffix the workflow name with a date-timestamp. A default value can be "
+        " set with the config item `workflow_name_add_timestamp`; otherwise set to "
+        "`True`."
+    ),
+)
+#: Standard option
+name_dir_option = click.option(
+    "--name-dir/--name-no-dir",
+    "name_use_dir",
+    is_flag=True,
+    default=None,
+    help=(
+        "If True, and `--name-timestamp` is also True, the workflow directory name "
+        "will be just the date-timestamp, and will be contained within a parent "
+        "directory corresponding to the workflow name. A default value can be set with "
+        "the config item `workflow_name_use_dir`; otherwise set to `False`."
+    ),
+)
+#: Standard option
+overwrite_option = click.option(
+    "--overwrite",
+    is_flag=True,
+    default=False,
+    help=(
+        "If True and the workflow directory (`path` + `name`) already exists, "
+        "the existing directory will be overwritten."
+    ),
+)
+#: Standard option
+store_option = click.option(
+    "--store",
+    type=click.Choice(ALL_STORE_FORMATS),
+    help="The persistent store type to use.",
+    default=DEFAULT_STORE_FORMAT,
+)
+
+#: Standard option
+ts_fmt_option = click.option(
+    "--ts-fmt",
+    help=(
+        "The datetime format to use for storing datetimes. Datetimes are always "
+        "stored in UTC (because Numpy does not store time zone info), so this "
+        "should not include a time zone name."
+    ),
+)
+#: Standard option
+ts_name_fmt_option = click.option(
+    "--ts-name-fmt",
+    help=(
+        "The datetime format to use when generating the workflow name, where it "
+        "includes a timestamp."
+    ),
+)
+
+#: Standard option
+variables_option = click.option(
+    "-v",
+    "--var",
+    "variables",
+    type=(str, str),
+    multiple=True,
+    help=(
+        "Workflow template variable value to be substituted in to the template file or "
+        "string. Multiple variable values can be specified."
+    ),
+)
+#: Standard option
+js_parallelism_option = click.option(
+    "--js-parallelism",
+    help=(
+        "If True, allow multiple jobscripts to execute simultaneously. If "
+        "'scheduled'/'direct', only allow simultaneous execution of scheduled/direct "
+        "jobscripts. Raises if set to True, 'scheduled', or 'direct', but the store type "
+        "does not support the `jobscript_parallelism` feature. If not set, jobscript "
+        "parallelism will be used if the store type supports it, for scheduled "
+        "jobscripts only."
+    ),
+    type=BoolOrString(["direct", "scheduled"]),
+)
+#: Standard option
+wait_option = click.option(
+    "--wait",
+    help=("If True, this command will block until the workflow execution is complete."),
+    is_flag=True,
+    default=False,
+)
+#: Standard option
+add_to_known_opt = click.option(
+    "--add-to-known/--no-add-to-known",
+    default=True,
+    help="If True, add this submission to the known-submissions file.",
+)
+#: Standard option
+print_idx_opt = click.option(
+    "--print-idx",
+    help="If True, print the submitted jobscript indices for each submission index.",
+    is_flag=True,
+    default=False,
+)
+#: Standard option
+tasks_opt = click.option(
+    "--tasks",
+    help=(
+        "List of comma-separated task indices to include in this submission. By default "
+        "all tasks are included."
+    ),
+    callback=sub_tasks_callback,
+)
+#: Standard option
+cancel_opt = click.option(
+    "--cancel",
+    help="Immediately cancel the submission. Useful for testing and benchmarking.",
+    is_flag=True,
+    default=False,
+)
+#: Standard option
+submit_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track submission progress.",
+    default=True,
+)
+#: Standard option
+force_arr_opt = click.option(
+    "--force-array",
+    help=(
+        "Used to force the use of job arrays, even if the scheduler does not support it. "
+        "This is provided for testing purposes only."
+    ),
+    is_flag=True,
+    default=False,
+)
+
+#: Standard option
+make_status_opt = click.option(
+    "--status/--no-status",
+    help="If True, display a live status to track workflow creation progress.",
+    default=True,
+)
+
+#: Standard option
+add_sub_opt = click.option(
+    "--add-submission",
+    help=("If True, add a submission to the workflow (but do not submit)."),
+    is_flag=True,
+    default=False,
+)
+
+#: Standard option
+zip_path_opt = click.option(
+    "--path",
+    default=".",
+    help=(
+        "Path at which to create the new zipped workflow. If this is an existing "
+        "directory, the zip file will be created within this directory. Otherwise, this "
+        "path is assumed to be the full file path to the new zip file."
+    ),
+)
+#: Standard option
+zip_overwrite_opt = click.option(
+    "--overwrite",
+    is_flag=True,
+    default=False,
+    help="If set, any existing file will be overwritten.",
+)
+#: Standard option
+zip_log_opt = click.option("--log", help="Path to a log file to use during zipping.")
+#: Standard option
+zip_include_execute_opt = click.option("--include-execute", is_flag=True)
+#: Standard option
+zip_include_rechunk_backups_opt = click.option("--include-rechunk-backups", is_flag=True)
+
+#: Standard option
+unzip_path_opt = click.option(
+    "--path",
+    default=".",
+    help=(
+        "Path at which to create the new unzipped workflow. If this is an existing "
+        "directory, the new workflow directory will be created within this directory. "
+        "Otherwise, this path will represent the new workflow directory path."
+    ),
+)
+#: Standard option
+unzip_log_opt = click.option("--log", help="Path to a log file to use during unzipping.")
+
+#: Standard option
+rechunk_backup_opt = click.option(
+    "--backup/--no-backup",
+    default=True,
+    help=("First copy a backup of the array to a directory ending in `.bak`."),
+)
+#: Standard option
+rechunk_chunk_size_opt = click.option(
+    "--chunk-size",
+    type=click.INT,
+    default=None,
+    help=(
+        "New chunk size (array items per chunk). If unset (as by default), the array "
+        "will be rechunked to a single chunk array (i.e with a chunk size equal to the "
+        "array's shape)."
+    ),
+)
+#: Standard option
+rechunk_status_opt = click.option(
+    "--status/--no-status",
+    default=True,
+    help="If True, display a live status to track rechunking progress.",
+)
+cancel_status_opt = click.option(
+    "--status/--no-status",
+    default=True,
+    help="If True, display a live status to track cancel progress.",
+)
+
+list_js_max_js_opt = click.option(
+    "--max-js", type=click.INT, help="Display up to this jobscript only."
+)
+list_js_jobscripts_opt = click.option(
+    "--jobscripts", help="Comma-separated list of jobscript indices to show."
+)
+list_task_js_max_js_opt = click.option(
+    "--max-js", type=click.INT, help="Include jobscripts up to this jobscript only."
+)
+list_task_js_task_names_opt = click.option(
+    "--task-names", help="Comma-separated list of task name sub-strings to show."
+)
+list_js_width_opt = click.option(
+    "--width", type=click.INT, help="Width in characters of the table to print."
+)
+jobscript_std_array_idx_opt = click.option(
+    "--array-idx",
+    type=click.INT,
+    help=(
+        "For array jobs only, the job array index whose standard stream is to be printed."
+    ),
+)
+
+
+def _add_doc_from_help(*args):
+    """
+    Attach the ``help`` field of each of its arguments as its ``__doc__``.
+    Only necessary because the wrappers in Click don't do this for us.
+
+    :meta private:
+    """
+    # Yes, this is ugly!
+    from types import SimpleNamespace
+
+    for opt in args:
+        ns = SimpleNamespace()
+        params = getattr(opt(ns), "__click_params__", [])
+        if params:
+            help = getattr(params[0], "help", "")
+            if help:
+                opt.__doc__ = f"Click option decorator: {help}"
+
+
+_add_doc_from_help(
+    format_option,
+    path_option,
+    name_option,
+    overwrite_option,
+    store_option,
+    ts_fmt_option,
+    ts_name_fmt_option,
+    variables_option,
+    js_parallelism_option,
+    wait_option,
+    add_to_known_opt,
+    print_idx_opt,
+    tasks_opt,
+    cancel_opt,
+    submit_status_opt,
+    make_status_opt,
+    zip_path_opt,
+    zip_overwrite_opt,
+    zip_log_opt,
+    zip_include_execute_opt,
+    zip_include_rechunk_backups_opt,
+    unzip_path_opt,
+    unzip_log_opt,
+    rechunk_backup_opt,
+    rechunk_chunk_size_opt,
+    rechunk_status_opt,
+    cancel_status_opt,
+    list_js_max_js_opt,
+    list_js_jobscripts_opt,
+    list_task_js_max_js_opt,
+    list_task_js_task_names_opt,
+    list_js_width_opt,
+    jobscript_std_array_idx_opt,
+)

hpcflow/sdk/config/__init__.py

@@ -0,0 +1,5 @@
+"""
+Configuration loading and manipulation.
+"""
+
+from hpcflow.sdk.config.config import Config, ConfigFile, ConfigOptions, DEFAULT_CONFIG

hpcflow/sdk/config/callbacks.py

@@ -0,0 +1,246 @@
+"""Module that defines built-in callback functions for configuration item values."""
+
+from __future__ import annotations
+import os
+import re
+import fsspec  # type: ignore
+from typing import overload, TYPE_CHECKING
+from hpcflow.sdk.core.errors import UnsupportedSchedulerError, UnsupportedShellError
+from hpcflow.sdk.submission.shells import get_supported_shells
+
+if TYPE_CHECKING:
+    from typing import Any, TypeVar
+    from .config import Config
+    from ..typing import PathLike
+
+    T = TypeVar("T")
+
+
+def callback_vars(config: Config, value) -> str:
+    """
+    Callback that substitutes configuration variables.
+    """
+
+    def vars_repl(match_obj: re.Match[str]) -> str:
+        return config._variables[match_obj[1]]
+
+    vars_regex = rf"\<\<({ '|'.join(config._variables) })\>\>"
+    return re.sub(
+        pattern=vars_regex,
+        repl=vars_repl,
+        string=str(value),
+    )
+
+
+@overload
+def callback_file_paths(config: Config, file_path: PathLike) -> PathLike: ...
+
+
+@overload
+def callback_file_paths(config: Config, file_path: list[PathLike]) -> list[PathLike]: ...
+
+
+def callback_file_paths(config: Config, file_path: PathLike | list[PathLike]):
+    """
+    Callback that resolves file paths.
+    """
+    if isinstance(file_path, list):
+        return [config._resolve_path(path) for path in file_path]
+    else:
+        return config._resolve_path(file_path)
+
+
+def callback_bool(config: Config, value: str | bool) -> bool:
+    """
+    Callback that coerces values to boolean.
+    """
+    if not isinstance(value, bool):
+        if value.lower() == "true":
+            return True
+        elif value.lower() == "false":
+            return False
+        else:
+            raise TypeError(f"Cannot cast {value!r} to a bool type.")
+    return value
+
+
+@overload
+def callback_lowercase(config: Config, value: list[str]) -> list[str]: ...
+
+
+@overload
+def callback_lowercase(config: Config, value: dict[str, T]) -> dict[str, T]: ...
+
+
+@overload
+def callback_lowercase(config: Config, value: str) -> str: ...
+
+
+def callback_lowercase(
+    config: Config, value: list[str] | dict[str, T] | str
+) -> list[str] | dict[str, T] | str:
+    """
+    Callback that forces a string to lower case.
+    """
+    if isinstance(value, list):
+        return [item.lower() for item in value]
+    elif isinstance(value, dict):
+        return {k.lower(): v for k, v in value.items()}
+    else:
+        return value.lower()
+
+
+def exists_in_schedulers(config: Config, value: T) -> T:
+    """
+    Callback that tests that a value is a supported scheduler name.
+    """
+    if value not in config.schedulers:
+        raise ValueError(
+            f"Cannot set default scheduler; {value!r} is not a supported scheduler "
+            f"according to the config file, which lists these schedulers as available "
+            f"on this machine: {config.schedulers!r}."
+        )
+    return value
+
+
+def callback_supported_schedulers(
+    config: Config, schedulers: dict[str, Any]
+) -> dict[str, Any]:
+    """
+    Callback that tests that all values are names of supported schedulers.
+    """
+    # validate against supported schedulers according to the OS - this won't validate that
+    # a particular scheduler actually exists on this system:
+    available = set(config._app.get_OS_supported_schedulers())
+    if any((witness := k) not in available for k in schedulers):
+        raise UnsupportedSchedulerError(scheduler=witness, available=available)
+    return schedulers
+
+
+def _hostname_in_invocation(config: Config) -> bool:
+    return "hostname" in config._file.get_invocation(config._config_key)["match"]
+
+
+def set_scheduler_invocation_match(config: Config, scheduler: str) -> None:
+    """Invoked on set of `default_scheduler`.
+
+    For clusters with "proper" schedulers (SGE, SLURM, etc.), login nodes are typically
+    named using the word "login". So we can use this knowledge to set a default for the
+    "hostname" invocation match key, if it is not manually set. However, it is preferable
+    that on clusters the hostname match is explicitly set.
+
+    """
+    sched = config._app.get_scheduler(
+        scheduler_name=scheduler,
+        os_name=os.name,
+        scheduler_args=config.get(f"schedulers.{scheduler}").get("defaults", {}),
+    )
+    if isinstance(sched, config._app.QueuedScheduler):
+        if not _hostname_in_invocation(config):
+            config._file.update_invocation(
+                config_key=config._config_key,
+                match={"hostname": sched.DEFAULT_LOGIN_NODE_MATCH},
+            )
+
+
+def callback_scheduler_set_up(
+    config: Config, schedulers: dict[str, Any]
+) -> dict[str, Any]:
+    """Invoked on set of `schedulers`.
+
+    Runs scheduler-specific config initialisation.
+    """
+    for k, v in schedulers.items():
+        sched = config._app.get_scheduler(
+            scheduler_name=k,
+            os_name=os.name,
+            scheduler_args=v.get("defaults", {}),
+        )
+
+        if isinstance(sched, config._app.SGEPosix):
+            # some `QueuedScheduler` classes have a `get_login_nodes` method which can be used
+            # to populate the names of login nodes explicitly, if not already set:
+            if not _hostname_in_invocation(config):
+                config._file.update_invocation(
+                    config_key=config._config_key,
+                    match={"hostname": sched.get_login_nodes()},
+                )
+    return schedulers
+
+
+def callback_supported_shells(config: Config, shell_name: str) -> str:
+    """
+    Callback that tests if a shell names is supported on this OS.
+    """
+    supported = get_supported_shells(os.name)
+    if shell_name not in supported:
+        raise UnsupportedShellError(shell=shell_name, supported=supported)
+    return shell_name
+
+
+def set_callback_file_paths(config: Config, value: PathLike | list[PathLike]) -> None:
+    """Check the file(s) is/are accessible. This is only done on `config.set` (and not on
+    `config.get` or `config._validate`) because it could be expensive in the case of remote
+    files."""
+    value = callback_file_paths(config, value)
+
+    to_check = value if isinstance(value, list) else [value]
+
+    for file_path in to_check:
+        if file_path is None:
+            continue
+        with fsspec.open(file_path, mode="rt") as fh:
+            pass
+            # TODO: also check something in it?
+        print(f"Checked access to: {file_path}")
+
+
+def check_load_data_files(config: Config, value: Any) -> None:
+    """Check data files (e.g., task schema files) can be loaded successfully. This is only
+    done on `config.set` (and not on `config.get` or `config._validate`) because it could
+    be expensive in the case of remote files."""
+    config._app.reload_template_components(warn=False)
+
+
+def callback_log_file_path(config, value):
+    value = value.strip()
+    if value:
+        return config._resolve_path(value)
+    else:
+        return value
+
+
+def callback_update_log_console_level(config: Config, value: str) -> None:
+    """
+    Callback to set the logging level.
+    """
+    config._app.log.update_console_level(new_level=value)
+
+
+def callback_unset_log_console_level(config: Config) -> None:
+    """Reset the console handler to the default level."""
+    config._app.log.update_console_level()
+
+
+def callback_update_log_file_level(config: Config, value: str) -> None:
+    """Callback to set the level of the log file handler."""
+    config._app.log.update_file_level(new_level=value)
+
+
+def callback_update_log_file_path(config: Config, value: str) -> None:
+    """
+    Callback to update the log file path, or remove the file handler if no path specifed.
+    """
+    config._app.log.remove_file_handler()
+    if value:
+        config._app.log.add_file_logger(path=value, level=config.get("log_file_level"))
+
+
+def callback_unset_log_file_level(config: Config) -> None:
+    """Callback to reset the file handler to the default level."""
+    config._app.log.update_file_level()
+
+
+def callback_unset_log_file_path(config: Config) -> None:
+    """Callback to remove the log file handler."""
+    config._app.log.remove_file_handler()