hpcflow 0.1.9__py3-none-any.whl → 0.2.0a271__py3-none-any.whl

hpcflow/sdk/demo/cli.py

@@ -0,0 +1,242 @@
+"""
+CLI components for demonstration code.
+"""
+
+from __future__ import annotations
+from pathlib import Path
+from random import randint
+from typing import TYPE_CHECKING
+import click
+
+from hpcflow.sdk.core.workflow import Workflow
+from hpcflow.sdk.core.utils import get_process_stamp
+from hpcflow.sdk.cli_common import (
+    format_option,
+    path_option,
+    name_option,
+    name_timestamp_option,
+    name_dir_option,
+    overwrite_option,
+    store_option,
+    ts_fmt_option,
+    ts_name_fmt_option,
+    variables_option,
+    js_parallelism_option,
+    wait_option,
+    add_to_known_opt,
+    print_idx_opt,
+    tasks_opt,
+    cancel_opt,
+    submit_status_opt,
+    make_status_opt,
+    add_sub_opt,
+)
+from hpcflow.sdk.submission.submission import Submission
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+    from typing import Literal
+    from ..app import BaseApp
+
+
+def get_demo_software_CLI(app: BaseApp):
+    """Generate the CLI to provide an example software."""
+
+    @click.group()
+    def demo_software():
+        pass
+
+    @demo_software.command("doSomething")
+    @click.option("--infile1", "-i1", type=click.Path(exists=True), required=True)
+    @click.option("--infile2", "-i2", type=click.Path(exists=True), required=True)
+    @click.option("--value", "-v")
+    @click.option("--out", "-o")
+    def demo_do_something(
+        infile1: Path, infile2: Path, value: str | None = None, out: str | None = None
+    ):
+        click.echo("trying to do something")
+
+        with Path(infile1).open("r") as handle:
+            file_id_1 = int(handle.readline().strip())
+        with Path(infile2).open("r") as handle:
+            file_id_2 = int(handle.readline().strip())
+
+        if out is None:
+            out = "outfile.txt"
+        out_path = Path(out)
+        with out_path.open("a") as handle:
+            handle.write("{}\n".format(randint(0, int(1e6))))
+            handle.write(
+                "{} Generated by `doSomething --infile1 {} --infile2 {}`.\n".format(
+                    get_process_stamp(), infile1, infile2
+                )
+            )
+            if value:
+                handle.write("{} Value: {}\n".format(get_process_stamp(), value))
+            handle.write(
+                "{} Original file ID: {}: {}\n".format(
+                    get_process_stamp(), infile1, file_id_1
+                )
+            )
+            handle.write(
+                "{} Original file ID: {}: {}\n".format(
+                    get_process_stamp(), infile2, file_id_2
+                )
+            )
+
+    return demo_software
+
+
+def get_demo_workflow_CLI(app: BaseApp):
+    """Generate the CLI to provide access to builtin demo workflows."""
+
+    def list_callback(ctx: click.Context, param, value: bool):
+        if not value or ctx.resilient_parsing:
+            return
+        # TODO: format with Rich with a one-line description
+        click.echo("\n".join(app.list_demo_workflows()))
+        ctx.exit()
+
+    @click.group()
+    @click.option(
+        "-l",
+        "--list",
+        help="Print available builtin demo workflows.",
+        is_flag=True,
+        is_eager=True,
+        expose_value=False,
+        callback=list_callback,
+    )
+    def demo_workflow():
+        """Interact with builtin demo workflows."""
+        pass
+
+    @demo_workflow.command("make")
+    @click.argument("workflow_name")
+    @format_option
+    @path_option
+    @name_option
+    @name_timestamp_option
+    @name_dir_option
+    @overwrite_option
+    @store_option
+    @ts_fmt_option
+    @ts_name_fmt_option
+    @variables_option
+    @make_status_opt
+    @add_sub_opt
+    def make_demo_workflow(
+        workflow_name: str,
+        format: Literal["json", "yaml"] | None,
+        path: Path | None,
+        name: str | None,
+        name_add_timestamp: bool | None,
+        name_use_dir: bool | None,
+        overwrite: bool,
+        store: str,
+        ts_fmt: str | None = None,
+        ts_name_fmt: str | None = None,
+        variables: Iterable[tuple[str, str]] = (),
+        status: bool = True,
+        add_submission: bool = False,
+    ):
+        wk_or_sub = app.make_demo_workflow(
+            workflow_name=workflow_name,
+            template_format=format,
+            path=path,
+            name=name,
+            name_add_timestamp=name_add_timestamp,
+            name_use_dir=name_use_dir,
+            overwrite=overwrite,
+            store=store,
+            ts_fmt=ts_fmt,
+            ts_name_fmt=ts_name_fmt,
+            variables=dict(variables),
+            status=status,
+            add_submission=add_submission,
+        )
+        if add_submission:
+            assert isinstance(wk_or_sub, Submission)
+            click.echo(wk_or_sub.workflow.path)
+        else:
+            assert isinstance(wk_or_sub, Workflow)
+            click.echo(wk_or_sub.path)
+
+    @demo_workflow.command("go")
+    @click.argument("workflow_name")
+    @format_option
+    @path_option
+    @name_option
+    @name_timestamp_option
+    @name_dir_option
+    @overwrite_option
+    @store_option
+    @ts_fmt_option
+    @ts_name_fmt_option
+    @variables_option
+    @js_parallelism_option
+    @wait_option
+    @add_to_known_opt
+    @print_idx_opt
+    @tasks_opt
+    @cancel_opt
+    @submit_status_opt
+    def make_and_submit_demo_workflow(
+        workflow_name: str,
+        format: Literal["json", "yaml"] | None,
+        path: Path | None,
+        name: str | None,
+        name_add_timestamp: bool | None,
+        name_use_dir: bool | None,
+        overwrite: bool,
+        store: str,
+        ts_fmt: str | None = None,
+        ts_name_fmt: str | None = None,
+        variables: Iterable[tuple[str, str]] = (),
+        js_parallelism: bool | None = None,
+        wait: bool = False,
+        add_to_known: bool = True,
+        print_idx: bool = False,
+        tasks: list[int] | None = None,
+        cancel: bool = False,
+        status: bool = True,
+    ):
+        out = app.make_and_submit_demo_workflow(
+            workflow_name=workflow_name,
+            template_format=format,
+            path=path,
+            name=name,
+            name_add_timestamp=name_add_timestamp,
+            name_use_dir=name_use_dir,
+            overwrite=overwrite,
+            store=store,
+            ts_fmt=ts_fmt,
+            ts_name_fmt=ts_name_fmt,
+            variables=dict(variables),
+            JS_parallelism=js_parallelism,
+            wait=wait,
+            add_to_known=add_to_known,
+            return_idx=print_idx,
+            tasks=tasks,
+            cancel=cancel,
+            status=status,
+        )
+        if print_idx:
+            assert isinstance(out, tuple)
+            click.echo(out[1])
+
+    @demo_workflow.command("copy")
+    @click.argument("workflow_name")
+    @click.argument("destination")
+    @click.option("--doc/--no-doc", default=True)
+    def copy_demo_workflow(workflow_name: str, destination: str, doc: bool):
+        app.copy_demo_workflow(name=workflow_name, dst=destination, doc=doc)
+
+    @demo_workflow.command("show")
+    @click.argument("workflow_name")
+    @click.option("--syntax/--no-syntax", default=True)
+    @click.option("--doc/--no-doc", default=True)
+    def show_demo_workflow(workflow_name: str, syntax: bool, doc: bool):
+        app.show_demo_workflow(workflow_name, syntax=syntax, doc=doc)
+
+    return demo_workflow

hpcflow/sdk/helper/__init__.py

@@ -0,0 +1,3 @@
+"""
+Helpers for the CLI.
+"""

hpcflow/sdk/helper/cli.py

@@ -0,0 +1,137 @@
+"""
+Common Click command line options related to the helper.
+"""
+
+from __future__ import annotations
+from typing import TYPE_CHECKING
+
+import click
+
+from hpcflow.sdk.helper.helper import (
+    DEFAULT_TIMEOUT,
+    DEFAULT_TIMEOUT_CHECK,
+    DEFAULT_WATCH_INTERVAL,
+    get_helper_log_path,
+    get_watcher_file_path,
+    get_helper_watch_list,
+    start_helper,
+    stop_helper,
+    restart_helper,
+    clear_helper,
+    run_helper,
+    get_helper_PID,
+    get_helper_uptime,
+)
+from hpcflow.sdk.cli_common import _add_doc_from_help
+
+if TYPE_CHECKING:
+    from ..app import BaseApp
+
+#: Helper option: ``--timeout``
+timeout_option = click.option(
+    "--timeout",
+    type=click.FLOAT,
+    default=DEFAULT_TIMEOUT,
+    show_default=True,
+    help="Helper timeout in seconds.",
+)
+#: Helper option: ``--timeout-check-interval``
+timeout_check_interval_option = click.option(
+    "--timeout-check-interval",
+    type=click.FLOAT,
+    default=DEFAULT_TIMEOUT_CHECK,
+    show_default=True,
+    help="Interval between testing if the timeout has been exceeded in seconds.",
+)
+#: Helper option: ``--watch interval``
+watch_interval_option = click.option(
+    "--watch-interval",
+    type=click.FLOAT,
+    default=DEFAULT_WATCH_INTERVAL,
+    show_default=True,
+    help=(
+        "Polling interval for watching workflows (and the workflow watch list) in "
+        "seconds."
+    ),
+)
+_add_doc_from_help(timeout_option, timeout_check_interval_option, watch_interval_option)
+
+
+def get_helper_CLI(app: BaseApp):
+    """Generate the CLI to provide some server-like functionality."""
+
+    @click.group()
+    def helper():
+        pass
+
+    @helper.command()
+    @timeout_option
+    @timeout_check_interval_option
+    @watch_interval_option
+    def start(timeout: float, timeout_check_interval: float, watch_interval: float):
+        """Start the helper process."""
+        start_helper(app, timeout, timeout_check_interval, watch_interval)
+
+    @helper.command()
+    def stop():
+        """Stop the helper process, if it is running."""
+        stop_helper(app)
+
+    @helper.command()
+    @timeout_option
+    @timeout_check_interval_option
+    @watch_interval_option
+    def run(timeout: float, timeout_check_interval: float, watch_interval: float):
+        """Run the helper functionality."""
+        run_helper(app, timeout, timeout_check_interval, watch_interval)
+
+    @helper.command()
+    @timeout_option
+    @timeout_check_interval_option
+    @watch_interval_option
+    def restart(timeout: float, timeout_check_interval: float, watch_interval: float):
+        """Restart (or start) the helper process."""
+        restart_helper(app, timeout, timeout_check_interval, watch_interval)
+
+    @helper.command()
+    @click.option("-f", "--file", is_flag=True)
+    def pid(file: bool):
+        """Get the process ID of the running helper, if running."""
+        pid_info = get_helper_PID(app)
+        if pid_info:
+            pid, pid_file = pid_info
+            if file:
+                click.echo(f"{pid} ({str(pid_file)})")
+            else:
+                click.echo(pid)
+
+    @helper.command()
+    def clear() -> None:
+        """Remove the PID file (and kill the helper process if it exists). This should not
+        normally be needed."""
+        clear_helper(app)
+
+    @helper.command()
+    def uptime() -> None:
+        """Get the uptime of the helper process, if it is running."""
+        out = get_helper_uptime(app)
+        if out:
+            click.echo(out)
+
+    @helper.command()
+    def log_path() -> None:
+        """Get the path to the helper log file (may not exist)."""
+        click.echo(get_helper_log_path(app))
+
+    @helper.command()
+    def watch_list_path() -> None:
+        """Get the path to the workflow watch list file (may not exist)."""
+        click.echo(get_watcher_file_path(app))
+
+    @helper.command()
+    def watch_list() -> None:
+        """Get the list of workflows currently being watched."""
+        for wk in get_helper_watch_list(app) or ():
+            click.echo(str(wk["path"]))
+
+    return helper

hpcflow/sdk/helper/helper.py

@@ -0,0 +1,300 @@
+"""
+Implementation of a helper process used to monitor jobs.
+"""
+
+from __future__ import annotations
+from datetime import datetime, timedelta
+import logging
+from logging.handlers import RotatingFileHandler
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+from typing import Any, TYPE_CHECKING
+import psutil
+
+from hpcflow.sdk.helper.watcher import MonitorController
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from ..app import BaseApp
+
+
+DEFAULT_TIMEOUT = 3600  # seconds
+DEFAULT_TIMEOUT_CHECK = 60  # seconds
+DEFAULT_WATCH_INTERVAL = 10  # seconds
+
+
+def kill_proc_tree(
+    pid: int,
+    sig=signal.SIGTERM,
+    include_parent: bool = True,
+    timeout: float | None = None,
+    on_terminate: Callable[[psutil.Process], object] | None = None,
+) -> tuple[list[psutil.Process], list[psutil.Process]]:
+    """Kill a process tree (including grandchildren) with signal
+    `sig` and return a (gone, still_alive) tuple.
+    `on_terminate`, if specified, is a callback function which is
+    called as soon as a child terminates.
+
+    Returns
+    -------
+    list[Process]:
+        The process and subprocesses that have died.
+    list[Process]:
+        The process and subprocesses that are still alive.
+    """
+    assert pid != os.getpid(), "won't kill myself"
+    parent = psutil.Process(pid)
+    children = parent.children(recursive=True)
+    if include_parent:
+        children.append(parent)
+    for p in children:
+        try:
+            p.send_signal(sig)
+        except psutil.NoSuchProcess:
+            pass
+    return psutil.wait_procs(children, timeout=timeout, callback=on_terminate)
+
+
+def get_PID_file_path(app: BaseApp) -> Path:
+    """Get the path to the file containing the process ID of the helper, if running."""
+    return app.user_data_dir / "pid.txt"
+
+
+def get_watcher_file_path(app: BaseApp) -> Path:
+    """Get the path to the watcher file, which contains a list of workflows to watch."""
+    return app.user_data_dir / "watch_workflows.txt"
+
+
+def get_helper_log_path(app: BaseApp) -> Path:
+    """Get the log file path for the helper."""
+    return app.user_data_dir / "helper.log"
+
+
+def get_helper_watch_list(app: BaseApp):
+    """Get the list of workflows currently being watched by the helper process."""
+    watch_file_path = get_watcher_file_path(app)
+    if watch_file_path.exists():
+        return MonitorController.parse_watch_workflows_file(
+            watch_file_path, get_helper_logger(app)
+        )
+    return None
+
+
+def start_helper(
+    app: BaseApp,
+    timeout: timedelta | float = DEFAULT_TIMEOUT,
+    timeout_check_interval: timedelta | float = DEFAULT_TIMEOUT_CHECK,
+    watch_interval: timedelta | float = DEFAULT_WATCH_INTERVAL,
+    logger: logging.Logger | None = None,
+):
+    """
+    Start the helper process.
+    """
+    PID_file = get_PID_file_path(app)
+    if PID_file.is_file():
+        with PID_file.open("rt") as fp:
+            helper_pid = int(fp.read().strip())
+            print(f"Helper already running, with process ID: {helper_pid}")
+
+    else:
+        logger = logger or get_helper_logger(app)
+        logger.info(
+            f"Starting helper with timeout={timeout!r}, timeout_check_interval="
+            f"{timeout_check_interval!r} and watch_interval={watch_interval!r}."
+        )
+        kwargs: dict[str, Any] = {}
+        if os.name == "nt":
+            kwargs["creationflags"] = getattr(subprocess, "CREATE_NO_WINDOW", 0)
+
+        if isinstance(timeout, timedelta):
+            timeout = timeout.total_seconds()
+        if isinstance(timeout_check_interval, timedelta):
+            timeout_check_interval = timeout_check_interval.total_seconds()
+        if isinstance(watch_interval, timedelta):
+            watch_interval = watch_interval.total_seconds()
+
+        args = [
+            *app.run_time_info.invocation_command,
+            "--config-dir",
+            str(app.config.config_directory),
+            "helper",
+            "run",
+            "--timeout",
+            str(timeout),
+            "--timeout-check-interval",
+            str(timeout_check_interval),
+            "--watch-interval",
+            str(watch_interval),
+        ]
+
+        proc = subprocess.Popen(
+            args=args,
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            **kwargs,
+        )
+
+        logger.info(f"Writing process ID {proc.pid} to file.")
+        try:
+            with PID_file.open("wt") as fp:
+                fp.write(f"{proc.pid}\n")
+        except FileNotFoundError as err:
+            logger.error(
+                f"Could not write to the PID file {PID_file!r}; killing helper process. "
+                f"Exception was: {err!r}"
+            )
+            proc.kill()
+            sys.exit(1)
+
+
+def restart_helper(
+    app: BaseApp,
+    timeout: timedelta | float = DEFAULT_TIMEOUT,
+    timeout_check_interval: timedelta | float = DEFAULT_TIMEOUT_CHECK,
+    watch_interval: timedelta | float = DEFAULT_WATCH_INTERVAL,
+):
+    """
+    Restart the helper process.
+    """
+    logger = stop_helper(app, return_logger=True)
+    start_helper(app, timeout, timeout_check_interval, watch_interval, logger=logger)
+
+
+def get_helper_PID(app: BaseApp):
+    """
+    Get the process ID of the helper process.
+    """
+    PID_file = get_PID_file_path(app)
+    if not PID_file.is_file():
+        print("Helper not running!")
+        return None
+    with PID_file.open("rt") as fp:
+        helper_pid = int(fp.read().strip())
+    return helper_pid, PID_file
+
+
+def stop_helper(app: BaseApp, return_logger: bool = False):
+    """
+    Stop the helper process.
+    """
+    logger = get_helper_logger(app)
+    if pid_info := get_helper_PID(app):
+        logger.info("Stopping helper.")
+        pid, pid_file = pid_info
+        kill_proc_tree(pid=pid)
+        pid_file.unlink()
+
+        workflow_dirs_file_path = get_watcher_file_path(app)
+        logger.info(f"Deleting watcher file: {str(workflow_dirs_file_path)}")
+        workflow_dirs_file_path.unlink()
+
+    return logger if return_logger else None
+
+
+def clear_helper(app: BaseApp):
+    """
+    Stop the helper or remove any stale information relating to it.
+    """
+    try:
+        stop_helper(app)
+    except psutil.NoSuchProcess:
+        if pid_info := get_helper_PID(app):
+            pid_file = pid_info[1]
+            print(f"Removing file {pid_file!r}")
+            pid_file.unlink()
+
+
+def get_helper_uptime(app: BaseApp) -> None | timedelta:
+    """
+    Get the amount of time that the helper has been running.
+    """
+    if not (pid_info := get_helper_PID(app)):
+        return None
+    proc = psutil.Process(pid_info[0])
+    create_time = datetime.fromtimestamp(proc.create_time())
+    return datetime.now() - create_time
+
+
+def get_helper_logger(app: BaseApp) -> logging.Logger:
+    """
+    Get the logger for helper-related messages.
+    """
+    log_path = get_helper_log_path(app)
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+    f_handler = RotatingFileHandler(log_path, maxBytes=(5 * 2**20), backupCount=3)
+    f_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+    f_handler.setFormatter(f_format)
+    logger.addHandler(f_handler)
+
+    return logger
+
+
+def helper_timeout(
+    app: BaseApp,
+    timeout: timedelta,
+    controller: MonitorController,
+    logger: logging.Logger,
+):
+    """Kill the helper due to running duration exceeding the timeout."""
+
+    logger.info(f"Helper exiting due to timeout ({timeout!r}).")
+    if pid_info := get_helper_PID(app):
+        pid_file = pid_info[1]
+        logger.info(f"Deleting PID file: {pid_file!r}.")
+        pid_file.unlink()
+
+    logger.info("Stopping all watchers.")
+    controller.stop()
+    controller.join()
+
+    logger.info(f"Deleting watcher file: {str(controller.workflow_dirs_file_path)}")
+    controller.workflow_dirs_file_path.unlink()
+
+    sys.exit(0)
+
+
+def run_helper(
+    app: BaseApp,
+    timeout: timedelta | float = DEFAULT_TIMEOUT,
+    timeout_check_interval: timedelta | float = DEFAULT_TIMEOUT_CHECK,
+    watch_interval: timedelta | float = DEFAULT_WATCH_INTERVAL,
+):
+    """
+    Run the helper core.
+    """
+    # TODO: when writing to watch_workflows from a workflow, copy, modify and then rename
+    # this will be atomic - so there will be only one event fired.
+    # Also return a local run ID (the position in the file) to be used in jobscript naming
+
+    # TODO: we will want to set the timeout to be slightly more than the largest allowable
+    # walltime in the case of scheduler submissions.
+
+    if not isinstance(timeout, timedelta):
+        timeout = timedelta(seconds=timeout)
+
+    if isinstance(timeout_check_interval, timedelta):
+        timeout_check_interval_s = timeout_check_interval.total_seconds()
+    else:
+        timeout_check_interval_s = timeout_check_interval
+        timeout_check_interval = timedelta(seconds=timeout_check_interval_s)
+
+    start_time = datetime.now()
+    logger = get_helper_logger(app)
+    controller = MonitorController(get_watcher_file_path(app), watch_interval, logger)
+    timeout_limit = timeout - timeout_check_interval
+    try:
+        while True:
+            if datetime.now() - start_time >= timeout_limit:
+                helper_timeout(app, timeout, controller, logger)
+            time.sleep(timeout_check_interval_s)
+
+    except KeyboardInterrupt:
+        controller.stop()
+
+    controller.join()  # wait for it to stop!