hpcflow-new2 0.2.0a213__tar.gz → 0.2.0a215__tar.gz

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hpcflow-new2
-Version: 0.2.0a213
+Version: 0.2.0a215
 Summary: Computational workflow management
 License: MPL-2.0
 Author: aplowman
@@ -36,7 +36,7 @@ Requires-Dist: scipy (>=1.15.2,<2.0.0) ; python_version >= "3.10"
 Requires-Dist: termcolor (>=1.1.0,<2.0.0)
 Requires-Dist: typing-extensions (>=4.12.2,<5.0.0)
 Requires-Dist: valida (>=0.7.5,<0.8.0)
-Requires-Dist: watchdog (>=2.1.9,<3.0.0)
+Requires-Dist: watchdog (>=6.0.0,<7.0.0)
 Requires-Dist: zarr (==2.17.2)
 Description-Content-Type: text/markdown
 

hpcflow_new2-0.2.0a215/hpcflow/_version.py

@@ -0,0 +1 @@
+__version__ = "0.2.0a215"

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/core/utils.py

@@ -832,8 +832,8 @@ class JSONLikeDirSnapShot(DirectorySnapshot):
 
         #: Where to take the snapshot based at.
         self.root_path = root_path
-        self._stat_info: dict[str, os.stat_result] = {}
-        self._inode_to_path: dict[tuple, str] = {}
+        self._stat_info: dict[bytes | str, os.stat_result] = {}
+        self._inode_to_path: dict[tuple[int, int], bytes | str] = {}
 
         if data:
             assert root_path
@@ -862,7 +862,7 @@ class JSONLikeDirSnapShot(DirectorySnapshot):
         # store efficiently:
         inode_invert = {v: k for k, v in self._inode_to_path.items()}
         data: dict[str, list] = {
-            str(PurePath(k).relative_to(root_path)): [
+            str(PurePath(cast("str", k)).relative_to(cast("str", root_path))): [
                 str(i) if use_strings else i for i in [*v, *inode_invert[k]]
             ]
             for k, v in self._stat_info.items()

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/data/config_schema.yaml

@@ -152,8 +152,9 @@ rules:
   - path: [schedulers, { type: map_value }, defaults]
     condition:
       value.allowed_keys: # TODO: split these up based on scheduler type
-        - shebang_args
-        - options
+        - shebang_executable
+        - directives # TODO: only actually allowed in `QueuedScheduler` sub-classes
+        - options # deprecated, replaced by directives
         - submit_cmd
         - show_cmd
         - del_cmd
@@ -211,6 +212,7 @@ rules:
     condition:
       value.allowed_keys:
         - executable
+        - executable_args
         - os_args
         - WSL_executable
         - WSL_distribution

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/helper/watcher.py

@@ -7,6 +7,7 @@ from collections.abc import Callable
 from datetime import timedelta
 from logging import Logger
 from pathlib import Path
+from typing import cast
 from watchdog.observers.polling import PollingObserver
 from watchdog.events import (
     FileSystemEvent,
@@ -17,7 +18,7 @@ from watchdog.events import (
 
 class _PMEHDelegate(PatternMatchingEventHandler):
     def __init__(self, pattern: str, on_modified: Callable[[FileSystemEvent], None]):
-        super().__init__([pattern])
+        super().__init__(patterns=[pattern])
         self.__on_modified = on_modified
 
     def on_modified(self, event: FileSystemEvent) -> None:
@@ -64,7 +65,7 @@ class MonitorController:
         self.observer = PollingObserver(timeout=self.watch_interval)
         self.observer.schedule(
             self.event_handler,
-            path=self.workflow_dirs_file_path.parent,
+            path=cast("str", self.workflow_dirs_file_path.parent),
             recursive=False,
         )
 
@@ -112,8 +113,10 @@ class MonitorController:
         """
         Callback when files are modified.
         """
-        self.logger.info(f"Watch file modified: {event.src_path}")
-        wks = self.parse_watch_workflows_file(event.src_path, logger=self.logger)
+        self.logger.info(f"Watch file modified: {event.src_path!r}")
+        wks = self.parse_watch_workflows_file(
+            cast("str", event.src_path), logger=self.logger
+        )
         self.workflow_monitor.update_workflow_paths(wks)
 
     def join(self) -> None:
@@ -157,7 +160,9 @@ class WorkflowMonitor:
         observer = PollingObserver(timeout=self.watch_interval)
         self.observer: PollingObserver | None = observer
         for i in self.workflow_paths:
-            observer.schedule(self.event_handler, path=i["path"], recursive=False)
+            observer.schedule(
+                self.event_handler, path=cast("str", i["path"]), recursive=False
+            )
             self.logger.info(f"Watching workflow: {i['path'].name}")
 
         observer.start()
@@ -166,7 +171,7 @@ class WorkflowMonitor:
         """
         Triggered on a workflow being modified.
         """
-        self.logger.info(f"Workflow modified: {event.src_path}")
+        self.logger.info(f"Workflow modified: {event.src_path!r}")
 
     def update_workflow_paths(self, new_paths: list[dict[str, Path]]):
         """

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/jobscript.py

@@ -1440,7 +1440,7 @@ class Jobscript(JSONLike):
         scheduler_name: str | None = None,
         scheduler_args: dict[str, Any] | None = None,
     ) -> str:
-        """Prepare the jobscript file string."""
+        """Prepare the jobscript file contents as a string."""
         scheduler_name = scheduler_name or self.scheduler_name
         assert scheduler_name
         assert os_name
@@ -1465,15 +1465,14 @@ class Jobscript(JSONLike):
         }
 
         shebang = shell.JS_SHEBANG.format(
-            shebang_executable=" ".join(shell.shebang_executable),
-            shebang_args=scheduler.shebang_args,
+            shebang=" ".join(scheduler.shebang_executable or shell.shebang_executable)
         )
         header = shell.JS_HEADER.format(**header_args)
 
         if isinstance(scheduler, QueuedScheduler):
             header = shell.JS_SCHEDULER_HEADER.format(
                 shebang=shebang,
-                scheduler_options=scheduler.format_options(
+                scheduler_options=scheduler.format_directives(
                     resources=self.resources,
                     num_elements=self.blocks[0].num_elements,  # only used for array jobs
                     is_array=self.is_array,

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/schedulers/__init__.py

@@ -7,6 +7,7 @@ from abc import ABC, abstractmethod
 import sys
 import time
 from typing import Generic, TypeVar, TYPE_CHECKING
+import warnings
 from typing_extensions import override
 from hpcflow.sdk.typing import hydrate
 from hpcflow.sdk.core.app_aware import AppAware
@@ -39,12 +40,9 @@ class Scheduler(ABC, Generic[JSRefType], AppAware):
 
     Parameters
     ----------
-    shell_args: str
-        Arguments to pass to the shell. Pre-quoted.
-    shebang_args: str
-        Arguments to set on the shebang line. Pre-quoted.
-    options: dict
-        Options to the scheduler.
+    shebang_executable: list[str]
+        If specified, this will be used in the jobscript's shebang line instead of the
+        shell's `executable` and `executable_args` attributes.
     """
 
     # This would be in the docstring except it renders really wrongly!
@@ -53,20 +51,8 @@ class Scheduler(ABC, Generic[JSRefType], AppAware):
     # T
     #     The type of a jobscript reference.
 
-    #: Default value for arguments to the shell.
-    DEFAULT_SHELL_ARGS: ClassVar[str] = ""
-    #: Default value for arguments on the shebang line.
-    DEFAULT_SHEBANG_ARGS: ClassVar[str] = ""
-
-    def __init__(
-        self,
-        shell_args: str | None = None,
-        shebang_args: str | None = None,
-        options: dict | None = None,
-    ):
-        self.shebang_args = shebang_args or self.DEFAULT_SHEBANG_ARGS
-        self.shell_args = shell_args or self.DEFAULT_SHELL_ARGS
-        self.options = options or {}
+    def __init__(self, shebang_executable: list[str] | None = None):
+        self.shebang_executable = shebang_executable
 
     @property
     def unique_properties(self) -> tuple[str, ...]:
@@ -168,6 +154,13 @@ class QueuedScheduler(Scheduler[str]):
 
     Parameters
     ----------
+    directives: dict
+        Scheduler directives. Each item is written verbatim in the jobscript as a
+        scheduler directive, and is not processed in any way. If a value is `None`, the
+        key is considered a flag-like directive. If a value is a list, multiple directives
+        will be printed to the jobscript with the same key, but different values.
+    options: dict
+        Deprecated. Please use `directives` instead.
     submit_cmd: str
         The submission command, if overridden from default.
     show_cmd: str
@@ -203,6 +196,8 @@ class QueuedScheduler(Scheduler[str]):
 
     def __init__(
         self,
+        directives: dict | None = None,
+        options: dict | None = None,
         submit_cmd: str | None = None,
         show_cmd: Sequence[str] | None = None,
         del_cmd: str | None = None,
@@ -215,6 +210,16 @@ class QueuedScheduler(Scheduler[str]):
     ) -> None:
         super().__init__(*args, **kwargs)
 
+        if options:
+            warnings.warn(
+                f"{self.__class__.__name__!r}: Please use `directives` instead of "
+                f"`options`, which will be removed in a future release.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            directives = options
+
+        self.directives = directives or {}
         self.submit_cmd: str = submit_cmd or self.DEFAULT_SUBMIT_CMD
         self.show_cmd = show_cmd or self.DEFAULT_SHOW_CMD
         self.del_cmd = del_cmd or self.DEFAULT_DEL_CMD
@@ -250,7 +255,7 @@ class QueuedScheduler(Scheduler[str]):
             time.sleep(2)
 
     @abstractmethod
-    def format_options(
+    def format_directives(
         self,
         resources: ElementResources,
         num_elements: int,
@@ -259,7 +264,7 @@ class QueuedScheduler(Scheduler[str]):
         js_idx: int,
     ) -> str:
         """
-        Render options in a way that the scheduler can handle.
+        Render directives in a way that the scheduler can handle.
         """
 
     def get_std_out_err_filename(

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/schedulers/direct.py

@@ -44,14 +44,6 @@ class DirectScheduler(Scheduler[DirectRef]):
     The correct subclass (:py:class:`DirectPosix` or :py:class:`DirectWindows`) should
     be used to create actual instances.
 
-    Keyword Args
-    ------------
-    shell_args: str
-        Arguments to pass to the shell. Pre-quoted.
-    shebang_args: str
-        Arguments to set on the shebang line. Pre-quoted.
-    options: dict
-        Options to the jobscript command.
     """
 
     @classmethod
@@ -219,36 +211,16 @@ class DirectPosix(DirectScheduler):
     """
     A direct scheduler for POSIX systems.
 
-    Keyword Args
-    ------------
-    shell_args: str
-        Arguments to pass to the shell. Pre-quoted.
-    shebang_args: str
-        Arguments to set on the shebang line. Pre-quoted.
-    options: dict
-        Options to the jobscript command.
     """
 
-    #: Default shell.
-    DEFAULT_SHELL_EXECUTABLE: ClassVar[str] = "/bin/bash"
-
 
 @hydrate
 class DirectWindows(DirectScheduler):
     """
     A direct scheduler for Windows.
 
-    Keyword Args
-    ------------
-    shell_args: str
-        Arguments to pass to the shell. Pre-quoted.
-    options: dict
-        Options to the jobscript command.
     """
 
-    #: Default shell.
-    DEFAULT_SHELL_EXECUTABLE: ClassVar[str] = "powershell.exe"
-
     @override
     def get_submit_command(
         self, shell: Shell, js_path: str, deps: dict[Any, tuple[Any, ...]]

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/schedulers/sge.py

@@ -37,12 +37,11 @@ class SGEPosix(QueuedScheduler):
     ------------
     cwd_switch: str
         Override of default switch to use to set the current working directory.
-    shell_args: str
-        Arguments to pass to the shell. Pre-quoted.
-    shebang_args: str
-        Arguments to set on the shebang line. Pre-quoted.
-    options: dict
-        Options to the jobscript command.
+    directives: dict
+        Scheduler directives. Each item is written verbatim in the jobscript as a
+        scheduler directive, and is not processed in any way. If a value is `None`, the
+        key is considered a flag-like directive. If a value is a list, multiple directives
+        will be printed to the jobscript with the same key, but different values.
 
     Notes
     -----
@@ -55,8 +54,6 @@ class SGEPosix(QueuedScheduler):
 
     """
 
-    #: Default args for shebang line.
-    DEFAULT_SHEBANG_ARGS: ClassVar[str] = ""
     #: Default submission command.
     DEFAULT_SUBMIT_CMD: ClassVar[str] = "qsub"
     #: Default command to show the queue state.
@@ -203,7 +200,7 @@ class SGEPosix(QueuedScheduler):
             yield f"{self.js_cmd} -e {base}"
 
     @override
-    def format_options(
+    def format_directives(
         self,
         resources: ElementResources,
         num_elements: int,
@@ -212,7 +209,7 @@ class SGEPosix(QueuedScheduler):
         js_idx: int,
     ) -> str:
         """
-        Format the options to the jobscript command.
+        Format the directives to the jobscript command.
         """
         opts: list[str] = []
         opts.append(self.format_switch(self.cwd_switch))
@@ -226,7 +223,7 @@ class SGEPosix(QueuedScheduler):
             )
         )
 
-        for opt_k, opt_v in self.options.items():
+        for opt_k, opt_v in self.directives.items():
             if opt_v is None:
                 opts.append(f"{self.js_cmd} {opt_k}")
             elif isinstance(opt_v, list):

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/schedulers/slurm.py

@@ -37,12 +37,11 @@ class SlurmPosix(QueuedScheduler):
 
     Keyword Args
     ------------
-    shell_args: str
-        Arguments to pass to the shell. Pre-quoted.
-    shebang_args: str
-        Arguments to set on the shebang line. Pre-quoted.
-    options: dict
-        Options to the jobscript command.
+    directives: dict
+        Scheduler directives. Each item is written verbatim in the jobscript as a
+        scheduler directive, and is not processed in any way. If a value is `None`, the
+        key is considered a flag-like directive. If a value is a list, multiple directives
+        will be printed to the jobscript with the same key, but different values.
 
     Notes
     -----
@@ -59,10 +58,6 @@ class SlurmPosix(QueuedScheduler):
 
     """
 
-    #: Default shell.
-    DEFAULT_SHELL_EXECUTABLE: ClassVar[str] = "/bin/bash"
-    #: Default args for shebang line.
-    DEFAULT_SHEBANG_ARGS: ClassVar[str] = ""
     #: Default submission command.
     DEFAULT_SUBMIT_CMD: ClassVar[str] = "sbatch"
     #: Default command to show the queue state.
@@ -368,7 +363,7 @@ class SlurmPosix(QueuedScheduler):
             yield f"{self.js_cmd} --error {base}.err"
 
     @override
-    def format_options(
+    def format_directives(
         self,
         resources: ElementResources,
         num_elements: int,
@@ -377,7 +372,7 @@ class SlurmPosix(QueuedScheduler):
         js_idx: int,
     ) -> str:
         """
-        Format the options to the scheduler.
+        Format the directives to the scheduler.
         """
         opts: list[str] = []
         opts.extend(self.__format_core_request_lines(resources))
@@ -391,7 +386,7 @@ class SlurmPosix(QueuedScheduler):
             )
         )
 
-        for opt_k, opt_v in self.options.items():
+        for opt_k, opt_v in self.directives.items():
             if isinstance(opt_v, list):
                 for i in opt_v:
                     opts.append(f"{self.js_cmd} {opt_k} {i}")

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/shells/base.py

@@ -79,30 +79,39 @@ class Shell(ABC):
     #: Template for the jobscript footer.
     JS_FOOTER: ClassVar[str]
 
-    __slots__ = ("_executable", "os_args")
+    __slots__ = ("_executable", "executable_args", "os_args")
 
     def __init__(
-        self, executable: str | None = None, os_args: dict[str, str] | None = None
+        self,
+        executable: str | None = None,
+        executable_args: list[str] | None = None,
+        os_args: dict[str, str] | None = None,
     ):
         #: Which executable implements the shell.
         self._executable = executable or self.DEFAULT_EXE
+        #: Arguments to provide to the shell executable (e.g. `--login`).
+        self.executable_args = executable_args or []
         #: Arguments to pass to the shell.
         self.os_args = os_args or {}
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, self.__class__):
             return False
-        return self._executable == other._executable and self.os_args == other.os_args
+        return (
+            self._executable == other._executable
+            and self.executable_args == other.executable_args
+            and self.os_args == other.os_args
+        )
 
     def __hash__(self):
-        return get_hash((self._executable, self.os_args))
+        return get_hash((self._executable, tuple(self.executable_args), self.os_args))
 
     @property
     def executable(self) -> list[str]:
         """
         The executable to use plus any mandatory arguments.
         """
-        return [self._executable]
+        return [self._executable, *self.executable_args]
 
     @property
     def shebang_executable(self) -> list[str]:

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/sdk/submission/shells/bash.py

@@ -38,7 +38,7 @@ class Bash(Shell):
     #: Indent for environment setup.
     JS_ENV_SETUP_INDENT: ClassVar[str] = 2 * JS_INDENT
     #: Template for the jobscript shebang line.
-    JS_SHEBANG: ClassVar[str] = """#!{shebang_executable} {shebang_args}"""
+    JS_SHEBANG: ClassVar[str] = """#!{shebang}"""
     #: Template for the jobscript functions file.
     JS_FUNCS: ClassVar[str] = dedent(
         """\
@@ -429,6 +429,10 @@ class WSLBash(Bash):
 
     @property
     def shebang_executable(self) -> list[str]:
+        """
+        The executable to use in a shebang line, overridden here to exclude the WSL
+        command.
+        """
         return super().executable
 
     def _get_OS_info_POSIX(self) -> Mapping[str, str]:

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/tests/unit/test_shell.py

@@ -1,4 +1,10 @@
 from __future__ import annotations
+from pathlib import Path
+import sys
+
+import pytest
+
+import hpcflow.app as hf
 from hpcflow.sdk.submission.shells import ALL_SHELLS
 
 
@@ -97,3 +103,27 @@ def test_format_array_bash():
 def test_format_array_get_item_bash():
     shell = ALL_SHELLS["bash"]["posix"]()
     assert shell.format_array_get_item("my_arr", 3) == r"${my_arr[3]}"
+
+
+@pytest.mark.integration
+@pytest.mark.skipif(condition=sys.platform == "win32", reason="This is a bash-only test.")
+def test_executable_args_bash_login(null_config, tmp_path: Path):
+    """Check if we provide a `--login` argument to the shell `executable_args`, we end up
+    in a login shell on bash."""
+
+    cmd = "shopt -q login_shell && echo 'Login shell' || echo 'Not login shell'"
+    s1 = hf.TaskSchema(
+        objective="t1",
+        actions=[hf.Action(commands=[hf.Command(command=cmd)])],
+    )
+    t1 = hf.Task(
+        schema=s1,
+        resources={"any": {"shell_args": {"executable_args": ["--login"]}}},
+    )
+    wkt = hf.WorkflowTemplate(name="test_bash_login", tasks=[t1])
+    wk = hf.Workflow.from_template(
+        template=wkt,
+        path=tmp_path,
+    )
+    wk.submit(wait=True, status=False, add_to_known=False)
+    assert wk.submissions[0].jobscripts[0].get_stdout().strip() == "Login shell"

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/hpcflow/tests/unit/test_submission.py

@@ -468,7 +468,11 @@ def test_unique_schedulers_two_direct_and_SLURM(new_null_config, tmp_path) -> No
 
 def test_scheduler_config_defaults(new_null_config, tmp_path) -> None:
     """Check default options defined in the config are merged into jobscript resources."""
-    hf.config.set("schedulers.direct.defaults.options", {"a": "c"})
+
+    # note we use the `shebang_executable` for this test. On Windows, this will not be
+    # included in the jobscript, so it is effectively ignored, but the test is still
+    # valid.
+    hf.config.set("schedulers.direct.defaults.shebang_executable", ["/bin/bash"])
 
     t1 = hf.Task(
         schema=hf.task_schemas.test_t1_ps,
@@ -479,7 +483,10 @@ def test_scheduler_config_defaults(new_null_config, tmp_path) -> None:
         schema=hf.task_schemas.test_t1_ps,
         inputs={"p1": 1},
         resources={
-            "any": {"scheduler": "direct", "scheduler_args": {"options": {"a": "b"}}}
+            "any": {
+                "scheduler": "direct",
+                "scheduler_args": {"shebang_executable": ["bash"]},
+            }
         },
     )
     wkt = hf.WorkflowTemplate(name="temp", tasks=[t1, t2])
@@ -489,5 +496,7 @@ def test_scheduler_config_defaults(new_null_config, tmp_path) -> None:
     )
     sub = wk.add_submission()
     assert sub is not None
-    assert sub.jobscripts[0].resources.scheduler_args == {"options": {"a": "c"}}
-    assert sub.jobscripts[1].resources.scheduler_args == {"options": {"a": "b"}}
+    assert sub.jobscripts[0].resources.scheduler_args == {
+        "shebang_executable": ["/bin/bash"]
+    }
+    assert sub.jobscripts[1].resources.scheduler_args == {"shebang_executable": ["bash"]}

{hpcflow_new2-0.2.0a213 → hpcflow_new2-0.2.0a215}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [tool.poetry]
 name = "hpcflow-new2"
-version = "0.2.0a213"
+version = "0.2.0a215"
 
 description = "Computational workflow management"
 authors = ["aplowman <adam.plowman@manchester.ac.uk>"]
@@ -32,7 +32,7 @@ click = "^8.1.3"
 msgpack = "^1.0.4"
 reretry = "^0.11.8"
 psutil = "^5.9.4"
-watchdog = "^2.1.9"
+watchdog = "^6.0.0"
 paramiko = "^3.2.0"
 aiohttp = "^3.8.4"
 rich = "^13.4.2"
@@ -100,7 +100,7 @@ hook-dirs = "hpcflow.__pyinstaller:get_hook_dirs"
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "0.2.0a213"
+version = "0.2.0a215"
 tag_format = "v$version"
 version_files = [ 
     "pyproject.toml:version",

hpcflow_new2-0.2.0a213/hpcflow/_version.py

@@ -1 +0,0 @@
-__version__ = "0.2.0a213"