aiida-hyperqueue 0.1.0__tar.gz → 0.2.0__tar.gz

aiida_hyperqueue-0.2.0/.github/workflows/cd.yml

@@ -0,0 +1,52 @@
+name: cd
+
+on:
+    push:
+        tags:
+        - 'v[0-9]+.[0-9]+.[0-9]+*'
+
+jobs:
+
+    validate-release-tag:
+
+        if: github.repository == 'aiidateam/aiida-hyperqueue'
+        runs-on: ubuntu-latest
+
+        steps:
+        -   name: Checkout source
+            uses: actions/checkout@v3
+
+        -   name: Set up Python 3.10
+            uses: actions/setup-python@v4
+            with:
+                python-version: '3.10'
+
+        -   name: Validate the tag version against the package version
+            run: python .github/workflows/validate_release_tag.py $GITHUB_REF
+
+    publish:
+
+        name: Publish to PyPI
+        needs: [validate-release-tag]
+        runs-on: ubuntu-latest
+        environment: deployment
+        permissions:
+            id-token: write
+
+        steps:
+        -   name: Checkout source
+            uses: actions/checkout@v3
+
+        -   name: Set up Python 3.10
+            uses: actions/setup-python@v4
+            with:
+                python-version: '3.10'
+
+        -   name: Install flit
+            run: pip install flit~=3.8
+
+        -   name: Build with flit
+            run: flit build
+
+        -   name: Publish to PyPI using Trusted Publishing
+            uses: pypa/gh-action-pypi-publish@release/v1

aiida_hyperqueue-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,30 @@
+name: ci
+
+on: [push, pull_request]
+
+jobs:
+
+  tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install python dependencies
+      run: |
+        pip install --upgrade pip
+        pip install .[tests]
+
+    - name: Run test suite
+      run: |
+        chmod og-rw ~
+        pytest --cov aiida_hyperqueue --cov-append .

aiida_hyperqueue-0.2.0/.github/workflows/validate_release_tag.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""Validate that the version in the tag label matches the version of the package."""
+
+import argparse
+import ast
+from pathlib import Path
+
+
+def get_version_from_module(content: str) -> str:
+    """Get the ``__version__`` attribute from a module.
+
+    .. note:: This has been adapted from :mod:`setuptools.config`.
+    """
+    try:
+        module = ast.parse(content)
+    except SyntaxError as exception:
+        raise OSError("Unable to parse module.") from exception
+
+    try:
+        return next(
+            ast.literal_eval(statement.value)
+            for statement in module.body
+            if isinstance(statement, ast.Assign)
+            for target in statement.targets
+            if isinstance(target, ast.Name) and target.id == "__version__"
+        )
+    except StopIteration as exception:
+        raise OSError(
+            "Unable to find the `__version__` attribute in the module."
+        ) from exception
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("GITHUB_REF", help="The GITHUB_REF environmental variable")
+    args = parser.parse_args()
+    assert args.GITHUB_REF.startswith(
+        "refs/tags/v"
+    ), f'GITHUB_REF should start with "refs/tags/v": {args.GITHUB_REF}'
+    tag_version = args.GITHUB_REF[11:]
+    package_version = get_version_from_module(
+        Path("aiida_hyperqueue/__init__.py").read_text(encoding="utf-8")
+    )
+    error_message = f"The tag version `{tag_version}` is different from the package version `{package_version}`"
+    assert tag_version == package_version, error_message

aiida_hyperqueue-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+# Install pre-commit hooks via:
+# pre-commit install
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.6.0
+      hooks:
+        - id: end-of-file-fixer
+        - id: fix-encoding-pragma
+        - id: mixed-line-ending
+        - id: trailing-whitespace
+        - id: check-json
+
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      rev: v0.6.5
+      hooks:
+      - id: ruff-format
+      - id: ruff

aiida_hyperqueue-0.2.0/.readthedocs.yml

@@ -0,0 +1,26 @@
+version: 2
+
+# YAML adapted from aiida-core .readthedocs.yml
+
+# Disable all unneeded formats
+formats: []
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: '3.11'
+  jobs:
+    # Use uv to speed up the build
+    # https://docs.readthedocs.io/en/stable/build-customization.html#install-dependencies-with-uv
+    post_create_environment:
+    - asdf plugin add uv
+    - asdf install uv 0.2.9
+    - asdf global uv 0.2.9
+    post_install:
+    - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv pip install .[docs]
+
+# Let the build fail if there are any warnings
+sphinx:
+  builder: html
+  configuration: docs/source/conf.py
+  fail_on_warning: true

aiida_hyperqueue-0.2.0/CHANGELOG.md

@@ -0,0 +1,21 @@
+## v0.2.0
+
+- [#18](https://github.com/aiidateam/aiida-hyperqueue/pull/18) A large refactoring with following changes:
+
+    - correctly support memory setup for resources.
+    - support turn on hyperthreading with latest version of hyperqueue.
+    - Support install hq to remote computer over CLI.
+    - adding unit tests with submit to real hq using the fixture from hyperqueue repo.
+    - Fix the submit bug for hq > 0.12 that resources are configured twice in job script and submit command.
+
+- [#26](https://github.com/aiidateam/aiida-hyperqueue/pull/26) Change alloc add worker name (name appear in slurm queue) `aiida` -> `ahq`
+- [#24](https://github.com/aiidateam/aiida-hyperqueue/pull/24) Use JSON format as output for hq job list and hq submit
+
+
+## v0.1.1
+
+Only a small fix to the `README.md` was added here, this release is mainly to test the automated deployment to PyPI.
+
+## v0.1.0
+
+First release on PyPI, tested with HyperQueue v0.15.0.

{aiida-hyperqueue-0.1.0 → aiida_hyperqueue-0.2.0}/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021 The AiiDA Team.
+Copyright (c) 2023 The AiiDA Team.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

aiida_hyperqueue-0.2.0/PKG-INFO

@@ -0,0 +1,64 @@
+Metadata-Version: 2.4
+Name: aiida-hyperqueue
+Version: 0.2.0
+Summary: AiiDA plugin for the HyperQueue metascheduler
+Author: Marnik Bercx, Jusong Yu
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Classifier: Framework :: AiiDA
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python
+License-File: LICENSE
+Requires-Dist: aiida-core~=2.0
+Requires-Dist: sphinx ; extra == "docs"
+Requires-Dist: myst-parser ; extra == "docs"
+Requires-Dist: sphinx-design ; extra == "docs"
+Requires-Dist: sphinx-book-theme ; extra == "docs"
+Requires-Dist: sphinxcontrib-details-directive~=0.1.0 ; extra == "docs"
+Requires-Dist: pre-commit~=3.5 ; extra == "pre-commit"
+Requires-Dist: mypy~=1.10.0 ; extra == "pre-commit"
+Requires-Dist: pgtest~=1.3.1 ; extra == "tests"
+Requires-Dist: coverage~=7.0 ; extra == "tests"
+Requires-Dist: pytest~=7.0 ; extra == "tests"
+Requires-Dist: pytest-cov~=4.1 ; extra == "tests"
+Project-URL: Source, https://github.com/aiidateam/aiida-hyperqueue
+Provides-Extra: docs
+Provides-Extra: pre-commit
+Provides-Extra: tests
+
+[![Build Status](https://github.com/aiidateam/aiida-hyperqueue/workflows/ci/badge.svg)](https://github.com/aiidateam/aiida-hyperqueue/actions)
+[![Docs status](https://readthedocs.org/projects/aiida-hyperqueue/badge)](http://aiida-hyperqueue.readthedocs.io/)
+[![PyPI version](https://badge.fury.io/py/aiida-hyperqueue.svg)](https://badge.fury.io/py/aiida-hyperqueue)
+
+# AiiDA HyperQueue plugin
+
+AiiDA plugin for the [HyperQueue](https://it4innovations.github.io/hyperqueue/stable/) metascheduler.
+
+| ❗️ This package is still in the early stages of development and we will most likely break the API regularly in new 0.X versions. Be sure to pin the version when installing this package in scripts.|
+|---|
+
+## Features
+
+Allows task farming on Slurm machines through the submission of AiiDA calculations to the [HyperQueue](https://github.com/It4innovations/hyperqueue) metascheduler.
+See the [Documentation](http://aiida-hyperqueue.readthedocs.io/) for more information on how to install and use the plugin.
+
+## For developers
+
+To control the loglevel of command, since we use the `echo` module from aiida, the CLI loglever can be set through `logging.verdi_loglevel`.
+
+## Acknowledgenement
+If you use this plugin for your research, please cite the following work:
+
+#### HyperQueue
+
+* J. Beránek *et al.*, *HyperQueue: Efficient and ergonomic task graphs on HPC clusters*, SoftwareX **27**, 101814 (2024); DOI: [10.1016/j.softx.2024.101814](https://doi.org/10.1016/j.softx.2024.101814)
+
+#### AiiDA
+
+* S. P. Huber *et al.*, *AiiDA 1.0, a scalable computational infrastructure for automated reproducible workflows and data provenance*, Scientific Data **7**, 300 (2020); DOI: [10.1038/s41597-020-00638-4](https://doi.org/10.1038/s41597-020-00638-4)
+* M. Uhrin *et al.*, *Workflows in AiiDA: Engineering a high-throughput, event-based engine for robust and modular computational workflows*, Computational Materials Science **187**, 110086 (2021); DOI: [10.1016/j.commatsci.2020.110086](https://doi.org/10.1016/j.commatsci.2020.110086)
+

aiida_hyperqueue-0.2.0/README.md

@@ -0,0 +1,31 @@
+[![Build Status](https://github.com/aiidateam/aiida-hyperqueue/workflows/ci/badge.svg)](https://github.com/aiidateam/aiida-hyperqueue/actions)
+[![Docs status](https://readthedocs.org/projects/aiida-hyperqueue/badge)](http://aiida-hyperqueue.readthedocs.io/)
+[![PyPI version](https://badge.fury.io/py/aiida-hyperqueue.svg)](https://badge.fury.io/py/aiida-hyperqueue)
+
+# AiiDA HyperQueue plugin
+
+AiiDA plugin for the [HyperQueue](https://it4innovations.github.io/hyperqueue/stable/) metascheduler.
+
+| ❗️ This package is still in the early stages of development and we will most likely break the API regularly in new 0.X versions. Be sure to pin the version when installing this package in scripts.|
+|---|
+
+## Features
+
+Allows task farming on Slurm machines through the submission of AiiDA calculations to the [HyperQueue](https://github.com/It4innovations/hyperqueue) metascheduler.
+See the [Documentation](http://aiida-hyperqueue.readthedocs.io/) for more information on how to install and use the plugin.
+
+## For developers
+
+To control the loglevel of command, since we use the `echo` module from aiida, the CLI loglever can be set through `logging.verdi_loglevel`.
+
+## Acknowledgenement
+If you use this plugin for your research, please cite the following work:
+
+#### HyperQueue
+
+* J. Beránek *et al.*, *HyperQueue: Efficient and ergonomic task graphs on HPC clusters*, SoftwareX **27**, 101814 (2024); DOI: [10.1016/j.softx.2024.101814](https://doi.org/10.1016/j.softx.2024.101814)
+
+#### AiiDA
+
+* S. P. Huber *et al.*, *AiiDA 1.0, a scalable computational infrastructure for automated reproducible workflows and data provenance*, Scientific Data **7**, 300 (2020); DOI: [10.1038/s41597-020-00638-4](https://doi.org/10.1038/s41597-020-00638-4)
+* M. Uhrin *et al.*, *Workflows in AiiDA: Engineering a high-throughput, event-based engine for robust and modular computational workflows*, Computational Materials Science **187**, 110086 (2021); DOI: [10.1016/j.commatsci.2020.110086](https://doi.org/10.1016/j.commatsci.2020.110086)

{aiida-hyperqueue-0.1.0 → aiida_hyperqueue-0.2.0}/aiida_hyperqueue/__init__.py

@@ -5,4 +5,4 @@ aiida_hyperqueue
 AiiDA plugin for the HyperQueue metascheduler
 """
 
-__version__ = '0.1.0'
+__version__ = "0.2.0"

aiida_hyperqueue-0.2.0/aiida_hyperqueue/cli/__init__.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+from .root import cmd_root  # noqa: F401
+from .install import cmd_install  # noqa: F401
+from .server import cmd_info, cmd_start, cmd_stop  # noqa: F401
+from .alloc import cmd_list, cmd_add, cmd_remove  # noqa: F401

aiida_hyperqueue-0.2.0/aiida_hyperqueue/cli/alloc.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+import click
+
+from aiida.cmdline.params import options, arguments
+from aiida.cmdline.utils import echo
+
+from .root import cmd_root
+
+
+@cmd_root.group("alloc")
+def alloc_group():
+    """Commands to configure HQ allocations."""
+
+
+@alloc_group.command("add")
+@click.argument("slurm-options", nargs=-1)
+@options.COMPUTER(required=True)
+@click.option(
+    "-t",
+    "--time-limit",
+    type=str,
+    required=True,
+    help=(
+        "Time limit for each job run by the allocation. The duration can be expressed using various shortcuts "
+        "recognised by HyperQueue, e.g. 30m, 2h, ... For the full list, see https://tinyurl.com/hq-duration."
+    ),
+)
+@click.option(
+    "--hyper-threading/--no-hyper-threading",
+    default=True,
+    type=click.BOOL,
+    help=("Allow HyperQueue to consider hyperthreads when assigning resources."),
+)
+@click.option(
+    "-b",
+    "--backlog",
+    type=click.INT,
+    required=False,
+    default=1,
+    help=(
+        "Set the backlog for the allocator. This is the number of allocations HyperQueue will make sure is waiting with"
+        " the job manager."
+    ),
+)
+@click.option(
+    "-w",
+    "--workers-per-alloc",
+    type=click.INT,
+    required=False,
+    default=1,
+    help=("Option to allow pooled jobs to launch on multiple nodes."),
+)
+def cmd_add(
+    slurm_options, computer, time_limit, hyper_threading, backlog, workers_per_alloc
+):
+    """Add a new allocation to the HQ server."""
+
+    # from hq==0.13.0: ``--cpus=no-ht`` is now changed to a flag ``--no-hyper-threading``
+    hyper = "" if hyper_threading else "--no-hyper-threading"
+
+    with computer.get_transport() as transport:
+        retval, _, stderr = transport.exec_command_wait(
+            f'hq alloc add slurm --backlog {backlog} --time-limit {time_limit} --name ahq {hyper} '
+            f'--workers-per-alloc {workers_per_alloc} -- {" ".join(slurm_options)}'
+        )
+
+    if retval != 0:
+        echo.echo_critical(f"failed to create new allocation: {stderr}\n")
+
+    echo.echo_success(f"{stderr}")
+
+
+@alloc_group.command("list")
+@arguments.COMPUTER()
+def cmd_list(computer):
+    """List the allocations on the HQ server."""
+
+    with computer.get_transport() as transport:
+        retval, stdout, stderr = transport.exec_command_wait("hq alloc list")
+
+    if retval != 0:
+        echo.echo_critical(f"failed to list allocations: {stderr}\n")
+
+    echo.echo(stdout)
+
+
+@alloc_group.command("remove")
+@click.argument("alloc_id")
+@options.COMPUTER(required=True)
+def cmd_remove(alloc_id, computer):
+    """Remove an allocation from the HQ server."""
+
+    with computer.get_transport() as transport:
+        retval, _, stderr = transport.exec_command_wait(f"hq alloc remove {alloc_id}")
+
+    if retval != 0:
+        echo.echo_critical(f"failed to remove allocation: {stderr}\n")
+
+    echo.echo_success(f"{stderr}")

aiida_hyperqueue-0.2.0/aiida_hyperqueue/cli/install.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+import click
+import tempfile
+import requests
+import tarfile
+from pathlib import Path
+
+from aiida import orm
+from aiida.cmdline.utils import echo
+
+from .params import arguments
+from .root import cmd_root
+
+
+@cmd_root.command("install")
+@arguments.COMPUTER()
+@click.option(
+    "-p",
+    "--remote-bin-dir",
+    type=click.Path(),
+    default=Path("$HOME/bin/"),
+    help="remote bin path hq will stored.",
+)
+@click.option(
+    "--write-bashrc/--no-write-bashrc",
+    default=True,
+    help="write the bin path to bashrc.",
+)
+@click.option(
+    "--hq-version", type=str, default="0.19.0", help="the hq version will be installed."
+)
+# TODO: should also support different arch binary??
+def cmd_install(
+    computer: orm.Computer, remote_bin_dir: Path, hq_version: str, write_bashrc: bool
+):
+    """Install the hq binary to the computer through the transport"""
+
+    # The minimal hq version we support is 0.13.0, check the minor version
+    try:
+        _, minor, _ = hq_version.split(".")
+    except ValueError as e:
+        echo.echo_critical(f"Cannot parse the version {hq_version}: {e}")
+    else:
+        if int(minor) < 13:
+            # `--no-hyper-threading` replace `--cpus=no-ht` from 0.13.0
+            # If older version installed, try to not use `--no-hyper-threading` for `aiida-hq alloc add`.
+            echo.echo_warning(
+                f"You are installing hq version {hq_version}, please do not use `--no-hyper-threading` for `aiida-hq alloc add`."
+                " Or install version >= 0.13.0"
+            )
+
+    # Download the hq binary with specific version to local temp folder
+    # raise if the version not found
+    # Then upload to the remote using opened transport of computer
+    with tempfile.TemporaryDirectory() as temp_dir:
+        url = f"https://github.com/It4innovations/hyperqueue/releases/download/v{hq_version}/hq-v{hq_version}-linux-x64.tar.gz"
+        response = requests.get(url, stream=True)
+        rcode = response.status_code
+
+        if rcode != 200:
+            echo.echo_error(
+                "Cannot download the hq, please check the version is exist."
+            )
+
+        temp_dir = Path(temp_dir)
+        tar_path = temp_dir / "hq.tar.gz"
+
+        with open(tar_path, "wb") as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+        with tarfile.open(tar_path, "r") as tar:
+            tar.extractall(path=temp_dir)
+
+        echo.echo_success(f"The hq version {hq_version} binary downloaded.")
+
+        bin_path = temp_dir / "hq"
+
+        # upload the binary to remote
+        # TODO: try not override if the binary exist, put has overwrite=True as default
+        with computer.get_transport() as transport:
+            # Get the abs path of remote bin dir
+            retval, stdout, stderr = transport.exec_command_wait(
+                f"echo {str(remote_bin_dir)}"
+            )
+            if retval != 0:
+                echo.echo_critical(
+                    f"Not able to parse remote bin dir {remote_bin_dir}, exit_code={retval}"
+                )
+            else:
+                remote_bin_dir = Path(stdout.strip())
+
+            # first check if the hq exist in the target folder
+            if transport.isfile(str(remote_bin_dir / "hq")):
+                echo.echo_info(
+                    f"hq exist in the {remote_bin_dir} on remote, will override it."
+                )
+
+            transport.makedirs(path=remote_bin_dir, ignore_existing=True)
+            transport.put(
+                localpath=str(bin_path.resolve()), remotepath=str(remote_bin_dir)
+            )
+
+            # XXX: should transport.put take care of this already??
+            transport.exec_command_wait(f"chmod +x {str(remote_bin_dir / 'hq')}")
+
+            # write to bashrc
+            if write_bashrc:
+                identity_str = "by aiida-hq"
+                retval, _, stderr = transport.exec_command_wait(
+                    f"grep -q '# {identity_str}' ~/.bashrc || echo '# {identity_str}\nexport PATH=$HOME/bin:$PATH' >> ~/.bashrc"
+                )
+
+                if retval != 0:
+                    echo.echo_critical(
+                        f"Not able to set set the path $HOME/bin to your remote bashrc, try to do it manually.\n"
+                        f"Info: {stderr}"
+                    )
+
+    echo.echo_success(f"The hq binary installed into remote {remote_bin_dir}")

aiida_hyperqueue-0.2.0/aiida_hyperqueue/cli/params/arguments.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from aiida.cmdline.params import arguments as core_arguments
+
+COMPUTER = core_arguments.COMPUTER

aiida_hyperqueue-0.2.0/aiida_hyperqueue/cli/params/options.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+"""Reusable options for CLI commands."""
+
+import functools
+
+import click
+from aiida.cmdline.params import options as core_options
+from aiida.cmdline.params import types as core_types
+
+__all__ = (
+    "PROFILE",
+    "VERBOSITY",
+    "VERSION",
+)
+
+PROFILE = functools.partial(
+    core_options.PROFILE,
+    type=core_types.ProfileParamType(load_profile=True),
+    expose_value=False,
+)
+
+# Clone the ``VERBOSITY`` option from ``aiida-core`` so the ``-v`` short flag can be removed, since that overlaps with
+# the flag of the ``VERSION`` option of this CLI.
+VERBOSITY = core_options.VERBOSITY.clone()
+VERBOSITY.args = ("--verbosity",)
+
+VERSION = core_options.OverridableOption(
+    "-v",
+    "--version",
+    type=click.STRING,
+    required=False,
+    help="Select the version of the installed configuration.",
+)

aiida_hyperqueue-0.2.0/aiida_hyperqueue/cli/root.py

@@ -0,0 +1,37 @@
+# -*- coding: utf-8 -*-
+"""Command line interface `aiida-hq` for aiida-hyperqueue.
+The CLI implementation prototype from `aiida-pseudo`.
+"""
+
+import click
+
+from aiida.cmdline.groups.verdi import VerdiCommandGroup
+
+from .params import options
+
+
+class CustomVerdiCommandGroup(VerdiCommandGroup):
+    """Subclass of :class:`aiida.cmdline.groups.verdi.VerdiCommandGroup` for the CLI.
+
+    This subclass overrides the verbosity option to use a custom one that removes the ``-v`` short version of the option
+    since that is used by other options in this CLI and so would clash.
+    """
+
+    @staticmethod
+    def add_verbosity_option(cmd):
+        """Apply the ``verbosity`` option to the command, which is common to all subcommands."""
+        if cmd is not None and "verbosity" not in [param.name for param in cmd.params]:
+            cmd = options.VERBOSITY()(cmd)
+
+        return cmd
+
+
+@click.group(
+    "aiida-hq",
+    cls=CustomVerdiCommandGroup,
+    context_settings={"help_option_names": ["-h", "--help"]},
+)
+@options.VERBOSITY()
+@options.PROFILE()
+def cmd_root():
+    """CLI for the ``aiida-hyperqueue`` plugin."""