dpdispatcher 0.6.5__tar.gz → 0.6.7__tar.gz

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/.git_archival.txt

@@ -1,4 +1,3 @@
 node: $Format:%H$
 node-date: $Format:%cI$
 describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$
-ref-names: $Format:%D$

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
   # Python
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.4.4
+    rev: v0.6.2
     hooks:
       - id: ruff
         args: ["--fix"]
@@ -31,7 +31,7 @@ repos:
         args: ["--write"]
   # Python inside docs
   - repo: https://github.com/asottile/blacken-docs
-    rev: 1.16.0
+    rev: 1.18.0
     hooks:
       - id: blacken-docs
   # markdown, yaml

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.5
+Version: 0.6.7
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE
@@ -199,7 +199,7 @@ Requires-Dist: sphinx_rtd_theme>=1.0.0rc1; extra == "docs"
 Requires-Dist: numpydoc; extra == "docs"
 Requires-Dist: deepmodeling_sphinx>=0.1.1; extra == "docs"
 Requires-Dist: dargs>=0.3.1; extra == "docs"
-Requires-Dist: sphinx-argparse; extra == "docs"
+Requires-Dist: sphinx-argparse<0.5.0; extra == "docs"
 Provides-Extra: cloudserver
 Requires-Dist: oss2; extra == "cloudserver"
 Requires-Dist: tqdm; extra == "cloudserver"

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/pbs/start-pbs.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-docker-compose up -d --no-build
+docker compose up -d --no-build
 while [ `docker exec -u pbsuser pbs_master pbsnodes -a | grep "Mom = pbs_slave" | wc -l` -ne 2 ]
 do
     echo "Waiting for PBS slave nodes to become available";

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/pbs.sh

@@ -2,13 +2,14 @@
 set -e
 
 cd ./ci/pbs
-docker-compose pull
+docker compose pull
 ./start-pbs.sh
 cd -
 
 docker exec pbs_master /bin/bash -c "chmod -R 777 /shared_space"
 docker exec pbs_master /bin/bash -c "chown -R pbsuser:pbsuser /home/pbsuser"
 
+docker exec pbs_master /bin/bash -c "yum install -y procps"
 docker exec pbs_master /bin/bash -c "cd /dpdispatcher && pip install uv && uv pip install --system .[test] coverage && chown -R pbsuser ."
 docker exec -u pbsuser pbs_master /bin/bash -c "cd /dpdispatcher && coverage run --source=./dpdispatcher -m unittest -v && coverage report"
 docker exec -u pbsuser --env-file <(env | grep GITHUB) pbs_master /bin/bash -c "cd /dpdispatcher && curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov"

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/slurm/register_cluster.sh

@@ -2,4 +2,4 @@
 set -e
 
 docker exec slurmctld bash -c "/usr/bin/sacctmgr --immediate add cluster name=linux" && \
-docker-compose restart slurmdbd slurmctld
+docker compose restart slurmdbd slurmctld

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/slurm/start-slurm.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-docker-compose up -d --no-build
+docker compose up -d --no-build
 
 while [ `./register_cluster.sh 2>&1 | grep "sacctmgr: error" | wc -l` -ne 0 ]
   do

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/slurm.sh

@@ -2,9 +2,10 @@
 set -e
 
 cd ./ci/slurm
-docker-compose pull
+docker compose pull
 ./start-slurm.sh
 cd -
 
+docker exec slurmctld /bin/bash -c "yum install -y procps"
 docker exec slurmctld /bin/bash -c "cd dpdispatcher && pip install uv && uv pip install --system .[test] coverage && coverage run --source=./dpdispatcher -m unittest -v && coverage report"
 docker exec --env-file <(env | grep -e GITHUB -e CODECOV) slurmctld /bin/bash -c "cd dpdispatcher && curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov"

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/ssh/start-ssh.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-docker-compose up -d --no-build
+docker compose up -d --no-build
 
 docker exec server /bin/bash -c "ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N \"\" && cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && chmod 600 /root/.ssh/authorized_keys"
 docker exec server /bin/bash -c "mkdir -p /dpdispatcher_working"

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/ssh.sh

@@ -2,7 +2,7 @@
 set -e
 
 cd ./ci/ssh
-docker-compose pull
+docker compose pull
 ./start-ssh.sh
 cd -
 

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/ci/ssh_rsync.sh

@@ -2,7 +2,7 @@
 set -e
 
 cd ./ci/ssh
-docker-compose pull
+docker compose pull
 ./start-ssh.sh
 cd -
 

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/doc/conf.py

@@ -21,7 +21,7 @@ from datetime import date
 
 project = "DPDispatcher"
 copyright = "2020-%d, Deep Modeling" % date.today().year
-author = "Deep Modeling"
+author = "DeepModeling"
 
 
 # -- General configuration ---------------------------------------------------
@@ -97,3 +97,7 @@ intersphinx_mapping = {
     "python": ("https://docs.python.org/", None),
     "dargs": ("https://docs.deepmodeling.com/projects/dargs/en/latest/", None),
 }
+
+myst_enable_extensions = [
+    "colon_fence",
+]

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/doc/context.md

@@ -9,16 +9,17 @@ One needs to set {dargs:argument}`context_type <machine/context_type>` to one of
 
 `LazyLocal` directly runs jobs in the local server and local directory.
 
-Since [`bash -l`](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) is used in the shebang line of the submission scripts, the [login shell startup files](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) will be executed, potentially overriding the current environmental variables. Therefore, it's advisable to explicitly set the environmental variables using {dargs:argument}`envs <resources/envs>` or {dargs:argument}`source_list <resources/source_list>`.
+Since [`bash -l`](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) is used in the shebang line of the submission scripts, the [login shell startup files](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) will be executed, potentially overriding the current environment variables. Therefore, it's advisable to explicitly set the environment variables using {dargs:argument}`envs <resources/envs>` or {dargs:argument}`source_list <resources/source_list>`.
 
 ## Local
 
 {dargs:argument}`context_type <machine/context_type>`: `Local`
 
 `Local` runs jobs in the local server, but in a different directory.
-Files will be copied to the remote directory before jobs start and copied back after jobs finish.
+Files will be symlinked to the remote directory before jobs start and copied back after jobs finish.
+If the local directory is not accessible with the [batch system](./batch.md), turn off {dargs:argument}`symlink <machine[LocalContext]/remote_profile/symlink>`, and then files on the local directory will be copied to the remote directory.
 
-Since [`bash -l`](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) is used in the shebang line of the submission scripts, the [login shell startup files](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) will be executed, potentially overriding the current environmental variables. Therefore, it's advisable to explicitly set the environmental variables using {dargs:argument}`envs <resources/envs>` or {dargs:argument}`source_list <resources/source_list>`.
+Since [`bash -l`](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) is used in the shebang line of the submission scripts, the [login shell startup files](https://www.gnu.org/software/bash/manual/bash.html#Invoking-Bash) will be executed, potentially overriding the current environment variables. Therefore, it's advisable to explicitly set the environment variables using {dargs:argument}`envs <resources/envs>` or {dargs:argument}`source_list <resources/source_list>`.
 
 ## SSH
 

dpdispatcher-0.6.7/doc/env.md

@@ -0,0 +1,35 @@
+# Environment variables
+
+When launching a job, dpdispatcher sets the following environment variables according to the resources, in addition to user-defined environment variables:
+
+:::{envvar} DPDISPATCHER_NUMBER_NODE
+
+The number of nodes required for each job.
+
+:::
+
+:::{envvar} DPDISPATCHER_CPU_PER_NODE
+
+CPU numbers of each node assigned to each job.
+
+:::
+
+:::{envvar} DPDISPATCHER_GPU_PER_NODE
+
+GPU numbers of each node assigned to each job.
+
+:::
+
+:::{envvar} DPDISPATCHER_QUEUE_NAME
+
+The queue name of batch job scheduler system.
+
+:::
+
+:::{envvar} DPDISPATCHER_GROUP_SIZE
+
+The number of tasks in a job. 0 means infinity.
+
+:::
+
+These environment variables can be used in the {dargs:argument}`command <task/command>`, for example, `mpirun -n ${DPDISPATCHER_CPU_PER_NODE} xx.run`.

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/doc/index.rst

@@ -22,6 +22,7 @@ DPDispatcher will monitor (poke) until these jobs finish and download the result
    machine
    resources
    task
+   env
    run
    cli
    api/api

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.5'
-__version_tuple__ = version_tuple = (0, 6, 5)
+__version__ = version = '0.6.7'
+__version_tuple__ = version_tuple = (0, 6, 7)

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/base_context.py

@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod
-from typing import List, Tuple
+from typing import Any, List, Tuple
 
 from dargs import Argument
 
@@ -73,6 +73,66 @@ class BaseContext(metaclass=ABCMeta):
     def check_finish(self, proc):
         raise NotImplementedError("abstract method")
 
+    def block_checkcall(self, cmd, asynchronously=False) -> Tuple[Any, Any, Any]:
+        """Run command with arguments. Wait for command to complete.
+
+        Parameters
+        ----------
+        cmd : str
+            The command to run.
+        asynchronously : bool, optional, default=False
+            Run command asynchronously. If True, `nohup` will be used to run the command.
+
+        Returns
+        -------
+        stdin
+            standard inout
+        stdout
+            standard output
+        stderr
+            standard error
+
+        Raises
+        ------
+        RuntimeError
+            when the return code is not zero
+        """
+        if asynchronously:
+            cmd = f"nohup {cmd} >/dev/null &"
+        exit_status, stdin, stdout, stderr = self.block_call(cmd)
+        if exit_status != 0:
+            raise RuntimeError(
+                "Get error code %d in calling %s with job: %s . message: %s"
+                % (
+                    exit_status,
+                    cmd,
+                    self.submission.submission_hash,
+                    stderr.read().decode("utf-8"),
+                )
+            )
+        return stdin, stdout, stderr
+
+    @abstractmethod
+    def block_call(self, cmd) -> Tuple[int, Any, Any, Any]:
+        """Run command with arguments. Wait for command to complete.
+
+        Parameters
+        ----------
+        cmd : str
+            The command to run.
+
+        Returns
+        -------
+        exit_status
+            exit code
+        stdin
+            standard inout
+        stdout
+            standard output
+        stderr
+            standard error
+        """
+
     @classmethod
     def machine_arginfo(cls) -> Argument:
         """Generate the machine arginfo.

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/contexts/dp_cloud_server_context.py

@@ -335,6 +335,11 @@ class BohriumContext(BaseContext):
             )
         ]
 
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )
+
 
 DpCloudServerContext = BohriumContext
 LebesgueContext = BohriumContext

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/contexts/hdfs_context.py

@@ -244,3 +244,8 @@ class HDFSContext(BaseContext):
 
     def read_file(self, fname):
         return HDFS.read_hdfs_file(os.path.join(self.remote_root, fname))
+
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/contexts/lazy_local_context.py

@@ -112,23 +112,6 @@ class LazyLocalContext(BaseContext):
     #                else:
     #                    raise RuntimeError('do not find download file ' + fname)
 
-    def block_checkcall(self, cmd):
-        # script_dir = os.path.join(self.local_root, self.submission.work_base)
-        # os.chdir(script_dir)
-        proc = sp.Popen(
-            cmd, cwd=self.local_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
-        )
-        o, e = proc.communicate()
-        stdout = SPRetObj(o)
-        stderr = SPRetObj(e)
-        code = proc.returncode
-        if code != 0:
-            raise RuntimeError(
-                "Get error code %d in locally calling %s with job: %s ",
-                (code, cmd, self.submission.submission_hash),
-            )
-        return None, stdout, stderr
-
     def block_call(self, cmd):
         proc = sp.Popen(
             cmd, cwd=self.local_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/contexts/local_context.py

@@ -3,6 +3,9 @@ import shutil
 import subprocess as sp
 from glob import glob
 from subprocess import TimeoutExpired
+from typing import List
+
+from dargs import Argument
 
 from dpdispatcher.base_context import BaseContext
 from dpdispatcher.dlog import dlog
@@ -60,6 +63,7 @@ class LocalContext(BaseContext):
         self.temp_local_root = os.path.abspath(local_root)
         self.temp_remote_root = os.path.abspath(remote_root)
         self.remote_profile = remote_profile
+        self.symlink = remote_profile.get("symlink", True)
 
     @classmethod
     def load_from_dict(cls, context_dict):
@@ -83,6 +87,25 @@ class LocalContext(BaseContext):
             self.temp_remote_root, submission.submission_hash
         )
 
+    def _copy_from_local_to_remote(self, local_path, remote_path):
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(
+                f"cannot find uploaded file {os.path.join(local_path)}"
+            )
+        if os.path.exists(remote_path):
+            os.remove(remote_path)
+        _check_file_path(remote_path)
+
+        if self.symlink:
+            # ensure the file exist
+            os.symlink(local_path, remote_path)
+        elif os.path.isfile(local_path):
+            shutil.copyfile(local_path, remote_path)
+        elif os.path.isdir(local_path):
+            shutil.copytree(local_path, remote_path)
+        else:
+            raise ValueError(f"Unknown file type: {local_path}")
+
     def upload(self, submission):
         os.makedirs(self.remote_root, exist_ok=True)
         for ii in submission.belonging_tasks:
@@ -103,14 +126,9 @@ class LocalContext(BaseContext):
                 file_list.extend(rel_file_list)
 
             for jj in file_list:
-                if not os.path.exists(os.path.join(local_job, jj)):
-                    raise FileNotFoundError(
-                        "cannot find upload file " + os.path.join(local_job, jj)
-                    )
-                if os.path.exists(os.path.join(remote_job, jj)):
-                    os.remove(os.path.join(remote_job, jj))
-                _check_file_path(os.path.join(remote_job, jj))
-                os.symlink(os.path.join(local_job, jj), os.path.join(remote_job, jj))
+                self._copy_from_local_to_remote(
+                    os.path.join(local_job, jj), os.path.join(remote_job, jj)
+                )
 
         local_job = self.local_root
         remote_job = self.remote_root
@@ -128,14 +146,9 @@ class LocalContext(BaseContext):
             file_list.extend(rel_file_list)
 
         for jj in file_list:
-            if not os.path.exists(os.path.join(local_job, jj)):
-                raise FileNotFoundError(
-                    "cannot find upload file " + os.path.join(local_job, jj)
-                )
-            if os.path.exists(os.path.join(remote_job, jj)):
-                os.remove(os.path.join(remote_job, jj))
-            _check_file_path(os.path.join(remote_job, jj))
-            os.symlink(os.path.join(local_job, jj), os.path.join(remote_job, jj))
+            self._copy_from_local_to_remote(
+                os.path.join(local_job, jj), os.path.join(remote_job, jj)
+            )
 
     def download(
         self, submission, check_exists=False, mark_failure=True, back_error=False
@@ -288,21 +301,6 @@ class LocalContext(BaseContext):
                 # no nothing in the case of linked files
                 pass
 
-    def block_checkcall(self, cmd):
-        proc = sp.Popen(
-            cmd, cwd=self.remote_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
-        )
-        o, e = proc.communicate()
-        stdout = SPRetObj(o)
-        stderr = SPRetObj(e)
-        code = proc.returncode
-        if code != 0:
-            raise RuntimeError(
-                f"Get error code {code} in locally calling {cmd} with job: {self.submission.submission_hash}"
-                f"\nStandard error: {stderr}"
-            )
-        return None, stdout, stderr
-
     def block_call(self, cmd):
         proc = sp.Popen(
             cmd, cwd=self.remote_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
@@ -351,3 +349,31 @@ class LocalContext(BaseContext):
                 stdout = None
                 stderr = None
         return ret, stdout, stderr
+
+    @classmethod
+    def machine_subfields(cls) -> List[Argument]:
+        """Generate the machine subfields.
+
+        Returns
+        -------
+        list[Argument]
+            machine subfields
+        """
+        doc_remote_profile = "The information used to maintain the local machine."
+        return [
+            Argument(
+                "remote_profile",
+                dict,
+                optional=True,
+                doc=doc_remote_profile,
+                sub_fields=[
+                    Argument(
+                        "symlink",
+                        bool,
+                        optional=True,
+                        default=True,
+                        doc="Whether to use symbolic links to replace copy. This option should be turned off if the local directory is not accessible on the Batch system.",
+                    ),
+                ],
+            )
+        ]

{dpdispatcher-0.6.5 → dpdispatcher-0.6.7}/dpdispatcher/contexts/openapi_context.py

@@ -258,3 +258,8 @@ class OpenAPIContext(BaseContext):
             dir_to_be_removed = os.path.join(local_root, "backup")
             if os.path.exists(dir_to_be_removed):
                 shutil.rmtree(dir_to_be_removed)
+
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )