dpdispatcher 0.6.6__py3-none-any.whl → 0.6.8__py3-none-any.whl

dpdispatcher/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.6'
-__version_tuple__ = version_tuple = (0, 6, 6)
+__version__ = version = '0.6.8'
+__version_tuple__ = version_tuple = (0, 6, 8)

dpdispatcher/base_context.py

@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod
-from typing import List, Tuple
+from typing import Any, List, Tuple
 
 from dargs import Argument
 
@@ -73,6 +73,65 @@ class BaseContext(metaclass=ABCMeta):
     def check_finish(self, proc):
         raise NotImplementedError("abstract method")
 
+    def block_checkcall(self, cmd, asynchronously=False) -> Tuple[Any, Any, Any]:
+        """Run command with arguments. Wait for command to complete.
+
+        Parameters
+        ----------
+        cmd : str
+            The command to run.
+        asynchronously : bool, optional, default=False
+            Run command asynchronously. If True, `nohup` will be used to run the command.
+
+        Returns
+        -------
+        stdin
+            standard inout
+        stdout
+            standard output
+        stderr
+            standard error
+
+        Raises
+        ------
+        RuntimeError
+            when the return code is not zero
+        """
+        if asynchronously:
+            cmd = f"nohup {cmd} >/dev/null &"
+        exit_status, stdin, stdout, stderr = self.block_call(cmd)
+        if exit_status != 0:
+            raise RuntimeError(
+                "Get error code {} in calling {} with job: {} . message: {}".format(
+                    exit_status,
+                    cmd,
+                    self.submission.submission_hash,
+                    stderr.read().decode("utf-8"),
+                )
+            )
+        return stdin, stdout, stderr
+
+    @abstractmethod
+    def block_call(self, cmd) -> Tuple[int, Any, Any, Any]:
+        """Run command with arguments. Wait for command to complete.
+
+        Parameters
+        ----------
+        cmd : str
+            The command to run.
+
+        Returns
+        -------
+        exit_status
+            exit code
+        stdin
+            standard inout
+        stdout
+            standard output
+        stderr
+            standard error
+        """
+
     @classmethod
     def machine_arginfo(cls) -> Argument:
         """Generate the machine arginfo.

dpdispatcher/contexts/dp_cloud_server_context.py

@@ -335,6 +335,11 @@ class BohriumContext(BaseContext):
             )
         ]
 
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )
+
 
 DpCloudServerContext = BohriumContext
 LebesgueContext = BohriumContext

dpdispatcher/contexts/hdfs_context.py

@@ -244,3 +244,8 @@ class HDFSContext(BaseContext):
 
     def read_file(self, fname):
         return HDFS.read_hdfs_file(os.path.join(self.remote_root, fname))
+
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )

dpdispatcher/contexts/lazy_local_context.py

@@ -112,23 +112,6 @@ class LazyLocalContext(BaseContext):
     #                else:
     #                    raise RuntimeError('do not find download file ' + fname)
 
-    def block_checkcall(self, cmd):
-        # script_dir = os.path.join(self.local_root, self.submission.work_base)
-        # os.chdir(script_dir)
-        proc = sp.Popen(
-            cmd, cwd=self.local_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
-        )
-        o, e = proc.communicate()
-        stdout = SPRetObj(o)
-        stderr = SPRetObj(e)
-        code = proc.returncode
-        if code != 0:
-            raise RuntimeError(
-                "Get error code %d in locally calling %s with job: %s ",
-                (code, cmd, self.submission.submission_hash),
-            )
-        return None, stdout, stderr
-
     def block_call(self, cmd):
         proc = sp.Popen(
             cmd, cwd=self.local_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE

dpdispatcher/contexts/local_context.py

@@ -3,6 +3,9 @@ import shutil
 import subprocess as sp
 from glob import glob
 from subprocess import TimeoutExpired
+from typing import List
+
+from dargs import Argument
 
 from dpdispatcher.base_context import BaseContext
 from dpdispatcher.dlog import dlog
@@ -60,6 +63,7 @@ class LocalContext(BaseContext):
         self.temp_local_root = os.path.abspath(local_root)
         self.temp_remote_root = os.path.abspath(remote_root)
         self.remote_profile = remote_profile
+        self.symlink = remote_profile.get("symlink", True)
 
     @classmethod
     def load_from_dict(cls, context_dict):
@@ -83,6 +87,25 @@ class LocalContext(BaseContext):
             self.temp_remote_root, submission.submission_hash
         )
 
+    def _copy_from_local_to_remote(self, local_path, remote_path):
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(
+                f"cannot find uploaded file {os.path.join(local_path)}"
+            )
+        if os.path.exists(remote_path):
+            os.remove(remote_path)
+        _check_file_path(remote_path)
+
+        if self.symlink:
+            # ensure the file exist
+            os.symlink(local_path, remote_path)
+        elif os.path.isfile(local_path):
+            shutil.copyfile(local_path, remote_path)
+        elif os.path.isdir(local_path):
+            shutil.copytree(local_path, remote_path)
+        else:
+            raise ValueError(f"Unknown file type: {local_path}")
+
     def upload(self, submission):
         os.makedirs(self.remote_root, exist_ok=True)
         for ii in submission.belonging_tasks:
@@ -103,14 +126,9 @@ class LocalContext(BaseContext):
                 file_list.extend(rel_file_list)
 
             for jj in file_list:
-                if not os.path.exists(os.path.join(local_job, jj)):
-                    raise FileNotFoundError(
-                        "cannot find upload file " + os.path.join(local_job, jj)
-                    )
-                if os.path.exists(os.path.join(remote_job, jj)):
-                    os.remove(os.path.join(remote_job, jj))
-                _check_file_path(os.path.join(remote_job, jj))
-                os.symlink(os.path.join(local_job, jj), os.path.join(remote_job, jj))
+                self._copy_from_local_to_remote(
+                    os.path.join(local_job, jj), os.path.join(remote_job, jj)
+                )
 
         local_job = self.local_root
         remote_job = self.remote_root
@@ -128,14 +146,9 @@ class LocalContext(BaseContext):
             file_list.extend(rel_file_list)
 
         for jj in file_list:
-            if not os.path.exists(os.path.join(local_job, jj)):
-                raise FileNotFoundError(
-                    "cannot find upload file " + os.path.join(local_job, jj)
-                )
-            if os.path.exists(os.path.join(remote_job, jj)):
-                os.remove(os.path.join(remote_job, jj))
-            _check_file_path(os.path.join(remote_job, jj))
-            os.symlink(os.path.join(local_job, jj), os.path.join(remote_job, jj))
+            self._copy_from_local_to_remote(
+                os.path.join(local_job, jj), os.path.join(remote_job, jj)
+            )
 
     def download(
         self, submission, check_exists=False, mark_failure=True, back_error=False
@@ -288,21 +301,6 @@ class LocalContext(BaseContext):
                 # no nothing in the case of linked files
                 pass
 
-    def block_checkcall(self, cmd):
-        proc = sp.Popen(
-            cmd, cwd=self.remote_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
-        )
-        o, e = proc.communicate()
-        stdout = SPRetObj(o)
-        stderr = SPRetObj(e)
-        code = proc.returncode
-        if code != 0:
-            raise RuntimeError(
-                f"Get error code {code} in locally calling {cmd} with job: {self.submission.submission_hash}"
-                f"\nStandard error: {stderr}"
-            )
-        return None, stdout, stderr
-
     def block_call(self, cmd):
         proc = sp.Popen(
             cmd, cwd=self.remote_root, shell=True, stdout=sp.PIPE, stderr=sp.PIPE
@@ -351,3 +349,31 @@ class LocalContext(BaseContext):
                 stdout = None
                 stderr = None
         return ret, stdout, stderr
+
+    @classmethod
+    def machine_subfields(cls) -> List[Argument]:
+        """Generate the machine subfields.
+
+        Returns
+        -------
+        list[Argument]
+            machine subfields
+        """
+        doc_remote_profile = "The information used to maintain the local machine."
+        return [
+            Argument(
+                "remote_profile",
+                dict,
+                optional=True,
+                doc=doc_remote_profile,
+                sub_fields=[
+                    Argument(
+                        "symlink",
+                        bool,
+                        optional=True,
+                        default=True,
+                        doc="Whether to use symbolic links to replace copy. This option should be turned off if the local directory is not accessible on the Batch system.",
+                    ),
+                ],
+            )
+        ]

dpdispatcher/contexts/openapi_context.py

@@ -258,3 +258,8 @@ class OpenAPIContext(BaseContext):
             dir_to_be_removed = os.path.join(local_root, "backup")
             if os.path.exists(dir_to_be_removed):
                 shutil.rmtree(dir_to_be_removed)
+
+    def block_call(self, cmd):
+        raise RuntimeError(
+            "Unsupported method. You may use an unsupported combination of the machine and the context."
+        )

dpdispatcher/contexts/ssh_context.py

@@ -44,6 +44,7 @@ class SSHSession:
         totp_secret=None,
         tar_compress=True,
         look_for_keys=True,
+        execute_command=None,
     ):
         self.hostname = hostname
         self.username = username
@@ -56,6 +57,7 @@ class SSHSession:
         self.ssh = None
         self.tar_compress = tar_compress
         self.look_for_keys = look_for_keys
+        self.execute_command = execute_command
         self._keyboard_interactive_auth = False
         self._setup_ssh()
 
@@ -88,8 +90,7 @@ class SSHSession:
         while not self._check_alive():
             if count == max_check:
                 raise RuntimeError(
-                    "cannot connect ssh after %d failures at interval %d s"
-                    % (max_check, sleep_time)
+                    f"cannot connect ssh after {max_check} failures at interval {sleep_time} s"
                 )
             dlog.info("connection check failed, try to reconnect to " + self.hostname)
             self._setup_ssh()
@@ -237,6 +238,8 @@ class SSHSession:
         self.ssh._transport = ts  # type: ignore
         # reset sftp
         self._sftp = None
+        if self.execute_command is not None:
+            self.exec_command(self.execute_command)
 
     def inter_handler(self, title, instructions, prompt_list):
         """inter_handler: the callback for paramiko.transport.auth_interactive.
@@ -338,6 +341,7 @@ class SSHSession:
         doc_look_for_keys = (
             "enable searching for discoverable private key files in ~/.ssh/"
         )
+        doc_execute_command = "execute command after ssh connection is established."
         ssh_remote_profile_args = [
             Argument("hostname", str, optional=False, doc=doc_hostname),
             Argument("username", str, optional=False, doc=doc_username),
@@ -379,6 +383,13 @@ class SSHSession:
                 default=True,
                 doc=doc_look_for_keys,
             ),
+            Argument(
+                "execute_command",
+                str,
+                optional=True,
+                default=None,
+                doc=doc_execute_command,
+            ),
         ]
         ssh_remote_profile_format = Argument(
             "ssh_session", dict, ssh_remote_profile_args
@@ -438,7 +449,9 @@ class SSHContext(BaseContext):
         self.init_local_root = local_root
         self.init_remote_root = remote_root
         self.temp_local_root = os.path.abspath(local_root)
-        assert os.path.isabs(remote_root), "remote_root must be a abspath"
+        assert os.path.isabs(os.path.realpath(remote_root)), (
+            "remote_root must be a abspath"
+        )
         self.temp_remote_root = remote_root
         self.remote_profile = remote_profile
         self.remote_root = None
@@ -755,41 +768,6 @@ class SSHContext(BaseContext):
                 tar_compress=self.remote_profile.get("tar_compress", None),
             )
 
-    def block_checkcall(self, cmd, asynchronously=False, stderr_whitelist=None):
-        """Run command with arguments. Wait for command to complete. If the return code
-        was zero then return, otherwise raise RuntimeError.
-
-        Parameters
-        ----------
-        cmd : str
-            The command to run.
-        asynchronously : bool, optional, default=False
-            Run command asynchronously. If True, `nohup` will be used to run the command.
-        stderr_whitelist : list of str, optional, default=None
-            If not None, the stderr will be checked against the whitelist. If the stderr
-            contains any of the strings in the whitelist, the command will be considered
-            successful.
-        """
-        assert self.remote_root is not None
-        self.ssh_session.ensure_alive()
-        if asynchronously:
-            cmd = f"nohup {cmd} >/dev/null &"
-        stdin, stdout, stderr = self.ssh_session.exec_command(
-            (f"cd {shlex.quote(self.remote_root)} ;") + cmd
-        )
-        exit_status = stdout.channel.recv_exit_status()
-        if exit_status != 0:
-            raise RuntimeError(
-                "Get error code %d in calling %s through ssh with job: %s . message: %s"
-                % (
-                    exit_status,
-                    cmd,
-                    self.submission.submission_hash,
-                    stderr.read().decode("utf-8"),
-                )
-            )
-        return stdin, stdout, stderr
-
     def block_call(self, cmd):
         assert self.remote_root is not None
         self.ssh_session.ensure_alive()

dpdispatcher/machine.py

@@ -161,6 +161,9 @@ class Machine(metaclass=ABCMeta):
             machine_dict["remote_profile"] = self.context.remote_profile
         else:
             machine_dict["remote_profile"] = {}
+        # normalize the dict
+        base = self.arginfo()
+        machine_dict = base.normalize_value(machine_dict, trim_pattern="_*")
         return machine_dict
 
     def __eq__(self, other):
@@ -265,6 +268,15 @@ class Machine(metaclass=ABCMeta):
 
         export_envs_part = ""
         envs = job.resources.envs
+        envs = {
+            # export resources information to the environment variables
+            "DPDISPATCHER_NUMBER_NODE": job.resources.number_node,
+            "DPDISPATCHER_CPU_PER_NODE": job.resources.cpu_per_node,
+            "DPDISPATCHER_GPU_PER_NODE": job.resources.gpu_per_node,
+            "DPDISPATCHER_QUEUE_NAME": job.resources.queue_name,
+            "DPDISPATCHER_GROUP_SIZE": job.resources.group_size,
+            **envs,
+        }
         for k, v in envs.items():
             if isinstance(v, list):
                 for each_value in v:

dpdispatcher/machines/JH_UniScheduler.py

@@ -39,7 +39,7 @@ class JH_UniScheduler(Machine):
         custom_gpu_line = resources.kwargs.get("custom_gpu_line", None)
         if not custom_gpu_line:
             script_header_dict["JH_UniScheduler_number_gpu_line"] = (
-                "" f"#JSUB -gpgpu {resources.gpu_per_node}"
+                f"#JSUB -gpgpu {resources.gpu_per_node}"
             )
         else:
             script_header_dict["JH_UniScheduler_number_gpu_line"] = custom_gpu_line
@@ -105,8 +105,7 @@ class JH_UniScheduler(Machine):
         elif ret != 0:
             # just retry when any unknown error raised.
             raise RetrySignal(
-                "Get error code %d in checking status through ssh with job: %s . message: %s"
-                % (ret, job.job_hash, err_str)
+                f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
             )
         status_out = stdout.read().decode("utf-8").split("\n")
         if len(status_out) < 2:

dpdispatcher/machines/distributed_shell.py

@@ -181,8 +181,7 @@ class DistributedShell(Machine):
         if ret != 0:
             err_str = stderr.decode("utf-8")
             raise RuntimeError(
-                "Command squeue fails to execute, error message:%s\nreturn code %d\n"
-                % (err_str, ret)
+                f"Command {cmd} fails to execute, error message:{err_str}\nreturn code {ret}\n"
             )
         job_id = int(stdout.decode("utf-8").strip())
 
@@ -200,8 +199,7 @@ class DistributedShell(Machine):
         if ret != 0:
             err_str = stderr.decode("utf-8")
             raise RuntimeError(
-                "Command fails to execute, error message:%s\nreturn code %d\n"
-                % (err_str, ret)
+                f"Command fails to execute, error message:{err_str}\nreturn code {ret}\n"
             )
 
         if_job_exists = bool(stdout.decode("utf-8").strip())

dpdispatcher/machines/lsf.py

@@ -129,8 +129,7 @@ class LSF(Machine):
         elif ret != 0:
             # just retry when any unknown error raised.
             raise RetrySignal(
-                "Get error code %d in checking status through ssh with job: %s . message: %s"
-                % (ret, job.job_hash, err_str)
+                f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
             )
         status_out = stdout.read().decode("utf-8").split("\n")
         if len(status_out) < 2:

dpdispatcher/machines/pbs.py

@@ -76,7 +76,8 @@ class PBS(Machine):
         job_id = job.job_id
         if job_id == "":
             return JobStatus.unsubmitted
-        ret, stdin, stdout, stderr = self.context.block_call("qstat -x " + job_id)
+        command = "qstat -x " + job_id
+        ret, stdin, stdout, stderr = self.context.block_call(command)
         err_str = stderr.read().decode("utf-8")
         if ret != 0:
             if "qstat: Unknown Job Id" in err_str or "Job has finished" in err_str:
@@ -86,8 +87,7 @@ class PBS(Machine):
                     return JobStatus.terminated
             else:
                 raise RuntimeError(
-                    "status command qstat fails to execute. erro info: %s return code %d"
-                    % (err_str, ret)
+                    f"status command {command} fails to execute. erro info: {err_str} return code {ret}"
                 )
         status_line = stdout.read().decode("utf-8").split("\n")[-2]
         status_word = status_line.split()[-2]
@@ -126,7 +126,8 @@ class Torque(PBS):
         job_id = job.job_id
         if job_id == "":
             return JobStatus.unsubmitted
-        ret, stdin, stdout, stderr = self.context.block_call("qstat -l " + job_id)
+        command = "qstat -l " + job_id
+        ret, stdin, stdout, stderr = self.context.block_call(command)
         err_str = stderr.read().decode("utf-8")
         if ret != 0:
             if "qstat: Unknown Job Id" in err_str or "Job has finished" in err_str:
@@ -136,8 +137,7 @@ class Torque(PBS):
                     return JobStatus.terminated
             else:
                 raise RuntimeError(
-                    "status command qstat fails to execute. erro info: %s return code %d"
-                    % (err_str, ret)
+                    f"status command {command} fails to execute. erro info: {err_str} return code {ret}"
                 )
         status_line = stdout.read().decode("utf-8").split("\n")[-2]
         status_word = status_line.split()[-2]
@@ -259,15 +259,17 @@ class SGE(PBS):
         pass
 
     def check_status(self, job):
+        ### https://softpanorama.org/HPC/Grid_engine/Queues/queue_states.shtml
         job_id = job.job_id
         status_line = None
         if job_id == "":
             return JobStatus.unsubmitted
-        ret, stdin, stdout, stderr = self.context.block_call("qstat")
+        command = "qstat"
+        ret, stdin, stdout, stderr = self.context.block_call(command)
         err_str = stderr.read().decode("utf-8")
         if ret != 0:
             raise RuntimeError(
-                f"status command qstat fails to execute. erro info: {err_str} return code {ret}"
+                f"status command {command} fails to execute. erro info: {err_str} return code {ret}"
             )
         status_text_list = stdout.read().decode("utf-8").split("\n")
         for txt in status_text_list:
@@ -291,10 +293,12 @@ class SGE(PBS):
         else:
             status_word = status_line.split()[4]
             # dlog.info (status_word)
-            if status_word in ["qw"]:
+            if status_word in ["qw", "hqw", "t"]:
                 return JobStatus.waiting
-            elif status_word in ["r"]:
+            elif status_word in ["r", "Rr"]:
                 return JobStatus.running
+            elif status_word in ["Eqw", "dr", "dt"]:
+                return JobStatus.terminated
             else:
                 return JobStatus.unknown
 

dpdispatcher/machines/shell.py

@@ -38,14 +38,12 @@ class Shell(Machine):
         script_run_str = self.gen_script_command(job)
         script_run_file_name = f"{job.script_file_name}.run"
         self.context.write_file(fname=script_run_file_name, write_str=script_run_str)
-        ret, stdin, stdout, stderr = self.context.block_call(
-            f"cd {shlex.quote(self.context.remote_root)} && {{ nohup bash {script_file_name} 1>>{output_name} 2>>{output_name} & }} && echo $!"
-        )
+        cmd = f"cd {shlex.quote(self.context.remote_root)} && {{ nohup bash {script_file_name} 1>>{output_name} 2>>{output_name} & }} && echo $!"
+        ret, stdin, stdout, stderr = self.context.block_call(cmd)
         if ret != 0:
             err_str = stderr.read().decode("utf-8")
             raise RuntimeError(
-                "status command squeue fails to execute\nerror message:%s\nreturn code %d\n"
-                % (err_str, ret)
+                f"status command {cmd} fails to execute\nerror message:{err_str}\nreturn code {ret}\n"
             )
         job_id = int(stdout.read().decode("utf-8").strip())
         self.context.write_file(job_id_name, str(job_id))
@@ -73,14 +71,15 @@ class Shell(Machine):
             return JobStatus.unsubmitted
 
         # mark defunct process as terminated
-        ret, stdin, stdout, stderr = self.context.block_call(
+        cmd = (
+            r"""command -v ps >/dev/null 2>&1 || { echo >&2 "I require ps but it's not installed. Aborting."; exit 1; };"""
             f"if ps -p {job_id} > /dev/null && ! (ps -o command -p {job_id} | grep defunct >/dev/null) ; then echo 1; fi"
         )
+        ret, stdin, stdout, stderr = self.context.block_call(cmd)
         if ret != 0:
             err_str = stderr.read().decode("utf-8")
             raise RuntimeError(
-                "status command squeue fails to execute\nerror message:%s\nreturn code %d\n"
-                % (err_str, ret)
+                f"status command {cmd} fails to execute\nerror message:{err_str}\nreturn code {ret}\n"
             )
 
         if_job_exists = bool(stdout.read().decode("utf-8").strip())

dpdispatcher/machines/slurm.py

@@ -83,13 +83,12 @@ class Slurm(Machine):
         script_run_file_name = f"{job.script_file_name}.run"
         self.context.write_file(fname=script_run_file_name, write_str=script_run_str)
         # self.context.write_file(fname=os.path.join(self.context.submission.work_base, script_file_name), write_str=script_str)
-        ret, stdin, stdout, stderr = self.context.block_call(
-            "cd {} && {} {}".format(
-                shlex.quote(self.context.remote_root),
-                "sbatch",
-                shlex.quote(script_file_name),
-            )
+        command = "cd {} && {} {}".format(
+            shlex.quote(self.context.remote_root),
+            "sbatch",
+            shlex.quote(script_file_name),
         )
+        ret, stdin, stdout, stderr = self.context.block_call(command)
         if ret != 0:
             err_str = stderr.read().decode("utf-8")
             if (
@@ -98,8 +97,7 @@ class Slurm(Machine):
             ):
                 # server network error, retry 3 times
                 raise RetrySignal(
-                    "Get error code %d in submitting through ssh with job: %s . message: %s"
-                    % (ret, job.job_hash, err_str)
+                    f"Get error code {ret} in submitting with job: {job.job_hash} . message: {err_str}"
                 )
             elif (
                 "Job violates accounting/QOS policy" in err_str
@@ -110,8 +108,7 @@ class Slurm(Machine):
                 # job number exceeds, skip the submitting
                 return ""
             raise RuntimeError(
-                "status command squeue fails to execute\nerror message:%s\nreturn code %d\n"
-                % (err_str, ret)
+                f"command {command} fails to execute\nerror message:{err_str}\nreturn code {ret}\n"
             )
         subret = stdout.readlines()
         # --parsable
@@ -129,9 +126,8 @@ class Slurm(Machine):
         job_id = job.job_id
         if job_id == "":
             return JobStatus.unsubmitted
-        ret, stdin, stdout, stderr = self.context.block_call(
-            'squeue -o "%.18i %.2t" -j ' + job_id
-        )
+        command = 'squeue -o "%.18i %.2t" -j ' + job_id
+        ret, stdin, stdout, stderr = self.context.block_call(command)
         if ret != 0:
             err_str = stderr.read().decode("utf-8")
             if "Invalid job id specified" in err_str:
@@ -147,13 +143,11 @@ class Slurm(Machine):
             ):
                 # retry 3 times
                 raise RetrySignal(
-                    "Get error code %d in checking status through ssh with job: %s . message: %s"
-                    % (ret, job.job_hash, err_str)
+                    f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
                 )
             raise RuntimeError(
-                "status command squeue fails to execute."
-                "job_id:%s \n error message:%s\n return code %d\n"
-                % (job_id, err_str, ret)
+                f"status command {command} fails to execute."
+                f"job_id:{job_id} \n error message:{err_str}\n return code {ret}\n"
             )
         status_line = stdout.read().decode("utf-8").split("\n")[-2]
         status_word = status_line.split()[-1]
@@ -257,7 +251,7 @@ class SlurmJobArray(Slurm):
             return super().gen_script_header(job) + "\n#SBATCH --array={}".format(
                 ",".join(map(str, job_array))
             )
-        return super().gen_script_header(job) + "\n#SBATCH --array=0-%d" % (
+        return super().gen_script_header(job) + "\n#SBATCH --array=0-%s" % (
             math.ceil(len(job.job_task_list) / slurm_job_size) - 1
         )
 
@@ -319,9 +313,8 @@ class SlurmJobArray(Slurm):
         job_id = job.job_id
         if job_id == "":
             return JobStatus.unsubmitted
-        ret, stdin, stdout, stderr = self.context.block_call(
-            'squeue -h -o "%.18i %.2t" -j ' + job_id
-        )
+        command = 'squeue -h -o "%.18i %.2t" -j ' + job_id
+        ret, stdin, stdout, stderr = self.context.block_call(command)
         if ret != 0:
             err_str = stderr.read().decode("utf-8")
             if "Invalid job id specified" in err_str:
@@ -336,13 +329,11 @@ class SlurmJobArray(Slurm):
             ):
                 # retry 3 times
                 raise RetrySignal(
-                    "Get error code %d in checking status through ssh with job: %s . message: %s"
-                    % (ret, job.job_hash, err_str)
+                    f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
                 )
             raise RuntimeError(
-                "status command squeue fails to execute."
-                "job_id:%s \n error message:%s\n return code %d\n"
-                % (job_id, err_str, ret)
+                f"status command {command} fails to execute."
+                f"job_id:{job_id} \n error message:{err_str}\n return code {ret}\n"
             )
         status_lines = stdout.read().decode("utf-8").split("\n")[:-1]
         status = []

dpdispatcher/submission.py

@@ -55,7 +55,6 @@ class Submission:
         *,
         task_list=[],
     ):
-        # self.submission_list = submission_list
         self.local_root = None
         self.work_base = work_base
         self._abs_work_base = os.path.abspath(work_base)
@@ -324,8 +323,7 @@ class Submission:
         kwargs = {**{"clean": False}, **kwargs}
         if kwargs["clean"]:
             dlog.warning(
-                "Using async submission with `clean=True`, "
-                "job may fail in queue system"
+                "Using async submission with `clean=True`, job may fail in queue system"
             )
         loop = asyncio.get_event_loop()
         wrapped_submission = functools.partial(self.run_submission, **kwargs)
@@ -515,12 +513,9 @@ class Submission:
     def submission_from_json(cls, json_file_name="submission.json"):
         with open(json_file_name) as f:
             submission_dict = json.load(f)
-        # submission_dict = machine.context.read_file(json_file_name)
         submission = cls.deserialize(submission_dict=submission_dict, machine=None)
         return submission
 
-    # def check_if_recover()
-
     def try_recover_from_json(self):
         submission_file_name = f"{self.submission_hash}.json"
         if_recover = self.machine.context.check_file_exists(submission_file_name)
@@ -545,7 +540,6 @@ class Submission:
                     f"machine.context.remote_root:{self.machine.context.remote_root}; "
                     f"submission.work_base:{submission.work_base};"
                 )
-                # self = submission.bind_machine(machine=self.machine)
             else:
                 print(self.serialize())
                 print(submission.serialize())
@@ -759,7 +753,6 @@ class Job:
         self.fail_count = 0
         self.job_uuid = uuid.uuid4()
 
-        # self.job_hash = self.get_hash()
         self.job_hash = self.get_hash()
         self.script_file_name = self.job_hash + ".sub"
 
@@ -1122,9 +1115,9 @@ class Resources:
 
     @staticmethod
     def arginfo(detail_kwargs=True):
-        doc_number_node = "The number of node need for each `job`"
-        doc_cpu_per_node = "cpu numbers of each node assigned to each job."
-        doc_gpu_per_node = "gpu numbers of each node assigned to each job."
+        doc_number_node = "The number of nodes required for each `job`."
+        doc_cpu_per_node = "CPU numbers of each node assigned to each job."
+        doc_gpu_per_node = "GPU numbers of each node assigned to each job."
         doc_queue_name = "The queue name of batch job scheduler system."
         doc_group_size = "The number of `tasks` in a `job`. 0 means infinity."
         doc_custom_flags = "The extra lines pass to job submitting script header"

dpdispatcher/utils/hdfs_cli.py

@@ -28,7 +28,7 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                f"Cannot check existence of hdfs uri[{uri}] " f"with cmd[{cmd}]"
+                f"Cannot check existence of hdfs uri[{uri}] with cmd[{cmd}]"
             ) from e
 
     @staticmethod
@@ -48,9 +48,7 @@ class HDFS:
                     f"with cmd[{cmd}]; ret[{ret}] output[{out}] stderr[{err}]"
                 )
         except Exception as e:
-            raise RuntimeError(
-                f"Cannot remove hdfs uri[{uri}] " f"with cmd[{cmd}]"
-            ) from e
+            raise RuntimeError(f"Cannot remove hdfs uri[{uri}] with cmd[{cmd}]") from e
 
     @staticmethod
     def mkdir(uri):
@@ -70,7 +68,7 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                f"Cannot mkdir of hdfs uri[{uri}] " f"with cmd[{cmd}]"
+                f"Cannot mkdir of hdfs uri[{uri}] with cmd[{cmd}]"
             ) from e
 
     @staticmethod
@@ -80,7 +78,7 @@ class HDFS:
         """
         # Make sure local_path is accessible
         if not os.path.exists(local_path) or not os.access(local_path, os.R_OK):
-            raise RuntimeError(f"try to access local_path[{local_path}] " "but failed")
+            raise RuntimeError(f"try to access local_path[{local_path}] but failed")
         cmd = f"hadoop fs -copyFromLocal -f {local_path} {to_uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
@@ -132,9 +130,7 @@ class HDFS:
                     f"cmd [{cmd}] ret[{ret}] output[{out}] stderr[{err}]"
                 )
         except Exception as e:
-            raise RuntimeError(
-                f"Cannot read text from uri[{uri}]" f"cmd [{cmd}]"
-            ) from e
+            raise RuntimeError(f"Cannot read text from uri[{uri}]cmd [{cmd}]") from e
 
     @staticmethod
     def move(from_uri, to_uri):
@@ -151,6 +147,5 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                f"Cannot move from_uri[{from_uri}] to "
-                f"to_uri[{to_uri}] with cmd[{cmd}]"
+                f"Cannot move from_uri[{from_uri}] to to_uri[{to_uri}] with cmd[{cmd}]"
             ) from e

dpdispatcher/utils/utils.py

@@ -191,7 +191,7 @@ def retry(
             else:
                 # raise all exceptions
                 raise RuntimeError(
-                    "Failed to run %s for %d times" % (func.__name__, current_retry)
+                    f"Failed to run {func.__name__} for {current_retry} times"
                 ) from errors[-1]
 
         return wrapper

{dpdispatcher-0.6.6.dist-info → dpdispatcher-0.6.8.dist-info}/METADATA

@@ -1,9 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: dpdispatcher
-Version: 0.6.6
+Version: 0.6.8
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
-License: GNU LESSER GENERAL PUBLIC LICENSE
+License:                    GNU LESSER GENERAL PUBLIC LICENSE
                                Version 3, 29 June 2007
         
          Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
@@ -186,32 +186,32 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: paramiko
-Requires-Dist: dargs >=0.4.1
+Requires-Dist: dargs>=0.4.1
 Requires-Dist: requests
-Requires-Dist: tqdm >=4.9.0
+Requires-Dist: tqdm>=4.9.0
+Requires-Dist: typing_extensions; python_version < "3.7"
 Requires-Dist: pyyaml
-Requires-Dist: tomli >=1.1.0 ; python_version < "3.11"
-Requires-Dist: typing-extensions ; python_version < "3.7"
-Provides-Extra: bohrium
-Requires-Dist: oss2 ; extra == 'bohrium'
-Requires-Dist: tqdm ; extra == 'bohrium'
-Requires-Dist: bohrium-sdk ; extra == 'bohrium'
-Provides-Extra: cloudserver
-Requires-Dist: oss2 ; extra == 'cloudserver'
-Requires-Dist: tqdm ; extra == 'cloudserver'
-Requires-Dist: bohrium-sdk ; extra == 'cloudserver'
+Requires-Dist: tomli>=1.1.0; python_version < "3.11"
 Provides-Extra: docs
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: myst-parser ; extra == 'docs'
-Requires-Dist: sphinx-rtd-theme >=1.0.0rc1 ; extra == 'docs'
-Requires-Dist: numpydoc ; extra == 'docs'
-Requires-Dist: deepmodeling-sphinx >=0.1.1 ; extra == 'docs'
-Requires-Dist: dargs >=0.3.1 ; extra == 'docs'
-Requires-Dist: sphinx-argparse <0.5.0 ; extra == 'docs'
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: sphinx-book-theme; extra == "docs"
+Requires-Dist: numpydoc; extra == "docs"
+Requires-Dist: deepmodeling-sphinx>=0.3.0; extra == "docs"
+Requires-Dist: dargs>=0.3.1; extra == "docs"
+Requires-Dist: sphinx-argparse<0.5.0; extra == "docs"
+Provides-Extra: cloudserver
+Requires-Dist: oss2; extra == "cloudserver"
+Requires-Dist: tqdm; extra == "cloudserver"
+Requires-Dist: bohrium-sdk; extra == "cloudserver"
+Provides-Extra: bohrium
+Requires-Dist: oss2; extra == "bohrium"
+Requires-Dist: tqdm; extra == "bohrium"
+Requires-Dist: bohrium-sdk; extra == "bohrium"
 Provides-Extra: gui
-Requires-Dist: dpgui ; extra == 'gui'
+Requires-Dist: dpgui; extra == "gui"
 Provides-Extra: test
-Requires-Dist: dpgui ; extra == 'test'
+Requires-Dist: dpgui; extra == "test"
 
 # DPDispatcher
 
@@ -221,7 +221,7 @@ Requires-Dist: dpgui ; extra == 'test'
 [![Documentation Status](https://readthedocs.org/projects/dpdispatcher/badge/)](https://dpdispatcher.readthedocs.io/)
 
 DPDispatcher is a Python package used to generate HPC (High-Performance Computing) scheduler systems (Slurm/PBS/LSF/Bohrium) jobs input scripts, submit them to HPC systems, and poke until they finish.
-​
+
 DPDispatcher will monitor (poke) until these jobs finish and download the results files (if these jobs are running on remote systems connected by SSH).
 
 For more information, check the [documentation](https://dpdispatcher.readthedocs.io/).

{dpdispatcher-0.6.6.dist-info → dpdispatcher-0.6.8.dist-info}/RECORD

@@ -1,49 +1,49 @@
 dpdispatcher/__init__.py,sha256=CLZP_N5CTp14ujWCykEHuJjoIfKR6CwrclXhjWUgNoE,517
 dpdispatcher/__main__.py,sha256=BFhG-mSBzVZUEezQJqXWZnt2WsnhAHT_zpT8Y6gpOz0,116
-dpdispatcher/_version.py,sha256=A5NOPsDJAvtNjXOWXcGEBcGThUtYnfklnJHouP0KaiU,411
+dpdispatcher/_version.py,sha256=jFQ70HGO-FOLhIy6SpujRlrxJLVWsy5CAtewPppbaOs,411
 dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
-dpdispatcher/base_context.py,sha256=NvaC_RHyspxq412z-eCq4Zn8-szZxvn8K6OkXvx7l4Y,3615
+dpdispatcher/base_context.py,sha256=W4eWDWVzYeL6EuEkivmJp-_h_B2mV9PtRWc09l1_Qzc,5242
 dpdispatcher/dlog.py,sha256=QJKAwB6gV3Zb6zQUL9dZ_uIoTIEy9Z7ecmVQ-8WNmD8,1081
 dpdispatcher/dpdisp.py,sha256=jhuTmwPY7KBF4WukaQomEwZcfYoISaMbKwuxdDGSluc,4206
-dpdispatcher/machine.py,sha256=EXrOckVsW9ZFOBc88eaSt2_WzDqNtjDTkGjOBFKWG04,16106
+dpdispatcher/machine.py,sha256=k53ycs_v7xrl4D93URc5ht0shoO9NPrVl0rYr4v5OiU,16696
 dpdispatcher/run.py,sha256=tFHbJAioXXpgHTE5bhRRAuc8w7cX1ET9SBbiAg3Rw-I,5382
-dpdispatcher/submission.py,sha256=0_PCpRyiUwCHwYAzdXs-3rzq8YzZs0VZBU6tS7SixG0,48361
+dpdispatcher/submission.py,sha256=zLzdKJkMXhvaicD2el33NxDHP_9LL29HBombxR1l-Sw,48086
 dpdispatcher/contexts/__init__.py,sha256=jlvcIppmUnS39yBlkZEDvIQFV-j_BR75ZTbZALF_RB0,336
-dpdispatcher/contexts/dp_cloud_server_context.py,sha256=6XK0B2sLGEDeZmV2SZzQdVrMcWAWYZVLLK-IaShEXIY,12245
-dpdispatcher/contexts/hdfs_context.py,sha256=B6pjGUD8Xaa0G_Zrnoci2DZnEXxojE9fAcexMMvAZCM,8930
-dpdispatcher/contexts/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
-dpdispatcher/contexts/local_context.py,sha256=AsIfOT24FV0_bNlD2xU-pqAJy-XHZ6XTsbll4Vt6bMM,14065
-dpdispatcher/contexts/openapi_context.py,sha256=DXaMS10SXN3VKEeEdzQyfOgRwUyHRJVCJHd2fKKdsmA,9499
-dpdispatcher/contexts/ssh_context.py,sha256=ApFhzK0c7zxclOSESEswpy_RsM1zLkeEYJ_hCtrALmQ,38682
+dpdispatcher/contexts/dp_cloud_server_context.py,sha256=PGRMef3q2hfK-o5dNIWWvzPca2NK1HrWEgungM4L9Go,12420
+dpdispatcher/contexts/hdfs_context.py,sha256=mYQzXMZ4A9EjjWBAH3Ba6HOErUhMMwCsKxOjpd5R57Y,9105
+dpdispatcher/contexts/lazy_local_context.py,sha256=FAClbLD2F4LizUqFzMOg3t0Z6NLeTDLJy7NkRcDELFs,5070
+dpdispatcher/contexts/local_context.py,sha256=VbaSXGAc_EDMT0K5WV_flBF0bX87ntrwO_hq_Bkcb04,14590
+dpdispatcher/contexts/openapi_context.py,sha256=M7L9axpjOrzvdTpLMDuEzZqe4ZuKIxjS0bzZUv8W2IQ,9674
+dpdispatcher/contexts/ssh_context.py,sha256=qaj8h2TdY1i-YYdDstUBs9IJaLwzytwnQkdntMEZ7vg,37664
 dpdispatcher/dpcloudserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dpdispatcher/dpcloudserver/client.py,sha256=k1niKjG6zFnMtHn_UuCjYoOcMju3o3PV-GdyVLr5-KM,165
 dpdispatcher/entrypoints/__init__.py,sha256=exKSFT3j2oCerGwtI8WbHQK-D0K-CyifocRji1xntT4,20
 dpdispatcher/entrypoints/gui.py,sha256=29lMXqbmSRbLj4rfBv7Jnw89NLU9syTB88IUP6IRJsU,830
 dpdispatcher/entrypoints/run.py,sha256=tRkHfeAktV6gF31yb2MVOSTlpNGZFw3N0jHBmM1YfIg,175
 dpdispatcher/entrypoints/submission.py,sha256=ikVwIZAQL0SsYO5xaMIdKXgO6qtc05w1vqmvtG7Nk5M,3401
-dpdispatcher/machines/JH_UniScheduler.py,sha256=f7Vs9_m4Th1GVSgsJTy9_nMAY8g9n0ZewnPY2DFECfI,5795
+dpdispatcher/machines/JH_UniScheduler.py,sha256=ZeUZXqyGrN5Zec4gwpwH5r6FJXaJLRUJMQWDCP7X3Nk,5756
 dpdispatcher/machines/__init__.py,sha256=tOQuPUlW1Ab4qcC0oSAIyDjZA_WyE67h_EIxPCWGhys,336
-dpdispatcher/machines/distributed_shell.py,sha256=LvWl6ktPlgmJ7rk90VWxp4douve8hYmuRf-B0saFBds,7534
+dpdispatcher/machines/distributed_shell.py,sha256=c0-lGeGz_M-PY2gPciT-uYZLQht5XTMaxJSNxkbMffc,7489
 dpdispatcher/machines/dp_cloud_server.py,sha256=SR69gsFb2BvOQCW1QnWfP3cQvu_qHLJNsycp5wzosJU,11706
 dpdispatcher/machines/fugaku.py,sha256=oY2hD2ldL2dztwtJ9WNisdsfPnaX-5yTRXewIT9r60I,4314
-dpdispatcher/machines/lsf.py,sha256=Q6IE4nCkNEKcW0AdBTKPOYgmCJAeXWmUVxZ9sQFkxos,7932
+dpdispatcher/machines/lsf.py,sha256=xGDq8OLAk83E9EjK_3-QtEOyahvBGspWbxT__7mnSTw,7896
 dpdispatcher/machines/openapi.py,sha256=Gzzbo8YOAybXGTrgMutexErcaEi3ts7uTUNvOhThFS8,8858
-dpdispatcher/machines/pbs.py,sha256=xPbdnT-g8pDMbq-yuI8G7TA0AZqn9gLXuqfWabQ2Whk,12437
-dpdispatcher/machines/shell.py,sha256=DnqMNb2nmBc3gVx8tA8oiUWdnWHKJwpIPs660i3Eq7A,4703
-dpdispatcher/machines/slurm.py,sha256=YM2Mv55jAFtDIiJoJLkD6p1Wi1ujjH6t4WlU8EtlbCw,15592
+dpdispatcher/machines/pbs.py,sha256=gUoj3OGQbZRBK4P-WXlhrxlQqTeUi9X8JGLOkAB__wE,12669
+dpdispatcher/machines/shell.py,sha256=EeYnRCowXdzO3Nh25Yh_t5xeM6frq4uChk4GVx7OjH8,4797
+dpdispatcher/machines/slurm.py,sha256=oyMX9iZQpVRR951zwz0wRNfl3_uJZzdtzxMbTJotlQU,15402
 dpdispatcher/utils/__init__.py,sha256=fwvwkMf7DFNQkNBiIce8Y8gRA6FhICwKjkKiXu_BEJg,13
-dpdispatcher/utils/hdfs_cli.py,sha256=n3EIfFIralsISlaEewawD35f0P8mabo-u8D8UW3k_7Y,5308
+dpdispatcher/utils/hdfs_cli.py,sha256=a1a9PJAzt3wsTcdaSw_oD1vcNw59pMooxpAHjYOaaGA,5209
 dpdispatcher/utils/job_status.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
 dpdispatcher/utils/record.py,sha256=c8jdPmCuLzRmFo_jOjR0j9zFR1EWX3NSHVuPEIYCycg,2147
-dpdispatcher/utils/utils.py,sha256=1One9eW-v3ejDcL6PB9PSCMZQkalnbxq0DfJoUwQaLs,5334
+dpdispatcher/utils/utils.py,sha256=Wo-8tGO05e2KkRyLXoIg3UlxzkuM-x1phRrTA1Hh7Ko,5328
 dpdispatcher/utils/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
 dpdispatcher/utils/dpcloudserver/client.py,sha256=CLfXswvzI4inDrW2bYkfMQ6gQJFcZOgLhiXBz_EI17M,12029
 dpdispatcher/utils/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
 dpdispatcher/utils/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
 dpdispatcher/utils/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
-dpdispatcher-0.6.6.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
-dpdispatcher-0.6.6.dist-info/METADATA,sha256=0sYP0wVNFK9e2SMke4jpCbjpBEDA691quZj60MO3p6k,12828
-dpdispatcher-0.6.6.dist-info/WHEEL,sha256=Wyh-_nZ0DJYolHNn1_hMa4lM7uDedD_RGVwbmTjyItk,91
-dpdispatcher-0.6.6.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
-dpdispatcher-0.6.6.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
-dpdispatcher-0.6.6.dist-info/RECORD,,
+dpdispatcher-0.6.8.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
+dpdispatcher-0.6.8.dist-info/METADATA,sha256=mneS5eFsvLVeWxt9dGHBLyDQv0NSdIdC9x00TqFHhGI,12811
+dpdispatcher-0.6.8.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+dpdispatcher-0.6.8.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
+dpdispatcher-0.6.8.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
+dpdispatcher-0.6.8.dist-info/RECORD,,

{dpdispatcher-0.6.6.dist-info → dpdispatcher-0.6.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (71.1.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any