dpdispatcher 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

dpdispatcher/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.2'
-__version_tuple__ = version_tuple = (0, 6, 2)
+__version__ = version = '0.6.4'
+__version_tuple__ = version_tuple = (0, 6, 4)

dpdispatcher/contexts/hdfs_context.py

@@ -88,7 +88,7 @@ class HDFSContext(BaseContext):
             for ff in task.forward_files:
                 abs_file_list = glob(os.path.join(local_job, ff))
                 if not abs_file_list:
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find upload file " + os.path.join(local_job, ff)
                     )
                 rel_file_list = [
@@ -100,7 +100,7 @@ class HDFSContext(BaseContext):
         for fc in submission.forward_common_files:
             abs_file_list = glob(os.path.join(local_job, fc))
             if not abs_file_list:
-                raise RuntimeError(
+                raise FileNotFoundError(
                     "cannot find upload file " + os.path.join(local_job, fc)
                 )
             rel_file_list = [
@@ -170,9 +170,11 @@ class HDFSContext(BaseContext):
                             ) as fp:
                                 pass
                         else:
-                            raise RuntimeError("do not find download file " + rfile)
+                            raise FileNotFoundError(
+                                "do not find download file " + rfile
+                            )
                     else:
-                        raise RuntimeError("do not find download file " + rfile)
+                        raise FileNotFoundError("do not find download file " + rfile)
                 else:
                     if os.path.exists(lfile):
                         dlog.info(f"find existing {lfile}, replacing by {rfile}")
@@ -203,9 +205,9 @@ class HDFSContext(BaseContext):
                         ) as fp:
                             pass
                     else:
-                        raise RuntimeError("do not find download file " + rfile)
+                        raise FileNotFoundError("do not find download file " + rfile)
                 else:
-                    raise RuntimeError("do not find download file " + rfile)
+                    raise FileNotFoundError("do not find download file " + rfile)
             else:
                 if os.path.exists(lfile):
                     dlog.info(f"find existing {lfile}, replacing by {rfile}")

dpdispatcher/contexts/local_context.py

@@ -94,7 +94,7 @@ class LocalContext(BaseContext):
             for kk in ii.forward_files:
                 abs_file_list = glob(os.path.join(local_job, kk))
                 if not abs_file_list:
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find upload file " + os.path.join(local_job, kk)
                     )
                 rel_file_list = [
@@ -104,7 +104,7 @@ class LocalContext(BaseContext):
 
             for jj in file_list:
                 if not os.path.exists(os.path.join(local_job, jj)):
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find upload file " + os.path.join(local_job, jj)
                     )
                 if os.path.exists(os.path.join(remote_job, jj)):
@@ -119,7 +119,7 @@ class LocalContext(BaseContext):
         for kk in submission.forward_common_files:
             abs_file_list = glob(os.path.join(local_job, kk))
             if not abs_file_list:
-                raise RuntimeError(
+                raise FileNotFoundError(
                     "cannot find upload file " + os.path.join(local_job, kk)
                 )
             rel_file_list = [
@@ -129,7 +129,7 @@ class LocalContext(BaseContext):
 
         for jj in file_list:
             if not os.path.exists(os.path.join(local_job, jj)):
-                raise RuntimeError(
+                raise FileNotFoundError(
                     "cannot find upload file " + os.path.join(local_job, jj)
                 )
             if os.path.exists(os.path.join(remote_job, jj)):
@@ -160,7 +160,7 @@ class LocalContext(BaseContext):
                         else:
                             pass
                     else:
-                        raise RuntimeError(
+                        raise FileNotFoundError(
                             "cannot find download file " + os.path.join(remote_job, kk)
                         )
                 rel_flist = [
@@ -188,7 +188,9 @@ class LocalContext(BaseContext):
                             else:
                                 pass
                         else:
-                            raise RuntimeError("do not find download file " + rfile)
+                            raise FileNotFoundError(
+                                "do not find download file " + rfile
+                            )
                     elif (not os.path.exists(rfile)) and (os.path.exists(lfile)):
                         # already downloaded
                         pass
@@ -206,7 +208,10 @@ class LocalContext(BaseContext):
                             shutil.rmtree(lfile, ignore_errors=True)
                         elif os.path.isfile(lfile) or os.path.islink(lfile):
                             os.remove(lfile)
-                        shutil.copyfile(rfile, lfile)
+                        if not os.path.islink(rfile):
+                            shutil.move(rfile, lfile)
+                        else:
+                            shutil.copyfile(rfile, lfile)
                     else:
                         raise RuntimeError("should not reach here!")
                 else:
@@ -229,7 +234,7 @@ class LocalContext(BaseContext):
                     else:
                         pass
                 else:
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find download file " + os.path.join(remote_job, kk)
                     )
             rel_flist = [os.path.relpath(ii, start=remote_job) for ii in abs_flist_r]
@@ -255,7 +260,7 @@ class LocalContext(BaseContext):
                         else:
                             pass
                     else:
-                        raise RuntimeError("do not find download file " + rfile)
+                        raise FileNotFoundError("do not find download file " + rfile)
                 elif (not os.path.exists(rfile)) and (os.path.exists(lfile)):
                     # already downloaded
                     pass
@@ -273,7 +278,10 @@ class LocalContext(BaseContext):
                         shutil.rmtree(lfile, ignore_errors=True)
                     elif os.path.isfile(lfile) or os.path.islink(lfile):
                         os.remove(lfile)
-                    shutil.copyfile(rfile, lfile)
+                    if not os.path.islink(rfile):
+                        shutil.move(rfile, lfile)
+                    else:
+                        shutil.copyfile(rfile, lfile)
                 else:
                     raise RuntimeError("should not reach here!")
             else:

dpdispatcher/contexts/ssh_context.py

@@ -295,7 +295,7 @@ class SSHSession:
         assert self.ssh is not None
         try:
             return self.ssh.exec_command(cmd)
-        except (paramiko.ssh_exception.SSHException, socket.timeout) as e:
+        except (paramiko.ssh_exception.SSHException, socket.timeout, EOFError) as e:
             # SSH session not active
             # retry for up to 3 times
             # ensure alive
@@ -569,7 +569,7 @@ class SSHContext(BaseContext):
                     rel_file_list, work_path, file_list, directory_list
                 )
             else:
-                raise RuntimeError(f"cannot find upload file {work_path} {jj}")
+                raise FileNotFoundError(f"cannot find upload file {work_path} {jj}")
 
     def upload(
         self,

dpdispatcher/dpdisp.py

@@ -54,6 +54,11 @@ def main_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Clean submission.",
     )
+    parser_submission_action.add_argument(
+        "--reset-fail-count",
+        action="store_true",
+        help="Reset fail count of all jobs to zero.",
+    )
     ##########################################
     # gui
     parser_gui = subparsers.add_parser(
@@ -105,6 +110,7 @@ def main():
             download_terminated_log=args.download_terminated_log,
             download_finished_task=args.download_finished_task,
             clean=args.clean,
+            reset_fail_count=args.reset_fail_count,
         )
     elif args.command == "gui":
         start_dpgui(

dpdispatcher/entrypoints/submission.py

@@ -12,6 +12,7 @@ def handle_submission(
     download_terminated_log: bool = False,
     download_finished_task: bool = False,
     clean: bool = False,
+    reset_fail_count: bool = False,
 ):
     """Handle terminated submission.
 
@@ -25,13 +26,21 @@ def handle_submission(
         Download finished tasks.
     clean : bool, optional
         Clean submission.
+    reset_fail_count : bool, optional
+        Reset fail count of all jobs to zero.
 
     Raises
     ------
     ValueError
         At least one action should be specified.
     """
-    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+    if (
+        int(download_terminated_log)
+        + int(download_finished_task)
+        + int(clean)
+        + int(reset_fail_count)
+        == 0
+    ):
         raise ValueError("At least one action should be specified.")
 
     submission_file = record.get_submission(submission_hash)
@@ -42,7 +51,18 @@ def handle_submission(
     # TODO: for unclear reason, the submission_hash may be changed
     submission.submission_hash = submission_hash
     submission.machine.context.bind_submission(submission)
+    if reset_fail_count:
+        for job in submission.belonging_jobs:
+            job.fail_count = 0
+        # save to remote and local
+        submission.submission_to_json()
+        record.write(submission)
+    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+        # if only reset_fail_count, no need to update submission state (expensive)
+        return
     submission.update_submission_state()
+    submission.submission_to_json()
+    record.write(submission)
 
     terminated_tasks = []
     finished_tasks = []

dpdispatcher/machines/pbs.py

@@ -177,3 +177,107 @@ class Torque(PBS):
                 **pbs_script_header_dict
             )
         return pbs_script_header
+
+
+sge_script_header_template = """
+#!/bin/bash
+#$ -N dpdispatcher_submit
+{select_node_line}
+#$ -cwd
+
+"""
+
+
+class SGE(PBS):
+    def __init__(
+        self,
+        batch_type=None,
+        context_type=None,
+        local_root=None,
+        remote_root=None,
+        remote_profile={},
+        *,
+        context=None,
+    ):
+        super(PBS, self).__init__(
+            batch_type,
+            context_type,
+            local_root,
+            remote_root,
+            remote_profile,
+            context=context,
+        )
+
+    def gen_script_header(self, job):
+        resources = job.resources
+        sge_script_header_dict = {}
+        # resources.number_node is not used
+        sge_script_header_dict[
+            "select_node_line"
+        ] = f"#$ -pe mpi {resources.cpu_per_node} "
+        # resources.queue_name is not necessary
+        sge_script_header = sge_script_header_template.format(**sge_script_header_dict)
+        return sge_script_header
+
+    def do_submit(self, job):
+        script_file_name = job.script_file_name
+        script_str = self.gen_script(job)
+        job_id_name = job.job_hash + "_job_id"
+        self.context.write_file(fname=script_file_name, write_str=script_str)
+        script_file_dir = self.context.remote_root
+        stdin, stdout, stderr = self.context.block_checkcall(
+            "cd {} && {} {}".format(script_file_dir, "qsub", script_file_name)
+        )
+        subret = stdout.readlines()
+        job_id = subret[0].split()[2]
+        self.context.write_file(job_id_name, job_id)
+        return job_id
+
+    def default_resources(self, resources):
+        pass
+
+    def check_status(self, job):
+        job_id = job.job_id
+        status_line = None
+        if job_id == "":
+            return JobStatus.unsubmitted
+        ret, stdin, stdout, stderr = self.context.block_call("qstat")
+        err_str = stderr.read().decode("utf-8")
+        if ret != 0:
+            raise RuntimeError(
+                "status command qstat fails to execute. erro info: %s return code %d"
+                % (err_str, ret)
+            )
+        status_text_list = stdout.read().decode("utf-8").split("\n")
+        for txt in status_text_list:
+            if job_id in txt:
+                status_line = txt
+
+        if status_line is None:
+            count = 0
+            while count <= 6:
+                if self.check_finish_tag(job=job):
+                    return JobStatus.finished
+                dlog.info(
+                    "not tag_finished detected, execute sync command and wait. count "
+                    + str(count)
+                )
+                self.context.block_call("sync")
+                import time
+
+                time.sleep(10)
+                count += 1
+            return JobStatus.terminated
+        else:
+            status_word = status_line.split()[4]
+            # dlog.info (status_word)
+            if status_word in ["qw"]:
+                return JobStatus.waiting
+            elif status_word in ["r"]:
+                return JobStatus.running
+            else:
+                return JobStatus.unknown
+
+    def check_finish_tag(self, job):
+        job_tag_finished = job.job_hash + "_job_tag_finished"
+        return self.context.check_file_exists(job_tag_finished)

{dpdispatcher-0.6.2.dist-info → dpdispatcher-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.2
+Version: 0.6.4
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License: GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.2.dist-info → dpdispatcher-0.6.4.dist-info}/RECORD

@@ -1,31 +1,31 @@
 dpdispatcher/__init__.py,sha256=CLZP_N5CTp14ujWCykEHuJjoIfKR6CwrclXhjWUgNoE,517
 dpdispatcher/__main__.py,sha256=BFhG-mSBzVZUEezQJqXWZnt2WsnhAHT_zpT8Y6gpOz0,116
-dpdispatcher/_version.py,sha256=PMhMumGW6FY6KtkQm1bSNdEAeyrHlBqpSQ7WpjsTyws,411
+dpdispatcher/_version.py,sha256=YeKWh9qHxBSK-fURNyCejbICk3LuDCjfwlZrUuEgWgM,411
 dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
 dpdispatcher/base_context.py,sha256=NvaC_RHyspxq412z-eCq4Zn8-szZxvn8K6OkXvx7l4Y,3615
 dpdispatcher/dlog.py,sha256=ndh12teQBbJRybXd8UjEmAi6QTsAXajRicDj5mAH5h0,799
-dpdispatcher/dpdisp.py,sha256=YuGb-HWLsDfSO2c7GH0eM20ciojGbx3yq9oZHP7u4yc,3498
+dpdispatcher/dpdisp.py,sha256=p-f_KCJxALBqP4StpNK2X_-VkGZ0L43MhROxhCpFwE4,3712
 dpdispatcher/machine.py,sha256=XFRH41gNCex_qs9gbg-S88_qab3_UAGfxKWUPxoipCM,16140
 dpdispatcher/submission.py,sha256=mVAHBlT0a3_1PtsEvvhvwNPkAhgLiBXXemX64BcwizU,48447
 dpdispatcher/contexts/__init__.py,sha256=s5M0ZJSrPttSyLdBwKD2m3W7a5AbYZdPB7IAND2j7EY,335
 dpdispatcher/contexts/dp_cloud_server_context.py,sha256=6XK0B2sLGEDeZmV2SZzQdVrMcWAWYZVLLK-IaShEXIY,12245
-dpdispatcher/contexts/hdfs_context.py,sha256=GJs_vmDCjTsnbfTdXpFTfpWTYXnZTDkEO2UJIdpV5F4,8908
+dpdispatcher/contexts/hdfs_context.py,sha256=GbV_o3i0NL43B7dCPnArXS5DPkkET4EAiHw1VgsMcdE,9000
 dpdispatcher/contexts/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
-dpdispatcher/contexts/local_context.py,sha256=7CoGzcX-RU6cpmSYcf4wMwncYaFVUb8Ljj4ksfXcx4s,13678
+dpdispatcher/contexts/local_context.py,sha256=vhZtdtduPokw6hU0YbaWNuoCOO6Tio2w99Fi8AegIw0,14052
 dpdispatcher/contexts/openapi_context.py,sha256=DXaMS10SXN3VKEeEdzQyfOgRwUyHRJVCJHd2fKKdsmA,9499
-dpdispatcher/contexts/ssh_context.py,sha256=1UbMIFpSGBcg3H1My4cx5vjAALvaxlZxWOcXwoX6Ff0,38597
+dpdispatcher/contexts/ssh_context.py,sha256=zhBM_qH4zGMws7Yww5txNLhkK7b3maiFEMoewuAJou0,38612
 dpdispatcher/dpcloudserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dpdispatcher/dpcloudserver/client.py,sha256=k1niKjG6zFnMtHn_UuCjYoOcMju3o3PV-GdyVLr5-KM,165
 dpdispatcher/entrypoints/__init__.py,sha256=exKSFT3j2oCerGwtI8WbHQK-D0K-CyifocRji1xntT4,20
 dpdispatcher/entrypoints/gui.py,sha256=29lMXqbmSRbLj4rfBv7Jnw89NLU9syTB88IUP6IRJsU,830
-dpdispatcher/entrypoints/submission.py,sha256=R2DXUGXTsk0Jw2y5Wjby40R5dxzXeqr4gn33ov6mdAI,2751
+dpdispatcher/entrypoints/submission.py,sha256=ikVwIZAQL0SsYO5xaMIdKXgO6qtc05w1vqmvtG7Nk5M,3401
 dpdispatcher/machines/__init__.py,sha256=9kSYkz2w3flp00IrHWTEwvoFGrathQAT3tvbieye83c,335
 dpdispatcher/machines/distributed_shell.py,sha256=7avNcoOzEj7UcJuKl6b1ka2bj5dixcJaMlZK-I-i_Tc,7571
 dpdispatcher/machines/dp_cloud_server.py,sha256=SR69gsFb2BvOQCW1QnWfP3cQvu_qHLJNsycp5wzosJU,11706
 dpdispatcher/machines/fugaku.py,sha256=9OP3qSaaruqypHAdcuBFQM_MUtFp3yrvhZ5bPyLwEEk,4308
 dpdispatcher/machines/lsf.py,sha256=Qruot39cPEpBNbbPmDwb1Gyfgyw3N36O0hs9PNEXyVU,7997
 dpdispatcher/machines/openapi.py,sha256=Gzzbo8YOAybXGTrgMutexErcaEi3ts7uTUNvOhThFS8,8858
-dpdispatcher/machines/pbs.py,sha256=HGBUf96AJ7hWOQPrENP5tFIDnEm4cb9deqJJ8wExbms,7079
+dpdispatcher/machines/pbs.py,sha256=YK0rKIsd8GAZYfxGLezA9RdCQ6AOeuPML_v5OwmioTo,10331
 dpdispatcher/machines/shell.py,sha256=qaia7mC_fz5Bqyelxmc1je-xg7NQ_6vQQ0qAjg2m4RQ,4796
 dpdispatcher/machines/slurm.py,sha256=SP5rQiCPWzq4rqgUgp0IGJXXD_1DURWl4OBRAJ-Kng4,15611
 dpdispatcher/utils/__init__.py,sha256=fwvwkMf7DFNQkNBiIce8Y8gRA6FhICwKjkKiXu_BEJg,13
@@ -38,9 +38,9 @@ dpdispatcher/utils/dpcloudserver/client.py,sha256=CLfXswvzI4inDrW2bYkfMQ6gQJFcZO
 dpdispatcher/utils/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
 dpdispatcher/utils/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
 dpdispatcher/utils/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
-dpdispatcher-0.6.2.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
-dpdispatcher-0.6.2.dist-info/METADATA,sha256=1JRjYaFcAF0_NBxNInlFOM2YmpbBKp7P5EAnvX99f-4,12752
-dpdispatcher-0.6.2.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-dpdispatcher-0.6.2.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
-dpdispatcher-0.6.2.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
-dpdispatcher-0.6.2.dist-info/RECORD,,
+dpdispatcher-0.6.4.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
+dpdispatcher-0.6.4.dist-info/METADATA,sha256=9NoY3DjM_USxl-2nBiNNLMCzJ6DZadgEMS1e11Fj72c,12752
+dpdispatcher-0.6.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+dpdispatcher-0.6.4.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
+dpdispatcher-0.6.4.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
+dpdispatcher-0.6.4.dist-info/RECORD,,

{dpdispatcher-0.6.2.dist-info → dpdispatcher-0.6.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any