dpdispatcher 0.6.3__py3-none-any.whl → 0.6.4__py3-none-any.whl

dpdispatcher/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.3'
-__version_tuple__ = version_tuple = (0, 6, 3)
+__version__ = version = '0.6.4'
+__version_tuple__ = version_tuple = (0, 6, 4)

dpdispatcher/contexts/local_context.py

@@ -208,7 +208,10 @@ class LocalContext(BaseContext):
                             shutil.rmtree(lfile, ignore_errors=True)
                         elif os.path.isfile(lfile) or os.path.islink(lfile):
                             os.remove(lfile)
-                        shutil.copyfile(rfile, lfile)
+                        if not os.path.islink(rfile):
+                            shutil.move(rfile, lfile)
+                        else:
+                            shutil.copyfile(rfile, lfile)
                     else:
                         raise RuntimeError("should not reach here!")
                 else:
@@ -275,7 +278,10 @@ class LocalContext(BaseContext):
                         shutil.rmtree(lfile, ignore_errors=True)
                     elif os.path.isfile(lfile) or os.path.islink(lfile):
                         os.remove(lfile)
-                    shutil.copyfile(rfile, lfile)
+                    if not os.path.islink(rfile):
+                        shutil.move(rfile, lfile)
+                    else:
+                        shutil.copyfile(rfile, lfile)
                 else:
                     raise RuntimeError("should not reach here!")
             else:

dpdispatcher/contexts/ssh_context.py

@@ -295,7 +295,7 @@ class SSHSession:
         assert self.ssh is not None
         try:
             return self.ssh.exec_command(cmd)
-        except (paramiko.ssh_exception.SSHException, socket.timeout) as e:
+        except (paramiko.ssh_exception.SSHException, socket.timeout, EOFError) as e:
             # SSH session not active
             # retry for up to 3 times
             # ensure alive

dpdispatcher/dpdisp.py

@@ -54,6 +54,11 @@ def main_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Clean submission.",
     )
+    parser_submission_action.add_argument(
+        "--reset-fail-count",
+        action="store_true",
+        help="Reset fail count of all jobs to zero.",
+    )
     ##########################################
     # gui
     parser_gui = subparsers.add_parser(
@@ -105,6 +110,7 @@ def main():
             download_terminated_log=args.download_terminated_log,
             download_finished_task=args.download_finished_task,
             clean=args.clean,
+            reset_fail_count=args.reset_fail_count,
         )
     elif args.command == "gui":
         start_dpgui(

dpdispatcher/entrypoints/submission.py

@@ -12,6 +12,7 @@ def handle_submission(
     download_terminated_log: bool = False,
     download_finished_task: bool = False,
     clean: bool = False,
+    reset_fail_count: bool = False,
 ):
     """Handle terminated submission.
 
@@ -25,13 +26,21 @@ def handle_submission(
         Download finished tasks.
     clean : bool, optional
         Clean submission.
+    reset_fail_count : bool, optional
+        Reset fail count of all jobs to zero.
 
     Raises
     ------
     ValueError
         At least one action should be specified.
     """
-    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+    if (
+        int(download_terminated_log)
+        + int(download_finished_task)
+        + int(clean)
+        + int(reset_fail_count)
+        == 0
+    ):
         raise ValueError("At least one action should be specified.")
 
     submission_file = record.get_submission(submission_hash)
@@ -42,7 +51,18 @@ def handle_submission(
     # TODO: for unclear reason, the submission_hash may be changed
     submission.submission_hash = submission_hash
     submission.machine.context.bind_submission(submission)
+    if reset_fail_count:
+        for job in submission.belonging_jobs:
+            job.fail_count = 0
+        # save to remote and local
+        submission.submission_to_json()
+        record.write(submission)
+    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+        # if only reset_fail_count, no need to update submission state (expensive)
+        return
     submission.update_submission_state()
+    submission.submission_to_json()
+    record.write(submission)
 
     terminated_tasks = []
     finished_tasks = []

dpdispatcher/machines/pbs.py

@@ -177,3 +177,107 @@ class Torque(PBS):
                 **pbs_script_header_dict
             )
         return pbs_script_header
+
+
+sge_script_header_template = """
+#!/bin/bash
+#$ -N dpdispatcher_submit
+{select_node_line}
+#$ -cwd
+
+"""
+
+
+class SGE(PBS):
+    def __init__(
+        self,
+        batch_type=None,
+        context_type=None,
+        local_root=None,
+        remote_root=None,
+        remote_profile={},
+        *,
+        context=None,
+    ):
+        super(PBS, self).__init__(
+            batch_type,
+            context_type,
+            local_root,
+            remote_root,
+            remote_profile,
+            context=context,
+        )
+
+    def gen_script_header(self, job):
+        resources = job.resources
+        sge_script_header_dict = {}
+        # resources.number_node is not used
+        sge_script_header_dict[
+            "select_node_line"
+        ] = f"#$ -pe mpi {resources.cpu_per_node} "
+        # resources.queue_name is not necessary
+        sge_script_header = sge_script_header_template.format(**sge_script_header_dict)
+        return sge_script_header
+
+    def do_submit(self, job):
+        script_file_name = job.script_file_name
+        script_str = self.gen_script(job)
+        job_id_name = job.job_hash + "_job_id"
+        self.context.write_file(fname=script_file_name, write_str=script_str)
+        script_file_dir = self.context.remote_root
+        stdin, stdout, stderr = self.context.block_checkcall(
+            "cd {} && {} {}".format(script_file_dir, "qsub", script_file_name)
+        )
+        subret = stdout.readlines()
+        job_id = subret[0].split()[2]
+        self.context.write_file(job_id_name, job_id)
+        return job_id
+
+    def default_resources(self, resources):
+        pass
+
+    def check_status(self, job):
+        job_id = job.job_id
+        status_line = None
+        if job_id == "":
+            return JobStatus.unsubmitted
+        ret, stdin, stdout, stderr = self.context.block_call("qstat")
+        err_str = stderr.read().decode("utf-8")
+        if ret != 0:
+            raise RuntimeError(
+                "status command qstat fails to execute. erro info: %s return code %d"
+                % (err_str, ret)
+            )
+        status_text_list = stdout.read().decode("utf-8").split("\n")
+        for txt in status_text_list:
+            if job_id in txt:
+                status_line = txt
+
+        if status_line is None:
+            count = 0
+            while count <= 6:
+                if self.check_finish_tag(job=job):
+                    return JobStatus.finished
+                dlog.info(
+                    "not tag_finished detected, execute sync command and wait. count "
+                    + str(count)
+                )
+                self.context.block_call("sync")
+                import time
+
+                time.sleep(10)
+                count += 1
+            return JobStatus.terminated
+        else:
+            status_word = status_line.split()[4]
+            # dlog.info (status_word)
+            if status_word in ["qw"]:
+                return JobStatus.waiting
+            elif status_word in ["r"]:
+                return JobStatus.running
+            else:
+                return JobStatus.unknown
+
+    def check_finish_tag(self, job):
+        job_tag_finished = job.job_hash + "_job_tag_finished"
+        return self.context.check_file_exists(job_tag_finished)

{dpdispatcher-0.6.3.dist-info → dpdispatcher-0.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.3
+Version: 0.6.4
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License: GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.3.dist-info → dpdispatcher-0.6.4.dist-info}/RECORD

@@ -1,31 +1,31 @@
 dpdispatcher/__init__.py,sha256=CLZP_N5CTp14ujWCykEHuJjoIfKR6CwrclXhjWUgNoE,517
 dpdispatcher/__main__.py,sha256=BFhG-mSBzVZUEezQJqXWZnt2WsnhAHT_zpT8Y6gpOz0,116
-dpdispatcher/_version.py,sha256=ztAzoNjubRfmXpwvac6nvVotoNE53YlNbwgehkHg0Ss,411
+dpdispatcher/_version.py,sha256=YeKWh9qHxBSK-fURNyCejbICk3LuDCjfwlZrUuEgWgM,411
 dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
 dpdispatcher/base_context.py,sha256=NvaC_RHyspxq412z-eCq4Zn8-szZxvn8K6OkXvx7l4Y,3615
 dpdispatcher/dlog.py,sha256=ndh12teQBbJRybXd8UjEmAi6QTsAXajRicDj5mAH5h0,799
-dpdispatcher/dpdisp.py,sha256=YuGb-HWLsDfSO2c7GH0eM20ciojGbx3yq9oZHP7u4yc,3498
+dpdispatcher/dpdisp.py,sha256=p-f_KCJxALBqP4StpNK2X_-VkGZ0L43MhROxhCpFwE4,3712
 dpdispatcher/machine.py,sha256=XFRH41gNCex_qs9gbg-S88_qab3_UAGfxKWUPxoipCM,16140
 dpdispatcher/submission.py,sha256=mVAHBlT0a3_1PtsEvvhvwNPkAhgLiBXXemX64BcwizU,48447
 dpdispatcher/contexts/__init__.py,sha256=s5M0ZJSrPttSyLdBwKD2m3W7a5AbYZdPB7IAND2j7EY,335
 dpdispatcher/contexts/dp_cloud_server_context.py,sha256=6XK0B2sLGEDeZmV2SZzQdVrMcWAWYZVLLK-IaShEXIY,12245
 dpdispatcher/contexts/hdfs_context.py,sha256=GbV_o3i0NL43B7dCPnArXS5DPkkET4EAiHw1VgsMcdE,9000
 dpdispatcher/contexts/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
-dpdispatcher/contexts/local_context.py,sha256=1RbApz-w0Syxghgk9tdCkELQNXmi5oU-s8J_0UTi-lU,13780
+dpdispatcher/contexts/local_context.py,sha256=vhZtdtduPokw6hU0YbaWNuoCOO6Tio2w99Fi8AegIw0,14052
 dpdispatcher/contexts/openapi_context.py,sha256=DXaMS10SXN3VKEeEdzQyfOgRwUyHRJVCJHd2fKKdsmA,9499
-dpdispatcher/contexts/ssh_context.py,sha256=155S3xDsfTPfbD7-oBUK4PLgpGD1vMEcU0MDThePJtQ,38602
+dpdispatcher/contexts/ssh_context.py,sha256=zhBM_qH4zGMws7Yww5txNLhkK7b3maiFEMoewuAJou0,38612
 dpdispatcher/dpcloudserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dpdispatcher/dpcloudserver/client.py,sha256=k1niKjG6zFnMtHn_UuCjYoOcMju3o3PV-GdyVLr5-KM,165
 dpdispatcher/entrypoints/__init__.py,sha256=exKSFT3j2oCerGwtI8WbHQK-D0K-CyifocRji1xntT4,20
 dpdispatcher/entrypoints/gui.py,sha256=29lMXqbmSRbLj4rfBv7Jnw89NLU9syTB88IUP6IRJsU,830
-dpdispatcher/entrypoints/submission.py,sha256=R2DXUGXTsk0Jw2y5Wjby40R5dxzXeqr4gn33ov6mdAI,2751
+dpdispatcher/entrypoints/submission.py,sha256=ikVwIZAQL0SsYO5xaMIdKXgO6qtc05w1vqmvtG7Nk5M,3401
 dpdispatcher/machines/__init__.py,sha256=9kSYkz2w3flp00IrHWTEwvoFGrathQAT3tvbieye83c,335
 dpdispatcher/machines/distributed_shell.py,sha256=7avNcoOzEj7UcJuKl6b1ka2bj5dixcJaMlZK-I-i_Tc,7571
 dpdispatcher/machines/dp_cloud_server.py,sha256=SR69gsFb2BvOQCW1QnWfP3cQvu_qHLJNsycp5wzosJU,11706
 dpdispatcher/machines/fugaku.py,sha256=9OP3qSaaruqypHAdcuBFQM_MUtFp3yrvhZ5bPyLwEEk,4308
 dpdispatcher/machines/lsf.py,sha256=Qruot39cPEpBNbbPmDwb1Gyfgyw3N36O0hs9PNEXyVU,7997
 dpdispatcher/machines/openapi.py,sha256=Gzzbo8YOAybXGTrgMutexErcaEi3ts7uTUNvOhThFS8,8858
-dpdispatcher/machines/pbs.py,sha256=HGBUf96AJ7hWOQPrENP5tFIDnEm4cb9deqJJ8wExbms,7079
+dpdispatcher/machines/pbs.py,sha256=YK0rKIsd8GAZYfxGLezA9RdCQ6AOeuPML_v5OwmioTo,10331
 dpdispatcher/machines/shell.py,sha256=qaia7mC_fz5Bqyelxmc1je-xg7NQ_6vQQ0qAjg2m4RQ,4796
 dpdispatcher/machines/slurm.py,sha256=SP5rQiCPWzq4rqgUgp0IGJXXD_1DURWl4OBRAJ-Kng4,15611
 dpdispatcher/utils/__init__.py,sha256=fwvwkMf7DFNQkNBiIce8Y8gRA6FhICwKjkKiXu_BEJg,13
@@ -38,9 +38,9 @@ dpdispatcher/utils/dpcloudserver/client.py,sha256=CLfXswvzI4inDrW2bYkfMQ6gQJFcZO
 dpdispatcher/utils/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
 dpdispatcher/utils/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
 dpdispatcher/utils/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
-dpdispatcher-0.6.3.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
-dpdispatcher-0.6.3.dist-info/METADATA,sha256=Y-EwJVwhTAOszfKFejHSPTsU1DH931Qrhwbdmkie6Mc,12752
-dpdispatcher-0.6.3.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-dpdispatcher-0.6.3.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
-dpdispatcher-0.6.3.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
-dpdispatcher-0.6.3.dist-info/RECORD,,
+dpdispatcher-0.6.4.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
+dpdispatcher-0.6.4.dist-info/METADATA,sha256=9NoY3DjM_USxl-2nBiNNLMCzJ6DZadgEMS1e11Fj72c,12752
+dpdispatcher-0.6.4.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+dpdispatcher-0.6.4.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
+dpdispatcher-0.6.4.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
+dpdispatcher-0.6.4.dist-info/RECORD,,

{dpdispatcher-0.6.3.dist-info → dpdispatcher-0.6.4.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any