dpdispatcher 0.6.2__tar.gz → 0.6.4__tar.gz

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/.github/workflows/pyright.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: '3.11'
     - run: pip install -e .[cloudserver,gui]

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Setup python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: 3.x
         architecture: x64

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/.github/workflows/test-bohrium.yml

@@ -16,7 +16,7 @@ jobs:
       with:
         ref: "${{ github.event.pull_request.merge_commit_sha }}"
     - name: Set up Python 3.12
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12'
         cache: 'pip'

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - run: pip install .[test] coverage

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/.pre-commit-config.yaml

@@ -18,7 +18,7 @@ repos:
 # Python
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.1.5
+    rev: v0.1.13
     hooks:
     - id: ruff
       args: ["--fix"]

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.2
+Version: 0.6.4
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/doc/batch.md

@@ -75,4 +75,11 @@ Read Fujitsu cloud service documentation for details.
 ## OpenAPI
 
 {dargs:argument}`batcy_type <resources/batch_type>`: `OpenAPI`
-OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
+OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
+
+
+## SGE
+
+{dargs:argument}`batch_type <resources/batch_type>`: `SGE`
+
+The [Sun Grid Engine (SGE) scheduler](https://gridscheduler.sourceforge.net) is a batch-queueing system distributed resource management. The commands and flags of SGE share a lot similarity with PBS except when checking job status. Use this argument if one is submitting job to SGE based batch system.

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.2'
-__version_tuple__ = version_tuple = (0, 6, 2)
+__version__ = version = '0.6.4'
+__version_tuple__ = version_tuple = (0, 6, 4)

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/contexts/hdfs_context.py

@@ -88,7 +88,7 @@ class HDFSContext(BaseContext):
             for ff in task.forward_files:
                 abs_file_list = glob(os.path.join(local_job, ff))
                 if not abs_file_list:
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find upload file " + os.path.join(local_job, ff)
                     )
                 rel_file_list = [
@@ -100,7 +100,7 @@ class HDFSContext(BaseContext):
         for fc in submission.forward_common_files:
             abs_file_list = glob(os.path.join(local_job, fc))
             if not abs_file_list:
-                raise RuntimeError(
+                raise FileNotFoundError(
                     "cannot find upload file " + os.path.join(local_job, fc)
                 )
             rel_file_list = [
@@ -170,9 +170,11 @@ class HDFSContext(BaseContext):
                             ) as fp:
                                 pass
                         else:
-                            raise RuntimeError("do not find download file " + rfile)
+                            raise FileNotFoundError(
+                                "do not find download file " + rfile
+                            )
                     else:
-                        raise RuntimeError("do not find download file " + rfile)
+                        raise FileNotFoundError("do not find download file " + rfile)
                 else:
                     if os.path.exists(lfile):
                         dlog.info(f"find existing {lfile}, replacing by {rfile}")
@@ -203,9 +205,9 @@ class HDFSContext(BaseContext):
                         ) as fp:
                             pass
                     else:
-                        raise RuntimeError("do not find download file " + rfile)
+                        raise FileNotFoundError("do not find download file " + rfile)
                 else:
-                    raise RuntimeError("do not find download file " + rfile)
+                    raise FileNotFoundError("do not find download file " + rfile)
             else:
                 if os.path.exists(lfile):
                     dlog.info(f"find existing {lfile}, replacing by {rfile}")

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/contexts/local_context.py

@@ -94,7 +94,7 @@ class LocalContext(BaseContext):
             for kk in ii.forward_files:
                 abs_file_list = glob(os.path.join(local_job, kk))
                 if not abs_file_list:
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find upload file " + os.path.join(local_job, kk)
                     )
                 rel_file_list = [
@@ -104,7 +104,7 @@ class LocalContext(BaseContext):
 
             for jj in file_list:
                 if not os.path.exists(os.path.join(local_job, jj)):
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find upload file " + os.path.join(local_job, jj)
                     )
                 if os.path.exists(os.path.join(remote_job, jj)):
@@ -119,7 +119,7 @@ class LocalContext(BaseContext):
         for kk in submission.forward_common_files:
             abs_file_list = glob(os.path.join(local_job, kk))
             if not abs_file_list:
-                raise RuntimeError(
+                raise FileNotFoundError(
                     "cannot find upload file " + os.path.join(local_job, kk)
                 )
             rel_file_list = [
@@ -129,7 +129,7 @@ class LocalContext(BaseContext):
 
         for jj in file_list:
             if not os.path.exists(os.path.join(local_job, jj)):
-                raise RuntimeError(
+                raise FileNotFoundError(
                     "cannot find upload file " + os.path.join(local_job, jj)
                 )
             if os.path.exists(os.path.join(remote_job, jj)):
@@ -160,7 +160,7 @@ class LocalContext(BaseContext):
                         else:
                             pass
                     else:
-                        raise RuntimeError(
+                        raise FileNotFoundError(
                             "cannot find download file " + os.path.join(remote_job, kk)
                         )
                 rel_flist = [
@@ -188,7 +188,9 @@ class LocalContext(BaseContext):
                             else:
                                 pass
                         else:
-                            raise RuntimeError("do not find download file " + rfile)
+                            raise FileNotFoundError(
+                                "do not find download file " + rfile
+                            )
                     elif (not os.path.exists(rfile)) and (os.path.exists(lfile)):
                         # already downloaded
                         pass
@@ -206,7 +208,10 @@ class LocalContext(BaseContext):
                             shutil.rmtree(lfile, ignore_errors=True)
                         elif os.path.isfile(lfile) or os.path.islink(lfile):
                             os.remove(lfile)
-                        shutil.copyfile(rfile, lfile)
+                        if not os.path.islink(rfile):
+                            shutil.move(rfile, lfile)
+                        else:
+                            shutil.copyfile(rfile, lfile)
                     else:
                         raise RuntimeError("should not reach here!")
                 else:
@@ -229,7 +234,7 @@ class LocalContext(BaseContext):
                     else:
                         pass
                 else:
-                    raise RuntimeError(
+                    raise FileNotFoundError(
                         "cannot find download file " + os.path.join(remote_job, kk)
                     )
             rel_flist = [os.path.relpath(ii, start=remote_job) for ii in abs_flist_r]
@@ -255,7 +260,7 @@ class LocalContext(BaseContext):
                         else:
                             pass
                     else:
-                        raise RuntimeError("do not find download file " + rfile)
+                        raise FileNotFoundError("do not find download file " + rfile)
                 elif (not os.path.exists(rfile)) and (os.path.exists(lfile)):
                     # already downloaded
                     pass
@@ -273,7 +278,10 @@ class LocalContext(BaseContext):
                         shutil.rmtree(lfile, ignore_errors=True)
                     elif os.path.isfile(lfile) or os.path.islink(lfile):
                         os.remove(lfile)
-                    shutil.copyfile(rfile, lfile)
+                    if not os.path.islink(rfile):
+                        shutil.move(rfile, lfile)
+                    else:
+                        shutil.copyfile(rfile, lfile)
                 else:
                     raise RuntimeError("should not reach here!")
             else:

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/contexts/ssh_context.py

@@ -295,7 +295,7 @@ class SSHSession:
         assert self.ssh is not None
         try:
             return self.ssh.exec_command(cmd)
-        except (paramiko.ssh_exception.SSHException, socket.timeout) as e:
+        except (paramiko.ssh_exception.SSHException, socket.timeout, EOFError) as e:
             # SSH session not active
             # retry for up to 3 times
             # ensure alive
@@ -569,7 +569,7 @@ class SSHContext(BaseContext):
                     rel_file_list, work_path, file_list, directory_list
                 )
             else:
-                raise RuntimeError(f"cannot find upload file {work_path} {jj}")
+                raise FileNotFoundError(f"cannot find upload file {work_path} {jj}")
 
     def upload(
         self,

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/dpdisp.py

@@ -54,6 +54,11 @@ def main_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Clean submission.",
     )
+    parser_submission_action.add_argument(
+        "--reset-fail-count",
+        action="store_true",
+        help="Reset fail count of all jobs to zero.",
+    )
     ##########################################
     # gui
     parser_gui = subparsers.add_parser(
@@ -105,6 +110,7 @@ def main():
             download_terminated_log=args.download_terminated_log,
             download_finished_task=args.download_finished_task,
             clean=args.clean,
+            reset_fail_count=args.reset_fail_count,
         )
     elif args.command == "gui":
         start_dpgui(

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/entrypoints/submission.py

@@ -12,6 +12,7 @@ def handle_submission(
     download_terminated_log: bool = False,
     download_finished_task: bool = False,
     clean: bool = False,
+    reset_fail_count: bool = False,
 ):
     """Handle terminated submission.
 
@@ -25,13 +26,21 @@ def handle_submission(
         Download finished tasks.
     clean : bool, optional
         Clean submission.
+    reset_fail_count : bool, optional
+        Reset fail count of all jobs to zero.
 
     Raises
     ------
     ValueError
         At least one action should be specified.
     """
-    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+    if (
+        int(download_terminated_log)
+        + int(download_finished_task)
+        + int(clean)
+        + int(reset_fail_count)
+        == 0
+    ):
         raise ValueError("At least one action should be specified.")
 
     submission_file = record.get_submission(submission_hash)
@@ -42,7 +51,18 @@ def handle_submission(
     # TODO: for unclear reason, the submission_hash may be changed
     submission.submission_hash = submission_hash
     submission.machine.context.bind_submission(submission)
+    if reset_fail_count:
+        for job in submission.belonging_jobs:
+            job.fail_count = 0
+        # save to remote and local
+        submission.submission_to_json()
+        record.write(submission)
+    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+        # if only reset_fail_count, no need to update submission state (expensive)
+        return
     submission.update_submission_state()
+    submission.submission_to_json()
+    record.write(submission)
 
     terminated_tasks = []
     finished_tasks = []

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher/machines/pbs.py

@@ -177,3 +177,107 @@ class Torque(PBS):
                 **pbs_script_header_dict
             )
         return pbs_script_header
+
+
+sge_script_header_template = """
+#!/bin/bash
+#$ -N dpdispatcher_submit
+{select_node_line}
+#$ -cwd
+
+"""
+
+
+class SGE(PBS):
+    def __init__(
+        self,
+        batch_type=None,
+        context_type=None,
+        local_root=None,
+        remote_root=None,
+        remote_profile={},
+        *,
+        context=None,
+    ):
+        super(PBS, self).__init__(
+            batch_type,
+            context_type,
+            local_root,
+            remote_root,
+            remote_profile,
+            context=context,
+        )
+
+    def gen_script_header(self, job):
+        resources = job.resources
+        sge_script_header_dict = {}
+        # resources.number_node is not used
+        sge_script_header_dict[
+            "select_node_line"
+        ] = f"#$ -pe mpi {resources.cpu_per_node} "
+        # resources.queue_name is not necessary
+        sge_script_header = sge_script_header_template.format(**sge_script_header_dict)
+        return sge_script_header
+
+    def do_submit(self, job):
+        script_file_name = job.script_file_name
+        script_str = self.gen_script(job)
+        job_id_name = job.job_hash + "_job_id"
+        self.context.write_file(fname=script_file_name, write_str=script_str)
+        script_file_dir = self.context.remote_root
+        stdin, stdout, stderr = self.context.block_checkcall(
+            "cd {} && {} {}".format(script_file_dir, "qsub", script_file_name)
+        )
+        subret = stdout.readlines()
+        job_id = subret[0].split()[2]
+        self.context.write_file(job_id_name, job_id)
+        return job_id
+
+    def default_resources(self, resources):
+        pass
+
+    def check_status(self, job):
+        job_id = job.job_id
+        status_line = None
+        if job_id == "":
+            return JobStatus.unsubmitted
+        ret, stdin, stdout, stderr = self.context.block_call("qstat")
+        err_str = stderr.read().decode("utf-8")
+        if ret != 0:
+            raise RuntimeError(
+                "status command qstat fails to execute. erro info: %s return code %d"
+                % (err_str, ret)
+            )
+        status_text_list = stdout.read().decode("utf-8").split("\n")
+        for txt in status_text_list:
+            if job_id in txt:
+                status_line = txt
+
+        if status_line is None:
+            count = 0
+            while count <= 6:
+                if self.check_finish_tag(job=job):
+                    return JobStatus.finished
+                dlog.info(
+                    "not tag_finished detected, execute sync command and wait. count "
+                    + str(count)
+                )
+                self.context.block_call("sync")
+                import time
+
+                time.sleep(10)
+                count += 1
+            return JobStatus.terminated
+        else:
+            status_word = status_line.split()[4]
+            # dlog.info (status_word)
+            if status_word in ["qw"]:
+                return JobStatus.waiting
+            elif status_word in ["r"]:
+                return JobStatus.running
+            else:
+                return JobStatus.unknown
+
+    def check_finish_tag(self, job):
+        job_tag_finished = job.job_hash + "_job_tag_finished"
+        return self.context.check_file_exists(job_tag_finished)

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/dpdispatcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.2
+Version: 0.6.4
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/tests/test_local_context.py

@@ -69,7 +69,7 @@ class TestLocalContext(unittest.TestCase):
 
         self.local_context.bind_submission(submission)
 
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(FileNotFoundError):
             self.local_context.upload(submission)
 
     def test_upload(self):
@@ -236,7 +236,7 @@ class TestLocalContextDownload(unittest.TestCase):
             submission_hash="0_md/",
         )
         self.local_context.bind_submission(submission)
-        with self.assertRaises(RuntimeError):
+        with self.assertRaises(FileNotFoundError):
             self.local_context.download(submission, check_exists=False)
 
     def test_download_mark_failure_tag(self):

{dpdispatcher-0.6.2 → dpdispatcher-0.6.4}/tests/test_run_submission.py

@@ -3,6 +3,7 @@ import os
 import random
 import shutil
 import sys
+import tempfile
 import traceback
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
@@ -72,6 +73,16 @@ class RunSubmission:
             )
             task_list.append(task)
 
+        for ii in range(2):
+            task = Task(
+                command=f"mkdir -p out_dir{ii} && touch out_dir{ii}/out{ii}",
+                task_work_path="./",
+                forward_files=[],
+                backward_files=[f"out_dir{ii}"],
+                outlog=f"out_dir{ii}.txt",
+            )
+            task_list.append(task)
+
         # test space in file name
         task_list.append(
             Task(
@@ -90,6 +101,11 @@ class RunSubmission:
             backward_common_files=[],
             task_list=task_list,
         )
+        # test override directory
+        os.makedirs(
+            os.path.join(self.machine_dict["local_root"], "test_dir", "out_dir1"),
+            exist_ok=True,
+        )
         submission.run_submission(check_interval=2)
 
         for ii in range(4):
@@ -251,6 +267,23 @@ class TestPBSRun(RunSubmission, unittest.TestCase):
         return super().test_async_run_submission()
 
 
+@unittest.skipIf(sys.platform == "win32", "Shell is not supported on Windows")
+class TestLocalContext(RunSubmission, unittest.TestCase):
+    def setUp(self):
+        super().setUp()
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.machine_dict["context_type"] = "LocalContext"
+        self.machine_dict["remote_root"] = self.temp_dir.name
+
+    def tearDown(self):
+        super().tearDown()
+        self.temp_dir.cleanup()
+
+    @unittest.skip("It seems the remote file may be deleted")
+    def test_async_run_submission(self):
+        return super().test_async_run_submission()
+
+
 @unittest.skipIf(sys.platform == "win32", "Shell is not supported on Windows")
 class TestLazyLocalContext(RunSubmission, unittest.TestCase):
     def setUp(self):