dpdispatcher 0.5.8__py3-none-any.whl → 0.5.10__py3-none-any.whl

dpdispatcher/__init__.py

@@ -49,6 +49,8 @@ from .lazy_local_context import LazyLocalContext
 from .local_context import LocalContext
 from .lsf import LSF
 from .machine import Machine
+from .openapi import OpenAPI
+from .openapi_context import OpenAPIContext
 from .pbs import PBS, Torque
 from .shell import Shell
 from .slurm import Slurm
@@ -77,6 +79,8 @@ __all__ = [
     "__version__",
     "DistributedShell",
     "DpCloudServer",
+    "OpenAPI",
+    "OpenAPIContext",
     "DpCloudServerContext",
     "HDFSContext",
     "LazyLocalContext",

dpdispatcher/_version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.5.8'
-__version_tuple__ = version_tuple = (0, 5, 8)
+__version__ = version = '0.5.10'
+__version_tuple__ = version_tuple = (0, 5, 10)

dpdispatcher/dp_cloud_server.py

@@ -31,6 +31,13 @@ class Bohrium(Machine):
         phone = context.remote_profile.get("phone", None)
         username = context.remote_profile.get("username", None)
         password = context.remote_profile.get("password", None)
+
+        ticket = os.environ.get("BOHR_TICKET", None)
+        if ticket:
+            self.api = Client(ticket=ticket)
+            self.group_id = None
+            return
+
         if email is None and username is not None:
             raise DeprecationWarning(
                 "username is no longer support in current version, "

dpdispatcher/dp_cloud_server_context.py

@@ -21,7 +21,7 @@ DP_CLOUD_SERVER_HOME_DIR = os.path.join(
     os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/"
 )
 ENDPOINT = "http://oss-cn-shenzhen.aliyuncs.com"
-BUCKET_NAME = "dpcloudserver"
+BUCKET_NAME = os.environ.get("BUCKET_NAME", "dpcloudserver")
 
 
 class BohriumContext(BaseContext):
@@ -39,9 +39,16 @@ class BohriumContext(BaseContext):
         self.init_remote_root = remote_root
         self.temp_local_root = os.path.abspath(local_root)
         self.remote_profile = remote_profile
+        ticket = os.environ.get("BOHR_TICKET", None)
         email = remote_profile.get("email", None)
         phone = remote_profile.get("phone", None)
         password = remote_profile.get("password")
+        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
+
+        if ticket is not None:
+            self.api = Client(ticket=ticket)
+            return
+
         if email is None and phone is None:
             raise ValueError(
                 "can not find email/phone number in remote_profile, please check your machine file."
@@ -57,8 +64,6 @@ class BohriumContext(BaseContext):
 
         self.api = Client(account, password)
 
-        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
-
     @classmethod
     def load_from_dict(cls, context_dict):
         local_root = context_dict["local_root"]
@@ -256,9 +261,7 @@ class BohriumContext(BaseContext):
         return os.path.isfile(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname))
 
     def clean(self):
-        submission_file_name = "{submission_hash}.json".format(
-            submission_hash=self.submission.submission_hash
-        )
+        submission_file_name = f"{self.submission.submission_hash}.json"
         submission_json = os.path.join(DP_CLOUD_SERVER_HOME_DIR, submission_file_name)
         os.remove(submission_json)
         return True
@@ -288,7 +291,7 @@ class BohriumContext(BaseContext):
                 dict,
                 [
                     Argument("email", str, optional=True, doc="Email"),
-                    Argument("password", str, optional=False, doc="Password"),
+                    Argument("password", str, optional=True, doc="Password"),
                     Argument(
                         "program_id",
                         int,

dpdispatcher/dpcloudserver/client.py

@@ -25,7 +25,9 @@ class RequestInfoException(Exception):
 
 
 class Client:
-    def __init__(self, email=None, password=None, debug=False, base_url=API_HOST):
+    def __init__(
+        self, email=None, password=None, debug=False, ticket=None, base_url=API_HOST
+    ):
         self.debug = debug
         self.debug = os.getenv("LBG_CLI_DEBUG_PRINT", debug)
         self.config = {}
@@ -35,6 +37,7 @@ class Client:
         self.config["password"] = password
         self.base_url = base_url
         self.last_log_offset = 0
+        self.ticket = ticket
 
     def post(self, url, data=None, header=None, params=None, retry=5):
         return self._req(
@@ -51,19 +54,26 @@ class Client:
             header = {}
         if not self.token:
             self.refresh_token()
+        self.ticket = os.environ.get("BOHR_TICKET", "")
         header["Authorization"] = f"jwt {self.token}"
+        header["Brm-Ticket"] = self.ticket
         resp_code = None
         err = None
         for i in range(retry):
             resp = None
-            if method == "GET":
-                resp = requests.get(url, params=params, headers=header)
-            else:
-                if self.debug:
-                    print(data)
-                resp = requests.post(url, json=data, params=params, headers=header)
-            if self.debug:
-                print(resp.text)
+            try:
+                if method == "GET":
+                    resp = requests.get(url, params=params, headers=header)
+                else:
+                    if self.debug:
+                        print(data)
+                    resp = requests.post(url, json=data, params=params, headers=header)
+            except Exception as e:
+                dlog.error(f"request({i}) error {e}", i, stack_info=ENABLE_STACK)
+                err = e
+                time.sleep(1 * i)
+                continue
+
             resp_code = resp.status_code
             if not resp.ok:
                 if self.debug:
@@ -96,6 +106,9 @@ class Client:
         self.user_id = resp["user_id"]
 
     def refresh_token(self, retry=3):
+        self.ticket = os.environ.get("BOHR_TICKET", "")
+        if self.ticket:
+            return
         url = "/account/login"
         post_data = {"email": self.config["email"], "password": self.config["password"]}
         resp_code = None

dpdispatcher/fugaku.py

@@ -24,9 +24,7 @@ class Fugaku(Machine):
         ] = f'#PJM -L "node={resources.number_node}" '
         fugaku_script_header_dict[
             "fugaku_ntasks_per_node_line"
-        ] = '#PJM --mpi "max-proc-per-node={cpu_per_node}"'.format(
-            cpu_per_node=resources.cpu_per_node
-        )
+        ] = f'#PJM --mpi "max-proc-per-node={resources.cpu_per_node}"'
         fugaku_script_header_dict[
             "queue_name_line"
         ] = f'#PJM -L "rscgrp={resources.queue_name}"'

dpdispatcher/hdfs_cli.py

@@ -90,9 +90,7 @@ class HDFS:
             raise RuntimeError(
                 "try to access local_path[{}] " "but failed".format(local_path)
             )
-        cmd = "hadoop fs -copyFromLocal -f {local} {remote}".format(
-            local=local_path, remote=to_uri
-        )
+        cmd = f"hadoop fs -copyFromLocal -f {local_path} {to_uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:
@@ -106,9 +104,7 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                "Cannot copy local[{}] to remote[{}] with cmd[{}]".format(
-                    local_path, to_uri, cmd
-                )
+                f"Cannot copy local[{local_path}] to remote[{to_uri}] with cmd[{cmd}]"
             ) from e
 
     @staticmethod
@@ -118,9 +114,7 @@ class HDFS:
             remote = from_uri
         elif isinstance(from_uri, list) or isinstance(from_uri, tuple):
             remote = " ".join(from_uri)
-        cmd = "hadoop fs -copyToLocal {remote} {local}".format(
-            remote=remote, local=local_path
-        )
+        cmd = f"hadoop fs -copyToLocal {remote} {local_path}"
 
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
@@ -135,9 +129,7 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                "Cannot copy remote[{}] to local[{}] with cmd[{}]".format(
-                    from_uri, local_path, cmd
-                )
+                f"Cannot copy remote[{from_uri}] to local[{local_path}] with cmd[{cmd}]"
             ) from e
 
     @staticmethod

dpdispatcher/hdfs_context.py

@@ -137,10 +137,7 @@ class HDFSContext(BaseContext):
         if os.path.exists(gz_dir):
             shutil.rmtree(gz_dir, ignore_errors=True)
         os.mkdir(os.path.join(self.local_root, "tmp"))
-        rfile_tgz = "{}/{}_*_download.tar.gz".format(
-            self.remote_root,
-            submission.submission_hash,
-        )
+        rfile_tgz = f"{self.remote_root}/{submission.submission_hash}_*_download.tar.gz"
         lfile_tgz = "%s/tmp/" % (self.local_root)
         HDFS.copy_to_local(rfile_tgz, lfile_tgz)
 

dpdispatcher/lsf.py

@@ -31,12 +31,8 @@ class LSF(Machine):
             "lsf_nodes_line": "#BSUB -n {number_cores}".format(
                 number_cores=resources.number_node * resources.cpu_per_node
             ),
-            "lsf_ptile_line": "#BSUB -R 'span[ptile={cpu_per_node}]'".format(
-                cpu_per_node=resources.cpu_per_node
-            ),
-            "lsf_partition_line": "#BSUB -q {queue_name}".format(
-                queue_name=resources.queue_name
-            ),
+            "lsf_ptile_line": f"#BSUB -R 'span[ptile={resources.cpu_per_node}]'",
+            "lsf_partition_line": f"#BSUB -q {resources.queue_name}",
         }
         gpu_usage_flag = resources.kwargs.get("gpu_usage", False)
         gpu_new_syntax_flag = resources.kwargs.get("gpu_new_syntax", False)

dpdispatcher/machine.py

@@ -208,9 +208,7 @@ class Machine(metaclass=ABCMeta):
 
     def check_if_recover(self, submission):
         submission_hash = submission.submission_hash
-        submission_file_name = "{submission_hash}.json".format(
-            submission_hash=submission_hash
-        )
+        submission_file_name = f"{submission_hash}.json"
         if_recover = self.context.check_file_exists(submission_file_name)
         return if_recover
 

dpdispatcher/openapi.py

@@ -0,0 +1,198 @@
+import os
+import shutil
+import time
+
+try:
+    from bohriumsdk.client import Client
+    from bohriumsdk.job import Job
+    from bohriumsdk.storage import Storage
+    from bohriumsdk.util import Util
+except ModuleNotFoundError:
+    found_bohriumsdk = False
+else:
+    found_bohriumsdk = True
+
+from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.machine import Machine
+
+shell_script_header_template = """
+#!/bin/bash -l
+"""
+
+
+class OpenAPI(Machine):
+    def __init__(self, context):
+        if not found_bohriumsdk:
+            raise ModuleNotFoundError(
+                "bohriumsdk not installed. Install dpdispatcher with `pip install dpdispatcher[bohrium]`"
+            )
+        self.context = context
+        self.remote_profile = context.remote_profile.copy()
+
+        self.grouped = self.remote_profile.get("grouped", True)
+        self.client = Client()
+        self.job = Job(client=self.client)
+        self.storage = Storage(client=self.client)
+        self.group_id = None
+
+    def gen_script(self, job):
+        shell_script = super().gen_script(job)
+        return shell_script
+
+    def gen_script_header(self, job):
+        shell_script_header = shell_script_header_template
+        return shell_script_header
+
+    def gen_local_script(self, job):
+        script_str = self.gen_script(job)
+        script_file_name = job.script_file_name
+        self.context.write_local_file(fname=script_file_name, write_str=script_str)
+        return script_file_name
+
+    def _gen_backward_files_list(self, job):
+        result_file_list = []
+        # result_file_list.extend(job.backward_common_files)
+        for task in job.job_task_list:
+            result_file_list.extend(
+                [os.path.join(task.task_work_path, b_f) for b_f in task.backward_files]
+            )
+        result_file_list = list(set(result_file_list))
+        return result_file_list
+
+    def do_submit(self, job):
+        self.gen_local_script(job)
+
+        project_id = self.remote_profile.get("project_id", 0)
+
+        openapi_params = {
+            "oss_path": job.upload_path,
+            "input_file_type": 3,
+            "input_file_method": 1,
+            "job_type": "container",
+            "job_name": self.remote_profile.get("job_name", "DP-GEN"),
+            "project_id": project_id,
+            "scass_type": self.remote_profile.get("machine_type", ""),
+            "cmd": f"bash {job.script_file_name}",
+            "log_files": os.path.join(
+                job.job_task_list[0].task_work_path, job.job_task_list[0].outlog
+            ),
+            "out_files": self._gen_backward_files_list(job),
+            "platform": self.remote_profile.get("platform", "ali"),
+            "image_address": self.remote_profile.get("image_address", ""),
+            "job_id": job.job_id,
+        }
+
+        data = self.job.insert(**openapi_params)
+
+        job.job_id = data.get("jobId", 0)  # type: ignore
+        # self.job_group_id = data.get("jobGroupId")
+        job.job_state = JobStatus.waiting
+        return job.job_id
+
+    def _get_job_detail(self, job_id, group_id):
+        check_return = self.job.detail(job_id)
+        assert check_return is not None, (
+            f"Failed to retrieve tasks information. To resubmit this job, please "
+            f"try again, if this problem still exists please delete the submission "
+            f"file and try again.\nYou can check submission.submission_hash in the "
+            f'previous log or type `grep -rl "{job_id}:job_group_id:{group_id}" '
+            f"~/.dpdispatcher/dp_cloud_server/` to find corresponding file. "
+            f"You can try with command:\n    "
+            f'rm $(grep -rl "{job_id}:job_group_id:{group_id}" ~/.dpdispatcher/dp_cloud_server/)'
+        )
+        return check_return
+
+    def check_status(self, job):
+        if job.job_id == "":
+            return JobStatus.unsubmitted
+        job_id = job.job_id
+        group_id = None
+        if hasattr(job, "jgid"):
+            group_id = job.jgid
+        check_return = self._get_job_detail(job_id, group_id)
+        try:
+            dp_job_status = check_return["status"]  # type: ignore
+        except IndexError as e:
+            dlog.error(
+                f"cannot find job information in bohrium for job {job.job_id}. check_return:{check_return}; retry one more time after 60 seconds"
+            )
+            time.sleep(60)
+            retry_return = self._get_job_detail(job_id, group_id)
+            try:
+                dp_job_status = retry_return["status"]  # type: ignore
+            except IndexError as e:
+                raise RuntimeError(
+                    f"cannot find job information in bohrium for job {job.job_id} {check_return} {retry_return}"
+                )
+
+        job_state = self.map_dp_job_state(dp_job_status)
+        if job_state == JobStatus.finished:
+            job_log = self.job.log(job_id)
+            if self.remote_profile.get("output_log"):
+                print(job_log, end="")
+            # print(job.job_id)
+            self._download_job(job)
+        elif self.remote_profile.get("output_log") and job_state == JobStatus.running:
+            job_log = self.job.log(job_id)
+            print(job_log, end="")
+        return job_state
+
+    def _download_job(self, job):
+        data = self.job.detail(job.job_id)
+        # print(data)
+        job_url = data["jobFiles"]["outFiles"][0]["url"]  # type: ignore
+        if not job_url:
+            return
+        job_hash = job.job_hash
+        result_filename = job_hash + "_back.zip"
+        target_result_zip = os.path.join(self.context.local_root, result_filename)
+        self.storage.download_from_url(job_url, target_result_zip)
+        Util.unzip_file(target_result_zip, out_dir=self.context.local_root)
+        try:
+            os.makedirs(os.path.join(self.context.local_root, "backup"), exist_ok=True)
+            shutil.move(
+                target_result_zip,
+                os.path.join(
+                    self.context.local_root,
+                    "backup",
+                    os.path.split(target_result_zip)[1],
+                ),
+            )
+        except (OSError, shutil.Error) as e:
+            dlog.exception("unable to backup file, " + str(e))
+
+    def check_finish_tag(self, job):
+        job_tag_finished = job.job_hash + "_job_tag_finished"
+        dlog.info("check if job finished: ", job.job_id, job_tag_finished)
+        return self.context.check_file_exists(job_tag_finished)
+        # return
+        # pass
+
+    def check_if_recover(self, submission):
+        return False
+        # pass
+
+    @staticmethod
+    def map_dp_job_state(status):
+        if isinstance(status, JobStatus):
+            return status
+        map_dict = {
+            -1: JobStatus.terminated,
+            0: JobStatus.waiting,
+            1: JobStatus.running,
+            2: JobStatus.finished,
+            3: JobStatus.waiting,
+            4: JobStatus.running,
+            5: JobStatus.terminated,
+            6: JobStatus.running,
+            9: JobStatus.waiting,
+        }
+        if status not in map_dict:
+            dlog.error(f"unknown job status {status}")
+            return JobStatus.unknown
+        return map_dict[status]
+
+    # def check_finish_tag(self, job):
+    #     job_tag_finished = job.job_hash + '_job_tag_finished'
+    #     return self.context.check_file_exists(job_tag_finished)

dpdispatcher/openapi_context.py

@@ -0,0 +1,259 @@
+import os
+import shutil
+import uuid
+
+import tqdm
+
+try:
+    from bohriumsdk.client import Client
+    from bohriumsdk.job import Job
+    from bohriumsdk.storage import Storage
+    from bohriumsdk.util import Util
+except ModuleNotFoundError:
+    found_bohriumsdk = False
+else:
+    found_bohriumsdk = True
+
+from dpdispatcher import dlog
+from dpdispatcher.base_context import BaseContext
+from dpdispatcher.JobStatus import JobStatus
+
+DP_CLOUD_SERVER_HOME_DIR = os.path.join(
+    os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/"
+)
+
+
+class OpenAPIContext(BaseContext):
+    def __init__(
+        self,
+        local_root,
+        remote_root=None,
+        remote_profile={},
+        *args,
+        **kwargs,
+    ):
+        if not found_bohriumsdk:
+            raise ModuleNotFoundError(
+                "bohriumsdk not installed. Install dpdispatcher with `pip install dpdispatcher[bohrium]`"
+            )
+        self.init_local_root = local_root
+        self.init_remote_root = remote_root
+        self.temp_local_root = os.path.abspath(local_root)
+        self.remote_profile = remote_profile
+        self.client = Client()
+        self.storage = Storage(client=self.client)
+        self.job = Job(client=self.client)
+        self.util = Util()
+        self.jgid = None
+
+    @classmethod
+    def load_from_dict(cls, context_dict):
+        local_root = context_dict.get("local_root", "./")
+        remote_root = context_dict.get("remote_root", None)
+        remote_profile = context_dict.get("remote_profile", {})
+
+        bohrium_context = cls(
+            local_root=local_root,
+            remote_root=remote_root,
+            remote_profile=remote_profile,
+        )
+        return bohrium_context
+
+    def bind_submission(self, submission):
+        self.submission = submission
+        self.local_root = os.path.join(self.temp_local_root, submission.work_base)
+        self.remote_root = "."
+
+        self.submission_hash = submission.submission_hash
+
+        self.machine = submission.machine
+
+    def _gen_object_key(self, job, zip_filename):
+        if hasattr(job, "upload_path") and job.upload_path:
+            return job.upload_path
+        else:
+            project_id = self.remote_profile.get("project_id")
+
+            uid = uuid.uuid4()
+            path = os.path.join(str(project_id), str(uid), zip_filename)
+            setattr(job, "upload_path", path)
+            return path
+
+    def upload_job(self, job, common_files=None):
+        if common_files is None:
+            common_files = []
+        self.machine.gen_local_script(job)
+        zip_filename = job.job_hash + ".zip"
+        zip_task_file = os.path.join(self.local_root, zip_filename)
+
+        upload_file_list = [
+            job.script_file_name,
+        ]
+
+        upload_file_list.extend(common_files)
+
+        for task in job.job_task_list:
+            for file in task.forward_files:
+                upload_file_list.append(os.path.join(task.task_work_path, file))
+
+        upload_zip = Util.zip_file_list(
+            self.local_root, zip_task_file, file_list=upload_file_list
+        )
+        project_id = self.remote_profile.get("project_id", 0)
+
+        data = self.job.create(
+            project_id=project_id,
+            name=self.remote_profile.get("job_name", "DP-GEN"),
+            group_id=self.jgid,  # type: ignore
+        )
+        self.jgid = data.get("jobGroupId", "")  # type: ignore
+        token = data.get("token", "")  # type: ignore
+
+        object_key = os.path.join(data["storePath"], zip_filename)  # type: ignore
+        job.upload_path = object_key
+        job.job_id = data["jobId"]  # type: ignore
+        job.jgid = data["jobGroupId"]  # type: ignore
+        self.storage.upload_From_file_multi_part(
+            object_key=object_key, file_path=upload_zip, token=token
+        )
+
+        # self._backup(self.local_root, upload_zip)
+
+    def upload(self, submission):
+        # oss_task_dir = os.path.join('%s/%s/%s.zip' % ('indicate', file_uuid, file_uuid))
+        # zip_filename = submission.submission_hash + '.zip'
+        # oss_task_zip = 'indicate/' + submission.submission_hash + '/' + zip_filename
+
+        # zip_path = "/home/felix/workplace/22_dpdispatcher/dpdispatcher-yfb/dpdispatcher/dpcloudserver/t.txt"
+        # zip_path = self.local_root
+        bar_format = "{l_bar}{bar}| {n:.02f}/{total:.02f} %  [{elapsed}<{remaining}, {rate_fmt}{postfix}]"
+        job_to_be_uploaded = []
+        result = None
+        dlog.info("checking all job has been uploaded")
+        for job in submission.belonging_jobs:
+            if job.job_state == JobStatus.unsubmitted:
+                job_to_be_uploaded.append(job)
+        if len(job_to_be_uploaded) == 0:
+            dlog.info("all job has been uploaded, continue")
+            return result
+        for job in tqdm.tqdm(
+            job_to_be_uploaded,
+            desc="Uploading to tiefblue",
+            bar_format=bar_format,
+            leave=False,
+            disable=None,
+        ):
+            self.upload_job(job, submission.forward_common_files)
+        return result
+        # return oss_task_zip
+        # api.upload(self.oss_task_dir, zip_task_file)
+
+    def download(self, submission):
+        jobs = submission.belonging_jobs
+        job_hashs = {}
+        job_infos = {}
+        job_result = []
+        for job in jobs:
+            jid = job.job_id
+            job_hashs[jid] = job.job_hash
+            jobinfo = self.job.detail(jid)
+            # jobinfo = self.api.get_job_detail(jid)
+            job_result.append(jobinfo)
+        # if group_id is not None:
+        #     job_result = self.api.get_tasks_list(group_id)
+        for each in job_result:
+            if "resultUrl" in each and each["resultUrl"] != "" and each["status"] == 2:
+                job_hash = ""
+                if each["id"] not in job_hashs:
+                    dlog.info(
+                        f"find unexpect job_hash, but task {each['id']} still been download."
+                    )
+                    dlog.debug(str(job_hashs))
+                    job_hash = str(each["id"])
+                else:
+                    job_hash = job_hashs[each["id"]]
+                job_infos[job_hash] = each
+        bar_format = "{l_bar}{bar}| {n:.02f}/{total:.02f} %  [{elapsed}<{remaining}, {rate_fmt}{postfix}]"
+        for job_hash, info in tqdm.tqdm(
+            job_infos.items(),
+            desc="Validating download file from Lebesgue",
+            bar_format=bar_format,
+            leave=False,
+            disable=None,
+        ):
+            result_filename = job_hash + "_back.zip"
+            target_result_zip = os.path.join(self.local_root, result_filename)
+            if self._check_if_job_has_already_downloaded(
+                target_result_zip, self.local_root
+            ):
+                continue
+            self.storage.download_from_url(info["resultUrl"], target_result_zip)
+            Util.unzip_file(target_result_zip, out_dir=self.local_root)
+            self._backup(self.local_root, target_result_zip)
+        self._clean_backup(
+            self.local_root, keep_backup=self.remote_profile.get("keep_backup", True)
+        )
+        return True
+
+    def write_file(self, fname, write_str):
+        result = self.write_home_file(fname, write_str)
+        return result
+
+    def write_local_file(self, fname, write_str):
+        local_filename = os.path.join(self.local_root, fname)
+        with open(local_filename, "w") as f:
+            f.write(write_str)
+        return local_filename
+
+    def read_file(self, fname):
+        result = self.read_home_file(fname)
+        return result
+
+    def write_home_file(self, fname, write_str):
+        # os.makedirs(self.remote_root, exist_ok = True)
+        with open(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname), "w") as fp:
+            fp.write(write_str)
+        return True
+
+    def read_home_file(self, fname):
+        with open(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname)) as fp:
+            ret = fp.read()
+        return ret
+
+    def check_file_exists(self, fname):
+        result = self.check_home_file_exits(fname)
+        return result
+
+    def check_home_file_exits(self, fname):
+        return os.path.isfile(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname))
+
+    def clean(self):
+        submission_file_name = f"{self.submission.submission_hash}.json"
+        submission_json = os.path.join(DP_CLOUD_SERVER_HOME_DIR, submission_file_name)
+        os.remove(submission_json)
+        return True
+
+    def _check_if_job_has_already_downloaded(self, target, local_root):
+        backup_file_location = os.path.join(
+            local_root, "backup", os.path.split(target)[1]
+        )
+        if os.path.exists(backup_file_location):
+            return True
+        else:
+            return False
+
+    def _backup(self, local_root, target):
+        try:
+            # move to backup directory
+            os.makedirs(os.path.join(local_root, "backup"), exist_ok=True)
+            shutil.move(
+                target, os.path.join(local_root, "backup", os.path.split(target)[1])
+            )
+        except (OSError, shutil.Error) as e:
+            dlog.exception("unable to backup file, " + str(e))
+
+    def _clean_backup(self, local_root, keep_backup=True):
+        if not keep_backup:
+            dir_to_be_removed = os.path.join(local_root, "backup")
+            if os.path.exists(dir_to_be_removed):
+                shutil.rmtree(dir_to_be_removed)

dpdispatcher/pbs.py

@@ -22,16 +22,12 @@ class PBS(Machine):
         pbs_script_header_dict = {}
         pbs_script_header_dict[
             "select_node_line"
-        ] = "#PBS -l select={number_node}:ncpus={cpu_per_node}".format(
-            number_node=resources.number_node, cpu_per_node=resources.cpu_per_node
-        )
+        ] = f"#PBS -l select={resources.number_node}:ncpus={resources.cpu_per_node}"
         if resources.gpu_per_node != 0:
             pbs_script_header_dict[
                 "select_node_line"
             ] += f":ngpus={resources.gpu_per_node}"
-        pbs_script_header_dict["queue_name_line"] = "#PBS -q {queue_name}".format(
-            queue_name=resources.queue_name
-        )
+        pbs_script_header_dict["queue_name_line"] = f"#PBS -q {resources.queue_name}"
         pbs_script_header = pbs_script_header_template.format(**pbs_script_header_dict)
         return pbs_script_header
 
@@ -147,15 +143,11 @@ class Torque(PBS):
         pbs_script_header_dict = {}
         pbs_script_header_dict[
             "select_node_line"
-        ] = "#PBS -l nodes={number_node}:ppn={cpu_per_node}".format(
-            number_node=resources.number_node, cpu_per_node=resources.cpu_per_node
-        )
+        ] = f"#PBS -l nodes={resources.number_node}:ppn={resources.cpu_per_node}"
         if resources.gpu_per_node != 0:
             pbs_script_header_dict["select_node_line"] += ":gpus={gpu_per_node}".format(
                 gpu_per_node=resources.gpu_per_node
             )
-        pbs_script_header_dict["queue_name_line"] = "#PBS -q {queue_name}".format(
-            queue_name=resources.queue_name
-        )
+        pbs_script_header_dict["queue_name_line"] = f"#PBS -q {resources.queue_name}"
         pbs_script_header = pbs_script_header_template.format(**pbs_script_header_dict)
         return pbs_script_header

dpdispatcher/slurm.py

@@ -34,16 +34,12 @@ class Slurm(Machine):
         )
         script_header_dict[
             "slurm_ntasks_per_node_line"
-        ] = "#SBATCH --ntasks-per-node {cpu_per_node}".format(
-            cpu_per_node=resources.cpu_per_node
-        )
+        ] = f"#SBATCH --ntasks-per-node {resources.cpu_per_node}"
         custom_gpu_line = resources.kwargs.get("custom_gpu_line", None)
         if not custom_gpu_line:
             script_header_dict[
                 "slurm_number_gpu_line"
-            ] = "#SBATCH --gres=gpu:{gpu_per_node}".format(
-                gpu_per_node=resources.gpu_per_node
-            )
+            ] = f"#SBATCH --gres=gpu:{resources.gpu_per_node}"
         else:
             script_header_dict["slurm_number_gpu_line"] = custom_gpu_line
         if resources.queue_name != "":

dpdispatcher/submission.py

@@ -266,7 +266,7 @@ class Submission:
 
     def try_download_result(self):
         start_time = time.time()
-        retry_interval = 60  # 每1分钟重试一次
+        retry_interval = 60  # retry every 1 minute
         success = False
         while not success:
             try:
@@ -275,14 +275,14 @@ class Submission:
             except (EOFError, Exception) as e:
                 dlog.exception(e)
                 elapsed_time = time.time() - start_time
-                if elapsed_time < 3600:  # 1小时内
+                if elapsed_time < 3600:  # in 1 h
                     dlog.info("Retrying in 1 minute...")
                     time.sleep(retry_interval)
-                elif elapsed_time < 86400:  # 1小时后，但在24小时内
-                    retry_interval = 600  # 每10分钟重试一次
+                elif elapsed_time < 86400:  # 1 h ~ 24 h
+                    retry_interval = 600  # retry every 10 min
                     dlog.info("Retrying in 10 minutes...")
                     time.sleep(retry_interval)
-                else:  # 超过24小时
+                else:  # > 24 h
                     dlog.info("Maximum retries time reached. Exiting.")
                     break
 
@@ -509,9 +509,7 @@ class Submission:
     def submission_to_json(self):
         # self.update_submission_state()
         write_str = json.dumps(self.serialize(), indent=4, default=str)
-        submission_file_name = "{submission_hash}.json".format(
-            submission_hash=self.submission_hash
-        )
+        submission_file_name = f"{self.submission_hash}.json"
         self.machine.context.write_file(submission_file_name, write_str=write_str)
 
     @classmethod
@@ -525,9 +523,7 @@ class Submission:
     # def check_if_recover()
 
     def try_recover_from_json(self):
-        submission_file_name = "{submission_hash}.json".format(
-            submission_hash=self.submission_hash
-        )
+        submission_file_name = f"{self.submission_hash}.json"
         if_recover = self.machine.context.check_file_exists(submission_file_name)
         submission = None
         submission_dict = {}
@@ -787,9 +783,7 @@ class Job:
         """
         if len(job_dict.keys()) != 1:
             raise RuntimeError(
-                "json file may be broken, len(job_dict.keys()) must be 1. {job_dict}".format(
-                    job_dict=job_dict
-                )
+                f"json file may be broken, len(job_dict.keys()) must be 1. {job_dict}"
             )
         job_hash = list(job_dict.keys())[0]
 
@@ -871,11 +865,7 @@ class Job:
             #     raise RuntimeError("job:job {job} failed 3 times".format(job=self))
             self.submit_job()
             if self.job_state != JobStatus.unsubmitted:
-                dlog.info(
-                    "job: {job_hash} submit; job_id is {job_id}".format(
-                        job_hash=self.job_hash, job_id=self.job_id
-                    )
-                )
+                dlog.info(f"job: {self.job_hash} submit; job_id is {self.job_id}")
             if self.resources.wait_time != 0:
                 time.sleep(self.resources.wait_time)
             # self.get_job_state()

{dpdispatcher-0.5.8.dist-info → dpdispatcher-0.5.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.5.8
+Version: 0.5.10
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License: GNU LESSER GENERAL PUBLIC LICENSE
@@ -190,9 +190,14 @@ Requires-Dist: dargs (>=0.2.9)
 Requires-Dist: requests
 Requires-Dist: tqdm (>=4.9.0)
 Requires-Dist: typing-extensions ; python_version < "3.7"
+Provides-Extra: bohrium
+Requires-Dist: oss2 ; extra == 'bohrium'
+Requires-Dist: tqdm ; extra == 'bohrium'
+Requires-Dist: bohrium-sdk ; extra == 'bohrium'
 Provides-Extra: cloudserver
 Requires-Dist: oss2 ; extra == 'cloudserver'
 Requires-Dist: tqdm ; extra == 'cloudserver'
+Requires-Dist: bohrium-sdk ; extra == 'cloudserver'
 Provides-Extra: docs
 Requires-Dist: sphinx ; extra == 'docs'
 Requires-Dist: myst-parser ; extra == 'docs'
@@ -223,6 +228,12 @@ DPDispatcher can be installed by `pip`:
 pip install dpdispatcher
 ```
 
+To add [Bohrium](https://bohrium.dp.tech/) support, execute
+
+```bash
+pip install dpdispatcher[bohrium]
+```
+
 ## Usage
 
 See [Getting Started](https://dpdispatcher.readthedocs.io/en/latest/getting-started.html) for usage.

dpdispatcher-0.5.10.dist-info/RECORD

@@ -0,0 +1,36 @@
+dpdispatcher/JobStatus.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
+dpdispatcher/__init__.py,sha256=i33piTZkPDYIm_qiTEI_J3bmHlFayEN1T8T8TyzYdqg,3017
+dpdispatcher/_version.py,sha256=RvtoZGalK6X9Bv5tbHI4epbrqfU-LImzqZ_rqfMFZHA,162
+dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
+dpdispatcher/base_context.py,sha256=Hfri0x41XC4MRUjxc0-WMiZB_E4NvLp94ZYaHfYCWHM,3610
+dpdispatcher/distributed_shell.py,sha256=XMcXt8g1f2DY5HYhhyiN5ehV2ihKULY5ng-sB0B7YaI,6933
+dpdispatcher/dp_cloud_server.py,sha256=dOoQk7GaUglnqmVFefuElLPjP_3pCw4asHz8PEUrpSw,10028
+dpdispatcher/dp_cloud_server_context.py,sha256=h_IBVyQ1Ddm70rSDd6AlWqTckqdd6oOCRZZXyGV23K4,11373
+dpdispatcher/dpdisp.py,sha256=_dyH8xEgUR-s2xKkB20D9FIYhSHUCmzc2PxWgo9ildQ,94
+dpdispatcher/fugaku.py,sha256=Nwj3tDMuv29wMZdZcQf7U9EkeoKxrdu0FZg_KuODgX8,3678
+dpdispatcher/hdfs_cli.py,sha256=eBU-1woqmFqVjJqhsDWLRHnU6YrYTptOgsexs9Iy-l4,5835
+dpdispatcher/hdfs_context.py,sha256=uuhDNudDf_7CZWN_1bY4jNJoKL7n8mNZwOWu7N57KwI,8893
+dpdispatcher/lazy_local_context.py,sha256=ZdWNqK3QF8SsoqnCjpFt3ZDRCIagjzJNlKPUYutRUC8,5692
+dpdispatcher/local_context.py,sha256=anYJqQASOnkcAhfckUcFD8_DcjNUZ1KE0GuksxR5Mxw,11772
+dpdispatcher/lsf.py,sha256=uBFWs25H1GZXCzben5ATstoGE8fayWVs1IsV9MkoMyo,7668
+dpdispatcher/machine.py,sha256=ogeQHJfslH4JWc9Mm5SW-Yo7mvbxoeBZQI9aMuvphkE,15235
+dpdispatcher/openapi.py,sha256=CGfYycov13EaBRJgz4cd-ezx8KLXNfdaOOIl8l4PtVc,7386
+dpdispatcher/openapi_context.py,sha256=8ji_ztA-wZdKpqOroXCWddu9FdbojCUM2h2t_dZpI8E,9420
+dpdispatcher/pbs.py,sha256=HGqoFympIpNNiAmtBZEKzsmVtt8-O3Xxp2rjYAUNMuM,6065
+dpdispatcher/shell.py,sha256=kEP7za-qN71y_21p0uBNkopZ5s63Adq54904hjUHv48,4141
+dpdispatcher/slurm.py,sha256=bNvFL6lCU5YveTGuSLbDdk-9jbDXyWbB-OBRFhWnfZg,14530
+dpdispatcher/ssh_context.py,sha256=7Xrm8biVA7tAEDJ6YJZzC3nbdQrVBr_5UOhQNQ7qJ2g,35032
+dpdispatcher/submission.py,sha256=GLbEfvcejQCzlK8V2U-tsZ4K3HrFffWDl9mIyPoHdgY,46295
+dpdispatcher/utils.py,sha256=RXUHJl3S2z26Em3SeltnxtdVM3kv7weXJKvBEjG6I34,5035
+dpdispatcher/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
+dpdispatcher/dpcloudserver/client.py,sha256=WaNYyuiBvzylyUlGWMNsKVKjnyfQmUn7Qi5kgLMiVfM,11588
+dpdispatcher/dpcloudserver/config.py,sha256=vBRtzExJXTGfXPeBObXrZNAhBNXoFFzMkzSuSrrjHEQ,635
+dpdispatcher/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
+dpdispatcher/dpcloudserver/temp_test.py,sha256=jklOSu7tZ_wW5gycGRiUsbBWMLZDqCBslSYOCb2hTHw,2932
+dpdispatcher/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
+dpdispatcher-0.5.10.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
+dpdispatcher-0.5.10.dist-info/METADATA,sha256=9-GohvpxCFjtHxunG_nTBh6FMLyy7_j40xs_239pKVE,12595
+dpdispatcher-0.5.10.dist-info/WHEEL,sha256=AtBG6SXL3KF_v0NxLf0ehyVOh0cold-JbJYXNGorC6Q,92
+dpdispatcher-0.5.10.dist-info/entry_points.txt,sha256=3bKn6IB6SYhKOUbbcOdBBevz4gsDmhmbogKMVn4ptOQ,52
+dpdispatcher-0.5.10.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
+dpdispatcher-0.5.10.dist-info/RECORD,,

{dpdispatcher-0.5.8.dist-info → dpdispatcher-0.5.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.40.0)
+Generator: bdist_wheel (0.41.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dpdispatcher-0.5.8.dist-info/RECORD

@@ -1,34 +0,0 @@
-dpdispatcher/JobStatus.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
-dpdispatcher/__init__.py,sha256=2GIz4niyzHTbxros1G7Mi4uBJbD3AMSnTPxXSJMJmUs,2907
-dpdispatcher/_version.py,sha256=iqWtoISytDDNpYe-atC8Kl-rZhTojPnDQKAEcFNtIhg,160
-dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
-dpdispatcher/base_context.py,sha256=Hfri0x41XC4MRUjxc0-WMiZB_E4NvLp94ZYaHfYCWHM,3610
-dpdispatcher/distributed_shell.py,sha256=XMcXt8g1f2DY5HYhhyiN5ehV2ihKULY5ng-sB0B7YaI,6933
-dpdispatcher/dp_cloud_server.py,sha256=xVpDI0exBwHNSZECLJdfrQsvBzeUn5a0gx5Bzt9UAdU,9857
-dpdispatcher/dp_cloud_server_context.py,sha256=VfRRo4ruorWC8NVjW19EjmxQ0Rbz6XzxrHrJKl4cCZk,11255
-dpdispatcher/dpdisp.py,sha256=_dyH8xEgUR-s2xKkB20D9FIYhSHUCmzc2PxWgo9ildQ,94
-dpdispatcher/fugaku.py,sha256=wSjY0XB3TNNWAPKHgMpoPl5jyYJIlijBcEkYXp6nrZQ,3733
-dpdispatcher/hdfs_cli.py,sha256=9Vrf7Kz_kJgXP2xEdZqNVNxRGbui5RrtnLtEjxfcq9A,6047
-dpdispatcher/hdfs_context.py,sha256=1jT1nzx7VGJFJ42MHTXoFWhfEu4KBkMBJO84klRAnPI,8938
-dpdispatcher/lazy_local_context.py,sha256=ZdWNqK3QF8SsoqnCjpFt3ZDRCIagjzJNlKPUYutRUC8,5692
-dpdispatcher/local_context.py,sha256=anYJqQASOnkcAhfckUcFD8_DcjNUZ1KE0GuksxR5Mxw,11772
-dpdispatcher/lsf.py,sha256=zy-WEnC7f2Dy5hJGnRBl5jpjYZ_H3-KMcE0lxDG6ejo,7790
-dpdispatcher/machine.py,sha256=31xG5ksN8mBVwD8taLsk5KXLhjM0ZTjlHlbbPgiig1c,15296
-dpdispatcher/pbs.py,sha256=LiULEKNDuisrKmOpZyB1af6sGDQ35xrAhMh7VMwpFbY,6327
-dpdispatcher/shell.py,sha256=kEP7za-qN71y_21p0uBNkopZ5s63Adq54904hjUHv48,4141
-dpdispatcher/slurm.py,sha256=krlyjzxK8gIhSsqcKHFvNiUwVE7411wTUwuW9xGzS-E,14648
-dpdispatcher/ssh_context.py,sha256=7Xrm8biVA7tAEDJ6YJZzC3nbdQrVBr_5UOhQNQ7qJ2g,35032
-dpdispatcher/submission.py,sha256=r_F05nHTpN86b2os8RZAjZsCILNarDko2BjAEUYSntw,46643
-dpdispatcher/utils.py,sha256=RXUHJl3S2z26Em3SeltnxtdVM3kv7weXJKvBEjG6I34,5035
-dpdispatcher/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
-dpdispatcher/dpcloudserver/client.py,sha256=w1wQ8g-FMQlyh00LIAbJLE1xirGXocpp7zAnhbeM4V0,11152
-dpdispatcher/dpcloudserver/config.py,sha256=vBRtzExJXTGfXPeBObXrZNAhBNXoFFzMkzSuSrrjHEQ,635
-dpdispatcher/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
-dpdispatcher/dpcloudserver/temp_test.py,sha256=jklOSu7tZ_wW5gycGRiUsbBWMLZDqCBslSYOCb2hTHw,2932
-dpdispatcher/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
-dpdispatcher-0.5.8.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
-dpdispatcher-0.5.8.dist-info/METADATA,sha256=o2oD8_6Ohc04mRTkJWi51-KOPamYqH0kvUD-E0iW-c0,12280
-dpdispatcher-0.5.8.dist-info/WHEEL,sha256=pkctZYzUS4AYVn6dJ-7367OJZivF2e8RA9b_ZBjif18,92
-dpdispatcher-0.5.8.dist-info/entry_points.txt,sha256=3bKn6IB6SYhKOUbbcOdBBevz4gsDmhmbogKMVn4ptOQ,52
-dpdispatcher-0.5.8.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
-dpdispatcher-0.5.8.dist-info/RECORD,,