dpdispatcher 0.5.8__tar.gz → 0.5.10__tar.gz

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/.pre-commit-config.yaml

@@ -17,12 +17,12 @@ repos:
     -   id: check-toml
 # Python
 -   repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.7.0
     hooks:
     -   id: black-jupyter
--   repo: https://github.com/charliermarsh/ruff-pre-commit
+-   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.0.275
+    rev: v0.0.278
     hooks:
     - id: ruff
       args: ["--fix"]
@@ -34,6 +34,6 @@ repos:
       args: ["--write"]
 # Python inside docs
 -   repo: https://github.com/asottile/blacken-docs
-    rev: 1.14.0
+    rev: 1.15.0
     hooks:
     -   id: blacken-docs

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/Dockerfile

@@ -2,4 +2,4 @@ FROM python:3.11
 
 WORKDIR /data/dpdispatcher
 COPY ./ ./
-RUN pip install .[cloudserver]
+RUN pip install .[bohrium]

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.5.8
+Version: 0.5.10
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE
@@ -186,6 +186,7 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 Provides-Extra: docs
 Provides-Extra: cloudserver
+Provides-Extra: bohrium
 Provides-Extra: test
 License-File: LICENSE
 
@@ -210,6 +211,12 @@ DPDispatcher can be installed by `pip`:
 pip install dpdispatcher
 ```
 
+To add [Bohrium](https://bohrium.dp.tech/) support, execute
+
+```bash
+pip install dpdispatcher[bohrium]
+```
+
 ## Usage
 
 See [Getting Started](https://dpdispatcher.readthedocs.io/en/latest/getting-started.html) for usage.

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/README.md

@@ -19,6 +19,12 @@ DPDispatcher can be installed by `pip`:
 pip install dpdispatcher
 ```
 
+To add [Bohrium](https://bohrium.dp.tech/) support, execute
+
+```bash
+pip install dpdispatcher[bohrium]
+```
+
 ## Usage
 
 See [Getting Started](https://dpdispatcher.readthedocs.io/en/latest/getting-started.html) for usage.

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/doc/batch.md

@@ -70,3 +70,9 @@ Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
 [Fujitsu cloud service](https://doc.cloud.global.fujitsu.com/lib/common/jp/hpc-user-manual/) is a job scheduling system used by Fujitsu's HPCs such as Fugaku, ITO and K computer. It should be noted that although the same job scheduling system is used, there are some differences in the details, Fagaku class cannot be directly used for other HPCs.
 
 Read Fujitsu cloud service documentation for details.
+
+
+## OpenAPI
+
+{dargs:argument}`batcy_type <resources/batch_type>`: `OpenAPI`
+OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/doc/context.md

@@ -42,3 +42,11 @@ To use Bohrium, one needs to provide necessary parameters in {dargs:argument}`re
 
 The Hadoop Distributed File System (HDFS) is a distributed file system.
 Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
+
+
+## OpenAPI
+
+{dargs:argument}`context_type <machine/context_type>`: `OpenAPI`
+
+OpenAPI is a new way to submit jobs to Bohrium. It using [AccessKey](https://bohrium.dp.tech/personal/setting) instead of username and password. Read Bohrium documentation for details.
+To use OpenAPI, one needs to provide necessary parameters in {dargs:argument}`remote_profile <machine[OpenAPIContext]/remote_profile>`.

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/doc/install.md

@@ -6,3 +6,9 @@ DPDispatcher can installed by `pip`:
 ```bash
 pip install dpdispatcher
 ```
+
+To add [Bohrium](https://bohrium.dp.tech/) support, execute
+
+```bash
+pip install dpdispatcher[bohrium]
+```

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/__init__.py

@@ -49,6 +49,8 @@ from .lazy_local_context import LazyLocalContext
 from .local_context import LocalContext
 from .lsf import LSF
 from .machine import Machine
+from .openapi import OpenAPI
+from .openapi_context import OpenAPIContext
 from .pbs import PBS, Torque
 from .shell import Shell
 from .slurm import Slurm
@@ -77,6 +79,8 @@ __all__ = [
     "__version__",
     "DistributedShell",
     "DpCloudServer",
+    "OpenAPI",
+    "OpenAPIContext",
     "DpCloudServerContext",
     "HDFSContext",
     "LazyLocalContext",

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/_version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.5.8'
-__version_tuple__ = version_tuple = (0, 5, 8)
+__version__ = version = '0.5.10'
+__version_tuple__ = version_tuple = (0, 5, 10)

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/dp_cloud_server.py

@@ -31,6 +31,13 @@ class Bohrium(Machine):
         phone = context.remote_profile.get("phone", None)
         username = context.remote_profile.get("username", None)
         password = context.remote_profile.get("password", None)
+
+        ticket = os.environ.get("BOHR_TICKET", None)
+        if ticket:
+            self.api = Client(ticket=ticket)
+            self.group_id = None
+            return
+
         if email is None and username is not None:
             raise DeprecationWarning(
                 "username is no longer support in current version, "

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/dp_cloud_server_context.py

@@ -21,7 +21,7 @@ DP_CLOUD_SERVER_HOME_DIR = os.path.join(
     os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/"
 )
 ENDPOINT = "http://oss-cn-shenzhen.aliyuncs.com"
-BUCKET_NAME = "dpcloudserver"
+BUCKET_NAME = os.environ.get("BUCKET_NAME", "dpcloudserver")
 
 
 class BohriumContext(BaseContext):
@@ -39,9 +39,16 @@ class BohriumContext(BaseContext):
         self.init_remote_root = remote_root
         self.temp_local_root = os.path.abspath(local_root)
         self.remote_profile = remote_profile
+        ticket = os.environ.get("BOHR_TICKET", None)
         email = remote_profile.get("email", None)
         phone = remote_profile.get("phone", None)
         password = remote_profile.get("password")
+        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
+
+        if ticket is not None:
+            self.api = Client(ticket=ticket)
+            return
+
         if email is None and phone is None:
             raise ValueError(
                 "can not find email/phone number in remote_profile, please check your machine file."
@@ -57,8 +64,6 @@ class BohriumContext(BaseContext):
 
         self.api = Client(account, password)
 
-        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
-
     @classmethod
     def load_from_dict(cls, context_dict):
         local_root = context_dict["local_root"]
@@ -256,9 +261,7 @@ class BohriumContext(BaseContext):
         return os.path.isfile(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname))
 
     def clean(self):
-        submission_file_name = "{submission_hash}.json".format(
-            submission_hash=self.submission.submission_hash
-        )
+        submission_file_name = f"{self.submission.submission_hash}.json"
         submission_json = os.path.join(DP_CLOUD_SERVER_HOME_DIR, submission_file_name)
         os.remove(submission_json)
         return True
@@ -288,7 +291,7 @@ class BohriumContext(BaseContext):
                 dict,
                 [
                     Argument("email", str, optional=True, doc="Email"),
-                    Argument("password", str, optional=False, doc="Password"),
+                    Argument("password", str, optional=True, doc="Password"),
                     Argument(
                         "program_id",
                         int,

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/dpcloudserver/client.py

@@ -25,7 +25,9 @@ class RequestInfoException(Exception):
 
 
 class Client:
-    def __init__(self, email=None, password=None, debug=False, base_url=API_HOST):
+    def __init__(
+        self, email=None, password=None, debug=False, ticket=None, base_url=API_HOST
+    ):
         self.debug = debug
         self.debug = os.getenv("LBG_CLI_DEBUG_PRINT", debug)
         self.config = {}
@@ -35,6 +37,7 @@ class Client:
         self.config["password"] = password
         self.base_url = base_url
         self.last_log_offset = 0
+        self.ticket = ticket
 
     def post(self, url, data=None, header=None, params=None, retry=5):
         return self._req(
@@ -51,19 +54,26 @@ class Client:
             header = {}
         if not self.token:
             self.refresh_token()
+        self.ticket = os.environ.get("BOHR_TICKET", "")
         header["Authorization"] = f"jwt {self.token}"
+        header["Brm-Ticket"] = self.ticket
         resp_code = None
         err = None
         for i in range(retry):
             resp = None
-            if method == "GET":
-                resp = requests.get(url, params=params, headers=header)
-            else:
-                if self.debug:
-                    print(data)
-                resp = requests.post(url, json=data, params=params, headers=header)
-            if self.debug:
-                print(resp.text)
+            try:
+                if method == "GET":
+                    resp = requests.get(url, params=params, headers=header)
+                else:
+                    if self.debug:
+                        print(data)
+                    resp = requests.post(url, json=data, params=params, headers=header)
+            except Exception as e:
+                dlog.error(f"request({i}) error {e}", i, stack_info=ENABLE_STACK)
+                err = e
+                time.sleep(1 * i)
+                continue
+
             resp_code = resp.status_code
             if not resp.ok:
                 if self.debug:
@@ -96,6 +106,9 @@ class Client:
         self.user_id = resp["user_id"]
 
     def refresh_token(self, retry=3):
+        self.ticket = os.environ.get("BOHR_TICKET", "")
+        if self.ticket:
+            return
         url = "/account/login"
         post_data = {"email": self.config["email"], "password": self.config["password"]}
         resp_code = None

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/fugaku.py

@@ -24,9 +24,7 @@ class Fugaku(Machine):
         ] = f'#PJM -L "node={resources.number_node}" '
         fugaku_script_header_dict[
             "fugaku_ntasks_per_node_line"
-        ] = '#PJM --mpi "max-proc-per-node={cpu_per_node}"'.format(
-            cpu_per_node=resources.cpu_per_node
-        )
+        ] = f'#PJM --mpi "max-proc-per-node={resources.cpu_per_node}"'
         fugaku_script_header_dict[
             "queue_name_line"
         ] = f'#PJM -L "rscgrp={resources.queue_name}"'

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/hdfs_cli.py

@@ -90,9 +90,7 @@ class HDFS:
             raise RuntimeError(
                 "try to access local_path[{}] " "but failed".format(local_path)
             )
-        cmd = "hadoop fs -copyFromLocal -f {local} {remote}".format(
-            local=local_path, remote=to_uri
-        )
+        cmd = f"hadoop fs -copyFromLocal -f {local_path} {to_uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:
@@ -106,9 +104,7 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                "Cannot copy local[{}] to remote[{}] with cmd[{}]".format(
-                    local_path, to_uri, cmd
-                )
+                f"Cannot copy local[{local_path}] to remote[{to_uri}] with cmd[{cmd}]"
             ) from e
 
     @staticmethod
@@ -118,9 +114,7 @@ class HDFS:
             remote = from_uri
         elif isinstance(from_uri, list) or isinstance(from_uri, tuple):
             remote = " ".join(from_uri)
-        cmd = "hadoop fs -copyToLocal {remote} {local}".format(
-            remote=remote, local=local_path
-        )
+        cmd = f"hadoop fs -copyToLocal {remote} {local_path}"
 
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
@@ -135,9 +129,7 @@ class HDFS:
                 )
         except Exception as e:
             raise RuntimeError(
-                "Cannot copy remote[{}] to local[{}] with cmd[{}]".format(
-                    from_uri, local_path, cmd
-                )
+                f"Cannot copy remote[{from_uri}] to local[{local_path}] with cmd[{cmd}]"
             ) from e
 
     @staticmethod

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/hdfs_context.py

@@ -137,10 +137,7 @@ class HDFSContext(BaseContext):
         if os.path.exists(gz_dir):
             shutil.rmtree(gz_dir, ignore_errors=True)
         os.mkdir(os.path.join(self.local_root, "tmp"))
-        rfile_tgz = "{}/{}_*_download.tar.gz".format(
-            self.remote_root,
-            submission.submission_hash,
-        )
+        rfile_tgz = f"{self.remote_root}/{submission.submission_hash}_*_download.tar.gz"
         lfile_tgz = "%s/tmp/" % (self.local_root)
         HDFS.copy_to_local(rfile_tgz, lfile_tgz)
 

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/lsf.py

@@ -31,12 +31,8 @@ class LSF(Machine):
             "lsf_nodes_line": "#BSUB -n {number_cores}".format(
                 number_cores=resources.number_node * resources.cpu_per_node
             ),
-            "lsf_ptile_line": "#BSUB -R 'span[ptile={cpu_per_node}]'".format(
-                cpu_per_node=resources.cpu_per_node
-            ),
-            "lsf_partition_line": "#BSUB -q {queue_name}".format(
-                queue_name=resources.queue_name
-            ),
+            "lsf_ptile_line": f"#BSUB -R 'span[ptile={resources.cpu_per_node}]'",
+            "lsf_partition_line": f"#BSUB -q {resources.queue_name}",
         }
         gpu_usage_flag = resources.kwargs.get("gpu_usage", False)
         gpu_new_syntax_flag = resources.kwargs.get("gpu_new_syntax", False)

{dpdispatcher-0.5.8 → dpdispatcher-0.5.10}/dpdispatcher/machine.py

@@ -208,9 +208,7 @@ class Machine(metaclass=ABCMeta):
 
     def check_if_recover(self, submission):
         submission_hash = submission.submission_hash
-        submission_file_name = "{submission_hash}.json".format(
-            submission_hash=submission_hash
-        )
+        submission_file_name = f"{submission_hash}.json"
         if_recover = self.context.check_file_exists(submission_file_name)
         return if_recover
 

dpdispatcher-0.5.10/dpdispatcher/openapi.py

@@ -0,0 +1,198 @@
+import os
+import shutil
+import time
+
+try:
+    from bohriumsdk.client import Client
+    from bohriumsdk.job import Job
+    from bohriumsdk.storage import Storage
+    from bohriumsdk.util import Util
+except ModuleNotFoundError:
+    found_bohriumsdk = False
+else:
+    found_bohriumsdk = True
+
+from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.machine import Machine
+
+shell_script_header_template = """
+#!/bin/bash -l
+"""
+
+
+class OpenAPI(Machine):
+    def __init__(self, context):
+        if not found_bohriumsdk:
+            raise ModuleNotFoundError(
+                "bohriumsdk not installed. Install dpdispatcher with `pip install dpdispatcher[bohrium]`"
+            )
+        self.context = context
+        self.remote_profile = context.remote_profile.copy()
+
+        self.grouped = self.remote_profile.get("grouped", True)
+        self.client = Client()
+        self.job = Job(client=self.client)
+        self.storage = Storage(client=self.client)
+        self.group_id = None
+
+    def gen_script(self, job):
+        shell_script = super().gen_script(job)
+        return shell_script
+
+    def gen_script_header(self, job):
+        shell_script_header = shell_script_header_template
+        return shell_script_header
+
+    def gen_local_script(self, job):
+        script_str = self.gen_script(job)
+        script_file_name = job.script_file_name
+        self.context.write_local_file(fname=script_file_name, write_str=script_str)
+        return script_file_name
+
+    def _gen_backward_files_list(self, job):
+        result_file_list = []
+        # result_file_list.extend(job.backward_common_files)
+        for task in job.job_task_list:
+            result_file_list.extend(
+                [os.path.join(task.task_work_path, b_f) for b_f in task.backward_files]
+            )
+        result_file_list = list(set(result_file_list))
+        return result_file_list
+
+    def do_submit(self, job):
+        self.gen_local_script(job)
+
+        project_id = self.remote_profile.get("project_id", 0)
+
+        openapi_params = {
+            "oss_path": job.upload_path,
+            "input_file_type": 3,
+            "input_file_method": 1,
+            "job_type": "container",
+            "job_name": self.remote_profile.get("job_name", "DP-GEN"),
+            "project_id": project_id,
+            "scass_type": self.remote_profile.get("machine_type", ""),
+            "cmd": f"bash {job.script_file_name}",
+            "log_files": os.path.join(
+                job.job_task_list[0].task_work_path, job.job_task_list[0].outlog
+            ),
+            "out_files": self._gen_backward_files_list(job),
+            "platform": self.remote_profile.get("platform", "ali"),
+            "image_address": self.remote_profile.get("image_address", ""),
+            "job_id": job.job_id,
+        }
+
+        data = self.job.insert(**openapi_params)
+
+        job.job_id = data.get("jobId", 0)  # type: ignore
+        # self.job_group_id = data.get("jobGroupId")
+        job.job_state = JobStatus.waiting
+        return job.job_id
+
+    def _get_job_detail(self, job_id, group_id):
+        check_return = self.job.detail(job_id)
+        assert check_return is not None, (
+            f"Failed to retrieve tasks information. To resubmit this job, please "
+            f"try again, if this problem still exists please delete the submission "
+            f"file and try again.\nYou can check submission.submission_hash in the "
+            f'previous log or type `grep -rl "{job_id}:job_group_id:{group_id}" '
+            f"~/.dpdispatcher/dp_cloud_server/` to find corresponding file. "
+            f"You can try with command:\n    "
+            f'rm $(grep -rl "{job_id}:job_group_id:{group_id}" ~/.dpdispatcher/dp_cloud_server/)'
+        )
+        return check_return
+
+    def check_status(self, job):
+        if job.job_id == "":
+            return JobStatus.unsubmitted
+        job_id = job.job_id
+        group_id = None
+        if hasattr(job, "jgid"):
+            group_id = job.jgid
+        check_return = self._get_job_detail(job_id, group_id)
+        try:
+            dp_job_status = check_return["status"]  # type: ignore
+        except IndexError as e:
+            dlog.error(
+                f"cannot find job information in bohrium for job {job.job_id}. check_return:{check_return}; retry one more time after 60 seconds"
+            )
+            time.sleep(60)
+            retry_return = self._get_job_detail(job_id, group_id)
+            try:
+                dp_job_status = retry_return["status"]  # type: ignore
+            except IndexError as e:
+                raise RuntimeError(
+                    f"cannot find job information in bohrium for job {job.job_id} {check_return} {retry_return}"
+                )
+
+        job_state = self.map_dp_job_state(dp_job_status)
+        if job_state == JobStatus.finished:
+            job_log = self.job.log(job_id)
+            if self.remote_profile.get("output_log"):
+                print(job_log, end="")
+            # print(job.job_id)
+            self._download_job(job)
+        elif self.remote_profile.get("output_log") and job_state == JobStatus.running:
+            job_log = self.job.log(job_id)
+            print(job_log, end="")
+        return job_state
+
+    def _download_job(self, job):
+        data = self.job.detail(job.job_id)
+        # print(data)
+        job_url = data["jobFiles"]["outFiles"][0]["url"]  # type: ignore
+        if not job_url:
+            return
+        job_hash = job.job_hash
+        result_filename = job_hash + "_back.zip"
+        target_result_zip = os.path.join(self.context.local_root, result_filename)
+        self.storage.download_from_url(job_url, target_result_zip)
+        Util.unzip_file(target_result_zip, out_dir=self.context.local_root)
+        try:
+            os.makedirs(os.path.join(self.context.local_root, "backup"), exist_ok=True)
+            shutil.move(
+                target_result_zip,
+                os.path.join(
+                    self.context.local_root,
+                    "backup",
+                    os.path.split(target_result_zip)[1],
+                ),
+            )
+        except (OSError, shutil.Error) as e:
+            dlog.exception("unable to backup file, " + str(e))
+
+    def check_finish_tag(self, job):
+        job_tag_finished = job.job_hash + "_job_tag_finished"
+        dlog.info("check if job finished: ", job.job_id, job_tag_finished)
+        return self.context.check_file_exists(job_tag_finished)
+        # return
+        # pass
+
+    def check_if_recover(self, submission):
+        return False
+        # pass
+
+    @staticmethod
+    def map_dp_job_state(status):
+        if isinstance(status, JobStatus):
+            return status
+        map_dict = {
+            -1: JobStatus.terminated,
+            0: JobStatus.waiting,
+            1: JobStatus.running,
+            2: JobStatus.finished,
+            3: JobStatus.waiting,
+            4: JobStatus.running,
+            5: JobStatus.terminated,
+            6: JobStatus.running,
+            9: JobStatus.waiting,
+        }
+        if status not in map_dict:
+            dlog.error(f"unknown job status {status}")
+            return JobStatus.unknown
+        return map_dict[status]
+
+    # def check_finish_tag(self, job):
+    #     job_tag_finished = job.job_hash + '_job_tag_finished'
+    #     return self.context.check_file_exists(job_tag_finished)