dpdispatcher 0.6.9__py3-none-any.whl → 0.6.11__py3-none-any.whl

dpdispatcher/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.6.9'
-__version_tuple__ = version_tuple = (0, 6, 9)
+__version__ = version = '0.6.11'
+__version_tuple__ = version_tuple = (0, 6, 11)
+
+__commit_id__ = commit_id = None

dpdispatcher/contexts/openapi_context.py

@@ -1,18 +1,20 @@
+import glob
 import os
 import shutil
 import uuid
+from zipfile import ZipFile
 
 import tqdm
 
 try:
-    from bohriumsdk.client import Client
-    from bohriumsdk.job import Job
-    from bohriumsdk.storage import Storage
-    from bohriumsdk.util import Util
-except ModuleNotFoundError:
+    from bohrium import Bohrium
+    from bohrium.resources import Job, Tiefblue
+except ModuleNotFoundError as e:
     found_bohriumsdk = False
+    import_bohrium_error = e
 else:
     found_bohriumsdk = True
+    import_bohrium_error = None
 
 from dpdispatcher.base_context import BaseContext
 from dpdispatcher.dlog import dlog
@@ -23,6 +25,36 @@ DP_CLOUD_SERVER_HOME_DIR = os.path.join(
 )
 
 
+def unzip_file(zip_file, out_dir="./"):
+    obj = ZipFile(zip_file, "r")
+    for item in obj.namelist():
+        obj.extract(item, out_dir)
+
+
+def zip_file_list(root_path, zip_filename, file_list=[]):
+    out_zip_file = os.path.join(root_path, zip_filename)
+    # print('debug: file_list', file_list)
+    zip_obj = ZipFile(out_zip_file, "w")
+    for f in file_list:
+        matched_files = os.path.join(root_path, f)
+        for ii in glob.glob(matched_files):
+            # print('debug: matched_files:ii', ii)
+            if os.path.isdir(ii):
+                arcname = os.path.relpath(ii, start=root_path)
+                zip_obj.write(ii, arcname)
+                for root, dirs, files in os.walk(ii):
+                    for file in files:
+                        filename = os.path.join(root, file)
+                        arcname = os.path.relpath(filename, start=root_path)
+                        # print('debug: filename:arcname:root_path', filename, arcname, root_path)
+                        zip_obj.write(filename, arcname)
+            else:
+                arcname = os.path.relpath(ii, start=root_path)
+                zip_obj.write(ii, arcname)
+    zip_obj.close()
+    return out_zip_file
+
+
 class OpenAPIContext(BaseContext):
     def __init__(
         self,
@@ -35,16 +67,41 @@ class OpenAPIContext(BaseContext):
         if not found_bohriumsdk:
             raise ModuleNotFoundError(
                 "bohriumsdk not installed. Install dpdispatcher with `pip install dpdispatcher[bohrium]`"
-            )
+            ) from import_bohrium_error
         self.init_local_root = local_root
         self.init_remote_root = remote_root
         self.temp_local_root = os.path.abspath(local_root)
         self.remote_profile = remote_profile
-        self.client = Client()
-        self.storage = Storage(client=self.client)
+        access_key = (
+            remote_profile.get("access_key", None)
+            or os.getenv("BOHRIUM_ACCESS_KEY", None)
+            or os.getenv("ACCESS_KEY", None)
+        )
+        project_id = (
+            remote_profile.get("project_id", None)
+            or os.getenv("BOHRIUM_PROJECT_ID", None)
+            or os.getenv("PROJECT_ID", None)
+        )
+        app_key = (
+            remote_profile.get("app_key", None)
+            or os.getenv("BOHRIUM_APP_KEY", None)
+            or os.getenv("APP_KEY", None)
+        )
+        if access_key is None:
+            raise ValueError(
+                "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'"
+            )
+        if project_id is None:
+            raise ValueError(
+                "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'"
+            )
+        self.client = Bohrium(
+            access_key=access_key, project_id=project_id, app_key=app_key
+        )
+        self.storage = Tiefblue()
         self.job = Job(client=self.client)
-        self.util = Util()
         self.jgid = None
+        os.makedirs(DP_CLOUD_SERVER_HOME_DIR, exist_ok=True)
 
     @classmethod
     def load_from_dict(cls, context_dict):
@@ -97,7 +154,7 @@ class OpenAPIContext(BaseContext):
             for file in task.forward_files:
                 upload_file_list.append(os.path.join(task.task_work_path, file))
 
-        upload_zip = Util.zip_file_list(
+        upload_zip = zip_file_list(
             self.local_root, zip_task_file, file_list=upload_file_list
         )
         project_id = self.remote_profile.get("project_id", 0)
@@ -189,7 +246,7 @@ class OpenAPIContext(BaseContext):
             ):
                 continue
             self.storage.download_from_url(info["resultUrl"], target_result_zip)
-            Util.unzip_file(target_result_zip, out_dir=self.local_root)
+            unzip_file(target_result_zip, out_dir=self.local_root)
             self._backup(self.local_root, target_result_zip)
         self._clean_backup(
             self.local_root, keep_backup=self.remote_profile.get("keep_backup", True)

dpdispatcher/contexts/ssh_context.py

@@ -163,7 +163,6 @@ class SSHSession:
             if os.path.exists(key_path):
                 for pkey_class in (
                     paramiko.RSAKey,
-                    paramiko.DSSKey,
                     paramiko.ECDSAKey,
                     paramiko.Ed25519Key,
                 ):
@@ -181,7 +180,6 @@ class SSHSession:
         elif self.look_for_keys:
             for keytype, name in [
                 (paramiko.RSAKey, "rsa"),
-                (paramiko.DSSKey, "dsa"),
                 (paramiko.ECDSAKey, "ecdsa"),
                 (paramiko.Ed25519Key, "ed25519"),
             ]:

dpdispatcher/machines/openapi.py

@@ -1,14 +1,13 @@
 import os
 import shutil
 import time
+from zipfile import ZipFile
 
 from dpdispatcher.utils.utils import customized_script_header_template
 
 try:
-    from bohriumsdk.client import Client
-    from bohriumsdk.job import Job
-    from bohriumsdk.storage import Storage
-    from bohriumsdk.util import Util
+    from bohrium import Bohrium
+    from bohrium.resources import Job, Tiefblue
 except ModuleNotFoundError:
     found_bohriumsdk = False
 else:
@@ -23,6 +22,12 @@ shell_script_header_template = """
 """
 
 
+def unzip_file(zip_file, out_dir="./"):
+    obj = ZipFile(zip_file, "r")
+    for item in obj.namelist():
+        obj.extract(item, out_dir)
+
+
 class OpenAPI(Machine):
     def __init__(self, context):
         if not found_bohriumsdk:
@@ -35,9 +40,35 @@ class OpenAPI(Machine):
         self.grouped = self.remote_profile.get("grouped", True)
         self.retry_count = self.remote_profile.get("retry_count", 3)
         self.ignore_exit_code = context.remote_profile.get("ignore_exit_code", True)
-        self.client = Client()
+
+        access_key = (
+            self.remote_profile.get("access_key", None)
+            or os.getenv("BOHRIUM_ACCESS_KEY", None)
+            or os.getenv("ACCESS_KEY", None)
+        )
+        project_id = (
+            self.remote_profile.get("project_id", None)
+            or os.getenv("BOHRIUM_PROJECT_ID", None)
+            or os.getenv("PROJECT_ID", None)
+        )
+        app_key = (
+            self.remote_profile.get("app_key", None)
+            or os.getenv("BOHRIUM_APP_KEY", None)
+            or os.getenv("APP_KEY", None)
+        )
+        if access_key is None:
+            raise ValueError(
+                "remote_profile must contain 'access_key' or set environment variable 'BOHRIUM_ACCESS_KEY'"
+            )
+        if project_id is None:
+            raise ValueError(
+                "remote_profile must contain 'project_id' or set environment variable 'BOHRIUM_PROJECT_ID'"
+            )
+        self.client = Bohrium(
+            access_key=access_key, project_id=project_id, app_key=app_key
+        )
+        self.storage = Tiefblue()
         self.job = Job(client=self.client)
-        self.storage = Storage(client=self.client)
         self.group_id = None
 
     def gen_script(self, job):
@@ -102,7 +133,6 @@ class OpenAPI(Machine):
         }
         if job.job_state == JobStatus.unsubmitted:
             openapi_params["job_id"] = job.job_id
-
         data = self.job.insert(**openapi_params)
 
         job.job_id = data.get("jobId", 0)  # type: ignore
@@ -170,7 +200,7 @@ class OpenAPI(Machine):
         result_filename = job_hash + "_back.zip"
         target_result_zip = os.path.join(self.context.local_root, result_filename)
         self.storage.download_from_url(job_url, target_result_zip)
-        Util.unzip_file(target_result_zip, out_dir=self.context.local_root)
+        unzip_file(target_result_zip, out_dir=self.context.local_root)
         try:
             os.makedirs(os.path.join(self.context.local_root, "backup"), exist_ok=True)
             shutil.move(

dpdispatcher/machines/slurm.py

@@ -85,7 +85,7 @@ class Slurm(Machine):
         # self.context.write_file(fname=os.path.join(self.context.submission.work_base, script_file_name), write_str=script_str)
         command = "cd {} && {} {}".format(
             shlex.quote(self.context.remote_root),
-            "sbatch",
+            "sbatch --parsable",
             shlex.quote(script_file_name),
         )
         ret, stdin, stdout, stderr = self.context.block_call(command)

{dpdispatcher-0.6.9.dist-info → dpdispatcher-0.6.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dpdispatcher
-Version: 0.6.9
+Version: 0.6.11
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.6.9.dist-info → dpdispatcher-0.6.11.dist-info}/RECORD

@@ -1,6 +1,6 @@
 dpdispatcher/__init__.py,sha256=CLZP_N5CTp14ujWCykEHuJjoIfKR6CwrclXhjWUgNoE,517
 dpdispatcher/__main__.py,sha256=BFhG-mSBzVZUEezQJqXWZnt2WsnhAHT_zpT8Y6gpOz0,116
-dpdispatcher/_version.py,sha256=mDo81caPh-9VsCFEkJjBxRgru5DzIfRLWeCvOiXWMJo,511
+dpdispatcher/_version.py,sha256=sw6_B_LEWeMBiN2AFkGTSamIubZxwWvM45JyuClFrPY,706
 dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
 dpdispatcher/base_context.py,sha256=W4eWDWVzYeL6EuEkivmJp-_h_B2mV9PtRWc09l1_Qzc,5242
 dpdispatcher/dlog.py,sha256=QJKAwB6gV3Zb6zQUL9dZ_uIoTIEy9Z7ecmVQ-8WNmD8,1081
@@ -13,8 +13,8 @@ dpdispatcher/contexts/dp_cloud_server_context.py,sha256=PGRMef3q2hfK-o5dNIWWvzPc
 dpdispatcher/contexts/hdfs_context.py,sha256=mYQzXMZ4A9EjjWBAH3Ba6HOErUhMMwCsKxOjpd5R57Y,9105
 dpdispatcher/contexts/lazy_local_context.py,sha256=FAClbLD2F4LizUqFzMOg3t0Z6NLeTDLJy7NkRcDELFs,5070
 dpdispatcher/contexts/local_context.py,sha256=VbaSXGAc_EDMT0K5WV_flBF0bX87ntrwO_hq_Bkcb04,14590
-dpdispatcher/contexts/openapi_context.py,sha256=M7L9axpjOrzvdTpLMDuEzZqe4ZuKIxjS0bzZUv8W2IQ,9674
-dpdispatcher/contexts/ssh_context.py,sha256=qaj8h2TdY1i-YYdDstUBs9IJaLwzytwnQkdntMEZ7vg,37664
+dpdispatcher/contexts/openapi_context.py,sha256=bWfg-y-QHAT1iSva6PvuLgMs1uJDLom4QC06IGMeUfs,11928
+dpdispatcher/contexts/ssh_context.py,sha256=VOpPRQX1WS6WbWxCYhuT62q203v1Qx_hygLsV41p3Fg,37585
 dpdispatcher/dpcloudserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 dpdispatcher/dpcloudserver/client.py,sha256=k1niKjG6zFnMtHn_UuCjYoOcMju3o3PV-GdyVLr5-KM,165
 dpdispatcher/entrypoints/__init__.py,sha256=exKSFT3j2oCerGwtI8WbHQK-D0K-CyifocRji1xntT4,20
@@ -27,10 +27,10 @@ dpdispatcher/machines/distributed_shell.py,sha256=c0-lGeGz_M-PY2gPciT-uYZLQht5XT
 dpdispatcher/machines/dp_cloud_server.py,sha256=SR69gsFb2BvOQCW1QnWfP3cQvu_qHLJNsycp5wzosJU,11706
 dpdispatcher/machines/fugaku.py,sha256=oY2hD2ldL2dztwtJ9WNisdsfPnaX-5yTRXewIT9r60I,4314
 dpdispatcher/machines/lsf.py,sha256=xGDq8OLAk83E9EjK_3-QtEOyahvBGspWbxT__7mnSTw,7896
-dpdispatcher/machines/openapi.py,sha256=Gzzbo8YOAybXGTrgMutexErcaEi3ts7uTUNvOhThFS8,8858
+dpdispatcher/machines/openapi.py,sha256=8unjG9HTCBwbkZTW_t-QKOSaBHfqhzsMARTVVCDGGHw,9929
 dpdispatcher/machines/pbs.py,sha256=gUoj3OGQbZRBK4P-WXlhrxlQqTeUi9X8JGLOkAB__wE,12669
 dpdispatcher/machines/shell.py,sha256=EeYnRCowXdzO3Nh25Yh_t5xeM6frq4uChk4GVx7OjH8,4797
-dpdispatcher/machines/slurm.py,sha256=oyMX9iZQpVRR951zwz0wRNfl3_uJZzdtzxMbTJotlQU,15402
+dpdispatcher/machines/slurm.py,sha256=rN51Qh_u9ZVmjkIClu4Tfc-Qesr19DxPaaeh0FmRx14,15413
 dpdispatcher/utils/__init__.py,sha256=fwvwkMf7DFNQkNBiIce8Y8gRA6FhICwKjkKiXu_BEJg,13
 dpdispatcher/utils/hdfs_cli.py,sha256=a1a9PJAzt3wsTcdaSw_oD1vcNw59pMooxpAHjYOaaGA,5209
 dpdispatcher/utils/job_status.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
@@ -41,9 +41,9 @@ dpdispatcher/utils/dpcloudserver/client.py,sha256=fp1e14MTgsMgasZSWowq-NqfCoi21P
 dpdispatcher/utils/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
 dpdispatcher/utils/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
 dpdispatcher/utils/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
-dpdispatcher-0.6.9.dist-info/licenses/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
-dpdispatcher-0.6.9.dist-info/METADATA,sha256=VL24cHeEqy4J29UGpgmfBaT7YiUvo0_iftaQDoMh_yU,12833
-dpdispatcher-0.6.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dpdispatcher-0.6.9.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
-dpdispatcher-0.6.9.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
-dpdispatcher-0.6.9.dist-info/RECORD,,
+dpdispatcher-0.6.11.dist-info/licenses/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
+dpdispatcher-0.6.11.dist-info/METADATA,sha256=z3SBMyU0Mq8f1Sv_4vbYAUYcjfh0Fjjbjdv9wNBf_EM,12834
+dpdispatcher-0.6.11.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dpdispatcher-0.6.11.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
+dpdispatcher-0.6.11.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
+dpdispatcher-0.6.11.dist-info/RECORD,,