dpdispatcher 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

dpdispatcher/__init__.py

@@ -1,103 +1,21 @@
-import logging
-import os
-import sys
-import warnings
-
-ROOT_PATH = tuple(__path__)[0]
-dlog = logging.getLogger(__name__)
-dlog.propagate = False
-dlog.setLevel(logging.INFO)
-try:
-    dlogf = logging.FileHandler(
-        os.getcwd() + os.sep + "dpdispatcher" + ".log", delay=True
-    )
-except PermissionError:
-    warnings.warn(
-        "dpdispatcher.log meet permission error. redirect the log to ~/dpdispatcher.log"
-    )
-    dlogf = logging.FileHandler(
-        os.path.join(os.path.expanduser("~"), "dpdispatcher.log")
-    )
-
-# dlogf = logging.FileHandler('./'+os.sep+SHORT_CMD+'.log')
-# dlogf = logging.FileHandler(os.path.join(os.environ['HOME'], SHORT_CMD+'.log'))
-# dlogf = logging.FileHandler(os.path.join(os.path.expanduser('~'), SHORT_CMD+'.log'))
-# dlogf = logging.FileHandler(os.path.join("/tmp/", SHORT_CMD+'.log'))
-dlogf_formatter = logging.Formatter("%(asctime)s - %(levelname)s : %(message)s")
-# dlogf_formatter=logging.Formatter('%(asctime)s - %(name)s - [%(filename)s:%(funcName)s - %(lineno)d ] - %(levelname)s \n %(message)s')
-dlogf.setFormatter(dlogf_formatter)
-dlog.addHandler(dlogf)
-
-dlog_stdout = logging.StreamHandler(sys.stdout)
-dlog_stdout.setFormatter(dlogf_formatter)
-dlog.addHandler(dlog_stdout)
-
 __author__ = "DeepModeling Team"
-__copyright__ = "Copyright 2019"
-__status__ = "Development"
+__copyright__ = "Copyright 2019-2023, DeepModeling"
+__status__ = "Production"
 try:
     from ._version import version as __version__
 except ImportError:
-    __version__ = "unkown"
-
-from .distributed_shell import DistributedShell
-from .dp_cloud_server import DpCloudServer, Lebesgue
-from .dp_cloud_server_context import DpCloudServerContext, LebesgueContext
-from .fugaku import Fugaku
-from .hdfs_context import HDFSContext
-from .lazy_local_context import LazyLocalContext
-from .local_context import LocalContext
-from .lsf import LSF
-from .machine import Machine
-from .openapi import OpenAPI
-from .openapi_context import OpenAPIContext
-from .pbs import PBS, Torque
-from .shell import Shell
-from .slurm import Slurm
-from .ssh_context import SSHContext
-from .submission import Job, Resources, Submission, Task
-
-
-def info():
-    """Show basic information about dpdispatcher, its location and version."""
-    print("DeepModeling\n------------")
-    print("Version: " + __version__)
-    print("Path:    " + ROOT_PATH)
-    print("")
-    print("Dependency")
-    print("------------")
-    for modui in ["psutil", "paramiko", "dargs", "oss2"]:
-        try:
-            mm = __import__(modui)
-            print("%10s %10s   %s" % (modui, mm.__version__, mm.__path__[0]))
-        except ImportError:
-            print("%10s %10s Not Found" % (modui, ""))
-    print()
+    __version__ = "unknown"
 
+import dpdispatcher.contexts  # noqa: F401
+import dpdispatcher.machines  # noqa: F401
+from dpdispatcher.machine import Machine
+from dpdispatcher.submission import Job, Resources, Submission, Task
 
 __all__ = [
     "__version__",
-    "DistributedShell",
-    "DpCloudServer",
-    "OpenAPI",
-    "OpenAPIContext",
-    "DpCloudServerContext",
-    "HDFSContext",
-    "LazyLocalContext",
-    "LocalContext",
-    "LSF",
     "Machine",
-    "PBS",
-    "Shell",
-    "Slurm",
-    "Fugaku",
-    "SSHContext",
     "Submission",
     "Task",
-    "Torque",
-    "info",
-    "Lebesgue",
-    "LebesgueContext",
     "Job",
     "Resources",
 ]

dpdispatcher/__main__.py

@@ -0,0 +1,8 @@
+"""Package dp entry point."""
+
+from dpdispatcher.dpdisp import (
+    main,
+)
+
+if __name__ == "__main__":
+    main()

dpdispatcher/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.6.0'
-__version_tuple__ = version_tuple = (0, 6, 0)
+__version__ = version = '0.6.1'
+__version_tuple__ = version_tuple = (0, 6, 1)

dpdispatcher/base_context.py

@@ -3,7 +3,7 @@ from typing import List, Tuple
 
 from dargs import Argument
 
-from dpdispatcher import dlog
+from dpdispatcher.dlog import dlog
 
 
 class BaseContext(metaclass=ABCMeta):

dpdispatcher/contexts/__init__.py

@@ -0,0 +1,11 @@
+"""Contexts."""
+import importlib
+from pathlib import Path
+
+PACKAGE_BASE = "dpdispatcher.contexts"
+NOT_LOADABLE = ("__init__.py",)
+
+for module_file in Path(__file__).parent.glob("*.py"):
+    if module_file.name not in NOT_LOADABLE:
+        module_name = f".{module_file.stem}"
+        importlib.import_module(module_name, PACKAGE_BASE)

dpdispatcher/{dp_cloud_server_context.py → contexts/dp_cloud_server_context.py}

@@ -8,13 +8,16 @@ from typing import List
 import tqdm
 from dargs.dargs import Argument
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
-from dpdispatcher.dpcloudserver.config import ALI_STS_BUCKET_NAME, ALI_STS_ENDPOINT
+from dpdispatcher.dlog import dlog
 
 # from dpdispatcher.submission import Machine
 # from . import dlog
-from .dpcloudserver import Client, zip_file
+from dpdispatcher.utils.dpcloudserver import Client, zip_file
+from dpdispatcher.utils.dpcloudserver.config import (
+    ALI_STS_BUCKET_NAME,
+    ALI_STS_ENDPOINT,
+)
 
 # from zip_file import zip_files
 

dpdispatcher/{hdfs_context.py → contexts/hdfs_context.py}

@@ -3,9 +3,9 @@ import shutil
 import tarfile
 from glob import glob
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
-from dpdispatcher.hdfs_cli import HDFS
+from dpdispatcher.dlog import dlog
+from dpdispatcher.utils.hdfs_cli import HDFS
 
 
 class HDFSContext(BaseContext):

dpdispatcher/{local_context.py → contexts/local_context.py}

@@ -1,12 +1,11 @@
-import hashlib
 import os
 import shutil
 import subprocess as sp
 from glob import glob
 from subprocess import TimeoutExpired
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
+from dpdispatcher.dlog import dlog
 
 
 class SPRetObj:
@@ -30,14 +29,6 @@ def _check_file_path(fname):
         os.makedirs(dirname, exist_ok=True)
 
 
-def _identical_files(fname0, fname1):
-    with open(fname0) as fp:
-        code0 = hashlib.sha1(fp.read().encode("utf-8")).hexdigest()
-    with open(fname1) as fp:
-        code1 = hashlib.sha1(fp.read().encode("utf-8")).hexdigest()
-    return code0 == code1
-
-
 class LocalContext(BaseContext):
     """Run jobs in the local server and remote directory.
 

dpdispatcher/{openapi_context.py → contexts/openapi_context.py}

@@ -14,9 +14,9 @@ except ModuleNotFoundError:
 else:
     found_bohriumsdk = True
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
+from dpdispatcher.utils.job_status import JobStatus
 
 DP_CLOUD_SERVER_HOME_DIR = os.path.join(
     os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/"

dpdispatcher/{ssh_context.py → contexts/ssh_context.py}

@@ -18,11 +18,17 @@ import paramiko
 import paramiko.ssh_exception
 from dargs.dargs import Argument
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
+from dpdispatcher.dlog import dlog
 
 # from dpdispatcher.submission import Machine
-from dpdispatcher.utils import RetrySignal, generate_totp, get_sha256, retry, rsync
+from dpdispatcher.utils.utils import (
+    RetrySignal,
+    generate_totp,
+    get_sha256,
+    retry,
+    rsync,
+)
 
 
 class SSHSession:
@@ -140,6 +146,7 @@ class SSHSession:
         # Make a Paramiko Transport object using the socket
         ts = paramiko.Transport(sock)
         ts.banner_timeout = 60
+        ts.auth_timeout = self.timeout + 20
         ts.use_compression(compress=True)
 
         # Tell Paramiko that the Transport is going to be used as a client
@@ -939,21 +946,35 @@ class SSHContext(BaseContext):
         per_nfile = 100
         ntar = len(files) // per_nfile + 1
         if ntar <= 1:
-            self.block_checkcall(
-                "tar {} {} {}".format(
-                    tar_command,
-                    shlex.quote(of),
-                    " ".join([shlex.quote(file) for file in files]),
+            try:
+                self.block_checkcall(
+                    "tar {} {} {}".format(
+                        tar_command,
+                        shlex.quote(of),
+                        " ".join([shlex.quote(file) for file in files]),
+                    )
                 )
-            )
+            except RuntimeError as e:
+                if "No such file or directory" in str(e):
+                    raise FileNotFoundError(
+                        "Any of the backward files does not exist in the remote directory."
+                    ) from e
+                raise e
         else:
             file_list_file = os.path.join(
                 self.remote_root, ".tmp.tar." + str(uuid.uuid4())
             )
             self.write_file(file_list_file, "\n".join(files))
-            self.block_checkcall(
-                f"tar {tar_command} {shlex.quote(of)} -T {shlex.quote(file_list_file)}"
-            )
+            try:
+                self.block_checkcall(
+                    f"tar {tar_command} {shlex.quote(of)} -T {shlex.quote(file_list_file)}"
+                )
+            except RuntimeError as e:
+                if "No such file or directory" in str(e):
+                    raise FileNotFoundError(
+                        "Any of the backward files does not exist in the remote directory."
+                    ) from e
+                raise e
         # trans
         from_f = pathlib.PurePath(os.path.join(self.remote_root, of)).as_posix()
         to_f = pathlib.PurePath(os.path.join(self.local_root, of)).as_posix()

dpdispatcher/dlog.py

@@ -0,0 +1,31 @@
+import logging
+import os
+import sys
+import warnings
+
+dlog = logging.getLogger("dpdispatcher")
+dlog.propagate = False
+dlog.setLevel(logging.INFO)
+try:
+    dlogf = logging.FileHandler(
+        os.getcwd() + os.sep + "dpdispatcher" + ".log", delay=True
+    )
+except PermissionError:
+    warnings.warn(
+        "dpdispatcher.log meet permission error. redirect the log to ~/dpdispatcher.log"
+    )
+    dlogf = logging.FileHandler(
+        os.path.join(os.path.expanduser("~"), "dpdispatcher.log"), delay=True
+    )
+
+dlogf_formatter = logging.Formatter("%(asctime)s - %(levelname)s : %(message)s")
+dlogf.setFormatter(dlogf_formatter)
+dlog.addHandler(dlogf)
+
+dlog_stdout = logging.StreamHandler(sys.stdout)
+dlog_stdout.setFormatter(dlogf_formatter)
+dlog.addHandler(dlog_stdout)
+
+__all__ = [
+    "dlog",
+]

dpdispatcher/dpdisp.py

@@ -2,7 +2,8 @@
 import argparse
 from typing import List, Optional
 
-from dpdispatcher.gui import start_dpgui
+from dpdispatcher.entrypoints.gui import start_dpgui
+from dpdispatcher.entrypoints.submission import handle_submission
 
 
 def main_parser() -> argparse.ArgumentParser:
@@ -23,6 +24,37 @@ def main_parser() -> argparse.ArgumentParser:
     )
     subparsers = parser.add_subparsers(title="Valid subcommands", dest="command")
     ##########################################
+    # backward
+    parser_submission = subparsers.add_parser(
+        "submission",
+        help="Handle terminated submission.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser_submission.add_argument(
+        "SUBMISSION_HASH",
+        type=str,
+        help="Submission hash to download.",
+    )
+    parser_submission_action = parser_submission.add_argument_group(
+        "Actions",
+        description="One or more actions to take on submission.",
+    )
+    parser_submission_action.add_argument(
+        "--download-terminated-log",
+        action="store_true",
+        help="Download log files of terminated tasks.",
+    )
+    parser_submission_action.add_argument(
+        "--download-finished-task",
+        action="store_true",
+        help="Download finished tasks.",
+    )
+    parser_submission_action.add_argument(
+        "--clean",
+        action="store_true",
+        help="Clean submission.",
+    )
+    ##########################################
     # gui
     parser_gui = subparsers.add_parser(
         "gui",
@@ -67,7 +99,14 @@ def parse_args(args: Optional[List[str]] = None):
 
 def main():
     args = parse_args()
-    if args.command == "gui":
+    if args.command == "submission":
+        handle_submission(
+            submission_hash=args.SUBMISSION_HASH,
+            download_terminated_log=args.download_terminated_log,
+            download_finished_task=args.download_finished_task,
+            clean=args.clean,
+        )
+    elif args.command == "gui":
         start_dpgui(
             port=args.port,
             bind_all=args.bind_all,

dpdispatcher/entrypoints/__init__.py

@@ -0,0 +1 @@
+"""Entry points."""

dpdispatcher/entrypoints/submission.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+
+from dpdispatcher.dlog import dlog
+from dpdispatcher.submission import Submission
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.record import record
+
+
+def handle_submission(
+    *,
+    submission_hash: str,
+    download_terminated_log: bool = False,
+    download_finished_task: bool = False,
+    clean: bool = False,
+):
+    """Handle terminated submission.
+
+    Parameters
+    ----------
+    submission_hash : str
+        Submission hash to download.
+    download_terminated_log : bool, optional
+        Download log files of terminated tasks.
+    download_finished_task : bool, optional
+        Download finished tasks.
+    clean : bool, optional
+        Clean submission.
+
+    Raises
+    ------
+    ValueError
+        At least one action should be specified.
+    """
+    if int(download_terminated_log) + int(download_finished_task) + int(clean) == 0:
+        raise ValueError("At least one action should be specified.")
+
+    submission_file = record.get_submission(submission_hash)
+    submission = Submission.submission_from_json(str(submission_file))
+    submission.belonging_tasks = [
+        task for job in submission.belonging_jobs for task in job.job_task_list
+    ]
+    # TODO: for unclear reason, the submission_hash may be changed
+    submission.submission_hash = submission_hash
+    submission.machine.context.bind_submission(submission)
+    submission.update_submission_state()
+
+    terminated_tasks = []
+    finished_tasks = []
+    for task in submission.belonging_tasks:
+        task.get_task_state(submission.machine.context)
+        if task.task_state == JobStatus.terminated:
+            terminated_tasks.append(task)
+        elif task.task_state == JobStatus.finished:
+            finished_tasks.append(task)
+    submission.belonging_tasks = []
+
+    if download_terminated_log:
+        for task in terminated_tasks:
+            task.backward_files = [task.outlog, task.errlog]
+        submission.belonging_tasks += terminated_tasks
+    if download_finished_task:
+        submission.belonging_tasks += finished_tasks
+
+    submission.download_jobs()
+
+    if download_terminated_log:
+        terminated_log_files = []
+        for task in terminated_tasks:
+            assert submission.local_root is not None
+            terminated_log_files.append(
+                Path(submission.local_root) / task.task_work_path / task.outlog
+            )
+            terminated_log_files.append(
+                Path(submission.local_root) / task.task_work_path / task.errlog
+            )
+
+        dlog.info(
+            "Terminated logs are downloaded into:\n  "
+            + "\n  ".join([str(f) for f in terminated_log_files])
+        )
+
+    if clean:
+        submission.clean_jobs()

dpdispatcher/machine.py

@@ -4,10 +4,11 @@ import shlex
 from abc import ABCMeta, abstractmethod
 from typing import List, Tuple
 
+import yaml
 from dargs import Argument, Variant
 
-from dpdispatcher import dlog
 from dpdispatcher.base_context import BaseContext
+from dpdispatcher.dlog import dlog
 
 script_template = """\
 {script_header}
@@ -124,6 +125,13 @@ class Machine(metaclass=ABCMeta):
         machine = cls.load_from_dict(machine_dict=machine_dict)
         return machine
 
+    @classmethod
+    def load_from_yaml(cls, yaml_path):
+        with open(yaml_path) as f:
+            machine_dict = yaml.safe_load(f)
+        machine = cls.load_from_dict(machine_dict=machine_dict)
+        return machine
+
     @classmethod
     def load_from_dict(cls, machine_dict):
         batch_type = machine_dict["batch_type"]

dpdispatcher/machines/__init__.py

@@ -0,0 +1,11 @@
+"""Machines."""
+import importlib
+from pathlib import Path
+
+PACKAGE_BASE = "dpdispatcher.machines"
+NOT_LOADABLE = ("__init__.py",)
+
+for module_file in Path(__file__).parent.glob("*.py"):
+    if module_file.name not in NOT_LOADABLE:
+        module_name = f".{module_file.stem}"
+        importlib.import_module(module_name, PACKAGE_BASE)

dpdispatcher/{distributed_shell.py → machines/distributed_shell.py}

@@ -1,7 +1,7 @@
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
-from dpdispatcher.utils import (
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import (
     customized_script_header_template,
     run_cmd_with_all_output,
 )

dpdispatcher/{dp_cloud_server.py → machines/dp_cloud_server.py}

@@ -4,12 +4,12 @@ import time
 import uuid
 import warnings
 
-from dpdispatcher import dlog
-from dpdispatcher.dpcloudserver import Client, zip_file
-from dpdispatcher.dpcloudserver.config import ALI_OSS_BUCKET_URL
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
-from dpdispatcher.utils import customized_script_header_template
+from dpdispatcher.utils.dpcloudserver import Client, zip_file
+from dpdispatcher.utils.dpcloudserver.config import ALI_OSS_BUCKET_URL
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import customized_script_header_template
 
 shell_script_header_template = """
 #!/bin/bash -l

dpdispatcher/{fugaku.py → machines/fugaku.py}

@@ -1,9 +1,9 @@
 import shlex
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
-from dpdispatcher.utils import customized_script_header_template
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import customized_script_header_template
 
 fugaku_script_header_template = """\
 {queue_name_line}

dpdispatcher/{lsf.py → machines/lsf.py}

@@ -3,10 +3,14 @@ from typing import List
 
 from dargs import Argument
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
-from dpdispatcher.utils import RetrySignal, customized_script_header_template, retry
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import (
+    RetrySignal,
+    customized_script_header_template,
+    retry,
+)
 
 lsf_script_header_template = """\
 #!/bin/bash -l

dpdispatcher/{openapi.py → machines/openapi.py}

@@ -2,7 +2,7 @@ import os
 import shutil
 import time
 
-from dpdispatcher.utils import customized_script_header_template
+from dpdispatcher.utils.utils import customized_script_header_template
 
 try:
     from bohriumsdk.client import Client
@@ -14,9 +14,9 @@ except ModuleNotFoundError:
 else:
     found_bohriumsdk = True
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
+from dpdispatcher.utils.job_status import JobStatus
 
 shell_script_header_template = """
 #!/bin/bash -l
@@ -147,7 +147,9 @@ class OpenAPI(Machine):
                 )
 
         job_state = self.map_dp_job_state(
-            dp_job_status, check_return.get("exitCode", 0), self.ignore_exit_code  # type: ignore
+            dp_job_status,
+            check_return.get("exitCode", 0),  # type: ignore
+            self.ignore_exit_code,
         )
         if job_state == JobStatus.finished:
             job_log = self.job.log(job_id)

dpdispatcher/{pbs.py → machines/pbs.py}

@@ -1,9 +1,9 @@
 import shlex
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
-from dpdispatcher.utils import customized_script_header_template
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import customized_script_header_template
 
 pbs_script_header_template = """
 #!/bin/bash -l

dpdispatcher/{shell.py → machines/shell.py}

@@ -1,9 +1,9 @@
 import shlex
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
-from dpdispatcher.utils import customized_script_header_template
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import customized_script_header_template
 
 shell_script_header_template = """
 #!/bin/bash -l

dpdispatcher/{slurm.py → machines/slurm.py}

@@ -5,10 +5,14 @@ from typing import List
 
 from dargs import Argument
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine, script_command_template
-from dpdispatcher.utils import RetrySignal, customized_script_header_template, retry
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.utils import (
+    RetrySignal,
+    customized_script_header_template,
+    retry,
+)
 
 # from dpdispatcher.submission import Resources
 
@@ -20,6 +24,12 @@ slurm_script_header_template = """\
 {slurm_number_gpu_line}
 {slurm_partition_line}"""
 
+slurm_job_array_script_end_template = """
+wait
+
+{append_script_part}
+"""
+
 
 class Slurm(Machine):
     def gen_script(self, job):
@@ -296,9 +306,13 @@ class SlurmJobArray(Slurm):
         return script_command
 
     def gen_script_end(self, job):
-        # We cannot have a end script for job array
+        # We cannot touch tag for job array
         # we may check task tag instead
-        return ""
+        append_script = job.resources.append_script
+        append_script_part = "\n".join(append_script)
+        return slurm_job_array_script_end_template.format(
+            append_script_part=append_script_part,
+        )
 
     @retry()
     def check_status(self, job):

dpdispatcher/submission.py

@@ -9,15 +9,16 @@ import random
 import time
 import uuid
 from hashlib import sha1
-from typing import Optional
+from typing import List, Optional
 
+import yaml
 from dargs.dargs import Argument, Variant
 
-from dpdispatcher import dlog
-from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.dlog import dlog
 from dpdispatcher.machine import Machine
+from dpdispatcher.utils.job_status import JobStatus
+from dpdispatcher.utils.record import record
 
-# from dpdispatcher.slurm import SlurmResources
 # %%
 default_strategy = dict(if_cuda_multi_devices=False, ratio_unfinished=0.0)
 
@@ -248,9 +249,11 @@ class Submission:
                 time.sleep(check_interval)
             except (Exception, KeyboardInterrupt, SystemExit) as e:
                 self.submission_to_json()
+                record_path = record.write(self)
                 dlog.exception(e)
                 dlog.info(f"submission exit: {self.submission_hash}")
                 dlog.info(f"at {self.machine.context.remote_root}")
+                dlog.info(f"Submission information is saved in {str(record_path)}.")
                 dlog.debug(self.serialize())
                 raise e
             else:
@@ -273,6 +276,9 @@ class Submission:
             try:
                 self.download_jobs()
                 success = True
+            except FileNotFoundError as e:
+                # retry will never success if the file is not found
+                raise e
             except (EOFError, Exception) as e:
                 dlog.exception(e)
                 elapsed_time = time.time() - start_time
@@ -340,7 +346,6 @@ class Submission:
             dlog.debug(
                 f"debug:update_submission_state: job: {job.job_hash}, {job.job_id}, {job.job_state}"
             )
-        # self.submission_to_json()
 
     def handle_unexpected_submission_state(self):
         """Handle unexpected job state of the submission.
@@ -353,25 +358,16 @@ class Submission:
                 job.handle_unexpected_job_state()
         except Exception as e:
             self.submission_to_json()
+            record_path = record.write(self)
             raise RuntimeError(
                 f"Meet errors will handle unexpected submission state.\n"
                 f"Debug information: remote_root=={self.machine.context.remote_root}.\n"
                 f"Debug information: submission_hash=={self.submission_hash}.\n"
-                f"Please check the dirs and scripts in remote_root. "
-                f"The job information mentioned above may help."
+                f"Please check error messages above and in remote_root. "
+                f"The submission information is saved in {str(record_path)}.\n"
+                f"For furthur actions, run the following command with proper flags: dpdisp submission {self.submission_hash}"
             ) from e
 
-    # not used here, submitting job is in handle_unexpected_submission_state.
-
-    # def submit_submission(self):
-    #     """submit the job belonging to the submission.
-    #     """
-    #     for job in self.belonging_jobs:
-    #         job.submit_job()
-    #     self.get_submission_state()
-
-    # def update_submi
-
     def check_ratio_unfinished(self, ratio_unfinished: float) -> bool:
         """Calculate the ratio of unfinished tasks in the submission.
 
@@ -506,6 +502,8 @@ class Submission:
 
     def clean_jobs(self):
         self.machine.context.clean()
+        assert self.submission_hash is not None
+        record.remove(self.submission_hash)
 
     def submission_to_json(self):
         # self.update_submission_state()
@@ -615,6 +613,13 @@ class Task:
             task_dict = json.load(f)
         return cls.load_from_dict(task_dict)
 
+    @classmethod
+    def load_from_yaml(cls, yaml_file):
+        with open(yaml_file) as f:
+            task_dict = yaml.safe_load(f)
+        task = cls.load_from_dict(task_dict=task_dict)
+        return task
+
     @classmethod
     def load_from_dict(cls, task_dict: dict) -> "Task":
         # check dict
@@ -670,26 +675,30 @@ class Task:
             Argument("command", str, optional=False, doc=doc_command),
             Argument("task_work_path", str, optional=False, doc=doc_task_work_path),
             Argument(
-                "forward_files", list, optional=False, doc=doc_forward_files, default=[]
+                "forward_files",
+                List[str],
+                optional=True,
+                doc=doc_forward_files,
+                default=[],
             ),
             Argument(
                 "backward_files",
-                list,
-                optional=False,
+                List[str],
+                optional=True,
                 doc=doc_backward_files,
                 default=[],
             ),
             Argument(
                 "outlog",
                 [type(None), str],
-                optional=False,
+                optional=True,
                 doc=doc_outlog,
                 default="log",
             ),
             Argument(
                 "errlog",
                 [type(None), str],
-                optional=False,
+                optional=True,
                 doc=doc_errlog,
                 default="err",
             ),
@@ -836,16 +845,18 @@ class Job:
         if job_state == JobStatus.terminated:
             self.fail_count += 1
             dlog.info(
-                f"job: {self.job_hash} {self.job_id} terminated;"
+                f"job: {self.job_hash} {self.job_id} terminated; "
                 f"fail_cout is {self.fail_count}; resubmitting job"
             )
             retry_count = 3
             assert self.machine is not None
-            if hasattr(self.machine, "retry_count") and self.machine.retry_count > 0:
+            if hasattr(self.machine, "retry_count") and self.machine.retry_count >= 0:
                 retry_count = self.machine.retry_count + 1
             if (self.fail_count) > 0 and (self.fail_count % retry_count == 0):
                 last_error_message = self.get_last_error_message()
-                err_msg = f"job:{self.job_hash} {self.job_id} failed {self.fail_count} times. job_detail:{self}"
+                err_msg = (
+                    f"job:{self.job_hash} {self.job_id} failed {self.fail_count} times."
+                )
                 if last_error_message is not None:
                     err_msg += f"\nPossible remote error message: {last_error_message}"
                 raise RuntimeError(err_msg)
@@ -1092,7 +1103,14 @@ class Resources:
     def load_from_json(cls, json_file):
         with open(json_file) as f:
             resources_dict = json.load(f)
-        resources = cls.deserialize(resources_dict=resources_dict)
+        resources = cls.load_from_dict(resources_dict=resources_dict)
+        return resources
+
+    @classmethod
+    def load_from_yaml(cls, yaml_file):
+        with open(yaml_file) as f:
+            resources_dict = yaml.safe_load(f)
+        resources = cls.load_from_dict(resources_dict=resources_dict)
         return resources
 
     @classmethod
@@ -1175,36 +1193,40 @@ class Resources:
             ),
             Argument("queue_name", str, optional=True, doc=doc_queue_name, default=""),
             Argument("group_size", int, optional=False, doc=doc_group_size),
-            Argument("custom_flags", list, optional=True, doc=doc_custom_flags),
+            Argument("custom_flags", List[str], optional=True, doc=doc_custom_flags),
             # Argument("strategy", dict, optional=True, doc=doc_strategy,default=default_strategy),
             strategy_format,
             Argument("para_deg", int, optional=True, doc=doc_para_deg, default=1),
             Argument(
-                "source_list", list, optional=True, doc=doc_source_list, default=[]
+                "source_list", List[str], optional=True, doc=doc_source_list, default=[]
             ),
             Argument(
                 "module_purge", bool, optional=True, doc=doc_module_purge, default=False
             ),
             Argument(
                 "module_unload_list",
-                list,
+                List[str],
                 optional=True,
                 doc=doc_module_unload_list,
                 default=[],
             ),
             Argument(
-                "module_list", list, optional=True, doc=doc_module_list, default=[]
+                "module_list", List[str], optional=True, doc=doc_module_list, default=[]
             ),
             Argument("envs", dict, optional=True, doc=doc_envs, default={}),
             Argument(
                 "prepend_script",
-                list,
+                List[str],
                 optional=True,
                 doc=doc_prepend_script,
                 default=[],
             ),
             Argument(
-                "append_script", list, optional=True, doc=doc_append_script, default=[]
+                "append_script",
+                List[str],
+                optional=True,
+                doc=doc_append_script,
+                default=[],
             ),
             Argument(
                 "wait_time", [int, float], optional=True, doc=doc_wait_time, default=0

dpdispatcher/utils/__init__.py

@@ -0,0 +1 @@
+"""Utils."""

dpdispatcher/{dpcloudserver → utils/dpcloudserver}/client.py

@@ -6,7 +6,7 @@ from urllib.parse import urljoin
 
 import requests
 
-from dpdispatcher import dlog
+from dpdispatcher.dlog import dlog
 
 from .config import API_HOST, API_LOGGER_STACK_INFO, HTTP_TIME_OUT
 from .retcode import RETCODE

dpdispatcher/{hdfs_cli.py → utils/hdfs_cli.py}

@@ -2,7 +2,7 @@
 
 import os
 
-from dpdispatcher.utils import run_cmd_with_all_output
+from dpdispatcher.utils.utils import run_cmd_with_all_output
 
 
 class HDFS:

dpdispatcher/utils/record.py

@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+from typing import List
+
+
+class Record:
+    """Record failed or canceled submissions."""
+
+    def __init__(self) -> None:
+        self.record_directory = Path.home() / ".dpdispatcher" / "submission"
+        self.record_directory.mkdir(parents=True, exist_ok=True)
+
+    def get_submissions(self) -> List[str]:
+        """Get all stored submission hashes.
+
+        Returns
+        -------
+        list[str]
+            List of submission hashes.
+        """
+        return [
+            f.stem
+            for f in self.record_directory.iterdir()
+            if (f.is_file() and f.suffix == ".json")
+        ]
+
+    def write(self, submission) -> Path:
+        """Write submission data to file.
+
+        Parameters
+        ----------
+        submission : dpdispatcher.Submission
+            Submission data.
+
+        Returns
+        -------
+        pathlib.Path
+            Path to submission data.
+        """
+        submission_path = self.record_directory / f"{submission.submission_hash}.json"
+        submission_path.write_text(json.dumps(submission.serialize(), indent=2))
+        return submission_path
+
+    def get_submission(self, hash: str, not_exist_ok: bool = False) -> Path:
+        """Get submission data by hash.
+
+        Parameters
+        ----------
+        hash : str
+            Hash of submission data.
+
+        Returns
+        -------
+        pathlib.Path
+            Path to submission data.
+        """
+        submission_file = self.record_directory / f"{hash}.json"
+        if not not_exist_ok and not submission_file.is_file():
+            raise FileNotFoundError(f"Submission file not found: {submission_file}")
+        return submission_file
+
+    def remove(self, hash: str):
+        """Remove submission data by hash.
+
+        Call this method when the remote directory is cleaned.
+
+        Parameters
+        ----------
+        hash : str
+            Hash of submission data.
+        """
+        path = self.get_submission(hash, not_exist_ok=True)
+        if path.is_file():
+            path.unlink()
+
+
+# the record object can be globally used
+record = Record()
+__all__ = ["record"]

dpdispatcher/{utils.py → utils/utils.py}

@@ -7,7 +7,7 @@ import subprocess
 import time
 from typing import TYPE_CHECKING, Callable, Optional, Type, Union
 
-from dpdispatcher import dlog
+from dpdispatcher.dlog import dlog
 
 if TYPE_CHECKING:
     from dpdispatcher import Resources

{dpdispatcher-0.6.0.dist-info → dpdispatcher-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.6.0
+Version: 0.6.1
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License: GNU LESSER GENERAL PUBLIC LICENSE
@@ -186,9 +186,10 @@ Requires-Python: >=3.7
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: paramiko
-Requires-Dist: dargs >=0.2.9
+Requires-Dist: dargs >=0.4.1
 Requires-Dist: requests
 Requires-Dist: tqdm >=4.9.0
+Requires-Dist: pyyaml
 Requires-Dist: typing-extensions ; python_version < "3.7"
 Provides-Extra: bohrium
 Requires-Dist: oss2 ; extra == 'bohrium'

dpdispatcher-0.6.1.dist-info/RECORD

@@ -0,0 +1,44 @@
+dpdispatcher/__init__.py,sha256=CLZP_N5CTp14ujWCykEHuJjoIfKR6CwrclXhjWUgNoE,517
+dpdispatcher/__main__.py,sha256=BFhG-mSBzVZUEezQJqXWZnt2WsnhAHT_zpT8Y6gpOz0,116
+dpdispatcher/_version.py,sha256=lgiCYGSijhLK71WmuudWf_AyhNAutwQWx2V8bV6a5VQ,411
+dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
+dpdispatcher/base_context.py,sha256=NvaC_RHyspxq412z-eCq4Zn8-szZxvn8K6OkXvx7l4Y,3615
+dpdispatcher/dlog.py,sha256=ndh12teQBbJRybXd8UjEmAi6QTsAXajRicDj5mAH5h0,799
+dpdispatcher/dpdisp.py,sha256=YuGb-HWLsDfSO2c7GH0eM20ciojGbx3yq9oZHP7u4yc,3498
+dpdispatcher/machine.py,sha256=XFRH41gNCex_qs9gbg-S88_qab3_UAGfxKWUPxoipCM,16140
+dpdispatcher/submission.py,sha256=mVAHBlT0a3_1PtsEvvhvwNPkAhgLiBXXemX64BcwizU,48447
+dpdispatcher/contexts/__init__.py,sha256=s5M0ZJSrPttSyLdBwKD2m3W7a5AbYZdPB7IAND2j7EY,335
+dpdispatcher/contexts/dp_cloud_server_context.py,sha256=6XK0B2sLGEDeZmV2SZzQdVrMcWAWYZVLLK-IaShEXIY,12245
+dpdispatcher/contexts/hdfs_context.py,sha256=GJs_vmDCjTsnbfTdXpFTfpWTYXnZTDkEO2UJIdpV5F4,8908
+dpdispatcher/contexts/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
+dpdispatcher/contexts/local_context.py,sha256=7CoGzcX-RU6cpmSYcf4wMwncYaFVUb8Ljj4ksfXcx4s,13678
+dpdispatcher/contexts/openapi_context.py,sha256=DXaMS10SXN3VKEeEdzQyfOgRwUyHRJVCJHd2fKKdsmA,9499
+dpdispatcher/contexts/ssh_context.py,sha256=1UbMIFpSGBcg3H1My4cx5vjAALvaxlZxWOcXwoX6Ff0,38597
+dpdispatcher/entrypoints/__init__.py,sha256=exKSFT3j2oCerGwtI8WbHQK-D0K-CyifocRji1xntT4,20
+dpdispatcher/entrypoints/gui.py,sha256=29lMXqbmSRbLj4rfBv7Jnw89NLU9syTB88IUP6IRJsU,830
+dpdispatcher/entrypoints/submission.py,sha256=R2DXUGXTsk0Jw2y5Wjby40R5dxzXeqr4gn33ov6mdAI,2751
+dpdispatcher/machines/__init__.py,sha256=9kSYkz2w3flp00IrHWTEwvoFGrathQAT3tvbieye83c,335
+dpdispatcher/machines/distributed_shell.py,sha256=7avNcoOzEj7UcJuKl6b1ka2bj5dixcJaMlZK-I-i_Tc,7571
+dpdispatcher/machines/dp_cloud_server.py,sha256=SR69gsFb2BvOQCW1QnWfP3cQvu_qHLJNsycp5wzosJU,11706
+dpdispatcher/machines/fugaku.py,sha256=9OP3qSaaruqypHAdcuBFQM_MUtFp3yrvhZ5bPyLwEEk,4308
+dpdispatcher/machines/lsf.py,sha256=Qruot39cPEpBNbbPmDwb1Gyfgyw3N36O0hs9PNEXyVU,7997
+dpdispatcher/machines/openapi.py,sha256=Gzzbo8YOAybXGTrgMutexErcaEi3ts7uTUNvOhThFS8,8858
+dpdispatcher/machines/pbs.py,sha256=HGBUf96AJ7hWOQPrENP5tFIDnEm4cb9deqJJ8wExbms,7079
+dpdispatcher/machines/shell.py,sha256=qaia7mC_fz5Bqyelxmc1je-xg7NQ_6vQQ0qAjg2m4RQ,4796
+dpdispatcher/machines/slurm.py,sha256=SP5rQiCPWzq4rqgUgp0IGJXXD_1DURWl4OBRAJ-Kng4,15611
+dpdispatcher/utils/__init__.py,sha256=fwvwkMf7DFNQkNBiIce8Y8gRA6FhICwKjkKiXu_BEJg,13
+dpdispatcher/utils/hdfs_cli.py,sha256=Fy36JTrfdhuxGbaHe1hYY0KrlNp06Tbjwo5wpj4ph-8,5434
+dpdispatcher/utils/job_status.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
+dpdispatcher/utils/record.py,sha256=c8jdPmCuLzRmFo_jOjR0j9zFR1EWX3NSHVuPEIYCycg,2147
+dpdispatcher/utils/utils.py,sha256=1One9eW-v3ejDcL6PB9PSCMZQkalnbxq0DfJoUwQaLs,5334
+dpdispatcher/utils/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
+dpdispatcher/utils/dpcloudserver/client.py,sha256=CLfXswvzI4inDrW2bYkfMQ6gQJFcZOgLhiXBz_EI17M,12029
+dpdispatcher/utils/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
+dpdispatcher/utils/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
+dpdispatcher/utils/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
+dpdispatcher-0.6.1.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
+dpdispatcher-0.6.1.dist-info/METADATA,sha256=TKf52k3Vt9neBJNWXCimoMSmEkvC5ubCjVqUCPOa8_8,12752
+dpdispatcher-0.6.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+dpdispatcher-0.6.1.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
+dpdispatcher-0.6.1.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
+dpdispatcher-0.6.1.dist-info/RECORD,,

{dpdispatcher-0.6.0.dist-info → dpdispatcher-0.6.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.2)
+Generator: bdist_wheel (0.41.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dpdispatcher/dpcloudserver/temp_test.py

@@ -1,90 +0,0 @@
-# %%
-import os
-import sys
-import unittest
-import uuid
-
-from dpdispatcher.dpcloudserver import api
-from dpdispatcher.dpcloudserver.zip_file import zip_files
-
-# import api
-# from zip_file import zip_files
-
-
-# %%
-class DPTest(unittest.TestCase):
-    test_data = {
-        "job_type": "indicate",
-        "log_file": "mylog",
-        "command": "( echo aa && lmp -i input.lammps && sleep 900 ) > dp.log 2>&1",
-        "backward_files": [],
-        "job_name": "dpdispatcher_lammps_test",
-        "machine": {
-            "platform": "ali",
-            "resources": {
-                "gpu_type": "1 * NVIDIA P100",
-                "cpu_num": 4,
-                "mem_limit": 28,
-                "time_limit": "2:00:00",
-                "image_name": "yfb-deepmd-kit-1.2.4-cuda10",
-            },
-        },
-        "job_resources": "http://dpcloudserver.oss-cn-shenzhen.aliyuncs.com/dpcloudserver/indicate/a657ff49722839f1ee54edeb3e9b1beb0ee5cc0e/a657ff49722839f1ee54edeb3e9b1beb0ee5cc0e.zip",
-    }
-
-    username = ""
-    password = ""
-
-    ENDPOINT = "http://oss-cn-shenzhen.aliyuncs.com"
-    BUCKET_NAME = "dpcloudserver"
-
-    @classmethod
-    def setUpClass(cls):
-        print("execute", sys._getframe().f_code.co_name)
-
-    @classmethod
-    def tearDownClass(cls):
-        print("execute", sys._getframe().f_code.co_name)
-
-    def setUp(self):
-        print("execute", sys._getframe().f_code.co_name)
-        api.login(self.username, self.password)
-
-    def test_commit_job(self):
-        print("----------", sys._getframe().f_code.co_name)
-        file_uuid = uuid.uuid1().hex
-        oss_task_zip = os.path.join(
-            "{}/{}/{}.zip".format("indicate", file_uuid, file_uuid)
-        )
-        zip_path = "/home/felix/workplace/22_dpdispatcher/dpdispatcher-yfb/dpdispatcher/dpcloudserver/t.txt"
-        zip_task_file = zip_path + ".zip"
-        zip_files(zip_path, zip_task_file, [])
-        api.upload(oss_task_zip, zip_task_file, self.ENDPOINT, self.BUCKET_NAME)
-        job_id = api.job_create(
-            self.test_data["job_type"], self.test_data["job_resources"], self.test_data
-        )
-        tasks = api.get_job_detail(job_id)
-        print(tasks)
-
-    def test_get_tasks(self):
-        print("----------", sys._getframe().f_code.co_name)
-        jobs = api.get_jobs()
-        for j in jobs:
-            tasks = api.get_job_detail(j["id"])
-            print(tasks)
-
-    # def test_download(self):
-    #     print('----------', sys._getframe().f_code.co_name)
-    #     oss_path = 'dpcloudserver/indicate/abe0febc92ce11eb990800163e094dc5/abe0febc92ce11eb990800163e094dc5.zip'
-    #     api.download(oss_path, "out.zip", self.ENDPOINT, self.BUCKET_NAME)
-
-
-if __name__ == "__main__":
-    suite = unittest.TestSuite()
-
-    suite.addTest(DPTest("test_commit_job"))
-    # suite.addTest(DPTest("test_get_tasks"))
-    # suite.addTest(DPTest("test_download"))
-
-    runner = unittest.TextTestRunner()
-    runner.run(suite)

dpdispatcher-0.6.0.dist-info/RECORD

@@ -1,37 +0,0 @@
-dpdispatcher/JobStatus.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
-dpdispatcher/__init__.py,sha256=i33piTZkPDYIm_qiTEI_J3bmHlFayEN1T8T8TyzYdqg,3017
-dpdispatcher/_version.py,sha256=2JKwcA-YQ0okV2N-gwTWy_n51igWrPcsKQFm0cnqsvw,411
-dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
-dpdispatcher/base_context.py,sha256=Hfri0x41XC4MRUjxc0-WMiZB_E4NvLp94ZYaHfYCWHM,3610
-dpdispatcher/distributed_shell.py,sha256=ZLz-1GKrjHjeW6_GXKfFufFbMaFuy3RpJz0hoWRRZP0,7553
-dpdispatcher/dp_cloud_server.py,sha256=I0yHJtovINmro0LPjWeHnbGahIpS3OkB4ooq4IgjELo,11676
-dpdispatcher/dp_cloud_server_context.py,sha256=S81jLpwdur9Z4TptkmLs_7vuleLVeRXyeD1nvUgDodY,12203
-dpdispatcher/dpdisp.py,sha256=M5yaAmpyQJ6q_kcclWGu7ixXm5qYN9XbXpqzrboVGwI,2138
-dpdispatcher/fugaku.py,sha256=_F-dRI8JniTLgldpfwj5lsX10YR6BfH-x95OPSPF4G4,4290
-dpdispatcher/gui.py,sha256=29lMXqbmSRbLj4rfBv7Jnw89NLU9syTB88IUP6IRJsU,830
-dpdispatcher/hdfs_cli.py,sha256=SkCA-dgZbcS2a6Z96XzUB3-T3Barkak_5S7wP7WF55o,5428
-dpdispatcher/hdfs_context.py,sha256=mO12yMoASGSh8erumVxMOPnHGdhOZuyaPyir9hfY0ZY,8897
-dpdispatcher/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
-dpdispatcher/local_context.py,sha256=yCmEDsxTUmDGsH9iPGamk7KOME2Yb7jI2reyDDpAsFI,13948
-dpdispatcher/lsf.py,sha256=fLJ1FaaJpSrdYxs-h2RGkOkIMPOnT_owNMwlmMbK_0g,7962
-dpdispatcher/machine.py,sha256=oUptUUhWjVfRe82WGGu0WEMovlIB5YqCgxQOAdx6wbw,15898
-dpdispatcher/openapi.py,sha256=RGBDQ2KOICm0rDJOtf9MydGQ6VhrsecGfPGGUCVlURo,8815
-dpdispatcher/openapi_context.py,sha256=IboZ5Y7pwKfrxe9tot1JroxQuo85tbVsmlgkWeKbJAs,9487
-dpdispatcher/pbs.py,sha256=JitABpPC84m14Gu5TJ6eyYXtFTN1Yc5WbMKWsAjy8OI,7061
-dpdispatcher/shell.py,sha256=_iKGxLcjEZViqjLNfXXMx_DywSnRHCdGjzg7176kND0,4778
-dpdispatcher/slurm.py,sha256=wqcFrP0oxXneZuLjMb3k4JYatfxlFo4dKGIHAqysRyc,15302
-dpdispatcher/ssh_context.py,sha256=r2Xn-yFGCtTTia-NpILgqiSeK_SecJRGGNdxwy1q1_k,37871
-dpdispatcher/submission.py,sha256=N9tEC6qMQBYoY4t3q7GZNaz6SvoWo6EyN1aRJOGT8yY,47588
-dpdispatcher/utils.py,sha256=9gYNpTkWoNXCcDakizlWFO5S5p_1eBHUglie_ZjeJoc,5329
-dpdispatcher/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
-dpdispatcher/dpcloudserver/client.py,sha256=52d_ftHk_8kpC3TNcGjXkWXLNQ0U8yYZ2l11L0DzxVg,12024
-dpdispatcher/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
-dpdispatcher/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
-dpdispatcher/dpcloudserver/temp_test.py,sha256=jklOSu7tZ_wW5gycGRiUsbBWMLZDqCBslSYOCb2hTHw,2932
-dpdispatcher/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
-dpdispatcher-0.6.0.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
-dpdispatcher-0.6.0.dist-info/METADATA,sha256=AmajK6Ht5EY_RD_cMwnK0Gnnd2zECCo0E_xS_r_ysHg,12730
-dpdispatcher-0.6.0.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-dpdispatcher-0.6.0.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
-dpdispatcher-0.6.0.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
-dpdispatcher-0.6.0.dist-info/RECORD,,