dpdispatcher 0.5.7__tar.gz → 0.5.8__tar.gz

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.5.7
+Version: 0.5.8
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/doc/batch.md

@@ -62,3 +62,11 @@ Read Bohrium documentation for details.
 
 `DistributedShell` is used to submit yarn jobs.
 Read [Support DPDispatcher on Yarn](dpdispatcher_on_yarn.md) for details.
+
+## Fugaku
+
+{dargs:argument}`batch_type <resources/batch_type>`: `Fugaku`
+
+[Fujitsu cloud service](https://doc.cloud.global.fujitsu.com/lib/common/jp/hpc-user-manual/) is a job scheduling system used by Fujitsu's HPCs such as Fugaku, ITO and K computer. It should be noted that although the same job scheduling system is used, there are some differences in the details, Fagaku class cannot be directly used for other HPCs.
+
+Read Fujitsu cloud service documentation for details.

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/dpdispatcher/__init__.py

@@ -43,6 +43,7 @@ except ImportError:
 from .distributed_shell import DistributedShell
 from .dp_cloud_server import DpCloudServer, Lebesgue
 from .dp_cloud_server_context import DpCloudServerContext, LebesgueContext
+from .fugaku import Fugaku
 from .hdfs_context import HDFSContext
 from .lazy_local_context import LazyLocalContext
 from .local_context import LocalContext
@@ -85,6 +86,7 @@ __all__ = [
     "PBS",
     "Shell",
     "Slurm",
+    "Fugaku",
     "SSHContext",
     "Submission",
     "Task",

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/dpdispatcher/_version.py

@@ -1,4 +1,4 @@
 # file generated by setuptools_scm
 # don't change, don't track in version control
-__version__ = version = '0.5.7'
-__version_tuple__ = version_tuple = (0, 5, 7)
+__version__ = version = '0.5.8'
+__version_tuple__ = version_tuple = (0, 5, 8)

dpdispatcher-0.5.8/dpdispatcher/fugaku.py

@@ -0,0 +1,94 @@
+import shlex
+
+from dpdispatcher import dlog
+from dpdispatcher.JobStatus import JobStatus
+from dpdispatcher.machine import Machine
+
+fugaku_script_header_template = """\
+{queue_name_line}
+{fugaku_node_number_line}
+{fugaku_ntasks_per_node_line}
+"""
+
+
+class Fugaku(Machine):
+    def gen_script(self, job):
+        fugaku_script = super().gen_script(job)
+        return fugaku_script
+
+    def gen_script_header(self, job):
+        resources = job.resources
+        fugaku_script_header_dict = {}
+        fugaku_script_header_dict[
+            "fugaku_node_number_line"
+        ] = f'#PJM -L "node={resources.number_node}" '
+        fugaku_script_header_dict[
+            "fugaku_ntasks_per_node_line"
+        ] = '#PJM --mpi "max-proc-per-node={cpu_per_node}"'.format(
+            cpu_per_node=resources.cpu_per_node
+        )
+        fugaku_script_header_dict[
+            "queue_name_line"
+        ] = f'#PJM -L "rscgrp={resources.queue_name}"'
+        fugaku_script_header = fugaku_script_header_template.format(
+            **fugaku_script_header_dict
+        )
+        return fugaku_script_header
+
+    def do_submit(self, job):
+        script_file_name = job.script_file_name
+        script_str = self.gen_script(job)
+        job_id_name = job.job_hash + "_job_id"
+        # script_str = self.sub_script(job_dirs, cmd, args=args, resources=resources, outlog=outlog, errlog=errlog)
+        self.context.write_file(fname=script_file_name, write_str=script_str)
+        # self.context.write_file(fname=os.path.join(self.context.submission.work_base, script_file_name), write_str=script_str)
+        # script_file_dir = os.path.join(self.context.submission.work_base)
+        script_file_dir = self.context.remote_root
+        # stdin, stdout, stderr = self.context.block_checkcall('cd %s && %s %s' % (self.context.remote_root, 'pjsub', script_file_name))
+
+        stdin, stdout, stderr = self.context.block_checkcall(
+            "cd {} && {} {}".format(
+                shlex.quote(script_file_dir), "pjsub", shlex.quote(script_file_name)
+            )
+        )
+        subret = stdout.readlines()
+        job_id = subret[0].split()[5]
+        self.context.write_file(job_id_name, job_id)
+        return job_id
+
+    def default_resources(self, resources):
+        pass
+
+    def check_status(self, job):
+        job_id = job.job_id
+        if job_id == "":
+            return JobStatus.unsubmitted
+        ret, stdin, stdout, stderr = self.context.block_call("pjstat " + job_id)
+        err_str = stderr.read().decode("utf-8")
+        try:
+            status_line = stdout.read().decode("utf-8").split("\n")[-2]
+        # pjstat only retrun 0 if the job is not waiting or running
+        except Exception:
+            ret, stdin, stdout, stderr = self.context.block_call("pjstat -H  " + job_id)
+            status_line = stdout.read().decode("utf-8").split("\n")[-2]
+            status_word = status_line.split()[3]
+            if status_word in ["EXT", "CCL", "ERR"]:
+                if self.check_finish_tag(job):
+                    dlog.info(f"job: {job.job_hash} {job.job_id} finished")
+                    return JobStatus.finished
+                else:
+                    return JobStatus.terminated
+            else:
+                return JobStatus.unknown
+        status_word = status_line.split()[3]
+        # dlog.info (status_word)
+        if status_word in ["QUE", "HLD", "RNA", "SPD"]:
+            return JobStatus.waiting
+        elif status_word in ["RUN", "RNE"]:
+            return JobStatus.running
+        else:
+            return JobStatus.unknown
+
+    def check_finish_tag(self, job):
+        job_tag_finished = job.job_hash + "_job_tag_finished"
+        return self.context.check_file_exists(job_tag_finished)

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/dpdispatcher/submission.py

@@ -1,5 +1,7 @@
 # %%
+import asyncio
 import copy
+import functools
 import json
 import os
 import pathlib
@@ -199,7 +201,9 @@ class Submission:
             self.local_root = machine.context.temp_local_root
         return self
 
-    def run_submission(self, *, dry_run=False, exit_on_submit=False, clean=True):
+    def run_submission(
+        self, *, dry_run=False, exit_on_submit=False, clean=True, check_interval=30
+    ):
         """Main method to execute the submission.
         First, check whether old Submission exists on the remote machine, and try to recover from it.
         Second, upload the local files to the remote machine where the tasks to be executed.
@@ -240,7 +244,7 @@ class Submission:
                 break
 
             try:
-                time.sleep(30)
+                time.sleep(check_interval)
             except (Exception, KeyboardInterrupt, SystemExit) as e:
                 self.submission_to_json()
                 dlog.exception(e)
@@ -254,12 +258,72 @@ class Submission:
             finally:
                 pass
         self.handle_unexpected_submission_state()
-        self.download_jobs()
+        self.try_download_result()
         self.submission_to_json()
         if clean:
             self.clean_jobs()
         return self.serialize()
 
+    def try_download_result(self):
+        start_time = time.time()
+        retry_interval = 60  # 每1分钟重试一次
+        success = False
+        while not success:
+            try:
+                self.download_jobs()
+                success = True
+            except (EOFError, Exception) as e:
+                dlog.exception(e)
+                elapsed_time = time.time() - start_time
+                if elapsed_time < 3600:  # 1小时内
+                    dlog.info("Retrying in 1 minute...")
+                    time.sleep(retry_interval)
+                elif elapsed_time < 86400:  # 1小时后，但在24小时内
+                    retry_interval = 600  # 每10分钟重试一次
+                    dlog.info("Retrying in 10 minutes...")
+                    time.sleep(retry_interval)
+                else:  # 超过24小时
+                    dlog.info("Maximum retries time reached. Exiting.")
+                    break
+
+    async def async_run_submission(self, **kwargs):
+        """Async interface of run_submission.
+
+        Examples
+        --------
+        >>> import asyncio
+        >>> from dpdispacher import Machine, Resource, Submission
+        >>> async def run_jobs():
+        ...     backgroud_task = set()
+        ...     # task1
+        ...     task1 = Task(...)
+        ...     submission1 = Submission(..., task_list=[task1])
+        ...     background_task = asyncio.create_task(
+        ...         submission1.async_run_submission(check_interval=2, clean=False)
+        ...     )
+        ...     # task2
+        ...     task2 = Task(...)
+        ...     submission2 = Submission(..., task_list=[task1])
+        ...     background_task = asyncio.create_task(
+        ...         submission2.async_run_submission(check_interval=2, clean=False)
+        ...     )
+        ...     background_tasks.add(background_task)
+        ...     result = await asyncio.gather(*background_tasks)
+        ...     return result
+        >>> run_jobs()
+
+        May raise Error if pass `clean=True` explicitly when submit to pbs or slurm.
+        """
+        kwargs = {**{"clean": False}, **kwargs}
+        if kwargs["clean"]:
+            dlog.warning(
+                "Using async submission with `clean=True`, "
+                "job may fail in queue system"
+            )
+        loop = asyncio.get_event_loop()
+        wrapped_submission = functools.partial(self.run_submission, **kwargs)
+        return await loop.run_in_executor(None, wrapped_submission)
+
     def update_submission_state(self):
         """Check whether all the jobs in the submission.
 

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/dpdispatcher.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dpdispatcher
-Version: 0.5.7
+Version: 0.5.8
 Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
 Author: DeepModeling
 License:                    GNU LESSER GENERAL PUBLIC LICENSE

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/dpdispatcher.egg-info/SOURCES.txt

@@ -55,6 +55,7 @@ dpdispatcher/distributed_shell.py
 dpdispatcher/dp_cloud_server.py
 dpdispatcher/dp_cloud_server_context.py
 dpdispatcher/dpdisp.py
+dpdispatcher/fugaku.py
 dpdispatcher/hdfs_cli.py
 dpdispatcher/hdfs_context.py
 dpdispatcher/lazy_local_context.py
@@ -131,10 +132,12 @@ tests/jsons/machine_ali_ehpc.json
 tests/jsons/machine_center.json
 tests/jsons/machine_diffenert.json
 tests/jsons/machine_dp_cloud_server.json
+tests/jsons/machine_fugaku.json
 tests/jsons/machine_if_cuda_multi_devices.json
 tests/jsons/machine_lazy_local_lsf.json
 tests/jsons/machine_lazy_local_slurm.json
 tests/jsons/machine_lazylocal_shell.json
+tests/jsons/machine_local_fugaku.json
 tests/jsons/machine_local_shell.json
 tests/jsons/machine_lsf.json
 tests/jsons/machine_slurm.json

dpdispatcher-0.5.8/tests/jsons/machine_fugaku.json

@@ -0,0 +1,24 @@
+{
+    "machine": {
+        "batch_type": "fugaku",
+        "context_type": "SSHContext",
+        "local_root" : "./",
+        "remote_profile": {
+                "hostname": "login.****.jp",
+                "key_filename":"/home/***/.ssh/***",
+                "passphrase":"******",
+                "username": "u*****"
+        },
+        "remote_root": "/vol*****/data/****"
+},
+    "resources": {
+            "number_node": 1,
+            "cpu_per_node": 48,
+            "source_list": [""],
+            "queue_name": "small",
+            "group_size": 1,
+            "custom_flags" : ["#PJM -L \"elapse=4:00:00\"",
+            "#PJM -x PJM_LLIO_GFSCACHE=/vol0004",
+            "#PJM -g hp******"]
+    }
+}

dpdispatcher-0.5.8/tests/jsons/machine_local_fugaku.json

@@ -0,0 +1,18 @@
+{
+    "machine": {
+        "batch_type": "fugaku",
+        "context_type": "local",
+        "local_root" : "./",
+        "remote_root": "./"
+    },
+    "resources": {
+            "number_node": 1,
+            "cpu_per_node": 48,
+            "source_list": [""],
+            "queue_name": "small",
+            "group_size": 1,
+            "custom_flags" : ["#PJM -L \"elapse=4:00:00\"",
+            "#PJM -x PJM_LLIO_GFSCACHE=/vol0004",
+            "#PJM -g hp******"]
+    }
+}

{dpdispatcher-0.5.7 → dpdispatcher-0.5.8}/tests/test_run_submission.py

@@ -1,4 +1,6 @@
+import asyncio
 import os
+import random
 import shutil
 import sys
 
@@ -85,7 +87,51 @@ class RunSubmission:
             backward_common_files=[],
             task_list=task_list,
         )
-        submission.run_submission()
+        submission.run_submission(check_interval=2)
+
+        for ii in range(4):
+            self.assertTrue(
+                os.path.isfile(
+                    os.path.join(
+                        self.machine_dict["local_root"], "test_dir/", f"out{ii}.txt"
+                    )
+                )
+            )
+
+    def test_async_run_submission(self):
+        machine = Machine.load_from_dict(self.machine_dict)
+        resources = Resources.load_from_dict(self.resources_dict)
+        ntask = 4
+
+        async def run_jobs(ntask):
+            background_tasks = set()
+            for ii in range(ntask):
+                sleep_time = random.random() * 5 + 2
+                task = Task(
+                    command=f"echo dpdispatcher_unittest_{ii} && sleep {sleep_time}",
+                    task_work_path="./",
+                    forward_files=[],
+                    backward_files=[f"out{ii}.txt"],
+                    outlog=f"out{ii}.txt",
+                )
+                submission = Submission(
+                    work_base="test_dir/",
+                    machine=machine,
+                    resources=resources,
+                    forward_common_files=[],
+                    backward_common_files=[],
+                    task_list=[task],
+                )
+                background_task = asyncio.create_task(
+                    submission.async_run_submission(check_interval=2, clean=False)
+                )
+                background_tasks.add(background_task)
+                # background_task.add_done_callback(background_tasks.discard)
+            res = await asyncio.gather(*background_tasks)
+            return res
+
+        res = asyncio.run(run_jobs(ntask=ntask))
+        print(res)
 
         for ii in range(4):
             self.assertTrue(
@@ -110,6 +156,10 @@ class TestSlurmRun(RunSubmission, unittest.TestCase):
         self.machine_dict["batch_type"] = "Slurm"
         self.resources_dict["queue_name"] = "normal"
 
+    @unittest.skip("Manaually skip")  # comment this line to open unittest
+    def test_async_run_submission(self):
+        return super().test_async_run_submission()
+
 
 @unittest.skipIf(
     os.environ.get("DPDISPATCHER_TEST") != "slurm",
@@ -121,6 +171,10 @@ class TestSlurmJobArrayRun(RunSubmission, unittest.TestCase):
         self.machine_dict["batch_type"] = "SlurmJobArray"
         self.resources_dict["queue_name"] = "normal"
 
+    @unittest.skip("Manaually skip")  # comment this line to open unittest
+    def test_async_run_submission(self):
+        return super().test_async_run_submission()
+
 
 @unittest.skipIf(
     os.environ.get("DPDISPATCHER_TEST") != "slurm",
@@ -133,6 +187,10 @@ class TestSlurmJobArrayRun2(RunSubmission, unittest.TestCase):
         self.resources_dict["queue_name"] = "normal"
         self.resources_dict["kwargs"] = {"slurm_job_size": 2}
 
+    @unittest.skip("Manaually skip")  # comment this line to open unittest
+    def test_async_run_submission(self):
+        return super().test_async_run_submission()
+
 
 @unittest.skipIf(
     os.environ.get("DPDISPATCHER_TEST") != "pbs", "outside the pbs testing environment"
@@ -143,6 +201,10 @@ class TestPBSRun(RunSubmission, unittest.TestCase):
         self.machine_dict["batch_type"] = "PBS"
         self.resources_dict["queue_name"] = "workq"
 
+    @unittest.skip("Manaually skip")  # comment this line to open unittest
+    def test_async_run_submission(self):
+        return super().test_async_run_submission()
+
 
 @unittest.skipIf(sys.platform == "win32", "Shell is not supported on Windows")
 class TestLazyLocalContext(RunSubmission, unittest.TestCase):