dpdispatcher 0.6.7__py3-none-any.whl → 0.6.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of dpdispatcher might be problematic. Click here for more details.
- dpdispatcher/_version.py +2 -2
- dpdispatcher/base_context.py +1 -2
- dpdispatcher/contexts/ssh_context.py +4 -3
- dpdispatcher/machines/JH_UniScheduler.py +2 -3
- dpdispatcher/machines/distributed_shell.py +2 -4
- dpdispatcher/machines/lsf.py +1 -2
- dpdispatcher/machines/pbs.py +7 -6
- dpdispatcher/machines/shell.py +2 -4
- dpdispatcher/machines/slurm.py +9 -15
- dpdispatcher/submission.py +1 -8
- dpdispatcher/utils/hdfs_cli.py +6 -11
- dpdispatcher/utils/utils.py +1 -1
- {dpdispatcher-0.6.7.dist-info → dpdispatcher-0.6.8.dist-info}/METADATA +25 -25
- {dpdispatcher-0.6.7.dist-info → dpdispatcher-0.6.8.dist-info}/RECORD +18 -18
- {dpdispatcher-0.6.7.dist-info → dpdispatcher-0.6.8.dist-info}/WHEEL +1 -1
- {dpdispatcher-0.6.7.dist-info → dpdispatcher-0.6.8.dist-info}/LICENSE +0 -0
- {dpdispatcher-0.6.7.dist-info → dpdispatcher-0.6.8.dist-info}/entry_points.txt +0 -0
- {dpdispatcher-0.6.7.dist-info → dpdispatcher-0.6.8.dist-info}/top_level.txt +0 -0
dpdispatcher/_version.py
CHANGED
dpdispatcher/base_context.py
CHANGED
|
@@ -102,8 +102,7 @@ class BaseContext(metaclass=ABCMeta):
|
|
|
102
102
|
exit_status, stdin, stdout, stderr = self.block_call(cmd)
|
|
103
103
|
if exit_status != 0:
|
|
104
104
|
raise RuntimeError(
|
|
105
|
-
"Get error code
|
|
106
|
-
% (
|
|
105
|
+
"Get error code {} in calling {} with job: {} . message: {}".format(
|
|
107
106
|
exit_status,
|
|
108
107
|
cmd,
|
|
109
108
|
self.submission.submission_hash,
|
|
@@ -90,8 +90,7 @@ class SSHSession:
|
|
|
90
90
|
while not self._check_alive():
|
|
91
91
|
if count == max_check:
|
|
92
92
|
raise RuntimeError(
|
|
93
|
-
"cannot connect ssh after
|
|
94
|
-
% (max_check, sleep_time)
|
|
93
|
+
f"cannot connect ssh after {max_check} failures at interval {sleep_time} s"
|
|
95
94
|
)
|
|
96
95
|
dlog.info("connection check failed, try to reconnect to " + self.hostname)
|
|
97
96
|
self._setup_ssh()
|
|
@@ -450,7 +449,9 @@ class SSHContext(BaseContext):
|
|
|
450
449
|
self.init_local_root = local_root
|
|
451
450
|
self.init_remote_root = remote_root
|
|
452
451
|
self.temp_local_root = os.path.abspath(local_root)
|
|
453
|
-
assert os.path.isabs(remote_root),
|
|
452
|
+
assert os.path.isabs(os.path.realpath(remote_root)), (
|
|
453
|
+
"remote_root must be a abspath"
|
|
454
|
+
)
|
|
454
455
|
self.temp_remote_root = remote_root
|
|
455
456
|
self.remote_profile = remote_profile
|
|
456
457
|
self.remote_root = None
|
|
@@ -39,7 +39,7 @@ class JH_UniScheduler(Machine):
|
|
|
39
39
|
custom_gpu_line = resources.kwargs.get("custom_gpu_line", None)
|
|
40
40
|
if not custom_gpu_line:
|
|
41
41
|
script_header_dict["JH_UniScheduler_number_gpu_line"] = (
|
|
42
|
-
|
|
42
|
+
f"#JSUB -gpgpu {resources.gpu_per_node}"
|
|
43
43
|
)
|
|
44
44
|
else:
|
|
45
45
|
script_header_dict["JH_UniScheduler_number_gpu_line"] = custom_gpu_line
|
|
@@ -105,8 +105,7 @@ class JH_UniScheduler(Machine):
|
|
|
105
105
|
elif ret != 0:
|
|
106
106
|
# just retry when any unknown error raised.
|
|
107
107
|
raise RetrySignal(
|
|
108
|
-
"Get error code
|
|
109
|
-
% (ret, job.job_hash, err_str)
|
|
108
|
+
f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
|
|
110
109
|
)
|
|
111
110
|
status_out = stdout.read().decode("utf-8").split("\n")
|
|
112
111
|
if len(status_out) < 2:
|
|
@@ -181,8 +181,7 @@ class DistributedShell(Machine):
|
|
|
181
181
|
if ret != 0:
|
|
182
182
|
err_str = stderr.decode("utf-8")
|
|
183
183
|
raise RuntimeError(
|
|
184
|
-
"Command
|
|
185
|
-
% (cmd, err_str, ret)
|
|
184
|
+
f"Command {cmd} fails to execute, error message:{err_str}\nreturn code {ret}\n"
|
|
186
185
|
)
|
|
187
186
|
job_id = int(stdout.decode("utf-8").strip())
|
|
188
187
|
|
|
@@ -200,8 +199,7 @@ class DistributedShell(Machine):
|
|
|
200
199
|
if ret != 0:
|
|
201
200
|
err_str = stderr.decode("utf-8")
|
|
202
201
|
raise RuntimeError(
|
|
203
|
-
"Command fails to execute, error message
|
|
204
|
-
% (err_str, ret)
|
|
202
|
+
f"Command fails to execute, error message:{err_str}\nreturn code {ret}\n"
|
|
205
203
|
)
|
|
206
204
|
|
|
207
205
|
if_job_exists = bool(stdout.decode("utf-8").strip())
|
dpdispatcher/machines/lsf.py
CHANGED
|
@@ -129,8 +129,7 @@ class LSF(Machine):
|
|
|
129
129
|
elif ret != 0:
|
|
130
130
|
# just retry when any unknown error raised.
|
|
131
131
|
raise RetrySignal(
|
|
132
|
-
"Get error code
|
|
133
|
-
% (ret, job.job_hash, err_str)
|
|
132
|
+
f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
|
|
134
133
|
)
|
|
135
134
|
status_out = stdout.read().decode("utf-8").split("\n")
|
|
136
135
|
if len(status_out) < 2:
|
dpdispatcher/machines/pbs.py
CHANGED
|
@@ -87,8 +87,7 @@ class PBS(Machine):
|
|
|
87
87
|
return JobStatus.terminated
|
|
88
88
|
else:
|
|
89
89
|
raise RuntimeError(
|
|
90
|
-
"status command
|
|
91
|
-
% (command, err_str, ret)
|
|
90
|
+
f"status command {command} fails to execute. erro info: {err_str} return code {ret}"
|
|
92
91
|
)
|
|
93
92
|
status_line = stdout.read().decode("utf-8").split("\n")[-2]
|
|
94
93
|
status_word = status_line.split()[-2]
|
|
@@ -138,8 +137,7 @@ class Torque(PBS):
|
|
|
138
137
|
return JobStatus.terminated
|
|
139
138
|
else:
|
|
140
139
|
raise RuntimeError(
|
|
141
|
-
"status command
|
|
142
|
-
% (command, err_str, ret)
|
|
140
|
+
f"status command {command} fails to execute. erro info: {err_str} return code {ret}"
|
|
143
141
|
)
|
|
144
142
|
status_line = stdout.read().decode("utf-8").split("\n")[-2]
|
|
145
143
|
status_word = status_line.split()[-2]
|
|
@@ -261,6 +259,7 @@ class SGE(PBS):
|
|
|
261
259
|
pass
|
|
262
260
|
|
|
263
261
|
def check_status(self, job):
|
|
262
|
+
### https://softpanorama.org/HPC/Grid_engine/Queues/queue_states.shtml
|
|
264
263
|
job_id = job.job_id
|
|
265
264
|
status_line = None
|
|
266
265
|
if job_id == "":
|
|
@@ -294,10 +293,12 @@ class SGE(PBS):
|
|
|
294
293
|
else:
|
|
295
294
|
status_word = status_line.split()[4]
|
|
296
295
|
# dlog.info (status_word)
|
|
297
|
-
if status_word in ["qw"]:
|
|
296
|
+
if status_word in ["qw", "hqw", "t"]:
|
|
298
297
|
return JobStatus.waiting
|
|
299
|
-
elif status_word in ["r"]:
|
|
298
|
+
elif status_word in ["r", "Rr"]:
|
|
300
299
|
return JobStatus.running
|
|
300
|
+
elif status_word in ["Eqw", "dr", "dt"]:
|
|
301
|
+
return JobStatus.terminated
|
|
301
302
|
else:
|
|
302
303
|
return JobStatus.unknown
|
|
303
304
|
|
dpdispatcher/machines/shell.py
CHANGED
|
@@ -43,8 +43,7 @@ class Shell(Machine):
|
|
|
43
43
|
if ret != 0:
|
|
44
44
|
err_str = stderr.read().decode("utf-8")
|
|
45
45
|
raise RuntimeError(
|
|
46
|
-
"status command
|
|
47
|
-
% (cmd, err_str, ret)
|
|
46
|
+
f"status command {cmd} fails to execute\nerror message:{err_str}\nreturn code {ret}\n"
|
|
48
47
|
)
|
|
49
48
|
job_id = int(stdout.read().decode("utf-8").strip())
|
|
50
49
|
self.context.write_file(job_id_name, str(job_id))
|
|
@@ -80,8 +79,7 @@ class Shell(Machine):
|
|
|
80
79
|
if ret != 0:
|
|
81
80
|
err_str = stderr.read().decode("utf-8")
|
|
82
81
|
raise RuntimeError(
|
|
83
|
-
"status command
|
|
84
|
-
% (cmd, err_str, ret)
|
|
82
|
+
f"status command {cmd} fails to execute\nerror message:{err_str}\nreturn code {ret}\n"
|
|
85
83
|
)
|
|
86
84
|
|
|
87
85
|
if_job_exists = bool(stdout.read().decode("utf-8").strip())
|
dpdispatcher/machines/slurm.py
CHANGED
|
@@ -97,8 +97,7 @@ class Slurm(Machine):
|
|
|
97
97
|
):
|
|
98
98
|
# server network error, retry 3 times
|
|
99
99
|
raise RetrySignal(
|
|
100
|
-
"Get error code
|
|
101
|
-
% (ret, job.job_hash, err_str)
|
|
100
|
+
f"Get error code {ret} in submitting with job: {job.job_hash} . message: {err_str}"
|
|
102
101
|
)
|
|
103
102
|
elif (
|
|
104
103
|
"Job violates accounting/QOS policy" in err_str
|
|
@@ -109,8 +108,7 @@ class Slurm(Machine):
|
|
|
109
108
|
# job number exceeds, skip the submitting
|
|
110
109
|
return ""
|
|
111
110
|
raise RuntimeError(
|
|
112
|
-
"command
|
|
113
|
-
% (command, err_str, ret)
|
|
111
|
+
f"command {command} fails to execute\nerror message:{err_str}\nreturn code {ret}\n"
|
|
114
112
|
)
|
|
115
113
|
subret = stdout.readlines()
|
|
116
114
|
# --parsable
|
|
@@ -145,13 +143,11 @@ class Slurm(Machine):
|
|
|
145
143
|
):
|
|
146
144
|
# retry 3 times
|
|
147
145
|
raise RetrySignal(
|
|
148
|
-
"Get error code
|
|
149
|
-
% (ret, job.job_hash, err_str)
|
|
146
|
+
f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
|
|
150
147
|
)
|
|
151
148
|
raise RuntimeError(
|
|
152
|
-
"status command
|
|
153
|
-
"job_id
|
|
154
|
-
% (command, job_id, err_str, ret)
|
|
149
|
+
f"status command {command} fails to execute."
|
|
150
|
+
f"job_id:{job_id} \n error message:{err_str}\n return code {ret}\n"
|
|
155
151
|
)
|
|
156
152
|
status_line = stdout.read().decode("utf-8").split("\n")[-2]
|
|
157
153
|
status_word = status_line.split()[-1]
|
|
@@ -255,7 +251,7 @@ class SlurmJobArray(Slurm):
|
|
|
255
251
|
return super().gen_script_header(job) + "\n#SBATCH --array={}".format(
|
|
256
252
|
",".join(map(str, job_array))
|
|
257
253
|
)
|
|
258
|
-
return super().gen_script_header(job) + "\n#SBATCH --array=0-%
|
|
254
|
+
return super().gen_script_header(job) + "\n#SBATCH --array=0-%s" % (
|
|
259
255
|
math.ceil(len(job.job_task_list) / slurm_job_size) - 1
|
|
260
256
|
)
|
|
261
257
|
|
|
@@ -333,13 +329,11 @@ class SlurmJobArray(Slurm):
|
|
|
333
329
|
):
|
|
334
330
|
# retry 3 times
|
|
335
331
|
raise RetrySignal(
|
|
336
|
-
"Get error code
|
|
337
|
-
% (ret, job.job_hash, err_str)
|
|
332
|
+
f"Get error code {ret} in checking status with job: {job.job_hash} . message: {err_str}"
|
|
338
333
|
)
|
|
339
334
|
raise RuntimeError(
|
|
340
|
-
"status command
|
|
341
|
-
"job_id
|
|
342
|
-
% (command, job_id, err_str, ret)
|
|
335
|
+
f"status command {command} fails to execute."
|
|
336
|
+
f"job_id:{job_id} \n error message:{err_str}\n return code {ret}\n"
|
|
343
337
|
)
|
|
344
338
|
status_lines = stdout.read().decode("utf-8").split("\n")[:-1]
|
|
345
339
|
status = []
|
dpdispatcher/submission.py
CHANGED
|
@@ -55,7 +55,6 @@ class Submission:
|
|
|
55
55
|
*,
|
|
56
56
|
task_list=[],
|
|
57
57
|
):
|
|
58
|
-
# self.submission_list = submission_list
|
|
59
58
|
self.local_root = None
|
|
60
59
|
self.work_base = work_base
|
|
61
60
|
self._abs_work_base = os.path.abspath(work_base)
|
|
@@ -324,8 +323,7 @@ class Submission:
|
|
|
324
323
|
kwargs = {**{"clean": False}, **kwargs}
|
|
325
324
|
if kwargs["clean"]:
|
|
326
325
|
dlog.warning(
|
|
327
|
-
"Using async submission with `clean=True`, "
|
|
328
|
-
"job may fail in queue system"
|
|
326
|
+
"Using async submission with `clean=True`, job may fail in queue system"
|
|
329
327
|
)
|
|
330
328
|
loop = asyncio.get_event_loop()
|
|
331
329
|
wrapped_submission = functools.partial(self.run_submission, **kwargs)
|
|
@@ -515,12 +513,9 @@ class Submission:
|
|
|
515
513
|
def submission_from_json(cls, json_file_name="submission.json"):
|
|
516
514
|
with open(json_file_name) as f:
|
|
517
515
|
submission_dict = json.load(f)
|
|
518
|
-
# submission_dict = machine.context.read_file(json_file_name)
|
|
519
516
|
submission = cls.deserialize(submission_dict=submission_dict, machine=None)
|
|
520
517
|
return submission
|
|
521
518
|
|
|
522
|
-
# def check_if_recover()
|
|
523
|
-
|
|
524
519
|
def try_recover_from_json(self):
|
|
525
520
|
submission_file_name = f"{self.submission_hash}.json"
|
|
526
521
|
if_recover = self.machine.context.check_file_exists(submission_file_name)
|
|
@@ -545,7 +540,6 @@ class Submission:
|
|
|
545
540
|
f"machine.context.remote_root:{self.machine.context.remote_root}; "
|
|
546
541
|
f"submission.work_base:{submission.work_base};"
|
|
547
542
|
)
|
|
548
|
-
# self = submission.bind_machine(machine=self.machine)
|
|
549
543
|
else:
|
|
550
544
|
print(self.serialize())
|
|
551
545
|
print(submission.serialize())
|
|
@@ -759,7 +753,6 @@ class Job:
|
|
|
759
753
|
self.fail_count = 0
|
|
760
754
|
self.job_uuid = uuid.uuid4()
|
|
761
755
|
|
|
762
|
-
# self.job_hash = self.get_hash()
|
|
763
756
|
self.job_hash = self.get_hash()
|
|
764
757
|
self.script_file_name = self.job_hash + ".sub"
|
|
765
758
|
|
dpdispatcher/utils/hdfs_cli.py
CHANGED
|
@@ -28,7 +28,7 @@ class HDFS:
|
|
|
28
28
|
)
|
|
29
29
|
except Exception as e:
|
|
30
30
|
raise RuntimeError(
|
|
31
|
-
f"Cannot check existence of hdfs uri[{uri}]
|
|
31
|
+
f"Cannot check existence of hdfs uri[{uri}] with cmd[{cmd}]"
|
|
32
32
|
) from e
|
|
33
33
|
|
|
34
34
|
@staticmethod
|
|
@@ -48,9 +48,7 @@ class HDFS:
|
|
|
48
48
|
f"with cmd[{cmd}]; ret[{ret}] output[{out}] stderr[{err}]"
|
|
49
49
|
)
|
|
50
50
|
except Exception as e:
|
|
51
|
-
raise RuntimeError(
|
|
52
|
-
f"Cannot remove hdfs uri[{uri}] " f"with cmd[{cmd}]"
|
|
53
|
-
) from e
|
|
51
|
+
raise RuntimeError(f"Cannot remove hdfs uri[{uri}] with cmd[{cmd}]") from e
|
|
54
52
|
|
|
55
53
|
@staticmethod
|
|
56
54
|
def mkdir(uri):
|
|
@@ -70,7 +68,7 @@ class HDFS:
|
|
|
70
68
|
)
|
|
71
69
|
except Exception as e:
|
|
72
70
|
raise RuntimeError(
|
|
73
|
-
f"Cannot mkdir of hdfs uri[{uri}]
|
|
71
|
+
f"Cannot mkdir of hdfs uri[{uri}] with cmd[{cmd}]"
|
|
74
72
|
) from e
|
|
75
73
|
|
|
76
74
|
@staticmethod
|
|
@@ -80,7 +78,7 @@ class HDFS:
|
|
|
80
78
|
"""
|
|
81
79
|
# Make sure local_path is accessible
|
|
82
80
|
if not os.path.exists(local_path) or not os.access(local_path, os.R_OK):
|
|
83
|
-
raise RuntimeError(f"try to access local_path[{local_path}]
|
|
81
|
+
raise RuntimeError(f"try to access local_path[{local_path}] but failed")
|
|
84
82
|
cmd = f"hadoop fs -copyFromLocal -f {local_path} {to_uri}"
|
|
85
83
|
try:
|
|
86
84
|
ret, out, err = run_cmd_with_all_output(cmd)
|
|
@@ -132,9 +130,7 @@ class HDFS:
|
|
|
132
130
|
f"cmd [{cmd}] ret[{ret}] output[{out}] stderr[{err}]"
|
|
133
131
|
)
|
|
134
132
|
except Exception as e:
|
|
135
|
-
raise RuntimeError(
|
|
136
|
-
f"Cannot read text from uri[{uri}]" f"cmd [{cmd}]"
|
|
137
|
-
) from e
|
|
133
|
+
raise RuntimeError(f"Cannot read text from uri[{uri}]cmd [{cmd}]") from e
|
|
138
134
|
|
|
139
135
|
@staticmethod
|
|
140
136
|
def move(from_uri, to_uri):
|
|
@@ -151,6 +147,5 @@ class HDFS:
|
|
|
151
147
|
)
|
|
152
148
|
except Exception as e:
|
|
153
149
|
raise RuntimeError(
|
|
154
|
-
f"Cannot move from_uri[{from_uri}] to "
|
|
155
|
-
f"to_uri[{to_uri}] with cmd[{cmd}]"
|
|
150
|
+
f"Cannot move from_uri[{from_uri}] to to_uri[{to_uri}] with cmd[{cmd}]"
|
|
156
151
|
) from e
|
dpdispatcher/utils/utils.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: dpdispatcher
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.8
|
|
4
4
|
Summary: Generate HPC scheduler systems jobs input scripts, submit these scripts to HPC systems, and poke until they finish
|
|
5
5
|
Author: DeepModeling
|
|
6
|
-
License:
|
|
6
|
+
License: GNU LESSER GENERAL PUBLIC LICENSE
|
|
7
7
|
Version 3, 29 June 2007
|
|
8
8
|
|
|
9
9
|
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
|
@@ -186,32 +186,32 @@ Requires-Python: >=3.7
|
|
|
186
186
|
Description-Content-Type: text/markdown
|
|
187
187
|
License-File: LICENSE
|
|
188
188
|
Requires-Dist: paramiko
|
|
189
|
-
Requires-Dist: dargs
|
|
189
|
+
Requires-Dist: dargs>=0.4.1
|
|
190
190
|
Requires-Dist: requests
|
|
191
|
-
Requires-Dist: tqdm
|
|
191
|
+
Requires-Dist: tqdm>=4.9.0
|
|
192
|
+
Requires-Dist: typing_extensions; python_version < "3.7"
|
|
192
193
|
Requires-Dist: pyyaml
|
|
193
|
-
Requires-Dist: tomli
|
|
194
|
-
Requires-Dist: typing-extensions ; python_version < "3.7"
|
|
195
|
-
Provides-Extra: bohrium
|
|
196
|
-
Requires-Dist: oss2 ; extra == 'bohrium'
|
|
197
|
-
Requires-Dist: tqdm ; extra == 'bohrium'
|
|
198
|
-
Requires-Dist: bohrium-sdk ; extra == 'bohrium'
|
|
199
|
-
Provides-Extra: cloudserver
|
|
200
|
-
Requires-Dist: oss2 ; extra == 'cloudserver'
|
|
201
|
-
Requires-Dist: tqdm ; extra == 'cloudserver'
|
|
202
|
-
Requires-Dist: bohrium-sdk ; extra == 'cloudserver'
|
|
194
|
+
Requires-Dist: tomli>=1.1.0; python_version < "3.11"
|
|
203
195
|
Provides-Extra: docs
|
|
204
|
-
Requires-Dist: sphinx
|
|
205
|
-
Requires-Dist: myst-parser
|
|
206
|
-
Requires-Dist: sphinx-
|
|
207
|
-
Requires-Dist: numpydoc
|
|
208
|
-
Requires-Dist: deepmodeling-sphinx
|
|
209
|
-
Requires-Dist: dargs
|
|
210
|
-
Requires-Dist: sphinx-argparse
|
|
196
|
+
Requires-Dist: sphinx; extra == "docs"
|
|
197
|
+
Requires-Dist: myst-parser; extra == "docs"
|
|
198
|
+
Requires-Dist: sphinx-book-theme; extra == "docs"
|
|
199
|
+
Requires-Dist: numpydoc; extra == "docs"
|
|
200
|
+
Requires-Dist: deepmodeling-sphinx>=0.3.0; extra == "docs"
|
|
201
|
+
Requires-Dist: dargs>=0.3.1; extra == "docs"
|
|
202
|
+
Requires-Dist: sphinx-argparse<0.5.0; extra == "docs"
|
|
203
|
+
Provides-Extra: cloudserver
|
|
204
|
+
Requires-Dist: oss2; extra == "cloudserver"
|
|
205
|
+
Requires-Dist: tqdm; extra == "cloudserver"
|
|
206
|
+
Requires-Dist: bohrium-sdk; extra == "cloudserver"
|
|
207
|
+
Provides-Extra: bohrium
|
|
208
|
+
Requires-Dist: oss2; extra == "bohrium"
|
|
209
|
+
Requires-Dist: tqdm; extra == "bohrium"
|
|
210
|
+
Requires-Dist: bohrium-sdk; extra == "bohrium"
|
|
211
211
|
Provides-Extra: gui
|
|
212
|
-
Requires-Dist: dpgui
|
|
212
|
+
Requires-Dist: dpgui; extra == "gui"
|
|
213
213
|
Provides-Extra: test
|
|
214
|
-
Requires-Dist: dpgui
|
|
214
|
+
Requires-Dist: dpgui; extra == "test"
|
|
215
215
|
|
|
216
216
|
# DPDispatcher
|
|
217
217
|
|
|
@@ -221,7 +221,7 @@ Requires-Dist: dpgui ; extra == 'test'
|
|
|
221
221
|
[](https://dpdispatcher.readthedocs.io/)
|
|
222
222
|
|
|
223
223
|
DPDispatcher is a Python package used to generate HPC (High-Performance Computing) scheduler systems (Slurm/PBS/LSF/Bohrium) jobs input scripts, submit them to HPC systems, and poke until they finish.
|
|
224
|
-
|
|
224
|
+
|
|
225
225
|
DPDispatcher will monitor (poke) until these jobs finish and download the results files (if these jobs are running on remote systems connected by SSH).
|
|
226
226
|
|
|
227
227
|
For more information, check the [documentation](https://dpdispatcher.readthedocs.io/).
|
|
@@ -1,49 +1,49 @@
|
|
|
1
1
|
dpdispatcher/__init__.py,sha256=CLZP_N5CTp14ujWCykEHuJjoIfKR6CwrclXhjWUgNoE,517
|
|
2
2
|
dpdispatcher/__main__.py,sha256=BFhG-mSBzVZUEezQJqXWZnt2WsnhAHT_zpT8Y6gpOz0,116
|
|
3
|
-
dpdispatcher/_version.py,sha256=
|
|
3
|
+
dpdispatcher/_version.py,sha256=jFQ70HGO-FOLhIy6SpujRlrxJLVWsy5CAtewPppbaOs,411
|
|
4
4
|
dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
|
|
5
|
-
dpdispatcher/base_context.py,sha256=
|
|
5
|
+
dpdispatcher/base_context.py,sha256=W4eWDWVzYeL6EuEkivmJp-_h_B2mV9PtRWc09l1_Qzc,5242
|
|
6
6
|
dpdispatcher/dlog.py,sha256=QJKAwB6gV3Zb6zQUL9dZ_uIoTIEy9Z7ecmVQ-8WNmD8,1081
|
|
7
7
|
dpdispatcher/dpdisp.py,sha256=jhuTmwPY7KBF4WukaQomEwZcfYoISaMbKwuxdDGSluc,4206
|
|
8
8
|
dpdispatcher/machine.py,sha256=k53ycs_v7xrl4D93URc5ht0shoO9NPrVl0rYr4v5OiU,16696
|
|
9
9
|
dpdispatcher/run.py,sha256=tFHbJAioXXpgHTE5bhRRAuc8w7cX1ET9SBbiAg3Rw-I,5382
|
|
10
|
-
dpdispatcher/submission.py,sha256=
|
|
10
|
+
dpdispatcher/submission.py,sha256=zLzdKJkMXhvaicD2el33NxDHP_9LL29HBombxR1l-Sw,48086
|
|
11
11
|
dpdispatcher/contexts/__init__.py,sha256=jlvcIppmUnS39yBlkZEDvIQFV-j_BR75ZTbZALF_RB0,336
|
|
12
12
|
dpdispatcher/contexts/dp_cloud_server_context.py,sha256=PGRMef3q2hfK-o5dNIWWvzPca2NK1HrWEgungM4L9Go,12420
|
|
13
13
|
dpdispatcher/contexts/hdfs_context.py,sha256=mYQzXMZ4A9EjjWBAH3Ba6HOErUhMMwCsKxOjpd5R57Y,9105
|
|
14
14
|
dpdispatcher/contexts/lazy_local_context.py,sha256=FAClbLD2F4LizUqFzMOg3t0Z6NLeTDLJy7NkRcDELFs,5070
|
|
15
15
|
dpdispatcher/contexts/local_context.py,sha256=VbaSXGAc_EDMT0K5WV_flBF0bX87ntrwO_hq_Bkcb04,14590
|
|
16
16
|
dpdispatcher/contexts/openapi_context.py,sha256=M7L9axpjOrzvdTpLMDuEzZqe4ZuKIxjS0bzZUv8W2IQ,9674
|
|
17
|
-
dpdispatcher/contexts/ssh_context.py,sha256=
|
|
17
|
+
dpdispatcher/contexts/ssh_context.py,sha256=qaj8h2TdY1i-YYdDstUBs9IJaLwzytwnQkdntMEZ7vg,37664
|
|
18
18
|
dpdispatcher/dpcloudserver/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
19
19
|
dpdispatcher/dpcloudserver/client.py,sha256=k1niKjG6zFnMtHn_UuCjYoOcMju3o3PV-GdyVLr5-KM,165
|
|
20
20
|
dpdispatcher/entrypoints/__init__.py,sha256=exKSFT3j2oCerGwtI8WbHQK-D0K-CyifocRji1xntT4,20
|
|
21
21
|
dpdispatcher/entrypoints/gui.py,sha256=29lMXqbmSRbLj4rfBv7Jnw89NLU9syTB88IUP6IRJsU,830
|
|
22
22
|
dpdispatcher/entrypoints/run.py,sha256=tRkHfeAktV6gF31yb2MVOSTlpNGZFw3N0jHBmM1YfIg,175
|
|
23
23
|
dpdispatcher/entrypoints/submission.py,sha256=ikVwIZAQL0SsYO5xaMIdKXgO6qtc05w1vqmvtG7Nk5M,3401
|
|
24
|
-
dpdispatcher/machines/JH_UniScheduler.py,sha256=
|
|
24
|
+
dpdispatcher/machines/JH_UniScheduler.py,sha256=ZeUZXqyGrN5Zec4gwpwH5r6FJXaJLRUJMQWDCP7X3Nk,5756
|
|
25
25
|
dpdispatcher/machines/__init__.py,sha256=tOQuPUlW1Ab4qcC0oSAIyDjZA_WyE67h_EIxPCWGhys,336
|
|
26
|
-
dpdispatcher/machines/distributed_shell.py,sha256=
|
|
26
|
+
dpdispatcher/machines/distributed_shell.py,sha256=c0-lGeGz_M-PY2gPciT-uYZLQht5XTMaxJSNxkbMffc,7489
|
|
27
27
|
dpdispatcher/machines/dp_cloud_server.py,sha256=SR69gsFb2BvOQCW1QnWfP3cQvu_qHLJNsycp5wzosJU,11706
|
|
28
28
|
dpdispatcher/machines/fugaku.py,sha256=oY2hD2ldL2dztwtJ9WNisdsfPnaX-5yTRXewIT9r60I,4314
|
|
29
|
-
dpdispatcher/machines/lsf.py,sha256=
|
|
29
|
+
dpdispatcher/machines/lsf.py,sha256=xGDq8OLAk83E9EjK_3-QtEOyahvBGspWbxT__7mnSTw,7896
|
|
30
30
|
dpdispatcher/machines/openapi.py,sha256=Gzzbo8YOAybXGTrgMutexErcaEi3ts7uTUNvOhThFS8,8858
|
|
31
|
-
dpdispatcher/machines/pbs.py,sha256=
|
|
32
|
-
dpdispatcher/machines/shell.py,sha256=
|
|
33
|
-
dpdispatcher/machines/slurm.py,sha256=
|
|
31
|
+
dpdispatcher/machines/pbs.py,sha256=gUoj3OGQbZRBK4P-WXlhrxlQqTeUi9X8JGLOkAB__wE,12669
|
|
32
|
+
dpdispatcher/machines/shell.py,sha256=EeYnRCowXdzO3Nh25Yh_t5xeM6frq4uChk4GVx7OjH8,4797
|
|
33
|
+
dpdispatcher/machines/slurm.py,sha256=oyMX9iZQpVRR951zwz0wRNfl3_uJZzdtzxMbTJotlQU,15402
|
|
34
34
|
dpdispatcher/utils/__init__.py,sha256=fwvwkMf7DFNQkNBiIce8Y8gRA6FhICwKjkKiXu_BEJg,13
|
|
35
|
-
dpdispatcher/utils/hdfs_cli.py,sha256=
|
|
35
|
+
dpdispatcher/utils/hdfs_cli.py,sha256=a1a9PJAzt3wsTcdaSw_oD1vcNw59pMooxpAHjYOaaGA,5209
|
|
36
36
|
dpdispatcher/utils/job_status.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
|
|
37
37
|
dpdispatcher/utils/record.py,sha256=c8jdPmCuLzRmFo_jOjR0j9zFR1EWX3NSHVuPEIYCycg,2147
|
|
38
|
-
dpdispatcher/utils/utils.py,sha256=
|
|
38
|
+
dpdispatcher/utils/utils.py,sha256=Wo-8tGO05e2KkRyLXoIg3UlxzkuM-x1phRrTA1Hh7Ko,5328
|
|
39
39
|
dpdispatcher/utils/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
|
|
40
40
|
dpdispatcher/utils/dpcloudserver/client.py,sha256=CLfXswvzI4inDrW2bYkfMQ6gQJFcZOgLhiXBz_EI17M,12029
|
|
41
41
|
dpdispatcher/utils/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
|
|
42
42
|
dpdispatcher/utils/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
|
|
43
43
|
dpdispatcher/utils/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
|
|
44
|
-
dpdispatcher-0.6.
|
|
45
|
-
dpdispatcher-0.6.
|
|
46
|
-
dpdispatcher-0.6.
|
|
47
|
-
dpdispatcher-0.6.
|
|
48
|
-
dpdispatcher-0.6.
|
|
49
|
-
dpdispatcher-0.6.
|
|
44
|
+
dpdispatcher-0.6.8.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
|
|
45
|
+
dpdispatcher-0.6.8.dist-info/METADATA,sha256=mneS5eFsvLVeWxt9dGHBLyDQv0NSdIdC9x00TqFHhGI,12811
|
|
46
|
+
dpdispatcher-0.6.8.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
|
|
47
|
+
dpdispatcher-0.6.8.dist-info/entry_points.txt,sha256=NRHUV0IU_u7_XtcmmEDnVzAcUmurhiEAGwENckrajo4,233
|
|
48
|
+
dpdispatcher-0.6.8.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
|
|
49
|
+
dpdispatcher-0.6.8.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|