dpdispatcher 0.5.11__py3-none-any.whl → 0.6.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dpdispatcher/__init__.py +7 -89
- dpdispatcher/__main__.py +8 -0
- dpdispatcher/_version.py +14 -2
- dpdispatcher/base_context.py +1 -1
- dpdispatcher/contexts/__init__.py +11 -0
- dpdispatcher/{dp_cloud_server_context.py → contexts/dp_cloud_server_context.py} +7 -3
- dpdispatcher/{hdfs_context.py → contexts/hdfs_context.py} +2 -2
- dpdispatcher/{local_context.py → contexts/local_context.py} +51 -14
- dpdispatcher/{openapi_context.py → contexts/openapi_context.py} +3 -2
- dpdispatcher/{ssh_context.py → contexts/ssh_context.py} +113 -34
- dpdispatcher/dlog.py +31 -0
- dpdispatcher/dpdisp.py +113 -1
- dpdispatcher/entrypoints/__init__.py +1 -0
- dpdispatcher/entrypoints/gui.py +31 -0
- dpdispatcher/entrypoints/submission.py +83 -0
- dpdispatcher/machine.py +18 -4
- dpdispatcher/machines/__init__.py +11 -0
- dpdispatcher/{distributed_shell.py → machines/distributed_shell.py} +20 -4
- dpdispatcher/{dp_cloud_server.py → machines/dp_cloud_server.py} +21 -5
- dpdispatcher/{fugaku.py → machines/fugaku.py} +18 -5
- dpdispatcher/{lsf.py → machines/lsf.py} +20 -4
- dpdispatcher/{openapi.py → machines/openapi.py} +23 -4
- dpdispatcher/{pbs.py → machines/pbs.py} +30 -4
- dpdispatcher/{shell.py → machines/shell.py} +17 -3
- dpdispatcher/{slurm.py → machines/slurm.py} +37 -6
- dpdispatcher/submission.py +83 -39
- dpdispatcher/utils/__init__.py +1 -0
- dpdispatcher/{dpcloudserver → utils/dpcloudserver}/client.py +1 -1
- dpdispatcher/{hdfs_cli.py → utils/hdfs_cli.py} +1 -1
- dpdispatcher/utils/record.py +79 -0
- dpdispatcher/{utils.py → utils/utils.py} +14 -2
- {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/METADATA +7 -2
- dpdispatcher-0.6.1.dist-info/RECORD +44 -0
- {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/WHEEL +1 -1
- dpdispatcher-0.6.1.dist-info/entry_points.txt +7 -0
- dpdispatcher/dpcloudserver/temp_test.py +0 -90
- dpdispatcher-0.5.11.dist-info/RECORD +0 -36
- dpdispatcher-0.5.11.dist-info/entry_points.txt +0 -2
- /dpdispatcher/{lazy_local_context.py → contexts/lazy_local_context.py} +0 -0
- /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/__init__.py +0 -0
- /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/config.py +0 -0
- /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/retcode.py +0 -0
- /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/zip_file.py +0 -0
- /dpdispatcher/{JobStatus.py → utils/job_status.py} +0 -0
- {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/LICENSE +0 -0
- {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/top_level.txt +0 -0
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
# %%
|
|
2
|
-
import os
|
|
3
|
-
import sys
|
|
4
|
-
import unittest
|
|
5
|
-
import uuid
|
|
6
|
-
|
|
7
|
-
from dpdispatcher.dpcloudserver import api
|
|
8
|
-
from dpdispatcher.dpcloudserver.zip_file import zip_files
|
|
9
|
-
|
|
10
|
-
# import api
|
|
11
|
-
# from zip_file import zip_files
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
# %%
|
|
15
|
-
class DPTest(unittest.TestCase):
|
|
16
|
-
test_data = {
|
|
17
|
-
"job_type": "indicate",
|
|
18
|
-
"log_file": "mylog",
|
|
19
|
-
"command": "( echo aa && lmp -i input.lammps && sleep 900 ) > dp.log 2>&1",
|
|
20
|
-
"backward_files": [],
|
|
21
|
-
"job_name": "dpdispatcher_lammps_test",
|
|
22
|
-
"machine": {
|
|
23
|
-
"platform": "ali",
|
|
24
|
-
"resources": {
|
|
25
|
-
"gpu_type": "1 * NVIDIA P100",
|
|
26
|
-
"cpu_num": 4,
|
|
27
|
-
"mem_limit": 28,
|
|
28
|
-
"time_limit": "2:00:00",
|
|
29
|
-
"image_name": "yfb-deepmd-kit-1.2.4-cuda10",
|
|
30
|
-
},
|
|
31
|
-
},
|
|
32
|
-
"job_resources": "http://dpcloudserver.oss-cn-shenzhen.aliyuncs.com/dpcloudserver/indicate/a657ff49722839f1ee54edeb3e9b1beb0ee5cc0e/a657ff49722839f1ee54edeb3e9b1beb0ee5cc0e.zip",
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
username = ""
|
|
36
|
-
password = ""
|
|
37
|
-
|
|
38
|
-
ENDPOINT = "http://oss-cn-shenzhen.aliyuncs.com"
|
|
39
|
-
BUCKET_NAME = "dpcloudserver"
|
|
40
|
-
|
|
41
|
-
@classmethod
|
|
42
|
-
def setUpClass(cls):
|
|
43
|
-
print("execute", sys._getframe().f_code.co_name)
|
|
44
|
-
|
|
45
|
-
@classmethod
|
|
46
|
-
def tearDownClass(cls):
|
|
47
|
-
print("execute", sys._getframe().f_code.co_name)
|
|
48
|
-
|
|
49
|
-
def setUp(self):
|
|
50
|
-
print("execute", sys._getframe().f_code.co_name)
|
|
51
|
-
api.login(self.username, self.password)
|
|
52
|
-
|
|
53
|
-
def test_commit_job(self):
|
|
54
|
-
print("----------", sys._getframe().f_code.co_name)
|
|
55
|
-
file_uuid = uuid.uuid1().hex
|
|
56
|
-
oss_task_zip = os.path.join(
|
|
57
|
-
"{}/{}/{}.zip".format("indicate", file_uuid, file_uuid)
|
|
58
|
-
)
|
|
59
|
-
zip_path = "/home/felix/workplace/22_dpdispatcher/dpdispatcher-yfb/dpdispatcher/dpcloudserver/t.txt"
|
|
60
|
-
zip_task_file = zip_path + ".zip"
|
|
61
|
-
zip_files(zip_path, zip_task_file, [])
|
|
62
|
-
api.upload(oss_task_zip, zip_task_file, self.ENDPOINT, self.BUCKET_NAME)
|
|
63
|
-
job_id = api.job_create(
|
|
64
|
-
self.test_data["job_type"], self.test_data["job_resources"], self.test_data
|
|
65
|
-
)
|
|
66
|
-
tasks = api.get_job_detail(job_id)
|
|
67
|
-
print(tasks)
|
|
68
|
-
|
|
69
|
-
def test_get_tasks(self):
|
|
70
|
-
print("----------", sys._getframe().f_code.co_name)
|
|
71
|
-
jobs = api.get_jobs()
|
|
72
|
-
for j in jobs:
|
|
73
|
-
tasks = api.get_job_detail(j["id"])
|
|
74
|
-
print(tasks)
|
|
75
|
-
|
|
76
|
-
# def test_download(self):
|
|
77
|
-
# print('----------', sys._getframe().f_code.co_name)
|
|
78
|
-
# oss_path = 'dpcloudserver/indicate/abe0febc92ce11eb990800163e094dc5/abe0febc92ce11eb990800163e094dc5.zip'
|
|
79
|
-
# api.download(oss_path, "out.zip", self.ENDPOINT, self.BUCKET_NAME)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if __name__ == "__main__":
|
|
83
|
-
suite = unittest.TestSuite()
|
|
84
|
-
|
|
85
|
-
suite.addTest(DPTest("test_commit_job"))
|
|
86
|
-
# suite.addTest(DPTest("test_get_tasks"))
|
|
87
|
-
# suite.addTest(DPTest("test_download"))
|
|
88
|
-
|
|
89
|
-
runner = unittest.TextTestRunner()
|
|
90
|
-
runner.run(suite)
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
dpdispatcher/JobStatus.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
|
|
2
|
-
dpdispatcher/__init__.py,sha256=i33piTZkPDYIm_qiTEI_J3bmHlFayEN1T8T8TyzYdqg,3017
|
|
3
|
-
dpdispatcher/_version.py,sha256=yRZChScuo0oPgmhW4ryd-ooMVmlzBEI2S2te8ZJWfMM,162
|
|
4
|
-
dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
|
|
5
|
-
dpdispatcher/base_context.py,sha256=Hfri0x41XC4MRUjxc0-WMiZB_E4NvLp94ZYaHfYCWHM,3610
|
|
6
|
-
dpdispatcher/distributed_shell.py,sha256=XMcXt8g1f2DY5HYhhyiN5ehV2ihKULY5ng-sB0B7YaI,6933
|
|
7
|
-
dpdispatcher/dp_cloud_server.py,sha256=bZYGh7Q14ZlXgFwCnn6ZVDgYKIVJrjkJtO61ZY1k2u0,11011
|
|
8
|
-
dpdispatcher/dp_cloud_server_context.py,sha256=zB-r5WmnD7NM2IDGzoNAH2nCsYfhHR0rqHOH-gYo_3Y,12160
|
|
9
|
-
dpdispatcher/dpdisp.py,sha256=_dyH8xEgUR-s2xKkB20D9FIYhSHUCmzc2PxWgo9ildQ,94
|
|
10
|
-
dpdispatcher/fugaku.py,sha256=Nwj3tDMuv29wMZdZcQf7U9EkeoKxrdu0FZg_KuODgX8,3678
|
|
11
|
-
dpdispatcher/hdfs_cli.py,sha256=SkCA-dgZbcS2a6Z96XzUB3-T3Barkak_5S7wP7WF55o,5428
|
|
12
|
-
dpdispatcher/hdfs_context.py,sha256=mO12yMoASGSh8erumVxMOPnHGdhOZuyaPyir9hfY0ZY,8897
|
|
13
|
-
dpdispatcher/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
|
|
14
|
-
dpdispatcher/local_context.py,sha256=8Slm8Y_i7Quffze5ASAWJGrP5toVJZREQ6hG5EebFpE,11776
|
|
15
|
-
dpdispatcher/lsf.py,sha256=ve8aJeeNDaC8Hktp09j1TU49ck2AiJ0MjuCJs8zkzdE,7391
|
|
16
|
-
dpdispatcher/machine.py,sha256=0Fg5mluZ1Xlo4OAM8oplji37qdGgvR0GDHEIucm4Ym8,15520
|
|
17
|
-
dpdispatcher/openapi.py,sha256=cNHbiijlbU-U7bbSQLXHIQp5CpPga-ZIvaRD5YxVA8s,8149
|
|
18
|
-
dpdispatcher/openapi_context.py,sha256=B_Rpw-xoJG6Y4ME6tAyEoIEZWv6afEcPgldjCzkNmBw,9444
|
|
19
|
-
dpdispatcher/pbs.py,sha256=HGqoFympIpNNiAmtBZEKzsmVtt8-O3Xxp2rjYAUNMuM,6065
|
|
20
|
-
dpdispatcher/shell.py,sha256=kEP7za-qN71y_21p0uBNkopZ5s63Adq54904hjUHv48,4141
|
|
21
|
-
dpdispatcher/slurm.py,sha256=bNvFL6lCU5YveTGuSLbDdk-9jbDXyWbB-OBRFhWnfZg,14530
|
|
22
|
-
dpdispatcher/ssh_context.py,sha256=Daibjh5hHoILv4wNKfOOJWsom9kTEdN-92Ra1rP4VtY,35035
|
|
23
|
-
dpdispatcher/submission.py,sha256=g_1zVsfLv8kNOc3Qnf8w9Br4_dOijvj40NmJxzecnpM,46491
|
|
24
|
-
dpdispatcher/utils.py,sha256=RXUHJl3S2z26Em3SeltnxtdVM3kv7weXJKvBEjG6I34,5035
|
|
25
|
-
dpdispatcher/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
|
|
26
|
-
dpdispatcher/dpcloudserver/client.py,sha256=52d_ftHk_8kpC3TNcGjXkWXLNQ0U8yYZ2l11L0DzxVg,12024
|
|
27
|
-
dpdispatcher/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
|
|
28
|
-
dpdispatcher/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
|
|
29
|
-
dpdispatcher/dpcloudserver/temp_test.py,sha256=jklOSu7tZ_wW5gycGRiUsbBWMLZDqCBslSYOCb2hTHw,2932
|
|
30
|
-
dpdispatcher/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
|
|
31
|
-
dpdispatcher-0.5.11.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
|
|
32
|
-
dpdispatcher-0.5.11.dist-info/METADATA,sha256=iRAA5v_xI37ltz6tdB_tsoEYwd2RYFeMmh33hp6-pA8,12585
|
|
33
|
-
dpdispatcher-0.5.11.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
|
|
34
|
-
dpdispatcher-0.5.11.dist-info/entry_points.txt,sha256=3bKn6IB6SYhKOUbbcOdBBevz4gsDmhmbogKMVn4ptOQ,52
|
|
35
|
-
dpdispatcher-0.5.11.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
|
|
36
|
-
dpdispatcher-0.5.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|