dpdispatcher 0.5.11__py3-none-any.whl → 0.6.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. dpdispatcher/__init__.py +7 -89
  2. dpdispatcher/__main__.py +8 -0
  3. dpdispatcher/_version.py +14 -2
  4. dpdispatcher/base_context.py +1 -1
  5. dpdispatcher/contexts/__init__.py +11 -0
  6. dpdispatcher/{dp_cloud_server_context.py → contexts/dp_cloud_server_context.py} +7 -3
  7. dpdispatcher/{hdfs_context.py → contexts/hdfs_context.py} +2 -2
  8. dpdispatcher/{local_context.py → contexts/local_context.py} +51 -14
  9. dpdispatcher/{openapi_context.py → contexts/openapi_context.py} +3 -2
  10. dpdispatcher/{ssh_context.py → contexts/ssh_context.py} +113 -34
  11. dpdispatcher/dlog.py +31 -0
  12. dpdispatcher/dpdisp.py +113 -1
  13. dpdispatcher/entrypoints/__init__.py +1 -0
  14. dpdispatcher/entrypoints/gui.py +31 -0
  15. dpdispatcher/entrypoints/submission.py +83 -0
  16. dpdispatcher/machine.py +18 -4
  17. dpdispatcher/machines/__init__.py +11 -0
  18. dpdispatcher/{distributed_shell.py → machines/distributed_shell.py} +20 -4
  19. dpdispatcher/{dp_cloud_server.py → machines/dp_cloud_server.py} +21 -5
  20. dpdispatcher/{fugaku.py → machines/fugaku.py} +18 -5
  21. dpdispatcher/{lsf.py → machines/lsf.py} +20 -4
  22. dpdispatcher/{openapi.py → machines/openapi.py} +23 -4
  23. dpdispatcher/{pbs.py → machines/pbs.py} +30 -4
  24. dpdispatcher/{shell.py → machines/shell.py} +17 -3
  25. dpdispatcher/{slurm.py → machines/slurm.py} +37 -6
  26. dpdispatcher/submission.py +83 -39
  27. dpdispatcher/utils/__init__.py +1 -0
  28. dpdispatcher/{dpcloudserver → utils/dpcloudserver}/client.py +1 -1
  29. dpdispatcher/{hdfs_cli.py → utils/hdfs_cli.py} +1 -1
  30. dpdispatcher/utils/record.py +79 -0
  31. dpdispatcher/{utils.py → utils/utils.py} +14 -2
  32. {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/METADATA +7 -2
  33. dpdispatcher-0.6.1.dist-info/RECORD +44 -0
  34. {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/WHEEL +1 -1
  35. dpdispatcher-0.6.1.dist-info/entry_points.txt +7 -0
  36. dpdispatcher/dpcloudserver/temp_test.py +0 -90
  37. dpdispatcher-0.5.11.dist-info/RECORD +0 -36
  38. dpdispatcher-0.5.11.dist-info/entry_points.txt +0 -2
  39. /dpdispatcher/{lazy_local_context.py → contexts/lazy_local_context.py} +0 -0
  40. /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/__init__.py +0 -0
  41. /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/config.py +0 -0
  42. /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/retcode.py +0 -0
  43. /dpdispatcher/{dpcloudserver → utils/dpcloudserver}/zip_file.py +0 -0
  44. /dpdispatcher/{JobStatus.py → utils/job_status.py} +0 -0
  45. {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/LICENSE +0 -0
  46. {dpdispatcher-0.5.11.dist-info → dpdispatcher-0.6.1.dist-info}/top_level.txt +0 -0
@@ -1,90 +0,0 @@
1
- # %%
2
- import os
3
- import sys
4
- import unittest
5
- import uuid
6
-
7
- from dpdispatcher.dpcloudserver import api
8
- from dpdispatcher.dpcloudserver.zip_file import zip_files
9
-
10
- # import api
11
- # from zip_file import zip_files
12
-
13
-
14
- # %%
15
- class DPTest(unittest.TestCase):
16
- test_data = {
17
- "job_type": "indicate",
18
- "log_file": "mylog",
19
- "command": "( echo aa && lmp -i input.lammps && sleep 900 ) > dp.log 2>&1",
20
- "backward_files": [],
21
- "job_name": "dpdispatcher_lammps_test",
22
- "machine": {
23
- "platform": "ali",
24
- "resources": {
25
- "gpu_type": "1 * NVIDIA P100",
26
- "cpu_num": 4,
27
- "mem_limit": 28,
28
- "time_limit": "2:00:00",
29
- "image_name": "yfb-deepmd-kit-1.2.4-cuda10",
30
- },
31
- },
32
- "job_resources": "http://dpcloudserver.oss-cn-shenzhen.aliyuncs.com/dpcloudserver/indicate/a657ff49722839f1ee54edeb3e9b1beb0ee5cc0e/a657ff49722839f1ee54edeb3e9b1beb0ee5cc0e.zip",
33
- }
34
-
35
- username = ""
36
- password = ""
37
-
38
- ENDPOINT = "http://oss-cn-shenzhen.aliyuncs.com"
39
- BUCKET_NAME = "dpcloudserver"
40
-
41
- @classmethod
42
- def setUpClass(cls):
43
- print("execute", sys._getframe().f_code.co_name)
44
-
45
- @classmethod
46
- def tearDownClass(cls):
47
- print("execute", sys._getframe().f_code.co_name)
48
-
49
- def setUp(self):
50
- print("execute", sys._getframe().f_code.co_name)
51
- api.login(self.username, self.password)
52
-
53
- def test_commit_job(self):
54
- print("----------", sys._getframe().f_code.co_name)
55
- file_uuid = uuid.uuid1().hex
56
- oss_task_zip = os.path.join(
57
- "{}/{}/{}.zip".format("indicate", file_uuid, file_uuid)
58
- )
59
- zip_path = "/home/felix/workplace/22_dpdispatcher/dpdispatcher-yfb/dpdispatcher/dpcloudserver/t.txt"
60
- zip_task_file = zip_path + ".zip"
61
- zip_files(zip_path, zip_task_file, [])
62
- api.upload(oss_task_zip, zip_task_file, self.ENDPOINT, self.BUCKET_NAME)
63
- job_id = api.job_create(
64
- self.test_data["job_type"], self.test_data["job_resources"], self.test_data
65
- )
66
- tasks = api.get_job_detail(job_id)
67
- print(tasks)
68
-
69
- def test_get_tasks(self):
70
- print("----------", sys._getframe().f_code.co_name)
71
- jobs = api.get_jobs()
72
- for j in jobs:
73
- tasks = api.get_job_detail(j["id"])
74
- print(tasks)
75
-
76
- # def test_download(self):
77
- # print('----------', sys._getframe().f_code.co_name)
78
- # oss_path = 'dpcloudserver/indicate/abe0febc92ce11eb990800163e094dc5/abe0febc92ce11eb990800163e094dc5.zip'
79
- # api.download(oss_path, "out.zip", self.ENDPOINT, self.BUCKET_NAME)
80
-
81
-
82
- if __name__ == "__main__":
83
- suite = unittest.TestSuite()
84
-
85
- suite.addTest(DPTest("test_commit_job"))
86
- # suite.addTest(DPTest("test_get_tasks"))
87
- # suite.addTest(DPTest("test_download"))
88
-
89
- runner = unittest.TextTestRunner()
90
- runner.run(suite)
@@ -1,36 +0,0 @@
1
- dpdispatcher/JobStatus.py,sha256=Eszs4TPLfszCuf6zLaFonf25feXDUguF28spYOjJpQE,233
2
- dpdispatcher/__init__.py,sha256=i33piTZkPDYIm_qiTEI_J3bmHlFayEN1T8T8TyzYdqg,3017
3
- dpdispatcher/_version.py,sha256=yRZChScuo0oPgmhW4ryd-ooMVmlzBEI2S2te8ZJWfMM,162
4
- dpdispatcher/arginfo.py,sha256=pNaxYIE6ahBidpR7OCKZdw8iGt003uTXGSlVzwiuvRg,188
5
- dpdispatcher/base_context.py,sha256=Hfri0x41XC4MRUjxc0-WMiZB_E4NvLp94ZYaHfYCWHM,3610
6
- dpdispatcher/distributed_shell.py,sha256=XMcXt8g1f2DY5HYhhyiN5ehV2ihKULY5ng-sB0B7YaI,6933
7
- dpdispatcher/dp_cloud_server.py,sha256=bZYGh7Q14ZlXgFwCnn6ZVDgYKIVJrjkJtO61ZY1k2u0,11011
8
- dpdispatcher/dp_cloud_server_context.py,sha256=zB-r5WmnD7NM2IDGzoNAH2nCsYfhHR0rqHOH-gYo_3Y,12160
9
- dpdispatcher/dpdisp.py,sha256=_dyH8xEgUR-s2xKkB20D9FIYhSHUCmzc2PxWgo9ildQ,94
10
- dpdispatcher/fugaku.py,sha256=Nwj3tDMuv29wMZdZcQf7U9EkeoKxrdu0FZg_KuODgX8,3678
11
- dpdispatcher/hdfs_cli.py,sha256=SkCA-dgZbcS2a6Z96XzUB3-T3Barkak_5S7wP7WF55o,5428
12
- dpdispatcher/hdfs_context.py,sha256=mO12yMoASGSh8erumVxMOPnHGdhOZuyaPyir9hfY0ZY,8897
13
- dpdispatcher/lazy_local_context.py,sha256=F8abWAJRY1Ewx1sErINKN1ltWerXzeCcJgjTvLvucKE,5696
14
- dpdispatcher/local_context.py,sha256=8Slm8Y_i7Quffze5ASAWJGrP5toVJZREQ6hG5EebFpE,11776
15
- dpdispatcher/lsf.py,sha256=ve8aJeeNDaC8Hktp09j1TU49ck2AiJ0MjuCJs8zkzdE,7391
16
- dpdispatcher/machine.py,sha256=0Fg5mluZ1Xlo4OAM8oplji37qdGgvR0GDHEIucm4Ym8,15520
17
- dpdispatcher/openapi.py,sha256=cNHbiijlbU-U7bbSQLXHIQp5CpPga-ZIvaRD5YxVA8s,8149
18
- dpdispatcher/openapi_context.py,sha256=B_Rpw-xoJG6Y4ME6tAyEoIEZWv6afEcPgldjCzkNmBw,9444
19
- dpdispatcher/pbs.py,sha256=HGqoFympIpNNiAmtBZEKzsmVtt8-O3Xxp2rjYAUNMuM,6065
20
- dpdispatcher/shell.py,sha256=kEP7za-qN71y_21p0uBNkopZ5s63Adq54904hjUHv48,4141
21
- dpdispatcher/slurm.py,sha256=bNvFL6lCU5YveTGuSLbDdk-9jbDXyWbB-OBRFhWnfZg,14530
22
- dpdispatcher/ssh_context.py,sha256=Daibjh5hHoILv4wNKfOOJWsom9kTEdN-92Ra1rP4VtY,35035
23
- dpdispatcher/submission.py,sha256=g_1zVsfLv8kNOc3Qnf8w9Br4_dOijvj40NmJxzecnpM,46491
24
- dpdispatcher/utils.py,sha256=RXUHJl3S2z26Em3SeltnxtdVM3kv7weXJKvBEjG6I34,5035
25
- dpdispatcher/dpcloudserver/__init__.py,sha256=FnX9HH-2dXADluNfucg98JPMfruMoBpN9ER9lZkVQvQ,49
26
- dpdispatcher/dpcloudserver/client.py,sha256=52d_ftHk_8kpC3TNcGjXkWXLNQ0U8yYZ2l11L0DzxVg,12024
27
- dpdispatcher/dpcloudserver/config.py,sha256=NteQzf1OeEkz2UbkXHHQ0B72cUu23zLVzpM9Yh4v1Cc,559
28
- dpdispatcher/dpcloudserver/retcode.py,sha256=1qAF8gFZx55u2sO8KbtYSIIrjcO-IGufEUlwbkSfC1g,721
29
- dpdispatcher/dpcloudserver/temp_test.py,sha256=jklOSu7tZ_wW5gycGRiUsbBWMLZDqCBslSYOCb2hTHw,2932
30
- dpdispatcher/dpcloudserver/zip_file.py,sha256=f9WrlktwHW0YipaWg5Y0kxjMZlhD1cJYa6EUpvu4Cro,2611
31
- dpdispatcher-0.5.11.dist-info/LICENSE,sha256=46mU2C5kSwOnkqkw9XQAJlhBL2JAf1_uCD8lVcXyMRg,7652
32
- dpdispatcher-0.5.11.dist-info/METADATA,sha256=iRAA5v_xI37ltz6tdB_tsoEYwd2RYFeMmh33hp6-pA8,12585
33
- dpdispatcher-0.5.11.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
34
- dpdispatcher-0.5.11.dist-info/entry_points.txt,sha256=3bKn6IB6SYhKOUbbcOdBBevz4gsDmhmbogKMVn4ptOQ,52
35
- dpdispatcher-0.5.11.dist-info/top_level.txt,sha256=35jAQoXY-b-e9fJ1_mxhZUiaCoJNt1ZI7mpFRf07Qjs,13
36
- dpdispatcher-0.5.11.dist-info/RECORD,,
@@ -1,2 +0,0 @@
1
- [console_scripts]
2
- dpdisp = dpdispatcher.dpdisp:main
File without changes