dma-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dma_cli/__init__.py +3 -0
- dma_cli/__main__.py +6 -0
- dma_cli/calc.py +42 -0
- dma_cli/cli.py +148 -0
- dma_cli/client.py +245 -0
- dma_cli/resources.py +251 -0
- dma_cli/state.py +65 -0
- dma_cli/submit.py +673 -0
- dma_cli-0.1.0.data/data/share/dma-cli/batch.example.json +16 -0
- dma_cli-0.1.0.data/data/share/dma-cli/config.example.yaml +18 -0
- dma_cli-0.1.0.data/data/share/dma-cli/input/calc.py +42 -0
- dma_cli-0.1.0.data/data/share/dma-cli/input/case1/calc.py +42 -0
- dma_cli-0.1.0.data/data/share/dma-cli/input/case2/calc.py +42 -0
- dma_cli-0.1.0.data/data/share/dma-cli/job.example.json +13 -0
- dma_cli-0.1.0.dist-info/METADATA +269 -0
- dma_cli-0.1.0.dist-info/RECORD +19 -0
- dma_cli-0.1.0.dist-info/WHEEL +5 -0
- dma_cli-0.1.0.dist-info/entry_points.txt +2 -0
- dma_cli-0.1.0.dist-info/top_level.txt +1 -0
dma_cli/submit.py
ADDED
|
@@ -0,0 +1,673 @@
|
|
|
1
|
+
"""submit: 7 步链路 + 单个/批量并发 + 统计 + check_machine_type 预检。
|
|
2
|
+
|
|
3
|
+
7 步链路(移植自 dma_job_submit_10000.jmx 的权威请求体,HAR 同构):
|
|
4
|
+
1. POST /irpapi/szc/v1/job/ body {"client":3,"createSource":2} -> data.jobId
|
|
5
|
+
2. GET /irpapi/szc/v1/job/{jobId}/input/token -> data.token
|
|
6
|
+
3. POST /dmaapi/szc/v1/file/upload/multipart/init X-Storage-Token; 网页 attachment 体 -> data.initialKey
|
|
7
|
+
4. POST /dmaapi/szc/v1/file/upload/multipart/upload X-Storage-Token+X-Storage-Param; 原文 -> data.partString
|
|
8
|
+
5. POST /dmaapi/szc/v1/file/upload/multipart/complete X-Storage-Token; {initialKey,partString[]}
|
|
9
|
+
6. POST /irpapi/szc/v1/job/{jobId}/snapshot HAR 同构体(当前选择 + 覆盖)
|
|
10
|
+
7. POST /irpapi/szc/v1/job/commit/{jobId} Content-Type: x-www-form-urlencoded; 空 body
|
|
11
|
+
每步断言 code==0,失败记录「哪步 + code/msg」。
|
|
12
|
+
"""
|
|
13
|
+
import base64
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import sys
|
|
17
|
+
import time
|
|
18
|
+
from collections import Counter
|
|
19
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Optional
|
|
22
|
+
|
|
23
|
+
import requests
|
|
24
|
+
|
|
25
|
+
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
26
|
+
|
|
27
|
+
from .client import DmaClient, STATUS_MAP
|
|
28
|
+
|
|
29
|
+
# 内置默认任务脚本(无 --input_directory 时上传它)
|
|
30
|
+
DEFAULT_INPUT = Path(__file__).parent / "calc.py"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def find_sku_by_machine_type(client: DmaClient, machine_type: str) -> dict:
|
|
34
|
+
"""根据机型名(skuName)查找 sku 信息。返回 {skuId, skuName, cpu, memory, gpuModel, gpuNum}。"""
|
|
35
|
+
for kind in ["cpu", "gpu"]:
|
|
36
|
+
resp = client.get("/dmaapi/szc/v1/softwares/skus",
|
|
37
|
+
params={"kind": kind, "create_source": 2})
|
|
38
|
+
if resp.get("code") != 0:
|
|
39
|
+
continue
|
|
40
|
+
for item in resp.get("data", {}).get("items", []):
|
|
41
|
+
if item.get("skuName") == machine_type:
|
|
42
|
+
return {
|
|
43
|
+
"skuId": item["skuId"],
|
|
44
|
+
"skuName": item["skuName"],
|
|
45
|
+
"cpu": item["cpu"],
|
|
46
|
+
"memory": item["memory"],
|
|
47
|
+
"gpuModel": item["gpuModel"],
|
|
48
|
+
"gpuNum": item["gpuNum"],
|
|
49
|
+
}
|
|
50
|
+
raise RuntimeError(f"未找到机型 {machine_type},可用 'dma sku list --kind cpu|gpu' 查询")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def find_software_by_image_address(client: DmaClient, image_address: str) -> dict:
|
|
54
|
+
"""根据镜像地址查找软件信息。返回 {softwareId, softwareVersionId, default_cmd}。
|
|
55
|
+
匹配顺序:
|
|
56
|
+
1. 版本的 url 字段精确匹配
|
|
57
|
+
2. softwareVersion 字段精确匹配
|
|
58
|
+
3. softwareName 模糊匹配
|
|
59
|
+
"""
|
|
60
|
+
resp = client.get("/dmaapi/szc/v1/softwares/deep-mining", params={"bizPlatform": 2})
|
|
61
|
+
if resp.get("code") != 0:
|
|
62
|
+
raise RuntimeError(f"查询软件目录失败: code={resp.get('code')}")
|
|
63
|
+
|
|
64
|
+
items = resp.get("data", {}).get("items", [])
|
|
65
|
+
|
|
66
|
+
# 1. 先尝试按 url 精确匹配
|
|
67
|
+
for item in items:
|
|
68
|
+
for version in item.get("versions", []):
|
|
69
|
+
if version.get("url") == image_address:
|
|
70
|
+
cmds = version.get("cmds", [])
|
|
71
|
+
default_cmd = cmds[0]["cmd"] if cmds else None
|
|
72
|
+
return {
|
|
73
|
+
"softwareId": item["softwareId"],
|
|
74
|
+
"softwareVersionId": version["softwareVersionId"],
|
|
75
|
+
"softwareName": item["softwareName"],
|
|
76
|
+
"softwareVersion": version["softwareVersion"],
|
|
77
|
+
"default_cmd": default_cmd,
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
# 2. 尝试按 softwareVersion 精确匹配
|
|
81
|
+
for item in items:
|
|
82
|
+
for version in item.get("versions", []):
|
|
83
|
+
if version.get("softwareVersion") == image_address:
|
|
84
|
+
cmds = version.get("cmds", [])
|
|
85
|
+
default_cmd = cmds[0]["cmd"] if cmds else None
|
|
86
|
+
return {
|
|
87
|
+
"softwareId": item["softwareId"],
|
|
88
|
+
"softwareVersionId": version["softwareVersionId"],
|
|
89
|
+
"softwareName": item["softwareName"],
|
|
90
|
+
"softwareVersion": version["softwareVersion"],
|
|
91
|
+
"default_cmd": default_cmd,
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
# 3. 再尝试按 softwareName 模糊匹配
|
|
95
|
+
for item in items:
|
|
96
|
+
if image_address in item.get("softwareName", "") or item.get("softwareName") in image_address:
|
|
97
|
+
version = item.get("versions", [{}])[0]
|
|
98
|
+
cmds = version.get("cmds", [])
|
|
99
|
+
default_cmd = cmds[0]["cmd"] if cmds else None
|
|
100
|
+
return {
|
|
101
|
+
"softwareId": item["softwareId"],
|
|
102
|
+
"softwareVersionId": version.get("softwareVersionId"),
|
|
103
|
+
"softwareName": item["softwareName"],
|
|
104
|
+
"softwareVersion": version.get("softwareVersion"),
|
|
105
|
+
"default_cmd": default_cmd,
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
raise RuntimeError(f"未找到镜像 {image_address},可用 'dma image list' 查询")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def load_json_config(config_file: str) -> dict:
|
|
112
|
+
"""加载 JSON 配置文件。"""
|
|
113
|
+
if not os.path.exists(config_file):
|
|
114
|
+
raise RuntimeError(f"配置文件不存在: {config_file}")
|
|
115
|
+
with open(config_file, "r", encoding="utf-8") as f:
|
|
116
|
+
data = json.load(f)
|
|
117
|
+
# 如果是数组形式(批量任务),外部做处理,此处包装成 dict 格式避免 attribute error
|
|
118
|
+
if isinstance(data, list):
|
|
119
|
+
return {"_is_batch_list": True, "_batch_data": data}
|
|
120
|
+
return data
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def check_machine_type(client, sku_name, n_node=1) -> dict:
|
|
124
|
+
"""预检机型合法性,返回完整响应(data.type 如 QUICK)。"""
|
|
125
|
+
return client.get("/irpapi/szc/v1/job/check_machine_type",
|
|
126
|
+
params={"machineType": sku_name, "from": "web", "nNode": n_node})
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def build_snapshot_body(client, job_id, user_id, *, name, cmd, overrides: dict) -> dict:
|
|
130
|
+
"""用当前项目 + 覆盖参数拼 HAR 同构 snapshot 体。"""
|
|
131
|
+
project = client.current_project()
|
|
132
|
+
defaults = client.defaults
|
|
133
|
+
|
|
134
|
+
def ov(key, current):
|
|
135
|
+
v = overrides.get(key)
|
|
136
|
+
return v if v is not None else current
|
|
137
|
+
|
|
138
|
+
project_id = ov("project_id", project and project["projectId"])
|
|
139
|
+
if not project_id:
|
|
140
|
+
raise RuntimeError("未设置项目,请先 'dma project use <projectId>' 或指定 --project_id")
|
|
141
|
+
|
|
142
|
+
# 从 overrides 中获取 sku 和 software 信息(已通过自动查找填充)
|
|
143
|
+
sku_id = ov("sku_id", None)
|
|
144
|
+
cpu = ov("cpu", None)
|
|
145
|
+
memory = ov("memory", None)
|
|
146
|
+
gpu_model = ov("gpu_model", None)
|
|
147
|
+
gpu_num = ov("gpu_num", None)
|
|
148
|
+
|
|
149
|
+
if not all([sku_id, cpu is not None, memory is not None]):
|
|
150
|
+
raise RuntimeError("未设置机型,请指定 --machine_type 或 --sku-id")
|
|
151
|
+
|
|
152
|
+
software_id = ov("software_id", None)
|
|
153
|
+
software_version_id = ov("software_version_id", None)
|
|
154
|
+
|
|
155
|
+
if not all([software_id, software_version_id]):
|
|
156
|
+
raise RuntimeError("未设置镜像,请指定 --image_address 或 --software-id")
|
|
157
|
+
|
|
158
|
+
# 处理 max_run_time(分钟 → maxRunTime + maxRunTimeUnit)
|
|
159
|
+
max_run_time_min = ov("max_run_time", 0)
|
|
160
|
+
if max_run_time_min and max_run_time_min > 0:
|
|
161
|
+
max_run_time = max_run_time_min
|
|
162
|
+
max_run_time_unit = "m"
|
|
163
|
+
else:
|
|
164
|
+
max_run_time = defaults.get("max_run_time", 15)
|
|
165
|
+
max_run_time_unit = defaults.get("max_run_time_unit", "d")
|
|
166
|
+
|
|
167
|
+
# 处理 log_files 和 out_files
|
|
168
|
+
log_files = []
|
|
169
|
+
if overrides.get("log_file"):
|
|
170
|
+
log_files = [overrides["log_file"]]
|
|
171
|
+
|
|
172
|
+
out_files = []
|
|
173
|
+
if overrides.get("out_files"):
|
|
174
|
+
out_files.extend(overrides["out_files"])
|
|
175
|
+
if overrides.get("result_path"):
|
|
176
|
+
out_files.append(overrides["result_path"])
|
|
177
|
+
out_files = list(set(out_files)) # 去重
|
|
178
|
+
|
|
179
|
+
return {
|
|
180
|
+
"name": name,
|
|
181
|
+
"projectId": project_id,
|
|
182
|
+
"bohrJobId": int(job_id),
|
|
183
|
+
"inputs": [f"{user_id}/job/{job_id}/input/"],
|
|
184
|
+
"softwareId": software_id,
|
|
185
|
+
"softwareVersionId": software_version_id,
|
|
186
|
+
"cmd": cmd,
|
|
187
|
+
"compatibilityUpgrade": {"enabled": False},
|
|
188
|
+
"nnode": ov("nnode", 1),
|
|
189
|
+
"restartPolicy": 0,
|
|
190
|
+
"restartDelaySeconds": 20,
|
|
191
|
+
"maxRescheduleTimes": ov("max_reschedule_times", 0),
|
|
192
|
+
"logFiles": log_files,
|
|
193
|
+
"outFiles": out_files,
|
|
194
|
+
"skuId": sku_id,
|
|
195
|
+
"cpu": cpu,
|
|
196
|
+
"gpuModel": gpu_model or "cpu",
|
|
197
|
+
"gpuNum": gpu_num or 0,
|
|
198
|
+
"memory": memory,
|
|
199
|
+
"maxRunTime": max_run_time,
|
|
200
|
+
"maxRunTimeUnit": max_run_time_unit,
|
|
201
|
+
"inputFileList": [],
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def submit_one(client: DmaClient, input_files: list, *, name: str, cmd: str, overrides: dict, check: bool = True) -> dict:
|
|
206
|
+
"""提交单个任务,走完整 7 步。input_files 是 [(filename, bytes), ...] 列表。返回结果 dict。"""
|
|
207
|
+
start = time.time()
|
|
208
|
+
user_id = client._user_id()
|
|
209
|
+
result = {"ok": False, "job_id": None, "bohr_job_id": None, "error": None, "elapsed_ms": 0}
|
|
210
|
+
|
|
211
|
+
def fail(step, resp):
|
|
212
|
+
result["error"] = f"{step}: code={resp.get('code')} msg={resp.get('msg') or resp.get('message')}"
|
|
213
|
+
result["elapsed_ms"] = int((time.time() - start) * 1000)
|
|
214
|
+
return result
|
|
215
|
+
|
|
216
|
+
try:
|
|
217
|
+
# 预检(可选)
|
|
218
|
+
if check:
|
|
219
|
+
sku_name = overrides.get("sku_name")
|
|
220
|
+
if sku_name:
|
|
221
|
+
try:
|
|
222
|
+
resp = check_machine_type(client, sku_name, overrides.get("nnode", 1))
|
|
223
|
+
if resp.get("code") != 0 or not (resp.get("data") or {}).get("type"):
|
|
224
|
+
return fail("check_machine_type", resp)
|
|
225
|
+
except Exception as e:
|
|
226
|
+
print(f" ⚠️ 预检异常(忽略): {e}")
|
|
227
|
+
|
|
228
|
+
# 步骤1 创建 job
|
|
229
|
+
resp = client.post_json("/irpapi/szc/v1/job/", {"client": 3, "createSource": 2})
|
|
230
|
+
if resp.get("code") != 0:
|
|
231
|
+
return fail("step1 create job", resp)
|
|
232
|
+
job_id = resp.get("data", {}).get("jobId")
|
|
233
|
+
|
|
234
|
+
# 步骤2 获取 input token
|
|
235
|
+
resp = client.get(f"/irpapi/szc/v1/job/{job_id}/input/token")
|
|
236
|
+
if resp.get("code") != 0:
|
|
237
|
+
return fail("step2 get token", resp)
|
|
238
|
+
storage_token = resp.get("data", {}).get("token")
|
|
239
|
+
|
|
240
|
+
# 步骤3-5 上传文件
|
|
241
|
+
for filename, file_bytes in input_files:
|
|
242
|
+
# 步骤3 init
|
|
243
|
+
init_body = {
|
|
244
|
+
"path": f"{user_id}/job/{job_id}/input/{filename}",
|
|
245
|
+
"option": {"contentDisposition": f'attachment; filename="{filename}"'}
|
|
246
|
+
}
|
|
247
|
+
resp = client.post_json(
|
|
248
|
+
"/dmaapi/szc/v1/file/upload/multipart/init",
|
|
249
|
+
init_body,
|
|
250
|
+
headers={"X-Storage-Token": storage_token}
|
|
251
|
+
)
|
|
252
|
+
if resp.get("code") != 0:
|
|
253
|
+
return fail(f"step3 init ({filename})", resp)
|
|
254
|
+
initial_key = resp.get("data", {}).get("initialKey")
|
|
255
|
+
|
|
256
|
+
# 用 initial_key 构造 X-Storage-Param
|
|
257
|
+
storage_param = base64.b64encode(
|
|
258
|
+
json.dumps({"initialKey": initial_key, "number": 1, "partSize": len(file_bytes)}).encode()
|
|
259
|
+
).decode()
|
|
260
|
+
|
|
261
|
+
# 步骤4 upload
|
|
262
|
+
resp = client.post_raw(
|
|
263
|
+
"/dmaapi/szc/v1/file/upload/multipart/upload",
|
|
264
|
+
file_bytes,
|
|
265
|
+
headers={
|
|
266
|
+
"X-Storage-Token": storage_token,
|
|
267
|
+
"X-Storage-Param": storage_param,
|
|
268
|
+
"Content-Type": "application/x-www-form-urlencoded",
|
|
269
|
+
}
|
|
270
|
+
)
|
|
271
|
+
if resp.get("code") != 0:
|
|
272
|
+
return fail(f"step4 upload ({filename})", resp)
|
|
273
|
+
part_string = resp.get("data", {}).get("partString")
|
|
274
|
+
if part_string is None:
|
|
275
|
+
return fail(f"step4 upload (no partString) ({filename})", resp)
|
|
276
|
+
|
|
277
|
+
# 步骤5 complete
|
|
278
|
+
complete_body = {"initialKey": initial_key, "partString": [part_string]}
|
|
279
|
+
resp = client.post_json(
|
|
280
|
+
"/dmaapi/szc/v1/file/upload/multipart/complete",
|
|
281
|
+
complete_body,
|
|
282
|
+
headers={"X-Storage-Token": storage_token},
|
|
283
|
+
)
|
|
284
|
+
if resp.get("code") != 0:
|
|
285
|
+
return fail(f"step5 complete ({filename})", resp)
|
|
286
|
+
|
|
287
|
+
# 步骤6 snapshot
|
|
288
|
+
snap = build_snapshot_body(client, job_id, user_id, name=name, cmd=cmd, overrides=overrides)
|
|
289
|
+
resp = client.post_json(f"/irpapi/szc/v1/job/{job_id}/snapshot", snap)
|
|
290
|
+
if resp.get("code") != 0:
|
|
291
|
+
return fail("step6 snapshot", resp)
|
|
292
|
+
|
|
293
|
+
# 步骤7 commit
|
|
294
|
+
resp = client.post_empty(f"/irpapi/szc/v1/job/commit/{job_id}",
|
|
295
|
+
headers={"Content-Type": "application/x-www-form-urlencoded"})
|
|
296
|
+
if resp.get("code") != 0:
|
|
297
|
+
return fail("step7 commit", resp)
|
|
298
|
+
|
|
299
|
+
data = resp.get("data") or {}
|
|
300
|
+
result["job_id"] = data.get("jobId") or job_id
|
|
301
|
+
result["bohr_job_id"] = data.get("bohrJobId") or job_id
|
|
302
|
+
result["ok"] = True
|
|
303
|
+
except Exception as e:
|
|
304
|
+
result["error"] = f"exception: {e}"
|
|
305
|
+
result["elapsed_ms"] = int((time.time() - start) * 1000)
|
|
306
|
+
return result
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def submit_batch(client: DmaClient, count: int, concurrency: int, input_files: list,
|
|
310
|
+
*, name_prefix: str, cmd: str, overrides: dict, check: bool) -> tuple:
|
|
311
|
+
"""批量并发提交,每线程独立 DmaClient + requests.Session。返回 (results, elapsed_s)。"""
|
|
312
|
+
start = time.time()
|
|
313
|
+
results = []
|
|
314
|
+
|
|
315
|
+
def task(i):
|
|
316
|
+
c = DmaClient(http_session=requests.Session())
|
|
317
|
+
return submit_one(c, input_files,
|
|
318
|
+
name=f"{name_prefix}-{i}", cmd=cmd, overrides=overrides, check=check)
|
|
319
|
+
|
|
320
|
+
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
|
321
|
+
futures = [ex.submit(task, i) for i in range(count)]
|
|
322
|
+
for fu in as_completed(futures):
|
|
323
|
+
r = fu.result()
|
|
324
|
+
results.append(r)
|
|
325
|
+
mark = "✅" if r["ok"] else "❌"
|
|
326
|
+
tail = "" if r["ok"] else f" {r['error']}"
|
|
327
|
+
print(f" [{len(results)}/{count}] {mark} job={r.get('job_id')} "
|
|
328
|
+
f"bohr={r.get('bohr_job_id')} {r['elapsed_ms']}ms{tail}")
|
|
329
|
+
return results, time.time() - start
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _pct(sorted_times, p):
|
|
333
|
+
if not sorted_times:
|
|
334
|
+
return 0
|
|
335
|
+
idx = max(0, min(len(sorted_times) - 1, int(len(sorted_times) * p / 100)))
|
|
336
|
+
return sorted_times[idx]
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def print_stats(results, elapsed_s):
|
|
340
|
+
total = len(results)
|
|
341
|
+
ok = sum(1 for r in results if r["ok"])
|
|
342
|
+
fail = total - ok
|
|
343
|
+
rate = ok / total * 100 if total else 0
|
|
344
|
+
rps = ok / elapsed_s if elapsed_s else 0
|
|
345
|
+
times = sorted(r["elapsed_ms"] for r in results if r["ok"])
|
|
346
|
+
avg = sum(times) / len(times) if times else 0
|
|
347
|
+
print(f"\n{'=' * 60}")
|
|
348
|
+
print("批量提交统计")
|
|
349
|
+
print(f"{'=' * 60}")
|
|
350
|
+
print(f"总数: {total} 成功: {ok} 失败: {fail} 成功率: {rate:.1f}%")
|
|
351
|
+
print(f"RPS: {rps:.2f} 总耗时: {elapsed_s:.2f}s")
|
|
352
|
+
if times:
|
|
353
|
+
print(f"单任务耗时(ms): avg={avg:.0f} P50={_pct(times, 50)} "
|
|
354
|
+
f"P95={_pct(times, 95)} P99={_pct(times, 99)}")
|
|
355
|
+
if fail:
|
|
356
|
+
steps = Counter()
|
|
357
|
+
for r in results:
|
|
358
|
+
if not r["ok"] and r["error"]:
|
|
359
|
+
steps[r["error"].split(":")[0]] += 1
|
|
360
|
+
print("\n失败按步骤归类:")
|
|
361
|
+
for k, v in steps.most_common():
|
|
362
|
+
print(f" {k}: {v}")
|
|
363
|
+
print(f"{'=' * 60}")
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def poll_task_status(client: DmaClient, job_id: int, timeout: int = 300) -> None:
|
|
367
|
+
"""轮询单个任务的状态直到结束。"""
|
|
368
|
+
import time
|
|
369
|
+
start_time = time.time()
|
|
370
|
+
last_status = None
|
|
371
|
+
|
|
372
|
+
while time.time() - start_time < timeout:
|
|
373
|
+
# 通过任务列表查询指定任务
|
|
374
|
+
resp = client.get("/irpapi/szc/v1/job/list",
|
|
375
|
+
params={"page": 1, "pageSize": 100, "queryType": "all", "createSource": 2})
|
|
376
|
+
if resp.get("code") == 0:
|
|
377
|
+
items = resp.get("data", {}).get("items", [])
|
|
378
|
+
# 查找目标任务
|
|
379
|
+
job = next((j for j in items if j.get("bohrId") == job_id or j.get("id") == job_id), None)
|
|
380
|
+
if job:
|
|
381
|
+
status = job.get("status")
|
|
382
|
+
status_name = STATUS_MAP.get(status, f"未知({status})")
|
|
383
|
+
|
|
384
|
+
if status != last_status:
|
|
385
|
+
print(f" 任务状态: {status_name}")
|
|
386
|
+
last_status = status
|
|
387
|
+
|
|
388
|
+
# 终态: 2=成功, 3=失败, 4=取消, 6=超时
|
|
389
|
+
if status in [2, 3, 4, 6]:
|
|
390
|
+
return
|
|
391
|
+
|
|
392
|
+
time.sleep(2)
|
|
393
|
+
|
|
394
|
+
print(f" ⚠️ 轮询超时({timeout}s),任务仍在运行")
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def handle_submit(client, args):
|
|
398
|
+
"""处理 submit 命令。"""
|
|
399
|
+
# 1. 加载配置并合并参数
|
|
400
|
+
overrides = {}
|
|
401
|
+
|
|
402
|
+
# 从 JSON 配置文件读取
|
|
403
|
+
if args.config_file:
|
|
404
|
+
json_config = load_json_config(args.config_file)
|
|
405
|
+
# 如果包装成批量列表,则提取 _batch_data[0] 作为合并参考,避免报错
|
|
406
|
+
if isinstance(json_config, dict) and json_config.get("_is_batch_list"):
|
|
407
|
+
json_config = json_config["_batch_data"][0] if json_config["_batch_data"] else {}
|
|
408
|
+
|
|
409
|
+
# 映射 JSON 字段到 overrides
|
|
410
|
+
if json_config.get("project_id"):
|
|
411
|
+
overrides["project_id"] = json_config["project_id"]
|
|
412
|
+
if json_config.get("job_name"):
|
|
413
|
+
args.job_name = json_config["job_name"]
|
|
414
|
+
if json_config.get("machine_type"):
|
|
415
|
+
overrides["machine_type"] = json_config["machine_type"]
|
|
416
|
+
if json_config.get("image_address"):
|
|
417
|
+
overrides["image_address"] = json_config["image_address"]
|
|
418
|
+
if json_config.get("command"):
|
|
419
|
+
args.command = json_config["command"]
|
|
420
|
+
if json_config.get("input_directory"):
|
|
421
|
+
args.input_directory = json_config["input_directory"]
|
|
422
|
+
if json_config.get("max_run_time"):
|
|
423
|
+
overrides["max_run_time"] = json_config["max_run_time"]
|
|
424
|
+
if json_config.get("max_reschedule_times"):
|
|
425
|
+
overrides["max_reschedule_times"] = json_config["max_reschedule_times"]
|
|
426
|
+
if json_config.get("nnode"):
|
|
427
|
+
overrides["nnode"] = json_config["nnode"]
|
|
428
|
+
if json_config.get("log_file"):
|
|
429
|
+
overrides["log_file"] = json_config["log_file"]
|
|
430
|
+
if json_config.get("out_files"):
|
|
431
|
+
overrides["out_files"] = json_config["out_files"]
|
|
432
|
+
if json_config.get("result_path"):
|
|
433
|
+
overrides["result_path"] = json_config["result_path"]
|
|
434
|
+
if json_config.get("dataset_path"):
|
|
435
|
+
overrides["dataset_path"] = json_config["dataset_path"]
|
|
436
|
+
|
|
437
|
+
# CLI 参数覆盖 JSON 配置
|
|
438
|
+
if args.project_id:
|
|
439
|
+
overrides["project_id"] = args.project_id
|
|
440
|
+
if args.machine_type:
|
|
441
|
+
overrides["machine_type"] = args.machine_type
|
|
442
|
+
if args.image_address:
|
|
443
|
+
overrides["image_address"] = args.image_address
|
|
444
|
+
if args.max_run_time:
|
|
445
|
+
overrides["max_run_time"] = args.max_run_time
|
|
446
|
+
if args.max_reschedule_times:
|
|
447
|
+
overrides["max_reschedule_times"] = args.max_reschedule_times
|
|
448
|
+
if args.nnode:
|
|
449
|
+
overrides["nnode"] = args.nnode
|
|
450
|
+
if args.log_file:
|
|
451
|
+
overrides["log_file"] = args.log_file
|
|
452
|
+
if args.out_files:
|
|
453
|
+
overrides["out_files"] = args.out_files
|
|
454
|
+
if args.result_path:
|
|
455
|
+
overrides["result_path"] = args.result_path
|
|
456
|
+
if args.dataset_path:
|
|
457
|
+
overrides["dataset_path"] = args.dataset_path
|
|
458
|
+
|
|
459
|
+
# 2. 自动查找 machine_type 和 image_address
|
|
460
|
+
if overrides.get("machine_type"):
|
|
461
|
+
print(f"🔍 查找机型: {overrides['machine_type']}")
|
|
462
|
+
sku_info = find_sku_by_machine_type(client, overrides["machine_type"])
|
|
463
|
+
overrides["sku_id"] = sku_info["skuId"]
|
|
464
|
+
overrides["sku_name"] = sku_info["skuName"]
|
|
465
|
+
overrides["cpu"] = sku_info["cpu"]
|
|
466
|
+
overrides["memory"] = sku_info["memory"]
|
|
467
|
+
overrides["gpu_model"] = sku_info["gpuModel"]
|
|
468
|
+
overrides["gpu_num"] = sku_info["gpuNum"]
|
|
469
|
+
print(f"✅ 找到机型: {sku_info['skuId']} | {sku_info['skuName']} "
|
|
470
|
+
f"(cpu={sku_info['cpu']} mem={sku_info['memory']} gpu={sku_info['gpuModel']}x{sku_info['gpuNum']})")
|
|
471
|
+
|
|
472
|
+
if overrides.get("image_address"):
|
|
473
|
+
print(f"🔍 查找镜像: {overrides['image_address']}")
|
|
474
|
+
sw_info = find_software_by_image_address(client, overrides["image_address"])
|
|
475
|
+
overrides["software_id"] = sw_info["softwareId"]
|
|
476
|
+
overrides["software_version_id"] = sw_info["softwareVersionId"]
|
|
477
|
+
print(f"✅ 找到镜像: {sw_info['softwareId']} | {sw_info['softwareName']} "
|
|
478
|
+
f"-> {sw_info['softwareVersionId']} | {sw_info['softwareVersion']}")
|
|
479
|
+
# 如果未指定命令,使用默认命令
|
|
480
|
+
if not args.command and sw_info.get("default_cmd"):
|
|
481
|
+
args.command = sw_info["default_cmd"]
|
|
482
|
+
print(f"✅ 使用默认命令: {args.command}")
|
|
483
|
+
|
|
484
|
+
# 3. 确定命令
|
|
485
|
+
cmd = args.command or client.defaults.get("cmd", "python calc.py")
|
|
486
|
+
|
|
487
|
+
# 4. 确定任务名
|
|
488
|
+
name_prefix = args.name_prefix or client.defaults.get("name_prefix", "cli-job")
|
|
489
|
+
name = args.job_name or f"{name_prefix}-1"
|
|
490
|
+
|
|
491
|
+
# 5. 准备输入文件
|
|
492
|
+
input_dir = args.input_directory
|
|
493
|
+
input_files = []
|
|
494
|
+
input_dir_specified = False
|
|
495
|
+
|
|
496
|
+
# 检查 JSON 配置文件中是否指定了 input_directory
|
|
497
|
+
if args.config_file:
|
|
498
|
+
json_config = load_json_config(args.config_file)
|
|
499
|
+
if json_config.get("input_directory"):
|
|
500
|
+
input_dir_specified = True
|
|
501
|
+
|
|
502
|
+
# 只有明确指定了 input_directory 才上传目录,否则使用默认文件
|
|
503
|
+
if input_dir and os.path.isdir(input_dir) and input_dir_specified:
|
|
504
|
+
# 上传目录下所有文件
|
|
505
|
+
for filepath in Path(input_dir).iterdir():
|
|
506
|
+
if filepath.is_file():
|
|
507
|
+
with open(filepath, "rb") as f:
|
|
508
|
+
input_files.append((filepath.name, f.read()))
|
|
509
|
+
print(f"📁 上传目录 {input_dir}: {len(input_files)} 个文件")
|
|
510
|
+
else:
|
|
511
|
+
# 使用默认文件
|
|
512
|
+
with open(DEFAULT_INPUT, "rb") as f:
|
|
513
|
+
input_files.append((DEFAULT_INPUT.name, f.read()))
|
|
514
|
+
|
|
515
|
+
if not input_files:
|
|
516
|
+
raise RuntimeError("无输入文件")
|
|
517
|
+
|
|
518
|
+
# 6. 确定并发数
|
|
519
|
+
concurrency = args.concurrency or client.defaults.get("concurrency", 5)
|
|
520
|
+
check = not args.no_check
|
|
521
|
+
|
|
522
|
+
# 7. 处理批量提交
|
|
523
|
+
if args.batch_file:
|
|
524
|
+
# 方式1: 从 JSON 数组文件批量提交
|
|
525
|
+
batch_res = load_json_config(args.batch_file)
|
|
526
|
+
if isinstance(batch_res, dict) and batch_res.get("_is_batch_list"):
|
|
527
|
+
batch_configs = batch_res["_batch_data"]
|
|
528
|
+
else:
|
|
529
|
+
batch_configs = batch_res
|
|
530
|
+
|
|
531
|
+
if not isinstance(batch_configs, list):
|
|
532
|
+
raise RuntimeError("batch_file 必须是 JSON 数组")
|
|
533
|
+
|
|
534
|
+
print(f"🚀 批量提交 {len(batch_configs)} 个任务(来自 JSON 数组),并发数 {concurrency}")
|
|
535
|
+
results = []
|
|
536
|
+
start_time = time.time()
|
|
537
|
+
|
|
538
|
+
def submit_batch_item(config: dict, idx: int):
|
|
539
|
+
"""提交单个批量任务"""
|
|
540
|
+
# 合并参数:config 覆盖 overrides
|
|
541
|
+
item_overrides = overrides.copy()
|
|
542
|
+
for key in ["project_id", "job_name", "machine_type", "image_address", "command",
|
|
543
|
+
"input_directory", "max_run_time", "max_reschedule_times", "nnode",
|
|
544
|
+
"log_file", "out_files", "result_path", "dataset_path"]:
|
|
545
|
+
if config.get(key):
|
|
546
|
+
item_overrides[key] = config[key]
|
|
547
|
+
|
|
548
|
+
# 自动查找
|
|
549
|
+
if item_overrides.get("machine_type"):
|
|
550
|
+
sku_info = find_sku_by_machine_type(client, item_overrides["machine_type"])
|
|
551
|
+
item_overrides["sku_id"] = sku_info["skuId"]
|
|
552
|
+
item_overrides["sku_name"] = sku_info["skuName"]
|
|
553
|
+
item_overrides["cpu"] = sku_info["cpu"]
|
|
554
|
+
item_overrides["memory"] = sku_info["memory"]
|
|
555
|
+
item_overrides["gpu_model"] = sku_info["gpuModel"]
|
|
556
|
+
item_overrides["gpu_num"] = sku_info["gpuNum"]
|
|
557
|
+
|
|
558
|
+
if item_overrides.get("image_address"):
|
|
559
|
+
sw_info = find_software_by_image_address(client, item_overrides["image_address"])
|
|
560
|
+
item_overrides["software_id"] = sw_info["softwareId"]
|
|
561
|
+
item_overrides["software_version_id"] = sw_info["softwareVersionId"]
|
|
562
|
+
|
|
563
|
+
# 准备输入文件
|
|
564
|
+
item_input_dir = config.get("input_directory", "./")
|
|
565
|
+
item_files = []
|
|
566
|
+
if item_input_dir and os.path.isdir(item_input_dir) and config.get("input_directory"):
|
|
567
|
+
for filepath in Path(item_input_dir).iterdir():
|
|
568
|
+
if filepath.is_file():
|
|
569
|
+
with open(filepath, "rb") as f:
|
|
570
|
+
item_files.append((filepath.name, f.read()))
|
|
571
|
+
else:
|
|
572
|
+
with open(DEFAULT_INPUT, "rb") as f:
|
|
573
|
+
item_files.append((DEFAULT_INPUT.name, f.read()))
|
|
574
|
+
|
|
575
|
+
# 确定参数
|
|
576
|
+
item_name = config.get("job_name") or f"{name_prefix}-{idx}"
|
|
577
|
+
item_cmd = config.get("command") or cmd
|
|
578
|
+
|
|
579
|
+
# 提交
|
|
580
|
+
c = DmaClient(http_session=requests.Session())
|
|
581
|
+
return submit_one(c, item_files, name=item_name, cmd=item_cmd,
|
|
582
|
+
overrides=item_overrides, check=check)
|
|
583
|
+
|
|
584
|
+
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
|
585
|
+
futures = [ex.submit(submit_batch_item, cfg, i) for i, cfg in enumerate(batch_configs, 1)]
|
|
586
|
+
for fu in as_completed(futures):
|
|
587
|
+
r = fu.result()
|
|
588
|
+
results.append(r)
|
|
589
|
+
mark = "✅" if r["ok"] else "❌"
|
|
590
|
+
tail = "" if r["ok"] else f" {r['error']}"
|
|
591
|
+
print(f" [{len(results)}/{len(batch_configs)}] {mark} job={r.get('job_id')} ({r['elapsed_ms']}ms){tail}")
|
|
592
|
+
|
|
593
|
+
elapsed = time.time() - start_time
|
|
594
|
+
print_stats(results, elapsed)
|
|
595
|
+
return
|
|
596
|
+
|
|
597
|
+
if args.batch_input_dir:
|
|
598
|
+
# 方式2: 扫描目录批量提交
|
|
599
|
+
if not os.path.isdir(args.batch_input_dir):
|
|
600
|
+
raise RuntimeError(f"目录不存在: {args.batch_input_dir}")
|
|
601
|
+
|
|
602
|
+
subdirs = [d for d in Path(args.batch_input_dir).iterdir() if d.is_dir()]
|
|
603
|
+
if not subdirs:
|
|
604
|
+
raise RuntimeError(f"目录 {args.batch_input_dir} 下无子目录")
|
|
605
|
+
|
|
606
|
+
print(f"🚀 批量提交 {len(subdirs)} 个任务(扫描目录),并发数 {concurrency}")
|
|
607
|
+
results = []
|
|
608
|
+
start_time = time.time()
|
|
609
|
+
|
|
610
|
+
def submit_dir_task(subdir: Path, idx: int):
|
|
611
|
+
"""提交目录任务"""
|
|
612
|
+
# 使用全局参数 + 该目录的 input_directory
|
|
613
|
+
item_overrides = overrides.copy()
|
|
614
|
+
item_overrides["input_directory"] = str(subdir)
|
|
615
|
+
|
|
616
|
+
# 准备输入文件
|
|
617
|
+
item_files = []
|
|
618
|
+
for filepath in subdir.iterdir():
|
|
619
|
+
if filepath.is_file():
|
|
620
|
+
with open(filepath, "rb") as f:
|
|
621
|
+
item_files.append((filepath.name, f.read()))
|
|
622
|
+
|
|
623
|
+
if not item_files:
|
|
624
|
+
with open(DEFAULT_INPUT, "rb") as f:
|
|
625
|
+
item_files.append((DEFAULT_INPUT.name, f.read()))
|
|
626
|
+
|
|
627
|
+
# 任务名使用子目录名
|
|
628
|
+
item_name = f"{name_prefix}-{subdir.name}"
|
|
629
|
+
|
|
630
|
+
# 提交
|
|
631
|
+
c = DmaClient(http_session=requests.Session())
|
|
632
|
+
return submit_one(c, item_files, name=item_name, cmd=cmd,
|
|
633
|
+
overrides=item_overrides, check=check)
|
|
634
|
+
|
|
635
|
+
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
|
636
|
+
futures = [ex.submit(submit_dir_task, d, i) for i, d in enumerate(subdirs, 1)]
|
|
637
|
+
for fu in as_completed(futures):
|
|
638
|
+
r = fu.result()
|
|
639
|
+
results.append(r)
|
|
640
|
+
mark = "✅" if r["ok"] else "❌"
|
|
641
|
+
tail = "" if r["ok"] else f" {r['error']}"
|
|
642
|
+
print(f" [{len(results)}/{len(subdirs)}] {mark} job={r.get('job_id')} ({r['elapsed_ms']}ms){tail}")
|
|
643
|
+
|
|
644
|
+
elapsed = time.time() - start_time
|
|
645
|
+
print_stats(results, elapsed)
|
|
646
|
+
return
|
|
647
|
+
|
|
648
|
+
# 8. dry-run 模式
|
|
649
|
+
if args.dry_run:
|
|
650
|
+
snap_body = build_snapshot_body(client, 0, client._user_id(), name=name, cmd=cmd, overrides=overrides)
|
|
651
|
+
print("【dry-run】将提交的 snapshot body(jobId/inputs 为占位):")
|
|
652
|
+
print(json.dumps(snap_body, indent=2, ensure_ascii=False))
|
|
653
|
+
return
|
|
654
|
+
|
|
655
|
+
# 9. 单个任务或 --count 批量
|
|
656
|
+
count = args.count
|
|
657
|
+
|
|
658
|
+
if count > 1:
|
|
659
|
+
print(f"🚀 批量提交 {count} 个任务,并发数 {concurrency}")
|
|
660
|
+
results, elapsed = submit_batch(client, count, concurrency, input_files,
|
|
661
|
+
name_prefix=name_prefix, cmd=cmd, overrides=overrides, check=check)
|
|
662
|
+
print_stats(results, elapsed)
|
|
663
|
+
else:
|
|
664
|
+
print(f"🚀 提交单个任务: {name}")
|
|
665
|
+
result = submit_one(client, input_files, name=name, cmd=cmd, overrides=overrides, check=check)
|
|
666
|
+
if result["ok"]:
|
|
667
|
+
print(f"✅ 提交成功 jobId={result['job_id']} ({result['elapsed_ms']}ms)")
|
|
668
|
+
# --poll: 轮询当前任务状态
|
|
669
|
+
if args.poll:
|
|
670
|
+
print("📊 轮询任务状态...")
|
|
671
|
+
poll_task_status(client, result['job_id'])
|
|
672
|
+
else:
|
|
673
|
+
print(f"❌ 提交失败: {result['error']}")
|