instance-repo 1.0.8__tar.gz → 1.0.9.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/PKG-INFO +1 -1
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/__init__.py +1 -1
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/cli.py +325 -5
- instance_repo-1.0.9.dev0/instance_repo/clients/__init__.py +12 -0
- instance_repo-1.0.9.dev0/instance_repo/clients/benchmarks.py +57 -0
- instance_repo-1.0.9.dev0/instance_repo/clients/datasets.py +413 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/clients/instances.py +200 -74
- instance_repo-1.0.9.dev0/instance_repo/clients/splits.py +170 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/clients/versions.py +132 -15
- instance_repo-1.0.9.dev0/instance_repo/clients/workflows.py +70 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/content.py +31 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/models.py +385 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/repo.py +22 -3
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/store/oss.py +9 -13
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/store/registry.py +36 -13
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/transport.py +28 -2
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo.egg-info/PKG-INFO +1 -1
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo.egg-info/SOURCES.txt +3 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/pyproject.toml +1 -1
- instance_repo-1.0.8/instance_repo/clients/__init__.py +0 -8
- instance_repo-1.0.8/instance_repo/clients/datasets.py +0 -158
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/README.md +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/_routing.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/_site_defaults.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/bootstrap.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/cache.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/clients/config.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/clients/images.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/clients/reports.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/clients/scaffold.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/concurrency.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/endpoints.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/errors.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/image.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/layout.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/loader.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/paths.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/release.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/retry.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/store/__init__.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/store/acr.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/store/base.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/tasktoml.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo/validate.py +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo.egg-info/dependency_links.txt +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo.egg-info/entry_points.txt +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo.egg-info/requires.txt +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/instance_repo.egg-info/top_level.txt +0 -0
- {instance_repo-1.0.8 → instance_repo-1.0.9.dev0}/setup.cfg +0 -0
|
@@ -60,8 +60,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
60
60
|
p.add_argument("--no-image", action="store_true")
|
|
61
61
|
p.add_argument("--overwrite", action="store_true")
|
|
62
62
|
p.add_argument("--no-register", action="store_true",
|
|
63
|
-
help="
|
|
64
|
-
"
|
|
63
|
+
help="只传 OSS 不写库(默认上传后自动 ingest 写库;"
|
|
64
|
+
"置上则跳过 ingest,之后自行 ingest 扫描入库)")
|
|
65
65
|
p.add_argument("--no-verify", action="store_true",
|
|
66
66
|
help="跳过上传后的回读校验(默认校验 OSS 对象/ACR 镜像确已落库)")
|
|
67
67
|
|
|
@@ -76,7 +76,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
76
76
|
pm.add_argument("--no-image", action="store_true")
|
|
77
77
|
pm.add_argument("--overwrite", action="store_true")
|
|
78
78
|
pm.add_argument("--register", action="store_true",
|
|
79
|
-
help="
|
|
79
|
+
help="整批上传完成后一次 ingest 写库(缺省只传 OSS,不写库)")
|
|
80
80
|
pm.add_argument("--no-verify", action="store_true")
|
|
81
81
|
pm.add_argument("--concurrency", type=int, default=8)
|
|
82
82
|
pm.add_argument("--continue-on-error", action="store_true",
|
|
@@ -109,6 +109,20 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
109
109
|
lst.add_argument("--version", default=None)
|
|
110
110
|
lst.add_argument("--split", default=None,
|
|
111
111
|
help="按 split 过滤;缺省为跨 split 检索")
|
|
112
|
+
# 1.0.10:分页/模糊搜索/过滤模式。任一给定 → 走 list_paged 单页语义;
|
|
113
|
+
# 都没给时仍走 list 自动翻页取全量(向后兼容)。
|
|
114
|
+
lst.add_argument("--q", default=None,
|
|
115
|
+
help="instance_id 子串模糊搜索(不区分大小写;给定即单页模式)")
|
|
116
|
+
lst.add_argument("--instance-id", default=None, dest="instance_id",
|
|
117
|
+
help="instance_id 精确等值(给定即单页模式)")
|
|
118
|
+
lst.add_argument("--tags", default=None,
|
|
119
|
+
help="k1:v1,k2:v2 等值过滤(key 含 ./$ 服务端拒绝;给定即单页模式)")
|
|
120
|
+
lst.add_argument("--difficulty", choices=["easy", "medium", "hard"], default=None,
|
|
121
|
+
help="按难度过滤(给定即单页模式)")
|
|
122
|
+
lst.add_argument("--page", type=int, default=1,
|
|
123
|
+
help="分页页码(默认 1;与 --q/--tags/--difficulty 配合生效)")
|
|
124
|
+
lst.add_argument("--page-size", type=int, default=50, dest="page_size",
|
|
125
|
+
help="分页大小(默认 50;服务端上限 200)")
|
|
112
126
|
|
|
113
127
|
pl = sub.add_parser("pull", help="拉取并解包(digest 校验)")
|
|
114
128
|
pl.add_argument("ref", nargs="?",
|
|
@@ -272,7 +286,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
272
286
|
|
|
273
287
|
ver = sub.add_parser("version", help="版本管理(create/list/status/update/set-run-type)")
|
|
274
288
|
ver.add_argument("action",
|
|
275
|
-
choices=["create", "list", "status", "update",
|
|
289
|
+
choices=["create", "list", "status", "update",
|
|
290
|
+
"set-run-type", "get"])
|
|
276
291
|
ver.add_argument("ref", nargs="?",
|
|
277
292
|
help="create/list: {L1}/{L2};"
|
|
278
293
|
"status/update/set-run-type: {L1}/{L2}/{version}"
|
|
@@ -285,12 +300,86 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
285
300
|
help="create 用:oss://{bucket}/{prefix}")
|
|
286
301
|
ver.add_argument("--storage-type", choices=["oss"], default="oss")
|
|
287
302
|
ver.add_argument("--splits", default=None, help="逗号分隔 split 名")
|
|
303
|
+
ver.add_argument("--split", default=None,
|
|
304
|
+
help="status 用:查 split 级真实记录(split_first 发布状态/目标路径)")
|
|
288
305
|
ver.add_argument("--status", choices=_STATUS_CHOICES, default=None,
|
|
289
306
|
help="update 用:版本状态流转")
|
|
290
307
|
ver.add_argument("--run-type", choices=["train", "eval"], default=None,
|
|
291
308
|
help="version 级用途标签")
|
|
292
309
|
ver.add_argument("--split-run-types", default=None,
|
|
293
310
|
help="split 级打标,格式 name=train,name2=eval(与 --run-type 互斥)")
|
|
311
|
+
ver.add_argument("--environment", default=None,
|
|
312
|
+
help="get 用:覆盖 profile 派生的 environment(pre/online)")
|
|
313
|
+
|
|
314
|
+
# ── 1.0.10 控制面管理子命令 ─────────────────────────────────────────────────
|
|
315
|
+
sp = sub.add_parser("split", help="split 记录管理(list/get/set-run-type)")
|
|
316
|
+
sp.add_argument("action", choices=["list", "get", "set-run-type"])
|
|
317
|
+
sp.add_argument("ref", nargs="?",
|
|
318
|
+
help="list: {L1}/{L2};get/set-run-type: {L1}/{L2}/{split}"
|
|
319
|
+
"(或用 --dataset/--split)")
|
|
320
|
+
sp.add_argument("--dataset", default=None, help="{L1}/{L2}")
|
|
321
|
+
sp.add_argument("--split", default=None,
|
|
322
|
+
help="目标 split 名(get/set-run-type 用)")
|
|
323
|
+
sp.add_argument("--version", default=None,
|
|
324
|
+
help="list 按 version 精确过滤;get 可选定位版本")
|
|
325
|
+
sp.add_argument("--scope", choices=["unversioned", "exact", "all"], default=None,
|
|
326
|
+
help="list 跨版本策略:缺省无 version → all(SDK 取全量哲学),"
|
|
327
|
+
"有 version 省略(服务端只接受 exact)")
|
|
328
|
+
sp.add_argument("--status", choices=_STATUS_CHOICES, default=None,
|
|
329
|
+
help="list 按状态过滤")
|
|
330
|
+
sp.add_argument("--run-type", choices=["train", "eval", ""], default=None,
|
|
331
|
+
dest="run_type",
|
|
332
|
+
help="set-run-type 用:train/eval 或空串清除")
|
|
333
|
+
sp.add_argument("--page", type=int, default=1)
|
|
334
|
+
sp.add_argument("--page-size", type=int, default=50, dest="page_size")
|
|
335
|
+
|
|
336
|
+
ds = sub.add_parser("dataset",
|
|
337
|
+
help="数据集/基准/工作流管理(list/get/update/access/"
|
|
338
|
+
"revoke/benchmarks/workflows)")
|
|
339
|
+
ds.add_argument("action",
|
|
340
|
+
choices=["list", "get", "update", "access", "revoke",
|
|
341
|
+
"benchmarks", "workflows"])
|
|
342
|
+
ds.add_argument("ref", nargs="?", metavar="dataset",
|
|
343
|
+
help="list/get/update/access/revoke: {L1}/{L2};"
|
|
344
|
+
"benchmarks: source_id(get);workflows: wf-id(get)")
|
|
345
|
+
ds.add_argument("--dataset", default=None, help="{L1}/{L2}")
|
|
346
|
+
ds.add_argument("--q", default=None,
|
|
347
|
+
help="list/benchmarks 模糊搜索;list 对 name+display_name,"
|
|
348
|
+
"benchmarks 对 name")
|
|
349
|
+
ds.add_argument("--status", default=None,
|
|
350
|
+
help="list/benchmarks 过滤;workflows 过滤"
|
|
351
|
+
"(pending/running/awaiting_approval/succeeded/failed/canceled)")
|
|
352
|
+
ds.add_argument("--visibility", default=None,
|
|
353
|
+
help="list 逗号分隔过滤(public/private)")
|
|
354
|
+
ds.add_argument("--owned-by-me", action="store_true", dest="owned_by_me",
|
|
355
|
+
help="list 按 owner=caller 过滤")
|
|
356
|
+
ds.add_argument("--sort-by", choices=["updated_at", "dataset_name"],
|
|
357
|
+
default=None, dest="sort_by")
|
|
358
|
+
ds.add_argument("--order", choices=["asc", "desc"], default=None)
|
|
359
|
+
ds.add_argument("--include-deprecated", action="store_true",
|
|
360
|
+
dest="include_deprecated")
|
|
361
|
+
ds.add_argument("--environment", default=None,
|
|
362
|
+
help="get/update 覆盖 profile 派生的 environment")
|
|
363
|
+
ds.add_argument("--metadata-model", choices=["split_first", "version_first"],
|
|
364
|
+
default=None, dest="metadata_model")
|
|
365
|
+
# update 用(至少一项)
|
|
366
|
+
ds.add_argument("--display-name", default=None, dest="display_name")
|
|
367
|
+
ds.add_argument("--description", default=None)
|
|
368
|
+
ds.add_argument("--benchmark-id", default=None, dest="benchmark_id")
|
|
369
|
+
ds.add_argument("--owner-team-id", default=None, dest="owner_team_id")
|
|
370
|
+
ds.add_argument("--default-scaffold", default=None, dest="default_scaffold")
|
|
371
|
+
# access/revoke 用
|
|
372
|
+
ds.add_argument("--principal", default=None, help="工号或 user_group id")
|
|
373
|
+
ds.add_argument("--role", choices=["reader", "writer", "admin"], default=None)
|
|
374
|
+
ds.add_argument("--principal-type", choices=["user", "user_group"],
|
|
375
|
+
default="user", dest="principal_type")
|
|
376
|
+
# benchmarks/workflows 用
|
|
377
|
+
ds.add_argument("--is-core", action="store_true", dest="is_core")
|
|
378
|
+
ds.add_argument("--source", default=None, help="benchmarks source 精确过滤")
|
|
379
|
+
ds.add_argument("--kind", default=None, help="workflows kind 过滤")
|
|
380
|
+
ds.add_argument("--creator", default=None, help="workflows creator 过滤")
|
|
381
|
+
ds.add_argument("--page", type=int, default=1)
|
|
382
|
+
ds.add_argument("--page-size", type=int, default=20, dest="page_size")
|
|
294
383
|
return ap
|
|
295
384
|
|
|
296
385
|
|
|
@@ -301,6 +390,36 @@ def _split_ref(ref: str, parts: int) -> list[str]:
|
|
|
301
390
|
return segs
|
|
302
391
|
|
|
303
392
|
|
|
393
|
+
def _page_count(res) -> int:
|
|
394
|
+
"""分页结果的总页数(total<0 表示服务端无分页信封 → 1)。"""
|
|
395
|
+
if getattr(res, "total", -1) < 0 or not getattr(res, "page_size", 0):
|
|
396
|
+
return 1
|
|
397
|
+
return max(1, (res.total + res.page_size - 1) // res.page_size)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _split_instance_count(split) -> int:
|
|
401
|
+
"""取 split 的实例计数:优先 manifest.instance_count,回落 0。"""
|
|
402
|
+
m = getattr(split, "manifest", None)
|
|
403
|
+
if m is not None:
|
|
404
|
+
c = getattr(m, "instance_count", None)
|
|
405
|
+
if isinstance(c, int):
|
|
406
|
+
return c
|
|
407
|
+
return 0
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _resolve_split_ref(args) -> tuple[str, str]:
|
|
411
|
+
"""定位 dataset + split:位置 ref({L1}/{L2}/{split})或 --dataset/--split。"""
|
|
412
|
+
if args.dataset:
|
|
413
|
+
if not args.split:
|
|
414
|
+
raise SystemExit("给定 --dataset 时 split 需要 --split")
|
|
415
|
+
return args.dataset, args.split
|
|
416
|
+
if args.ref:
|
|
417
|
+
segs = _split_ref(args.ref, 3)
|
|
418
|
+
return f"{segs[0]}/{segs[1]}", segs[2]
|
|
419
|
+
raise SystemExit("split get/set-run-type 需要 {L1}/{L2}/{split} 或 "
|
|
420
|
+
"--dataset/--split")
|
|
421
|
+
|
|
422
|
+
|
|
304
423
|
def _split_dataset(dataset: str) -> list[str]:
|
|
305
424
|
"""校验 dataset 为 ``{L1}/{L2}`` 两段式。
|
|
306
425
|
|
|
@@ -454,6 +573,11 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
454
573
|
verify=not args.no_verify)
|
|
455
574
|
print(f"{res.instance_id} uploaded={res.uploaded} "
|
|
456
575
|
f"image_pushed={res.image_pushed}", file=out)
|
|
576
|
+
if res.ingest:
|
|
577
|
+
stats = res.ingest.get("stats", {}) or {}
|
|
578
|
+
print(f"ingest {res.ingest.get('ingest_id')} "
|
|
579
|
+
f"status={res.ingest.get('status')} "
|
|
580
|
+
f"instances={stats.get('instance_count')}", file=out)
|
|
457
581
|
return 0
|
|
458
582
|
|
|
459
583
|
if args.cmd == "push-many":
|
|
@@ -505,6 +629,29 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
505
629
|
|
|
506
630
|
if args.cmd == "list":
|
|
507
631
|
dataset, version, _ = _resolve_target(args, 3)
|
|
632
|
+
# 1.0.10:任一过滤/分页 flag 给定 → 走 list_paged 单页;否则仍走
|
|
633
|
+
# list 自动翻页取全量(向后兼容)。
|
|
634
|
+
paged = any([args.q, args.instance_id, args.tags, args.difficulty,
|
|
635
|
+
args.page != 1, args.page_size != 50])
|
|
636
|
+
if paged:
|
|
637
|
+
tags_dict = None
|
|
638
|
+
if args.tags:
|
|
639
|
+
tags_dict = {}
|
|
640
|
+
for pair in args.tags.split(","):
|
|
641
|
+
if ":" not in pair:
|
|
642
|
+
raise SystemExit(f"--tags expects k1:v1,k2:v2, got {args.tags!r}")
|
|
643
|
+
k, v = pair.split(":", 1)
|
|
644
|
+
tags_dict[k] = v
|
|
645
|
+
res = r.instances.list_paged(dataset, args.version, split=args.split,
|
|
646
|
+
q=args.q, instance_id=args.instance_id,
|
|
647
|
+
tags=tags_dict, difficulty=args.difficulty,
|
|
648
|
+
page=args.page, page_size=args.page_size)
|
|
649
|
+
for inst in res.items:
|
|
650
|
+
print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}",
|
|
651
|
+
file=out)
|
|
652
|
+
print(f"page={res.page}/{(res.total+res.page_size-1)//res.page_size if res.total>0 else 1} "
|
|
653
|
+
f"total={res.total} has_more={res.has_more}", file=out)
|
|
654
|
+
return 0
|
|
508
655
|
for inst in r.instances.list(dataset, version, args.split):
|
|
509
656
|
print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}", file=out)
|
|
510
657
|
return 0
|
|
@@ -711,7 +858,34 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
711
858
|
return 0
|
|
712
859
|
dataset, version, _ = _resolve_target(args, 3)
|
|
713
860
|
if args.action == "status":
|
|
714
|
-
|
|
861
|
+
if args.split:
|
|
862
|
+
sp = r.versions.split_detail(dataset, version, args.split)
|
|
863
|
+
print(f"{sp.get('name', args.split)}\t{sp.get('status', '')}\t"
|
|
864
|
+
f"{sp.get('storage_path', '')}\t"
|
|
865
|
+
f"published_at={sp.get('published_at') or ''}", file=out)
|
|
866
|
+
else:
|
|
867
|
+
print(r.versions.status(dataset, version), file=out)
|
|
868
|
+
return 0
|
|
869
|
+
if args.action == "get":
|
|
870
|
+
# 1.0.10:取版本完整详情(splits 明细 + manifest + 时间戳)。
|
|
871
|
+
# split_first 分支时服务端聚合 legacy 投影。
|
|
872
|
+
dv = r.versions.get(dataset, version, environment=args.environment)
|
|
873
|
+
print(f"version\t{dv.get('version','')}\tstatus={dv.get('status','')}"
|
|
874
|
+
f"\tstorage_path={dv.get('storage_path','')}", file=out)
|
|
875
|
+
print(f"published_at={dv.get('published_at') or ''}"
|
|
876
|
+
f"\tlast_ingested_at={dv.get('last_ingested_at') or ''}",
|
|
877
|
+
file=out)
|
|
878
|
+
print(f"dataset_version_id={dv.get('dataset_version_id','')}",
|
|
879
|
+
file=out)
|
|
880
|
+
splits = dv.get("splits") or []
|
|
881
|
+
for s in splits:
|
|
882
|
+
print(f" split\t{s.get('name','')}\tstatus={s.get('status','')}"
|
|
883
|
+
f"\toss_prefix={s.get('oss_prefix','')}"
|
|
884
|
+
f"\tinstance_count={s.get('instance_count') or 0}", file=out)
|
|
885
|
+
manifest = dv.get("manifest") or {}
|
|
886
|
+
print(f"manifest\tinstance_count={manifest.get('instance_count') or 0}"
|
|
887
|
+
f"\tfile_count={manifest.get('file_count') or 0}"
|
|
888
|
+
f"\tsize_bytes={manifest.get('size_bytes') or 0}", file=out)
|
|
715
889
|
return 0
|
|
716
890
|
if args.action == "set-run-type":
|
|
717
891
|
r.versions.set_run_type(dataset, version,
|
|
@@ -727,6 +901,152 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
727
901
|
print(f"updated {dataset}/{version}", file=out)
|
|
728
902
|
return 0
|
|
729
903
|
|
|
904
|
+
# ── 1.0.10 控制面管理子命令 dispatch ────────────────────────────────────────
|
|
905
|
+
if args.cmd == "split":
|
|
906
|
+
if args.action == "list":
|
|
907
|
+
dataset = (args.dataset or args.ref or "").strip()
|
|
908
|
+
if not dataset:
|
|
909
|
+
raise SystemExit("split list 需要 --dataset 或位置 ref({L1}/{L2})")
|
|
910
|
+
res = r.splits.list(dataset,
|
|
911
|
+
version=args.version, version_scope=args.scope,
|
|
912
|
+
status=args.status,
|
|
913
|
+
page=args.page, page_size=args.page_size)
|
|
914
|
+
for s in res.items:
|
|
915
|
+
print(f"{s.name}\tversion={s.version or ''}\tstatus={s.status}"
|
|
916
|
+
f"\trun_type={s.run_type or ''}\t"
|
|
917
|
+
f"instance_count={_split_instance_count(s)}", file=out)
|
|
918
|
+
print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
|
|
919
|
+
return 0
|
|
920
|
+
# get / set-run-type:需要定位到具体 split
|
|
921
|
+
dataset, split_name = _resolve_split_ref(args)
|
|
922
|
+
if args.action == "get":
|
|
923
|
+
sp = r.splits.get(dataset, split_name, version=args.version)
|
|
924
|
+
if sp is None:
|
|
925
|
+
raise SystemExit(f"split {dataset}/{split_name} not found")
|
|
926
|
+
print(f"split\t{sp.name}\tversion={sp.version or ''}"
|
|
927
|
+
f"\tstatus={sp.status}\trun_type={sp.run_type or ''}", file=out)
|
|
928
|
+
print(f"storage_path={sp.storage_path}", file=out)
|
|
929
|
+
print(f"published_at={sp.published_at or ''}"
|
|
930
|
+
f"\tlast_ingested_at={sp.last_ingested_at or ''}", file=out)
|
|
931
|
+
print(f"instance_count={_split_instance_count(sp)}", file=out)
|
|
932
|
+
return 0
|
|
933
|
+
# set-run-type
|
|
934
|
+
if args.run_type is None:
|
|
935
|
+
raise SystemExit("split set-run-type 需要 --run-type(train/eval/'')")
|
|
936
|
+
r.splits.update(dataset, split_name,
|
|
937
|
+
version=args.version, run_type=args.run_type)
|
|
938
|
+
print(f"split {dataset}/{split_name} run_type={args.run_type or '(cleared)'}",
|
|
939
|
+
file=out)
|
|
940
|
+
return 0
|
|
941
|
+
|
|
942
|
+
if args.cmd == "dataset":
|
|
943
|
+
if args.action == "list":
|
|
944
|
+
res = r.datasets.list_paged(
|
|
945
|
+
q=args.q, visibility=args.visibility,
|
|
946
|
+
owned_by_me=args.owned_by_me or None,
|
|
947
|
+
sort_by=args.sort_by, order=args.order,
|
|
948
|
+
include_deprecated=args.include_deprecated or None,
|
|
949
|
+
environment=args.environment,
|
|
950
|
+
metadata_model=args.metadata_model,
|
|
951
|
+
page=args.page, page_size=args.page_size)
|
|
952
|
+
for ds in res.items:
|
|
953
|
+
print(f"{ds.dataset_name}\tvisibility={ds.visibility}"
|
|
954
|
+
f"\tversion_count={ds.version_count}"
|
|
955
|
+
f"\tsplit_count={ds.split_count}"
|
|
956
|
+
f"\tlatest={ds.latest_version or ''}", file=out)
|
|
957
|
+
print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
|
|
958
|
+
return 0
|
|
959
|
+
# 其余动作需要 dataset 名(位置 ref 或 --dataset)
|
|
960
|
+
dataset = (args.dataset or args.ref or "").strip()
|
|
961
|
+
if args.action in ("get", "update", "access", "revoke") and not dataset:
|
|
962
|
+
raise SystemExit(f"dataset {args.action} 需要 --dataset 或位置 ref")
|
|
963
|
+
if args.action == "get":
|
|
964
|
+
d = r.datasets.get_detail(dataset, environment=args.environment,
|
|
965
|
+
metadata_model=args.metadata_model)
|
|
966
|
+
print(f"dataset\t{d.dataset_name}\tid={d.dataset_id}"
|
|
967
|
+
f"\tvisibility={d.visibility}", file=out)
|
|
968
|
+
print(f"version_count={d.version_count}\tsplit_count={d.split_count}"
|
|
969
|
+
f"\tlatest={d.latest_version or ''}", file=out)
|
|
970
|
+
print(f"instance_count={d.stats.instance_count}"
|
|
971
|
+
f"\tfile_count={d.stats.file_count}"
|
|
972
|
+
f"\tsize_bytes={d.stats.size_bytes}", file=out)
|
|
973
|
+
return 0
|
|
974
|
+
if args.action == "update":
|
|
975
|
+
kwargs = {}
|
|
976
|
+
if args.display_name is not None:
|
|
977
|
+
kwargs["display_name"] = args.display_name
|
|
978
|
+
if args.description is not None:
|
|
979
|
+
kwargs["description"] = args.description
|
|
980
|
+
if args.visibility is not None:
|
|
981
|
+
kwargs["visibility"] = args.visibility
|
|
982
|
+
if args.benchmark_id is not None:
|
|
983
|
+
kwargs["benchmark_id"] = args.benchmark_id
|
|
984
|
+
if args.owner_team_id is not None:
|
|
985
|
+
kwargs["owner_team_id"] = args.owner_team_id
|
|
986
|
+
if args.default_scaffold is not None:
|
|
987
|
+
kwargs["default_scaffold"] = args.default_scaffold
|
|
988
|
+
if not kwargs:
|
|
989
|
+
raise SystemExit(
|
|
990
|
+
"dataset update 至少给一个:--display-name / --description / "
|
|
991
|
+
"--visibility / --benchmark-id / --owner-team-id / "
|
|
992
|
+
"--default-scaffold")
|
|
993
|
+
r.datasets.update(dataset, **kwargs)
|
|
994
|
+
print(f"updated {dataset}", file=out)
|
|
995
|
+
return 0
|
|
996
|
+
if args.action in ("access", "revoke"):
|
|
997
|
+
if args.action == "access":
|
|
998
|
+
res = r.datasets.list_access(dataset,
|
|
999
|
+
page=args.page, page_size=args.page_size)
|
|
1000
|
+
for rb in res.items:
|
|
1001
|
+
print(f"{rb.principal_type}\t{rb.principal_id}\t{rb.role_id}"
|
|
1002
|
+
f"\tadded_by={rb.added_by}\tadded_at={rb.added_at}",
|
|
1003
|
+
file=out)
|
|
1004
|
+
print(f"page={res.page}/{_page_count(res)} total={res.total}",
|
|
1005
|
+
file=out)
|
|
1006
|
+
return 0
|
|
1007
|
+
if not (args.principal and args.role):
|
|
1008
|
+
raise SystemExit("dataset revoke 需要 --principal + --role")
|
|
1009
|
+
r.datasets.revoke(dataset, args.principal, args.role,
|
|
1010
|
+
principal_type=args.principal_type)
|
|
1011
|
+
print(f"revoked {args.principal} ({args.principal_type}) "
|
|
1012
|
+
f"{args.role} from {dataset}", file=out)
|
|
1013
|
+
return 0
|
|
1014
|
+
if args.action == "benchmarks":
|
|
1015
|
+
if args.ref and not args.q:
|
|
1016
|
+
b = r.benchmarks.get(args.ref)
|
|
1017
|
+
if b is None:
|
|
1018
|
+
raise SystemExit(f"benchmark {args.ref} not found")
|
|
1019
|
+
print(f"benchmark\t{b.source_id}\tname={b.name}"
|
|
1020
|
+
f"\tteacher_directions={','.join(b.teacher_directions)}"
|
|
1021
|
+
f"\tis_core={b.is_core}", file=out)
|
|
1022
|
+
return 0
|
|
1023
|
+
res = r.benchmarks.list(
|
|
1024
|
+
q=args.q, source=args.source, is_core=args.is_core or None,
|
|
1025
|
+
page=args.page, page_size=args.page_size)
|
|
1026
|
+
for b in res.items:
|
|
1027
|
+
print(f"{b.source_id}\tname={b.name}\tis_core={b.is_core}"
|
|
1028
|
+
f"\tdirections={','.join(b.teacher_directions)}", file=out)
|
|
1029
|
+
print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
|
|
1030
|
+
return 0
|
|
1031
|
+
# workflows
|
|
1032
|
+
if args.ref:
|
|
1033
|
+
wf = r.workflows.get(args.ref)
|
|
1034
|
+
print(f"workflow\t{wf.id}\tkind={wf.kind}\tstatus={wf.status}"
|
|
1035
|
+
f"\tcurrent_step={wf.current_step}", file=out)
|
|
1036
|
+
if wf.approval:
|
|
1037
|
+
print(f"approval\tprovider={wf.approval.provider}"
|
|
1038
|
+
f"\tticket={wf.approval.ticket_link}"
|
|
1039
|
+
f"\tstatus={wf.approval.status}", file=out)
|
|
1040
|
+
return 0
|
|
1041
|
+
res = r.workflows.list(
|
|
1042
|
+
kind=args.kind, status=args.status, creator=args.creator,
|
|
1043
|
+
page=args.page, page_size=args.page_size)
|
|
1044
|
+
for wf in res.items:
|
|
1045
|
+
print(f"{wf.id}\tkind={wf.kind}\tstatus={wf.status}"
|
|
1046
|
+
f"\tcurrent_step={wf.current_step}", file=out)
|
|
1047
|
+
print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
|
|
1048
|
+
return 0
|
|
1049
|
+
|
|
730
1050
|
return 2 # pragma: no cover - argparse 已保证 cmd 合法
|
|
731
1051
|
|
|
732
1052
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""clients — 各资源客户端 + config。"""
|
|
2
|
+
from .config import Profile, load_profile
|
|
3
|
+
from .datasets import DatasetsClient
|
|
4
|
+
from .versions import VersionsClient
|
|
5
|
+
from .instances import InstancesClient
|
|
6
|
+
from .splits import SplitsClient
|
|
7
|
+
from .benchmarks import BenchmarksClient
|
|
8
|
+
from .workflows import WorkflowsClient
|
|
9
|
+
|
|
10
|
+
__all__ = ["Profile", "load_profile", "DatasetsClient", "VersionsClient",
|
|
11
|
+
"InstancesClient", "SplitsClient", "BenchmarksClient",
|
|
12
|
+
"WorkflowsClient"]
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""clients/benchmarks.py — benchmark 目录查询(→ apiserver /apis/v1/benchmarks)。
|
|
2
|
+
|
|
3
|
+
服务端契约(meta_handler / meta_service 实测):
|
|
4
|
+
* ``GET /benchmarks``:分页信封(默认 page=1/page_size=20,上限 200);
|
|
5
|
+
``q`` 对 name 模糊匹配(不区分大小写子串);``is_core`` 布尔(非法值服务端
|
|
6
|
+
静默忽略,SDK 不做额外校验);``source`` 精确。按 updated_at 倒序。
|
|
7
|
+
* ``GET /benchmarks/{source_id}``:点查;不存在 404。认证可选(combinedAuthOptional)。
|
|
8
|
+
|
|
9
|
+
主键是 ``source_id``(无独立 id 字段)。claim 弹窗的 Benchmark 候选即
|
|
10
|
+
``list(q=...)``;控制台"Benchmark maintainer is not allowed"的禁用态是**前端
|
|
11
|
+
逻辑**(apiserver 无 enabled/allowed 字段),SDK 不模拟——调用方按自身权限过滤。
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from urllib.parse import quote
|
|
16
|
+
|
|
17
|
+
from ..errors import TransportError
|
|
18
|
+
from ..models import Benchmark, PagedResult
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BenchmarksClient:
|
|
22
|
+
_BASE = "/apis/v1/benchmarks"
|
|
23
|
+
|
|
24
|
+
def __init__(self, transport) -> None:
|
|
25
|
+
self._t = transport
|
|
26
|
+
|
|
27
|
+
def list(self, *, q: str | None = None, is_core: bool | None = None,
|
|
28
|
+
source: str | None = None, page: int = 1,
|
|
29
|
+
page_size: int = 20) -> PagedResult:
|
|
30
|
+
"""benchmark 分页列表(单页)。``q`` 模糊搜索 name(不区分大小写子串)。"""
|
|
31
|
+
params: list[tuple[str, str]] = []
|
|
32
|
+
if (q or "").strip():
|
|
33
|
+
params.append(("q", q.strip()))
|
|
34
|
+
if is_core is not None:
|
|
35
|
+
params.append(("is_core", "true" if is_core else "false"))
|
|
36
|
+
if (source or "").strip():
|
|
37
|
+
params.append(("source", source.strip()))
|
|
38
|
+
params.append(("page", str(int(page))))
|
|
39
|
+
params.append(("page_size", str(int(page_size))))
|
|
40
|
+
query = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
|
|
41
|
+
data, pagination = self._t.request_with_meta("GET", f"{self._BASE}?{query}")
|
|
42
|
+
rows = data if isinstance(data, list) else (
|
|
43
|
+
data.get("benchmarks", []) if isinstance(data, dict) else [])
|
|
44
|
+
items = [Benchmark.from_dict(r) for r in rows if isinstance(r, dict)]
|
|
45
|
+
return PagedResult.from_response(items, pagination, int(page), int(page_size))
|
|
46
|
+
|
|
47
|
+
def get(self, source_id: str) -> Benchmark | None:
|
|
48
|
+
"""按 source_id 点查;不存在返回 None("没有"不是错误)。"""
|
|
49
|
+
if not (source_id or "").strip():
|
|
50
|
+
raise ValueError("source_id is required")
|
|
51
|
+
try:
|
|
52
|
+
data = self._t.get(f"{self._BASE}/{quote(source_id.strip(), safe='')}")
|
|
53
|
+
except TransportError as e:
|
|
54
|
+
if e.status == 404:
|
|
55
|
+
return None
|
|
56
|
+
raise
|
|
57
|
+
return Benchmark.from_dict(data) if isinstance(data, dict) else None
|