instance-repo 1.3.5.dev7__tar.gz → 1.4.0.dev3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/PKG-INFO +1 -1
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/__init__.py +8 -7
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/cli.py +104 -42
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/config.py +25 -1
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/datasets.py +18 -11
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/images.py +13 -5
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/instances.py +221 -71
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/reports.py +3 -1
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/versions.py +195 -84
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/concurrency.py +21 -12
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/errors.py +17 -3
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/layout.py +13 -14
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/models.py +58 -1
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/paths.py +94 -36
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/release.py +36 -29
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/repo.py +142 -85
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/store/oss_discovery.py +23 -22
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/store/registry.py +5 -3
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/transport.py +27 -12
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo.egg-info/PKG-INFO +1 -1
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/pyproject.toml +1 -1
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/README.md +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/_routing.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/_site_defaults.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/artifacts.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/bootstrap.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/cache.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/__init__.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/benchmarks.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/scaffold.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/splits.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/clients/workflows.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/content.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/endpoints.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/image.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/loader.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/retry.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/store/__init__.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/store/acr.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/store/base.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/store/oss.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/tasktoml.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo/validate.py +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo.egg-info/SOURCES.txt +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo.egg-info/dependency_links.txt +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo.egg-info/entry_points.txt +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo.egg-info/requires.txt +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/instance_repo.egg-info/top_level.txt +0 -0
- {instance_repo-1.3.5.dev7 → instance_repo-1.4.0.dev3}/setup.cfg +0 -0
|
@@ -8,8 +8,8 @@ from .loader import load_instances
|
|
|
8
8
|
from .content import compute_source_hash
|
|
9
9
|
from .models import (
|
|
10
10
|
DatasetSeries, DatasetVersion, DatasetInstance, ManifestEntry,
|
|
11
|
-
OwnerReference, Feedback, Identity,
|
|
12
|
-
RolloutReport, RolloutJob, ScaffoldConfig,
|
|
11
|
+
OwnerReference, Feedback, Identity, IngestReport, RegistrationReport,
|
|
12
|
+
PushResult, PushManyReport, RolloutReport, RolloutJob, ScaffoldConfig,
|
|
13
13
|
DatasetObjectDirectory, DatasetObjectFile, DatasetObjectListResult,
|
|
14
14
|
canonical_json, parse_feedbacks, append_feedback,
|
|
15
15
|
)
|
|
@@ -17,15 +17,16 @@ from .errors import (
|
|
|
17
17
|
InstanceRepoError, LayoutError, SchemaError, OwnerRefMissing, ImageMissing,
|
|
18
18
|
Immutable, DigestMismatch, Forbidden, NotVisible, TargetNotAllowed,
|
|
19
19
|
NetworkDenied, CredExpired, PrivateNetworkRequired, WriteOutcomeUnknown,
|
|
20
|
-
TransportError,
|
|
20
|
+
RegistrationFailed, TransportError,
|
|
21
21
|
)
|
|
22
22
|
|
|
23
|
-
__version__ = "1.
|
|
23
|
+
__version__ = "1.4.0.dev3"
|
|
24
24
|
|
|
25
25
|
__all__ = [
|
|
26
26
|
"Repo", "load_instances",
|
|
27
27
|
"DatasetSeries", "DatasetVersion", "DatasetInstance", "ManifestEntry",
|
|
28
|
-
"OwnerReference", "Feedback", "Identity", "
|
|
28
|
+
"OwnerReference", "Feedback", "Identity", "IngestReport",
|
|
29
|
+
"RegistrationReport", "PushResult", "PushManyReport",
|
|
29
30
|
"RolloutReport", "RolloutJob", "ScaffoldConfig",
|
|
30
31
|
"DatasetObjectDirectory", "DatasetObjectFile", "DatasetObjectListResult",
|
|
31
32
|
"canonical_json", "parse_feedbacks", "append_feedback",
|
|
@@ -33,6 +34,6 @@ __all__ = [
|
|
|
33
34
|
"InstanceRepoError", "LayoutError", "SchemaError", "OwnerRefMissing",
|
|
34
35
|
"ImageMissing", "Immutable", "DigestMismatch", "Forbidden", "NotVisible",
|
|
35
36
|
"TargetNotAllowed", "NetworkDenied", "CredExpired",
|
|
36
|
-
"PrivateNetworkRequired", "WriteOutcomeUnknown", "
|
|
37
|
-
"__version__",
|
|
37
|
+
"PrivateNetworkRequired", "WriteOutcomeUnknown", "RegistrationFailed",
|
|
38
|
+
"TransportError", "__version__",
|
|
38
39
|
]
|
|
@@ -19,9 +19,10 @@ import argparse
|
|
|
19
19
|
import json
|
|
20
20
|
import sys
|
|
21
21
|
|
|
22
|
+
from . import paths as P
|
|
22
23
|
from .concurrency import ItemError
|
|
23
24
|
from .errors import InstanceRepoError
|
|
24
|
-
from .models import
|
|
25
|
+
from .models import DatasetSeries, Feedback, OwnerReference, RolloutReport
|
|
25
26
|
from .repo import Repo
|
|
26
27
|
|
|
27
28
|
_STATUS_CHOICES = ["draft", "published", "deprecated", "archived"]
|
|
@@ -42,10 +43,6 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
42
43
|
ap.add_argument("--storage-env", dest="global_storage_env", default=None,
|
|
43
44
|
help="数据面存储环境(亦可用 IR_STORAGE_ENV);SDK 据此自动向 "
|
|
44
45
|
"apiserver 的 repo-config 拉取 OSS/ACR 寻址并缓存")
|
|
45
|
-
ap.add_argument("--metadata-model", default="split_first",
|
|
46
|
-
choices=["split_first", "version_first"],
|
|
47
|
-
help="元数据模型;缺省 split_first(split 必填、version 选填)。"
|
|
48
|
-
"旧数据集用 version_first(version 必填、split 选填)")
|
|
49
46
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
50
47
|
|
|
51
48
|
v = sub.add_parser("validate", help="本地校验 Instance 目录")
|
|
@@ -54,7 +51,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
54
51
|
p = sub.add_parser("push", help="上传 Instance(元数据先行)")
|
|
55
52
|
p.add_argument("dir")
|
|
56
53
|
p.add_argument("--dataset", required=True)
|
|
57
|
-
p.add_argument("--version",
|
|
54
|
+
p.add_argument("--version", default="",
|
|
55
|
+
help="版本号;split-first 可省略为空版本")
|
|
58
56
|
p.add_argument("--owner-ref", required=True, help="OwnerReference JSON")
|
|
59
57
|
p.add_argument("--split", default="default")
|
|
60
58
|
p.add_argument("--no-image", action="store_true")
|
|
@@ -70,31 +68,34 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
70
68
|
pm.add_argument("dirs", nargs="+",
|
|
71
69
|
help="实例路径,可给多个;整批父目录用通配展开,如 ./staging/*")
|
|
72
70
|
pm.add_argument("--dataset", required=True, help="{L1}/{L2}")
|
|
73
|
-
pm.add_argument("--version",
|
|
71
|
+
pm.add_argument("--version", default="",
|
|
72
|
+
help="版本号;split-first 可省略为空版本")
|
|
74
73
|
pm.add_argument("--owner-ref", required=True, help="OwnerReference JSON(整批共用)")
|
|
75
74
|
pm.add_argument("--split", default="default")
|
|
76
75
|
pm.add_argument("--no-image", action="store_true")
|
|
77
76
|
pm.add_argument("--overwrite", action="store_true")
|
|
78
77
|
pm.add_argument("--register", action="store_true",
|
|
79
|
-
help="
|
|
78
|
+
help="按成功实例ID分块ingest写库(缺省只传OSS,不写库)")
|
|
80
79
|
pm.add_argument("--no-verify", action="store_true")
|
|
81
80
|
pm.add_argument("--concurrency", type=int, default=8)
|
|
82
81
|
pm.add_argument("--continue-on-error", action="store_true",
|
|
83
|
-
help="
|
|
82
|
+
help="缺省任一失败即非零退出(已完成上传不回滚);置上则逐项汇报")
|
|
84
83
|
|
|
85
84
|
dl = sub.add_parser("deliver",
|
|
86
|
-
help="批量交付:并发上传 +
|
|
85
|
+
help="批量交付:并发上传 + 成功实例定向ingest")
|
|
87
86
|
dl.add_argument("dirs", nargs="+",
|
|
88
87
|
help="实例路径,可给多个;整批父目录用通配展开,如 ./staging/*")
|
|
89
88
|
dl.add_argument("--dataset", required=True, help="{L1}/{L2}")
|
|
90
|
-
dl.add_argument("--version",
|
|
89
|
+
dl.add_argument("--version", default="",
|
|
90
|
+
help="版本号;split-first 可省略为空版本")
|
|
91
91
|
dl.add_argument("--owner-ref", required=True, help="OwnerReference JSON(整批共用)")
|
|
92
92
|
dl.add_argument("--split", default="default")
|
|
93
93
|
dl.add_argument("--no-image", action="store_true")
|
|
94
94
|
dl.add_argument("--overwrite", action="store_true")
|
|
95
95
|
dl.add_argument("--concurrency", type=int, default=8)
|
|
96
96
|
dl.add_argument("--continue-on-error", action="store_true",
|
|
97
|
-
help="
|
|
97
|
+
help="缺省任一失败即非零退出(已完成上传不回滚);"
|
|
98
|
+
"置上则只注册成功子集并逐项汇报失败")
|
|
98
99
|
dl.add_argument("--no-ingest", action="store_true", help="只上传不入库")
|
|
99
100
|
dl.add_argument("--publish", action="store_true",
|
|
100
101
|
help="ingest 完成后把 version 置 published")
|
|
@@ -169,8 +170,8 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
169
170
|
iref = isub.add_parser("ref", help="按当前元数据模型算某实例的 ACR 引用")
|
|
170
171
|
iref.add_argument("--dataset", required=True, help="{L1}/{L2}")
|
|
171
172
|
iref.add_argument("--instance-id", required=True)
|
|
172
|
-
iref.add_argument("--split", default=None, help="
|
|
173
|
-
iref.add_argument("--version", default=None, help="
|
|
173
|
+
iref.add_argument("--split", default=None, help="split-first 正常业务下必填")
|
|
174
|
+
iref.add_argument("--version", default=None, help="可选版本号")
|
|
174
175
|
iref.add_argument("--acr-host", default=None, help="缺省取自动发现结果")
|
|
175
176
|
iref.add_argument("--acr-namespace", default=None, help="缺省取自动发现结果")
|
|
176
177
|
|
|
@@ -313,8 +314,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
313
314
|
help="split 级打标,格式 name=train,name2=eval(与 --run-type 互斥)")
|
|
314
315
|
ver.add_argument("--environment", default=None,
|
|
315
316
|
help="create/get/ingest 用:覆盖 profile 派生的 environment(pre/online)")
|
|
316
|
-
ver.add_argument("--
|
|
317
|
-
|
|
317
|
+
ver.add_argument("--instance-id", action="append", default=None,
|
|
318
|
+
dest="instance_ids",
|
|
319
|
+
help="ingest用:定向注册实例ID;可重复,也可逗号分隔")
|
|
318
320
|
ver.add_argument("--dry-run", action="store_true",
|
|
319
321
|
help="ingest 用:只预览扫描结果不落库")
|
|
320
322
|
ver.add_argument("--publish", action="store_true",
|
|
@@ -384,8 +386,6 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
384
386
|
dest="include_deprecated")
|
|
385
387
|
ds.add_argument("--environment", default=None,
|
|
386
388
|
help="get/update 覆盖 profile 派生的 environment")
|
|
387
|
-
ds.add_argument("--metadata-model", choices=["split_first", "version_first"],
|
|
388
|
-
default="split_first", dest="metadata_model")
|
|
389
389
|
# update 用(至少一项)
|
|
390
390
|
ds.add_argument("--display-name", default=None, dest="display_name")
|
|
391
391
|
ds.add_argument("--description", default=None)
|
|
@@ -487,16 +487,15 @@ def _resolve_split_ref(args) -> tuple[str, str]:
|
|
|
487
487
|
"--dataset/--split")
|
|
488
488
|
|
|
489
489
|
|
|
490
|
-
def _split_dataset(dataset: str) ->
|
|
491
|
-
"""校验 dataset
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
return segs
|
|
490
|
+
def _split_dataset(dataset: str) -> tuple[str, str]:
|
|
491
|
+
"""校验 ``--dataset`` 并返回模型回显所需的 ``(L1, L2)``。"""
|
|
492
|
+
try:
|
|
493
|
+
name = P.DatasetName.parse(dataset)
|
|
494
|
+
except InstanceRepoError as exc:
|
|
495
|
+
raise SystemExit(
|
|
496
|
+
f"--dataset must be one or two canonical segments, got "
|
|
497
|
+
f"{dataset!r}") from exc
|
|
498
|
+
return name.l1, name.l2
|
|
500
499
|
|
|
501
500
|
|
|
502
501
|
_TARGET_SHAPE = {
|
|
@@ -632,8 +631,7 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
632
631
|
cluster = _effective_cluster(args)
|
|
633
632
|
r = repo or Repo(profile=args.profile, api_env=getattr(args, "api_env", None),
|
|
634
633
|
api_base=getattr(args, "api_base", None), cluster=cluster,
|
|
635
|
-
storage_env=getattr(args, "global_storage_env", None)
|
|
636
|
-
metadata_model=getattr(args, "metadata_model", None))
|
|
634
|
+
storage_env=getattr(args, "global_storage_env", None))
|
|
637
635
|
|
|
638
636
|
if args.cmd == "validate":
|
|
639
637
|
rep = r.instances.validate(args.dir)
|
|
@@ -655,6 +653,9 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
655
653
|
print(f"ingest {res.ingest.get('ingest_id')} "
|
|
656
654
|
f"status={res.ingest.get('status')} "
|
|
657
655
|
f"instances={stats.get('instance_count')}", file=out)
|
|
656
|
+
if not args.no_register:
|
|
657
|
+
status = ((res.ingest or {}).get("status") or "").lower()
|
|
658
|
+
return 0 if status == "completed" else 1
|
|
658
659
|
return 0
|
|
659
660
|
|
|
660
661
|
if args.cmd == "push-many":
|
|
@@ -679,8 +680,14 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
679
680
|
else:
|
|
680
681
|
print(f"{res.instance_id}\tuploaded={res.uploaded}\t"
|
|
681
682
|
f"image_pushed={res.image_pushed}", file=out)
|
|
683
|
+
unfinished_registration = (
|
|
684
|
+
args.register and any(
|
|
685
|
+
not isinstance(res, Exception)
|
|
686
|
+
and ((res.ingest or {}).get("status") or "").lower()
|
|
687
|
+
!= "completed"
|
|
688
|
+
for res in results))
|
|
682
689
|
print(f"pushed={len(results) - failed} failed={failed}", file=out)
|
|
683
|
-
return 1 if failed else 0
|
|
690
|
+
return 1 if failed or unfinished_registration else 0
|
|
684
691
|
|
|
685
692
|
if args.cmd == "deliver":
|
|
686
693
|
if args.concurrency < 1:
|
|
@@ -699,10 +706,19 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
699
706
|
print(f"FAILED\t{getattr(e, 'item', '?')}\t{getattr(e, 'error', e)}",
|
|
700
707
|
file=out)
|
|
701
708
|
job = res.get("ingest") or {}
|
|
709
|
+
registration = res.get("registration")
|
|
710
|
+
pending = len(getattr(registration, "pending", []))
|
|
711
|
+
reg_failed = len(getattr(registration, "failed", []))
|
|
712
|
+
unknown = len(getattr(registration, "unknown", []))
|
|
713
|
+
not_submitted = len(getattr(registration, "not_submitted", []))
|
|
702
714
|
print(f"pushed={len(res['pushed'])} failed={len(res['failed'])} "
|
|
703
|
-
f"
|
|
704
|
-
|
|
705
|
-
|
|
715
|
+
f"skipped={len(res.get('skipped', []))} jobs={len(res.get('ingests', []))} "
|
|
716
|
+
f"ingest={job.get('ingest_id', '-')} status={job.get('status', '-')} "
|
|
717
|
+
f"pending={pending} registration_failed={reg_failed} "
|
|
718
|
+
f"unknown={unknown} not_submitted={not_submitted}", file=out)
|
|
719
|
+
unfinished = (res["failed"] or pending or reg_failed
|
|
720
|
+
or unknown or not_submitted)
|
|
721
|
+
return 1 if unfinished else 0
|
|
706
722
|
|
|
707
723
|
if args.cmd == "list":
|
|
708
724
|
dataset, version, _ = _resolve_target(args, 3)
|
|
@@ -948,8 +964,7 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
948
964
|
storage_path=args.storage_path, splits=splits,
|
|
949
965
|
status=args.status or "draft",
|
|
950
966
|
storage_type=args.storage_type,
|
|
951
|
-
environment=args.environment
|
|
952
|
-
metadata_model=args.metadata_model)
|
|
967
|
+
environment=args.environment)
|
|
953
968
|
print(f"created version {dv.dataset}/{dv.version} status={dv.status}",
|
|
954
969
|
file=out)
|
|
955
970
|
return 0
|
|
@@ -959,11 +974,59 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
959
974
|
dataset, _, _ = _resolve_target(args, 2)
|
|
960
975
|
splits = ([s.strip() for s in args.splits.split(",") if s.strip()]
|
|
961
976
|
if args.splits else None)
|
|
977
|
+
selected_ids = None
|
|
978
|
+
if args.instance_ids is not None:
|
|
979
|
+
unsupported = []
|
|
980
|
+
if args.dry_run:
|
|
981
|
+
unsupported.append("--dry-run")
|
|
982
|
+
if args.publish:
|
|
983
|
+
unsupported.append("--publish")
|
|
984
|
+
if args.allow_layout_inference:
|
|
985
|
+
unsupported.append("--allow-layout-inference")
|
|
986
|
+
if args.storage_path is not None:
|
|
987
|
+
unsupported.append("--storage-path")
|
|
988
|
+
if args.run_type is not None:
|
|
989
|
+
unsupported.append("--run-type")
|
|
990
|
+
if args.split_run_types is not None:
|
|
991
|
+
unsupported.append("--split-run-types")
|
|
992
|
+
if unsupported:
|
|
993
|
+
raise SystemExit(
|
|
994
|
+
"--instance-id selected ingest 不支持 "
|
|
995
|
+
+ ", ".join(unsupported))
|
|
996
|
+
selected_ids = sorted(set(
|
|
997
|
+
value.strip()
|
|
998
|
+
for item in args.instance_ids
|
|
999
|
+
for value in item.split(",") if value.strip()))
|
|
1000
|
+
if not selected_ids:
|
|
1001
|
+
raise SystemExit("--instance-id 至少需要一个非空ID")
|
|
1002
|
+
if splits:
|
|
1003
|
+
raise SystemExit(
|
|
1004
|
+
"--instance-id 只允许一个 --split,不能与 --splits 同用")
|
|
1005
|
+
registration = r.instances.ingest(
|
|
1006
|
+
dataset, args.version or None, selected_ids,
|
|
1007
|
+
split=args.split or "default",
|
|
1008
|
+
environment=args.environment,
|
|
1009
|
+
wait=not args.no_wait, poll_sec=args.poll_sec,
|
|
1010
|
+
timeout_sec=args.timeout_sec, cluster=args.cluster)
|
|
1011
|
+
print(
|
|
1012
|
+
f"jobs={len(registration.ingests)} "
|
|
1013
|
+
f"completed={len(registration.completed)} "
|
|
1014
|
+
f"pending={len(registration.pending)} "
|
|
1015
|
+
f"failed={len(registration.failed)} "
|
|
1016
|
+
f"unknown={len(registration.unknown)} "
|
|
1017
|
+
f"not_submitted={len(registration.not_submitted)}",
|
|
1018
|
+
file=out)
|
|
1019
|
+
unfinished = (registration.pending or registration.failed
|
|
1020
|
+
or registration.unknown
|
|
1021
|
+
or registration.not_submitted
|
|
1022
|
+
or len(registration.completed)
|
|
1023
|
+
!= len(registration.selected_ids))
|
|
1024
|
+
return 1 if unfinished else 0
|
|
1025
|
+
|
|
962
1026
|
job = r.versions.ingest(
|
|
963
1027
|
dataset, args.version or None,
|
|
964
1028
|
storage_path=args.storage_path, splits=splits,
|
|
965
|
-
split=args.split,
|
|
966
|
-
environment=args.environment,
|
|
1029
|
+
split=args.split, environment=args.environment,
|
|
967
1030
|
allow_layout_inference=args.allow_layout_inference,
|
|
968
1031
|
publish=args.publish, dry_run=args.dry_run,
|
|
969
1032
|
wait=not args.no_wait, poll_sec=args.poll_sec,
|
|
@@ -974,11 +1037,12 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
974
1037
|
f"instances={dv.get('instance_count') or 0}"
|
|
975
1038
|
f"\tsplits={len(dv.get('splits') or [])}", file=out)
|
|
976
1039
|
return 0
|
|
1040
|
+
status = (job.get("status") or "").lower()
|
|
977
1041
|
print(f"ingest {job.get('ingest_id', '')} "
|
|
978
1042
|
f"status={job.get('status', '')} "
|
|
979
1043
|
f"instances={job.get('stats', {}).get('instance_count') or 0}",
|
|
980
1044
|
file=out)
|
|
981
|
-
return 0
|
|
1045
|
+
return 0 if status == "completed" else 1
|
|
982
1046
|
dataset, version, _ = _resolve_target(args, 3)
|
|
983
1047
|
if args.action == "status":
|
|
984
1048
|
if args.split:
|
|
@@ -1086,7 +1150,6 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
1086
1150
|
sort_by=args.sort_by, order=args.order,
|
|
1087
1151
|
include_deprecated=args.include_deprecated or None,
|
|
1088
1152
|
environment=args.environment,
|
|
1089
|
-
metadata_model=args.metadata_model,
|
|
1090
1153
|
page=args.page, page_size=args.page_size)
|
|
1091
1154
|
for ds in res.items:
|
|
1092
1155
|
print(f"{ds.dataset_name}\tvisibility={ds.visibility}"
|
|
@@ -1100,8 +1163,7 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
1100
1163
|
if args.action in ("get", "update", "access", "revoke") and not dataset:
|
|
1101
1164
|
raise SystemExit(f"dataset {args.action} 需要 --dataset 或位置 ref")
|
|
1102
1165
|
if args.action == "get":
|
|
1103
|
-
d = r.datasets.get_detail(dataset, environment=args.environment
|
|
1104
|
-
metadata_model=args.metadata_model)
|
|
1166
|
+
d = r.datasets.get_detail(dataset, environment=args.environment)
|
|
1105
1167
|
src = "oss" if not d.registered else "mongodb"
|
|
1106
1168
|
print(f"dataset\t{d.dataset_name}\tid={d.dataset_id}"
|
|
1107
1169
|
f"\tvisibility={d.visibility}\t[{src}]", file=out)
|
|
@@ -148,6 +148,29 @@ def normalize_storage_env(storage_env: str | None) -> str:
|
|
|
148
148
|
return (storage_env or "").strip().lower()
|
|
149
149
|
|
|
150
150
|
|
|
151
|
+
def business_metadata_model(metadata_model: str | None) -> str:
|
|
152
|
+
"""Normalize a normal business entry to the only supported model.
|
|
153
|
+
|
|
154
|
+
``None``, the empty compatibility value, and ``split_first`` are accepted.
|
|
155
|
+
``version_first`` remains available only through low-level ``Layout`` and
|
|
156
|
+
legacy discovery helpers, never through Repo/client business I/O.
|
|
157
|
+
"""
|
|
158
|
+
from ..errors import SchemaError
|
|
159
|
+
|
|
160
|
+
if metadata_model is not None and not isinstance(metadata_model, str):
|
|
161
|
+
raise SchemaError("metadata_model must be a string or None")
|
|
162
|
+
requested = (metadata_model or "").strip()
|
|
163
|
+
if requested not in ("", "split_first"):
|
|
164
|
+
if requested == "version_first":
|
|
165
|
+
raise SchemaError(
|
|
166
|
+
"normal business entry is fixed to metadata_model=split_first; "
|
|
167
|
+
"version_first is retained only for low-level legacy reads")
|
|
168
|
+
raise SchemaError(
|
|
169
|
+
f"invalid metadata_model {requested!r}; normal business entry only "
|
|
170
|
+
"accepts split_first")
|
|
171
|
+
return "split_first"
|
|
172
|
+
|
|
173
|
+
|
|
151
174
|
def dataset_environment_for(storage_env: str | None) -> str:
|
|
152
175
|
"""由 storage_env 派生 ingest 的 ``environment``;无法派生时返回空串。
|
|
153
176
|
|
|
@@ -255,6 +278,7 @@ def load_profile(name: str = "default", *, overrides: dict | None = None,
|
|
|
255
278
|
关键字段(如 oss_bucket / acr_host)允许为空——仅在对应数据面操作真正用到时才校验,
|
|
256
279
|
纯控制面元数据操作(list/get/create/claim/whoami)不需要它们。
|
|
257
280
|
"""
|
|
281
|
+
requested_model = business_metadata_model(metadata_model)
|
|
258
282
|
base, env = resolve_api_base(api_env, api_base=api_base)
|
|
259
283
|
cluster_value = resolve_config_value(cluster, ENV_CLUSTER)
|
|
260
284
|
storage_value = resolve_config_value(
|
|
@@ -302,7 +326,7 @@ def load_profile(name: str = "default", *, overrides: dict | None = None,
|
|
|
302
326
|
return Profile(name=name, api_env=env,
|
|
303
327
|
cluster=str(cluster_value).strip(),
|
|
304
328
|
storage_env=normalize_storage_env(storage_value),
|
|
305
|
-
metadata_model=
|
|
329
|
+
metadata_model=requested_model,
|
|
306
330
|
prefer_public_registry=bool(prefer_public),
|
|
307
331
|
sources=sources, **cfg)
|
|
308
332
|
|
|
@@ -3,8 +3,8 @@ from __future__ import annotations
|
|
|
3
3
|
|
|
4
4
|
from urllib.parse import quote
|
|
5
5
|
|
|
6
|
+
from .config import Profile, business_metadata_model
|
|
6
7
|
from ..errors import SchemaError, TransportError
|
|
7
|
-
from ..layout import normalize_metadata_model
|
|
8
8
|
from ..models import (ClaimWorkspace, DatasetDetail, DatasetSeries,
|
|
9
9
|
PagedResult, RoleBinding, VISIBILITY_VALUES)
|
|
10
10
|
|
|
@@ -15,14 +15,15 @@ class DatasetsClient:
|
|
|
15
15
|
# 曾误写成 /apis/v1/datasets,该路径在 apiserver 上不存在,所有调用恒 404。
|
|
16
16
|
_BASE = "/apis/v1/datasets/series"
|
|
17
17
|
|
|
18
|
-
def __init__(self, transport, *, content_store=None,
|
|
18
|
+
def __init__(self, transport, *, profile: Profile | None = None, content_store=None,
|
|
19
19
|
oss_prefix: str = "swe/datasets",
|
|
20
20
|
metadata_model: str | None = None) -> None:
|
|
21
21
|
self._t = transport
|
|
22
|
+
self._p = profile
|
|
22
23
|
self._content = content_store
|
|
23
24
|
self._oss_prefix = oss_prefix
|
|
24
|
-
#
|
|
25
|
-
self._metadata_model = metadata_model
|
|
25
|
+
# 正常Dataset业务客户端固定split_first;legacy discovery仍可直接调用低层helper。
|
|
26
|
+
self._metadata_model = business_metadata_model(metadata_model)
|
|
26
27
|
|
|
27
28
|
def list(self, l1: str | None = None) -> list[DatasetSeries]:
|
|
28
29
|
"""列出 dataset(便利语义:自动翻页取全量,见 :meth:`list_paged` 单页对照)。
|
|
@@ -67,7 +68,7 @@ class DatasetsClient:
|
|
|
67
68
|
``updated_from/to`` RFC3339 闭区间;``sort_by`` updated_at(默认)|
|
|
68
69
|
dataset_name;``order`` desc(默认)|asc;``benchmark_id`` 精确(
|
|
69
70
|
``"none"`` 查未绑定);``environment`` online(默认)|pre;
|
|
70
|
-
``metadata_model`` split_first
|
|
71
|
+
``metadata_model`` 为兼容参数,正常业务只接受并显式发送 split_first。
|
|
71
72
|
|
|
72
73
|
items 为 :class:`~instance_repo.models.DatasetSeries`(含服务端返回的
|
|
73
74
|
version_count/split_count/latest_version/claimable 统计字段,见模块
|
|
@@ -113,7 +114,7 @@ class DatasetsClient:
|
|
|
113
114
|
"true" if include_deprecated else "false"))
|
|
114
115
|
if (environment or "").strip():
|
|
115
116
|
params.append(("environment", environment.strip()))
|
|
116
|
-
mm =
|
|
117
|
+
mm = business_metadata_model(metadata_model)
|
|
117
118
|
params.append(("metadata_model", mm))
|
|
118
119
|
params.append(("page", str(int(page))))
|
|
119
120
|
params.append(("page_size", str(int(page_size))))
|
|
@@ -125,15 +126,21 @@ class DatasetsClient:
|
|
|
125
126
|
items = [DatasetSeries(**_pick_series(r)) for r in rows if isinstance(r, dict)]
|
|
126
127
|
return PagedResult.from_response(items, pagination, int(page), int(page_size))
|
|
127
128
|
|
|
128
|
-
def get(self, name: str, *,
|
|
129
|
+
def get(self, name: str, *, environment: str | None = None,
|
|
130
|
+
metadata_model: str | None = None,
|
|
129
131
|
fallback_to_oss: bool = True) -> DatasetSeries:
|
|
132
|
+
mm = business_metadata_model(metadata_model)
|
|
130
133
|
try:
|
|
131
|
-
|
|
132
|
-
|
|
134
|
+
q = (f"dataset_name={quote(name, safe='')}"
|
|
135
|
+
f"&metadata_model={quote(mm, safe='')}")
|
|
136
|
+
if (environment or "").strip():
|
|
137
|
+
q += f"&environment={quote(environment.strip(), safe='')}"
|
|
138
|
+
data = self._t.get(f"{self._BASE}/detail?{q}")
|
|
133
139
|
return DatasetSeries(**_pick_series(data))
|
|
134
140
|
except TransportError as e:
|
|
135
141
|
if (e.biz_code == 92001 and fallback_to_oss
|
|
136
|
-
and self._content is not None
|
|
142
|
+
and self._content is not None
|
|
143
|
+
and self._discovery_env_match(environment)):
|
|
137
144
|
return self._discover_series(name)
|
|
138
145
|
raise
|
|
139
146
|
|
|
@@ -150,11 +157,11 @@ class DatasetsClient:
|
|
|
150
157
|
"""
|
|
151
158
|
if not (name or "").strip():
|
|
152
159
|
raise SchemaError("dataset name is required for get_detail")
|
|
160
|
+
mm = business_metadata_model(metadata_model)
|
|
153
161
|
try:
|
|
154
162
|
q = f"dataset_name={quote(name, safe='')}"
|
|
155
163
|
if (environment or "").strip():
|
|
156
164
|
q += f"&environment={quote(environment.strip(), safe='')}"
|
|
157
|
-
mm = normalize_metadata_model(metadata_model)
|
|
158
165
|
q += f"&metadata_model={quote(mm, safe='')}"
|
|
159
166
|
data = self._t.get(f"{self._BASE}/detail?{q}")
|
|
160
167
|
return DatasetDetail.from_dict(data) if isinstance(data, dict) else DatasetDetail()
|
|
@@ -8,9 +8,9 @@ v0.8 之前,镜像推送只作为「推实例」的内部副作用存在:没
|
|
|
8
8
|
* :meth:`copy` registry → registry(跨环境搬运,digest 固定)
|
|
9
9
|
* :meth:`exists` 存在性探测(带一致性重试)
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
命名不用调用方自己拼:正常业务固定 ``split_first``,:meth:`ref_for` 经
|
|
12
|
+
``DatasetName`` 解析后产出 ``{host}/{ns}/{sanitize(leaf)}:{split}-{id}``
|
|
13
|
+
(单段 L1 与两段 L1/L2 均支持)。旧布局仅由低层 ``Layout`` 读取存量数据。
|
|
14
14
|
|
|
15
15
|
网络策略沿用数据面统一口径:push / copy / exists 私网优先、连通性失败时受限回退公网;
|
|
16
16
|
**pull 归类为下载动作,强制私网**,公网回退被禁用,不可达即抛
|
|
@@ -21,8 +21,10 @@ from __future__ import annotations
|
|
|
21
21
|
from pathlib import Path
|
|
22
22
|
|
|
23
23
|
from .. import bootstrap
|
|
24
|
+
from .. import paths as P
|
|
24
25
|
from ..errors import SchemaError
|
|
25
26
|
from ..layout import Layout
|
|
27
|
+
from .config import business_metadata_model
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
class ImageClient:
|
|
@@ -47,12 +49,18 @@ class ImageClient:
|
|
|
47
49
|
``acr_host`` / ``namespace`` 缺省取当前 profile(含自动发现结果);显式传入可用于
|
|
48
50
|
计算**目标环境**的引用,配合 :meth:`copy` 做跨环境搬运。
|
|
49
51
|
"""
|
|
52
|
+
requested_model = metadata_model
|
|
53
|
+
if requested_model is None:
|
|
54
|
+
requested_model = self._p.metadata_model
|
|
55
|
+
model = business_metadata_model(requested_model)
|
|
50
56
|
host = acr_host or bootstrap.require(
|
|
51
57
|
self._p, "effective_acr_host", action="拼 ACR 镜像引用")
|
|
52
58
|
ns = namespace or bootstrap.require(
|
|
53
59
|
self._p, "acr_namespace", action="拼 ACR 镜像引用")
|
|
54
|
-
|
|
55
|
-
|
|
60
|
+
name = P.DatasetName.parse(dataset)
|
|
61
|
+
return self._layout(model).image_ref(
|
|
62
|
+
host, ns, name.canonical, instance_id,
|
|
63
|
+
version=version, split=split)
|
|
56
64
|
|
|
57
65
|
# ── 动作 ──
|
|
58
66
|
def push(self, tar_path, ref: str, *, overwrite: bool = False) -> str:
|