instance-repo 1.3.0.dev0__tar.gz → 1.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/PKG-INFO +7 -1
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/README.md +6 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/__init__.py +1 -1
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/_routing.py +4 -4
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/bootstrap.py +2 -1
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/cli.py +21 -9
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/config.py +37 -3
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/datasets.py +70 -15
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/images.py +1 -1
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/instances.py +133 -12
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/splits.py +116 -8
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/versions.py +102 -20
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/models.py +16 -6
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/repo.py +18 -6
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/acr.py +32 -17
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/oss.py +99 -8
- instance_repo-1.3.2/instance_repo/store/oss_discovery.py +690 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/transport.py +15 -7
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/PKG-INFO +7 -1
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/SOURCES.txt +1 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/pyproject.toml +1 -1
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/_site_defaults.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/artifacts.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/cache.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/__init__.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/benchmarks.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/reports.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/scaffold.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/workflows.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/concurrency.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/content.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/endpoints.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/errors.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/image.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/layout.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/loader.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/paths.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/release.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/retry.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/__init__.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/base.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/registry.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/tasktoml.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/validate.py +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/dependency_links.txt +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/entry_points.txt +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/requires.txt +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/top_level.txt +0 -0
- {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: instance-repo
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.2
|
|
4
4
|
Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -58,6 +58,12 @@ pip install "instance-repo[oss]" # 需要数据面 OSS 读写时,附带 os
|
|
|
58
58
|
`profile_overrides={"acr_namespace": ...}`;默认 namespace 由 repo-config 返回,但**必须由部署方
|
|
59
59
|
在 ACR 企业版实例中预建**,SDK 不会创建)。
|
|
60
60
|
|
|
61
|
+
**公网 ACR registry(runner 不在 VPC 内网时)**:默认使用 repo-config 返回的 `registry`
|
|
62
|
+
(VPC 内网域名)。runner 不在对应 VPC 内时,设环境变量 `IR_ACR_PREFER_PUBLIC=1`(或
|
|
63
|
+
`Repo(profile_overrides={"prefer_public_registry": True})`),SDK 会改走 `public_registry`
|
|
64
|
+
(公网域名);公网值缺失时静默回落到 VPC 值。也可用 `IR_ACR_PUBLIC_HOST` /
|
|
65
|
+
`profile_overrides={"acr_public_host": ...}` 显式指定公网 registry。
|
|
66
|
+
|
|
61
67
|
兼容别名表:
|
|
62
68
|
|
|
63
69
|
| 旧名 | 新名或状态 | 优先级 | 告警 |
|
|
@@ -44,6 +44,12 @@ pip install "instance-repo[oss]" # 需要数据面 OSS 读写时,附带 os
|
|
|
44
44
|
`profile_overrides={"acr_namespace": ...}`;默认 namespace 由 repo-config 返回,但**必须由部署方
|
|
45
45
|
在 ACR 企业版实例中预建**,SDK 不会创建)。
|
|
46
46
|
|
|
47
|
+
**公网 ACR registry(runner 不在 VPC 内网时)**:默认使用 repo-config 返回的 `registry`
|
|
48
|
+
(VPC 内网域名)。runner 不在对应 VPC 内时,设环境变量 `IR_ACR_PREFER_PUBLIC=1`(或
|
|
49
|
+
`Repo(profile_overrides={"prefer_public_registry": True})`),SDK 会改走 `public_registry`
|
|
50
|
+
(公网域名);公网值缺失时静默回落到 VPC 值。也可用 `IR_ACR_PUBLIC_HOST` /
|
|
51
|
+
`profile_overrides={"acr_public_host": ...}` 显式指定公网 registry。
|
|
52
|
+
|
|
47
53
|
兼容别名表:
|
|
48
54
|
|
|
49
55
|
| 旧名 | 新名或状态 | 优先级 | 告警 |
|
|
@@ -45,14 +45,14 @@ def _split_scheme(endpoint: str) -> tuple[str, str]:
|
|
|
45
45
|
return "", endpoint
|
|
46
46
|
|
|
47
47
|
|
|
48
|
-
def _probe_private_endpoint(endpoint: str, timeout: float =
|
|
48
|
+
def _probe_private_endpoint(endpoint: str, timeout: float = 0.5) -> bool:
|
|
49
49
|
"""Quick TCP connect probe to decide whether a private endpoint is reachable.
|
|
50
50
|
|
|
51
51
|
Performs a raw TCP connect (no TLS handshake) to the endpoint host/port.
|
|
52
52
|
Returns True if the connect succeeds, False on any error. The default
|
|
53
|
-
timeout is
|
|
54
|
-
|
|
55
|
-
|
|
53
|
+
timeout is 0.5 seconds — 内网 TCP 握手通常在几十毫秒内完成,0.5s 足够
|
|
54
|
+
判定可达性,同时把公网环境下私网不可达的探测开销从 3s 压到 0.5s
|
|
55
|
+
(结果按 store 缓存 5 分钟,仅端点变更或缓存过期才重探)。
|
|
56
56
|
"""
|
|
57
57
|
_, host = _split_scheme(endpoint)
|
|
58
58
|
# Determine port: default 443 for https, 80 for http.
|
|
@@ -36,7 +36,8 @@ _logger = logging.getLogger(__name__)
|
|
|
36
36
|
DISCOVERY_TTL_SECONDS = 3600.0
|
|
37
37
|
|
|
38
38
|
#: 由自动发现填充的字段(scaffold 两项不在 repo-config 返回范围内,仍须显式提供)。
|
|
39
|
-
DISCOVERABLE_FIELDS = ("oss_bucket", "oss_prefix", "acr_host",
|
|
39
|
+
DISCOVERABLE_FIELDS = ("oss_bucket", "oss_prefix", "acr_host",
|
|
40
|
+
"acr_public_host", "acr_namespace")
|
|
40
41
|
|
|
41
42
|
_cache = TTLCache(safety=0.0)
|
|
42
43
|
_lock = threading.Lock()
|
|
@@ -42,7 +42,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
42
42
|
ap.add_argument("--storage-env", dest="global_storage_env", default=None,
|
|
43
43
|
help="数据面存储环境(亦可用 IR_STORAGE_ENV);SDK 据此自动向 "
|
|
44
44
|
"apiserver 的 repo-config 拉取 OSS/ACR 寻址并缓存")
|
|
45
|
-
ap.add_argument("--metadata-model", default=
|
|
45
|
+
ap.add_argument("--metadata-model", default="split_first",
|
|
46
46
|
choices=["split_first", "version_first"],
|
|
47
47
|
help="元数据模型;缺省 split_first(split 必填、version 选填)。"
|
|
48
48
|
"旧数据集用 version_first(version 必填、split 选填)")
|
|
@@ -313,7 +313,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
313
313
|
help="split 级打标,格式 name=train,name2=eval(与 --run-type 互斥)")
|
|
314
314
|
ver.add_argument("--environment", default=None,
|
|
315
315
|
help="create/get/ingest 用:覆盖 profile 派生的 environment(pre/online)")
|
|
316
|
-
ver.add_argument("--metadata-model", default=
|
|
316
|
+
ver.add_argument("--metadata-model", default="split_first",
|
|
317
317
|
help="create/ingest 用:显式指定元数据模型(split_first/version_first)")
|
|
318
318
|
ver.add_argument("--dry-run", action="store_true",
|
|
319
319
|
help="ingest 用:只预览扫描结果不落库")
|
|
@@ -385,7 +385,7 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
385
385
|
ds.add_argument("--environment", default=None,
|
|
386
386
|
help="get/update 覆盖 profile 派生的 environment")
|
|
387
387
|
ds.add_argument("--metadata-model", choices=["split_first", "version_first"],
|
|
388
|
-
default=
|
|
388
|
+
default="split_first", dest="metadata_model")
|
|
389
389
|
# update 用(至少一项)
|
|
390
390
|
ds.add_argument("--display-name", default=None, dest="display_name")
|
|
391
391
|
ds.add_argument("--description", default=None)
|
|
@@ -724,13 +724,16 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
724
724
|
tags=tags_dict, difficulty=args.difficulty,
|
|
725
725
|
page=args.page, page_size=args.page_size)
|
|
726
726
|
for inst in res.items:
|
|
727
|
-
|
|
727
|
+
src = "\t[oss]" if not inst.registered else ""
|
|
728
|
+
print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}{src}",
|
|
728
729
|
file=out)
|
|
729
730
|
print(f"page={res.page}/{(res.total+res.page_size-1)//res.page_size if res.total>0 else 1} "
|
|
730
731
|
f"total={res.total} has_more={res.has_more}", file=out)
|
|
731
732
|
return 0
|
|
732
733
|
for inst in r.instances.list(dataset, version, args.split):
|
|
733
|
-
|
|
734
|
+
src = "\t[oss]" if not inst.registered else ""
|
|
735
|
+
print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}{src}",
|
|
736
|
+
file=out)
|
|
734
737
|
return 0
|
|
735
738
|
|
|
736
739
|
if args.cmd == "pull":
|
|
@@ -910,7 +913,9 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
910
913
|
# 实例不存在时库层返回 None(apiserver 92003),显式区分"查不到"与"字段为空"。
|
|
911
914
|
if inst is None:
|
|
912
915
|
raise SystemExit(f"instance {dataset}/{version}/{iid} not found")
|
|
913
|
-
|
|
916
|
+
src = "\t[oss]" if not inst.registered else ""
|
|
917
|
+
print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}{src}",
|
|
918
|
+
file=out)
|
|
914
919
|
return 0
|
|
915
920
|
|
|
916
921
|
if args.cmd == "version":
|
|
@@ -931,7 +936,9 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
931
936
|
dataset, _, _ = _resolve_target(args, 2)
|
|
932
937
|
if args.action == "list":
|
|
933
938
|
for v in r.versions.list(dataset):
|
|
934
|
-
|
|
939
|
+
src = "\t[oss]" if not v.registered else ""
|
|
940
|
+
print(f"{v.version}\t{v.status}\t{v.storage_path}{src}",
|
|
941
|
+
file=out)
|
|
935
942
|
return 0
|
|
936
943
|
if not args.version:
|
|
937
944
|
raise SystemExit("version create 需要 --version")
|
|
@@ -1026,7 +1033,8 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
1026
1033
|
status=args.status,
|
|
1027
1034
|
page=args.page, page_size=args.page_size)
|
|
1028
1035
|
for s in res.items:
|
|
1029
|
-
src = "projected" if s.projected else
|
|
1036
|
+
src = "projected" if s.projected else (
|
|
1037
|
+
"oss" if not s.registered else "record")
|
|
1030
1038
|
print(f"{s.name}\tversion={s.version or ''}\tstatus={s.status}"
|
|
1031
1039
|
f"\trun_type={s.run_type or ''}\t"
|
|
1032
1040
|
f"instance_count={_split_instance_count(s)}\t[{src}]",
|
|
@@ -1094,13 +1102,17 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
1094
1102
|
if args.action == "get":
|
|
1095
1103
|
d = r.datasets.get_detail(dataset, environment=args.environment,
|
|
1096
1104
|
metadata_model=args.metadata_model)
|
|
1105
|
+
src = "oss" if not d.registered else "mongodb"
|
|
1097
1106
|
print(f"dataset\t{d.dataset_name}\tid={d.dataset_id}"
|
|
1098
|
-
f"\tvisibility={d.visibility}", file=out)
|
|
1107
|
+
f"\tvisibility={d.visibility}\t[{src}]", file=out)
|
|
1099
1108
|
print(f"version_count={d.version_count}\tsplit_count={d.split_count}"
|
|
1100
1109
|
f"\tlatest={d.latest_version or ''}", file=out)
|
|
1101
1110
|
print(f"instance_count={d.stats.instance_count}"
|
|
1102
1111
|
f"\tfile_count={d.stats.file_count}"
|
|
1103
1112
|
f"\tsize_bytes={d.stats.size_bytes}", file=out)
|
|
1113
|
+
if not d.registered:
|
|
1114
|
+
print("warning: dataset not registered in MongoDB, "
|
|
1115
|
+
"data from OSS discovery", file=sys.stderr)
|
|
1104
1116
|
return 0
|
|
1105
1117
|
if args.action == "update":
|
|
1106
1118
|
kwargs = {}
|
|
@@ -76,6 +76,11 @@ ENV_OSS_PREFIX = "IR_OSS_PREFIX"
|
|
|
76
76
|
ENV_SCAFFOLD_BUCKET = "IR_SCAFFOLD_BUCKET"
|
|
77
77
|
ENV_SCAFFOLD_ROOT = "IR_SCAFFOLD_ROOT"
|
|
78
78
|
|
|
79
|
+
# 公网 ACR registry:runner 不在 VPC 内网时,可显式切换使用公网 registry。
|
|
80
|
+
ENV_ACR_PUBLIC_HOST = "IR_ACR_PUBLIC_HOST"
|
|
81
|
+
# 是否优先使用公网 registry("1"/"true"/"on",大小写不敏感)。
|
|
82
|
+
ENV_ACR_PREFER_PUBLIC = "IR_ACR_PREFER_PUBLIC"
|
|
83
|
+
|
|
79
84
|
#: v0.8 弃用的手工寻址环境变量 → 建议替代。scaffold 两项不在其中:scaffold 包不在
|
|
80
85
|
#: repo-config 的返回范围内,仍须显式提供。
|
|
81
86
|
DEPRECATED_ADDRESS_ENVS = {
|
|
@@ -179,6 +184,7 @@ class Profile:
|
|
|
179
184
|
name: str
|
|
180
185
|
oss_bucket: str = ""
|
|
181
186
|
acr_host: str = ""
|
|
187
|
+
acr_public_host: str = ""
|
|
182
188
|
acr_namespace: str = ""
|
|
183
189
|
oss_prefix: str = DEFAULT_OSS_PREFIX
|
|
184
190
|
scaffold_bucket: str = ""
|
|
@@ -197,6 +203,21 @@ class Profile:
|
|
|
197
203
|
#: ``bootstrap.discover()`` 捕获异常时写入;``require()`` 据此在报错消息里
|
|
198
204
|
#: 透传根因,避免"返回里没有 acr_host"这类误导性口径。
|
|
199
205
|
discovery_error: str = ""
|
|
206
|
+
#: 优先使用公网 ACR registry(默认 False=走 VPC 内网)。runner 不在 VPC
|
|
207
|
+
#: 内网时置 True,配合 ``acr_public_host``(repo-config 的 public_registry)
|
|
208
|
+
#: 生效;公网值缺失时静默回落到 ``acr_host``。
|
|
209
|
+
prefer_public_registry: bool = False
|
|
210
|
+
|
|
211
|
+
@property
|
|
212
|
+
def effective_acr_host(self) -> str:
|
|
213
|
+
"""ACR 操作实际使用的 registry host。
|
|
214
|
+
|
|
215
|
+
默认返回 ``acr_host``(repo-config 的 registry,通常是 VPC 内网域名);
|
|
216
|
+
``prefer_public_registry=True`` 且 ``acr_public_host`` 非空时返回公网域名。
|
|
217
|
+
"""
|
|
218
|
+
if self.prefer_public_registry and self.acr_public_host:
|
|
219
|
+
return self.acr_public_host
|
|
220
|
+
return self.acr_host
|
|
200
221
|
|
|
201
222
|
@property
|
|
202
223
|
def dataset_environment(self) -> str:
|
|
@@ -240,12 +261,14 @@ def load_profile(name: str = "default", *, overrides: dict | None = None,
|
|
|
240
261
|
storage_env, ENV_STORAGE_ENV, LEGACY_ENV_STORAGE_ENV)
|
|
241
262
|
env_values = {k: os.environ.get(k, "") for k in
|
|
242
263
|
(ENV_OSS_BUCKET, ENV_ACR_HOST, ENV_ACR_NAMESPACE,
|
|
264
|
+
ENV_ACR_PUBLIC_HOST,
|
|
243
265
|
ENV_OSS_PREFIX, ENV_SCAFFOLD_BUCKET, ENV_SCAFFOLD_ROOT)}
|
|
244
266
|
_warn_deprecated_envs(env_values)
|
|
245
267
|
cfg = {
|
|
246
268
|
"oss_bucket": env_values[ENV_OSS_BUCKET],
|
|
247
269
|
"acr_host": env_values[ENV_ACR_HOST],
|
|
248
270
|
"acr_namespace": env_values[ENV_ACR_NAMESPACE],
|
|
271
|
+
"acr_public_host": env_values[ENV_ACR_PUBLIC_HOST],
|
|
249
272
|
"oss_prefix": env_values[ENV_OSS_PREFIX] or DEFAULT_OSS_PREFIX,
|
|
250
273
|
"scaffold_bucket": env_values[ENV_SCAFFOLD_BUCKET],
|
|
251
274
|
"scaffold_root": env_values[ENV_SCAFFOLD_ROOT],
|
|
@@ -259,19 +282,28 @@ def load_profile(name: str = "default", *, overrides: dict | None = None,
|
|
|
259
282
|
sources["api_env"] = "env"
|
|
260
283
|
else:
|
|
261
284
|
sources["api_env"] = ""
|
|
285
|
+
prefer_public = False
|
|
262
286
|
if overrides:
|
|
263
|
-
# 只接受已知字段,避免把 api_env
|
|
264
|
-
|
|
265
|
-
|
|
287
|
+
# 只接受已知字段,避免把 api_env 之类误当数据面字段;
|
|
288
|
+
# prefer_public_registry 单独取(非寻址字段,不进 **cfg 传 Profile)。
|
|
289
|
+
for k in ("oss_bucket", "acr_host", "acr_namespace", "acr_public_host",
|
|
290
|
+
"oss_prefix", "scaffold_bucket", "scaffold_root", "api_base"):
|
|
266
291
|
if k in overrides and overrides[k] is not None:
|
|
267
292
|
cfg[k] = overrides[k]
|
|
268
293
|
sources[k] = "explicit"
|
|
294
|
+
if overrides.get("prefer_public_registry") is not None:
|
|
295
|
+
prefer_public = bool(overrides["prefer_public_registry"])
|
|
269
296
|
if cfg["api_base"]:
|
|
270
297
|
cfg["api_base"] = str(cfg["api_base"]).rstrip("/")
|
|
298
|
+
if not prefer_public:
|
|
299
|
+
# 环境变量开关:IR_ACR_PREFER_PUBLIC=1/true/on(大小写不敏感)
|
|
300
|
+
prefer_public = os.environ.get(ENV_ACR_PREFER_PUBLIC, "").strip().lower() \
|
|
301
|
+
in ("1", "true", "on")
|
|
271
302
|
return Profile(name=name, api_env=env,
|
|
272
303
|
cluster=str(cluster_value).strip(),
|
|
273
304
|
storage_env=normalize_storage_env(storage_value),
|
|
274
305
|
metadata_model=(metadata_model or "").strip(),
|
|
306
|
+
prefer_public_registry=bool(prefer_public),
|
|
275
307
|
sources=sources, **cfg)
|
|
276
308
|
|
|
277
309
|
|
|
@@ -323,6 +355,8 @@ def fetch_repo_config(transport, *, cluster: str = "", storage_env: str = "") ->
|
|
|
323
355
|
if acr := data.get("acr"):
|
|
324
356
|
if acr.get("registry"):
|
|
325
357
|
result["acr_host"] = acr["registry"]
|
|
358
|
+
if acr.get("public_registry"):
|
|
359
|
+
result["acr_public_host"] = acr["public_registry"]
|
|
326
360
|
if acr.get("namespace"):
|
|
327
361
|
result["acr_namespace"] = acr["namespace"]
|
|
328
362
|
# api_env 由 apiserver 在 repo-config 响应中下发(omitempty,旧版不返回)。
|
|
@@ -4,6 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
from urllib.parse import quote
|
|
5
5
|
|
|
6
6
|
from ..errors import SchemaError, TransportError
|
|
7
|
+
from ..layout import normalize_metadata_model
|
|
7
8
|
from ..models import (ClaimWorkspace, DatasetDetail, DatasetSeries,
|
|
8
9
|
PagedResult, RoleBinding, VISIBILITY_VALUES)
|
|
9
10
|
|
|
@@ -14,8 +15,14 @@ class DatasetsClient:
|
|
|
14
15
|
# 曾误写成 /apis/v1/datasets,该路径在 apiserver 上不存在,所有调用恒 404。
|
|
15
16
|
_BASE = "/apis/v1/datasets/series"
|
|
16
17
|
|
|
17
|
-
def __init__(self, transport
|
|
18
|
+
def __init__(self, transport, *, content_store=None,
|
|
19
|
+
oss_prefix: str = "swe/datasets",
|
|
20
|
+
metadata_model: str | None = None) -> None:
|
|
18
21
|
self._t = transport
|
|
22
|
+
self._content = content_store
|
|
23
|
+
self._oss_prefix = oss_prefix
|
|
24
|
+
# OSS 回退探测的布局语义(split_first 默认;version_first 显式切换)
|
|
25
|
+
self._metadata_model = metadata_model or "split_first"
|
|
19
26
|
|
|
20
27
|
def list(self, l1: str | None = None) -> list[DatasetSeries]:
|
|
21
28
|
"""列出 dataset(便利语义:自动翻页取全量,见 :meth:`list_paged` 单页对照)。
|
|
@@ -60,7 +67,7 @@ class DatasetsClient:
|
|
|
60
67
|
``updated_from/to`` RFC3339 闭区间;``sort_by`` updated_at(默认)|
|
|
61
68
|
dataset_name;``order`` desc(默认)|asc;``benchmark_id`` 精确(
|
|
62
69
|
``"none"`` 查未绑定);``environment`` online(默认)|pre;
|
|
63
|
-
``metadata_model`` version_first
|
|
70
|
+
``metadata_model`` split_first(默认)|version_first(决定统计来源)。
|
|
64
71
|
|
|
65
72
|
items 为 :class:`~instance_repo.models.DatasetSeries`(含服务端返回的
|
|
66
73
|
version_count/split_count/latest_version/claimable 统计字段,见模块
|
|
@@ -106,8 +113,8 @@ class DatasetsClient:
|
|
|
106
113
|
"true" if include_deprecated else "false"))
|
|
107
114
|
if (environment or "").strip():
|
|
108
115
|
params.append(("environment", environment.strip()))
|
|
109
|
-
|
|
110
|
-
|
|
116
|
+
mm = normalize_metadata_model(metadata_model)
|
|
117
|
+
params.append(("metadata_model", mm))
|
|
111
118
|
params.append(("page", str(int(page))))
|
|
112
119
|
params.append(("page_size", str(int(page_size))))
|
|
113
120
|
query = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
|
|
@@ -118,12 +125,21 @@ class DatasetsClient:
|
|
|
118
125
|
items = [DatasetSeries(**_pick_series(r)) for r in rows if isinstance(r, dict)]
|
|
119
126
|
return PagedResult.from_response(items, pagination, int(page), int(page_size))
|
|
120
127
|
|
|
121
|
-
def get(self, name: str
|
|
122
|
-
|
|
123
|
-
|
|
128
|
+
def get(self, name: str, *,
|
|
129
|
+
fallback_to_oss: bool = True) -> DatasetSeries:
|
|
130
|
+
try:
|
|
131
|
+
data = self._t.get(
|
|
132
|
+
f"{self._BASE}/detail?dataset_name={quote(name, safe='')}")
|
|
133
|
+
return DatasetSeries(**_pick_series(data))
|
|
134
|
+
except TransportError as e:
|
|
135
|
+
if (e.biz_code == 92001 and fallback_to_oss
|
|
136
|
+
and self._content is not None):
|
|
137
|
+
return self._discover_series(name)
|
|
138
|
+
raise
|
|
124
139
|
|
|
125
140
|
def get_detail(self, name: str, *, environment: str | None = None,
|
|
126
|
-
metadata_model: str | None = None
|
|
141
|
+
metadata_model: str | None = None,
|
|
142
|
+
fallback_to_oss: bool = True) -> DatasetDetail:
|
|
127
143
|
"""dataset 富详情(``GET /datasets/series/detail`` 完整响应)。
|
|
128
144
|
|
|
129
145
|
相比 :meth:`get` 的轻量 series 字段:新增 version_count / split_count /
|
|
@@ -134,13 +150,20 @@ class DatasetsClient:
|
|
|
134
150
|
"""
|
|
135
151
|
if not (name or "").strip():
|
|
136
152
|
raise SchemaError("dataset name is required for get_detail")
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
153
|
+
try:
|
|
154
|
+
q = f"dataset_name={quote(name, safe='')}"
|
|
155
|
+
if (environment or "").strip():
|
|
156
|
+
q += f"&environment={quote(environment.strip(), safe='')}"
|
|
157
|
+
mm = normalize_metadata_model(metadata_model)
|
|
158
|
+
q += f"&metadata_model={quote(mm, safe='')}"
|
|
159
|
+
data = self._t.get(f"{self._BASE}/detail?{q}")
|
|
160
|
+
return DatasetDetail.from_dict(data) if isinstance(data, dict) else DatasetDetail()
|
|
161
|
+
except TransportError as e:
|
|
162
|
+
if (e.biz_code == 92001 and fallback_to_oss
|
|
163
|
+
and self._content is not None):
|
|
164
|
+
return self._discover_detail(name, storage_env=environment,
|
|
165
|
+
metadata_model=metadata_model)
|
|
166
|
+
raise
|
|
144
167
|
|
|
145
168
|
def update(self, name: str, *, display_name: str | None = None,
|
|
146
169
|
visibility: str | None = None, owner_team_id: str | None = None,
|
|
@@ -371,6 +394,38 @@ class DatasetsClient:
|
|
|
371
394
|
return self._t.post(
|
|
372
395
|
f"{self._BASE}/{quote(dataset_id, safe='')}/claim", body)
|
|
373
396
|
|
|
397
|
+
# ── OSS fallback discovery (private) ─────────────────────────────────
|
|
398
|
+
|
|
399
|
+
def _discover_detail(self, name: str, *,
|
|
400
|
+
storage_env: str | None = None,
|
|
401
|
+
metadata_model: str | None = None) -> DatasetDetail:
|
|
402
|
+
from ..store.oss_discovery import (
|
|
403
|
+
parse_dataset_level, build_synthetic_dataset_detail,
|
|
404
|
+
_warn_if_truncated)
|
|
405
|
+
info = parse_dataset_level(
|
|
406
|
+
self._content, name, oss_prefix=self._oss_prefix,
|
|
407
|
+
metadata_model=metadata_model or self._metadata_model)
|
|
408
|
+
_warn_if_truncated(info, name, 500)
|
|
409
|
+
if not info.versions and not info.unversioned_splits:
|
|
410
|
+
from ..errors import DatasetNotFound
|
|
411
|
+
raise DatasetNotFound(f"dataset {name!r} not found in OSS")
|
|
412
|
+
return build_synthetic_dataset_detail(
|
|
413
|
+
name, info, storage_env=storage_env)
|
|
414
|
+
|
|
415
|
+
def _discover_series(self, name: str) -> DatasetSeries:
|
|
416
|
+
detail = self._discover_detail(name)
|
|
417
|
+
return DatasetSeries(
|
|
418
|
+
dataset_name=detail.dataset_name,
|
|
419
|
+
dataset_l1=detail.dataset_l1,
|
|
420
|
+
dataset_l2=detail.dataset_l2,
|
|
421
|
+
dataset_id=detail.dataset_id or None,
|
|
422
|
+
version_count=detail.version_count,
|
|
423
|
+
split_count=detail.split_count,
|
|
424
|
+
latest_version=detail.latest_version,
|
|
425
|
+
available_envs=detail.available_envs,
|
|
426
|
+
registered=False,
|
|
427
|
+
)
|
|
428
|
+
|
|
374
429
|
|
|
375
430
|
# 与 apiserver `DatasetListItem`(pkg/server/dataset_service.go:231)逐字段对齐。
|
|
376
431
|
_SERIES_FIELDS = ("dataset_name", "dataset_l1", "dataset_l2", "dataset_id",
|
|
@@ -48,7 +48,7 @@ class ImageClient:
|
|
|
48
48
|
计算**目标环境**的引用,配合 :meth:`copy` 做跨环境搬运。
|
|
49
49
|
"""
|
|
50
50
|
host = acr_host or bootstrap.require(
|
|
51
|
-
self._p, "
|
|
51
|
+
self._p, "effective_acr_host", action="拼 ACR 镜像引用")
|
|
52
52
|
ns = namespace or bootstrap.require(
|
|
53
53
|
self._p, "acr_namespace", action="拼 ACR 镜像引用")
|
|
54
54
|
return self._layout(metadata_model).image_ref(
|
|
@@ -70,6 +70,8 @@ class InstancesClient:
|
|
|
70
70
|
# 收尾改为 ingest)。ingester 形如 (dataset, version) -> ingest job dict;
|
|
71
71
|
# 由 Repo 注入 versions.ingest。直接构造未注入时 register=True 会报错。
|
|
72
72
|
self._ingester = ingester
|
|
73
|
+
# OSS 回退发现使用的 bucket 内根前缀(profile 可覆盖,缺省 swe/datasets)。
|
|
74
|
+
self._oss_prefix = getattr(profile, 'oss_prefix', 'swe/datasets')
|
|
73
75
|
|
|
74
76
|
def _check_split_not_locked(self, dataset: str, version: str,
|
|
75
77
|
split: str) -> None:
|
|
@@ -155,24 +157,55 @@ class InstancesClient:
|
|
|
155
157
|
def get(self, dataset: str, version: str, instance_id: str,
|
|
156
158
|
split: str | None = None, *,
|
|
157
159
|
environment: str | None = None,
|
|
158
|
-
metadata_model: str | None = None
|
|
160
|
+
metadata_model: str | None = None,
|
|
161
|
+
fallback_to_oss: bool = True) -> DatasetInstance | None:
|
|
159
162
|
"""取单个实例;不存在返回 None。
|
|
160
163
|
|
|
161
164
|
``environment`` / ``metadata_model`` 传 None 用 profile 派生的值(默认行为
|
|
162
165
|
不变);显式传值可覆盖本次调用——CLI 需要读 online 集合时不必重建 Repo。
|
|
166
|
+
|
|
167
|
+
**OSS 双触发回退**(设计文档 §4.4.4):
|
|
168
|
+
a. ``TransportError(biz_code=92001)``(dataset 不存在)→ OSS 回退。
|
|
169
|
+
b. ``result is None``(registry 吞了 92003 instance 不存在)→ OSS 回退。
|
|
163
170
|
"""
|
|
164
171
|
# 只在调用方显式指定时才下发覆盖参数:注册表是可注入实现,
|
|
165
172
|
# 保持缺省调用形态与历史一致,避免破坏既有实现。
|
|
166
173
|
overrides = _registry_overrides(environment, metadata_model)
|
|
167
|
-
|
|
174
|
+
try:
|
|
175
|
+
result = self._reg.get(
|
|
176
|
+
dataset, version, instance_id, split, **overrides)
|
|
177
|
+
except TransportError as e:
|
|
178
|
+
# 路径 1: dataset 不存在(92001) → OSS 回退
|
|
179
|
+
if (e.biz_code == 92001
|
|
180
|
+
and fallback_to_oss and self._content is not None):
|
|
181
|
+
return self._discover_instance(
|
|
182
|
+
dataset, version, instance_id, split)
|
|
183
|
+
raise
|
|
184
|
+
# 路径 2: instance 不存在(None, registry 吞了 92003) → OSS 回退
|
|
185
|
+
if result is None and fallback_to_oss and self._content is not None:
|
|
186
|
+
return self._discover_instance(
|
|
187
|
+
dataset, version, instance_id, split)
|
|
188
|
+
return result
|
|
168
189
|
|
|
169
190
|
def list(self, dataset: str, version: str,
|
|
170
191
|
split: str | None = None, *,
|
|
171
192
|
environment: str | None = None,
|
|
172
|
-
metadata_model: str | None = None
|
|
173
|
-
|
|
193
|
+
metadata_model: str | None = None,
|
|
194
|
+
fallback_to_oss: bool = True) -> list[DatasetInstance]:
|
|
195
|
+
"""自动翻页取全量;``environment`` / ``metadata_model`` 语义同 :meth:`get`。
|
|
196
|
+
|
|
197
|
+
**OSS 回退**:当 ``TransportError(biz_code=92001)``(dataset 不存在)时,
|
|
198
|
+
若 ``fallback_to_oss=True`` 且 ``content_store`` 已注入,自动回退到 OSS
|
|
199
|
+
存储层列举实例。
|
|
200
|
+
"""
|
|
174
201
|
overrides = _registry_overrides(environment, metadata_model)
|
|
175
|
-
|
|
202
|
+
try:
|
|
203
|
+
return self._reg.query(dataset, version, split, **overrides)
|
|
204
|
+
except TransportError as e:
|
|
205
|
+
if (e.biz_code == 92001
|
|
206
|
+
and fallback_to_oss and self._content is not None):
|
|
207
|
+
return self._discover_instances(dataset, version, split)
|
|
208
|
+
raise
|
|
176
209
|
|
|
177
210
|
def list_paged(self, dataset: str, version: str = "", *,
|
|
178
211
|
split: str | None = None, q: str | None = None,
|
|
@@ -181,7 +214,8 @@ class InstancesClient:
|
|
|
181
214
|
difficulty: str | None = None,
|
|
182
215
|
page: int = 1, page_size: int = 50,
|
|
183
216
|
environment: str | None = None,
|
|
184
|
-
metadata_model: str | None = None
|
|
217
|
+
metadata_model: str | None = None,
|
|
218
|
+
fallback_to_oss: bool = True) -> "PagedResult":
|
|
185
219
|
"""instance 分页列表(单页原始语义,``GET /datasets/instances``)。
|
|
186
220
|
|
|
187
221
|
与 :meth:`list` 的自动翻页取全量互补:精确控制页码/过滤/搜索。
|
|
@@ -190,6 +224,9 @@ class InstancesClient:
|
|
|
190
224
|
CSV("k1:v1,k2:v2",key 含 ``.``/``$`` 服务端拒绝——SDK 前置校验并
|
|
191
225
|
给出修复提示);``difficulty`` easy/medium/hard。**服务端无排序参数**
|
|
192
226
|
(固定 instance_id asc),SDK 不暴露。
|
|
227
|
+
|
|
228
|
+
**OSS 回退**:当 ``TransportError(biz_code=92001)`` 时回退到 OSS 存储层,
|
|
229
|
+
在客户端做过滤(q/tags/difficulty/instance_id)+ 内存分页。
|
|
193
230
|
"""
|
|
194
231
|
from urllib.parse import quote
|
|
195
232
|
from ..models import PagedResult
|
|
@@ -235,18 +272,100 @@ class InstancesClient:
|
|
|
235
272
|
if mm is None:
|
|
236
273
|
mm = normalize_metadata_model(
|
|
237
274
|
getattr(self._p, "metadata_model", None))
|
|
238
|
-
|
|
239
|
-
params.append(("metadata_model", mm))
|
|
275
|
+
params.append(("metadata_model", mm))
|
|
240
276
|
params.append(("page", str(int(page))))
|
|
241
277
|
params.append(("page_size", str(int(page_size))))
|
|
242
278
|
query = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
|
|
243
|
-
|
|
244
|
-
|
|
279
|
+
try:
|
|
280
|
+
data, pagination = self._t.request_with_meta(
|
|
281
|
+
"GET", f"{self._BASE}?{query}")
|
|
282
|
+
except TransportError as e:
|
|
283
|
+
if (e.biz_code == 92001
|
|
284
|
+
and fallback_to_oss and self._content is not None):
|
|
285
|
+
return self._discover_instances_paged(
|
|
286
|
+
dataset, version, split,
|
|
287
|
+
q=q, instance_id=instance_id,
|
|
288
|
+
tags=tags, difficulty=difficulty,
|
|
289
|
+
page=page, page_size=page_size)
|
|
290
|
+
raise
|
|
245
291
|
rows = data.get("instances", data) if isinstance(data, dict) else data
|
|
246
292
|
rows = rows if isinstance(rows, list) else []
|
|
247
293
|
items = [DatasetInstance.from_dict(r) for r in rows if isinstance(r, dict)]
|
|
248
294
|
return PagedResult.from_response(items, pagination, int(page), int(page_size))
|
|
249
295
|
|
|
296
|
+
# ── OSS 回退发现(未注册 dataset 的 instance 合成) ──
|
|
297
|
+
|
|
298
|
+
def _discover_instances(self, dataset: str, version: str,
|
|
299
|
+
split: str | None = None) -> list[DatasetInstance]:
|
|
300
|
+
"""OSS 回退的 instances.list — 从 OSS 列举指定 version/split 下的实例。"""
|
|
301
|
+
from ..store.oss_discovery import build_synthetic_instances
|
|
302
|
+
return build_synthetic_instances(
|
|
303
|
+
self._content, dataset, version, split,
|
|
304
|
+
oss_prefix=getattr(self, '_oss_prefix', 'swe/datasets'))
|
|
305
|
+
|
|
306
|
+
def _discover_instance(self, dataset: str, version: str,
|
|
307
|
+
instance_id: str,
|
|
308
|
+
split: str | None = None) -> DatasetInstance | None:
|
|
309
|
+
"""OSS 回退的 instances.get — 精确查找单个实例。"""
|
|
310
|
+
from ..store.oss_discovery import build_synthetic_instance
|
|
311
|
+
return build_synthetic_instance(
|
|
312
|
+
self._content, dataset, version, instance_id, split,
|
|
313
|
+
oss_prefix=getattr(self, '_oss_prefix', 'swe/datasets'))
|
|
314
|
+
|
|
315
|
+
def _discover_instances_paged(
|
|
316
|
+
self, dataset: str, version: str, split: str | None, *,
|
|
317
|
+
q: str | None, instance_id: str | None,
|
|
318
|
+
tags: dict[str, str] | str | None,
|
|
319
|
+
difficulty: str | None,
|
|
320
|
+
page: int, page_size: int,
|
|
321
|
+
) -> "PagedResult":
|
|
322
|
+
"""OSS 回退的 list_paged:全量列举 + 客户端过滤 + 内存分页。
|
|
323
|
+
|
|
324
|
+
``instance_id`` 精确匹配时直接 GET,不列举全量。
|
|
325
|
+
"""
|
|
326
|
+
from ..store.oss_discovery import _in_memory_paged
|
|
327
|
+
|
|
328
|
+
# 优化:instance_id 精确匹配时直接 GET,不列举全量
|
|
329
|
+
if instance_id:
|
|
330
|
+
inst = self._discover_instance(dataset, version, instance_id, split)
|
|
331
|
+
items: list[DatasetInstance] = [inst] if inst else []
|
|
332
|
+
else:
|
|
333
|
+
items = self._discover_instances(dataset, version, split)
|
|
334
|
+
|
|
335
|
+
# 客户端过滤
|
|
336
|
+
if q:
|
|
337
|
+
q_lower = q.lower()
|
|
338
|
+
items = [i for i in items
|
|
339
|
+
if q_lower in i.instance_id.lower()]
|
|
340
|
+
if difficulty:
|
|
341
|
+
items = [i for i in items
|
|
342
|
+
if i.difficulty == difficulty]
|
|
343
|
+
if tags:
|
|
344
|
+
tags_dict = (
|
|
345
|
+
tags if isinstance(tags, dict)
|
|
346
|
+
else self._parse_tags_csv(tags)
|
|
347
|
+
)
|
|
348
|
+
items = [i for i in items
|
|
349
|
+
if all(i.tags.get(k) == v
|
|
350
|
+
for k, v in tags_dict.items())]
|
|
351
|
+
|
|
352
|
+
return _in_memory_paged(items, int(page), int(page_size))
|
|
353
|
+
|
|
354
|
+
@staticmethod
|
|
355
|
+
def _parse_tags_csv(tags_str: str) -> dict[str, str]:
|
|
356
|
+
"""解析 ``"k1:v1,k2:v2"`` 格式的 tags CSV 字符串为 dict。"""
|
|
357
|
+
result: dict[str, str] = {}
|
|
358
|
+
for pair in str(tags_str).split(","):
|
|
359
|
+
pair = pair.strip()
|
|
360
|
+
if not pair:
|
|
361
|
+
continue
|
|
362
|
+
parts = pair.split(":", 1)
|
|
363
|
+
if len(parts) == 2:
|
|
364
|
+
result[parts[0]] = parts[1]
|
|
365
|
+
else:
|
|
366
|
+
result[parts[0]] = ""
|
|
367
|
+
return result
|
|
368
|
+
|
|
250
369
|
def update(self, dataset: str, version: str, instance_id: str, *,
|
|
251
370
|
split: str | None = None, dataset_split_id: str | None = None,
|
|
252
371
|
manifest: list[dict] | None = None,
|
|
@@ -377,7 +496,7 @@ class InstancesClient:
|
|
|
377
496
|
if push_image:
|
|
378
497
|
# 自带 tar 且会推 ACR:docker_image 记它将被推到的 ACR 目标地址。
|
|
379
498
|
# 仅在 push_image 时记——否则该 ACR 地址指向从未推送过的镜像,是假的。
|
|
380
|
-
docker_ref = lay.image_ref(self._p.
|
|
499
|
+
docker_ref = lay.image_ref(self._p.effective_acr_host, self._p.acr_namespace,
|
|
381
500
|
dataset, instance_id,
|
|
382
501
|
version=version, split=split)
|
|
383
502
|
else:
|
|
@@ -399,7 +518,7 @@ class InstancesClient:
|
|
|
399
518
|
if image_release_dst_acr:
|
|
400
519
|
from .. import release as _R # 局部导入避免循环依赖
|
|
401
520
|
extend_meta = _R.image_release_meta(
|
|
402
|
-
docker_ref, self._p.
|
|
521
|
+
docker_ref, self._p.effective_acr_host, image_release_dst_acr)
|
|
403
522
|
|
|
404
523
|
# ② 写元数据(pending) → 拿稳定 id(register=False 时跳过,用本地 id)
|
|
405
524
|
meta = DatasetInstance(
|
|
@@ -702,6 +821,8 @@ class InstancesClient:
|
|
|
702
821
|
warm(sts_prefix)
|
|
703
822
|
# 推镜像时同样预热 ACR 凭据缓存(否则首轮各 worker 会并发重复签同一 host)
|
|
704
823
|
if push_image and getattr(self._p, "acr_host", None):
|
|
824
|
+
# 签名恒用 apiserver 登记的 host(acr_host,VPC);token 实例级,
|
|
825
|
+
# 对公网 endpoint 同样有效(prefer_public 时连接走公网、签名仍走这里)。
|
|
705
826
|
try:
|
|
706
827
|
self._t.get_acr_credentials(self._p.acr_host)
|
|
707
828
|
except Exception: # noqa: BLE001 — 预热失败不阻断,真正推镜像时再报
|