instance-repo 1.3.0.dev0__tar.gz → 1.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/PKG-INFO +7 -1
  2. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/README.md +6 -0
  3. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/__init__.py +1 -1
  4. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/_routing.py +4 -4
  5. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/bootstrap.py +2 -1
  6. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/cli.py +21 -9
  7. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/config.py +37 -3
  8. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/datasets.py +70 -15
  9. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/images.py +1 -1
  10. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/instances.py +133 -12
  11. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/splits.py +116 -8
  12. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/versions.py +102 -20
  13. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/models.py +16 -6
  14. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/repo.py +18 -6
  15. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/acr.py +32 -17
  16. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/oss.py +99 -8
  17. instance_repo-1.3.2/instance_repo/store/oss_discovery.py +690 -0
  18. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/transport.py +15 -7
  19. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/PKG-INFO +7 -1
  20. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/SOURCES.txt +1 -0
  21. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/pyproject.toml +1 -1
  22. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/_site_defaults.py +0 -0
  23. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/artifacts.py +0 -0
  24. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/cache.py +0 -0
  25. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/__init__.py +0 -0
  26. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/benchmarks.py +0 -0
  27. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/reports.py +0 -0
  28. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/scaffold.py +0 -0
  29. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/clients/workflows.py +0 -0
  30. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/concurrency.py +0 -0
  31. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/content.py +0 -0
  32. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/endpoints.py +0 -0
  33. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/errors.py +0 -0
  34. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/image.py +0 -0
  35. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/layout.py +0 -0
  36. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/loader.py +0 -0
  37. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/paths.py +0 -0
  38. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/release.py +0 -0
  39. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/retry.py +0 -0
  40. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/__init__.py +0 -0
  41. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/base.py +0 -0
  42. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/store/registry.py +0 -0
  43. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/tasktoml.py +0 -0
  44. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo/validate.py +0 -0
  45. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/dependency_links.txt +0 -0
  46. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/entry_points.txt +0 -0
  47. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/requires.txt +0 -0
  48. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/instance_repo.egg-info/top_level.txt +0 -0
  49. {instance_repo-1.3.0.dev0 → instance_repo-1.3.2}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: instance-repo
3
- Version: 1.3.0.dev0
3
+ Version: 1.3.2
4
4
  Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
@@ -58,6 +58,12 @@ pip install "instance-repo[oss]" # 需要数据面 OSS 读写时,附带 os
58
58
  `profile_overrides={"acr_namespace": ...}`;默认 namespace 由 repo-config 返回,但**必须由部署方
59
59
  在 ACR 企业版实例中预建**,SDK 不会创建)。
60
60
 
61
+ **公网 ACR registry(runner 不在 VPC 内网时)**:默认使用 repo-config 返回的 `registry`
62
+ (VPC 内网域名)。runner 不在对应 VPC 内时,设环境变量 `IR_ACR_PREFER_PUBLIC=1`(或
63
+ `Repo(profile_overrides={"prefer_public_registry": True})`),SDK 会改走 `public_registry`
64
+ (公网域名);公网值缺失时静默回落到 VPC 值。也可用 `IR_ACR_PUBLIC_HOST` /
65
+ `profile_overrides={"acr_public_host": ...}` 显式指定公网 registry。
66
+
61
67
  兼容别名表:
62
68
 
63
69
  | 旧名 | 新名或状态 | 优先级 | 告警 |
@@ -44,6 +44,12 @@ pip install "instance-repo[oss]" # 需要数据面 OSS 读写时,附带 os
44
44
  `profile_overrides={"acr_namespace": ...}`;默认 namespace 由 repo-config 返回,但**必须由部署方
45
45
  在 ACR 企业版实例中预建**,SDK 不会创建)。
46
46
 
47
+ **公网 ACR registry(runner 不在 VPC 内网时)**:默认使用 repo-config 返回的 `registry`
48
+ (VPC 内网域名)。runner 不在对应 VPC 内时,设环境变量 `IR_ACR_PREFER_PUBLIC=1`(或
49
+ `Repo(profile_overrides={"prefer_public_registry": True})`),SDK 会改走 `public_registry`
50
+ (公网域名);公网值缺失时静默回落到 VPC 值。也可用 `IR_ACR_PUBLIC_HOST` /
51
+ `profile_overrides={"acr_public_host": ...}` 显式指定公网 registry。
52
+
47
53
  兼容别名表:
48
54
 
49
55
  | 旧名 | 新名或状态 | 优先级 | 告警 |
@@ -18,7 +18,7 @@ from .errors import (
18
18
  TransportError,
19
19
  )
20
20
 
21
- __version__ = "1.3.0.dev0"
21
+ __version__ = "1.3.2"
22
22
 
23
23
  __all__ = [
24
24
  "Repo", "load_instances",
@@ -45,14 +45,14 @@ def _split_scheme(endpoint: str) -> tuple[str, str]:
45
45
  return "", endpoint
46
46
 
47
47
 
48
- def _probe_private_endpoint(endpoint: str, timeout: float = 3.0) -> bool:
48
+ def _probe_private_endpoint(endpoint: str, timeout: float = 0.5) -> bool:
49
49
  """Quick TCP connect probe to decide whether a private endpoint is reachable.
50
50
 
51
51
  Performs a raw TCP connect (no TLS handshake) to the endpoint host/port.
52
52
  Returns True if the connect succeeds, False on any error. The default
53
- timeout is 3 seconds — short enough to save 60+ seconds of waiting when
54
- the private endpoint is unreachable, yet long enough for a normal TCP
55
- handshake on a reachable host.
53
+ timeout is 0.5 seconds — 内网 TCP 握手通常在几十毫秒内完成,0.5s 足够
54
+ 判定可达性,同时把公网环境下私网不可达的探测开销从 3s 压到 0.5s
55
+ (结果按 store 缓存 5 分钟,仅端点变更或缓存过期才重探)。
56
56
  """
57
57
  _, host = _split_scheme(endpoint)
58
58
  # Determine port: default 443 for https, 80 for http.
@@ -36,7 +36,8 @@ _logger = logging.getLogger(__name__)
36
36
  DISCOVERY_TTL_SECONDS = 3600.0
37
37
 
38
38
  #: 由自动发现填充的字段(scaffold 两项不在 repo-config 返回范围内,仍须显式提供)。
39
- DISCOVERABLE_FIELDS = ("oss_bucket", "oss_prefix", "acr_host", "acr_namespace")
39
+ DISCOVERABLE_FIELDS = ("oss_bucket", "oss_prefix", "acr_host",
40
+ "acr_public_host", "acr_namespace")
40
41
 
41
42
  _cache = TTLCache(safety=0.0)
42
43
  _lock = threading.Lock()
@@ -42,7 +42,7 @@ def build_parser() -> argparse.ArgumentParser:
42
42
  ap.add_argument("--storage-env", dest="global_storage_env", default=None,
43
43
  help="数据面存储环境(亦可用 IR_STORAGE_ENV);SDK 据此自动向 "
44
44
  "apiserver 的 repo-config 拉取 OSS/ACR 寻址并缓存")
45
- ap.add_argument("--metadata-model", default=None,
45
+ ap.add_argument("--metadata-model", default="split_first",
46
46
  choices=["split_first", "version_first"],
47
47
  help="元数据模型;缺省 split_first(split 必填、version 选填)。"
48
48
  "旧数据集用 version_first(version 必填、split 选填)")
@@ -313,7 +313,7 @@ def build_parser() -> argparse.ArgumentParser:
313
313
  help="split 级打标,格式 name=train,name2=eval(与 --run-type 互斥)")
314
314
  ver.add_argument("--environment", default=None,
315
315
  help="create/get/ingest 用:覆盖 profile 派生的 environment(pre/online)")
316
- ver.add_argument("--metadata-model", default=None,
316
+ ver.add_argument("--metadata-model", default="split_first",
317
317
  help="create/ingest 用:显式指定元数据模型(split_first/version_first)")
318
318
  ver.add_argument("--dry-run", action="store_true",
319
319
  help="ingest 用:只预览扫描结果不落库")
@@ -385,7 +385,7 @@ def build_parser() -> argparse.ArgumentParser:
385
385
  ds.add_argument("--environment", default=None,
386
386
  help="get/update 覆盖 profile 派生的 environment")
387
387
  ds.add_argument("--metadata-model", choices=["split_first", "version_first"],
388
- default=None, dest="metadata_model")
388
+ default="split_first", dest="metadata_model")
389
389
  # update 用(至少一项)
390
390
  ds.add_argument("--display-name", default=None, dest="display_name")
391
391
  ds.add_argument("--description", default=None)
@@ -724,13 +724,16 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
724
724
  tags=tags_dict, difficulty=args.difficulty,
725
725
  page=args.page, page_size=args.page_size)
726
726
  for inst in res.items:
727
- print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}",
727
+ src = "\t[oss]" if not inst.registered else ""
728
+ print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}{src}",
728
729
  file=out)
729
730
  print(f"page={res.page}/{(res.total+res.page_size-1)//res.page_size if res.total>0 else 1} "
730
731
  f"total={res.total} has_more={res.has_more}", file=out)
731
732
  return 0
732
733
  for inst in r.instances.list(dataset, version, args.split):
733
- print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}", file=out)
734
+ src = "\t[oss]" if not inst.registered else ""
735
+ print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}{src}",
736
+ file=out)
734
737
  return 0
735
738
 
736
739
  if args.cmd == "pull":
@@ -910,7 +913,9 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
910
913
  # 实例不存在时库层返回 None(apiserver 92003),显式区分"查不到"与"字段为空"。
911
914
  if inst is None:
912
915
  raise SystemExit(f"instance {dataset}/{version}/{iid} not found")
913
- print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}", file=out)
916
+ src = "\t[oss]" if not inst.registered else ""
917
+ print(f"{inst.instance_id}\t{inst.difficulty}\t{inst.docker_image}{src}",
918
+ file=out)
914
919
  return 0
915
920
 
916
921
  if args.cmd == "version":
@@ -931,7 +936,9 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
931
936
  dataset, _, _ = _resolve_target(args, 2)
932
937
  if args.action == "list":
933
938
  for v in r.versions.list(dataset):
934
- print(f"{v.version}\t{v.status}\t{v.storage_path}", file=out)
939
+ src = "\t[oss]" if not v.registered else ""
940
+ print(f"{v.version}\t{v.status}\t{v.storage_path}{src}",
941
+ file=out)
935
942
  return 0
936
943
  if not args.version:
937
944
  raise SystemExit("version create 需要 --version")
@@ -1026,7 +1033,8 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
1026
1033
  status=args.status,
1027
1034
  page=args.page, page_size=args.page_size)
1028
1035
  for s in res.items:
1029
- src = "projected" if s.projected else "record"
1036
+ src = "projected" if s.projected else (
1037
+ "oss" if not s.registered else "record")
1030
1038
  print(f"{s.name}\tversion={s.version or ''}\tstatus={s.status}"
1031
1039
  f"\trun_type={s.run_type or ''}\t"
1032
1040
  f"instance_count={_split_instance_count(s)}\t[{src}]",
@@ -1094,13 +1102,17 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
1094
1102
  if args.action == "get":
1095
1103
  d = r.datasets.get_detail(dataset, environment=args.environment,
1096
1104
  metadata_model=args.metadata_model)
1105
+ src = "oss" if not d.registered else "mongodb"
1097
1106
  print(f"dataset\t{d.dataset_name}\tid={d.dataset_id}"
1098
- f"\tvisibility={d.visibility}", file=out)
1107
+ f"\tvisibility={d.visibility}\t[{src}]", file=out)
1099
1108
  print(f"version_count={d.version_count}\tsplit_count={d.split_count}"
1100
1109
  f"\tlatest={d.latest_version or ''}", file=out)
1101
1110
  print(f"instance_count={d.stats.instance_count}"
1102
1111
  f"\tfile_count={d.stats.file_count}"
1103
1112
  f"\tsize_bytes={d.stats.size_bytes}", file=out)
1113
+ if not d.registered:
1114
+ print("warning: dataset not registered in MongoDB, "
1115
+ "data from OSS discovery", file=sys.stderr)
1104
1116
  return 0
1105
1117
  if args.action == "update":
1106
1118
  kwargs = {}
@@ -76,6 +76,11 @@ ENV_OSS_PREFIX = "IR_OSS_PREFIX"
76
76
  ENV_SCAFFOLD_BUCKET = "IR_SCAFFOLD_BUCKET"
77
77
  ENV_SCAFFOLD_ROOT = "IR_SCAFFOLD_ROOT"
78
78
 
79
+ # 公网 ACR registry:runner 不在 VPC 内网时,可显式切换使用公网 registry。
80
+ ENV_ACR_PUBLIC_HOST = "IR_ACR_PUBLIC_HOST"
81
+ # 是否优先使用公网 registry("1"/"true"/"on",大小写不敏感)。
82
+ ENV_ACR_PREFER_PUBLIC = "IR_ACR_PREFER_PUBLIC"
83
+
79
84
  #: v0.8 弃用的手工寻址环境变量 → 建议替代。scaffold 两项不在其中:scaffold 包不在
80
85
  #: repo-config 的返回范围内,仍须显式提供。
81
86
  DEPRECATED_ADDRESS_ENVS = {
@@ -179,6 +184,7 @@ class Profile:
179
184
  name: str
180
185
  oss_bucket: str = ""
181
186
  acr_host: str = ""
187
+ acr_public_host: str = ""
182
188
  acr_namespace: str = ""
183
189
  oss_prefix: str = DEFAULT_OSS_PREFIX
184
190
  scaffold_bucket: str = ""
@@ -197,6 +203,21 @@ class Profile:
197
203
  #: ``bootstrap.discover()`` 捕获异常时写入;``require()`` 据此在报错消息里
198
204
  #: 透传根因,避免"返回里没有 acr_host"这类误导性口径。
199
205
  discovery_error: str = ""
206
+ #: 优先使用公网 ACR registry(默认 False=走 VPC 内网)。runner 不在 VPC
207
+ #: 内网时置 True,配合 ``acr_public_host``(repo-config 的 public_registry)
208
+ #: 生效;公网值缺失时静默回落到 ``acr_host``。
209
+ prefer_public_registry: bool = False
210
+
211
+ @property
212
+ def effective_acr_host(self) -> str:
213
+ """ACR 操作实际使用的 registry host。
214
+
215
+ 默认返回 ``acr_host``(repo-config 的 registry,通常是 VPC 内网域名);
216
+ ``prefer_public_registry=True`` 且 ``acr_public_host`` 非空时返回公网域名。
217
+ """
218
+ if self.prefer_public_registry and self.acr_public_host:
219
+ return self.acr_public_host
220
+ return self.acr_host
200
221
 
201
222
  @property
202
223
  def dataset_environment(self) -> str:
@@ -240,12 +261,14 @@ def load_profile(name: str = "default", *, overrides: dict | None = None,
240
261
  storage_env, ENV_STORAGE_ENV, LEGACY_ENV_STORAGE_ENV)
241
262
  env_values = {k: os.environ.get(k, "") for k in
242
263
  (ENV_OSS_BUCKET, ENV_ACR_HOST, ENV_ACR_NAMESPACE,
264
+ ENV_ACR_PUBLIC_HOST,
243
265
  ENV_OSS_PREFIX, ENV_SCAFFOLD_BUCKET, ENV_SCAFFOLD_ROOT)}
244
266
  _warn_deprecated_envs(env_values)
245
267
  cfg = {
246
268
  "oss_bucket": env_values[ENV_OSS_BUCKET],
247
269
  "acr_host": env_values[ENV_ACR_HOST],
248
270
  "acr_namespace": env_values[ENV_ACR_NAMESPACE],
271
+ "acr_public_host": env_values[ENV_ACR_PUBLIC_HOST],
249
272
  "oss_prefix": env_values[ENV_OSS_PREFIX] or DEFAULT_OSS_PREFIX,
250
273
  "scaffold_bucket": env_values[ENV_SCAFFOLD_BUCKET],
251
274
  "scaffold_root": env_values[ENV_SCAFFOLD_ROOT],
@@ -259,19 +282,28 @@ def load_profile(name: str = "default", *, overrides: dict | None = None,
259
282
  sources["api_env"] = "env"
260
283
  else:
261
284
  sources["api_env"] = ""
285
+ prefer_public = False
262
286
  if overrides:
263
- # 只接受已知字段,避免把 api_env 之类误当数据面字段
264
- for k in ("oss_bucket", "acr_host", "acr_namespace", "oss_prefix",
265
- "scaffold_bucket", "scaffold_root", "api_base"):
287
+ # 只接受已知字段,避免把 api_env 之类误当数据面字段;
288
+ # prefer_public_registry 单独取(非寻址字段,不进 **cfg 传 Profile)。
289
+ for k in ("oss_bucket", "acr_host", "acr_namespace", "acr_public_host",
290
+ "oss_prefix", "scaffold_bucket", "scaffold_root", "api_base"):
266
291
  if k in overrides and overrides[k] is not None:
267
292
  cfg[k] = overrides[k]
268
293
  sources[k] = "explicit"
294
+ if overrides.get("prefer_public_registry") is not None:
295
+ prefer_public = bool(overrides["prefer_public_registry"])
269
296
  if cfg["api_base"]:
270
297
  cfg["api_base"] = str(cfg["api_base"]).rstrip("/")
298
+ if not prefer_public:
299
+ # 环境变量开关:IR_ACR_PREFER_PUBLIC=1/true/on(大小写不敏感)
300
+ prefer_public = os.environ.get(ENV_ACR_PREFER_PUBLIC, "").strip().lower() \
301
+ in ("1", "true", "on")
271
302
  return Profile(name=name, api_env=env,
272
303
  cluster=str(cluster_value).strip(),
273
304
  storage_env=normalize_storage_env(storage_value),
274
305
  metadata_model=(metadata_model or "").strip(),
306
+ prefer_public_registry=bool(prefer_public),
275
307
  sources=sources, **cfg)
276
308
 
277
309
 
@@ -323,6 +355,8 @@ def fetch_repo_config(transport, *, cluster: str = "", storage_env: str = "") ->
323
355
  if acr := data.get("acr"):
324
356
  if acr.get("registry"):
325
357
  result["acr_host"] = acr["registry"]
358
+ if acr.get("public_registry"):
359
+ result["acr_public_host"] = acr["public_registry"]
326
360
  if acr.get("namespace"):
327
361
  result["acr_namespace"] = acr["namespace"]
328
362
  # api_env 由 apiserver 在 repo-config 响应中下发(omitempty,旧版不返回)。
@@ -4,6 +4,7 @@ from __future__ import annotations
4
4
  from urllib.parse import quote
5
5
 
6
6
  from ..errors import SchemaError, TransportError
7
+ from ..layout import normalize_metadata_model
7
8
  from ..models import (ClaimWorkspace, DatasetDetail, DatasetSeries,
8
9
  PagedResult, RoleBinding, VISIBILITY_VALUES)
9
10
 
@@ -14,8 +15,14 @@ class DatasetsClient:
14
15
  # 曾误写成 /apis/v1/datasets,该路径在 apiserver 上不存在,所有调用恒 404。
15
16
  _BASE = "/apis/v1/datasets/series"
16
17
 
17
- def __init__(self, transport) -> None:
18
+ def __init__(self, transport, *, content_store=None,
19
+ oss_prefix: str = "swe/datasets",
20
+ metadata_model: str | None = None) -> None:
18
21
  self._t = transport
22
+ self._content = content_store
23
+ self._oss_prefix = oss_prefix
24
+ # OSS 回退探测的布局语义(split_first 默认;version_first 显式切换)
25
+ self._metadata_model = metadata_model or "split_first"
19
26
 
20
27
  def list(self, l1: str | None = None) -> list[DatasetSeries]:
21
28
  """列出 dataset(便利语义:自动翻页取全量,见 :meth:`list_paged` 单页对照)。
@@ -60,7 +67,7 @@ class DatasetsClient:
60
67
  ``updated_from/to`` RFC3339 闭区间;``sort_by`` updated_at(默认)|
61
68
  dataset_name;``order`` desc(默认)|asc;``benchmark_id`` 精确(
62
69
  ``"none"`` 查未绑定);``environment`` online(默认)|pre;
63
- ``metadata_model`` version_first(默认)|split_first(决定统计来源)。
70
+ ``metadata_model`` split_first(默认)|version_first(决定统计来源)。
64
71
 
65
72
  items 为 :class:`~instance_repo.models.DatasetSeries`(含服务端返回的
66
73
  version_count/split_count/latest_version/claimable 统计字段,见模块
@@ -106,8 +113,8 @@ class DatasetsClient:
106
113
  "true" if include_deprecated else "false"))
107
114
  if (environment or "").strip():
108
115
  params.append(("environment", environment.strip()))
109
- if (metadata_model or "").strip():
110
- params.append(("metadata_model", metadata_model.strip()))
116
+ mm = normalize_metadata_model(metadata_model)
117
+ params.append(("metadata_model", mm))
111
118
  params.append(("page", str(int(page))))
112
119
  params.append(("page_size", str(int(page_size))))
113
120
  query = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
@@ -118,12 +125,21 @@ class DatasetsClient:
118
125
  items = [DatasetSeries(**_pick_series(r)) for r in rows if isinstance(r, dict)]
119
126
  return PagedResult.from_response(items, pagination, int(page), int(page_size))
120
127
 
121
- def get(self, name: str) -> DatasetSeries:
122
- data = self._t.get(f"{self._BASE}/detail?dataset_name={quote(name, safe='')}")
123
- return DatasetSeries(**_pick_series(data))
128
+ def get(self, name: str, *,
129
+ fallback_to_oss: bool = True) -> DatasetSeries:
130
+ try:
131
+ data = self._t.get(
132
+ f"{self._BASE}/detail?dataset_name={quote(name, safe='')}")
133
+ return DatasetSeries(**_pick_series(data))
134
+ except TransportError as e:
135
+ if (e.biz_code == 92001 and fallback_to_oss
136
+ and self._content is not None):
137
+ return self._discover_series(name)
138
+ raise
124
139
 
125
140
  def get_detail(self, name: str, *, environment: str | None = None,
126
- metadata_model: str | None = None) -> DatasetDetail:
141
+ metadata_model: str | None = None,
142
+ fallback_to_oss: bool = True) -> DatasetDetail:
127
143
  """dataset 富详情(``GET /datasets/series/detail`` 完整响应)。
128
144
 
129
145
  相比 :meth:`get` 的轻量 series 字段:新增 version_count / split_count /
@@ -134,13 +150,20 @@ class DatasetsClient:
134
150
  """
135
151
  if not (name or "").strip():
136
152
  raise SchemaError("dataset name is required for get_detail")
137
- q = f"dataset_name={quote(name, safe='')}"
138
- if (environment or "").strip():
139
- q += f"&environment={quote(environment.strip(), safe='')}"
140
- if (metadata_model or "").strip():
141
- q += f"&metadata_model={quote(metadata_model.strip(), safe='')}"
142
- data = self._t.get(f"{self._BASE}/detail?{q}")
143
- return DatasetDetail.from_dict(data) if isinstance(data, dict) else DatasetDetail()
153
+ try:
154
+ q = f"dataset_name={quote(name, safe='')}"
155
+ if (environment or "").strip():
156
+ q += f"&environment={quote(environment.strip(), safe='')}"
157
+ mm = normalize_metadata_model(metadata_model)
158
+ q += f"&metadata_model={quote(mm, safe='')}"
159
+ data = self._t.get(f"{self._BASE}/detail?{q}")
160
+ return DatasetDetail.from_dict(data) if isinstance(data, dict) else DatasetDetail()
161
+ except TransportError as e:
162
+ if (e.biz_code == 92001 and fallback_to_oss
163
+ and self._content is not None):
164
+ return self._discover_detail(name, storage_env=environment,
165
+ metadata_model=metadata_model)
166
+ raise
144
167
 
145
168
  def update(self, name: str, *, display_name: str | None = None,
146
169
  visibility: str | None = None, owner_team_id: str | None = None,
@@ -371,6 +394,38 @@ class DatasetsClient:
371
394
  return self._t.post(
372
395
  f"{self._BASE}/{quote(dataset_id, safe='')}/claim", body)
373
396
 
397
+ # ── OSS fallback discovery (private) ─────────────────────────────────
398
+
399
+ def _discover_detail(self, name: str, *,
400
+ storage_env: str | None = None,
401
+ metadata_model: str | None = None) -> DatasetDetail:
402
+ from ..store.oss_discovery import (
403
+ parse_dataset_level, build_synthetic_dataset_detail,
404
+ _warn_if_truncated)
405
+ info = parse_dataset_level(
406
+ self._content, name, oss_prefix=self._oss_prefix,
407
+ metadata_model=metadata_model or self._metadata_model)
408
+ _warn_if_truncated(info, name, 500)
409
+ if not info.versions and not info.unversioned_splits:
410
+ from ..errors import DatasetNotFound
411
+ raise DatasetNotFound(f"dataset {name!r} not found in OSS")
412
+ return build_synthetic_dataset_detail(
413
+ name, info, storage_env=storage_env)
414
+
415
+ def _discover_series(self, name: str) -> DatasetSeries:
416
+ detail = self._discover_detail(name)
417
+ return DatasetSeries(
418
+ dataset_name=detail.dataset_name,
419
+ dataset_l1=detail.dataset_l1,
420
+ dataset_l2=detail.dataset_l2,
421
+ dataset_id=detail.dataset_id or None,
422
+ version_count=detail.version_count,
423
+ split_count=detail.split_count,
424
+ latest_version=detail.latest_version,
425
+ available_envs=detail.available_envs,
426
+ registered=False,
427
+ )
428
+
374
429
 
375
430
  # 与 apiserver `DatasetListItem`(pkg/server/dataset_service.go:231)逐字段对齐。
376
431
  _SERIES_FIELDS = ("dataset_name", "dataset_l1", "dataset_l2", "dataset_id",
@@ -48,7 +48,7 @@ class ImageClient:
48
48
  计算**目标环境**的引用,配合 :meth:`copy` 做跨环境搬运。
49
49
  """
50
50
  host = acr_host or bootstrap.require(
51
- self._p, "acr_host", action="拼 ACR 镜像引用")
51
+ self._p, "effective_acr_host", action="拼 ACR 镜像引用")
52
52
  ns = namespace or bootstrap.require(
53
53
  self._p, "acr_namespace", action="拼 ACR 镜像引用")
54
54
  return self._layout(metadata_model).image_ref(
@@ -70,6 +70,8 @@ class InstancesClient:
70
70
  # 收尾改为 ingest)。ingester 形如 (dataset, version) -> ingest job dict;
71
71
  # 由 Repo 注入 versions.ingest。直接构造未注入时 register=True 会报错。
72
72
  self._ingester = ingester
73
+ # OSS 回退发现使用的 bucket 内根前缀(profile 可覆盖,缺省 swe/datasets)。
74
+ self._oss_prefix = getattr(profile, 'oss_prefix', 'swe/datasets')
73
75
 
74
76
  def _check_split_not_locked(self, dataset: str, version: str,
75
77
  split: str) -> None:
@@ -155,24 +157,55 @@ class InstancesClient:
155
157
  def get(self, dataset: str, version: str, instance_id: str,
156
158
  split: str | None = None, *,
157
159
  environment: str | None = None,
158
- metadata_model: str | None = None) -> DatasetInstance | None:
160
+ metadata_model: str | None = None,
161
+ fallback_to_oss: bool = True) -> DatasetInstance | None:
159
162
  """取单个实例;不存在返回 None。
160
163
 
161
164
  ``environment`` / ``metadata_model`` 传 None 用 profile 派生的值(默认行为
162
165
  不变);显式传值可覆盖本次调用——CLI 需要读 online 集合时不必重建 Repo。
166
+
167
+ **OSS 双触发回退**(设计文档 §4.4.4):
168
+ a. ``TransportError(biz_code=92001)``(dataset 不存在)→ OSS 回退。
169
+ b. ``result is None``(registry 吞了 92003 instance 不存在)→ OSS 回退。
163
170
  """
164
171
  # 只在调用方显式指定时才下发覆盖参数:注册表是可注入实现,
165
172
  # 保持缺省调用形态与历史一致,避免破坏既有实现。
166
173
  overrides = _registry_overrides(environment, metadata_model)
167
- return self._reg.get(dataset, version, instance_id, split, **overrides)
174
+ try:
175
+ result = self._reg.get(
176
+ dataset, version, instance_id, split, **overrides)
177
+ except TransportError as e:
178
+ # 路径 1: dataset 不存在(92001) → OSS 回退
179
+ if (e.biz_code == 92001
180
+ and fallback_to_oss and self._content is not None):
181
+ return self._discover_instance(
182
+ dataset, version, instance_id, split)
183
+ raise
184
+ # 路径 2: instance 不存在(None, registry 吞了 92003) → OSS 回退
185
+ if result is None and fallback_to_oss and self._content is not None:
186
+ return self._discover_instance(
187
+ dataset, version, instance_id, split)
188
+ return result
168
189
 
169
190
  def list(self, dataset: str, version: str,
170
191
  split: str | None = None, *,
171
192
  environment: str | None = None,
172
- metadata_model: str | None = None) -> list[DatasetInstance]:
173
- """自动翻页取全量;``environment`` / ``metadata_model`` 语义同 :meth:`get`。"""
193
+ metadata_model: str | None = None,
194
+ fallback_to_oss: bool = True) -> list[DatasetInstance]:
195
+ """自动翻页取全量;``environment`` / ``metadata_model`` 语义同 :meth:`get`。
196
+
197
+ **OSS 回退**:当 ``TransportError(biz_code=92001)``(dataset 不存在)时,
198
+ 若 ``fallback_to_oss=True`` 且 ``content_store`` 已注入,自动回退到 OSS
199
+ 存储层列举实例。
200
+ """
174
201
  overrides = _registry_overrides(environment, metadata_model)
175
- return self._reg.query(dataset, version, split, **overrides)
202
+ try:
203
+ return self._reg.query(dataset, version, split, **overrides)
204
+ except TransportError as e:
205
+ if (e.biz_code == 92001
206
+ and fallback_to_oss and self._content is not None):
207
+ return self._discover_instances(dataset, version, split)
208
+ raise
176
209
 
177
210
  def list_paged(self, dataset: str, version: str = "", *,
178
211
  split: str | None = None, q: str | None = None,
@@ -181,7 +214,8 @@ class InstancesClient:
181
214
  difficulty: str | None = None,
182
215
  page: int = 1, page_size: int = 50,
183
216
  environment: str | None = None,
184
- metadata_model: str | None = None) -> "PagedResult":
217
+ metadata_model: str | None = None,
218
+ fallback_to_oss: bool = True) -> "PagedResult":
185
219
  """instance 分页列表(单页原始语义,``GET /datasets/instances``)。
186
220
 
187
221
  与 :meth:`list` 的自动翻页取全量互补:精确控制页码/过滤/搜索。
@@ -190,6 +224,9 @@ class InstancesClient:
190
224
  CSV("k1:v1,k2:v2",key 含 ``.``/``$`` 服务端拒绝——SDK 前置校验并
191
225
  给出修复提示);``difficulty`` easy/medium/hard。**服务端无排序参数**
192
226
  (固定 instance_id asc),SDK 不暴露。
227
+
228
+ **OSS 回退**:当 ``TransportError(biz_code=92001)`` 时回退到 OSS 存储层,
229
+ 在客户端做过滤(q/tags/difficulty/instance_id)+ 内存分页。
193
230
  """
194
231
  from urllib.parse import quote
195
232
  from ..models import PagedResult
@@ -235,18 +272,100 @@ class InstancesClient:
235
272
  if mm is None:
236
273
  mm = normalize_metadata_model(
237
274
  getattr(self._p, "metadata_model", None))
238
- if mm:
239
- params.append(("metadata_model", mm))
275
+ params.append(("metadata_model", mm))
240
276
  params.append(("page", str(int(page))))
241
277
  params.append(("page_size", str(int(page_size))))
242
278
  query = "&".join(f"{k}={quote(v, safe='')}" for k, v in params)
243
- data, pagination = self._t.request_with_meta(
244
- "GET", f"{self._BASE}?{query}")
279
+ try:
280
+ data, pagination = self._t.request_with_meta(
281
+ "GET", f"{self._BASE}?{query}")
282
+ except TransportError as e:
283
+ if (e.biz_code == 92001
284
+ and fallback_to_oss and self._content is not None):
285
+ return self._discover_instances_paged(
286
+ dataset, version, split,
287
+ q=q, instance_id=instance_id,
288
+ tags=tags, difficulty=difficulty,
289
+ page=page, page_size=page_size)
290
+ raise
245
291
  rows = data.get("instances", data) if isinstance(data, dict) else data
246
292
  rows = rows if isinstance(rows, list) else []
247
293
  items = [DatasetInstance.from_dict(r) for r in rows if isinstance(r, dict)]
248
294
  return PagedResult.from_response(items, pagination, int(page), int(page_size))
249
295
 
296
+ # ── OSS 回退发现(未注册 dataset 的 instance 合成) ──
297
+
298
+ def _discover_instances(self, dataset: str, version: str,
299
+ split: str | None = None) -> list[DatasetInstance]:
300
+ """OSS 回退的 instances.list — 从 OSS 列举指定 version/split 下的实例。"""
301
+ from ..store.oss_discovery import build_synthetic_instances
302
+ return build_synthetic_instances(
303
+ self._content, dataset, version, split,
304
+ oss_prefix=getattr(self, '_oss_prefix', 'swe/datasets'))
305
+
306
+ def _discover_instance(self, dataset: str, version: str,
307
+ instance_id: str,
308
+ split: str | None = None) -> DatasetInstance | None:
309
+ """OSS 回退的 instances.get — 精确查找单个实例。"""
310
+ from ..store.oss_discovery import build_synthetic_instance
311
+ return build_synthetic_instance(
312
+ self._content, dataset, version, instance_id, split,
313
+ oss_prefix=getattr(self, '_oss_prefix', 'swe/datasets'))
314
+
315
+ def _discover_instances_paged(
316
+ self, dataset: str, version: str, split: str | None, *,
317
+ q: str | None, instance_id: str | None,
318
+ tags: dict[str, str] | str | None,
319
+ difficulty: str | None,
320
+ page: int, page_size: int,
321
+ ) -> "PagedResult":
322
+ """OSS 回退的 list_paged:全量列举 + 客户端过滤 + 内存分页。
323
+
324
+ ``instance_id`` 精确匹配时直接 GET,不列举全量。
325
+ """
326
+ from ..store.oss_discovery import _in_memory_paged
327
+
328
+ # 优化:instance_id 精确匹配时直接 GET,不列举全量
329
+ if instance_id:
330
+ inst = self._discover_instance(dataset, version, instance_id, split)
331
+ items: list[DatasetInstance] = [inst] if inst else []
332
+ else:
333
+ items = self._discover_instances(dataset, version, split)
334
+
335
+ # 客户端过滤
336
+ if q:
337
+ q_lower = q.lower()
338
+ items = [i for i in items
339
+ if q_lower in i.instance_id.lower()]
340
+ if difficulty:
341
+ items = [i for i in items
342
+ if i.difficulty == difficulty]
343
+ if tags:
344
+ tags_dict = (
345
+ tags if isinstance(tags, dict)
346
+ else self._parse_tags_csv(tags)
347
+ )
348
+ items = [i for i in items
349
+ if all(i.tags.get(k) == v
350
+ for k, v in tags_dict.items())]
351
+
352
+ return _in_memory_paged(items, int(page), int(page_size))
353
+
354
+ @staticmethod
355
+ def _parse_tags_csv(tags_str: str) -> dict[str, str]:
356
+ """解析 ``"k1:v1,k2:v2"`` 格式的 tags CSV 字符串为 dict。"""
357
+ result: dict[str, str] = {}
358
+ for pair in str(tags_str).split(","):
359
+ pair = pair.strip()
360
+ if not pair:
361
+ continue
362
+ parts = pair.split(":", 1)
363
+ if len(parts) == 2:
364
+ result[parts[0]] = parts[1]
365
+ else:
366
+ result[parts[0]] = ""
367
+ return result
368
+
250
369
  def update(self, dataset: str, version: str, instance_id: str, *,
251
370
  split: str | None = None, dataset_split_id: str | None = None,
252
371
  manifest: list[dict] | None = None,
@@ -377,7 +496,7 @@ class InstancesClient:
377
496
  if push_image:
378
497
  # 自带 tar 且会推 ACR:docker_image 记它将被推到的 ACR 目标地址。
379
498
  # 仅在 push_image 时记——否则该 ACR 地址指向从未推送过的镜像,是假的。
380
- docker_ref = lay.image_ref(self._p.acr_host, self._p.acr_namespace,
499
+ docker_ref = lay.image_ref(self._p.effective_acr_host, self._p.acr_namespace,
381
500
  dataset, instance_id,
382
501
  version=version, split=split)
383
502
  else:
@@ -399,7 +518,7 @@ class InstancesClient:
399
518
  if image_release_dst_acr:
400
519
  from .. import release as _R # 局部导入避免循环依赖
401
520
  extend_meta = _R.image_release_meta(
402
- docker_ref, self._p.acr_host, image_release_dst_acr)
521
+ docker_ref, self._p.effective_acr_host, image_release_dst_acr)
403
522
 
404
523
  # ② 写元数据(pending) → 拿稳定 id(register=False 时跳过,用本地 id)
405
524
  meta = DatasetInstance(
@@ -702,6 +821,8 @@ class InstancesClient:
702
821
  warm(sts_prefix)
703
822
  # 推镜像时同样预热 ACR 凭据缓存(否则首轮各 worker 会并发重复签同一 host)
704
823
  if push_image and getattr(self._p, "acr_host", None):
824
+ # 签名恒用 apiserver 登记的 host(acr_host,VPC);token 实例级,
825
+ # 对公网 endpoint 同样有效(prefer_public 时连接走公网、签名仍走这里)。
705
826
  try:
706
827
  self._t.get_acr_credentials(self._p.acr_host)
707
828
  except Exception: # noqa: BLE001 — 预热失败不阻断,真正推镜像时再报