instance-repo 1.3.2.dev4__py3-none-any.whl → 1.3.3.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
instance_repo/__init__.py CHANGED
@@ -18,7 +18,7 @@ from .errors import (
18
18
  TransportError,
19
19
  )
20
20
 
21
- __version__ = "1.3.2.dev4"
21
+ __version__ = "1.3.3.dev2"
22
22
 
23
23
  __all__ = [
24
24
  "Repo", "load_instances",
@@ -160,7 +160,8 @@ class DatasetsClient:
160
160
  return DatasetDetail.from_dict(data) if isinstance(data, dict) else DatasetDetail()
161
161
  except TransportError as e:
162
162
  if (e.biz_code == 92001 and fallback_to_oss
163
- and self._content is not None):
163
+ and self._content is not None
164
+ and self._discovery_env_match(environment)):
164
165
  return self._discover_detail(name, storage_env=environment,
165
166
  metadata_model=metadata_model)
166
167
  raise
@@ -396,6 +397,20 @@ class DatasetsClient:
396
397
 
397
398
  # ── OSS fallback discovery (private) ─────────────────────────────────
398
399
 
400
+ def _discovery_env_match(self, environment: str | None) -> bool:
401
+ """OSS 回退环境兼容性检查:请求的 environment 须与 profile storage_env 派生环境一致。"""
402
+ req = (environment or "").strip()
403
+ if not req:
404
+ return True
405
+ if self._p is None:
406
+ return True
407
+ from .config import dataset_environment_for
408
+ derived = dataset_environment_for(
409
+ getattr(self._p, "storage_env", ""))
410
+ if not derived:
411
+ return True
412
+ return req == derived
413
+
399
414
  def _discover_detail(self, name: str, *,
400
415
  storage_env: str | None = None,
401
416
  metadata_model: str | None = None) -> DatasetDetail:
@@ -177,12 +177,14 @@ class InstancesClient:
177
177
  except TransportError as e:
178
178
  # 路径 1: dataset 不存在(92001) → OSS 回退
179
179
  if (e.biz_code == 92001
180
- and fallback_to_oss and self._content is not None):
180
+ and fallback_to_oss and self._content is not None
181
+ and self._discovery_env_match(environment)):
181
182
  return self._discover_instance(
182
183
  dataset, version, instance_id, split)
183
184
  raise
184
185
  # 路径 2: instance 不存在(None, registry 吞了 92003) → OSS 回退
185
- if result is None and fallback_to_oss and self._content is not None:
186
+ if result is None and fallback_to_oss and self._content is not None \
187
+ and self._discovery_env_match(environment):
186
188
  return self._discover_instance(
187
189
  dataset, version, instance_id, split)
188
190
  return result
@@ -203,7 +205,8 @@ class InstancesClient:
203
205
  return self._reg.query(dataset, version, split, **overrides)
204
206
  except TransportError as e:
205
207
  if (e.biz_code == 92001
206
- and fallback_to_oss and self._content is not None):
208
+ and fallback_to_oss and self._content is not None
209
+ and self._discovery_env_match(environment)):
207
210
  return self._discover_instances(dataset, version, split)
208
211
  raise
209
212
 
@@ -281,7 +284,8 @@ class InstancesClient:
281
284
  "GET", f"{self._BASE}?{query}")
282
285
  except TransportError as e:
283
286
  if (e.biz_code == 92001
284
- and fallback_to_oss and self._content is not None):
287
+ and fallback_to_oss and self._content is not None
288
+ and self._discovery_env_match(environment)):
285
289
  return self._discover_instances_paged(
286
290
  dataset, version, split,
287
291
  q=q, instance_id=instance_id,
@@ -295,6 +299,20 @@ class InstancesClient:
295
299
 
296
300
  # ── OSS 回退发现(未注册 dataset 的 instance 合成) ──
297
301
 
302
+ def _discovery_env_match(self, environment: str | None) -> bool:
303
+ """OSS 回退环境兼容性检查:请求的 environment 须与 profile storage_env 派生环境一致。"""
304
+ req = (environment or "").strip()
305
+ if not req:
306
+ return True
307
+ if self._p is None:
308
+ return True
309
+ from .config import dataset_environment_for
310
+ derived = dataset_environment_for(
311
+ getattr(self._p, "storage_env", ""))
312
+ if not derived:
313
+ return True
314
+ return req == derived
315
+
298
316
  def _discover_instances(self, dataset: str, version: str,
299
317
  split: str | None = None) -> list[DatasetInstance]:
300
318
  """OSS 回退的 instances.list — 从 OSS 列举指定 version/split 下的实例。"""
@@ -850,7 +868,8 @@ class InstancesClient:
850
868
  return results
851
869
 
852
870
  def pull(self, dataset: str, version: str, instance_id: str, dest_dir,
853
- split: str = "default", *, verify: bool = True) -> Path:
871
+ split: str = "default", *, verify: bool = True,
872
+ fallback_to_oss: bool = True) -> Path:
854
873
  """下载实例内容。自动检测格式:harbor 下载 content.tgz,swe 下载 {id}.json,
855
874
  custom 下载 {id}.json + assets 目录。
856
875
 
@@ -860,6 +879,8 @@ class InstancesClient:
860
879
  2. 无元数据或 manifest 无法区分 swe/custom → 下载 metadata JSON 检查
861
880
  metadata_model 字段(custom → 下载 assets 目录;否则 swe)
862
881
  3. metadata JSON 不存在 → fallback 到 harbor(下载 content.tgz)
882
+ 4. dataset 未注册(92001)→ OSS 回退:list-only 枚举文件树 + 逐文件下载
883
+ (仍走 DOWNLOAD 意图:仅私网,公网下载照旧禁止)
863
884
  """
864
885
  validate_instance_id(instance_id, source="pull()")
865
886
  expected = None
@@ -869,7 +890,15 @@ class InstancesClient:
869
890
  meta_bytes: bytes | None = None
870
891
 
871
892
  if verify:
872
- meta = self._reg.get(dataset, version, instance_id, split)
893
+ try:
894
+ meta = self._reg.get(dataset, version, instance_id, split)
895
+ except TransportError as e:
896
+ if (e.biz_code == 92001 and fallback_to_oss
897
+ and self._content is not None):
898
+ # dataset 未注册 → OSS 回退下载(私网 + 系统调用方语义)
899
+ return self._pull_unregistered(
900
+ dataset, version, instance_id, dest_dir, split)
901
+ raise
873
902
  if meta is not None and meta.manifest:
874
903
  has_tgz = False
875
904
  for m in meta.manifest:
@@ -1044,6 +1073,47 @@ class InstancesClient:
1044
1073
  downloaded.append(local_path)
1045
1074
  return downloaded
1046
1075
 
1076
+ def _pull_unregistered(self, dataset: str, version: str,
1077
+ instance_id: str, dest_dir, split: str) -> Path:
1078
+ """未注册 dataset 的 pull 回退:list-only 枚举文件树 + 逐文件下载。
1079
+
1080
+ 下载仍走 :meth:`~instance_repo.store.oss.OssContentStore.get_object`
1081
+ 的 DOWNLOAD 意图——**仅私网可用,公网下载照旧被拒**(与注册路径一致)。
1082
+
1083
+ STS 语义:未注册 dataset 无 RBAC 记录,GetObject STS 仅 system 白名单
1084
+ 调用方可签发(非 system 调用方在签发阶段收到明确错误),与历史行为一致。
1085
+
1086
+ 产物结构(harbor 同层,便于 CLI 整体搬移)::
1087
+
1088
+ {dest_dir}/{instance_id}/{instance_id}.json # metadata
1089
+ {dest_dir}/{instance_id}/<assets 相对路径> # assets 文件
1090
+ {dest_dir}/{instance_id}/report/<report 相对路径> # report 文件
1091
+
1092
+ Returns the metadata JSON 本地路径。
1093
+ """
1094
+ from ..store.oss_discovery import discover_instance_file_tree
1095
+
1096
+ tree = discover_instance_file_tree(
1097
+ self._content, dataset, version, instance_id, split,
1098
+ oss_prefix=self._oss_prefix)
1099
+ if tree is None:
1100
+ from ..errors import DatasetNotFound
1101
+ raise DatasetNotFound(
1102
+ f"instance {instance_id!r} not found in OSS for "
1103
+ f"dataset {dataset!r}")
1104
+
1105
+ metadata_key, files = tree
1106
+ dest_root = Path(dest_dir) / instance_id
1107
+ dest_root.mkdir(parents=True, exist_ok=True)
1108
+ metadata_dest = dest_root / f"{instance_id}.json"
1109
+
1110
+ self._content.get_object(metadata_key, metadata_dest, dataset=dataset)
1111
+ for key, rel in files:
1112
+ local_path = dest_root / rel
1113
+ local_path.parent.mkdir(parents=True, exist_ok=True)
1114
+ self._content.get_object(key, local_path, dataset=dataset)
1115
+ return metadata_dest
1116
+
1047
1117
  def pull_many(self, instance_ids: list[str], dataset: str, version: str,
1048
1118
  dest_dir, *, split: str = "default",
1049
1119
  concurrency: int = 8, continue_on_error: bool = False,
@@ -103,7 +103,8 @@ class SplitsClient:
103
103
  except TransportError as e:
104
104
  # ── 2. 92001 异常 → 直接进入 OSS 回退 ──
105
105
  if (e.biz_code == 92001
106
- and fallback_to_oss and self._content is not None):
106
+ and fallback_to_oss and self._content is not None
107
+ and self._discovery_env_match(environment)):
107
108
  return self._discover_splits(
108
109
  dataset, version, scope, status, page, page_size)
109
110
  raise
@@ -114,7 +115,8 @@ class SplitsClient:
114
115
  if not items and not (status or "").strip():
115
116
  items = self._legacy_list(dataset, version, self._env(environment))
116
117
  # ── 3b. 仍为空 → OSS 回退 ──
117
- if not items and fallback_to_oss and self._content is not None:
118
+ if not items and fallback_to_oss and self._content is not None \
119
+ and self._discovery_env_match(environment):
118
120
  return self._discover_splits(
119
121
  dataset, version, scope, status, page, page_size)
120
122
  return PagedResult.from_response(items, pagination, int(page), int(page_size))
@@ -156,7 +158,8 @@ class SplitsClient:
156
158
  if result is not None:
157
159
  return result
158
160
  # Legacy 回退无果 → OSS 回退
159
- if fallback_to_oss and self._content is not None:
161
+ if fallback_to_oss and self._content is not None \
162
+ and self._discovery_env_match(environment):
160
163
  return self._discover_split(dataset, split, version)
161
164
  return None
162
165
  raise
@@ -261,11 +264,14 @@ class SplitsClient:
261
264
  page: int, page_size: int) -> PagedResult:
262
265
  """OSS 回退的 splits.list — 处理 version_scope 和 status 过滤。
263
266
 
264
- 注意:回退场景(dataset 未注册)下 ``version_scope`` 是 MongoDB 语义,
265
- OSS 探测无从区分 versioned/unversioned——**忽略 scope 并返回全部探测结果**
266
- (G4:否则 ap CLI 无 --version 的 split list 恒空)。``version`` 非空时
267
- 仍精确匹配该 version;version 为空时额外包含 split_first 无版本 split
268
- (version="" 归属)。
267
+ split_first 模式下 OSS 探测可区分无版本 split(level-1 目录只有 .json)
268
+ 和有版本 split(level-1 目录含子目录),因此 ``version_scope`` 生效:
269
+
270
+ - ``"unversioned"``: 只返回无版本 split
271
+ - ``"versioned"``: 只返回有版本 split
272
+ - ``"all"`` / None: 返回全部(SDK 默认行为)
273
+
274
+ ``version`` 非空时仍精确匹配该 version。
269
275
  """
270
276
  from ..store.oss_discovery import (
271
277
  parse_dataset_level, build_synthetic_split,
@@ -276,21 +282,23 @@ class SplitsClient:
276
282
  metadata_model=self._discovery_mm())
277
283
  _warn_if_truncated(info, dataset, 500)
278
284
  splits: list[DatasetSplit] = []
285
+ scope = (version_scope or "all").strip().lower()
279
286
  # split_first 无版本布局:dataset 根下直接是 split 目录
280
- if not version:
287
+ if not version and scope in ("all", "unversioned"):
281
288
  for us in info.unversioned_splits:
282
289
  splits.append(build_synthetic_split(
283
290
  dataset, "", us.name, instance_count=us.instance_count))
284
- for v in info.versions:
285
- if version and v.version != version:
286
- continue
287
- if v.has_default_instances:
288
- splits.append(build_synthetic_split(
289
- dataset, v.version, "default",
290
- instance_count=v.default_instance_count))
291
- for s in v.splits:
292
- splits.append(build_synthetic_split(
293
- dataset, v.version, s))
291
+ if scope in ("all", "exact"):
292
+ for v in info.versions:
293
+ if version and v.version != version:
294
+ continue
295
+ if v.has_default_instances:
296
+ splits.append(build_synthetic_split(
297
+ dataset, v.version, "default",
298
+ instance_count=v.default_instance_count))
299
+ for s in v.splits:
300
+ splits.append(build_synthetic_split(
301
+ dataset, v.version, s))
294
302
  if (status or "").strip():
295
303
  splits = [s for s in splits if s.status == status.strip()]
296
304
  return _in_memory_paged(splits, int(page), int(page_size))
@@ -329,6 +337,27 @@ class SplitsClient:
329
337
  getattr(self._p, "metadata_model", None)
330
338
  if self._p is not None else None)
331
339
 
340
+ def _discovery_env_match(self, environment: str | None) -> bool:
341
+ """OSS 回退环境兼容性检查。
342
+
343
+ 请求的 ``environment`` 必须与 profile 的 storage_env 派生环境一致,
344
+ 否则 OSS 回退返回空——因为 bucket 是按 storage_env 绑定的,dev 桶里
345
+ 的数据不应作为 online 结果返回(反之亦然)。
346
+
347
+ ``environment`` 为空时视为兼容(跟随 profile 默认环境)。
348
+ """
349
+ req = (environment or "").strip()
350
+ if not req:
351
+ return True
352
+ if self._p is None:
353
+ return True
354
+ from .config import dataset_environment_for
355
+ derived = dataset_environment_for(
356
+ getattr(self._p, "storage_env", ""))
357
+ if not derived:
358
+ return True # 无法派生时不做限制
359
+ return req == derived
360
+
332
361
  # ── 写 ──
333
362
 
334
363
  def _post_create(self, dataset: str, split: str, version: str,
@@ -228,7 +228,8 @@ class VersionsClient:
228
228
  return [DatasetVersion(**_pick(r)) for r in (rows or [])]
229
229
  except TransportError as e:
230
230
  if (e.biz_code == 92001 and fallback_to_oss
231
- and self._content is not None):
231
+ and self._content is not None
232
+ and self._discovery_env_match(environment)):
232
233
  return self._discover_versions(dataset, metadata_model=metadata_model)
233
234
  raise
234
235
 
@@ -262,7 +263,8 @@ class VersionsClient:
262
263
  data, pagination = self._t.request_with_meta("GET", f"{self._BASE}?{q}")
263
264
  except TransportError as e:
264
265
  if (e.biz_code == 92001 and fallback_to_oss
265
- and self._content is not None):
266
+ and self._content is not None
267
+ and self._discovery_env_match(environment)):
266
268
  return self._discover_versions_paged(
267
269
  dataset, status, int(page), int(page_size),
268
270
  metadata_model=metadata_model)
@@ -295,7 +297,8 @@ class VersionsClient:
295
297
  return DatasetVersion.from_dict_full(data) if isinstance(data, dict) else DatasetVersion(version=version, dataset=dataset)
296
298
  except TransportError as e:
297
299
  if (e.biz_code == 92001 and fallback_to_oss
298
- and self._content is not None):
300
+ and self._content is not None
301
+ and self._discovery_env_match(environment)):
299
302
  return self._discover_version(dataset, version,
300
303
  metadata_model=metadata_model)
301
304
  raise
@@ -939,6 +942,20 @@ class VersionsClient:
939
942
  getattr(self._p, "metadata_model", None)
940
943
  if self._p is not None else None)
941
944
 
945
+ def _discovery_env_match(self, environment: str | None) -> bool:
946
+ """OSS 回退环境兼容性检查:请求的 environment 须与 profile storage_env 派生环境一致。"""
947
+ req = (environment or "").strip()
948
+ if not req:
949
+ return True
950
+ if self._p is None:
951
+ return True
952
+ from .config import dataset_environment_for
953
+ derived = dataset_environment_for(
954
+ getattr(self._p, "storage_env", ""))
955
+ if not derived:
956
+ return True
957
+ return req == derived
958
+
942
959
 
943
960
  # 与 apiserver `DatasetVersionListItem`(pkg/server/dataset_service.go:315)对齐:
944
961
  # 列表项带 instance_count/split_count/updated_at/last_ingested_at,详情接口才带
@@ -100,6 +100,22 @@ def _unconfirmed_private_endpoint_error(endpoint: str) -> PrivateNetworkRequired
100
100
  )
101
101
 
102
102
 
103
+ def _import_oss2(): # pragma: no cover - 真实 oss2
104
+ """惰性 import oss2,屏蔽其编译期 SyntaxWarning。
105
+
106
+ oss2 2.19.1 的 ``sign_url`` docstring 含非 raw 字符串里的 ``\\&``
107
+ (示例 URL 的 OSSAccessKeyId=...&Expires=...),首次 import 时 Python
108
+ 编译器对其发出 ``SyntaxWarning: invalid escape sequence '\\&'``——纯第三方
109
+ 文档噪音,与请求拼接无关。仅抑制 SyntaxWarning(编译期),不影响运行时
110
+ 其他警告。
111
+ """
112
+ import warnings as _warnings
113
+ with _warnings.catch_warnings():
114
+ _warnings.simplefilter("ignore", SyntaxWarning)
115
+ import oss2
116
+ return oss2
117
+
118
+
103
119
  def _default_bucket_factory(sts: dict, bucket: str, *,
104
120
  private_download: bool = False): # pragma: no cover - 真实 oss2
105
121
  """Build the default oss2 bucket, hardening private-download transport.
@@ -108,7 +124,7 @@ def _default_bucket_factory(sts: dict, bucket: str, *,
108
124
  this transport guarantee. The built-in private-download path forces HTTPS, ignores ambient
109
125
  proxy variables, and rejects redirects before requests can follow them.
110
126
  """
111
- import oss2
127
+ oss2 = _import_oss2()
112
128
 
113
129
  endpoint = sts["endpoint"]
114
130
  session = None
@@ -139,7 +155,7 @@ def _default_bucket_factory(sts: dict, bucket: str, *,
139
155
 
140
156
 
141
157
  def _default_resumable_upload(bucket, key: str, filename: str): # pragma: no cover - 真实 oss2
142
- import oss2
158
+ oss2 = _import_oss2()
143
159
  oss2.resumable_upload(bucket, key, filename)
144
160
 
145
161
 
@@ -451,7 +467,8 @@ class OssContentStore(BaseContentStore):
451
467
  lambda bucket: bucket.object_exists(key),
452
468
  )
453
469
 
454
- def list_objects(self, prefix: str, *, list_only: bool = False) -> list[str]:
470
+ def list_objects(self, prefix: str, *, list_only: bool = False,
471
+ dataset: str | None = None) -> list[str]:
455
472
  def list_all(bucket):
456
473
  keys: list[str] = []
457
474
  marker = ""
@@ -465,10 +482,11 @@ class OssContentStore(BaseContentStore):
465
482
  return keys
466
483
 
467
484
  return self._execute_oss(
468
- _OperationIntent.NON_DOWNLOAD, prefix, None, list_all,
485
+ _OperationIntent.NON_DOWNLOAD, prefix, dataset, list_all,
469
486
  list_only=list_only)
470
487
 
471
- def list_prefixes(self, prefix: str, *, list_only: bool = False) -> list[str]:
488
+ def list_prefixes(self, prefix: str, *, list_only: bool = False,
489
+ dataset: str | None = None) -> list[str]:
472
490
  """List immediate child *directories* under *prefix* using ``delimiter='/'``.
473
491
 
474
492
  Returns stripped directory names (without the parent prefix or trailing
@@ -499,11 +517,12 @@ class OssContentStore(BaseContentStore):
499
517
  return names
500
518
 
501
519
  return self._execute_oss(
502
- _OperationIntent.NON_DOWNLOAD, prefix, None, list_dirs,
520
+ _OperationIntent.NON_DOWNLOAD, prefix, dataset, list_dirs,
503
521
  list_only=list_only)
504
522
 
505
523
  def list_level(self, prefix: str, *,
506
- list_only: bool = False) -> tuple[list[str], list[str]]:
524
+ list_only: bool = False,
525
+ dataset: str | None = None) -> tuple[list[str], list[str]]:
507
526
  """Single-level listing with ``delimiter='/'``.
508
527
 
509
528
  Returns ``(dir_names, file_keys)`` where *dir_names* are stripped
@@ -533,7 +552,7 @@ class OssContentStore(BaseContentStore):
533
552
  return names, keys
534
553
 
535
554
  return self._execute_oss(
536
- _OperationIntent.NON_DOWNLOAD, prefix, None, list_one_level,
555
+ _OperationIntent.NON_DOWNLOAD, prefix, dataset, list_one_level,
537
556
  list_only=list_only)
538
557
 
539
558
  # ── directory-level helpers(自定义格式实例 = 目录级上传/下载)──
@@ -235,7 +235,7 @@ def parse_dataset_level(
235
235
  # -- Level 1: list immediate children under the dataset root -----------
236
236
  try:
237
237
  level1_names, level1_all_keys = content_store.list_level(
238
- root, list_only=True)
238
+ root, list_only=True, dataset=dataset)
239
239
  except TransportError as e:
240
240
  if e.status == 403:
241
241
  # OSS fallback requires ListObjects permission, which is only granted
@@ -293,7 +293,7 @@ def parse_dataset_level(
293
293
  group_dir = f"{root}{name}/"
294
294
  try:
295
295
  level2_names, level2_keys = content_store.list_level(
296
- group_dir, list_only=True)
296
+ group_dir, list_only=True, dataset=dataset)
297
297
  except Exception: # noqa: BLE001
298
298
  level2_names, level2_keys = [], []
299
299
  l2_groups = _group_level_names(level2_names)
@@ -316,7 +316,7 @@ def parse_dataset_level(
316
316
  split_dir = f"{group_dir}{s_name}/"
317
317
  try:
318
318
  _s_dirs, s_keys = content_store.list_level(
319
- split_dir, list_only=True)
319
+ split_dir, list_only=True, dataset=dataset)
320
320
  instance_count += len(_direct_json_keys(s_keys, split_dir))
321
321
  except Exception: # noqa: BLE001
322
322
  pass # best-effort
@@ -540,7 +540,8 @@ def build_synthetic_instances(
540
540
  """
541
541
  metadata_prefix = _metadata_prefix(dataset, version, split, oss_prefix)
542
542
 
543
- all_keys = content_store.list_objects(metadata_prefix, list_only=True)
543
+ all_keys = content_store.list_objects(
544
+ metadata_prefix, list_only=True, dataset=dataset)
544
545
  json_keys = _direct_json_keys(all_keys, metadata_prefix)
545
546
 
546
547
  instances: list[DatasetInstance] = []
@@ -585,6 +586,62 @@ def _format_from_key(key: str) -> str:
585
586
  return ""
586
587
 
587
588
 
589
+ def discover_instance_file_tree(
590
+ content_store,
591
+ dataset: str,
592
+ version: str,
593
+ instance_id: str,
594
+ split: str | None,
595
+ *,
596
+ oss_prefix: str,
597
+ ) -> tuple[str, list[tuple[str, str]]] | None:
598
+ """用 LIST 枚举实例的完整文件树(发现模式,list-only STS)。
599
+
600
+ Returns ``(metadata_key, files)`` where *files* is a list of
601
+ ``(oss_key, rel_path)`` tuples covering ``{split}-assets/{id}/`` 与
602
+ ``{split}-report/{id}/`` 下的所有对象。*rel_path* 相对实例根(assets
603
+ 文件为原相对路径,report 文件加 ``report/`` 前缀避免与 assets 同名冲突)。
604
+
605
+ 实例不存在(metadata .json 的精确 key LIST 未命中)时返回 ``None``。
606
+ """
607
+ if version:
608
+ metadata_key = P.metadata_key(
609
+ dataset, version, instance_id, split, prefix=oss_prefix)
610
+ else:
611
+ l1, l2 = P.split_dataset(dataset)
612
+ base = f"{oss_prefix}/{l1}/{l2}"
613
+ metadata_key = (f"{base}/{split}/{instance_id}.json"
614
+ if split and split != "default"
615
+ else f"{base}/{instance_id}.json")
616
+
617
+ # 存在性判定用 LIST(list-only STS 无 GetObject,不能 HEAD)。
618
+ # 不能用「prefix=精确 key」的 LIST:OSS 的 prefix 过滤是字符串前缀语义,
619
+ # 且 RAM 条件只放行 {parent}/*(精确 key 会带出同前缀兄弟对象,见 apiserver
620
+ # sts/client.go),改为父目录 LIST + 客户端成员判断。
621
+ parent = metadata_key.rsplit("/", 1)[0] + "/"
622
+ try:
623
+ if metadata_key not in content_store.list_objects(
624
+ parent, list_only=True, dataset=dataset):
625
+ return None
626
+ except Exception: # noqa: BLE001
627
+ return None
628
+
629
+ files: list[tuple[str, str]] = []
630
+ for suffix, rel_prefix in (("assets", ""), ("report", "report/")):
631
+ dir_prefix = _instance_dir(dataset, version, split, instance_id,
632
+ suffix, oss_prefix)
633
+ try:
634
+ for k in content_store.list_objects(
635
+ dir_prefix, list_only=True, dataset=dataset):
636
+ if k.endswith("/") or not k.startswith(dir_prefix):
637
+ continue # 目录标记 / 异常 key
638
+ rel = rel_prefix + k[len(dir_prefix):]
639
+ files.append((k, rel))
640
+ except Exception: # noqa: BLE001
641
+ pass # best-effort:无 assets/report 目录时忽略
642
+ return metadata_key, files
643
+
644
+
588
645
  def _instance_manifest(
589
646
  content_store,
590
647
  dataset: str,
@@ -600,15 +657,13 @@ def _instance_manifest(
600
657
  发现模式(list-only STS)无 GetObject,读不到 metadata JSON 内容,但目录
601
658
  结构可通过 LIST 完整呈现——恰好满足「查看实例的目录文件结构」需求。
602
659
  """
603
- keys = [metadata_key]
604
- for suffix in ("assets", "report"):
605
- try:
606
- keys.extend(k for k in content_store.list_objects(
607
- _instance_dir(dataset, version, split, instance_id,
608
- suffix, oss_prefix),
609
- list_only=True) if not k.endswith("/"))
610
- except Exception: # noqa: BLE001
611
- pass # best-effort:无 assets/report 目录时忽略
660
+ tree = discover_instance_file_tree(
661
+ content_store, dataset, version, instance_id, split,
662
+ oss_prefix=oss_prefix)
663
+ if tree is None:
664
+ return []
665
+ _meta_key, files = tree
666
+ keys = [_meta_key] + [k for k, _ in files]
612
667
  bucket = getattr(content_store, "bucket_name", "") or ""
613
668
  return [
614
669
  ManifestEntry(
@@ -657,8 +712,12 @@ def build_synthetic_instance(
657
712
  # 存在性判定用 LIST(list_only=True 仅需 ListObjects)而非 HEAD——
658
713
  # list-only STS 无 oss:GetObject,object_exists 的 HEAD 会在 STS 签发或
659
714
  # OSS 侧被拒,导致已存在的实例恒被误判为不存在。
715
+ # 不能用「prefix=精确 key」的 LIST(OSS 字符串前缀语义会带出兄弟对象,
716
+ # RAM 条件也只放行父目录形态),改父目录 LIST + 客户端成员判断。
717
+ parent = key.rsplit("/", 1)[0] + "/"
660
718
  try:
661
- keys = content_store.list_objects(key, list_only=True)
719
+ keys = content_store.list_objects(
720
+ parent, list_only=True, dataset=dataset)
662
721
  except Exception: # noqa: BLE001
663
722
  return None
664
723
 
@@ -1,10 +1,10 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: instance-repo
3
- Version: 1.3.2.dev4
3
+ Version: 1.3.3.dev2
4
4
  Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
5
5
  Requires-Python: >=3.10
6
6
  Description-Content-Type: text/markdown
7
- Requires-Dist: irepo-artifact<0.2.0,>=0.1.1
7
+ Requires-Dist: irepo-artifact<0.3.0,>=0.1.1
8
8
  Requires-Dist: tomli>=1.1.0; python_version < "3.11"
9
9
  Provides-Extra: oss
10
10
  Requires-Dist: oss2>=2.19; extra == "oss"
@@ -1,4 +1,4 @@
1
- instance_repo/__init__.py,sha256=lAyyofR-xkJ6HHVt7KUbO5Ar3_hFdKTJxyAK13u7hnE,1422
1
+ instance_repo/__init__.py,sha256=0xETAU6eMcC3NKU8yJlOMlE68yKJcasRlHsUBQ9TgHg,1422
2
2
  instance_repo/_routing.py,sha256=fX3oKJNca1XTD1pKK304eq0clf7I8iEdFkd2WFO3oV4,8414
3
3
  instance_repo/_site_defaults.py,sha256=ck7aHHTup8ibBo0l5MSR03XpTtxJ11hYo7oM2DzWs4A,2383
4
4
  instance_repo/artifacts.py,sha256=OOcSI1ehB_dw1ZNScjyhzghPqOdL534kQp_iI2Hz7IU,10659
@@ -23,22 +23,22 @@ instance_repo/validate.py,sha256=NMR-vFBIU_FQ0WKdBZuQ-ua4jtkAzvWwEjnAHyAFwDs,118
23
23
  instance_repo/clients/__init__.py,sha256=5V7Pe0b6nnMiwIhACA4ctQowovrL4iHTiHKKcjmsBQo,488
24
24
  instance_repo/clients/benchmarks.py,sha256=RzHY8VzDP1o2jT4M8gxxbcr3amuNr4mlSblZhhJqG_w,2778
25
25
  instance_repo/clients/config.py,sha256=UdfNDqeQnFaOBBAl-Aj-W2JvuouvJ11q-_gB4da9yBg,21209
26
- instance_repo/clients/datasets.py,sha256=snSqr4tHVzOCj4HnQlGHoKmqtz1HrxEoMNoja4yrSuI,24497
26
+ instance_repo/clients/datasets.py,sha256=vqoqxhTJMt5rd6JH5vBPfT8vIg3IDHpA7jUEZHIJgkI,25109
27
27
  instance_repo/clients/images.py,sha256=8i0FXxAAL2mOUUTrVgDdWSYM2imJ5nOEXzpVRBa-G-k,4911
28
- instance_repo/clients/instances.py,sha256=IOAKl3MAlW3UcftX8b5cGIEdC5_1qfouaVSE_0-OAxE,58125
28
+ instance_repo/clients/instances.py,sha256=WK07M8VOfpruwQCPs1e_4W-M1wLU8coWXNbAH097wQ0,61512
29
29
  instance_repo/clients/reports.py,sha256=4Z8bpcSi-mmfkQLKpMklelLI4mgHcnjfb0725bE2nkI,3001
30
30
  instance_repo/clients/scaffold.py,sha256=VLicNFnqKi2iYB-gjOkQ9wa1unNOJXO7UxBZlz7El5A,4998
31
- instance_repo/clients/splits.py,sha256=o8ygCVAMrxECNojzfUdaHQtuwL4Imzm7ghNKWPxG-VE,28255
32
- instance_repo/clients/versions.py,sha256=IOzGckKjQGTWC4w1Zqg12hN6bdzX2gClOZdgNCHdeyA,53639
31
+ instance_repo/clients/splits.py,sha256=sgipm2ShveXI516uUoC3fJpSQ6ZErlUOrPeu1xaTb0Q,29480
32
+ instance_repo/clients/versions.py,sha256=RIOsBPoHrSgXlFfVcjOdv4C84WNsROaJUBqVvHBm2dM,54377
33
33
  instance_repo/clients/workflows.py,sha256=6ftl1tuyEGx8BtMLSjfExP9_VcQhK4veo7ex1jDOdlc,3686
34
34
  instance_repo/store/__init__.py,sha256=fGwePNjKuojqx1yagSaY-svONE5pjaEz2BZmG0ISUZ0,674
35
35
  instance_repo/store/acr.py,sha256=OD7LH3GZY4HrrbEht86Mf3NfW_lh-8MhCTOGyX6gLT4,19468
36
36
  instance_repo/store/base.py,sha256=7feHdDXDvSkq60wXYvF6p8TK9KRj7YTLlESdgZ5MQHo,2622
37
- instance_repo/store/oss.py,sha256=-1dBPlUBLD0z-aI4GOuO6kwGyIgZX1zRvYCD-IPp-Zs,30672
38
- instance_repo/store/oss_discovery.py,sha256=gFZmHkpl1136hsXGL2CEBcHEbnpbTz3kAt2OZA71U-E,25894
37
+ instance_repo/store/oss.py,sha256=hrUIIOdVZD_Ck09k_7kBg8tBmOemfpSZetwQtBgpTYs,31516
38
+ instance_repo/store/oss_discovery.py,sha256=tzbwgYUVf9vnXCPRBA3vzzDIzaPmeqqiJ06GamTyBgg,28508
39
39
  instance_repo/store/registry.py,sha256=1R2GqXsuDZ8wc3IGpncF9yL6IHk-FeiwUlZN7v2ndSc,10837
40
- instance_repo-1.3.2.dev4.dist-info/METADATA,sha256=CY7IFFYrk54y2DSipSc5lSGMsFgGQfZiHfdh2ykx518,14227
41
- instance_repo-1.3.2.dev4.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
42
- instance_repo-1.3.2.dev4.dist-info/entry_points.txt,sha256=tn6jgwYChSx7eAqw9YNed_JFYYgoRtE7WkFn8xqa8Rw,53
43
- instance_repo-1.3.2.dev4.dist-info/top_level.txt,sha256=mblK0Bkwwp6jCZioS6q3sSuFFs8NR9DKTJ0DJpIRDmY,14
44
- instance_repo-1.3.2.dev4.dist-info/RECORD,,
40
+ instance_repo-1.3.3.dev2.dist-info/METADATA,sha256=6C4Oi51yH7xO4gC_CKN7ZGadOYJTT52Fw4eNMjua_gQ,14227
41
+ instance_repo-1.3.3.dev2.dist-info/WHEEL,sha256=SmOxYU7pzNKBqASvQJ7DjX3XGUF92lrGhMb3R6_iiqI,91
42
+ instance_repo-1.3.3.dev2.dist-info/entry_points.txt,sha256=tn6jgwYChSx7eAqw9YNed_JFYYgoRtE7WkFn8xqa8Rw,53
43
+ instance_repo-1.3.3.dev2.dist-info/top_level.txt,sha256=mblK0Bkwwp6jCZioS6q3sSuFFs8NR9DKTJ0DJpIRDmY,14
44
+ instance_repo-1.3.3.dev2.dist-info/RECORD,,