instance-repo 1.3.2.dev5__tar.gz → 1.3.3.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/PKG-INFO +2 -2
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/__init__.py +1 -1
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/datasets.py +16 -1
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/instances.py +76 -6
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/splits.py +48 -19
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/versions.py +20 -3
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/store/oss.py +9 -6
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/store/oss_discovery.py +73 -14
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/PKG-INFO +2 -2
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/requires.txt +1 -1
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/pyproject.toml +2 -2
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/README.md +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/_routing.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/_site_defaults.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/artifacts.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/bootstrap.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/cache.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/cli.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/__init__.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/benchmarks.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/config.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/images.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/reports.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/scaffold.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/clients/workflows.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/concurrency.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/content.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/endpoints.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/errors.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/image.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/layout.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/loader.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/models.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/paths.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/release.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/repo.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/retry.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/store/__init__.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/store/acr.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/store/base.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/store/registry.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/tasktoml.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/transport.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo/validate.py +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/SOURCES.txt +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/dependency_links.txt +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/entry_points.txt +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/top_level.txt +0 -0
- {instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/setup.cfg +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: instance-repo
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.3.dev2
|
|
4
4
|
Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
|
-
Requires-Dist: irepo-artifact<0.
|
|
7
|
+
Requires-Dist: irepo-artifact<0.3.0,>=0.1.1
|
|
8
8
|
Requires-Dist: tomli>=1.1.0; python_version < "3.11"
|
|
9
9
|
Provides-Extra: oss
|
|
10
10
|
Requires-Dist: oss2>=2.19; extra == "oss"
|
|
@@ -160,7 +160,8 @@ class DatasetsClient:
|
|
|
160
160
|
return DatasetDetail.from_dict(data) if isinstance(data, dict) else DatasetDetail()
|
|
161
161
|
except TransportError as e:
|
|
162
162
|
if (e.biz_code == 92001 and fallback_to_oss
|
|
163
|
-
and self._content is not None
|
|
163
|
+
and self._content is not None
|
|
164
|
+
and self._discovery_env_match(environment)):
|
|
164
165
|
return self._discover_detail(name, storage_env=environment,
|
|
165
166
|
metadata_model=metadata_model)
|
|
166
167
|
raise
|
|
@@ -396,6 +397,20 @@ class DatasetsClient:
|
|
|
396
397
|
|
|
397
398
|
# ── OSS fallback discovery (private) ─────────────────────────────────
|
|
398
399
|
|
|
400
|
+
def _discovery_env_match(self, environment: str | None) -> bool:
|
|
401
|
+
"""OSS 回退环境兼容性检查:请求的 environment 须与 profile storage_env 派生环境一致。"""
|
|
402
|
+
req = (environment or "").strip()
|
|
403
|
+
if not req:
|
|
404
|
+
return True
|
|
405
|
+
if self._p is None:
|
|
406
|
+
return True
|
|
407
|
+
from .config import dataset_environment_for
|
|
408
|
+
derived = dataset_environment_for(
|
|
409
|
+
getattr(self._p, "storage_env", ""))
|
|
410
|
+
if not derived:
|
|
411
|
+
return True
|
|
412
|
+
return req == derived
|
|
413
|
+
|
|
399
414
|
def _discover_detail(self, name: str, *,
|
|
400
415
|
storage_env: str | None = None,
|
|
401
416
|
metadata_model: str | None = None) -> DatasetDetail:
|
|
@@ -177,12 +177,14 @@ class InstancesClient:
|
|
|
177
177
|
except TransportError as e:
|
|
178
178
|
# 路径 1: dataset 不存在(92001) → OSS 回退
|
|
179
179
|
if (e.biz_code == 92001
|
|
180
|
-
and fallback_to_oss and self._content is not None
|
|
180
|
+
and fallback_to_oss and self._content is not None
|
|
181
|
+
and self._discovery_env_match(environment)):
|
|
181
182
|
return self._discover_instance(
|
|
182
183
|
dataset, version, instance_id, split)
|
|
183
184
|
raise
|
|
184
185
|
# 路径 2: instance 不存在(None, registry 吞了 92003) → OSS 回退
|
|
185
|
-
if result is None and fallback_to_oss and self._content is not None
|
|
186
|
+
if result is None and fallback_to_oss and self._content is not None \
|
|
187
|
+
and self._discovery_env_match(environment):
|
|
186
188
|
return self._discover_instance(
|
|
187
189
|
dataset, version, instance_id, split)
|
|
188
190
|
return result
|
|
@@ -203,7 +205,8 @@ class InstancesClient:
|
|
|
203
205
|
return self._reg.query(dataset, version, split, **overrides)
|
|
204
206
|
except TransportError as e:
|
|
205
207
|
if (e.biz_code == 92001
|
|
206
|
-
and fallback_to_oss and self._content is not None
|
|
208
|
+
and fallback_to_oss and self._content is not None
|
|
209
|
+
and self._discovery_env_match(environment)):
|
|
207
210
|
return self._discover_instances(dataset, version, split)
|
|
208
211
|
raise
|
|
209
212
|
|
|
@@ -281,7 +284,8 @@ class InstancesClient:
|
|
|
281
284
|
"GET", f"{self._BASE}?{query}")
|
|
282
285
|
except TransportError as e:
|
|
283
286
|
if (e.biz_code == 92001
|
|
284
|
-
and fallback_to_oss and self._content is not None
|
|
287
|
+
and fallback_to_oss and self._content is not None
|
|
288
|
+
and self._discovery_env_match(environment)):
|
|
285
289
|
return self._discover_instances_paged(
|
|
286
290
|
dataset, version, split,
|
|
287
291
|
q=q, instance_id=instance_id,
|
|
@@ -295,6 +299,20 @@ class InstancesClient:
|
|
|
295
299
|
|
|
296
300
|
# ── OSS 回退发现(未注册 dataset 的 instance 合成) ──
|
|
297
301
|
|
|
302
|
+
def _discovery_env_match(self, environment: str | None) -> bool:
|
|
303
|
+
"""OSS 回退环境兼容性检查:请求的 environment 须与 profile storage_env 派生环境一致。"""
|
|
304
|
+
req = (environment or "").strip()
|
|
305
|
+
if not req:
|
|
306
|
+
return True
|
|
307
|
+
if self._p is None:
|
|
308
|
+
return True
|
|
309
|
+
from .config import dataset_environment_for
|
|
310
|
+
derived = dataset_environment_for(
|
|
311
|
+
getattr(self._p, "storage_env", ""))
|
|
312
|
+
if not derived:
|
|
313
|
+
return True
|
|
314
|
+
return req == derived
|
|
315
|
+
|
|
298
316
|
def _discover_instances(self, dataset: str, version: str,
|
|
299
317
|
split: str | None = None) -> list[DatasetInstance]:
|
|
300
318
|
"""OSS 回退的 instances.list — 从 OSS 列举指定 version/split 下的实例。"""
|
|
@@ -850,7 +868,8 @@ class InstancesClient:
|
|
|
850
868
|
return results
|
|
851
869
|
|
|
852
870
|
def pull(self, dataset: str, version: str, instance_id: str, dest_dir,
|
|
853
|
-
split: str = "default", *, verify: bool = True
|
|
871
|
+
split: str = "default", *, verify: bool = True,
|
|
872
|
+
fallback_to_oss: bool = True) -> Path:
|
|
854
873
|
"""下载实例内容。自动检测格式:harbor 下载 content.tgz,swe 下载 {id}.json,
|
|
855
874
|
custom 下载 {id}.json + assets 目录。
|
|
856
875
|
|
|
@@ -860,6 +879,8 @@ class InstancesClient:
|
|
|
860
879
|
2. 无元数据或 manifest 无法区分 swe/custom → 下载 metadata JSON 检查
|
|
861
880
|
metadata_model 字段(custom → 下载 assets 目录;否则 swe)
|
|
862
881
|
3. metadata JSON 不存在 → fallback 到 harbor(下载 content.tgz)
|
|
882
|
+
4. dataset 未注册(92001)→ OSS 回退:list-only 枚举文件树 + 逐文件下载
|
|
883
|
+
(仍走 DOWNLOAD 意图:仅私网,公网下载照旧禁止)
|
|
863
884
|
"""
|
|
864
885
|
validate_instance_id(instance_id, source="pull()")
|
|
865
886
|
expected = None
|
|
@@ -869,7 +890,15 @@ class InstancesClient:
|
|
|
869
890
|
meta_bytes: bytes | None = None
|
|
870
891
|
|
|
871
892
|
if verify:
|
|
872
|
-
|
|
893
|
+
try:
|
|
894
|
+
meta = self._reg.get(dataset, version, instance_id, split)
|
|
895
|
+
except TransportError as e:
|
|
896
|
+
if (e.biz_code == 92001 and fallback_to_oss
|
|
897
|
+
and self._content is not None):
|
|
898
|
+
# dataset 未注册 → OSS 回退下载(私网 + 系统调用方语义)
|
|
899
|
+
return self._pull_unregistered(
|
|
900
|
+
dataset, version, instance_id, dest_dir, split)
|
|
901
|
+
raise
|
|
873
902
|
if meta is not None and meta.manifest:
|
|
874
903
|
has_tgz = False
|
|
875
904
|
for m in meta.manifest:
|
|
@@ -1044,6 +1073,47 @@ class InstancesClient:
|
|
|
1044
1073
|
downloaded.append(local_path)
|
|
1045
1074
|
return downloaded
|
|
1046
1075
|
|
|
1076
|
+
def _pull_unregistered(self, dataset: str, version: str,
|
|
1077
|
+
instance_id: str, dest_dir, split: str) -> Path:
|
|
1078
|
+
"""未注册 dataset 的 pull 回退:list-only 枚举文件树 + 逐文件下载。
|
|
1079
|
+
|
|
1080
|
+
下载仍走 :meth:`~instance_repo.store.oss.OssContentStore.get_object`
|
|
1081
|
+
的 DOWNLOAD 意图——**仅私网可用,公网下载照旧被拒**(与注册路径一致)。
|
|
1082
|
+
|
|
1083
|
+
STS 语义:未注册 dataset 无 RBAC 记录,GetObject STS 仅 system 白名单
|
|
1084
|
+
调用方可签发(非 system 调用方在签发阶段收到明确错误),与历史行为一致。
|
|
1085
|
+
|
|
1086
|
+
产物结构(harbor 同层,便于 CLI 整体搬移)::
|
|
1087
|
+
|
|
1088
|
+
{dest_dir}/{instance_id}/{instance_id}.json # metadata
|
|
1089
|
+
{dest_dir}/{instance_id}/<assets 相对路径> # assets 文件
|
|
1090
|
+
{dest_dir}/{instance_id}/report/<report 相对路径> # report 文件
|
|
1091
|
+
|
|
1092
|
+
Returns the metadata JSON 本地路径。
|
|
1093
|
+
"""
|
|
1094
|
+
from ..store.oss_discovery import discover_instance_file_tree
|
|
1095
|
+
|
|
1096
|
+
tree = discover_instance_file_tree(
|
|
1097
|
+
self._content, dataset, version, instance_id, split,
|
|
1098
|
+
oss_prefix=self._oss_prefix)
|
|
1099
|
+
if tree is None:
|
|
1100
|
+
from ..errors import DatasetNotFound
|
|
1101
|
+
raise DatasetNotFound(
|
|
1102
|
+
f"instance {instance_id!r} not found in OSS for "
|
|
1103
|
+
f"dataset {dataset!r}")
|
|
1104
|
+
|
|
1105
|
+
metadata_key, files = tree
|
|
1106
|
+
dest_root = Path(dest_dir) / instance_id
|
|
1107
|
+
dest_root.mkdir(parents=True, exist_ok=True)
|
|
1108
|
+
metadata_dest = dest_root / f"{instance_id}.json"
|
|
1109
|
+
|
|
1110
|
+
self._content.get_object(metadata_key, metadata_dest, dataset=dataset)
|
|
1111
|
+
for key, rel in files:
|
|
1112
|
+
local_path = dest_root / rel
|
|
1113
|
+
local_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1114
|
+
self._content.get_object(key, local_path, dataset=dataset)
|
|
1115
|
+
return metadata_dest
|
|
1116
|
+
|
|
1047
1117
|
def pull_many(self, instance_ids: list[str], dataset: str, version: str,
|
|
1048
1118
|
dest_dir, *, split: str = "default",
|
|
1049
1119
|
concurrency: int = 8, continue_on_error: bool = False,
|
|
@@ -103,7 +103,8 @@ class SplitsClient:
|
|
|
103
103
|
except TransportError as e:
|
|
104
104
|
# ── 2. 92001 异常 → 直接进入 OSS 回退 ──
|
|
105
105
|
if (e.biz_code == 92001
|
|
106
|
-
and fallback_to_oss and self._content is not None
|
|
106
|
+
and fallback_to_oss and self._content is not None
|
|
107
|
+
and self._discovery_env_match(environment)):
|
|
107
108
|
return self._discover_splits(
|
|
108
109
|
dataset, version, scope, status, page, page_size)
|
|
109
110
|
raise
|
|
@@ -114,7 +115,8 @@ class SplitsClient:
|
|
|
114
115
|
if not items and not (status or "").strip():
|
|
115
116
|
items = self._legacy_list(dataset, version, self._env(environment))
|
|
116
117
|
# ── 3b. 仍为空 → OSS 回退 ──
|
|
117
|
-
if not items and fallback_to_oss and self._content is not None
|
|
118
|
+
if not items and fallback_to_oss and self._content is not None \
|
|
119
|
+
and self._discovery_env_match(environment):
|
|
118
120
|
return self._discover_splits(
|
|
119
121
|
dataset, version, scope, status, page, page_size)
|
|
120
122
|
return PagedResult.from_response(items, pagination, int(page), int(page_size))
|
|
@@ -156,7 +158,8 @@ class SplitsClient:
|
|
|
156
158
|
if result is not None:
|
|
157
159
|
return result
|
|
158
160
|
# Legacy 回退无果 → OSS 回退
|
|
159
|
-
if fallback_to_oss and self._content is not None
|
|
161
|
+
if fallback_to_oss and self._content is not None \
|
|
162
|
+
and self._discovery_env_match(environment):
|
|
160
163
|
return self._discover_split(dataset, split, version)
|
|
161
164
|
return None
|
|
162
165
|
raise
|
|
@@ -261,11 +264,14 @@ class SplitsClient:
|
|
|
261
264
|
page: int, page_size: int) -> PagedResult:
|
|
262
265
|
"""OSS 回退的 splits.list — 处理 version_scope 和 status 过滤。
|
|
263
266
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
267
|
+
split_first 模式下 OSS 探测可区分无版本 split(level-1 目录只有 .json)
|
|
268
|
+
和有版本 split(level-1 目录含子目录),因此 ``version_scope`` 生效:
|
|
269
|
+
|
|
270
|
+
- ``"unversioned"``: 只返回无版本 split
|
|
271
|
+
- ``"versioned"``: 只返回有版本 split
|
|
272
|
+
- ``"all"`` / None: 返回全部(SDK 默认行为)
|
|
273
|
+
|
|
274
|
+
``version`` 非空时仍精确匹配该 version。
|
|
269
275
|
"""
|
|
270
276
|
from ..store.oss_discovery import (
|
|
271
277
|
parse_dataset_level, build_synthetic_split,
|
|
@@ -276,21 +282,23 @@ class SplitsClient:
|
|
|
276
282
|
metadata_model=self._discovery_mm())
|
|
277
283
|
_warn_if_truncated(info, dataset, 500)
|
|
278
284
|
splits: list[DatasetSplit] = []
|
|
285
|
+
scope = (version_scope or "all").strip().lower()
|
|
279
286
|
# split_first 无版本布局:dataset 根下直接是 split 目录
|
|
280
|
-
if not version:
|
|
287
|
+
if not version and scope in ("all", "unversioned"):
|
|
281
288
|
for us in info.unversioned_splits:
|
|
282
289
|
splits.append(build_synthetic_split(
|
|
283
290
|
dataset, "", us.name, instance_count=us.instance_count))
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
splits
|
|
293
|
-
|
|
291
|
+
if scope in ("all", "exact"):
|
|
292
|
+
for v in info.versions:
|
|
293
|
+
if version and v.version != version:
|
|
294
|
+
continue
|
|
295
|
+
if v.has_default_instances:
|
|
296
|
+
splits.append(build_synthetic_split(
|
|
297
|
+
dataset, v.version, "default",
|
|
298
|
+
instance_count=v.default_instance_count))
|
|
299
|
+
for s in v.splits:
|
|
300
|
+
splits.append(build_synthetic_split(
|
|
301
|
+
dataset, v.version, s))
|
|
294
302
|
if (status or "").strip():
|
|
295
303
|
splits = [s for s in splits if s.status == status.strip()]
|
|
296
304
|
return _in_memory_paged(splits, int(page), int(page_size))
|
|
@@ -329,6 +337,27 @@ class SplitsClient:
|
|
|
329
337
|
getattr(self._p, "metadata_model", None)
|
|
330
338
|
if self._p is not None else None)
|
|
331
339
|
|
|
340
|
+
def _discovery_env_match(self, environment: str | None) -> bool:
|
|
341
|
+
"""OSS 回退环境兼容性检查。
|
|
342
|
+
|
|
343
|
+
请求的 ``environment`` 必须与 profile 的 storage_env 派生环境一致,
|
|
344
|
+
否则 OSS 回退返回空——因为 bucket 是按 storage_env 绑定的,dev 桶里
|
|
345
|
+
的数据不应作为 online 结果返回(反之亦然)。
|
|
346
|
+
|
|
347
|
+
``environment`` 为空时视为兼容(跟随 profile 默认环境)。
|
|
348
|
+
"""
|
|
349
|
+
req = (environment or "").strip()
|
|
350
|
+
if not req:
|
|
351
|
+
return True
|
|
352
|
+
if self._p is None:
|
|
353
|
+
return True
|
|
354
|
+
from .config import dataset_environment_for
|
|
355
|
+
derived = dataset_environment_for(
|
|
356
|
+
getattr(self._p, "storage_env", ""))
|
|
357
|
+
if not derived:
|
|
358
|
+
return True # 无法派生时不做限制
|
|
359
|
+
return req == derived
|
|
360
|
+
|
|
332
361
|
# ── 写 ──
|
|
333
362
|
|
|
334
363
|
def _post_create(self, dataset: str, split: str, version: str,
|
|
@@ -228,7 +228,8 @@ class VersionsClient:
|
|
|
228
228
|
return [DatasetVersion(**_pick(r)) for r in (rows or [])]
|
|
229
229
|
except TransportError as e:
|
|
230
230
|
if (e.biz_code == 92001 and fallback_to_oss
|
|
231
|
-
and self._content is not None
|
|
231
|
+
and self._content is not None
|
|
232
|
+
and self._discovery_env_match(environment)):
|
|
232
233
|
return self._discover_versions(dataset, metadata_model=metadata_model)
|
|
233
234
|
raise
|
|
234
235
|
|
|
@@ -262,7 +263,8 @@ class VersionsClient:
|
|
|
262
263
|
data, pagination = self._t.request_with_meta("GET", f"{self._BASE}?{q}")
|
|
263
264
|
except TransportError as e:
|
|
264
265
|
if (e.biz_code == 92001 and fallback_to_oss
|
|
265
|
-
and self._content is not None
|
|
266
|
+
and self._content is not None
|
|
267
|
+
and self._discovery_env_match(environment)):
|
|
266
268
|
return self._discover_versions_paged(
|
|
267
269
|
dataset, status, int(page), int(page_size),
|
|
268
270
|
metadata_model=metadata_model)
|
|
@@ -295,7 +297,8 @@ class VersionsClient:
|
|
|
295
297
|
return DatasetVersion.from_dict_full(data) if isinstance(data, dict) else DatasetVersion(version=version, dataset=dataset)
|
|
296
298
|
except TransportError as e:
|
|
297
299
|
if (e.biz_code == 92001 and fallback_to_oss
|
|
298
|
-
and self._content is not None
|
|
300
|
+
and self._content is not None
|
|
301
|
+
and self._discovery_env_match(environment)):
|
|
299
302
|
return self._discover_version(dataset, version,
|
|
300
303
|
metadata_model=metadata_model)
|
|
301
304
|
raise
|
|
@@ -939,6 +942,20 @@ class VersionsClient:
|
|
|
939
942
|
getattr(self._p, "metadata_model", None)
|
|
940
943
|
if self._p is not None else None)
|
|
941
944
|
|
|
945
|
+
def _discovery_env_match(self, environment: str | None) -> bool:
|
|
946
|
+
"""OSS 回退环境兼容性检查:请求的 environment 须与 profile storage_env 派生环境一致。"""
|
|
947
|
+
req = (environment or "").strip()
|
|
948
|
+
if not req:
|
|
949
|
+
return True
|
|
950
|
+
if self._p is None:
|
|
951
|
+
return True
|
|
952
|
+
from .config import dataset_environment_for
|
|
953
|
+
derived = dataset_environment_for(
|
|
954
|
+
getattr(self._p, "storage_env", ""))
|
|
955
|
+
if not derived:
|
|
956
|
+
return True
|
|
957
|
+
return req == derived
|
|
958
|
+
|
|
942
959
|
|
|
943
960
|
# 与 apiserver `DatasetVersionListItem`(pkg/server/dataset_service.go:315)对齐:
|
|
944
961
|
# 列表项带 instance_count/split_count/updated_at/last_ingested_at,详情接口才带
|
|
@@ -467,7 +467,8 @@ class OssContentStore(BaseContentStore):
|
|
|
467
467
|
lambda bucket: bucket.object_exists(key),
|
|
468
468
|
)
|
|
469
469
|
|
|
470
|
-
def list_objects(self, prefix: str, *, list_only: bool = False
|
|
470
|
+
def list_objects(self, prefix: str, *, list_only: bool = False,
|
|
471
|
+
dataset: str | None = None) -> list[str]:
|
|
471
472
|
def list_all(bucket):
|
|
472
473
|
keys: list[str] = []
|
|
473
474
|
marker = ""
|
|
@@ -481,10 +482,11 @@ class OssContentStore(BaseContentStore):
|
|
|
481
482
|
return keys
|
|
482
483
|
|
|
483
484
|
return self._execute_oss(
|
|
484
|
-
_OperationIntent.NON_DOWNLOAD, prefix,
|
|
485
|
+
_OperationIntent.NON_DOWNLOAD, prefix, dataset, list_all,
|
|
485
486
|
list_only=list_only)
|
|
486
487
|
|
|
487
|
-
def list_prefixes(self, prefix: str, *, list_only: bool = False
|
|
488
|
+
def list_prefixes(self, prefix: str, *, list_only: bool = False,
|
|
489
|
+
dataset: str | None = None) -> list[str]:
|
|
488
490
|
"""List immediate child *directories* under *prefix* using ``delimiter='/'``.
|
|
489
491
|
|
|
490
492
|
Returns stripped directory names (without the parent prefix or trailing
|
|
@@ -515,11 +517,12 @@ class OssContentStore(BaseContentStore):
|
|
|
515
517
|
return names
|
|
516
518
|
|
|
517
519
|
return self._execute_oss(
|
|
518
|
-
_OperationIntent.NON_DOWNLOAD, prefix,
|
|
520
|
+
_OperationIntent.NON_DOWNLOAD, prefix, dataset, list_dirs,
|
|
519
521
|
list_only=list_only)
|
|
520
522
|
|
|
521
523
|
def list_level(self, prefix: str, *,
|
|
522
|
-
list_only: bool = False
|
|
524
|
+
list_only: bool = False,
|
|
525
|
+
dataset: str | None = None) -> tuple[list[str], list[str]]:
|
|
523
526
|
"""Single-level listing with ``delimiter='/'``.
|
|
524
527
|
|
|
525
528
|
Returns ``(dir_names, file_keys)`` where *dir_names* are stripped
|
|
@@ -549,7 +552,7 @@ class OssContentStore(BaseContentStore):
|
|
|
549
552
|
return names, keys
|
|
550
553
|
|
|
551
554
|
return self._execute_oss(
|
|
552
|
-
_OperationIntent.NON_DOWNLOAD, prefix,
|
|
555
|
+
_OperationIntent.NON_DOWNLOAD, prefix, dataset, list_one_level,
|
|
553
556
|
list_only=list_only)
|
|
554
557
|
|
|
555
558
|
# ── directory-level helpers(自定义格式实例 = 目录级上传/下载)──
|
|
@@ -235,7 +235,7 @@ def parse_dataset_level(
|
|
|
235
235
|
# -- Level 1: list immediate children under the dataset root -----------
|
|
236
236
|
try:
|
|
237
237
|
level1_names, level1_all_keys = content_store.list_level(
|
|
238
|
-
root, list_only=True)
|
|
238
|
+
root, list_only=True, dataset=dataset)
|
|
239
239
|
except TransportError as e:
|
|
240
240
|
if e.status == 403:
|
|
241
241
|
# OSS fallback requires ListObjects permission, which is only granted
|
|
@@ -293,7 +293,7 @@ def parse_dataset_level(
|
|
|
293
293
|
group_dir = f"{root}{name}/"
|
|
294
294
|
try:
|
|
295
295
|
level2_names, level2_keys = content_store.list_level(
|
|
296
|
-
group_dir, list_only=True)
|
|
296
|
+
group_dir, list_only=True, dataset=dataset)
|
|
297
297
|
except Exception: # noqa: BLE001
|
|
298
298
|
level2_names, level2_keys = [], []
|
|
299
299
|
l2_groups = _group_level_names(level2_names)
|
|
@@ -316,7 +316,7 @@ def parse_dataset_level(
|
|
|
316
316
|
split_dir = f"{group_dir}{s_name}/"
|
|
317
317
|
try:
|
|
318
318
|
_s_dirs, s_keys = content_store.list_level(
|
|
319
|
-
split_dir, list_only=True)
|
|
319
|
+
split_dir, list_only=True, dataset=dataset)
|
|
320
320
|
instance_count += len(_direct_json_keys(s_keys, split_dir))
|
|
321
321
|
except Exception: # noqa: BLE001
|
|
322
322
|
pass # best-effort
|
|
@@ -540,7 +540,8 @@ def build_synthetic_instances(
|
|
|
540
540
|
"""
|
|
541
541
|
metadata_prefix = _metadata_prefix(dataset, version, split, oss_prefix)
|
|
542
542
|
|
|
543
|
-
all_keys = content_store.list_objects(
|
|
543
|
+
all_keys = content_store.list_objects(
|
|
544
|
+
metadata_prefix, list_only=True, dataset=dataset)
|
|
544
545
|
json_keys = _direct_json_keys(all_keys, metadata_prefix)
|
|
545
546
|
|
|
546
547
|
instances: list[DatasetInstance] = []
|
|
@@ -585,6 +586,62 @@ def _format_from_key(key: str) -> str:
|
|
|
585
586
|
return ""
|
|
586
587
|
|
|
587
588
|
|
|
589
|
+
def discover_instance_file_tree(
|
|
590
|
+
content_store,
|
|
591
|
+
dataset: str,
|
|
592
|
+
version: str,
|
|
593
|
+
instance_id: str,
|
|
594
|
+
split: str | None,
|
|
595
|
+
*,
|
|
596
|
+
oss_prefix: str,
|
|
597
|
+
) -> tuple[str, list[tuple[str, str]]] | None:
|
|
598
|
+
"""用 LIST 枚举实例的完整文件树(发现模式,list-only STS)。
|
|
599
|
+
|
|
600
|
+
Returns ``(metadata_key, files)`` where *files* is a list of
|
|
601
|
+
``(oss_key, rel_path)`` tuples covering ``{split}-assets/{id}/`` 与
|
|
602
|
+
``{split}-report/{id}/`` 下的所有对象。*rel_path* 相对实例根(assets
|
|
603
|
+
文件为原相对路径,report 文件加 ``report/`` 前缀避免与 assets 同名冲突)。
|
|
604
|
+
|
|
605
|
+
实例不存在(metadata .json 的精确 key LIST 未命中)时返回 ``None``。
|
|
606
|
+
"""
|
|
607
|
+
if version:
|
|
608
|
+
metadata_key = P.metadata_key(
|
|
609
|
+
dataset, version, instance_id, split, prefix=oss_prefix)
|
|
610
|
+
else:
|
|
611
|
+
l1, l2 = P.split_dataset(dataset)
|
|
612
|
+
base = f"{oss_prefix}/{l1}/{l2}"
|
|
613
|
+
metadata_key = (f"{base}/{split}/{instance_id}.json"
|
|
614
|
+
if split and split != "default"
|
|
615
|
+
else f"{base}/{instance_id}.json")
|
|
616
|
+
|
|
617
|
+
# 存在性判定用 LIST(list-only STS 无 GetObject,不能 HEAD)。
|
|
618
|
+
# 不能用「prefix=精确 key」的 LIST:OSS 的 prefix 过滤是字符串前缀语义,
|
|
619
|
+
# 且 RAM 条件只放行 {parent}/*(精确 key 会带出同前缀兄弟对象,见 apiserver
|
|
620
|
+
# sts/client.go),改为父目录 LIST + 客户端成员判断。
|
|
621
|
+
parent = metadata_key.rsplit("/", 1)[0] + "/"
|
|
622
|
+
try:
|
|
623
|
+
if metadata_key not in content_store.list_objects(
|
|
624
|
+
parent, list_only=True, dataset=dataset):
|
|
625
|
+
return None
|
|
626
|
+
except Exception: # noqa: BLE001
|
|
627
|
+
return None
|
|
628
|
+
|
|
629
|
+
files: list[tuple[str, str]] = []
|
|
630
|
+
for suffix, rel_prefix in (("assets", ""), ("report", "report/")):
|
|
631
|
+
dir_prefix = _instance_dir(dataset, version, split, instance_id,
|
|
632
|
+
suffix, oss_prefix)
|
|
633
|
+
try:
|
|
634
|
+
for k in content_store.list_objects(
|
|
635
|
+
dir_prefix, list_only=True, dataset=dataset):
|
|
636
|
+
if k.endswith("/") or not k.startswith(dir_prefix):
|
|
637
|
+
continue # 目录标记 / 异常 key
|
|
638
|
+
rel = rel_prefix + k[len(dir_prefix):]
|
|
639
|
+
files.append((k, rel))
|
|
640
|
+
except Exception: # noqa: BLE001
|
|
641
|
+
pass # best-effort:无 assets/report 目录时忽略
|
|
642
|
+
return metadata_key, files
|
|
643
|
+
|
|
644
|
+
|
|
588
645
|
def _instance_manifest(
|
|
589
646
|
content_store,
|
|
590
647
|
dataset: str,
|
|
@@ -600,15 +657,13 @@ def _instance_manifest(
|
|
|
600
657
|
发现模式(list-only STS)无 GetObject,读不到 metadata JSON 内容,但目录
|
|
601
658
|
结构可通过 LIST 完整呈现——恰好满足「查看实例的目录文件结构」需求。
|
|
602
659
|
"""
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
except Exception: # noqa: BLE001
|
|
611
|
-
pass # best-effort:无 assets/report 目录时忽略
|
|
660
|
+
tree = discover_instance_file_tree(
|
|
661
|
+
content_store, dataset, version, instance_id, split,
|
|
662
|
+
oss_prefix=oss_prefix)
|
|
663
|
+
if tree is None:
|
|
664
|
+
return []
|
|
665
|
+
_meta_key, files = tree
|
|
666
|
+
keys = [_meta_key] + [k for k, _ in files]
|
|
612
667
|
bucket = getattr(content_store, "bucket_name", "") or ""
|
|
613
668
|
return [
|
|
614
669
|
ManifestEntry(
|
|
@@ -657,8 +712,12 @@ def build_synthetic_instance(
|
|
|
657
712
|
# 存在性判定用 LIST(list_only=True 仅需 ListObjects)而非 HEAD——
|
|
658
713
|
# list-only STS 无 oss:GetObject,object_exists 的 HEAD 会在 STS 签发或
|
|
659
714
|
# OSS 侧被拒,导致已存在的实例恒被误判为不存在。
|
|
715
|
+
# 不能用「prefix=精确 key」的 LIST(OSS 字符串前缀语义会带出兄弟对象,
|
|
716
|
+
# RAM 条件也只放行父目录形态),改父目录 LIST + 客户端成员判断。
|
|
717
|
+
parent = key.rsplit("/", 1)[0] + "/"
|
|
660
718
|
try:
|
|
661
|
-
keys = content_store.list_objects(
|
|
719
|
+
keys = content_store.list_objects(
|
|
720
|
+
parent, list_only=True, dataset=dataset)
|
|
662
721
|
except Exception: # noqa: BLE001
|
|
663
722
|
return None
|
|
664
723
|
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: instance-repo
|
|
3
|
-
Version: 1.3.
|
|
3
|
+
Version: 1.3.3.dev2
|
|
4
4
|
Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
|
-
Requires-Dist: irepo-artifact<0.
|
|
7
|
+
Requires-Dist: irepo-artifact<0.3.0,>=0.1.1
|
|
8
8
|
Requires-Dist: tomli>=1.1.0; python_version < "3.11"
|
|
9
9
|
Provides-Extra: oss
|
|
10
10
|
Requires-Dist: oss2>=2.19; extra == "oss"
|
|
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "instance-repo"
|
|
7
|
-
version = "1.3.
|
|
7
|
+
version = "1.3.3.dev2"
|
|
8
8
|
description = "InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)"
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
# tomllib 是 3.11 内置;3.10 上回退到 tomli(见 validate.py),故按 Python 版本条件依赖。
|
|
12
12
|
dependencies = [
|
|
13
|
-
"irepo-artifact>=0.1.1,<0.
|
|
13
|
+
"irepo-artifact>=0.1.1,<0.3.0",
|
|
14
14
|
"tomli>=1.1.0; python_version < '3.11'",
|
|
15
15
|
]
|
|
16
16
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{instance_repo-1.3.2.dev5 → instance_repo-1.3.3.dev2}/instance_repo.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|