instance-repo 1.0.9.dev0__tar.gz → 1.0.9.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/PKG-INFO +1 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/__init__.py +1 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/cli.py +17 -7
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/datasets.py +4 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/instances.py +29 -4
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/splits.py +156 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/versions.py +10 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/models.py +32 -2
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/store/registry.py +25 -10
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/PKG-INFO +1 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/pyproject.toml +1 -1
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/README.md +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/_routing.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/_site_defaults.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/bootstrap.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/cache.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/__init__.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/benchmarks.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/config.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/images.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/reports.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/scaffold.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/clients/workflows.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/concurrency.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/content.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/endpoints.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/errors.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/image.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/layout.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/loader.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/paths.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/release.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/repo.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/retry.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/store/__init__.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/store/acr.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/store/base.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/store/oss.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/tasktoml.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/transport.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo/validate.py +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/SOURCES.txt +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/dependency_links.txt +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/entry_points.txt +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/requires.txt +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/top_level.txt +0 -0
- {instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/setup.cfg +0 -0
|
@@ -912,9 +912,11 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
912
912
|
status=args.status,
|
|
913
913
|
page=args.page, page_size=args.page_size)
|
|
914
914
|
for s in res.items:
|
|
915
|
+
src = "projected" if s.projected else "record"
|
|
915
916
|
print(f"{s.name}\tversion={s.version or ''}\tstatus={s.status}"
|
|
916
917
|
f"\trun_type={s.run_type or ''}\t"
|
|
917
|
-
f"instance_count={_split_instance_count(s)}",
|
|
918
|
+
f"instance_count={_split_instance_count(s)}\t[{src}]",
|
|
919
|
+
file=out)
|
|
918
920
|
print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
|
|
919
921
|
return 0
|
|
920
922
|
# get / set-run-type:需要定位到具体 split
|
|
@@ -923,8 +925,10 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
923
925
|
sp = r.splits.get(dataset, split_name, version=args.version)
|
|
924
926
|
if sp is None:
|
|
925
927
|
raise SystemExit(f"split {dataset}/{split_name} not found")
|
|
928
|
+
src = "projected" if sp.projected else "record"
|
|
926
929
|
print(f"split\t{sp.name}\tversion={sp.version or ''}"
|
|
927
|
-
f"\tstatus={sp.status}\trun_type={sp.run_type or ''}"
|
|
930
|
+
f"\tstatus={sp.status}\trun_type={sp.run_type or ''}"
|
|
931
|
+
f"\t[{src}]", file=out)
|
|
928
932
|
print(f"storage_path={sp.storage_path}", file=out)
|
|
929
933
|
print(f"published_at={sp.published_at or ''}"
|
|
930
934
|
f"\tlast_ingested_at={sp.last_ingested_at or ''}", file=out)
|
|
@@ -995,14 +999,20 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
995
999
|
return 0
|
|
996
1000
|
if args.action in ("access", "revoke"):
|
|
997
1001
|
if args.action == "access":
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1002
|
+
# 自动翻页取全量(服务端默认 20 会漏绑定量大的 dataset)
|
|
1003
|
+
page_n, all_items = 1, []
|
|
1004
|
+
while True:
|
|
1005
|
+
res = r.datasets.list_access(dataset,
|
|
1006
|
+
page=page_n, page_size=200)
|
|
1007
|
+
all_items.extend(res.items)
|
|
1008
|
+
if not res.has_more or not res.items:
|
|
1009
|
+
break
|
|
1010
|
+
page_n += 1
|
|
1011
|
+
for rb in all_items:
|
|
1001
1012
|
print(f"{rb.principal_type}\t{rb.principal_id}\t{rb.role_id}"
|
|
1002
1013
|
f"\tadded_by={rb.added_by}\tadded_at={rb.added_at}",
|
|
1003
1014
|
file=out)
|
|
1004
|
-
print(f"
|
|
1005
|
-
file=out)
|
|
1015
|
+
print(f"total={len(all_items)}", file=out)
|
|
1006
1016
|
return 0
|
|
1007
1017
|
if not (args.principal and args.role):
|
|
1008
1018
|
raise SystemExit("dataset revoke 需要 --principal + --role")
|
|
@@ -372,13 +372,16 @@ class DatasetsClient:
|
|
|
372
372
|
f"{self._BASE}/{quote(dataset_id, safe='')}/claim", body)
|
|
373
373
|
|
|
374
374
|
|
|
375
|
+
# 与 apiserver `DatasetListItem`(pkg/server/dataset_service.go:231)逐字段对齐。
|
|
375
376
|
_SERIES_FIELDS = ("dataset_name", "dataset_l1", "dataset_l2", "dataset_id",
|
|
376
377
|
"teacher_direction", "teacher_type", "benchmark_id", "visibility",
|
|
377
378
|
"owner_team_id", "display_name", "description", "default_scaffold",
|
|
378
|
-
|
|
379
|
+
|
|
380
|
+
"owner", "status", "version_count", "split_count", "latest_version",
|
|
379
381
|
"claimable", "updated_at", "available_envs")
|
|
380
382
|
|
|
381
383
|
|
|
384
|
+
|
|
382
385
|
# TEACHER_ENUMS 是 teacher 级联标签(teacher_direction → 允许的 teacher_type 列表)的
|
|
383
386
|
# **参考快照**,供本地填参/校验提示使用。
|
|
384
387
|
#
|
|
@@ -28,6 +28,17 @@ from ..validate import (check_layout, check_swe_json, check_custom,
|
|
|
28
28
|
logger = logging.getLogger(__name__)
|
|
29
29
|
|
|
30
30
|
|
|
31
|
+
def _registry_overrides(environment: str | None,
|
|
32
|
+
metadata_model: str | None) -> dict:
|
|
33
|
+
"""把显式指定的 environment / metadata_model 变成注册表调用参数(缺省不下发)。"""
|
|
34
|
+
overrides: dict = {}
|
|
35
|
+
if environment is not None:
|
|
36
|
+
overrides["environment"] = environment
|
|
37
|
+
if metadata_model is not None:
|
|
38
|
+
overrides["metadata_model"] = metadata_model
|
|
39
|
+
return overrides
|
|
40
|
+
|
|
41
|
+
|
|
31
42
|
class InstancesClient:
|
|
32
43
|
# registry 的 BASE 同值;此处独立声明使 list_paged/update 等纯控制面方法
|
|
33
44
|
# 不依赖注入的 registry(直构客户端亦可调用)。
|
|
@@ -81,12 +92,26 @@ class InstancesClient:
|
|
|
81
92
|
return check_layout(local_path)
|
|
82
93
|
|
|
83
94
|
def get(self, dataset: str, version: str, instance_id: str,
|
|
84
|
-
split: str | None = None
|
|
85
|
-
|
|
95
|
+
split: str | None = None, *,
|
|
96
|
+
environment: str | None = None,
|
|
97
|
+
metadata_model: str | None = None) -> DatasetInstance | None:
|
|
98
|
+
"""取单个实例;不存在返回 None。
|
|
99
|
+
|
|
100
|
+
``environment`` / ``metadata_model`` 传 None 用 profile 派生的值(默认行为
|
|
101
|
+
不变);显式传值可覆盖本次调用——CLI 需要读 online 集合时不必重建 Repo。
|
|
102
|
+
"""
|
|
103
|
+
# 只在调用方显式指定时才下发覆盖参数:注册表是可注入实现,
|
|
104
|
+
# 保持缺省调用形态与历史一致,避免破坏既有实现。
|
|
105
|
+
overrides = _registry_overrides(environment, metadata_model)
|
|
106
|
+
return self._reg.get(dataset, version, instance_id, split, **overrides)
|
|
86
107
|
|
|
87
108
|
def list(self, dataset: str, version: str,
|
|
88
|
-
split: str | None = None
|
|
89
|
-
|
|
109
|
+
split: str | None = None, *,
|
|
110
|
+
environment: str | None = None,
|
|
111
|
+
metadata_model: str | None = None) -> list[DatasetInstance]:
|
|
112
|
+
"""自动翻页取全量;``environment`` / ``metadata_model`` 语义同 :meth:`get`。"""
|
|
113
|
+
overrides = _registry_overrides(environment, metadata_model)
|
|
114
|
+
return self._reg.query(dataset, version, split, **overrides)
|
|
90
115
|
|
|
91
116
|
def list_paged(self, dataset: str, version: str = "", *,
|
|
92
117
|
split: str | None = None, q: str | None = None,
|
|
@@ -88,6 +88,11 @@ class SplitsClient:
|
|
|
88
88
|
rows = data.get("splits", data) if isinstance(data, dict) else data
|
|
89
89
|
rows = rows if isinstance(rows, list) else []
|
|
90
90
|
items = [DatasetSplit.from_dict(r) for r in rows if isinstance(r, dict)]
|
|
91
|
+
# 存量数据回退:split-first 集合无记录(version_first 存量)时,聚合
|
|
92
|
+
# version 内嵌 splits[] 投影。仅当直查**零记录**且未按 status 过滤时
|
|
93
|
+
# 触发(投影无 status,过滤必为空,回退无意义)。
|
|
94
|
+
if not items and not (status or "").strip():
|
|
95
|
+
items = self._legacy_list(dataset, version, self._env(environment))
|
|
91
96
|
return PagedResult.from_response(items, pagination, int(page), int(page_size))
|
|
92
97
|
|
|
93
98
|
def get(self, dataset: str, split: str, *, version: str | None = None,
|
|
@@ -97,6 +102,12 @@ class SplitsClient:
|
|
|
97
102
|
只吞 HTTP 404(服务端 split 不存在);dataset 名拼错等服务端也按 404 报,
|
|
98
103
|
此处不区分(split detail 端点无独立业务码可判——与 instances.get 的
|
|
99
104
|
92003/92001 二分不同,这里统一 None,调用方拿不准时用 datasets.get 核对)。
|
|
105
|
+
|
|
106
|
+
**存量数据回退(1.0.10 真机反馈)**:split 记录只存在于 split_first
|
|
107
|
+
新数据(split-first 集合);存量 version_first 数据的 split 内嵌在
|
|
108
|
+
version 文档 ``splits[]`` 里,split detail 端点 404。此时回退到
|
|
109
|
+
versions.get 的内嵌投影(``projected=True``,无 status/published_at,
|
|
110
|
+
storage_path 取自 oss_prefix);仍找不到才返回 None。
|
|
100
111
|
"""
|
|
101
112
|
if not (dataset or "").strip():
|
|
102
113
|
raise SchemaError("dataset is required for splits.get")
|
|
@@ -113,10 +124,102 @@ class SplitsClient:
|
|
|
113
124
|
data = self._t.get(f"{self._BASE}/detail?{q}")
|
|
114
125
|
except TransportError as e:
|
|
115
126
|
if e.status == 404:
|
|
116
|
-
return
|
|
127
|
+
return self._legacy_get(dataset, split, version, env)
|
|
117
128
|
raise
|
|
118
129
|
return DatasetSplit.from_dict(data) if isinstance(data, dict) else None
|
|
119
130
|
|
|
131
|
+
# ── 存量数据兼容投影(version_first 的 split 内嵌在 version 文档里)──
|
|
132
|
+
|
|
133
|
+
def _legacy_get(self, dataset: str, split: str, version: str | None,
|
|
134
|
+
env: str) -> DatasetSplit | None:
|
|
135
|
+
"""split detail 404 后的回退:读 version 内嵌 splits[] 投影。
|
|
136
|
+
|
|
137
|
+
有 version 时单查该 version;无 version 时扫描全部 version(分页)
|
|
138
|
+
直到命中。返回 projected=True 的 DatasetSplit;仍无则 None。
|
|
139
|
+
TransportError 按 status 区分:404 重试/继续(version 不存在/实例
|
|
140
|
+
未找到属正常),5xx 与连接错误重抛——不把网络故障伪装成"数据不存在"。
|
|
141
|
+
"""
|
|
142
|
+
from .versions import VersionsClient
|
|
143
|
+
vc = VersionsClient(self._t, self._p)
|
|
144
|
+
if (version or "").strip():
|
|
145
|
+
try:
|
|
146
|
+
detail = vc.get(dataset, version, environment=env or None)
|
|
147
|
+
except TransportError as e:
|
|
148
|
+
if e.status == 404:
|
|
149
|
+
return None # version 不存在 → 回退无果
|
|
150
|
+
raise
|
|
151
|
+
return _project_embedded_split(detail, split, version, dataset)
|
|
152
|
+
# 无 version:扫描全部 version
|
|
153
|
+
page = 1
|
|
154
|
+
while True:
|
|
155
|
+
try:
|
|
156
|
+
res = vc.list_paged(dataset, page=page, page_size=200,
|
|
157
|
+
environment=env or None)
|
|
158
|
+
except TransportError as e:
|
|
159
|
+
if e.status == 404:
|
|
160
|
+
return None # dataset 不存在 → 终止扫描
|
|
161
|
+
raise
|
|
162
|
+
for item in res.items:
|
|
163
|
+
try:
|
|
164
|
+
detail = vc.get(dataset, item.version,
|
|
165
|
+
environment=env or None)
|
|
166
|
+
except TransportError as e:
|
|
167
|
+
if e.status == 404:
|
|
168
|
+
continue # 该 version 不存在,继续找下一个
|
|
169
|
+
raise
|
|
170
|
+
found = _project_embedded_split(detail, split,
|
|
171
|
+
item.version, dataset)
|
|
172
|
+
if found is not None:
|
|
173
|
+
return found
|
|
174
|
+
if not res.has_more or not res.items:
|
|
175
|
+
break
|
|
176
|
+
page += 1
|
|
177
|
+
return None
|
|
178
|
+
|
|
179
|
+
def _legacy_list(self, dataset: str, version: str | None, env: str
|
|
180
|
+
) -> list[DatasetSplit]:
|
|
181
|
+
"""split 集合无记录时的回退:聚合全部 version 的内嵌 splits[] 投影。
|
|
182
|
+
|
|
183
|
+
投影项无 status/published_at(内嵌字段集只有 name/run_type/oss_prefix/
|
|
184
|
+
is_default/instance_count 等),因此 status 过滤对存量数据天然为空——
|
|
185
|
+
如实返回空列表而非伪造状态。
|
|
186
|
+
TransportError 按 status 区分:404→跳过/终止;其他重抛。
|
|
187
|
+
"""
|
|
188
|
+
from .versions import VersionsClient
|
|
189
|
+
vc = VersionsClient(self._t, self._p)
|
|
190
|
+
out: list[DatasetSplit] = []
|
|
191
|
+
if (version or "").strip():
|
|
192
|
+
try:
|
|
193
|
+
detail = vc.get(dataset, version, environment=env or None)
|
|
194
|
+
except TransportError as e:
|
|
195
|
+
if e.status == 404:
|
|
196
|
+
return out # version 不存在,无数据可投影
|
|
197
|
+
raise
|
|
198
|
+
return _project_all_embedded_splits(detail, version, dataset)
|
|
199
|
+
page = 1
|
|
200
|
+
while True:
|
|
201
|
+
try:
|
|
202
|
+
res = vc.list_paged(dataset, page=page, page_size=200,
|
|
203
|
+
environment=env or None)
|
|
204
|
+
except TransportError as e:
|
|
205
|
+
if e.status == 404:
|
|
206
|
+
break # dataset 不存在,终止扫描
|
|
207
|
+
raise
|
|
208
|
+
for item in res.items:
|
|
209
|
+
try:
|
|
210
|
+
detail = vc.get(dataset, item.version,
|
|
211
|
+
environment=env or None)
|
|
212
|
+
except TransportError as e:
|
|
213
|
+
if e.status == 404:
|
|
214
|
+
continue # 该 version 不存在
|
|
215
|
+
raise
|
|
216
|
+
out.extend(_project_all_embedded_splits(
|
|
217
|
+
detail, item.version, dataset))
|
|
218
|
+
if not res.has_more or not res.items:
|
|
219
|
+
break
|
|
220
|
+
page += 1
|
|
221
|
+
return out
|
|
222
|
+
|
|
120
223
|
# ── 写 ──
|
|
121
224
|
|
|
122
225
|
def update(self, dataset: str, split: str | None = None, *,
|
|
@@ -168,3 +271,55 @@ def normalize_split_version_scope(version: str | None,
|
|
|
168
271
|
f"{', '.join(VERSION_SCOPE_VALUES)}")
|
|
169
272
|
return scope
|
|
170
273
|
return "all" if not (version or "").strip() else "exact"
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
# ── 存量数据内嵌 splits[] 投影 ────────────────────────────────────────────────
|
|
277
|
+
|
|
278
|
+
def _embedded_splits_of(detail) -> list:
|
|
279
|
+
"""取 version 详情的内嵌 splits[](兼容 dict 与 DatasetVersion 对象)。"""
|
|
280
|
+
if detail is None:
|
|
281
|
+
return []
|
|
282
|
+
if isinstance(detail, dict):
|
|
283
|
+
return detail.get("splits") or []
|
|
284
|
+
return getattr(detail, "splits", None) or []
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _project_embedded_split(detail, split: str, version: str,
|
|
288
|
+
dataset: str) -> "DatasetSplit | None":
|
|
289
|
+
"""从 versions/detail 响应投影单个 split(projected=True)。
|
|
290
|
+
|
|
291
|
+
内嵌字段集:{name, run_type?, oss_prefix, is_default, instance_count?,
|
|
292
|
+
file_count?, size_bytes?}——无 status/published_at/storage_path。
|
|
293
|
+
"""
|
|
294
|
+
for sp in _embedded_splits_of(detail):
|
|
295
|
+
if not isinstance(sp, dict) or sp.get("name") != split:
|
|
296
|
+
continue
|
|
297
|
+
return _build_projected_split(sp, version, dataset)
|
|
298
|
+
return None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _project_all_embedded_splits(detail, version: str,
|
|
302
|
+
dataset: str) -> list["DatasetSplit"]:
|
|
303
|
+
"""从 versions/detail 响应投影全部内嵌 split(projected=True)。"""
|
|
304
|
+
return [_build_projected_split(sp, version, dataset)
|
|
305
|
+
for sp in _embedded_splits_of(detail) if isinstance(sp, dict)]
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _build_projected_split(sp: dict, version: str,
|
|
309
|
+
dataset: str) -> "DatasetSplit":
|
|
310
|
+
from ..models import SplitManifest
|
|
311
|
+
manifest = SplitManifest(
|
|
312
|
+
instance_count=sp.get("instance_count"),
|
|
313
|
+
file_count=sp.get("file_count"),
|
|
314
|
+
size_bytes=sp.get("size_bytes"))
|
|
315
|
+
return DatasetSplit(
|
|
316
|
+
dataset=dataset,
|
|
317
|
+
dataset_id="",
|
|
318
|
+
dataset_version_id=None, # 内嵌投影拿不到 version 记录 id
|
|
319
|
+
version=version,
|
|
320
|
+
name=sp.get("name", ""),
|
|
321
|
+
run_type=sp.get("run_type"),
|
|
322
|
+
status="", # 内嵌无 status
|
|
323
|
+
storage_path=sp.get("oss_prefix", ""),
|
|
324
|
+
manifest=manifest,
|
|
325
|
+
projected=True)
|
|
@@ -799,8 +799,14 @@ class VersionsClient:
|
|
|
799
799
|
return last
|
|
800
800
|
|
|
801
801
|
|
|
802
|
+
# 与 apiserver `DatasetVersionListItem`(pkg/server/dataset_service.go:315)对齐:
|
|
803
|
+
# 列表项带 instance_count/split_count/updated_at/last_ingested_at,详情接口才带
|
|
804
|
+
# splits/manifest/run_type/created_by/published_at —— 两者都收进模型。
|
|
802
805
|
_FIELDS = ("version", "dataset", "status", "dataset_version_id",
|
|
803
|
-
|
|
806
|
+
|
|
807
|
+
"storage_path", "published_at",
|
|
808
|
+
"instance_count", "split_count", "last_ingested_at", "updated_at")
|
|
809
|
+
|
|
804
810
|
|
|
805
811
|
|
|
806
812
|
def _pick(d: dict) -> dict:
|
|
@@ -808,4 +814,7 @@ def _pick(d: dict) -> dict:
|
|
|
808
814
|
out.setdefault("version", d.get("version", ""))
|
|
809
815
|
out.setdefault("dataset", d.get("dataset", ""))
|
|
810
816
|
out.setdefault("status", d.get("status", "draft"))
|
|
817
|
+
# 列表项**不带** storage_type,「缺省即 oss」的模型默认值会让调用方以为服务端
|
|
818
|
+
# 返回了它(CLI 原样透出服务端响应,凭空多字段就是契约变更)。这里显式给 None。
|
|
819
|
+
out.setdefault("storage_type", d.get("storage_type"))
|
|
811
820
|
return out
|
|
@@ -80,12 +80,17 @@ class ManifestEntry:
|
|
|
80
80
|
content_type: str = "file"
|
|
81
81
|
content_format: str = "json"
|
|
82
82
|
content_digest: str | None = None
|
|
83
|
+
#: 服务端返回的对象相对路径(`GET /datasets/instances` 的 manifest 项带
|
|
84
|
+
#: `object_path`);本地扫描出的条目为 None,不进 to_dict,保证推送载荷不变。
|
|
85
|
+
object_path: str | None = None
|
|
83
86
|
|
|
84
87
|
def to_dict(self) -> dict:
|
|
85
88
|
d = {"file_path": self.file_path, "content_type": self.content_type,
|
|
86
89
|
"content_format": self.content_format}
|
|
87
90
|
if self.content_digest is not None:
|
|
88
91
|
d["content_digest"] = self.content_digest
|
|
92
|
+
if self.object_path is not None:
|
|
93
|
+
d["object_path"] = self.object_path
|
|
89
94
|
return d
|
|
90
95
|
|
|
91
96
|
|
|
@@ -117,6 +122,9 @@ class DatasetSeries:
|
|
|
117
122
|
claimable: bool | None = None
|
|
118
123
|
updated_at: str | None = None
|
|
119
124
|
available_envs: list[str] = field(default_factory=list)
|
|
125
|
+
# 服务端 `DatasetSeriesStatus`("" 正常 / "deprecated");列表项
|
|
126
|
+
# `DatasetListItem.Status` 带 `omitempty`,正常态不出现。
|
|
127
|
+
# 与 class 开头 status 字段合并到此位置。
|
|
120
128
|
|
|
121
129
|
def validate(self) -> None:
|
|
122
130
|
if self.visibility not in VISIBILITY_VALUES:
|
|
@@ -163,6 +171,11 @@ class DatasetVersion:
|
|
|
163
171
|
updated_at: str | None = None
|
|
164
172
|
parent_shell: bool = False
|
|
165
173
|
|
|
174
|
+
# ── 列表项统计字段(GET /datasets/versions 每项返回 instance_count/split_count)──
|
|
175
|
+
|
|
176
|
+
instance_count: int | None = None
|
|
177
|
+
split_count: int | None = None
|
|
178
|
+
|
|
166
179
|
def validate(self) -> None:
|
|
167
180
|
if self.status not in VERSION_STATUS_VALUES:
|
|
168
181
|
raise SchemaError(f"invalid status '{self.status}'; valid values: {', '.join(VERSION_STATUS_VALUES)}")
|
|
@@ -208,6 +221,9 @@ class DatasetInstance:
|
|
|
208
221
|
instance_version: str | None = None
|
|
209
222
|
feedbacks: str | None = None
|
|
210
223
|
schema_version: str = "1.0"
|
|
224
|
+
#: 服务端返回的时间戳;本地扫描构造的实例为 None,不进 to_dict(推送载荷不变)。
|
|
225
|
+
created_at: str | None = None
|
|
226
|
+
updated_at: str | None = None
|
|
211
227
|
|
|
212
228
|
def validate(self) -> None:
|
|
213
229
|
if not self.instance_id:
|
|
@@ -229,7 +245,7 @@ class DatasetInstance:
|
|
|
229
245
|
"schema_version": self.schema_version,
|
|
230
246
|
}
|
|
231
247
|
for k in ("id", "difficulty", "docker_image", "instance_status",
|
|
232
|
-
"instance_version", "feedbacks"):
|
|
248
|
+
"instance_version", "feedbacks", "created_at", "updated_at"):
|
|
233
249
|
v = getattr(self, k)
|
|
234
250
|
if v is not None:
|
|
235
251
|
d[k] = v
|
|
@@ -248,6 +264,7 @@ class DatasetInstance:
|
|
|
248
264
|
content_type=m.get("content_type", "file"),
|
|
249
265
|
content_format=m.get("content_format", "json"),
|
|
250
266
|
content_digest=m.get("content_digest"),
|
|
267
|
+
object_path=m.get("object_path"),
|
|
251
268
|
)
|
|
252
269
|
for m in (d.get("manifest") or [])
|
|
253
270
|
]
|
|
@@ -270,6 +287,8 @@ class DatasetInstance:
|
|
|
270
287
|
instance_version=d.get("instance_version"),
|
|
271
288
|
feedbacks=d.get("feedbacks"),
|
|
272
289
|
schema_version=d.get("schema_version", "1.0"),
|
|
290
|
+
created_at=d.get("created_at"),
|
|
291
|
+
updated_at=d.get("updated_at"),
|
|
273
292
|
)
|
|
274
293
|
|
|
275
294
|
|
|
@@ -514,6 +533,10 @@ class PagedResult:
|
|
|
514
533
|
page: int = 1
|
|
515
534
|
page_size: int = 0
|
|
516
535
|
has_more: bool = False
|
|
536
|
+
#: 服务端返回的**原始**分页信封(键名原样,如 ``pageSize``);无信封时为 None。
|
|
537
|
+
#: 给需要逐字透出服务端响应的调用方(CLI 的 ``pagination`` 字段就是原样透出,
|
|
538
|
+
#: 用归一化后的 ``page_size`` 会改变既有 JSON 契约)。
|
|
539
|
+
raw: dict | None = None
|
|
517
540
|
|
|
518
541
|
@classmethod
|
|
519
542
|
def from_response(cls, items: list, pagination: dict | None,
|
|
@@ -523,7 +546,8 @@ class PagedResult:
|
|
|
523
546
|
total = total if isinstance(total, (int, float)) else -1
|
|
524
547
|
return cls(items=items, total=int(total), page=page,
|
|
525
548
|
page_size=page_size,
|
|
526
|
-
has_more=page * page_size < total
|
|
549
|
+
has_more=page * page_size < total,
|
|
550
|
+
raw=dict(pagination))
|
|
527
551
|
return cls(items=items, total=-1, page=page, page_size=page_size,
|
|
528
552
|
has_more=False)
|
|
529
553
|
|
|
@@ -554,8 +578,13 @@ class DatasetSplit:
|
|
|
554
578
|
"""split-first 记录(GET /datasets/splits 系列返回的 DatasetSplitRecord)。
|
|
555
579
|
|
|
556
580
|
``version`` 为空表示 unversioned split(split_first 无版本布局)。
|
|
581
|
+
``projected=True`` 表示本对象来自 version 内嵌 ``splits[]`` 的兼容投影
|
|
582
|
+
(存量 version_first 数据无独立 split 记录,SplitsClient 回退投影):
|
|
583
|
+
``dataset_split_id``/``status``/``published_at`` 均为空,``storage_path``
|
|
584
|
+
取自内嵌 ``oss_prefix``。
|
|
557
585
|
"""
|
|
558
586
|
dataset_split_id: str = ""
|
|
587
|
+
projected: bool = False
|
|
559
588
|
dataset: str = ""
|
|
560
589
|
dataset_id: str = ""
|
|
561
590
|
dataset_version_id: str | None = None
|
|
@@ -577,6 +606,7 @@ class DatasetSplit:
|
|
|
577
606
|
m = d.get("manifest")
|
|
578
607
|
return cls(
|
|
579
608
|
dataset_split_id=d.get("dataset_split_id", ""),
|
|
609
|
+
projected=bool(d.get("projected", False)),
|
|
580
610
|
dataset=d.get("dataset", ""),
|
|
581
611
|
dataset_id=d.get("dataset_id", ""),
|
|
582
612
|
dataset_version_id=d.get("dataset_version_id"),
|
|
@@ -62,7 +62,9 @@ class ApiMetadataRegistry(BaseMetadataRegistry):
|
|
|
62
62
|
self._metadata_model = (metadata_model or "").strip()
|
|
63
63
|
|
|
64
64
|
def _q(self, dataset: str, version: str, split: str | None = None,
|
|
65
|
-
instance_id: str | None = None
|
|
65
|
+
instance_id: str | None = None, *,
|
|
66
|
+
environment: str | None = None,
|
|
67
|
+
metadata_model: str | None = None) -> str:
|
|
66
68
|
"""读路径查询串:与写路径(ingest)对称下发 metadata_model 与 environment。
|
|
67
69
|
|
|
68
70
|
两个字段都**必须**对称:写路径由 storage_env 派生 environment(dev→pre)写入 pre
|
|
@@ -70,14 +72,21 @@ class ApiMetadataRegistry(BaseMetadataRegistry):
|
|
|
70
72
|
服务端对读请求同样把空 metadata_model 归一为 version_first,而 SDK 默认
|
|
71
73
|
split_first,不显式下发就会用旧语义去查 split-first 数据。environment 为空时
|
|
72
74
|
省略(服务端按 online 缺省,保留未配置 storage_env 的旧行为)。
|
|
75
|
+
|
|
76
|
+
``environment`` / ``metadata_model`` 传 None 用构造期值,传非 None(含空串)
|
|
77
|
+
覆盖本次调用——调用方(如 CLI 的 ``--environment``)需要显式指定集合时用它,
|
|
78
|
+
不必按环境重建整个 Repo。
|
|
73
79
|
"""
|
|
80
|
+
model = (self._metadata_model if metadata_model is None
|
|
81
|
+
else metadata_model).strip()
|
|
82
|
+
env = (self._environment if environment is None else environment).strip()
|
|
74
83
|
base = _q(dataset, version, split, instance_id,
|
|
75
|
-
keep_default_split=(
|
|
84
|
+
keep_default_split=(model == "split_first"))
|
|
76
85
|
extra = []
|
|
77
|
-
if
|
|
78
|
-
extra.append(f"metadata_model={quote(
|
|
79
|
-
if
|
|
80
|
-
extra.append(f"environment={quote(
|
|
86
|
+
if model:
|
|
87
|
+
extra.append(f"metadata_model={quote(model, safe='')}")
|
|
88
|
+
if env:
|
|
89
|
+
extra.append(f"environment={quote(env, safe='')}")
|
|
81
90
|
return base + ("&" + "&".join(extra) if extra else "")
|
|
82
91
|
|
|
83
92
|
def register(self, meta: DatasetInstance) -> DatasetInstance:
|
|
@@ -137,7 +146,9 @@ class ApiMetadataRegistry(BaseMetadataRegistry):
|
|
|
137
146
|
return DatasetInstance.from_dict(data)
|
|
138
147
|
|
|
139
148
|
def get(self, dataset: str, version: str, instance_id: str,
|
|
140
|
-
split: str | None = None
|
|
149
|
+
split: str | None = None, *,
|
|
150
|
+
environment: str | None = None,
|
|
151
|
+
metadata_model: str | None = None) -> DatasetInstance | None:
|
|
141
152
|
"""取单个实例;**实例不存在返回 None**("没有"不是错误)。
|
|
142
153
|
|
|
143
154
|
apiserver 对"实例不存在"回 HTTP 404 + 业务码 92003,对"数据集不存在"回 404 +
|
|
@@ -145,7 +156,8 @@ class ApiMetadataRegistry(BaseMetadataRegistry):
|
|
|
145
156
|
"""
|
|
146
157
|
try:
|
|
147
158
|
data = self._t.get(
|
|
148
|
-
f"{self._BASE}/detail?
|
|
159
|
+
f"{self._BASE}/detail?"
|
|
160
|
+
f"{self._q(dataset, version, split, instance_id, environment=environment, metadata_model=metadata_model)}")
|
|
149
161
|
except TransportError as e:
|
|
150
162
|
if e.status == 404 and e.biz_code == CODE_INSTANCE_NOT_FOUND:
|
|
151
163
|
return None
|
|
@@ -155,7 +167,9 @@ class ApiMetadataRegistry(BaseMetadataRegistry):
|
|
|
155
167
|
return DatasetInstance.from_dict(data)
|
|
156
168
|
|
|
157
169
|
def query(self, dataset: str, version: str,
|
|
158
|
-
split: str | None = None
|
|
170
|
+
split: str | None = None, *,
|
|
171
|
+
environment: str | None = None,
|
|
172
|
+
metadata_model: str | None = None) -> list[DatasetInstance]:
|
|
159
173
|
# 分页遍历:服务端 page_size 上限 200,默认只回第一页(真机踩坑:1580 实例
|
|
160
174
|
# 只读回 50)。显式传 page_size=200 并按 pagination.total 翻页取满;
|
|
161
175
|
# 旧服务端不带 pagination 信封时单页即全部,行为与修复前一致。
|
|
@@ -163,7 +177,8 @@ class ApiMetadataRegistry(BaseMetadataRegistry):
|
|
|
163
177
|
page = 1
|
|
164
178
|
while True:
|
|
165
179
|
data, pagination = self._t.request_with_meta(
|
|
166
|
-
"GET", f"{self._BASE}?
|
|
180
|
+
"GET", f"{self._BASE}?"
|
|
181
|
+
f"{self._q(dataset, version, split, environment=environment, metadata_model=metadata_model)}"
|
|
167
182
|
f"&page={page}&page_size=200")
|
|
168
183
|
rows = data.get("instances", data) if isinstance(data, dict) else data
|
|
169
184
|
rows = rows if isinstance(rows, list) else [] # 非列表(裸 dict 等)→ 空结果
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{instance_repo-1.0.9.dev0 → instance_repo-1.0.9.dev1}/instance_repo.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|