instance-repo 1.0.9.dev4__tar.gz → 1.0.9.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/PKG-INFO +2 -3
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/README.md +1 -2
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/__init__.py +1 -1
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/_routing.py +36 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/cli.py +19 -37
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/splits.py +1 -58
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/versions.py +28 -12
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/models.py +3 -4
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/retry.py +5 -1
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/store/oss.py +23 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/transport.py +1 -1
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/PKG-INFO +2 -3
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/pyproject.toml +1 -1
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/_site_defaults.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/bootstrap.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/cache.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/__init__.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/benchmarks.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/config.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/datasets.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/images.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/instances.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/reports.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/scaffold.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/clients/workflows.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/concurrency.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/content.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/endpoints.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/errors.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/image.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/layout.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/loader.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/paths.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/release.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/repo.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/store/__init__.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/store/acr.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/store/base.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/store/registry.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/tasktoml.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo/validate.py +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/SOURCES.txt +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/dependency_links.txt +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/entry_points.txt +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/requires.txt +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/top_level.txt +0 -0
- {instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: instance-repo
|
|
3
|
-
Version: 1.0.9.
|
|
3
|
+
Version: 1.0.9.dev5
|
|
4
4
|
Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -149,10 +149,9 @@ keys = repo.list_user_data(uid)
|
|
|
149
149
|
irepo --help
|
|
150
150
|
# 全局参数:--api-env / --api-base / --cluster / --storage-env / --metadata-model / --profile
|
|
151
151
|
# 无 --api-key:凭据只从 AP_API_KEY(旧别名 INSTANCEREPO_TOKEN)读取,避免进 shell history
|
|
152
|
-
#
|
|
152
|
+
# 18 个子命令:validate / push / push-many / deliver / pull / list / get
|
|
153
153
|
# publish / grant / feedback / report / whoami / user-data
|
|
154
154
|
# repo-config / create / claim / version / image
|
|
155
|
-
# split(list/get/set-run-type)/ dataset(list/get/update/access/revoke/benchmarks/workflows)
|
|
156
155
|
irepo image ref --dataset alibaba/mybench --instance-id inst-1 --split test
|
|
157
156
|
irepo image pull '<acr>/<ns>/mybench:test-inst-1' ./out.tar
|
|
158
157
|
```
|
|
@@ -136,10 +136,9 @@ keys = repo.list_user_data(uid)
|
|
|
136
136
|
irepo --help
|
|
137
137
|
# 全局参数:--api-env / --api-base / --cluster / --storage-env / --metadata-model / --profile
|
|
138
138
|
# 无 --api-key:凭据只从 AP_API_KEY(旧别名 INSTANCEREPO_TOKEN)读取,避免进 shell history
|
|
139
|
-
#
|
|
139
|
+
# 18 个子命令:validate / push / push-many / deliver / pull / list / get
|
|
140
140
|
# publish / grant / feedback / report / whoami / user-data
|
|
141
141
|
# repo-config / create / claim / version / image
|
|
142
|
-
# split(list/get/set-run-type)/ dataset(list/get/update/access/revoke/benchmarks/workflows)
|
|
143
142
|
irepo image ref --dataset alibaba/mybench --instance-id inst-1 --split test
|
|
144
143
|
irepo image pull '<acr>/<ns>/mybench:test-inst-1' ./out.tar
|
|
145
144
|
```
|
|
@@ -45,6 +45,42 @@ def _split_scheme(endpoint: str) -> tuple[str, str]:
|
|
|
45
45
|
return "", endpoint
|
|
46
46
|
|
|
47
47
|
|
|
48
|
+
def _probe_private_endpoint(endpoint: str, timeout: float = 3.0) -> bool:
|
|
49
|
+
"""Quick TCP connect probe to decide whether a private endpoint is reachable.
|
|
50
|
+
|
|
51
|
+
Performs a raw TCP connect (no TLS handshake) to the endpoint host/port.
|
|
52
|
+
Returns True if the connect succeeds, False on any error. The default
|
|
53
|
+
timeout is 3 seconds — short enough to save 60+ seconds of waiting when
|
|
54
|
+
the private endpoint is unreachable, yet long enough for a normal TCP
|
|
55
|
+
handshake on a reachable host.
|
|
56
|
+
"""
|
|
57
|
+
_, host = _split_scheme(endpoint)
|
|
58
|
+
# Determine port: default 443 for https, 80 for http.
|
|
59
|
+
scheme, _ = _split_scheme(endpoint)
|
|
60
|
+
default_port = 443 if scheme == "https://" else 80
|
|
61
|
+
# Parse host:port — handle IPv4 with optional port.
|
|
62
|
+
if ":" in host:
|
|
63
|
+
parts = host.rsplit(":", 1)
|
|
64
|
+
hostname = parts[0]
|
|
65
|
+
try:
|
|
66
|
+
port = int(parts[1])
|
|
67
|
+
except (ValueError, IndexError):
|
|
68
|
+
hostname = host
|
|
69
|
+
port = default_port
|
|
70
|
+
else:
|
|
71
|
+
hostname = host
|
|
72
|
+
port = default_port
|
|
73
|
+
# Strip any trailing path from hostname.
|
|
74
|
+
if "/" in hostname:
|
|
75
|
+
hostname = hostname.split("/", 1)[0]
|
|
76
|
+
try:
|
|
77
|
+
conn = socket.create_connection((hostname, port), timeout=timeout)
|
|
78
|
+
conn.close()
|
|
79
|
+
return True
|
|
80
|
+
except Exception: # noqa: BLE001 - any failure means unreachable
|
|
81
|
+
return False
|
|
82
|
+
|
|
83
|
+
|
|
48
84
|
def _plan_oss_route(endpoint: str) -> _EndpointRoute:
|
|
49
85
|
value = (endpoint or "").strip()
|
|
50
86
|
scheme, host = _split_scheme(value)
|
|
@@ -312,16 +312,16 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
312
312
|
help="get 用:覆盖 profile 派生的 environment(pre/online)")
|
|
313
313
|
|
|
314
314
|
# ── 1.0.10 控制面管理子命令 ─────────────────────────────────────────────────
|
|
315
|
-
sp = sub.add_parser("split", help="split 记录管理(list/get/
|
|
316
|
-
sp.add_argument("action", choices=["list", "get", "
|
|
315
|
+
sp = sub.add_parser("split", help="split 记录管理(list/get/set-run-type)")
|
|
316
|
+
sp.add_argument("action", choices=["list", "get", "set-run-type"])
|
|
317
317
|
sp.add_argument("ref", nargs="?",
|
|
318
|
-
help="list: {L1}/{L2};get/set-run-type: {L1}/{L2}/{split}
|
|
319
|
-
"
|
|
318
|
+
help="list: {L1}/{L2};get/set-run-type: {L1}/{L2}/{split}"
|
|
319
|
+
"(或用 --dataset/--split)")
|
|
320
320
|
sp.add_argument("--dataset", default=None, help="{L1}/{L2}")
|
|
321
321
|
sp.add_argument("--split", default=None,
|
|
322
322
|
help="目标 split 名(get/set-run-type 用)")
|
|
323
323
|
sp.add_argument("--version", default=None,
|
|
324
|
-
help="list 按 version 精确过滤;get
|
|
324
|
+
help="list 按 version 精确过滤;get 可选定位版本")
|
|
325
325
|
sp.add_argument("--scope", choices=["unversioned", "exact", "all"], default=None,
|
|
326
326
|
help="list 跨版本策略:缺省无 version → all(SDK 取全量哲学),"
|
|
327
327
|
"有 version 省略(服务端只接受 exact)")
|
|
@@ -329,13 +329,9 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
329
329
|
help="list 按状态过滤")
|
|
330
330
|
sp.add_argument("--run-type", choices=["train", "eval", ""], default=None,
|
|
331
331
|
dest="run_type",
|
|
332
|
-
help="set-run-type
|
|
333
|
-
sp.add_argument("--storage-path", default=None, dest="storage_path",
|
|
334
|
-
help="create 用:split 壳的 storage_path(缺省自动计算)")
|
|
332
|
+
help="set-run-type 用:train/eval 或空串清除")
|
|
335
333
|
sp.add_argument("--page", type=int, default=1)
|
|
336
334
|
sp.add_argument("--page-size", type=int, default=50, dest="page_size")
|
|
337
|
-
sp.add_argument("--environment", default=None,
|
|
338
|
-
help="create 覆盖 profile 派生的 environment(pre/online)")
|
|
339
335
|
|
|
340
336
|
ds = sub.add_parser("dataset",
|
|
341
337
|
help="数据集/基准/工作流管理(list/get/update/access/"
|
|
@@ -420,7 +416,7 @@ def _resolve_split_ref(args) -> tuple[str, str]:
|
|
|
420
416
|
if args.ref:
|
|
421
417
|
segs = _split_ref(args.ref, 3)
|
|
422
418
|
return f"{segs[0]}/{segs[1]}", segs[2]
|
|
423
|
-
raise SystemExit("split get/
|
|
419
|
+
raise SystemExit("split get/set-run-type 需要 {L1}/{L2}/{split} 或 "
|
|
424
420
|
"--dataset/--split")
|
|
425
421
|
|
|
426
422
|
|
|
@@ -874,17 +870,19 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
874
870
|
# 1.0.10:取版本完整详情(splits 明细 + manifest + 时间戳)。
|
|
875
871
|
# split_first 分支时服务端聚合 legacy 投影。
|
|
876
872
|
dv = r.versions.get(dataset, version, environment=args.environment)
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
print(f"dataset_version_id={dv.dataset_version_id
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
873
|
+
print(f"version\t{dv.get('version','')}\tstatus={dv.get('status','')}"
|
|
874
|
+
f"\tstorage_path={dv.get('storage_path','')}", file=out)
|
|
875
|
+
print(f"published_at={dv.get('published_at') or ''}"
|
|
876
|
+
f"\tlast_ingested_at={dv.get('last_ingested_at') or ''}",
|
|
877
|
+
file=out)
|
|
878
|
+
print(f"dataset_version_id={dv.get('dataset_version_id','')}",
|
|
879
|
+
file=out)
|
|
880
|
+
splits = dv.get("splits") or []
|
|
881
|
+
for s in splits:
|
|
882
|
+
print(f" split\t{s.get('name','')}\tstatus={s.get('status','')}"
|
|
883
|
+
f"\toss_prefix={s.get('oss_prefix','')}"
|
|
886
884
|
f"\tinstance_count={s.get('instance_count') or 0}", file=out)
|
|
887
|
-
manifest = dv.manifest or {}
|
|
885
|
+
manifest = dv.get("manifest") or {}
|
|
888
886
|
print(f"manifest\tinstance_count={manifest.get('instance_count') or 0}"
|
|
889
887
|
f"\tfile_count={manifest.get('file_count') or 0}"
|
|
890
888
|
f"\tsize_bytes={manifest.get('size_bytes') or 0}", file=out)
|
|
@@ -921,22 +919,6 @@ def main(argv=None, *, repo: Repo | None = None, out=None) -> int:
|
|
|
921
919
|
file=out)
|
|
922
920
|
print(f"page={res.page}/{_page_count(res)} total={res.total}", file=out)
|
|
923
921
|
return 0
|
|
924
|
-
if args.action == "create":
|
|
925
|
-
dataset, split_name = _resolve_split_ref(args)
|
|
926
|
-
if not (args.version or "").strip():
|
|
927
|
-
raise SystemExit(
|
|
928
|
-
"split create 需要 --version(apiserver CreateVersion 硬约束)")
|
|
929
|
-
sp = r.splits.create(dataset, split_name,
|
|
930
|
-
version=args.version,
|
|
931
|
-
run_type=args.run_type,
|
|
932
|
-
storage_path=args.storage_path,
|
|
933
|
-
environment=args.environment)
|
|
934
|
-
src = "projected" if sp.projected else "record"
|
|
935
|
-
print(f"created split\t{sp.name}\tversion={sp.version or ''}"
|
|
936
|
-
f"\tstatus={sp.status}\trun_type={sp.run_type or ''}"
|
|
937
|
-
f"\t[{src}]", file=out)
|
|
938
|
-
print(f"storage_path={sp.storage_path}", file=out)
|
|
939
|
-
return 0
|
|
940
922
|
# get / set-run-type:需要定位到具体 split
|
|
941
923
|
dataset, split_name = _resolve_split_ref(args)
|
|
942
924
|
if args.action == "get":
|
|
@@ -222,68 +222,13 @@ class SplitsClient:
|
|
|
222
222
|
|
|
223
223
|
# ── 写 ──
|
|
224
224
|
|
|
225
|
-
def create(self, dataset: str, split: str, *,
|
|
226
|
-
version: str, run_type: str | None = None,
|
|
227
|
-
storage_path: str | None = None,
|
|
228
|
-
environment: str | None = None) -> DatasetSplit:
|
|
229
|
-
"""创建 split 记录——组合 versions.create(splits=[split]) 的便利封装。
|
|
230
|
-
|
|
231
|
-
apiserver **没有** split create 端点(``POST /datasets/splits`` 不存在):
|
|
232
|
-
split 记录由 ingest 扫描 OSS 或 versions.create/update(splits=[...])
|
|
233
|
-
间接产生(pre 集合 find-or-upsert ensure 真实 split record)。本方法是
|
|
234
|
-
"先建空 split 壳"的官方路径升级为 SDK 直接 API:
|
|
235
|
-
|
|
236
|
-
1. ``versions.create(dataset, version, splits=[split])``(version **必填**
|
|
237
|
-
——apiserver CreateVersion 硬约束;storage_path 缺省自动计算);
|
|
238
|
-
2. 可选 ``run_type``:create 后经 versions.update(split_run_types) 打标;
|
|
239
|
-
3. 读回 ``splits.get``(同 environment)返回记录。
|
|
240
|
-
|
|
241
|
-
参数:
|
|
242
|
-
dataset : dataset 名(必填)。
|
|
243
|
-
split : split 名(必填)。
|
|
244
|
-
version : version(必填——CreateVersion 硬约束)。
|
|
245
|
-
run_type : 可选,train/eval(create 后打标)。
|
|
246
|
-
storage_path : 可选,缺省按 profile 自动计算(与 versions.create 同源)。
|
|
247
|
-
environment : 可选,缺省由 profile 派生(pre/online)。
|
|
248
|
-
|
|
249
|
-
返回 :class:`~instance_repo.models.DatasetSplit`(读回;pre 集合 ensure
|
|
250
|
-
的 record 或 version 内嵌投影)。
|
|
251
|
-
"""
|
|
252
|
-
from .versions import VersionsClient
|
|
253
|
-
if not (dataset or "").strip():
|
|
254
|
-
raise SchemaError("dataset is required for splits.create")
|
|
255
|
-
if not (split or "").strip():
|
|
256
|
-
raise SchemaError("split is required for splits.create")
|
|
257
|
-
if not (version or "").strip():
|
|
258
|
-
raise SchemaError(
|
|
259
|
-
"version is required for splits.create (apiserver CreateVersion "
|
|
260
|
-
"requires version; split records are created via versions.create)")
|
|
261
|
-
rt = (run_type or "").strip()
|
|
262
|
-
if rt and rt not in RUN_TYPE_VALUES:
|
|
263
|
-
raise SchemaError(
|
|
264
|
-
f"invalid run_type {rt!r}; must be one of "
|
|
265
|
-
f"{', '.join(RUN_TYPE_VALUES)} or None")
|
|
266
|
-
env = self._env(environment)
|
|
267
|
-
vc = VersionsClient(self._t, self._p)
|
|
268
|
-
vc.create(dataset, version, splits=[split],
|
|
269
|
-
storage_path=storage_path, environment=env or None)
|
|
270
|
-
if rt:
|
|
271
|
-
# run_type 打标与 create 互斥字段:经 split_run_types 单独打(apiserver
|
|
272
|
-
# UpdateVersion 的 run_type/split_run_types/splits 两两互斥)。
|
|
273
|
-
vc.update(dataset, version, split_run_types={split: rt})
|
|
274
|
-
return self.get(dataset, split, version=version,
|
|
275
|
-
environment=env or None)
|
|
276
|
-
|
|
277
225
|
def update(self, dataset: str, split: str | None = None, *,
|
|
278
226
|
version: str | None = None, run_type: str = "",
|
|
279
|
-
dataset_split_id: str | None = None
|
|
280
|
-
environment: str | None = None) -> DatasetSplit:
|
|
227
|
+
dataset_split_id: str | None = None) -> DatasetSplit:
|
|
281
228
|
"""更新 split 记录(``PATCH /datasets/splits/detail``)。
|
|
282
229
|
|
|
283
230
|
服务端 body 仅接受 ``run_type``(**必填**:"train"/"eval",空串=清除);
|
|
284
231
|
仅 online 环境(split 记录 pre 环境的写经 versions.update 走集合路由)。
|
|
285
|
-
``environment`` 缺省由 profile 派生(同 get/list);profile 派生为 pre
|
|
286
|
-
而目标 split 记录在 online 集合时,须显式传 ``environment="online"``。
|
|
287
232
|
|
|
288
233
|
寻址二选一:``dataset_split_id``(优先,split-first 精确记录 id)或
|
|
289
234
|
``dataset``+``split``(+可选 ``version``)name 三元组。
|
|
@@ -303,8 +248,6 @@ class SplitsClient:
|
|
|
303
248
|
raise SchemaError(
|
|
304
249
|
f"invalid run_type {rt!r}; must be one of "
|
|
305
250
|
f"{', '.join(RUN_TYPE_VALUES)} (empty string to clear)")
|
|
306
|
-
if (env := self._env(environment)):
|
|
307
|
-
q += f"&environment={quote(env, safe='')}"
|
|
308
251
|
data = self._t.patch(f"{self._BASE}/detail?{q}", {"run_type": rt})
|
|
309
252
|
return DatasetSplit.from_dict(data) if isinstance(data, dict) else DatasetSplit()
|
|
310
253
|
|
|
@@ -209,8 +209,9 @@ class VersionsClient:
|
|
|
209
209
|
out[iid] = img
|
|
210
210
|
return out
|
|
211
211
|
|
|
212
|
-
def list(self, dataset: str
|
|
213
|
-
|
|
212
|
+
def list(self, dataset: str, *, environment: str | None = None,
|
|
213
|
+
metadata_model: str | None = None) -> list[DatasetVersion]:
|
|
214
|
+
extra = self._version_extra_q(environment, metadata_model)
|
|
214
215
|
data = self._t.get(
|
|
215
216
|
f"{self._BASE}?dataset_name={quote(dataset, safe='')}{extra}")
|
|
216
217
|
rows = data.get("versions", data) if isinstance(data, dict) else data
|
|
@@ -264,8 +265,10 @@ class VersionsClient:
|
|
|
264
265
|
data = self._t.get(f"{self._BASE}/detail?{q}")
|
|
265
266
|
return DatasetVersion.from_dict_full(data) if isinstance(data, dict) else DatasetVersion(version=version, dataset=dataset)
|
|
266
267
|
|
|
267
|
-
def status(self, dataset: str, version: str
|
|
268
|
-
|
|
268
|
+
def status(self, dataset: str, version: str, *,
|
|
269
|
+
environment: str | None = None,
|
|
270
|
+
metadata_model: str | None = None) -> str:
|
|
271
|
+
extra = self._version_extra_q(environment, metadata_model)
|
|
269
272
|
data = self._t.get(
|
|
270
273
|
f"{self._BASE}/detail?dataset_name={quote(dataset, safe='')}"
|
|
271
274
|
f"&version={quote(version, safe='')}{extra}")
|
|
@@ -382,7 +385,9 @@ class VersionsClient:
|
|
|
382
385
|
def update(self, dataset: str, version: str, *, status: str | None = None,
|
|
383
386
|
splits: list[str] | None = None, run_type: str | None = None,
|
|
384
387
|
split_run_types=None,
|
|
385
|
-
manifest: dict | None = None
|
|
388
|
+
manifest: dict | None = None,
|
|
389
|
+
environment: str | None = None,
|
|
390
|
+
metadata_model: str | None = None) -> None:
|
|
386
391
|
"""通用版本更新(``PATCH /apis/v1/datasets/versions/detail``)。
|
|
387
392
|
|
|
388
393
|
只发送非 None 字段。apiserver 侧 ``run_type`` / ``split_run_types`` / ``splits``
|
|
@@ -398,6 +403,8 @@ class VersionsClient:
|
|
|
398
403
|
manifest : 版本 manifest 整体替换(``{instance_count, file_count,
|
|
399
404
|
size_bytes, validation_status, validation_message}``
|
|
400
405
|
子集均可)——服务端支持但此前 SDK 未暴露(1.0.10 补)。
|
|
406
|
+
environment : 显式 environment 覆盖 profile 派生值(如发布后操作 online)。
|
|
407
|
+
metadata_model : 显式 metadata_model 覆盖 profile 派生值。
|
|
401
408
|
"""
|
|
402
409
|
norm_srt = _normalize_split_run_types(split_run_types) if split_run_types else None
|
|
403
410
|
# 三选一互斥(对齐 apiserver:run_type / split_run_types / splits 两两互斥)。
|
|
@@ -427,10 +434,13 @@ class VersionsClient:
|
|
|
427
434
|
body["manifest"] = manifest
|
|
428
435
|
if not body:
|
|
429
436
|
raise SchemaError("update requires at least one field to change")
|
|
430
|
-
self._patch_detail(dataset, version, body
|
|
437
|
+
self._patch_detail(dataset, version, body,
|
|
438
|
+
environment=environment, metadata_model=metadata_model)
|
|
431
439
|
|
|
432
440
|
def set_run_type(self, dataset: str, version: str, *,
|
|
433
|
-
run_type: str | None = None, split_run_types=None
|
|
441
|
+
run_type: str | None = None, split_run_types=None,
|
|
442
|
+
environment: str | None = None,
|
|
443
|
+
metadata_model: str | None = None) -> None:
|
|
434
444
|
"""便捷入口:仅给 version/split 打 run_type 标签(train/eval)。
|
|
435
445
|
|
|
436
446
|
version 级与 split 级二选一:传 ``run_type`` 打整个版本(无 logical split 的版本),
|
|
@@ -440,14 +450,20 @@ class VersionsClient:
|
|
|
440
450
|
raise SchemaError(
|
|
441
451
|
"set_run_type requires run_type or split_run_types")
|
|
442
452
|
self.update(dataset, version, run_type=run_type,
|
|
443
|
-
split_run_types=split_run_types
|
|
453
|
+
split_run_types=split_run_types,
|
|
454
|
+
environment=environment, metadata_model=metadata_model)
|
|
444
455
|
|
|
445
|
-
def deprecate(self, dataset: str, version: str
|
|
456
|
+
def deprecate(self, dataset: str, version: str, *,
|
|
457
|
+
environment: str | None = None,
|
|
458
|
+
metadata_model: str | None = None) -> None:
|
|
446
459
|
"""把版本置为 deprecated(``update(status="deprecated")`` 的薄封装)。"""
|
|
447
|
-
self.update(dataset, version, status="deprecated"
|
|
460
|
+
self.update(dataset, version, status="deprecated",
|
|
461
|
+
environment=environment, metadata_model=metadata_model)
|
|
448
462
|
|
|
449
|
-
def _patch_detail(self, dataset: str, version: str, body: dict
|
|
450
|
-
|
|
463
|
+
def _patch_detail(self, dataset: str, version: str, body: dict, *,
|
|
464
|
+
environment: str | None = None,
|
|
465
|
+
metadata_model: str | None = None) -> None:
|
|
466
|
+
extra = self._version_extra_q(environment, metadata_model)
|
|
451
467
|
self._t.patch(
|
|
452
468
|
f"{self._BASE}/detail?dataset_name={quote(dataset, safe='')}"
|
|
453
469
|
f"&version={quote(version, safe='')}{extra}",
|
|
@@ -124,7 +124,7 @@ class DatasetSeries:
|
|
|
124
124
|
available_envs: list[str] = field(default_factory=list)
|
|
125
125
|
# 服务端 `DatasetSeriesStatus`("" 正常 / "deprecated");列表项
|
|
126
126
|
# `DatasetListItem.Status` 带 `omitempty`,正常态不出现。
|
|
127
|
-
|
|
127
|
+
# 与 class 开头 status 字段合并到此位置。
|
|
128
128
|
|
|
129
129
|
def validate(self) -> None:
|
|
130
130
|
if self.visibility not in VISIBILITY_VALUES:
|
|
@@ -139,12 +139,11 @@ class DatasetSeries:
|
|
|
139
139
|
"""create 请求体:剔除服务端派生/服务端分配的字段。
|
|
140
140
|
|
|
141
141
|
teacher_direction / teacher_type 由 apiserver 从 benchmark 派生(传了也被忽略),
|
|
142
|
-
dataset_id 由服务端分配;dataset_l1 / dataset_l2 由 dataset_name
|
|
143
|
-
是服务端 DatasetSeriesStatus(deprecated 打标),create 不接受。都不发送,
|
|
142
|
+
dataset_id 由服务端分配;dataset_l1 / dataset_l2 由 dataset_name 推导。都不发送,
|
|
144
143
|
避免让调用方误以为这些值能由客户端决定。
|
|
145
144
|
"""
|
|
146
145
|
drop = {"teacher_direction", "teacher_type", "dataset_id",
|
|
147
|
-
"dataset_l1", "dataset_l2"
|
|
146
|
+
"dataset_l1", "dataset_l2"}
|
|
148
147
|
return {k: v for k, v in self.to_dict().items() if k not in drop}
|
|
149
148
|
|
|
150
149
|
|
|
@@ -22,6 +22,9 @@ _DEFAULT_SLEEP = time.sleep
|
|
|
22
22
|
#: 可重试的 4xx:429=限流(退避后可成功)、408=请求超时(服务端主动断,重发安全)。
|
|
23
23
|
RETRIABLE_4XX = frozenset({408, 429})
|
|
24
24
|
|
|
25
|
+
#: Tengine 等反向代理偶发返回的重定向状态码;SDK 应视为瞬时故障并重试。
|
|
26
|
+
REDIRECT_STATUSES = frozenset({301, 302, 307, 308})
|
|
27
|
+
|
|
25
28
|
|
|
26
29
|
def _status_of(exc: BaseException) -> int | None:
|
|
27
30
|
"""尽力从异常里取 HTTP 状态码:TransportError.status / oss2.*.status / .code。"""
|
|
@@ -46,7 +49,8 @@ def is_transient(exc: BaseException) -> bool:
|
|
|
46
49
|
status = _status_of(exc)
|
|
47
50
|
if status is not None:
|
|
48
51
|
return (status < 0 or 500 <= status <= 599
|
|
49
|
-
or status in RETRIABLE_4XX
|
|
52
|
+
or status in RETRIABLE_4XX
|
|
53
|
+
or status in REDIRECT_STATUSES)
|
|
50
54
|
return False
|
|
51
55
|
|
|
52
56
|
|
|
@@ -15,6 +15,7 @@ from .._routing import (
|
|
|
15
15
|
_OperationIntent,
|
|
16
16
|
_is_reachability_error,
|
|
17
17
|
_plan_oss_route,
|
|
18
|
+
_probe_private_endpoint,
|
|
18
19
|
)
|
|
19
20
|
from ..content import sha256_file, verify_file_digest
|
|
20
21
|
from ..errors import DigestMismatch, PrivateNetworkRequired, TransportError
|
|
@@ -157,6 +158,9 @@ class OssContentStore(BaseContentStore):
|
|
|
157
158
|
self._retry_attempts = retry_attempts
|
|
158
159
|
self._cache_buckets = cache_buckets
|
|
159
160
|
self._local = threading.local()
|
|
161
|
+
# Injectable TCP probe for private endpoint reachability. Defaults to
|
|
162
|
+
# the real raw-TCP-connect probe; tests may replace it with a stub.
|
|
163
|
+
self._probe_private_endpoint = _probe_private_endpoint
|
|
160
164
|
|
|
161
165
|
@property
|
|
162
166
|
def bucket_name(self) -> str:
|
|
@@ -251,6 +255,25 @@ class OssContentStore(BaseContentStore):
|
|
|
251
255
|
|
|
252
256
|
private_endpoint = route.private_endpoint
|
|
253
257
|
private_failure: BaseException | None = None
|
|
258
|
+
|
|
259
|
+
# Quick TCP probe to decide whether to attempt the private endpoint.
|
|
260
|
+
# If the route is standard (has a known public fallback) and the probe
|
|
261
|
+
# fails, skip the private attempt entirely to save 60+ seconds of
|
|
262
|
+
# waiting for a full timeout. Downloads are NOT affected — they
|
|
263
|
+
# always try private only (no public fallback).
|
|
264
|
+
if route.standard and route.public_endpoint is not None:
|
|
265
|
+
if not self._probe_private_endpoint(private_endpoint):
|
|
266
|
+
private_failure = ConnectionRefusedError(
|
|
267
|
+
f"TCP probe failed for {private_endpoint}")
|
|
268
|
+
try:
|
|
269
|
+
bucket = self._open_bucket(
|
|
270
|
+
sts, prefix, route.public_endpoint,
|
|
271
|
+
private_download=False)
|
|
272
|
+
return self._do(lambda: operation(bucket))
|
|
273
|
+
except Exception as public_exc: # noqa: BLE001
|
|
274
|
+
public_exc.private_route_failure = private_failure
|
|
275
|
+
raise
|
|
276
|
+
|
|
254
277
|
try:
|
|
255
278
|
bucket = self._open_bucket(
|
|
256
279
|
sts, prefix, private_endpoint, private_download=False)
|
|
@@ -196,7 +196,7 @@ class Transport:
|
|
|
196
196
|
raise SchemaError(
|
|
197
197
|
"apiserver 根地址未配置;请通过 AP_BASE_URL 提供"
|
|
198
198
|
"(旧别名 INSTANCEREPO_API_BASE 仍兼容;SDK 不内置任何域名,见部署文档)")
|
|
199
|
-
retry =
|
|
199
|
+
retry = True if idempotent is None else idempotent
|
|
200
200
|
if not retry:
|
|
201
201
|
return self._request_once(method, path, body)
|
|
202
202
|
return retry_transient(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: instance-repo
|
|
3
|
-
Version: 1.0.9.
|
|
3
|
+
Version: 1.0.9.dev5
|
|
4
4
|
Summary: InstanceRepo SDK — 评测 Instance 统一存储客户端(SDK-only;控制面由 apiserver 承载)
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -149,10 +149,9 @@ keys = repo.list_user_data(uid)
|
|
|
149
149
|
irepo --help
|
|
150
150
|
# 全局参数:--api-env / --api-base / --cluster / --storage-env / --metadata-model / --profile
|
|
151
151
|
# 无 --api-key:凭据只从 AP_API_KEY(旧别名 INSTANCEREPO_TOKEN)读取,避免进 shell history
|
|
152
|
-
#
|
|
152
|
+
# 18 个子命令:validate / push / push-many / deliver / pull / list / get
|
|
153
153
|
# publish / grant / feedback / report / whoami / user-data
|
|
154
154
|
# repo-config / create / claim / version / image
|
|
155
|
-
# split(list/get/set-run-type)/ dataset(list/get/update/access/revoke/benchmarks/workflows)
|
|
156
155
|
irepo image ref --dataset alibaba/mybench --instance-id inst-1 --split test
|
|
157
156
|
irepo image pull '<acr>/<ns>/mybench:test-inst-1' ./out.tar
|
|
158
157
|
```
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
{instance_repo-1.0.9.dev4 → instance_repo-1.0.9.dev5}/instance_repo.egg-info/entry_points.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|