ap-client 0.6.1.dev13__tar.gz → 0.7.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/PKG-INFO +4 -4
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/cli.py +19 -2
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/dataset_commands.py +117 -205
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/dataset_register.py +7 -76
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/fs_commands.py +20 -199
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/instance_commands.py +52 -519
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/irepo_sdk.py +4 -129
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/repo_read_guard.py +2 -6
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/split_publish.py +1 -1
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/pyproject.toml +4 -5
- ap_client-0.6.1.dev13/ap_client/dataset_name.py +0 -53
- ap_client-0.6.1.dev13/ap_client/dfs_commands.py +0 -702
- ap_client-0.6.1.dev13/ap_client/fs_storage.py +0 -146
- ap_client-0.6.1.dev13/ap_client/fs_transfer.py +0 -329
- ap_client-0.6.1.dev13/ap_client/instance_options.py +0 -94
- ap_client-0.6.1.dev13/ap_client/ossutil_hash.py +0 -329
- ap_client-0.6.1.dev13/ap_client/ossutil_io.py +0 -1247
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/__init__.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/api.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/config.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/exporter.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/managed_ak.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/manifest_view.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/profile_commands.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/share_api.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/tbb.py +0 -0
- {ap_client-0.6.1.dev13 → ap_client-0.7.0.dev1}/ap_client/waiter.py +0 -0
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0.dev1
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
|
-
Requires-Dist: irepo-artifact==0.2.
|
|
6
|
+
Requires-Dist: irepo-artifact==0.2.3.dev0
|
|
7
7
|
Requires-Dist: pyyaml>=6.0
|
|
8
8
|
Requires-Dist: requests>=2.28.0
|
|
9
9
|
Requires-Dist: rich>=13.0.0
|
|
10
10
|
Requires-Dist: typer>=0.9.0
|
|
11
11
|
Requires-Dist: websockets>=13.0
|
|
12
12
|
Provides-Extra: all
|
|
13
|
-
Requires-Dist: instance-repo[oss]<2.0.0,>=1.
|
|
13
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.5; extra == 'all'
|
|
14
14
|
Provides-Extra: dataset
|
|
15
|
-
Requires-Dist: instance-repo[oss]<2.0.0,>=1.
|
|
15
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.5; extra == 'dataset'
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
|
|
18
18
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -23,7 +23,6 @@ from ap_client.api import (
|
|
|
23
23
|
)
|
|
24
24
|
from ap_client.config import ENV_VAR_SPECS, ConfigurationError, _parse_bool, normalize_output_format
|
|
25
25
|
from ap_client.dataset_commands import register as _register_dataset_commands
|
|
26
|
-
from ap_client.dfs_commands import register as _register_dfs_commands
|
|
27
26
|
from ap_client.exporter import export_group, export_job, export_jobs
|
|
28
27
|
from ap_client.fs_commands import register as _register_fs_commands
|
|
29
28
|
from ap_client.instance_commands import register as _register_instance_commands
|
|
@@ -166,7 +165,6 @@ app.add_typer(checkpoint_app, name="checkpoint")
|
|
|
166
165
|
# ap dataset version/split/access 嵌套在 dataset 下;instance/fs 是一等资源。
|
|
167
166
|
_register_dataset_commands(dataset_app)
|
|
168
167
|
_register_instance_commands(app)
|
|
169
|
-
_register_dfs_commands(app)
|
|
170
168
|
_register_fs_commands(app)
|
|
171
169
|
_register_akpool_commands(app)
|
|
172
170
|
|
|
@@ -955,6 +953,25 @@ def _print_job_get_plain(job: dict) -> None:
|
|
|
955
953
|
]
|
|
956
954
|
)
|
|
957
955
|
|
|
956
|
+
bindings = job.get("ak_bindings") or []
|
|
957
|
+
rows = [
|
|
958
|
+
{**parameter, "ak_id": binding["ak_id"]}
|
|
959
|
+
for binding in bindings
|
|
960
|
+
for parameter in binding.get("parameters", [])
|
|
961
|
+
]
|
|
962
|
+
if "ak_bindings.parameters" in (job.get("restricted_fields") or []):
|
|
963
|
+
typer.echo("\nManaged AK parameters: content access required")
|
|
964
|
+
elif rows:
|
|
965
|
+
typer.echo("\nManaged AK parameters")
|
|
966
|
+
for row in rows:
|
|
967
|
+
typer.echo(f" {row['parameter']}: {row['value']} -> {row['ak_id']}")
|
|
968
|
+
if any("parameters" not in binding for binding in bindings):
|
|
969
|
+
typer.echo(" Some historical parameter mappings are unavailable")
|
|
970
|
+
elif response_ak_ids(job):
|
|
971
|
+
typer.echo("\nManaged AK parameters: historical mapping unavailable")
|
|
972
|
+
elif "ak_bindings" in job:
|
|
973
|
+
typer.echo("\nManaged AK parameters\n No managed AK used")
|
|
974
|
+
|
|
958
975
|
meta_pairs = [
|
|
959
976
|
("meta_model", job.get("meta_model")),
|
|
960
977
|
("meta_job_type", job.get("meta_job_type")),
|
|
@@ -123,10 +123,7 @@ _NO_BENCHMARK = "none"
|
|
|
123
123
|
_FORMAT_HELP = "Output format: plain/table/json/yaml (default: AP_FORMAT or command default)"
|
|
124
124
|
|
|
125
125
|
_METADATA_MODELS = ("version_first", "split_first")
|
|
126
|
-
|
|
127
|
-
# version_first(仅保留给低层 legacy 读)。带 version 的 split 查询在两种模型下
|
|
128
|
-
# 结果一致(实测),显式 split_first 与 SDK 归一值一致且新旧 SDK 均接受。
|
|
129
|
-
_VERSION_SPLIT_METADATA_MODEL = "split_first"
|
|
126
|
+
_VERSION_SPLIT_METADATA_MODEL = "version_first"
|
|
130
127
|
|
|
131
128
|
#: CLI **不替 SDK 决定元数据模型**:不下发 ``metadata_model``,由 SDK 按 profile 归一
|
|
132
129
|
#: (当前为 ``split_first``)。
|
|
@@ -694,74 +691,6 @@ def _unversioned_split(
|
|
|
694
691
|
return detail if detail is not None and not _version_value(detail.get("version")) else None
|
|
695
692
|
|
|
696
693
|
|
|
697
|
-
def _versioned_split(
|
|
698
|
-
dataset: str, version: str, split: str, *, environment: Optional[str] = None
|
|
699
|
-
) -> Optional[dict]:
|
|
700
|
-
"""取指定 version 里的单个 split 记录;不存在返回 ``None``(不当作错误)。
|
|
701
|
-
|
|
702
|
-
与 ``_unversioned_split`` 同源,走 recordsvc 的 ``splits.get``(SDK 自带
|
|
703
|
-
version_first 内嵌投影的 legacy 回退)。
|
|
704
|
-
"""
|
|
705
|
-
from .irepo_sdk import _reason
|
|
706
|
-
|
|
707
|
-
env = _environment(environment)
|
|
708
|
-
repo = _dataset_repo()
|
|
709
|
-
try:
|
|
710
|
-
record = repo.splits.get(
|
|
711
|
-
dataset, split, version=version, **_read_env_kwargs(repo, repo.splits, "get", env)
|
|
712
|
-
)
|
|
713
|
-
except Exception as exc: # noqa: BLE001
|
|
714
|
-
raise _fail(
|
|
715
|
-
f"dataset {dataset} version {version} split {split}: query failed: {_reason(exc)}",
|
|
716
|
-
f"Check the dataset name and permission: ap dataset get {dataset}",
|
|
717
|
-
) from exc
|
|
718
|
-
detail = _model_dict(record) if record is not None else None
|
|
719
|
-
return detail if detail is not None and _version_value(detail.get("version")) == version else None
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
def _versioned_splits(
|
|
723
|
-
dataset: str, version: str, *, stage: str, environment: Optional[str] = None
|
|
724
|
-
) -> list[dict]:
|
|
725
|
-
"""读**指定 version**的 split——走 SDK ``repo.splits``(recordsvc)。
|
|
726
|
-
|
|
727
|
-
与 ``_unversioned_splits`` 同源:version 文档不内嵌 ``splits[]``(服务端只给
|
|
728
|
-
``split_count`` 聚合,split 记录由 recordsvc 的 ``/datasets/splits`` 承载),
|
|
729
|
-
SDK ``splits.list(version=...)`` 是唯一正确读法,且自带存量 version_first
|
|
730
|
-
内嵌投影的 legacy 回退。
|
|
731
|
-
"""
|
|
732
|
-
from .irepo_sdk import _reason
|
|
733
|
-
|
|
734
|
-
env = _environment(environment)
|
|
735
|
-
repo = _dataset_repo()
|
|
736
|
-
try:
|
|
737
|
-
records: list[dict] = []
|
|
738
|
-
page = 1
|
|
739
|
-
while True:
|
|
740
|
-
result = repo.splits.list(
|
|
741
|
-
dataset,
|
|
742
|
-
version=version,
|
|
743
|
-
page_size=_SPLIT_PAGE_SIZE,
|
|
744
|
-
**({"page": page} if page > 1 else {}),
|
|
745
|
-
**_read_env_kwargs(repo, repo.splits, "list", env),
|
|
746
|
-
)
|
|
747
|
-
items = getattr(result, "items", None) or []
|
|
748
|
-
for item in items:
|
|
749
|
-
record = _model_dict(item)
|
|
750
|
-
# 与 ``_versioned_split`` 同口径:只收 version 匹配的记录(服务端
|
|
751
|
-
# exact scope 保证;legacy 投影与 OSS 发现也会带 version)。
|
|
752
|
-
if _version_value(record.get("version")) == version:
|
|
753
|
-
records.append(record)
|
|
754
|
-
if not items or getattr(result, "has_more", False) is not True:
|
|
755
|
-
break
|
|
756
|
-
page += 1
|
|
757
|
-
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
758
|
-
raise _fail(
|
|
759
|
-
f"dataset {dataset} version {version}: {stage} failed: {_reason(exc)}",
|
|
760
|
-
f"Check the version name: ap dataset version get {version} --dataset {dataset}",
|
|
761
|
-
) from exc
|
|
762
|
-
return records
|
|
763
|
-
|
|
764
|
-
|
|
765
694
|
def _split_records(detail: dict) -> list[dict]:
|
|
766
695
|
"""version 文档内嵌的 ``splits[]``(设计 §4.3:split 没有独立资源)。"""
|
|
767
696
|
splits = detail.get("splits")
|
|
@@ -795,12 +724,12 @@ def _split_names(detail: dict) -> list[str]:
|
|
|
795
724
|
return names
|
|
796
725
|
|
|
797
726
|
|
|
798
|
-
def _dataset_repo(
|
|
727
|
+
def _dataset_repo() -> "Repo":
|
|
799
728
|
"""dataset 写路径统一走 InstanceRepo SDK;未装 ``dataset`` extra 时退出 2。"""
|
|
800
729
|
from .irepo_sdk import SdkMissing, load_repo
|
|
801
730
|
|
|
802
731
|
try:
|
|
803
|
-
return load_repo(
|
|
732
|
+
return load_repo()
|
|
804
733
|
except SdkMissing as exc:
|
|
805
734
|
typer.echo(str(exc), err=True)
|
|
806
735
|
raise typer.Exit(2) from exc
|
|
@@ -1257,26 +1186,9 @@ def dataset_get(
|
|
|
1257
1186
|
"dataset_name": dataset,
|
|
1258
1187
|
"registered": False,
|
|
1259
1188
|
}
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
parse_dataset_name,
|
|
1264
|
-
)
|
|
1265
|
-
|
|
1266
|
-
try:
|
|
1267
|
-
name = parse_dataset_name(dataset)
|
|
1268
|
-
except DatasetNameUpgradeRequired as exc:
|
|
1269
|
-
raise _fail(
|
|
1270
|
-
f"Failed to query dataset {dataset}: {exc}",
|
|
1271
|
-
"pip install 'ap-client[dataset]' --upgrade",
|
|
1272
|
-
exit_code=2,
|
|
1273
|
-
) from exc
|
|
1274
|
-
except DatasetNameError:
|
|
1275
|
-
# 非规范名:不凭空发明 level 字段(与历史行为一致)。
|
|
1276
|
-
pass
|
|
1277
|
-
else:
|
|
1278
|
-
rendered["dataset_l1"] = name.l1
|
|
1279
|
-
rendered["dataset_l2"] = name.l2
|
|
1189
|
+
parts = dataset.split("/")
|
|
1190
|
+
if len(parts) == 2 and all(parts):
|
|
1191
|
+
rendered.update(dataset_l1=parts[0], dataset_l2=parts[1])
|
|
1280
1192
|
else:
|
|
1281
1193
|
rendered = _registered_dict(detail)
|
|
1282
1194
|
_emit_detail(rendered, output_format, order=_DATASET_DETAIL_ORDER, always=("status",))
|
|
@@ -1285,10 +1197,7 @@ def dataset_get(
|
|
|
1285
1197
|
|
|
1286
1198
|
|
|
1287
1199
|
def dataset_create(
|
|
1288
|
-
dataset: str = typer.Argument(
|
|
1289
|
-
..., help="Dataset name: L1, L1/L2, or a deeper path "
|
|
1290
|
-
"(e.g. my-bench, alibaba/my-bench, or gdpval/synthetic/my-bench)"
|
|
1291
|
-
),
|
|
1200
|
+
dataset: str = typer.Argument(..., help="Dataset name (e.g. alibaba/my-bench)"),
|
|
1292
1201
|
benchmark: str = typer.Option(..., "--benchmark", help="Benchmark name or source_id"),
|
|
1293
1202
|
description: Optional[str] = typer.Option(None, "--description", help="Dataset description"),
|
|
1294
1203
|
visibility: str = typer.Option(
|
|
@@ -1307,32 +1216,20 @@ def dataset_create(
|
|
|
1307
1216
|
|
|
1308
1217
|
from instance_repo.models import DatasetSeries
|
|
1309
1218
|
|
|
1310
|
-
from .dataset_name import (
|
|
1311
|
-
DatasetNameError,
|
|
1312
|
-
DatasetNameUpgradeRequired,
|
|
1313
|
-
parse_dataset_name,
|
|
1314
|
-
)
|
|
1315
1219
|
from .irepo_sdk import _reason
|
|
1316
1220
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
except DatasetNameUpgradeRequired as exc:
|
|
1221
|
+
parts = dataset.split("/")
|
|
1222
|
+
if len(parts) != 2 or not all(parts):
|
|
1320
1223
|
raise _fail(
|
|
1321
|
-
f"Failed to create dataset {dataset}:
|
|
1322
|
-
"
|
|
1323
|
-
exit_code=2,
|
|
1324
|
-
) from exc
|
|
1325
|
-
except DatasetNameError:
|
|
1326
|
-
raise _fail(
|
|
1327
|
-
f"Failed to create dataset {dataset}: the name must be one or more "
|
|
1328
|
-
"canonical path segments",
|
|
1329
|
-
"e.g. my-bench, your-team/my-bench, or gdpval/synthetic/my-bench",
|
|
1224
|
+
f"Failed to create dataset {dataset}: the name must be two segments <team>/<name>",
|
|
1225
|
+
"e.g. your-team/my-bench",
|
|
1330
1226
|
exit_code=2,
|
|
1331
1227
|
)
|
|
1228
|
+
l1, l2 = parts
|
|
1332
1229
|
series = DatasetSeries(
|
|
1333
1230
|
dataset_name=dataset,
|
|
1334
|
-
dataset_l1=
|
|
1335
|
-
dataset_l2=
|
|
1231
|
+
dataset_l1=l1,
|
|
1232
|
+
dataset_l2=l2,
|
|
1336
1233
|
benchmark_id=benchmark_id,
|
|
1337
1234
|
visibility=resolved_visibility,
|
|
1338
1235
|
description=description or None,
|
|
@@ -1916,10 +1813,15 @@ def dataset_split_list(
|
|
|
1916
1813
|
resolved_run_type = _check_choice(run_type, _RUN_TYPES, "--run-type") if run_type else ""
|
|
1917
1814
|
|
|
1918
1815
|
if resolved_version:
|
|
1919
|
-
|
|
1920
|
-
dataset,
|
|
1816
|
+
detail = _fetch_version(
|
|
1817
|
+
dataset,
|
|
1818
|
+
resolved_version,
|
|
1819
|
+
stage="split query",
|
|
1820
|
+
environment=environment,
|
|
1821
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
1921
1822
|
)
|
|
1922
|
-
|
|
1823
|
+
records = _split_records(detail)
|
|
1824
|
+
rows = [_split_row(record, detail) for record in records]
|
|
1923
1825
|
else:
|
|
1924
1826
|
records = _unversioned_splits(dataset, stage="split query", environment=environment)
|
|
1925
1827
|
rows = [_split_row(record, {}) for record in records]
|
|
@@ -1946,26 +1848,22 @@ def dataset_split_get(
|
|
|
1946
1848
|
resolved_version = _version_value(version)
|
|
1947
1849
|
|
|
1948
1850
|
if resolved_version:
|
|
1949
|
-
|
|
1950
|
-
dataset,
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
payload.setdefault("version", resolved_version)
|
|
1956
|
-
payload.setdefault("registered", True)
|
|
1957
|
-
_emit_detail(payload, output_format)
|
|
1958
|
-
return
|
|
1959
|
-
known = (
|
|
1960
|
-
", ".join(
|
|
1961
|
-
_split_names_of(
|
|
1962
|
-
_versioned_splits(
|
|
1963
|
-
dataset, resolved_version, stage="split query", environment=environment
|
|
1964
|
-
)
|
|
1965
|
-
)
|
|
1966
|
-
)
|
|
1967
|
-
or "<none>"
|
|
1851
|
+
detail = _fetch_version(
|
|
1852
|
+
dataset,
|
|
1853
|
+
resolved_version,
|
|
1854
|
+
stage="split query",
|
|
1855
|
+
environment=environment,
|
|
1856
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
1968
1857
|
)
|
|
1858
|
+
for record in _split_records(detail):
|
|
1859
|
+
if str(record.get("name") or "") == split:
|
|
1860
|
+
payload = dict(record)
|
|
1861
|
+
payload.setdefault("dataset", dataset)
|
|
1862
|
+
payload.setdefault("version", resolved_version)
|
|
1863
|
+
payload.setdefault("registered", detail.get("registered", True))
|
|
1864
|
+
_emit_detail(payload, output_format)
|
|
1865
|
+
return
|
|
1866
|
+
known = ", ".join(_split_names(detail)) or "<none>"
|
|
1969
1867
|
else:
|
|
1970
1868
|
record = _unversioned_split(dataset, split, environment=environment)
|
|
1971
1869
|
if record is not None:
|
|
@@ -2022,40 +1920,71 @@ def dataset_split_create(
|
|
|
2022
1920
|
exit_code=2,
|
|
2023
1921
|
)
|
|
2024
1922
|
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
1923
|
+
detail = _fetch_version(
|
|
1924
|
+
dataset,
|
|
1925
|
+
resolved_version,
|
|
1926
|
+
stage="split create precheck",
|
|
1927
|
+
environment=environment,
|
|
1928
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
1929
|
+
)
|
|
1930
|
+
|
|
1931
|
+
existing = _split_names(detail)
|
|
1932
|
+
if split in existing:
|
|
2031
1933
|
# 幂等:已存在同名 split(设计 §4.3)。
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
1934
|
+
for record in _split_records(detail):
|
|
1935
|
+
if str(record.get("name") or "") == split:
|
|
1936
|
+
payload = dict(record)
|
|
1937
|
+
payload.setdefault("dataset", dataset)
|
|
1938
|
+
payload.setdefault("version", resolved_version)
|
|
1939
|
+
payload["created"] = False
|
|
1940
|
+
_emit_detail(payload, output_format)
|
|
1941
|
+
return
|
|
1942
|
+
_emit_detail(
|
|
1943
|
+
{"dataset": dataset, "version": resolved_version, "split": split, "created": False},
|
|
1944
|
+
output_format,
|
|
1945
|
+
)
|
|
2037
1946
|
return
|
|
2038
1947
|
|
|
2039
|
-
|
|
2040
|
-
|
|
1948
|
+
# ① splits 全量 PATCH(现有 ∪ 新增)。
|
|
1949
|
+
names = existing + [split]
|
|
1950
|
+
_patch_version(
|
|
1951
|
+
dataset,
|
|
1952
|
+
resolved_version,
|
|
1953
|
+
{"splits": names},
|
|
1954
|
+
stage="split create",
|
|
1955
|
+
environment=environment,
|
|
1956
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
1957
|
+
)
|
|
2041
1958
|
|
|
2042
|
-
|
|
2043
|
-
create_kwargs = _sdk_read_kwargs(repo.splits, "create", environment=env, metadata_model="")
|
|
1959
|
+
# ② run_type 只能第二次 PATCH:服务端 splits 与 split_run_types 互斥。
|
|
2044
1960
|
if resolved_run_type:
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
1961
|
+
retry = (
|
|
1962
|
+
f"ap dataset split update {split} --dataset {dataset} --run-type {resolved_run_type}"
|
|
1963
|
+
)
|
|
1964
|
+
if resolved_version:
|
|
1965
|
+
retry += f" --version {resolved_version}"
|
|
1966
|
+
_patch_version(
|
|
1967
|
+
dataset,
|
|
1968
|
+
resolved_version,
|
|
1969
|
+
{"split_run_types": [{"name": split, "run_type": resolved_run_type}]},
|
|
1970
|
+
stage=f"split {split} set run_type",
|
|
1971
|
+
# 第一段 PATCH 已建出 split,失败后的恢复动作是补打 run_type,不是重跑 create。
|
|
1972
|
+
fallback_hint=f"split {split} was created but run_type is not set; set it now: {retry}",
|
|
1973
|
+
environment=environment,
|
|
1974
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
1975
|
+
)
|
|
1976
|
+
|
|
1977
|
+
_emit_detail(
|
|
1978
|
+
{
|
|
1979
|
+
"dataset": dataset,
|
|
1980
|
+
"version": resolved_version,
|
|
1981
|
+
"split": split,
|
|
1982
|
+
"run_type": resolved_run_type,
|
|
1983
|
+
"splits": names,
|
|
1984
|
+
"created": True,
|
|
1985
|
+
},
|
|
1986
|
+
output_format,
|
|
1987
|
+
)
|
|
2059
1988
|
|
|
2060
1989
|
|
|
2061
1990
|
@dataset_split_app.command("update")
|
|
@@ -2135,48 +2064,31 @@ def dataset_split_update(
|
|
|
2135
2064
|
)
|
|
2136
2065
|
return
|
|
2137
2066
|
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
)
|
|
2148
|
-
|
|
2149
|
-
if record is None:
|
|
2150
|
-
known = (
|
|
2151
|
-
", ".join(
|
|
2152
|
-
_split_names_of(
|
|
2153
|
-
_versioned_splits(
|
|
2154
|
-
dataset, resolved_version, stage="split update precheck",
|
|
2155
|
-
environment=environment,
|
|
2156
|
-
)
|
|
2157
|
-
)
|
|
2158
|
-
)
|
|
2159
|
-
or "<none>"
|
|
2160
|
-
)
|
|
2067
|
+
detail = _fetch_version(
|
|
2068
|
+
dataset,
|
|
2069
|
+
resolved_version,
|
|
2070
|
+
stage="split update precheck",
|
|
2071
|
+
environment=environment,
|
|
2072
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
2073
|
+
)
|
|
2074
|
+
|
|
2075
|
+
if split not in _split_names(detail):
|
|
2076
|
+
known = ", ".join(_split_names(detail)) or "<none>"
|
|
2077
|
+
hint = f"Create it first: ap dataset split create {split} --dataset {dataset} (existing: {known})"
|
|
2161
2078
|
raise _fail(
|
|
2162
2079
|
f"dataset {dataset} version {_version_label(resolved_version)} split {split}: "
|
|
2163
2080
|
f"update failed: not found",
|
|
2164
|
-
|
|
2165
|
-
f"--version {resolved_version} (existing: {known})",
|
|
2081
|
+
hint,
|
|
2166
2082
|
)
|
|
2167
2083
|
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
raise _fail(
|
|
2177
|
-
f"Failed to update split {split} of dataset {dataset} version {resolved_version}: {_reason(exc)}",
|
|
2178
|
-
"Write permission on this dataset is required; check the split name and the online environment, then retry",
|
|
2179
|
-
) from exc
|
|
2084
|
+
_patch_version(
|
|
2085
|
+
dataset,
|
|
2086
|
+
resolved_version,
|
|
2087
|
+
{"split_run_types": [{"name": split, "run_type": resolved_run_type}]},
|
|
2088
|
+
stage="split update",
|
|
2089
|
+
environment=environment,
|
|
2090
|
+
metadata_model=_VERSION_SPLIT_METADATA_MODEL,
|
|
2091
|
+
)
|
|
2180
2092
|
_emit_detail(
|
|
2181
2093
|
{
|
|
2182
2094
|
"dataset": dataset,
|
|
@@ -4,7 +4,6 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import math
|
|
7
|
-
import os
|
|
8
7
|
import sys
|
|
9
8
|
from typing import Any, List, Optional
|
|
10
9
|
from urllib.parse import quote
|
|
@@ -34,42 +33,6 @@ def _emit_preview_stderr(payload: dict, output_format: str) -> None:
|
|
|
34
33
|
typer.echo(json.dumps(payload, ensure_ascii=False, indent=2), err=True)
|
|
35
34
|
|
|
36
35
|
|
|
37
|
-
def _parse_request_timeout() -> float:
|
|
38
|
-
"""Read ``AP_REQUEST_TIMEOUT`` from the environment, falling back to 30 s.
|
|
39
|
-
|
|
40
|
-
Empty, missing, or non-numeric values all resolve to the default rather
|
|
41
|
-
than raising ``ValueError`` at import time (which would break ``ap --help``).
|
|
42
|
-
"""
|
|
43
|
-
raw = os.environ.get("AP_REQUEST_TIMEOUT", "").strip()
|
|
44
|
-
if not raw:
|
|
45
|
-
return 30.0
|
|
46
|
-
try:
|
|
47
|
-
value = float(raw)
|
|
48
|
-
except (TypeError, ValueError):
|
|
49
|
-
return 30.0
|
|
50
|
-
return value if math.isfinite(value) and value > 0 else 30.0
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def _set_sdk_request_timeout(repo: Any, timeout: float) -> None:
|
|
54
|
-
"""Override the IR SDK per-request HTTP timeout on the shared Transport.
|
|
55
|
-
|
|
56
|
-
All sub-clients (``datasets``, ``versions``, ``instances``) share
|
|
57
|
-
``repo.transport``, so a single assignment covers every control-plane
|
|
58
|
-
call the register command makes (preview and submission alike).
|
|
59
|
-
Falls back to setting the attribute on each sub-client's ``_t`` when the
|
|
60
|
-
``transport`` attribute is absent (older SDK shapes).
|
|
61
|
-
"""
|
|
62
|
-
transport = getattr(repo, "transport", None)
|
|
63
|
-
if transport is not None and hasattr(transport, "timeout"):
|
|
64
|
-
transport.timeout = timeout
|
|
65
|
-
return
|
|
66
|
-
for name in ("datasets", "versions", "instances", "images"):
|
|
67
|
-
client = getattr(repo, name, None)
|
|
68
|
-
t = getattr(client, "_t", None)
|
|
69
|
-
if t is not None and hasattr(t, "timeout"):
|
|
70
|
-
t.timeout = timeout
|
|
71
|
-
|
|
72
|
-
|
|
73
36
|
def _confirm(dataset: str, *, yes: bool) -> None:
|
|
74
37
|
if yes:
|
|
75
38
|
return
|
|
@@ -103,26 +66,11 @@ def _claim_without_workspace(
|
|
|
103
66
|
|
|
104
67
|
|
|
105
68
|
def _create_owned_dataset(repo: Any, dataset: str, benchmark_id: str) -> dict:
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
DatasetNameUpgradeRequired,
|
|
109
|
-
parse_dataset_name,
|
|
110
|
-
)
|
|
111
|
-
|
|
112
|
-
try:
|
|
113
|
-
name = parse_dataset_name(dataset)
|
|
114
|
-
except DatasetNameUpgradeRequired as exc:
|
|
69
|
+
parts = dataset.split("/")
|
|
70
|
+
if len(parts) != 2 or not all(parts):
|
|
115
71
|
raise datasets._fail(
|
|
116
|
-
f"Failed to create dataset {dataset}:
|
|
117
|
-
"
|
|
118
|
-
exit_code=2,
|
|
119
|
-
) from exc
|
|
120
|
-
except DatasetNameError:
|
|
121
|
-
raise datasets._fail(
|
|
122
|
-
f"Failed to create dataset {dataset}: the name must be one or more "
|
|
123
|
-
"canonical path segments",
|
|
124
|
-
"Use a dataset name such as my-bench, your-team/my-bench, "
|
|
125
|
-
"or gdpval/synthetic/my-bench",
|
|
72
|
+
f"Failed to create dataset {dataset}: the name must be two segments <team>/<name>",
|
|
73
|
+
"Use a dataset name such as your-team/my-bench",
|
|
126
74
|
exit_code=2,
|
|
127
75
|
)
|
|
128
76
|
|
|
@@ -130,8 +78,8 @@ def _create_owned_dataset(repo: Any, dataset: str, benchmark_id: str) -> dict:
|
|
|
130
78
|
|
|
131
79
|
fields: dict[str, Any] = {
|
|
132
80
|
"dataset_name": dataset,
|
|
133
|
-
"dataset_l1":
|
|
134
|
-
"dataset_l2":
|
|
81
|
+
"dataset_l1": parts[0],
|
|
82
|
+
"dataset_l2": parts[1],
|
|
135
83
|
"visibility": "private",
|
|
136
84
|
}
|
|
137
85
|
if benchmark_id:
|
|
@@ -258,13 +206,6 @@ def dataset_register(
|
|
|
258
206
|
yes: bool = typer.Option(False, "--yes", "-y", help="Skip the registration confirmation"),
|
|
259
207
|
wait: bool = typer.Option(False, "--wait", help="Wait for the ingest task to finish"),
|
|
260
208
|
timeout: float = typer.Option(1800.0, "--timeout", help="Local wait timeout in seconds"),
|
|
261
|
-
request_timeout: float = typer.Option(
|
|
262
|
-
_parse_request_timeout(),
|
|
263
|
-
"--request-timeout",
|
|
264
|
-
help="Per-request HTTP timeout in seconds for control-plane calls "
|
|
265
|
-
"(env: AP_REQUEST_TIMEOUT, default: 30). "
|
|
266
|
-
"Increase for datasets with many objects where the preview scan takes longer.",
|
|
267
|
-
),
|
|
268
209
|
output_format: Optional[str] = typer.Option(None, "--format", help=datasets._FORMAT_HELP),
|
|
269
210
|
owner: Optional[str] = typer.Option(
|
|
270
211
|
None, "--owner", help="Ensure ownership before ingest; the only supported value is @me"
|
|
@@ -318,17 +259,7 @@ def dataset_register(
|
|
|
318
259
|
)
|
|
319
260
|
|
|
320
261
|
env = datasets._environment(environment)
|
|
321
|
-
repo = datasets._dataset_repo(
|
|
322
|
-
# Override the IR SDK per-request HTTP timeout. All sub-clients
|
|
323
|
-
# (datasets, versions, instances) share repo.transport, so one
|
|
324
|
-
# assignment covers preview and submission alike.
|
|
325
|
-
if not math.isfinite(request_timeout) or request_timeout <= 0:
|
|
326
|
-
raise datasets._fail(
|
|
327
|
-
"--request-timeout must be a positive finite number",
|
|
328
|
-
"Pass a positive number of seconds",
|
|
329
|
-
exit_code=2,
|
|
330
|
-
)
|
|
331
|
-
_set_sdk_request_timeout(repo, request_timeout)
|
|
262
|
+
repo = datasets._dataset_repo()
|
|
332
263
|
datasets._check_storage_side(repo, env)
|
|
333
264
|
ingest = getattr(getattr(repo, "versions", None), "ingest", None)
|
|
334
265
|
if not callable(ingest):
|