ap-client 0.2.1__tar.gz → 0.2.1.dev0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/PKG-INFO +3 -3
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/api.py +266 -83
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/cli.py +28 -82
- ap_client-0.2.1.dev0/ap_client/dataset_commands.py +1374 -0
- ap_client-0.2.1.dev0/ap_client/fs_commands.py +566 -0
- ap_client-0.2.1.dev0/ap_client/instance_commands.py +506 -0
- ap_client-0.2.1.dev0/ap_client/irepo_sdk.py +326 -0
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/profile_commands.py +2 -1
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/pyproject.toml +3 -3
- ap_client-0.2.1/ap_client/irepo_commands.py +0 -131
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/__init__.py +0 -0
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/config.py +0 -0
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/exporter.py +0 -0
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/tbb.py +0 -0
- {ap_client-0.2.1 → ap_client-0.2.1.dev0}/ap_client/waiter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.2.1
|
|
3
|
+
Version: 0.2.1.dev0
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -9,9 +9,9 @@ Requires-Dist: rich>=13.0.0
|
|
|
9
9
|
Requires-Dist: typer>=0.9.0
|
|
10
10
|
Requires-Dist: websockets>=13.0
|
|
11
11
|
Provides-Extra: all
|
|
12
|
-
Requires-Dist: instance-repo[oss]>=0.
|
|
12
|
+
Requires-Dist: instance-repo[oss]>=1.0.3; extra == 'all'
|
|
13
13
|
Provides-Extra: dataset
|
|
14
|
-
Requires-Dist: instance-repo[oss]>=0.
|
|
14
|
+
Requires-Dist: instance-repo[oss]>=1.0.3; extra == 'dataset'
|
|
15
15
|
Description-Content-Type: text/markdown
|
|
16
16
|
|
|
17
17
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -268,20 +268,6 @@ class APIError(Exception):
|
|
|
268
268
|
return f"API Error {self.status_code}: {self.detail} (request_id={self.request_id})"
|
|
269
269
|
|
|
270
270
|
|
|
271
|
-
class PaginationProtocolError(RuntimeError):
|
|
272
|
-
"""The server stopped honoring a pagination protocol already in use."""
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
def _ensure_next_token_honored(response: dict, *, next_token: Optional[str], endpoint: str) -> None:
|
|
276
|
-
# An old server ignores the unknown next_token param and answers with its
|
|
277
|
-
# default first page, so a `response.get("next_token")` loop would silently
|
|
278
|
-
# repeat that page and drop the rest of the collection.
|
|
279
|
-
if next_token and "next_token" not in response:
|
|
280
|
-
raise PaginationProtocolError(
|
|
281
|
-
f"server stopped returning next_token while paginating {endpoint}; upgrade the server"
|
|
282
|
-
)
|
|
283
|
-
|
|
284
|
-
|
|
285
271
|
def _secret_ws_params(workspace_id: Optional[str]) -> Optional[dict]:
|
|
286
272
|
"""Build the ``?workspace_id=`` query for name-addressed secret endpoints.
|
|
287
273
|
|
|
@@ -293,6 +279,48 @@ def _secret_ws_params(workspace_id: Optional[str]) -> Optional[dict]:
|
|
|
293
279
|
return {"workspace_id": workspace_id} if workspace_id else None
|
|
294
280
|
|
|
295
281
|
|
|
282
|
+
def _unwrap_data(payload: Any) -> Any:
|
|
283
|
+
"""Unwrap the apiserver ``{code,message,data}`` envelope when present.
|
|
284
|
+
|
|
285
|
+
The dataset-domain detail/create/patch endpoints return a bare JSON body,
|
|
286
|
+
but the same handlers are occasionally wrapped by the generic success
|
|
287
|
+
envelope. Tolerate both so command code never has to branch: a mapping that
|
|
288
|
+
carries ``data`` alongside ``code``/``message`` is treated as an envelope,
|
|
289
|
+
anything else is returned verbatim.
|
|
290
|
+
"""
|
|
291
|
+
if (
|
|
292
|
+
isinstance(payload, dict)
|
|
293
|
+
and "data" in payload
|
|
294
|
+
and ("code" in payload or "message" in payload)
|
|
295
|
+
):
|
|
296
|
+
return payload["data"]
|
|
297
|
+
return payload
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _split_paged(payload: Any) -> tuple[list[dict], dict]:
|
|
301
|
+
"""Split a ``PagedSuccess`` envelope into ``(records, pagination)``.
|
|
302
|
+
|
|
303
|
+
``PagedSuccess`` is ``{code,message,data:[...],pagination:{total,page,page_size}}``.
|
|
304
|
+
A bare list (or a null ``data``) yields an empty pagination dict so callers
|
|
305
|
+
can render results without probing the response shape.
|
|
306
|
+
"""
|
|
307
|
+
if isinstance(payload, list):
|
|
308
|
+
return [item for item in payload if isinstance(item, dict)], {}
|
|
309
|
+
if not isinstance(payload, dict):
|
|
310
|
+
return [], {}
|
|
311
|
+
records = payload.get("data")
|
|
312
|
+
pagination = payload.get("pagination")
|
|
313
|
+
return (
|
|
314
|
+
[item for item in (records or []) if isinstance(item, dict)],
|
|
315
|
+
pagination if isinstance(pagination, dict) else {},
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _as_record_list(payload: Any) -> list[dict]:
|
|
320
|
+
"""Coerce a list-returning endpoint to ``list[dict]``, envelope or not."""
|
|
321
|
+
return _split_paged(payload)[0]
|
|
322
|
+
|
|
323
|
+
|
|
296
324
|
class APIClient:
|
|
297
325
|
"""Agent Platform API client."""
|
|
298
326
|
|
|
@@ -844,6 +872,211 @@ class APIClient:
|
|
|
844
872
|
"instance_ids": all_instance_ids,
|
|
845
873
|
}
|
|
846
874
|
|
|
875
|
+
# ==================== Dataset series (apiserver dataset domain) ====================
|
|
876
|
+
#
|
|
877
|
+
# 这一节对接 Go apiserver 的 dataset 领域(/apis/v1/datasets/...),与上面
|
|
878
|
+
# ossdata 目录的 /api/datasets 路由完全无关。方法名统一带 `_series`,
|
|
879
|
+
# 以免和 `ap job create --dataset` 依赖的 list_all_datasets /
|
|
880
|
+
# list_dataset_versions / list_all_dataset_instances 混淆。
|
|
881
|
+
#
|
|
882
|
+
# 列表端点返回 PagedSuccess envelope:
|
|
883
|
+
# {"code":0,"message":"ok","data":[...],"pagination":{"total","page","page_size"}}
|
|
884
|
+
# 详情/创建/更新端点返回裸 JSON body。
|
|
885
|
+
|
|
886
|
+
_DATASET_SERIES_BASE = "/apis/v1/datasets/series"
|
|
887
|
+
_DATASET_VERSIONS_BASE = "/apis/v1/datasets/versions"
|
|
888
|
+
_DATASET_INSTANCES_BASE = "/apis/v1/datasets/instances"
|
|
889
|
+
|
|
890
|
+
def list_dataset_series_page(
|
|
891
|
+
self,
|
|
892
|
+
*,
|
|
893
|
+
q: Optional[str] = None,
|
|
894
|
+
keyword: Optional[str] = None,
|
|
895
|
+
visibility: Optional[str] = None,
|
|
896
|
+
benchmark_id: Optional[str] = None,
|
|
897
|
+
include_deprecated: bool = False,
|
|
898
|
+
environment: Optional[str] = None,
|
|
899
|
+
page: int = 1,
|
|
900
|
+
page_size: int = 20,
|
|
901
|
+
) -> dict:
|
|
902
|
+
"""一页 dataset series(原始 PagedSuccess envelope)。
|
|
903
|
+
|
|
904
|
+
服务端没有 owner/mine/l1 过滤参数,调用方需要时只能在客户端做页内过滤。
|
|
905
|
+
"""
|
|
906
|
+
params: dict = {"page": page, "page_size": page_size}
|
|
907
|
+
if q:
|
|
908
|
+
params["q"] = q
|
|
909
|
+
if keyword:
|
|
910
|
+
params["keyword"] = keyword
|
|
911
|
+
if visibility:
|
|
912
|
+
params["visibility"] = visibility
|
|
913
|
+
if benchmark_id:
|
|
914
|
+
params["benchmark_id"] = benchmark_id
|
|
915
|
+
if include_deprecated:
|
|
916
|
+
params["include_deprecated"] = "true"
|
|
917
|
+
if environment:
|
|
918
|
+
params["environment"] = environment
|
|
919
|
+
return self._get(self._DATASET_SERIES_BASE, params=params)
|
|
920
|
+
|
|
921
|
+
def list_dataset_series(self, **kw) -> tuple[list[dict], dict]:
|
|
922
|
+
"""列出 dataset series,返回 ``(records, pagination)``。"""
|
|
923
|
+
return _split_paged(self.list_dataset_series_page(**kw))
|
|
924
|
+
|
|
925
|
+
def get_dataset_series(self, dataset_name: str, *, environment: Optional[str] = None) -> dict:
|
|
926
|
+
"""按名字取 dataset series 详情(含 dataset_id / claimable / stats)。"""
|
|
927
|
+
params: dict = {"dataset_name": dataset_name}
|
|
928
|
+
if environment:
|
|
929
|
+
params["environment"] = environment
|
|
930
|
+
return _unwrap_data(self._get(f"{self._DATASET_SERIES_BASE}/detail", params=params))
|
|
931
|
+
|
|
932
|
+
def create_dataset_series(self, body: dict) -> dict:
|
|
933
|
+
"""创建 dataset series。owner 由服务端按调用者身份填充。"""
|
|
934
|
+
return _unwrap_data(self._post(self._DATASET_SERIES_BASE, body))
|
|
935
|
+
|
|
936
|
+
def update_dataset_series(self, dataset_name: str, body: dict) -> dict:
|
|
937
|
+
"""PATCH dataset series 元数据(visibility/status 需要 dataset-admin)。"""
|
|
938
|
+
return _unwrap_data(
|
|
939
|
+
self._request(
|
|
940
|
+
"PATCH",
|
|
941
|
+
f"{self._DATASET_SERIES_BASE}/detail",
|
|
942
|
+
params={"dataset_name": dataset_name},
|
|
943
|
+
json_body=body,
|
|
944
|
+
)
|
|
945
|
+
)
|
|
946
|
+
|
|
947
|
+
def claim_dataset_series(self, dataset_id: str, body: dict) -> dict:
|
|
948
|
+
"""认领 dataset series(``workspace_id`` 服务端必填);成功后调用方成为 dataset admin。"""
|
|
949
|
+
return _unwrap_data(
|
|
950
|
+
self._post(f"{self._DATASET_SERIES_BASE}/{quote(dataset_id, safe='')}/claim", body)
|
|
951
|
+
)
|
|
952
|
+
|
|
953
|
+
def list_claim_workspaces(self) -> list[dict]:
|
|
954
|
+
"""列出当前用户可用于认领的 workspace。"""
|
|
955
|
+
return _as_record_list(self._get("/apis/v1/datasets/claim-workspaces"))
|
|
956
|
+
|
|
957
|
+
# ---- dataset versions ----
|
|
958
|
+
|
|
959
|
+
def list_dataset_series_versions(
|
|
960
|
+
self,
|
|
961
|
+
dataset_name: str,
|
|
962
|
+
*,
|
|
963
|
+
status: Optional[str] = None,
|
|
964
|
+
page: int = 1,
|
|
965
|
+
page_size: int = 20,
|
|
966
|
+
environment: Optional[str] = None,
|
|
967
|
+
) -> tuple[list[dict], dict]:
|
|
968
|
+
"""列出某 dataset 的版本,返回 ``(records, pagination)``。列表项不含 splits 数组。"""
|
|
969
|
+
params: dict = {"dataset_name": dataset_name, "page": page, "page_size": page_size}
|
|
970
|
+
if status:
|
|
971
|
+
params["status"] = status
|
|
972
|
+
if environment:
|
|
973
|
+
params["environment"] = environment
|
|
974
|
+
return _split_paged(self._get(self._DATASET_VERSIONS_BASE, params=params))
|
|
975
|
+
|
|
976
|
+
def get_dataset_series_version(
|
|
977
|
+
self,
|
|
978
|
+
dataset_name: str,
|
|
979
|
+
version: str,
|
|
980
|
+
*,
|
|
981
|
+
environment: Optional[str] = None,
|
|
982
|
+
) -> dict:
|
|
983
|
+
"""取单个版本详情(含 run_type / splits[] / manifest / published_at)。
|
|
984
|
+
|
|
985
|
+
``version=""`` 表示 split_first 的**无版本空间**,原样下发空串——绝不隐式取 latest。
|
|
986
|
+
"""
|
|
987
|
+
params: dict = {"dataset_name": dataset_name, "version": version}
|
|
988
|
+
if environment:
|
|
989
|
+
params["environment"] = environment
|
|
990
|
+
return _unwrap_data(self._get(f"{self._DATASET_VERSIONS_BASE}/detail", params=params))
|
|
991
|
+
|
|
992
|
+
def create_dataset_series_version(self, dataset_name: str, body: dict) -> dict:
|
|
993
|
+
"""创建版本(``version``/``storage_type``/``storage_path``/``splits``/``status``)。"""
|
|
994
|
+
return _unwrap_data(
|
|
995
|
+
self._request(
|
|
996
|
+
"POST",
|
|
997
|
+
self._DATASET_VERSIONS_BASE,
|
|
998
|
+
params={"dataset_name": dataset_name},
|
|
999
|
+
json_body=body,
|
|
1000
|
+
)
|
|
1001
|
+
)
|
|
1002
|
+
|
|
1003
|
+
def update_dataset_series_version(self, dataset_name: str, version: str, body: dict) -> dict:
|
|
1004
|
+
"""PATCH 版本(``status``/``splits``/``manifest``/``run_type``/``split_run_types``)。
|
|
1005
|
+
|
|
1006
|
+
服务端要求 ``splits`` 与 ``split_run_types`` 互斥,调用方需分两次 PATCH。
|
|
1007
|
+
"""
|
|
1008
|
+
return _unwrap_data(
|
|
1009
|
+
self._request(
|
|
1010
|
+
"PATCH",
|
|
1011
|
+
f"{self._DATASET_VERSIONS_BASE}/detail",
|
|
1012
|
+
params={"dataset_name": dataset_name, "version": version},
|
|
1013
|
+
json_body=body,
|
|
1014
|
+
)
|
|
1015
|
+
)
|
|
1016
|
+
|
|
1017
|
+
# ---- dataset instances (metadata only) ----
|
|
1018
|
+
|
|
1019
|
+
def list_dataset_series_instances(
|
|
1020
|
+
self,
|
|
1021
|
+
dataset_name: str,
|
|
1022
|
+
*,
|
|
1023
|
+
version: Optional[str] = None,
|
|
1024
|
+
split: Optional[str] = None,
|
|
1025
|
+
instance_id: Optional[str] = None,
|
|
1026
|
+
page: int = 1,
|
|
1027
|
+
page_size: int = 50,
|
|
1028
|
+
environment: Optional[str] = None,
|
|
1029
|
+
) -> tuple[list[dict], dict]:
|
|
1030
|
+
"""列出 instance 元数据,返回 ``(records, pagination)``。
|
|
1031
|
+
|
|
1032
|
+
``version=""`` / ``split=""`` 原样下发(无版本空间语义),``None`` 才省略该参数。
|
|
1033
|
+
"""
|
|
1034
|
+
params: dict = {"dataset_name": dataset_name, "page": page, "page_size": page_size}
|
|
1035
|
+
if version is not None:
|
|
1036
|
+
params["version"] = version
|
|
1037
|
+
if split is not None:
|
|
1038
|
+
params["split"] = split
|
|
1039
|
+
if instance_id:
|
|
1040
|
+
params["instance_id"] = instance_id
|
|
1041
|
+
if environment:
|
|
1042
|
+
params["environment"] = environment
|
|
1043
|
+
return _split_paged(self._get(self._DATASET_INSTANCES_BASE, params=params))
|
|
1044
|
+
|
|
1045
|
+
def get_dataset_series_instance(
|
|
1046
|
+
self,
|
|
1047
|
+
dataset_name: str,
|
|
1048
|
+
version: str,
|
|
1049
|
+
split: str,
|
|
1050
|
+
instance_id: str,
|
|
1051
|
+
*,
|
|
1052
|
+
environment: Optional[str] = None,
|
|
1053
|
+
) -> dict:
|
|
1054
|
+
"""取单个 instance 的元数据详情。``version=""`` 表示无版本空间。"""
|
|
1055
|
+
params: dict = {
|
|
1056
|
+
"dataset_name": dataset_name,
|
|
1057
|
+
"version": version,
|
|
1058
|
+
"split": split,
|
|
1059
|
+
"instance_id": instance_id,
|
|
1060
|
+
}
|
|
1061
|
+
if environment:
|
|
1062
|
+
params["environment"] = environment
|
|
1063
|
+
return _unwrap_data(self._get(f"{self._DATASET_INSTANCES_BASE}/detail", params=params))
|
|
1064
|
+
|
|
1065
|
+
# ---- permissions ----
|
|
1066
|
+
|
|
1067
|
+
def get_my_resource_permissions(self, resource_type: str, resource_id: str) -> list[str]:
|
|
1068
|
+
"""当前用户在某资源上的 action 列表(``{"actions":[...]}``)。"""
|
|
1069
|
+
payload = self._get(
|
|
1070
|
+
"/apis/v1/me/resource-permissions",
|
|
1071
|
+
params={"resource_type": resource_type, "resource_id": resource_id},
|
|
1072
|
+
)
|
|
1073
|
+
payload = _unwrap_data(payload)
|
|
1074
|
+
if isinstance(payload, dict):
|
|
1075
|
+
actions = payload.get("actions")
|
|
1076
|
+
if isinstance(actions, list):
|
|
1077
|
+
return [str(action) for action in actions]
|
|
1078
|
+
return []
|
|
1079
|
+
|
|
847
1080
|
# ==================== Meta operations ====================
|
|
848
1081
|
|
|
849
1082
|
def list_meta_models(self) -> dict:
|
|
@@ -1281,9 +1514,6 @@ class APIClient:
|
|
|
1281
1514
|
meta_job_type: Optional[str] = None,
|
|
1282
1515
|
upstream_platform: Optional[str] = None,
|
|
1283
1516
|
upstream_job_id: Optional[str] = None,
|
|
1284
|
-
next_token: Optional[str] = None,
|
|
1285
|
-
pagination: Optional[str] = None,
|
|
1286
|
-
include_total: bool = True,
|
|
1287
1517
|
) -> dict:
|
|
1288
1518
|
"""List jobs."""
|
|
1289
1519
|
params = {"skip": skip, "limit": limit}
|
|
@@ -1341,15 +1571,7 @@ class APIClient:
|
|
|
1341
1571
|
params["created_at_sort"] = created_at_sort
|
|
1342
1572
|
if finished_at_sort is not None:
|
|
1343
1573
|
params["finished_at_sort"] = finished_at_sort
|
|
1344
|
-
|
|
1345
|
-
params["next_token"] = next_token
|
|
1346
|
-
if pagination:
|
|
1347
|
-
params["pagination"] = pagination
|
|
1348
|
-
if not include_total:
|
|
1349
|
-
params["include_total"] = False
|
|
1350
|
-
response = self._get("/jobs", params=params)
|
|
1351
|
-
_ensure_next_token_honored(response, next_token=next_token, endpoint="/jobs")
|
|
1352
|
-
return response
|
|
1574
|
+
return self._get("/jobs", params=params)
|
|
1353
1575
|
|
|
1354
1576
|
# ==================== Group operations ====================
|
|
1355
1577
|
|
|
@@ -1540,9 +1762,6 @@ class APIClient:
|
|
|
1540
1762
|
meta_job_type: Optional[str] = None,
|
|
1541
1763
|
upstream_platform: Optional[str] = None,
|
|
1542
1764
|
upstream_job_id: Optional[str] = None,
|
|
1543
|
-
next_token: Optional[str] = None,
|
|
1544
|
-
pagination: Optional[str] = None,
|
|
1545
|
-
include_total: bool = True,
|
|
1546
1765
|
) -> dict:
|
|
1547
1766
|
"""List jobs in a group."""
|
|
1548
1767
|
params: dict = {"skip": skip, "limit": limit}
|
|
@@ -1566,16 +1785,7 @@ class APIClient:
|
|
|
1566
1785
|
params["upstream_job_id"] = upstream_job_id
|
|
1567
1786
|
if include_post_process:
|
|
1568
1787
|
params["include_post_process"] = True
|
|
1569
|
-
|
|
1570
|
-
params["next_token"] = next_token
|
|
1571
|
-
if pagination:
|
|
1572
|
-
params["pagination"] = pagination
|
|
1573
|
-
if not include_total:
|
|
1574
|
-
params["include_total"] = False
|
|
1575
|
-
endpoint = f"/groups/{quote(group_id)}/jobs"
|
|
1576
|
-
response = self._get(endpoint, params=params)
|
|
1577
|
-
_ensure_next_token_honored(response, next_token=next_token, endpoint=endpoint)
|
|
1578
|
-
return response
|
|
1788
|
+
return self._get(f"/groups/{quote(group_id)}/jobs", params=params)
|
|
1579
1789
|
|
|
1580
1790
|
def get_group_post_process_job_id(
|
|
1581
1791
|
self, group_id: str, timeout: TimeoutType = None
|
|
@@ -2053,30 +2263,19 @@ class APIClient:
|
|
|
2053
2263
|
skip: int = 0,
|
|
2054
2264
|
limit: int = _GROUP_ARTIFACTS_PAGE_SIZE,
|
|
2055
2265
|
include_post_process: bool = False,
|
|
2056
|
-
next_token: Optional[str] = None,
|
|
2057
|
-
pagination: Optional[str] = None,
|
|
2058
|
-
include_total: bool = True,
|
|
2059
2266
|
) -> dict:
|
|
2060
2267
|
"""Get one page of artifact download links for a group."""
|
|
2061
2268
|
params: dict[str, object] = {"skip": skip, "limit": limit}
|
|
2062
2269
|
if include_post_process:
|
|
2063
2270
|
params["include_post_process"] = True
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
if not include_total:
|
|
2069
|
-
params["include_total"] = False
|
|
2070
|
-
endpoint = f"/groups/{quote(group_id)}/artifacts"
|
|
2071
|
-
response = self._get(endpoint, params=params)
|
|
2072
|
-
_ensure_next_token_honored(response, next_token=next_token, endpoint=endpoint)
|
|
2073
|
-
return response
|
|
2271
|
+
return self._get(
|
|
2272
|
+
f"/groups/{quote(group_id)}/artifacts",
|
|
2273
|
+
params=params,
|
|
2274
|
+
)
|
|
2074
2275
|
|
|
2075
2276
|
def get_group_artifacts(self, group_id: str, include_post_process: bool = False) -> dict:
|
|
2076
2277
|
"""Get artifact download links for a group."""
|
|
2077
|
-
next_token: str | None = None
|
|
2078
2278
|
skip = 0
|
|
2079
|
-
mode: str | None = None
|
|
2080
2279
|
artifacts: list[dict] = []
|
|
2081
2280
|
last_page: dict | None = None
|
|
2082
2281
|
|
|
@@ -2086,41 +2285,25 @@ class APIClient:
|
|
|
2086
2285
|
skip=skip,
|
|
2087
2286
|
limit=_GROUP_ARTIFACTS_PAGE_SIZE,
|
|
2088
2287
|
include_post_process=include_post_process,
|
|
2089
|
-
next_token=next_token,
|
|
2090
|
-
pagination="cursor" if mode in {None, "cursor"} else None,
|
|
2091
|
-
include_total=False,
|
|
2092
2288
|
)
|
|
2093
2289
|
last_page = page
|
|
2094
2290
|
page_artifacts = page.get("artifacts") or []
|
|
2095
2291
|
artifacts.extend(page_artifacts)
|
|
2096
|
-
has_token_field = "next_token" in page
|
|
2097
2292
|
|
|
2098
|
-
|
|
2099
|
-
|
|
2293
|
+
total = page.get("total")
|
|
2294
|
+
if total is not None and len(artifacts) >= total:
|
|
2295
|
+
break
|
|
2296
|
+
if len(page_artifacts) < _GROUP_ARTIFACTS_PAGE_SIZE:
|
|
2297
|
+
break
|
|
2100
2298
|
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
break
|
|
2110
|
-
if total is not None and skip >= total:
|
|
2111
|
-
break
|
|
2112
|
-
|
|
2113
|
-
result = dict(last_page or {})
|
|
2114
|
-
result.pop("next_token", None)
|
|
2115
|
-
result.update(
|
|
2116
|
-
{
|
|
2117
|
-
"total": len(artifacts),
|
|
2118
|
-
"skip": 0,
|
|
2119
|
-
"limit": len(artifacts),
|
|
2120
|
-
"artifacts": artifacts,
|
|
2121
|
-
}
|
|
2122
|
-
)
|
|
2123
|
-
return result
|
|
2299
|
+
skip += len(page_artifacts)
|
|
2300
|
+
|
|
2301
|
+
return {
|
|
2302
|
+
**last_page,
|
|
2303
|
+
"skip": 0,
|
|
2304
|
+
"limit": len(artifacts),
|
|
2305
|
+
"artifacts": artifacts,
|
|
2306
|
+
}
|
|
2124
2307
|
|
|
2125
2308
|
def get_job_artifacts(self, job_ids: list) -> list:
|
|
2126
2309
|
"""Get artifact download links for one or more jobs."""
|
|
@@ -16,14 +16,15 @@ from ap_client import __version__, get_client, get_config, tbb
|
|
|
16
16
|
from ap_client.api import (
|
|
17
17
|
_OTHERS_MODEL,
|
|
18
18
|
APIError,
|
|
19
|
-
PaginationProtocolError,
|
|
20
19
|
_merge_meta_tags,
|
|
21
20
|
set_cluster_override,
|
|
22
21
|
set_verbose_override,
|
|
23
22
|
)
|
|
24
23
|
from ap_client.config import ENV_VAR_SPECS, ConfigurationError, _parse_bool, normalize_output_format
|
|
24
|
+
from ap_client.dataset_commands import register as _register_dataset_commands
|
|
25
25
|
from ap_client.exporter import export_group, export_job
|
|
26
|
-
from ap_client.
|
|
26
|
+
from ap_client.fs_commands import register as _register_fs_commands
|
|
27
|
+
from ap_client.instance_commands import register as _register_instance_commands
|
|
27
28
|
from ap_client.profile_commands import profile_app
|
|
28
29
|
from ap_client.waiter import (
|
|
29
30
|
WaitTimeoutError,
|
|
@@ -144,8 +145,11 @@ app.add_typer(meta_job_type_app, name="meta-job-type")
|
|
|
144
145
|
app.add_typer(benchmark_app, name="benchmark")
|
|
145
146
|
app.add_typer(checkpoint_app, name="checkpoint")
|
|
146
147
|
|
|
147
|
-
#
|
|
148
|
-
|
|
148
|
+
# Dataset 领域(见 docs/ap-cli-dataset-refactor-design.md):
|
|
149
|
+
# ap dataset version/split/access 嵌套在 dataset 下;instance/fs 是一等资源。
|
|
150
|
+
_register_dataset_commands(dataset_app)
|
|
151
|
+
_register_instance_commands(app)
|
|
152
|
+
_register_fs_commands(app)
|
|
149
153
|
|
|
150
154
|
_PAI_RUNTIME_ENV_TAGS: tuple[tuple[str, str], ...] = (
|
|
151
155
|
("DLC_JOB_ID", "dlc_job_id"),
|
|
@@ -924,7 +928,10 @@ def _print_params_plain(params: Any, masked_params: list) -> None:
|
|
|
924
928
|
rows: list[tuple[str, Any]] = []
|
|
925
929
|
for key, value in params.items():
|
|
926
930
|
display_value = _format_plain_value(value)
|
|
927
|
-
|
|
931
|
+
# 直接信任 masked_params 路径:服务端掩码形态已扩展为保留首尾(如
|
|
932
|
+
# sk-***890 / ****),不能再按值严格等于 "***" 判断,否则形态变化后
|
|
933
|
+
# (masked) 标识静默丢失
|
|
934
|
+
if str(key) in masked_paths:
|
|
928
935
|
display_value = f"{display_value} (masked)"
|
|
929
936
|
rows.append((str(key), display_value))
|
|
930
937
|
_print_key_values(rows, skip_empty=False)
|
|
@@ -2015,24 +2022,17 @@ def template_fetch(
|
|
|
2015
2022
|
|
|
2016
2023
|
# ==================== Dataset operations ====================
|
|
2017
2024
|
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
2030
|
-
client = get_client()
|
|
2031
|
-
result = client.list_all_datasets(search)
|
|
2032
|
-
if output_format == "table":
|
|
2033
|
-
_print_records_table(result)
|
|
2034
|
-
else:
|
|
2035
|
-
_print_formatted(result, output_format)
|
|
2025
|
+
# 设计 §6 / 上游 §13.2:`ap dataset versions` 与 `ap dataset instances` 行为保持不变
|
|
2026
|
+
# (仍打 ossdata 路由,`ap job create` 的配套流程依赖它们),只追加固定弃用文案。
|
|
2027
|
+
# 文案**只写 stderr**——JSON/YAML 的 stdout 必须仍可被 json.loads 直接消费。
|
|
2028
|
+
_DATASET_VERSIONS_DEPRECATION = (
|
|
2029
|
+
"Warning: `ap dataset versions` is deprecated.\n"
|
|
2030
|
+
"Use `ap dataset version list --dataset <dataset>` instead."
|
|
2031
|
+
)
|
|
2032
|
+
_DATASET_INSTANCES_DEPRECATION = (
|
|
2033
|
+
"Warning: `ap dataset instances` is deprecated.\n"
|
|
2034
|
+
"Use `ap instance list --dataset <dataset> --split <split> [--version <version>]` instead."
|
|
2035
|
+
)
|
|
2036
2036
|
|
|
2037
2037
|
|
|
2038
2038
|
@dataset_app.command("versions")
|
|
@@ -2044,8 +2044,9 @@ def dataset_versions(
|
|
|
2044
2044
|
help="Output format: plain/table/json/yaml (default: AP_FORMAT or command default)",
|
|
2045
2045
|
),
|
|
2046
2046
|
):
|
|
2047
|
-
"""List dataset versions."""
|
|
2047
|
+
"""List dataset versions (deprecated: use 'ap dataset version list')."""
|
|
2048
2048
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
2049
|
+
_emit_progress(_DATASET_VERSIONS_DEPRECATION)
|
|
2049
2050
|
client = get_client()
|
|
2050
2051
|
versions = client.list_dataset_versions(dataset)
|
|
2051
2052
|
if output_format == "table":
|
|
@@ -2065,8 +2066,9 @@ def dataset_instances(
|
|
|
2065
2066
|
help="Output format: plain/table/json/yaml (default: AP_FORMAT or command default)",
|
|
2066
2067
|
),
|
|
2067
2068
|
):
|
|
2068
|
-
"""List dataset instances."""
|
|
2069
|
+
"""List dataset instances (deprecated: use 'ap instance list')."""
|
|
2069
2070
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
2071
|
+
_emit_progress(_DATASET_INSTANCES_DEPRECATION)
|
|
2070
2072
|
client = get_client()
|
|
2071
2073
|
result = client.list_all_dataset_instances(dataset_version)
|
|
2072
2074
|
if output_format == "table":
|
|
@@ -2634,21 +2636,6 @@ def job_list(
|
|
|
2634
2636
|
desc: bool = typer.Option(False, "--desc", help="Sort descending; requires --sort"),
|
|
2635
2637
|
skip: int = typer.Option(0, "--skip", help="Skip the first N entries"),
|
|
2636
2638
|
limit: int = typer.Option(100, "--limit", help="Maximum number of entries to return"),
|
|
2637
|
-
next_token: Optional[str] = typer.Option(
|
|
2638
|
-
None,
|
|
2639
|
-
"--next-token",
|
|
2640
|
-
help="Opaque token returned by the previous exact-group job list page; requires --group-id",
|
|
2641
|
-
),
|
|
2642
|
-
cursor: bool = typer.Option(
|
|
2643
|
-
False,
|
|
2644
|
-
"--cursor",
|
|
2645
|
-
help="Opt this page into next-token pagination; requires --group-id",
|
|
2646
|
-
),
|
|
2647
|
-
no_total: bool = typer.Option(
|
|
2648
|
-
False,
|
|
2649
|
-
"--no-total",
|
|
2650
|
-
help="Skip the server-side count; total becomes null in json/yaml output",
|
|
2651
|
-
),
|
|
2652
2639
|
columns: Optional[str] = typer.Option(
|
|
2653
2640
|
None,
|
|
2654
2641
|
"--columns",
|
|
@@ -2674,16 +2661,7 @@ def job_list(
|
|
|
2674
2661
|
raise typer.BadParameter("--tag-pattern cannot be used with --tag")
|
|
2675
2662
|
if normalized_tag_logic != "and" and not tag:
|
|
2676
2663
|
raise typer.BadParameter("--tag-logic requires --tag")
|
|
2677
|
-
if next_token and not group_id:
|
|
2678
|
-
raise typer.BadParameter("--next-token requires --group-id")
|
|
2679
|
-
if cursor and not group_id:
|
|
2680
|
-
raise typer.BadParameter("--cursor requires --group-id")
|
|
2681
|
-
if next_token and skip > 0:
|
|
2682
|
-
raise typer.BadParameter("--next-token cannot be used with --skip greater than 0")
|
|
2683
|
-
if cursor and skip > 0:
|
|
2684
|
-
raise typer.BadParameter("--cursor cannot be used with --skip greater than 0")
|
|
2685
2664
|
sort_kwargs = _job_list_sort_kwargs(sort, asc=asc, desc=desc)
|
|
2686
|
-
cursor_request = bool(next_token) or cursor
|
|
2687
2665
|
client = get_client()
|
|
2688
2666
|
result = client.list_jobs(
|
|
2689
2667
|
template=template,
|
|
@@ -2707,9 +2685,6 @@ def job_list(
|
|
|
2707
2685
|
**({"meta_model": meta_model} if meta_model is not None else {}),
|
|
2708
2686
|
**({"meta_job_type": meta_job_type} if meta_job_type is not None else {}),
|
|
2709
2687
|
**sort_kwargs,
|
|
2710
|
-
**({"next_token": next_token} if next_token else {}),
|
|
2711
|
-
**({"pagination": "cursor"} if cursor_request else {}),
|
|
2712
|
-
include_total=not no_total,
|
|
2713
2688
|
skip=skip,
|
|
2714
2689
|
limit=limit,
|
|
2715
2690
|
)
|
|
@@ -2719,9 +2694,6 @@ def job_list(
|
|
|
2719
2694
|
_print_job_list_plain(result, columns=selected_columns)
|
|
2720
2695
|
else:
|
|
2721
2696
|
_print_formatted(result, output_format)
|
|
2722
|
-
next_token = result.get("next_token")
|
|
2723
|
-
if output_format in {"table", "plain"} and next_token:
|
|
2724
|
-
typer.echo(f"Next token: {next_token}", err=True)
|
|
2725
2697
|
|
|
2726
2698
|
|
|
2727
2699
|
@job_app.command("create")
|
|
@@ -5774,21 +5746,6 @@ def group_jobs(
|
|
|
5774
5746
|
),
|
|
5775
5747
|
skip: int = typer.Option(0, "--skip", help="Skip the first N entries"),
|
|
5776
5748
|
limit: int = typer.Option(100, "--limit", help="Maximum number of entries to return"),
|
|
5777
|
-
next_token: Optional[str] = typer.Option(
|
|
5778
|
-
None,
|
|
5779
|
-
"--next-token",
|
|
5780
|
-
help="Opaque token returned by the previous Group jobs page",
|
|
5781
|
-
),
|
|
5782
|
-
cursor: bool = typer.Option(
|
|
5783
|
-
False,
|
|
5784
|
-
"--cursor",
|
|
5785
|
-
help="Opt this page into next-token pagination",
|
|
5786
|
-
),
|
|
5787
|
-
no_total: bool = typer.Option(
|
|
5788
|
-
False,
|
|
5789
|
-
"--no-total",
|
|
5790
|
-
help="Skip the server-side count; total becomes null in json/yaml output",
|
|
5791
|
-
),
|
|
5792
5749
|
output_format: str = typer.Option(
|
|
5793
5750
|
None,
|
|
5794
5751
|
"--format",
|
|
@@ -5797,11 +5754,6 @@ def group_jobs(
|
|
|
5797
5754
|
):
|
|
5798
5755
|
"""List jobs in a Group."""
|
|
5799
5756
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
5800
|
-
if next_token and skip > 0:
|
|
5801
|
-
raise typer.BadParameter("--next-token cannot be used with --skip greater than 0")
|
|
5802
|
-
if cursor and skip > 0:
|
|
5803
|
-
raise typer.BadParameter("--cursor cannot be used with --skip greater than 0")
|
|
5804
|
-
cursor_request = bool(next_token) or cursor
|
|
5805
5757
|
client = get_client()
|
|
5806
5758
|
result = client.list_group_jobs(
|
|
5807
5759
|
group_id,
|
|
@@ -5809,9 +5761,6 @@ def group_jobs(
|
|
|
5809
5761
|
include_post_process=include_post_process,
|
|
5810
5762
|
skip=skip,
|
|
5811
5763
|
limit=limit,
|
|
5812
|
-
**({"next_token": next_token} if next_token else {}),
|
|
5813
|
-
**({"pagination": "cursor"} if cursor_request else {}),
|
|
5814
|
-
include_total=not no_total,
|
|
5815
5764
|
)
|
|
5816
5765
|
if output_format == "table":
|
|
5817
5766
|
_print_job_list_plain(result, rich=True)
|
|
@@ -5819,9 +5768,6 @@ def group_jobs(
|
|
|
5819
5768
|
_print_job_list_plain(result)
|
|
5820
5769
|
else:
|
|
5821
5770
|
_print_formatted(result, output_format)
|
|
5822
|
-
following_token = result.get("next_token")
|
|
5823
|
-
if output_format in {"table", "plain"} and following_token:
|
|
5824
|
-
typer.echo(f"Next token: {following_token}", err=True)
|
|
5825
5771
|
|
|
5826
5772
|
|
|
5827
5773
|
@group_app.command("stats")
|
|
@@ -6092,7 +6038,7 @@ def main() -> None:
|
|
|
6092
6038
|
except APIError as exc:
|
|
6093
6039
|
typer.echo(_format_api_error(exc), err=True)
|
|
6094
6040
|
raise SystemExit(1) from None
|
|
6095
|
-
except
|
|
6041
|
+
except ConfigurationError as exc:
|
|
6096
6042
|
typer.echo(f"error: {exc}", err=True)
|
|
6097
6043
|
raise SystemExit(1) from None
|
|
6098
6044
|
|