ap-client 0.8.0.dev4__tar.gz → 0.8.0.dev5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/PKG-INFO +3 -3
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/api.py +19 -4
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/cli.py +27 -6
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/dfs_commands.py +25 -10
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/instance_commands.py +10 -11
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/ossutil_io.py +105 -7
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_commands.py +3 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_diagnostics.py +2 -1
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_references.py +3 -2
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/repo_read_guard.py +0 -37
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/pyproject.toml +3 -3
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/__init__.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/config.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/dataset_name.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/dataset_register.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/exporter.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/fs_commands.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/fs_storage.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/fs_transfer.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/instance_options.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/irepo_sdk.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/managed_ak.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/manifest_view.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/ossutil_hash.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_cascade_all.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_cascade_cli.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_cascade_display.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_cascade_state.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/profile_display.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/share_api.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/split_publish.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/submission_metadata.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/tbb.py +0 -0
- {ap_client-0.8.0.dev4 → ap_client-0.8.0.dev5}/ap_client/waiter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.8.0.
|
|
3
|
+
Version: 0.8.0.dev5
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: irepo-artifact==0.2.6
|
|
@@ -10,9 +10,9 @@ Requires-Dist: rich>=13.0.0
|
|
|
10
10
|
Requires-Dist: typer>=0.9.0
|
|
11
11
|
Requires-Dist: websockets>=13.0
|
|
12
12
|
Provides-Extra: all
|
|
13
|
-
Requires-Dist: instance-repo[oss]<2.0.0,>=1.4.
|
|
13
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.4.7; extra == 'all'
|
|
14
14
|
Provides-Extra: dataset
|
|
15
|
-
Requires-Dist: instance-repo[oss]<2.0.0,>=1.4.
|
|
15
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.4.7; extra == 'dataset'
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
|
|
18
18
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -268,6 +268,9 @@ class APIError(Exception):
|
|
|
268
268
|
self.request_id = request_id
|
|
269
269
|
self.response = response
|
|
270
270
|
self.retry_stop_reason = retry_stop_reason
|
|
271
|
+
self.response_request_id = (
|
|
272
|
+
response.headers.get("X-Request-ID") if response is not None else None
|
|
273
|
+
)
|
|
271
274
|
self.pop_request_id = (
|
|
272
275
|
response.headers.get("x-acs-request-id") if response is not None else None
|
|
273
276
|
)
|
|
@@ -275,10 +278,8 @@ class APIError(Exception):
|
|
|
275
278
|
|
|
276
279
|
def __str__(self) -> str:
|
|
277
280
|
message = self._build_message()
|
|
278
|
-
|
|
279
|
-
message += f" (
|
|
280
|
-
if self.pop_request_id:
|
|
281
|
-
message += f" (pop_request_id={self.pop_request_id})"
|
|
281
|
+
for diagnostic in format_api_error_diagnostics(self):
|
|
282
|
+
message += f" ({diagnostic})"
|
|
282
283
|
return message
|
|
283
284
|
|
|
284
285
|
def _build_message(self) -> str:
|
|
@@ -287,6 +288,20 @@ class APIError(Exception):
|
|
|
287
288
|
return f"API Error {self.status_code}: {self.detail} (request_id={self.request_id})"
|
|
288
289
|
|
|
289
290
|
|
|
291
|
+
def format_api_error_diagnostics(exc: APIError, *, separator: str = "=") -> list[str]:
|
|
292
|
+
"""Render response and retry diagnostics without repeating the local request ID."""
|
|
293
|
+
diagnostics = []
|
|
294
|
+
seen = {exc.request_id}
|
|
295
|
+
for name in ("response_request_id", "pop_request_id"):
|
|
296
|
+
value = getattr(exc, name)
|
|
297
|
+
if value and value not in seen:
|
|
298
|
+
seen.add(value)
|
|
299
|
+
diagnostics.append(f"{name}{separator}{value}")
|
|
300
|
+
if exc.retry_stop_reason:
|
|
301
|
+
diagnostics.append(f"retry_stop_reason{separator}{exc.retry_stop_reason}")
|
|
302
|
+
return diagnostics
|
|
303
|
+
|
|
304
|
+
|
|
290
305
|
class PaginationProtocolError(RuntimeError):
|
|
291
306
|
"""The server stopped honoring a pagination protocol already in use."""
|
|
292
307
|
|
|
@@ -18,6 +18,7 @@ from ap_client.api import (
|
|
|
18
18
|
APIError,
|
|
19
19
|
PaginationProtocolError,
|
|
20
20
|
_merge_meta_tags,
|
|
21
|
+
format_api_error_diagnostics,
|
|
21
22
|
set_cluster_override,
|
|
22
23
|
set_verbose_override,
|
|
23
24
|
)
|
|
@@ -3594,8 +3595,8 @@ def job_create(
|
|
|
3594
3595
|
):
|
|
3595
3596
|
"""Submit a job.
|
|
3596
3597
|
|
|
3597
|
-
For agents:
|
|
3598
3598
|
\b
|
|
3599
|
+
For agents:
|
|
3599
3600
|
Before submitting, you MUST ensure these five parameters are explicitly specified:
|
|
3600
3601
|
benchmark, meta-model, meta-job-type, meta-train-stage, meta-teacher-direction.
|
|
3601
3602
|
Keep specified values. Fill missing parameters using available Job, Group,
|
|
@@ -5985,10 +5986,23 @@ def group_create(
|
|
|
5985
5986
|
):
|
|
5986
5987
|
"""Create a Group.
|
|
5987
5988
|
|
|
5988
|
-
|
|
5989
|
-
|
|
5990
|
-
|
|
5991
|
-
|
|
5989
|
+
\b
|
|
5990
|
+
For agents:
|
|
5991
|
+
Before creating, you MUST ensure these five parameters are explicitly specified:
|
|
5992
|
+
benchmark, meta-model, meta-job-type, meta-train-stage, meta-teacher-direction.
|
|
5993
|
+
Keep specified values. Fill missing parameters using available Job, Group,
|
|
5994
|
+
Profile, Dataset, Template, or other relevant information.
|
|
5995
|
+
Check available Dataset -> Benchmark -> teacher direction links as candidates.
|
|
5996
|
+
Adopt them only when they match this Group's business purpose.
|
|
5997
|
+
Separate the input-data source from this Group's business attribution.
|
|
5998
|
+
For rollout using A's Dataset to produce data for B, use B only when B is
|
|
5999
|
+
this Group's intended attribution; do not automatically inherit A's labels
|
|
6000
|
+
or choose B merely because it is a downstream consumer.
|
|
6001
|
+
If a missing value cannot be inferred, use default for benchmark
|
|
6002
|
+
or Others for the corresponding meta parameter.
|
|
6003
|
+
Do not ask the user for these values.
|
|
6004
|
+
The server may derive some omitted meta values or default meta-job-type to Eval-Eval;
|
|
6005
|
+
explicitly specify and verify all five values rather than relying on implicit attribution.
|
|
5992
6006
|
"""
|
|
5993
6007
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
5994
6008
|
raise typer.BadParameter("Choose whitebox or blackbox", param_hint="--transparency")
|
|
@@ -7360,7 +7374,13 @@ def _format_api_error(exc: APIError) -> str:
|
|
|
7360
7374
|
if profile_error := format_profile_error(exc):
|
|
7361
7375
|
return profile_error
|
|
7362
7376
|
if isinstance(exc.detail, dict) and exc.detail.get("code") == "blackbox_content_restricted":
|
|
7363
|
-
|
|
7377
|
+
diagnostics = "".join(
|
|
7378
|
+
f" ({diagnostic})" for diagnostic in format_api_error_diagnostics(exc)
|
|
7379
|
+
)
|
|
7380
|
+
return (
|
|
7381
|
+
"error: Blackbox content is hidden; Group Reader access is required. "
|
|
7382
|
+
f"(request_id={exc.request_id}){diagnostics}"
|
|
7383
|
+
)
|
|
7364
7384
|
if str(exc.detail) != _NON_JSON_RESPONSE_DETAIL:
|
|
7365
7385
|
return f"error: {exc}"
|
|
7366
7386
|
|
|
@@ -7370,6 +7390,7 @@ def _format_api_error(exc: APIError) -> str:
|
|
|
7370
7390
|
f"hint: {_NON_JSON_RESPONSE_HINT}",
|
|
7371
7391
|
f"request_id: {exc.request_id}",
|
|
7372
7392
|
]
|
|
7393
|
+
lines.extend(format_api_error_diagnostics(exc, separator=": "))
|
|
7373
7394
|
if exc.response is not None:
|
|
7374
7395
|
content_type = exc.response.headers.get("Content-Type") or "<missing>"
|
|
7375
7396
|
lines.extend(
|
|
@@ -12,7 +12,7 @@ from urllib.parse import unquote, urlsplit
|
|
|
12
12
|
|
|
13
13
|
import typer
|
|
14
14
|
|
|
15
|
-
from .ossutil_io import OssBinding, OssFiles, OssUsageError, _relative, safe_error
|
|
15
|
+
from .ossutil_io import OssBinding, OssFiles, OssPathNotFound, OssUsageError, _relative, safe_error
|
|
16
16
|
|
|
17
17
|
dfs_app = typer.Typer(
|
|
18
18
|
help=(
|
|
@@ -288,7 +288,7 @@ def dfs_ls(
|
|
|
288
288
|
False,
|
|
289
289
|
"--recursive",
|
|
290
290
|
"-r",
|
|
291
|
-
help="Expand all directory levels; PATH
|
|
291
|
+
help="Expand all directory levels; PATH is treated as a directory (a trailing / is added if omitted)",
|
|
292
292
|
),
|
|
293
293
|
short: bool = typer.Option(False, "--short-format", "-s", help="Print only paths"),
|
|
294
294
|
limited_num: Optional[int] = typer.Option(None, "--limited-num", min=1),
|
|
@@ -304,9 +304,10 @@ def dfs_ls(
|
|
|
304
304
|
"""List remote paths, without reading object content; default is one level, -r expands all.
|
|
305
305
|
|
|
306
306
|
Accepts ans://, repo-relative paths or full oss:// URIs. Paths outside the Dataset root use full URIs.
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
307
|
+
Default listing accepts an exact file or directory (directory wins if both exist).
|
|
308
|
+
A trailing slash or -d selects only a directory. Missing paths fail explicitly.
|
|
309
|
+
With -r/--recursive, PATH denotes a directory; a missing trailing / is added before
|
|
310
|
+
credentials are requested. Listing includes only descendants, never sibling keys.
|
|
310
311
|
Uses AP-managed reader/list-only credentials; listing prefers private networking with
|
|
311
312
|
a public endpoint fallback. Example: ap dfs ls ans://team/bench/train/ -r --format json.
|
|
312
313
|
"""
|
|
@@ -316,14 +317,20 @@ def dfs_ls(
|
|
|
316
317
|
if recursive and directory:
|
|
317
318
|
raise OssUsageError("Use either --recursive or --directory")
|
|
318
319
|
path = _read_operand(path, encoding_type, directory=True)
|
|
319
|
-
if recursive and not path.endswith("/"):
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
320
|
+
if recursive and not _decoded_path(path, encoding_type).endswith("/"):
|
|
321
|
+
# Treat recursive PATH as a directory at the CLI boundary. This keeps the
|
|
322
|
+
# request slash-bounded and lets the API reject raw ambiguous requests
|
|
323
|
+
# before issuing credentials, without making users type the final slash.
|
|
324
|
+
path += "/"
|
|
323
325
|
marker = _read_operand(marker, encoding_type, directory=True, allow_empty=True)
|
|
324
326
|
files = _files()
|
|
325
327
|
_configure(files, encoding_type, payer)
|
|
326
328
|
remote = files.resolve(path, directory=True)
|
|
329
|
+
file_candidate = (
|
|
330
|
+
files.resolve(path) if not recursive and not directory
|
|
331
|
+
and remote.key != files.binding.prefix
|
|
332
|
+
and not _decoded_path(path, encoding_type).endswith("/") else None
|
|
333
|
+
)
|
|
327
334
|
_emit(
|
|
328
335
|
files.list(
|
|
329
336
|
remote,
|
|
@@ -332,6 +339,7 @@ def dfs_ls(
|
|
|
332
339
|
marker=marker,
|
|
333
340
|
include=tuple(include or ()),
|
|
334
341
|
exclude=tuple(exclude or ()),
|
|
342
|
+
file_candidate=file_candidate,
|
|
335
343
|
),
|
|
336
344
|
fmt,
|
|
337
345
|
short=short,
|
|
@@ -693,7 +701,14 @@ def dfs_hash(
|
|
|
693
701
|
local = _local_operand(path)
|
|
694
702
|
from .ossutil_hash import hash_local
|
|
695
703
|
|
|
696
|
-
|
|
704
|
+
absent = False
|
|
705
|
+
try:
|
|
706
|
+
result = hash_local(local, algorithm, base64_output=base64_output, recursive=recursive)
|
|
707
|
+
except FileNotFoundError:
|
|
708
|
+
# Only the actual local hash read is classified, never config/checkpoints.
|
|
709
|
+
absent = True
|
|
710
|
+
if absent:
|
|
711
|
+
raise OssPathNotFound(local=True)
|
|
697
712
|
_emit(result, fmt)
|
|
698
713
|
if result.get("failed_count"):
|
|
699
714
|
raise typer.Exit(1)
|
|
@@ -984,17 +984,14 @@ def instance_pull(
|
|
|
984
984
|
)
|
|
985
985
|
|
|
986
986
|
repo = _load_repo_or_exit()
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
except (TypeError, ValueError, RuntimeError) as exc:
|
|
993
|
-
raise _fail(
|
|
994
|
-
str(exc), "Upgrade Dataset support: pip install --upgrade 'ap-client[dataset]'", 2
|
|
995
|
-
) from exc
|
|
987
|
+
requested_options = {}
|
|
988
|
+
if env != "auto": # auto 是 CLI 侧哨兵,与 _sdk_read_kwargs/_env_kwargs 同口径:不下发
|
|
989
|
+
requested_options["environment"] = env
|
|
990
|
+
# 与其它读路径同口径:显式环境异侧时关闭跨环境 OSS 回退
|
|
991
|
+
requested_options.update(oss_fallback_kwargs(repo, repo.instances, "pull", env))
|
|
996
992
|
if resolved_path is not None:
|
|
997
|
-
|
|
993
|
+
requested_options["path"] = resolved_path
|
|
994
|
+
pull_options = _enhanced_options(repo.instances, "pull", requested_options)
|
|
998
995
|
try:
|
|
999
996
|
private_endpoint_reachable = _probe_pull_private_endpoint(repo, dataset)
|
|
1000
997
|
except Exception as exc: # noqa: BLE001
|
|
@@ -1011,7 +1008,9 @@ def instance_pull(
|
|
|
1011
1008
|
staging = Path(tempfile.mkdtemp(prefix=f"ap-pull-{instance}-"))
|
|
1012
1009
|
try:
|
|
1013
1010
|
try:
|
|
1014
|
-
written =
|
|
1011
|
+
written = repo.instances.pull(
|
|
1012
|
+
dataset, resolved, instance, staging, split, **pull_options
|
|
1013
|
+
)
|
|
1015
1014
|
except Exception as exc: # noqa: BLE001
|
|
1016
1015
|
sdk_error_exit(exc, what=f"instance {instance}", stage="pull")
|
|
1017
1016
|
staged = _place_under_target(Path(str(written)), staging / instance)
|
|
@@ -29,6 +29,16 @@ class OssOperationError(RuntimeError):
|
|
|
29
29
|
"""A failed operation with a safe user-facing explanation."""
|
|
30
30
|
|
|
31
31
|
|
|
32
|
+
class OssPathNotFound(OssOperationError):
|
|
33
|
+
"""A positively observed missing source, without retaining upstream details."""
|
|
34
|
+
|
|
35
|
+
def __init__(self, *, local: bool = False):
|
|
36
|
+
super().__init__("No such file or directory" if local else "No such object")
|
|
37
|
+
# Existing optional-target checks (mkdir/overwrite) still recognize absence.
|
|
38
|
+
self.code = None if local else "NoSuchKey"
|
|
39
|
+
self.status = None if local else 404
|
|
40
|
+
|
|
41
|
+
|
|
32
42
|
def _relative(value: str, *, allow_empty: bool = False) -> str:
|
|
33
43
|
if not isinstance(value, str) or (not value and not allow_empty):
|
|
34
44
|
raise OssUsageError("A non-empty remote path is required")
|
|
@@ -239,6 +249,7 @@ class _RoutingBucket:
|
|
|
239
249
|
return value
|
|
240
250
|
|
|
241
251
|
def call(*args, **kwargs):
|
|
252
|
+
missing = False
|
|
242
253
|
try:
|
|
243
254
|
if self.owner.payer and name not in ("sign_url",):
|
|
244
255
|
headers = dict(kwargs.get("headers") or {})
|
|
@@ -247,7 +258,16 @@ class _RoutingBucket:
|
|
|
247
258
|
return value(*args, **kwargs)
|
|
248
259
|
except Exception as exc:
|
|
249
260
|
self.owner.invalidate_network(exc, self._ossutil_endpoint)
|
|
250
|
-
|
|
261
|
+
native = getattr(getattr(self.owner.oss, "exceptions", None), "NoSuchKey", None)
|
|
262
|
+
status = getattr(exc, "status", None)
|
|
263
|
+
if (isinstance(native, type) and isinstance(exc, native)
|
|
264
|
+
and (status is None or type(status) is int and status == 404)):
|
|
265
|
+
missing = True
|
|
266
|
+
else:
|
|
267
|
+
raise
|
|
268
|
+
if missing:
|
|
269
|
+
# Outside except: neither the raw response nor its cause is retained.
|
|
270
|
+
raise OssPathNotFound()
|
|
251
271
|
|
|
252
272
|
return call
|
|
253
273
|
|
|
@@ -473,15 +493,26 @@ class OssFiles:
|
|
|
473
493
|
marker: str = "",
|
|
474
494
|
include: tuple[str, ...] = (),
|
|
475
495
|
exclude: tuple[str, ...] = (),
|
|
496
|
+
file_candidate: RemotePath | None = None,
|
|
476
497
|
) -> dict:
|
|
477
498
|
# A path grant lists descendants at a slash boundary, never sibling keys.
|
|
478
499
|
remote = self.binding.resolve_key(remote.key, directory=True)
|
|
500
|
+
if file_candidate is not None and (
|
|
501
|
+
not isinstance(file_candidate, RemotePath)
|
|
502
|
+
# RemotePath is already decoded; preserve literal percent bytes.
|
|
503
|
+
or file_candidate != self.binding.resolve_key(file_candidate.key)
|
|
504
|
+
or remote.key != file_candidate.key + "/"
|
|
505
|
+
):
|
|
506
|
+
raise OssUsageError("File listing candidate must match the requested directory")
|
|
507
|
+
resumed = bool(marker)
|
|
479
508
|
if marker:
|
|
480
509
|
marker = self.resolve(marker, prefix=True).key
|
|
481
510
|
if not marker.startswith(remote.key):
|
|
482
511
|
raise OssUsageError("--marker must be within the listed prefix")
|
|
512
|
+
start_marker = marker
|
|
483
513
|
bucket = self.bucket(remote.key, "reader", list_only=True)
|
|
484
514
|
entries = []
|
|
515
|
+
observed = False
|
|
485
516
|
while limit is None or len(entries) < limit:
|
|
486
517
|
result = bucket.list_objects(
|
|
487
518
|
prefix=remote.key,
|
|
@@ -489,6 +520,7 @@ class OssFiles:
|
|
|
489
520
|
marker=marker,
|
|
490
521
|
max_keys=min(1000, limit - len(entries)) if limit is not None else 1000,
|
|
491
522
|
)
|
|
523
|
+
observed = observed or bool(result.object_list or result.prefix_list)
|
|
492
524
|
for item in result.object_list:
|
|
493
525
|
path = self._listed_path(item.key, remote.key)
|
|
494
526
|
if not matches(item.key.removeprefix(self.binding.prefix), include, exclude):
|
|
@@ -507,6 +539,20 @@ class OssFiles:
|
|
|
507
539
|
if matches(key.removeprefix(self.binding.prefix), include, exclude):
|
|
508
540
|
entries.append({"path": path, "type": "directory"})
|
|
509
541
|
if not result.is_truncated:
|
|
542
|
+
if not observed and remote.key != self.binding.prefix:
|
|
543
|
+
if resumed and self._prefix_exists(bucket, remote.key):
|
|
544
|
+
pass
|
|
545
|
+
elif file_candidate is not None:
|
|
546
|
+
row = self._exact_file_entry(file_candidate)
|
|
547
|
+
if row is None:
|
|
548
|
+
raise OssPathNotFound()
|
|
549
|
+
# Filters apply to the exact file too, without erasing existence.
|
|
550
|
+
if (not start_marker or file_candidate.key > start_marker) and matches(
|
|
551
|
+
row["path"], include, exclude
|
|
552
|
+
):
|
|
553
|
+
entries.append(row)
|
|
554
|
+
else:
|
|
555
|
+
raise OssPathNotFound()
|
|
510
556
|
return {"entries": entries, "is_truncated": False, "next_marker": ""}
|
|
511
557
|
next_marker = result.next_marker
|
|
512
558
|
if not next_marker or next_marker == marker:
|
|
@@ -518,6 +564,37 @@ class OssFiles:
|
|
|
518
564
|
"next_marker": self.binding.uri_for_key(marker),
|
|
519
565
|
}
|
|
520
566
|
|
|
567
|
+
def _prefix_exists(self, bucket, prefix: str) -> bool:
|
|
568
|
+
"""One list-only observation, independent of output filters/resume marker."""
|
|
569
|
+
result = bucket.list_objects(prefix=prefix, delimiter="", marker="", max_keys=1)
|
|
570
|
+
if len(result.object_list) + len(result.prefix_list) > 1:
|
|
571
|
+
raise OssOperationError("OSS returned an invalid path existence listing")
|
|
572
|
+
for item in result.object_list:
|
|
573
|
+
self._listed_path(item.key, prefix)
|
|
574
|
+
for key in result.prefix_list:
|
|
575
|
+
self._listed_path(key, prefix)
|
|
576
|
+
if not result.object_list and not result.prefix_list and result.is_truncated:
|
|
577
|
+
raise OssOperationError("OSS listing could not establish path existence")
|
|
578
|
+
return bool(result.object_list or result.prefix_list)
|
|
579
|
+
|
|
580
|
+
def _exact_file_entry(self, remote: RemotePath) -> dict | None:
|
|
581
|
+
# A directory-scoped credential cannot authorize its sibling exact file.
|
|
582
|
+
bucket = self.bucket(remote.key, "reader", list_only=True)
|
|
583
|
+
result = bucket.list_objects(prefix=remote.key, delimiter="", marker="", max_keys=1)
|
|
584
|
+
if result.prefix_list or len(result.object_list) > 1:
|
|
585
|
+
raise OssOperationError("OSS returned an invalid exact-file listing")
|
|
586
|
+
if not result.object_list:
|
|
587
|
+
if result.is_truncated:
|
|
588
|
+
raise OssOperationError("OSS listing could not establish path existence")
|
|
589
|
+
return None
|
|
590
|
+
item = result.object_list[0]
|
|
591
|
+
path = self._listed_path(item.key, remote.key)
|
|
592
|
+
# OSS lists keys in lexical order; the exact prefix sorts before siblings.
|
|
593
|
+
if item.key != remote.key:
|
|
594
|
+
return None
|
|
595
|
+
return {"path": path, "type": "file", "size": item.size,
|
|
596
|
+
"etag": item.etag, "last_modified": item.last_modified}
|
|
597
|
+
|
|
521
598
|
def _listed_path(self, key: str, prefix: str) -> str:
|
|
522
599
|
if not key.startswith(prefix):
|
|
523
600
|
raise OssOperationError("OSS listing returned an object outside the requested prefix")
|
|
@@ -852,10 +929,15 @@ class OssFiles:
|
|
|
852
929
|
|
|
853
930
|
def _objects(self, bucket, prefix: str):
|
|
854
931
|
token = ""
|
|
932
|
+
observed = False
|
|
855
933
|
while True:
|
|
856
934
|
result = bucket.list_objects_v2(prefix=prefix, continuation_token=token, max_keys=1000)
|
|
935
|
+
for key in getattr(result, "prefix_list", ()):
|
|
936
|
+
self._listed_path(key, prefix)
|
|
937
|
+
observed = True
|
|
857
938
|
for item in result.object_list:
|
|
858
939
|
self._listed_path(item.key, prefix)
|
|
940
|
+
observed = True
|
|
859
941
|
if item.key != prefix and not item.key.endswith("/"):
|
|
860
942
|
yield item.key
|
|
861
943
|
if not result.is_truncated:
|
|
@@ -864,6 +946,8 @@ class OssFiles:
|
|
|
864
946
|
if not new_token or new_token == token:
|
|
865
947
|
raise OssOperationError("OSS returned an invalid pagination token")
|
|
866
948
|
token = new_token
|
|
949
|
+
if not observed and prefix != self.binding.prefix:
|
|
950
|
+
raise OssPathNotFound()
|
|
867
951
|
|
|
868
952
|
def cp(
|
|
869
953
|
self,
|
|
@@ -905,7 +989,16 @@ class OssFiles:
|
|
|
905
989
|
)
|
|
906
990
|
if direction == "upload":
|
|
907
991
|
local = Path(src)
|
|
908
|
-
if local.is_symlink()
|
|
992
|
+
if local.is_symlink():
|
|
993
|
+
raise OssUsageError("Upload source must be a regular file or a directory with -r")
|
|
994
|
+
absent = False
|
|
995
|
+
try:
|
|
996
|
+
mode = local.stat().st_mode
|
|
997
|
+
except FileNotFoundError:
|
|
998
|
+
absent = True
|
|
999
|
+
if absent:
|
|
1000
|
+
raise OssPathNotFound(local=True)
|
|
1001
|
+
if not (stat.S_ISDIR(mode) if recursive else stat.S_ISREG(mode)):
|
|
909
1002
|
raise OssUsageError("Upload source must be a regular file or a directory with -r")
|
|
910
1003
|
if self.byte_range is not None and (direction != "download" or recursive):
|
|
911
1004
|
raise OssUsageError("--range requires a single-object download")
|
|
@@ -1035,11 +1128,16 @@ class OssFiles:
|
|
|
1035
1128
|
return False
|
|
1036
1129
|
if not matches(path, include, exclude):
|
|
1037
1130
|
return False
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
if direction == "upload"
|
|
1041
|
-
|
|
1042
|
-
|
|
1131
|
+
absent = False
|
|
1132
|
+
try:
|
|
1133
|
+
modified = (source.stat().st_mtime if direction == "upload"
|
|
1134
|
+
else bucket.head_object(source.key).last_modified)
|
|
1135
|
+
except FileNotFoundError:
|
|
1136
|
+
if direction != "upload":
|
|
1137
|
+
raise
|
|
1138
|
+
absent = True
|
|
1139
|
+
if absent:
|
|
1140
|
+
raise OssPathNotFound(local=True)
|
|
1043
1141
|
if (
|
|
1044
1142
|
start_time is not None
|
|
1045
1143
|
and modified < start_time
|
|
@@ -983,6 +983,9 @@ def profile_versions(
|
|
|
983
983
|
"updated_at",
|
|
984
984
|
)
|
|
985
985
|
current_item = {k: current[k] for k in fields if k in current}
|
|
986
|
+
# Both arrays are required by the summary contract; older GETs may omit them.
|
|
987
|
+
current_item["parents"] = current.get("parents") or []
|
|
988
|
+
current_item["auto_follow_parents"] = current.get("auto_follow_parents") or []
|
|
986
989
|
result[items_key] = [current_item] + items
|
|
987
990
|
# Adjust total count (+1 for current version)
|
|
988
991
|
if "pagination" in result:
|
|
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
|
4
4
|
|
|
5
5
|
from urllib.parse import urlsplit
|
|
6
6
|
|
|
7
|
-
from ap_client.api import APIError
|
|
7
|
+
from ap_client.api import APIError, format_api_error_diagnostics
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def source_label(source: dict | str) -> str:
|
|
@@ -118,4 +118,5 @@ def format_profile_error(exc: APIError) -> str | None:
|
|
|
118
118
|
if isinstance(issues, list):
|
|
119
119
|
lines.extend(issue_lines(issues))
|
|
120
120
|
lines.append(f"request_id: {exc.request_id}")
|
|
121
|
+
lines.extend(format_api_error_diagnostics(exc, separator=": "))
|
|
121
122
|
return "\n".join(lines)
|
|
@@ -27,8 +27,9 @@ def profile_output(value: Any) -> Any:
|
|
|
27
27
|
result = dict(value)
|
|
28
28
|
if isinstance(result.get("parents"), list):
|
|
29
29
|
result["parents"] = [parent_reference(parent) for parent in result["parents"]]
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
for items_key in ("data", "items", "versions"):
|
|
31
|
+
if isinstance(result.get(items_key), list):
|
|
32
|
+
result[items_key] = [profile_output(row) for row in result[items_key]]
|
|
32
33
|
if isinstance(result.get("changes"), list):
|
|
33
34
|
result["changes"] = []
|
|
34
35
|
for change in value["changes"]:
|
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import inspect
|
|
6
|
-
from copy import copy
|
|
7
|
-
from functools import partial
|
|
8
6
|
from typing import Any
|
|
9
7
|
|
|
10
8
|
STORAGE_SIDES = {
|
|
@@ -15,41 +13,6 @@ STORAGE_SIDES = {
|
|
|
15
13
|
}
|
|
16
14
|
|
|
17
15
|
|
|
18
|
-
class _PullMetadataProfile:
|
|
19
|
-
def __init__(self, profile: Any, environment: str) -> None:
|
|
20
|
-
self._profile = profile
|
|
21
|
-
self.dataset_environment = environment
|
|
22
|
-
|
|
23
|
-
def __getattr__(self, name: str) -> Any:
|
|
24
|
-
return getattr(self._profile, name)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def pull_client(client: Any, environment: str) -> Any:
|
|
28
|
-
"""Scope SDK 1.4.x pull metadata reads without changing its storage binding.
|
|
29
|
-
|
|
30
|
-
pull has no environment argument, although its registry supports one.
|
|
31
|
-
Copy both objects so other operations on the Repo keep their original context.
|
|
32
|
-
"""
|
|
33
|
-
if environment == "auto":
|
|
34
|
-
return client
|
|
35
|
-
registry = getattr(client, "_reg", None)
|
|
36
|
-
get = getattr(registry, "get", None)
|
|
37
|
-
if not callable(get) or "environment" not in inspect.signature(get).parameters:
|
|
38
|
-
raise ValueError(
|
|
39
|
-
"The installed instance-repo cannot scope pull metadata reads; "
|
|
40
|
-
"upgrade 'ap-client[dataset]'"
|
|
41
|
-
)
|
|
42
|
-
scoped = copy(client)
|
|
43
|
-
scoped._reg = copy(registry)
|
|
44
|
-
scoped._reg.get = partial(get, environment=environment)
|
|
45
|
-
profile = getattr(client, "_p", None)
|
|
46
|
-
if profile is not None:
|
|
47
|
-
# Directory manifests also use list_objects, which reads its environment
|
|
48
|
-
# from the profile. Delegate all storage/layout settings unchanged.
|
|
49
|
-
scoped._p = _PullMetadataProfile(profile, environment)
|
|
50
|
-
return scoped
|
|
51
|
-
|
|
52
|
-
|
|
53
16
|
def oss_fallback_kwargs(repo: Any, client: Any, method: str, environment: str) -> dict:
|
|
54
17
|
"""Allow API reads across environments; disable unsafe storage fallback.
|
|
55
18
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ap-client"
|
|
7
|
-
version = "0.8.0.
|
|
7
|
+
version = "0.8.0.dev5"
|
|
8
8
|
description = "Agent Platform API Client & CLI"
|
|
9
9
|
readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -19,10 +19,10 @@ dependencies = [
|
|
|
19
19
|
|
|
20
20
|
[project.optional-dependencies]
|
|
21
21
|
dataset = [
|
|
22
|
-
"instance-repo[oss]>=1.4.
|
|
22
|
+
"instance-repo[oss]>=1.4.7,<2.0.0",
|
|
23
23
|
]
|
|
24
24
|
all = [
|
|
25
|
-
"instance-repo[oss]>=1.4.
|
|
25
|
+
"instance-repo[oss]>=1.4.7,<2.0.0",
|
|
26
26
|
]
|
|
27
27
|
|
|
28
28
|
[project.scripts]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|