ap-client 0.7.1.dev2__py3-none-any.whl → 0.7.1.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ap_client/__init__.py +1 -1
- ap_client/api.py +282 -6
- ap_client/canonical_get.py +210 -0
- ap_client/canonical_put.py +914 -0
- ap_client/cli.py +165 -58
- ap_client/cluster_layout_fetcher.py +357 -0
- ap_client/cluster_region_resolver.py +394 -0
- ap_client/control_http.py +415 -0
- ap_client/dataset_commands.py +344 -41
- ap_client/dfs_commands.py +44 -14
- ap_client/fs_commands.py +424 -520
- ap_client/fs_storage.py +46 -139
- ap_client/instance_commands.py +30 -4
- ap_client/irepo_sdk.py +374 -0
- ap_client/ossutil_io.py +177 -77
- ap_client/profile_cascade_all.py +149 -0
- ap_client/profile_cascade_cli.py +356 -0
- ap_client/profile_cascade_display.py +279 -0
- ap_client/profile_cascade_state.py +207 -0
- ap_client/profile_commands.py +538 -48
- ap_client/profile_diagnostics.py +121 -0
- ap_client/profile_display.py +194 -0
- ap_client/profile_references.py +41 -0
- ap_client/replica_policy.py +114 -0
- ap_client/repo_read_guard.py +14 -3
- ap_client/shared_adapter.py +1442 -0
- ap_client/shared_commands.py +1114 -0
- ap_client/submission_metadata.py +162 -0
- ap_client/tool_manifest.py +184 -0
- {ap_client-0.7.1.dev2.dist-info → ap_client-0.7.1.dev4.dist-info}/METADATA +4 -4
- ap_client-0.7.1.dev4.dist-info/RECORD +47 -0
- {ap_client-0.7.1.dev2.dist-info → ap_client-0.7.1.dev4.dist-info}/WHEEL +1 -1
- ap_client-0.7.1.dev2.dist-info/RECORD +0 -30
- {ap_client-0.7.1.dev2.dist-info → ap_client-0.7.1.dev4.dist-info}/entry_points.txt +0 -0
ap_client/__init__.py
CHANGED
ap_client/api.py
CHANGED
|
@@ -6,6 +6,7 @@ import os
|
|
|
6
6
|
import platform
|
|
7
7
|
import re
|
|
8
8
|
import sys
|
|
9
|
+
import threading
|
|
9
10
|
import time
|
|
10
11
|
import uuid
|
|
11
12
|
from email.utils import parsedate_to_datetime
|
|
@@ -287,6 +288,14 @@ class APIError(Exception):
|
|
|
287
288
|
return f"API Error {self.status_code}: {self.detail} (request_id={self.request_id})"
|
|
288
289
|
|
|
289
290
|
|
|
291
|
+
class CanonicalControlAPIError(APIError):
|
|
292
|
+
"""Operation-only sanitized error; never holds a native response or cause."""
|
|
293
|
+
def __init__(self, reason, *, status_code=None, biz_code=None):
|
|
294
|
+
self.reason = reason
|
|
295
|
+
detail = {"code": biz_code} if type(biz_code) is int else "canonical control request failed"
|
|
296
|
+
super().__init__(status_code, detail, None, None)
|
|
297
|
+
|
|
298
|
+
|
|
290
299
|
class PaginationProtocolError(RuntimeError):
|
|
291
300
|
"""The server stopped honoring a pagination protocol already in use."""
|
|
292
301
|
|
|
@@ -371,6 +380,8 @@ class APIClient:
|
|
|
371
380
|
|
|
372
381
|
def __init__(self, config: Config):
|
|
373
382
|
self.config = config
|
|
383
|
+
self._control_channel = None
|
|
384
|
+
self._control_channel_lock = threading.Lock()
|
|
374
385
|
self.session = requests.Session()
|
|
375
386
|
headers = dict(config.headers)
|
|
376
387
|
user_agent = _build_user_agent(headers.get("User-Agent"))
|
|
@@ -426,8 +437,13 @@ class APIClient:
|
|
|
426
437
|
timeout: TimeoutType = None,
|
|
427
438
|
deadline: Optional[float] = None,
|
|
428
439
|
_return_response: bool = False,
|
|
440
|
+
max_response_bytes: Optional[int] = None,
|
|
429
441
|
) -> Any:
|
|
430
|
-
"""Send
|
|
442
|
+
"""Send through the normal channel unless bounded control is opted in."""
|
|
443
|
+
if max_response_bytes is not None:
|
|
444
|
+
return self._request_control_json(method, path, params=params,
|
|
445
|
+
json_body=json_body, timeout=timeout, deadline=deadline,
|
|
446
|
+
max_response_bytes=max_response_bytes, return_response=_return_response)
|
|
431
447
|
self._require_cluster_config()
|
|
432
448
|
return self._request_http(
|
|
433
449
|
method,
|
|
@@ -596,6 +612,130 @@ class APIClient:
|
|
|
596
612
|
raise APIError(resp.status_code, detail, request_id, resp) from exc
|
|
597
613
|
raise RuntimeError("unreachable retry state")
|
|
598
614
|
|
|
615
|
+
def _request_control_json(self, method, path, *, params, json_body, timeout,
|
|
616
|
+
deadline, max_response_bytes, return_response):
|
|
617
|
+
from .control_http import (ControlHTTPError, OwnedControlChannel, validate_limit,
|
|
618
|
+
_check_deadline)
|
|
619
|
+
|
|
620
|
+
if return_response:
|
|
621
|
+
raise CanonicalControlAPIError("invalid_request")
|
|
622
|
+
self._require_cluster_config()
|
|
623
|
+
failure = None
|
|
624
|
+
try:
|
|
625
|
+
validate_limit(max_response_bytes)
|
|
626
|
+
_check_deadline(deadline)
|
|
627
|
+
if isinstance(timeout, bool):
|
|
628
|
+
raise ControlHTTPError('invalid_deadline')
|
|
629
|
+
if isinstance(timeout, (int, float)):
|
|
630
|
+
if not math.isfinite(timeout) or timeout < 0:
|
|
631
|
+
raise ControlHTTPError('invalid_deadline')
|
|
632
|
+
if deadline is None:
|
|
633
|
+
# A finite SDK per-request remaining duration owns one
|
|
634
|
+
# deadline across this request's retries, never per attempt.
|
|
635
|
+
deadline = time.monotonic() + timeout
|
|
636
|
+
_check_deadline(deadline)
|
|
637
|
+
except ControlHTTPError as exc:
|
|
638
|
+
failure = CanonicalControlAPIError(exc.reason)
|
|
639
|
+
if failure is not None:
|
|
640
|
+
raise failure
|
|
641
|
+
with self._control_channel_lock:
|
|
642
|
+
if self._control_channel is None:
|
|
643
|
+
self._control_channel = OwnedControlChannel(self.session)
|
|
644
|
+
channel = self._control_channel
|
|
645
|
+
timeout = _default_timeout() if timeout is None else _normalize_explicit_timeout(timeout)
|
|
646
|
+
url = self._url(path)
|
|
647
|
+
attempts = self._max_attempts_for(method)
|
|
648
|
+
for attempt in range(1, attempts + 1):
|
|
649
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
650
|
+
raise CanonicalControlAPIError("deadline")
|
|
651
|
+
failure = response = None
|
|
652
|
+
try:
|
|
653
|
+
response = channel.request(method, url, headers={"X-Request-ID": str(uuid.uuid4())},
|
|
654
|
+
params=params, json_body=json_body,
|
|
655
|
+
timeout=_deadline_timeout(timeout, deadline), deadline=deadline,
|
|
656
|
+
max_response_bytes=max_response_bytes)
|
|
657
|
+
except ControlHTTPError as exc:
|
|
658
|
+
failure = CanonicalControlAPIError(exc.reason, status_code=exc.status)
|
|
659
|
+
retryable = exc.retryable or (exc.status is not None and
|
|
660
|
+
_is_retryable_status(exc.status) and exc.reason != "redirect")
|
|
661
|
+
except APIError:
|
|
662
|
+
failure = CanonicalControlAPIError("deadline")
|
|
663
|
+
retryable = False
|
|
664
|
+
if failure is not None:
|
|
665
|
+
if attempt < attempts and retryable:
|
|
666
|
+
delay = _retry_delay(attempt)
|
|
667
|
+
if deadline is not None:
|
|
668
|
+
remaining = deadline - time.monotonic()
|
|
669
|
+
if remaining <= 0:
|
|
670
|
+
raise CanonicalControlAPIError("deadline")
|
|
671
|
+
delay = min(delay, remaining)
|
|
672
|
+
time.sleep(delay)
|
|
673
|
+
continue
|
|
674
|
+
raise failure
|
|
675
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
676
|
+
raise CanonicalControlAPIError("deadline")
|
|
677
|
+
if not 200 <= response.status_code < 300:
|
|
678
|
+
if attempt < attempts and _is_retryable_status(response.status_code):
|
|
679
|
+
try:
|
|
680
|
+
delay = _retry_delay(attempt, response)
|
|
681
|
+
except (ValueError, TypeError, OverflowError, OSError):
|
|
682
|
+
delay = _retry_delay(attempt)
|
|
683
|
+
if not math.isfinite(delay):
|
|
684
|
+
delay = _retry_delay(attempt)
|
|
685
|
+
if deadline is not None:
|
|
686
|
+
remaining = deadline - time.monotonic()
|
|
687
|
+
if remaining <= 0:
|
|
688
|
+
raise CanonicalControlAPIError("deadline")
|
|
689
|
+
delay = min(delay, remaining)
|
|
690
|
+
time.sleep(delay)
|
|
691
|
+
continue
|
|
692
|
+
parsed = None
|
|
693
|
+
if response.refusal is None:
|
|
694
|
+
try:
|
|
695
|
+
parsed = json.loads(response.content.decode("utf-8", errors="replace"))
|
|
696
|
+
except (ValueError, UnicodeError, RecursionError):
|
|
697
|
+
pass
|
|
698
|
+
# Preserve both existing API envelope shapes while retaining
|
|
699
|
+
# only the typed numeric classification, never message/data.
|
|
700
|
+
detail = parsed.get("detail", parsed) if isinstance(parsed, dict) else None
|
|
701
|
+
if isinstance(detail, str):
|
|
702
|
+
decoded = None
|
|
703
|
+
try:
|
|
704
|
+
decoded = json.loads(detail)
|
|
705
|
+
except (ValueError, RecursionError):
|
|
706
|
+
pass
|
|
707
|
+
detail = decoded
|
|
708
|
+
code = detail.get("code") if isinstance(detail, dict) else None
|
|
709
|
+
raise CanonicalControlAPIError("http_error", status_code=response.status_code,
|
|
710
|
+
biz_code=code if type(code) is int else None)
|
|
711
|
+
if not response.content:
|
|
712
|
+
return None
|
|
713
|
+
text = None
|
|
714
|
+
try:
|
|
715
|
+
text = response.content.decode("utf-8", errors="strict")
|
|
716
|
+
except UnicodeError:
|
|
717
|
+
pass
|
|
718
|
+
if text is None:
|
|
719
|
+
raise CanonicalControlAPIError("invalid_utf8", status_code=response.status_code)
|
|
720
|
+
parsed = None
|
|
721
|
+
valid_json = False
|
|
722
|
+
try:
|
|
723
|
+
parsed = json.loads(text)
|
|
724
|
+
valid_json = True
|
|
725
|
+
except (ValueError, RecursionError):
|
|
726
|
+
pass
|
|
727
|
+
if not valid_json:
|
|
728
|
+
raise CanonicalControlAPIError("invalid_json", status_code=response.status_code)
|
|
729
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
730
|
+
raise CanonicalControlAPIError("deadline")
|
|
731
|
+
return parsed
|
|
732
|
+
raise CanonicalControlAPIError("network")
|
|
733
|
+
|
|
734
|
+
def _max_attempts_for(self, method: str) -> int:
|
|
735
|
+
if method.upper() not in _RETRYABLE_METHODS:
|
|
736
|
+
return 1
|
|
737
|
+
return max(1, self.config.http_max_attempts)
|
|
738
|
+
|
|
599
739
|
def _require_cluster_config(self) -> None:
|
|
600
740
|
if has_cluster_header(self.config.headers):
|
|
601
741
|
return
|
|
@@ -1204,6 +1344,7 @@ class APIClient:
|
|
|
1204
1344
|
priority_weight: Optional[int] = None,
|
|
1205
1345
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1206
1346
|
idempotency_key: Optional[str] = None,
|
|
1347
|
+
group_concurrency_policy: Optional[dict] = None,
|
|
1207
1348
|
) -> dict:
|
|
1208
1349
|
"""Create a group with optional scheduling defaults for its Jobs.
|
|
1209
1350
|
|
|
@@ -1254,6 +1395,8 @@ class APIClient:
|
|
|
1254
1395
|
effective_queue_id = queue_id if queue_id is not None else queue
|
|
1255
1396
|
if effective_queue_id is not None:
|
|
1256
1397
|
body["queue_id"] = effective_queue_id
|
|
1398
|
+
if group_concurrency_policy is not None:
|
|
1399
|
+
body["group_concurrency_policy"] = group_concurrency_policy
|
|
1257
1400
|
if max_concurrency is not None:
|
|
1258
1401
|
body["max_concurrency"] = max_concurrency
|
|
1259
1402
|
if priority is not None:
|
|
@@ -2048,6 +2191,7 @@ class APIClient:
|
|
|
2048
2191
|
priority: Optional[str] = None,
|
|
2049
2192
|
priority_weight: Optional[int] = None,
|
|
2050
2193
|
priority_update_mode: Optional[str] = None,
|
|
2194
|
+
group_concurrency_policy: Optional[dict] = None,
|
|
2051
2195
|
) -> dict:
|
|
2052
2196
|
"""Update group properties.
|
|
2053
2197
|
|
|
@@ -2075,6 +2219,8 @@ class APIClient:
|
|
|
2075
2219
|
raise ValueError("priority_update_mode requires priority or priority_weight")
|
|
2076
2220
|
body: dict = {}
|
|
2077
2221
|
|
|
2222
|
+
if group_concurrency_policy is not None:
|
|
2223
|
+
body["group_concurrency_policy"] = group_concurrency_policy
|
|
2078
2224
|
if max_concurrency is not None:
|
|
2079
2225
|
body["max_concurrency"] = max_concurrency
|
|
2080
2226
|
if eval_config is not None:
|
|
@@ -3072,6 +3218,8 @@ class APIClient:
|
|
|
3072
3218
|
account_pool: Optional[str] = None,
|
|
3073
3219
|
metadata: Optional[dict] = None,
|
|
3074
3220
|
tags: Optional[list[str]] = None,
|
|
3221
|
+
parents: Optional[list[str | dict]] = None,
|
|
3222
|
+
auto_follow_parents: Optional[list[str]] = None,
|
|
3075
3223
|
) -> dict:
|
|
3076
3224
|
"""Create a new submission profile."""
|
|
3077
3225
|
body: dict = {"name": name, "spec": spec}
|
|
@@ -3083,6 +3231,10 @@ class APIClient:
|
|
|
3083
3231
|
body["metadata"] = metadata
|
|
3084
3232
|
if tags is not None:
|
|
3085
3233
|
body["tags"] = tags
|
|
3234
|
+
if parents is not None:
|
|
3235
|
+
body["parents"] = parents
|
|
3236
|
+
if auto_follow_parents is not None:
|
|
3237
|
+
body["auto_follow_parents"] = auto_follow_parents
|
|
3086
3238
|
return self._post(self._PROFILES_BASE, body)
|
|
3087
3239
|
|
|
3088
3240
|
def list_profiles(
|
|
@@ -3224,6 +3376,9 @@ class APIClient:
|
|
|
3224
3376
|
account_pool: Optional[str] = None,
|
|
3225
3377
|
metadata: Optional[dict] = None,
|
|
3226
3378
|
reason: Optional[str] = None,
|
|
3379
|
+
version: Optional[int] = None,
|
|
3380
|
+
parents: Optional[list[str | dict]] = None,
|
|
3381
|
+
auto_follow_parents: Optional[list[str]] = None,
|
|
3227
3382
|
) -> dict:
|
|
3228
3383
|
"""Update a submission profile, preserving an omitted default Pool.
|
|
3229
3384
|
|
|
@@ -3239,7 +3394,108 @@ class APIClient:
|
|
|
3239
3394
|
body["metadata"] = metadata
|
|
3240
3395
|
if reason is not None:
|
|
3241
3396
|
body["reason"] = reason
|
|
3242
|
-
|
|
3397
|
+
if version is not None:
|
|
3398
|
+
if type(version) is not int or version < 1:
|
|
3399
|
+
raise ValueError("version must be a positive integer")
|
|
3400
|
+
body["version"] = version
|
|
3401
|
+
if parents is not None:
|
|
3402
|
+
body["parents"] = parents
|
|
3403
|
+
if auto_follow_parents is not None:
|
|
3404
|
+
body["auto_follow_parents"] = auto_follow_parents
|
|
3405
|
+
return self._put(f"{self._PROFILES_BASE}/{quote(profile_id, safe='')}", body)
|
|
3406
|
+
|
|
3407
|
+
def patch_profile(self, profile_id: str, fields: dict) -> dict:
|
|
3408
|
+
"""Patch own definition; callers should supply their original version."""
|
|
3409
|
+
return self._request(
|
|
3410
|
+
"PATCH", f"{self._PROFILES_BASE}/{quote(profile_id, safe='')}", json_body=fields
|
|
3411
|
+
)
|
|
3412
|
+
|
|
3413
|
+
def resolve_profile(self, identifier: str, *, version: Optional[int] = None) -> dict:
|
|
3414
|
+
return self._get(
|
|
3415
|
+
f"{self._PROFILES_BASE}/{quote(identifier, safe='')}/resolved",
|
|
3416
|
+
params={"version": version} if version is not None else None,
|
|
3417
|
+
)
|
|
3418
|
+
|
|
3419
|
+
def preview_profile(self, draft: dict) -> dict:
|
|
3420
|
+
return self._post(f"{self._PROFILES_BASE}/resolve", draft)
|
|
3421
|
+
|
|
3422
|
+
def profile_children(
|
|
3423
|
+
self,
|
|
3424
|
+
identifier: str,
|
|
3425
|
+
*,
|
|
3426
|
+
page: int = 1,
|
|
3427
|
+
page_size: int = 20,
|
|
3428
|
+
target_version: Optional[int] = None,
|
|
3429
|
+
eligible_only: bool = False,
|
|
3430
|
+
current_only: bool = False,
|
|
3431
|
+
) -> dict:
|
|
3432
|
+
params = {"page": page, "page_size": page_size}
|
|
3433
|
+
if target_version is not None:
|
|
3434
|
+
params["target_version"] = target_version
|
|
3435
|
+
if eligible_only:
|
|
3436
|
+
params["eligible_only"] = "true"
|
|
3437
|
+
if current_only:
|
|
3438
|
+
params["current_only"] = "true"
|
|
3439
|
+
return self._get(
|
|
3440
|
+
f"{self._PROFILES_BASE}/{quote(identifier, safe='')}/children",
|
|
3441
|
+
params=params,
|
|
3442
|
+
)
|
|
3443
|
+
|
|
3444
|
+
def profile_diff(
|
|
3445
|
+
self,
|
|
3446
|
+
identifier: str,
|
|
3447
|
+
from_version: Optional[int] = None,
|
|
3448
|
+
to_version: Optional[int] = None,
|
|
3449
|
+
view: str = "own",
|
|
3450
|
+
*,
|
|
3451
|
+
compare_to: Optional[str] = None,
|
|
3452
|
+
) -> dict:
|
|
3453
|
+
"""Compare versions of one Profile, or two Profiles (omitted versions use current)."""
|
|
3454
|
+
params = {"from": from_version, "to": to_version, "view": view}
|
|
3455
|
+
if compare_to is not None:
|
|
3456
|
+
params["compare_to"] = compare_to
|
|
3457
|
+
result = self._get(
|
|
3458
|
+
f"{self._PROFILES_BASE}/{quote(identifier, safe='')}/diff",
|
|
3459
|
+
params=params,
|
|
3460
|
+
)
|
|
3461
|
+
if compare_to is not None:
|
|
3462
|
+
# Older servers ignore unknown query fields. Do not present their
|
|
3463
|
+
# single-Profile response as a successful cross-Profile comparison.
|
|
3464
|
+
for field in ("from_profile", "to_profile"):
|
|
3465
|
+
ref = result.get(field) if isinstance(result, dict) else None
|
|
3466
|
+
if (
|
|
3467
|
+
not isinstance(ref, dict)
|
|
3468
|
+
or not isinstance(ref.get("profile_id"), str)
|
|
3469
|
+
or not ref["profile_id"].startswith("sp-")
|
|
3470
|
+
or type(ref.get("version")) is not int
|
|
3471
|
+
or ref["version"] < 1
|
|
3472
|
+
):
|
|
3473
|
+
raise APIError(
|
|
3474
|
+
200,
|
|
3475
|
+
"Server did not confirm cross-Profile diff support; upgrade the server before comparing different Profiles",
|
|
3476
|
+
None,
|
|
3477
|
+
None,
|
|
3478
|
+
)
|
|
3479
|
+
return result
|
|
3480
|
+
|
|
3481
|
+
def cascade_profiles(self, identifier: str, request: dict) -> dict:
|
|
3482
|
+
return self._post(f"{self._PROFILES_BASE}/{quote(identifier, safe='')}/cascades", request)
|
|
3483
|
+
|
|
3484
|
+
def get_profile_cascade(self, operation_id: str) -> dict:
|
|
3485
|
+
return self._get(f"/apis/v1/profile-cascades/{quote(operation_id, safe='')}")
|
|
3486
|
+
|
|
3487
|
+
def profile_cascade_history(
|
|
3488
|
+
self, profile_id: str, *, page: int = 1, page_size: int = 20
|
|
3489
|
+
) -> dict:
|
|
3490
|
+
return self._get(
|
|
3491
|
+
f"{self._PROFILES_BASE}/{quote(profile_id, safe='')}/cascades",
|
|
3492
|
+
params={"page": page, "page_size": page_size},
|
|
3493
|
+
)
|
|
3494
|
+
|
|
3495
|
+
def retry_profile_cascade(self, operation_id: str, items: list[dict]) -> dict:
|
|
3496
|
+
return self._post(
|
|
3497
|
+
f"/apis/v1/profile-cascades/{quote(operation_id, safe='')}/retry", {"items": items}
|
|
3498
|
+
)
|
|
3243
3499
|
|
|
3244
3500
|
def delete_profile(self, profile_id: str) -> dict:
|
|
3245
3501
|
"""Delete a submission profile."""
|
|
@@ -3523,12 +3779,34 @@ def _redact_url_query_secrets(value: str) -> str:
|
|
|
3523
3779
|
|
|
3524
3780
|
|
|
3525
3781
|
def _response_error_detail(resp: requests.Response) -> Any:
|
|
3782
|
+
"""错误响应体 → detail(兼容两代服务端错误形态,G2 修复)。
|
|
3783
|
+
|
|
3784
|
+
* FastAPI ``{"detail": <str|dict>}``:保持旧语义——detail 键优先
|
|
3785
|
+
(dict 时**整个内层 dict** 作为 detail 返回,不只是取 message);
|
|
3786
|
+
* Go/Gin 裸信封 ``{code, message, data}``(apiserver
|
|
3787
|
+
``pkg/api/response/response.go writeErrorData``):**整个响应 dict**
|
|
3788
|
+
作为 detail 返回——9404x 数字 code 与 ``data``(error_code/
|
|
3789
|
+
apply_url 等结构化负载)不再被降级成 message 字符串丢失;
|
|
3790
|
+
* 旧 ``{"message": ...}``(无 detail)形态:保持旧语义取 message
|
|
3791
|
+
字符串(detail/message 缺省回退 resp.text)。
|
|
3792
|
+
|
|
3793
|
+
detail 消费方(如 shared_adapter._decode_api_error)按"dict 即结构化
|
|
3794
|
+
错误体"自行解 code/data;str detail 继续支持(旧展示路径不变)。
|
|
3795
|
+
"""
|
|
3526
3796
|
try:
|
|
3527
3797
|
err_data = resp.json()
|
|
3528
3798
|
except Exception:
|
|
3529
3799
|
return resp.text
|
|
3530
3800
|
if isinstance(err_data, dict):
|
|
3531
|
-
|
|
3801
|
+
if "detail" in err_data:
|
|
3802
|
+
inner = err_data["detail"]
|
|
3803
|
+
if isinstance(inner, dict):
|
|
3804
|
+
return inner # FastAPI detail=dict:内层即结构化错误体
|
|
3805
|
+
return inner # detail=str:旧语义
|
|
3806
|
+
if "code" in err_data:
|
|
3807
|
+
# Go 裸信封 {code, message, data}:保留完整结构化错误体
|
|
3808
|
+
return err_data
|
|
3809
|
+
return err_data.get("message", resp.text)
|
|
3532
3810
|
return err_data
|
|
3533
3811
|
|
|
3534
3812
|
|
|
@@ -3550,9 +3828,7 @@ def _retry_delay(
|
|
|
3550
3828
|
*,
|
|
3551
3829
|
max_wait: float | None = None,
|
|
3552
3830
|
) -> float:
|
|
3553
|
-
delay = _HTTP_RETRY_INITIAL_DELAY * (
|
|
3554
|
-
_HTTP_RETRY_BACKOFF_FACTOR ** min(2, max(0, attempt - 1))
|
|
3555
|
-
)
|
|
3831
|
+
delay = _HTTP_RETRY_INITIAL_DELAY * (_HTTP_RETRY_BACKOFF_FACTOR ** min(2, max(0, attempt - 1)))
|
|
3556
3832
|
# Local backoff may be shortened to the wait ceiling; a gateway hint never is.
|
|
3557
3833
|
if max_wait is not None:
|
|
3558
3834
|
delay = min(delay, max_wait)
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""Thin user-data read orchestration over the SDK's typed read boundary.
|
|
2
|
+
|
|
3
|
+
The SDK owns wire/URL/policy validation, streaming and atomic publication.
|
|
4
|
+
This module owns the invocation's absolute budget, operator input guards,
|
|
5
|
+
matching typed results and honest per-file display. It does not replace the
|
|
6
|
+
runtime fs_get entry before the cross-repository integration gates pass.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import math
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import time
|
|
13
|
+
from typing import Any, Optional
|
|
14
|
+
|
|
15
|
+
from .canonical_put import _bind_operation_deadline
|
|
16
|
+
|
|
17
|
+
__all__ = ["CanonicalReadOrchestrationError", "ReadOutcome", "get_user_data"]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class CanonicalReadOrchestrationError(Exception):
|
|
21
|
+
"""A safe input, dependency, budget or typed-boundary refusal."""
|
|
22
|
+
|
|
23
|
+
def __init__(self, kind: str, message: str):
|
|
24
|
+
super().__init__(message)
|
|
25
|
+
self.kind = kind
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ReadOutcome:
|
|
29
|
+
"""Presentation result; only a matching SDK verified result is verified."""
|
|
30
|
+
|
|
31
|
+
def __init__(self, mode: str, *, canonical_key: str = "",
|
|
32
|
+
legacy_profile: str = "", verified: bool = False,
|
|
33
|
+
target: Optional[Path] = None):
|
|
34
|
+
self.mode = mode
|
|
35
|
+
self.canonical_key = canonical_key
|
|
36
|
+
self.legacy_profile = legacy_profile
|
|
37
|
+
self.verified = verified
|
|
38
|
+
self.target = target
|
|
39
|
+
|
|
40
|
+
def notice(self) -> str:
|
|
41
|
+
if self.mode == "canonical" and self.verified:
|
|
42
|
+
return (
|
|
43
|
+
"canonical read: content verified against the committed "
|
|
44
|
+
"anchor (sha256/size/generation) by the SDK reader"
|
|
45
|
+
)
|
|
46
|
+
if self.mode == "legacy-unanchored":
|
|
47
|
+
return (
|
|
48
|
+
"legacy read: NO sha256/generation guarantee "
|
|
49
|
+
"(unanchored); served via the trusted legacy profile "
|
|
50
|
+
f"{self.legacy_profile!r}"
|
|
51
|
+
)
|
|
52
|
+
return f"read completed (mode={self.mode!r})"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _remaining(absolute: Optional[float]) -> Optional[float]:
|
|
56
|
+
if absolute is None:
|
|
57
|
+
return None
|
|
58
|
+
remaining = absolute - time.monotonic()
|
|
59
|
+
if remaining <= 0:
|
|
60
|
+
raise CanonicalReadOrchestrationError(
|
|
61
|
+
"budget_exhausted", "the read operation exceeded its time budget")
|
|
62
|
+
return remaining
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _reader_failure() -> None:
|
|
66
|
+
# No repr of a resolution/result: they can contain signed URLs.
|
|
67
|
+
raise CanonicalReadOrchestrationError(
|
|
68
|
+
"reader_failure", "the SDK reader returned an inconsistent typed result")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class _ReadProgress:
|
|
72
|
+
"""Use the existing display lifecycle without inventing byte callbacks.
|
|
73
|
+
|
|
74
|
+
The reader exposes file completion, so the display changes to the known
|
|
75
|
+
completed size only after a matching result and completion budget check.
|
|
76
|
+
Hashing or bytes merely requested are never reported as transferred.
|
|
77
|
+
"""
|
|
78
|
+
|
|
79
|
+
def __init__(self, enabled, key: str, size: int):
|
|
80
|
+
self.enabled, self.key, self.size = enabled, key, size
|
|
81
|
+
self.report = None
|
|
82
|
+
|
|
83
|
+
def __enter__(self):
|
|
84
|
+
if self.enabled is False:
|
|
85
|
+
return self
|
|
86
|
+
from .fs_transfer import TransferProgress
|
|
87
|
+
|
|
88
|
+
report = TransferProgress("Downloading", self.enabled)
|
|
89
|
+
report.start({self.key: self.size})
|
|
90
|
+
report.__enter__()
|
|
91
|
+
self.report = report
|
|
92
|
+
return self
|
|
93
|
+
|
|
94
|
+
def finish(self):
|
|
95
|
+
if self.report is not None:
|
|
96
|
+
self.report.finish(self.key)
|
|
97
|
+
|
|
98
|
+
def __exit__(self, kind, value, traceback):
|
|
99
|
+
if self.report is not None:
|
|
100
|
+
self.report.__exit__(kind, value, traceback)
|
|
101
|
+
return False
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def get_user_data(
|
|
105
|
+
owner: str,
|
|
106
|
+
relative_path: str,
|
|
107
|
+
cluster: str,
|
|
108
|
+
target: Path,
|
|
109
|
+
*,
|
|
110
|
+
reader: Any,
|
|
111
|
+
transport: Any = None,
|
|
112
|
+
deadline: Optional[float] = None,
|
|
113
|
+
overwrite: bool = True,
|
|
114
|
+
progress: Optional[bool] = None,
|
|
115
|
+
) -> ReadOutcome:
|
|
116
|
+
"""Resolve and download once with a shared absolute monotonic deadline.
|
|
117
|
+
|
|
118
|
+
Public SDK ``deadline`` values are durations, freshly computed before
|
|
119
|
+
each phase. The accepted adapter receives the absolute deadline before
|
|
120
|
+
discovery, including when reached through the actual reader's transport.
|
|
121
|
+
None retains the SDK's optional deadline semantics. SDK typed errors
|
|
122
|
+
propagate without a fallback, retry or mode switch in this module.
|
|
123
|
+
"""
|
|
124
|
+
if reader is None:
|
|
125
|
+
raise CanonicalReadOrchestrationError(
|
|
126
|
+
"reader_missing", "the typed user-data SDK reader is not available")
|
|
127
|
+
if (any(not isinstance(v, str) or not v
|
|
128
|
+
for v in (owner, relative_path, cluster))
|
|
129
|
+
or any(char.isspace() for value in (owner, cluster) for char in value)):
|
|
130
|
+
raise CanonicalReadOrchestrationError(
|
|
131
|
+
"invalid_input", "owner and cluster must be nonempty without whitespace; relative_path must be nonempty")
|
|
132
|
+
# A relative path is an exact object identity. Leading/trailing spaces are
|
|
133
|
+
# legal bytes and must reach the SDK unchanged; validation rejects aliases
|
|
134
|
+
# instead of silently selecting a different object.
|
|
135
|
+
if deadline is not None and (
|
|
136
|
+
isinstance(deadline, bool) or not isinstance(deadline, (int, float))
|
|
137
|
+
or not math.isfinite(deadline)):
|
|
138
|
+
raise CanonicalReadOrchestrationError(
|
|
139
|
+
"invalid_input", "the read deadline must be a finite absolute monotonic time")
|
|
140
|
+
if not isinstance(overwrite, bool) or (
|
|
141
|
+
progress is not None and not isinstance(progress, bool)):
|
|
142
|
+
raise CanonicalReadOrchestrationError(
|
|
143
|
+
"invalid_input", "overwrite and progress must be boolean options")
|
|
144
|
+
if not callable(getattr(reader, "resolve", None)) or not callable(
|
|
145
|
+
getattr(reader, "download", None)):
|
|
146
|
+
_reader_failure()
|
|
147
|
+
|
|
148
|
+
from instance_repo.clients.user_data_reads import (
|
|
149
|
+
CanonicalReadResolution, LegacyReadResolution,
|
|
150
|
+
VerifiedReadResult, LegacyReadResult,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
_remaining(deadline)
|
|
154
|
+
_bind_operation_deadline(reader, deadline)
|
|
155
|
+
if transport is not None:
|
|
156
|
+
_bind_operation_deadline(transport, deadline)
|
|
157
|
+
resolution = reader.resolve(
|
|
158
|
+
owner=owner, relative_path=relative_path, cluster=cluster,
|
|
159
|
+
deadline=_remaining(deadline))
|
|
160
|
+
_remaining(deadline)
|
|
161
|
+
|
|
162
|
+
if type(resolution) is CanonicalReadResolution:
|
|
163
|
+
if resolution.mode != "canonical":
|
|
164
|
+
_reader_failure()
|
|
165
|
+
key, size = resolution.canonical_key, resolution.anchor.size
|
|
166
|
+
elif type(resolution) is LegacyReadResolution:
|
|
167
|
+
if resolution.mode != "legacy-unanchored":
|
|
168
|
+
_reader_failure()
|
|
169
|
+
key, size = resolution.legacy_key, resolution.size
|
|
170
|
+
else:
|
|
171
|
+
_reader_failure()
|
|
172
|
+
if (resolution.owner, resolution.relative_path, resolution.cluster) != (
|
|
173
|
+
owner, relative_path, cluster):
|
|
174
|
+
_reader_failure()
|
|
175
|
+
|
|
176
|
+
target = Path(target)
|
|
177
|
+
with _ReadProgress(progress, key, size) as display:
|
|
178
|
+
result = reader.download(
|
|
179
|
+
resolution, target, overwrite=overwrite, deadline=_remaining(deadline))
|
|
180
|
+
_remaining(deadline)
|
|
181
|
+
if type(resolution) is CanonicalReadResolution:
|
|
182
|
+
if type(result) is not VerifiedReadResult or (
|
|
183
|
+
result.mode != "canonical"
|
|
184
|
+
or result.canonical_key != resolution.canonical_key
|
|
185
|
+
or result.anchor != resolution.anchor
|
|
186
|
+
or result.sha256_verified != resolution.anchor.sha256
|
|
187
|
+
or type(result.bytes_written) is not int
|
|
188
|
+
or result.bytes_written != size
|
|
189
|
+
or result.target != target):
|
|
190
|
+
_reader_failure()
|
|
191
|
+
outcome = ReadOutcome("canonical", canonical_key=key,
|
|
192
|
+
verified=True, target=target)
|
|
193
|
+
else:
|
|
194
|
+
if type(result) is not LegacyReadResult or (
|
|
195
|
+
result.mode != "legacy-unanchored"
|
|
196
|
+
or result.legacy_key != resolution.legacy_key
|
|
197
|
+
or result.unanchored is not True
|
|
198
|
+
or type(result.bytes_written) is not int
|
|
199
|
+
or result.bytes_written != size
|
|
200
|
+
or result.target != target):
|
|
201
|
+
_reader_failure()
|
|
202
|
+
outcome = ReadOutcome("legacy-unanchored",
|
|
203
|
+
legacy_profile=resolution.legacy_profile,
|
|
204
|
+
verified=False, target=target)
|
|
205
|
+
_remaining(deadline)
|
|
206
|
+
display.finish()
|
|
207
|
+
# Display completion/cleanup belongs to this invocation too. Never report
|
|
208
|
+
# success after that work has consumed the caller's remaining budget.
|
|
209
|
+
_remaining(deadline)
|
|
210
|
+
return outcome
|