ap-client 0.8.1.dev0__tar.gz → 0.8.1.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/PKG-INFO +5 -4
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/api.py +454 -13
- ap_client-0.8.1.dev1/ap_client/canonical_get.py +210 -0
- ap_client-0.8.1.dev1/ap_client/canonical_put.py +956 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/cli.py +621 -200
- ap_client-0.8.1.dev1/ap_client/cli_help.py +159 -0
- ap_client-0.8.1.dev1/ap_client/cli_submission_defaults.py +95 -0
- ap_client-0.8.1.dev1/ap_client/cluster_layout_fetcher.py +362 -0
- ap_client-0.8.1.dev1/ap_client/cluster_region_resolver.py +394 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/config.py +32 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/dfs_commands.py +8 -3
- ap_client-0.8.1.dev1/ap_client/fs_commands.py +827 -0
- ap_client-0.8.1.dev1/ap_client/fs_storage.py +53 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/instance_commands.py +70 -37
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/irepo_sdk.py +372 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/ossutil_io.py +65 -8
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_commands.py +317 -93
- ap_client-0.8.1.dev1/ap_client/replica_policy.py +114 -0
- ap_client-0.8.1.dev1/ap_client/shared_adapter.py +1476 -0
- ap_client-0.8.1.dev1/ap_client/shared_commands.py +1138 -0
- ap_client-0.8.1.dev1/ap_client/tool_manifest.py +184 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/waiter.py +3 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/pyproject.toml +5 -4
- ap_client-0.8.1.dev0/ap_client/fs_commands.py +0 -861
- ap_client-0.8.1.dev0/ap_client/fs_storage.py +0 -146
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/__init__.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/dataset_name.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/dataset_register.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/exporter.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/fs_transfer.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/instance_options.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/managed_ak.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/manifest_view.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/ossutil_hash.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_cascade_all.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_cascade_cli.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_cascade_display.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_cascade_state.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_diagnostics.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_display.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/profile_references.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/repo_read_guard.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/share_api.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/split_publish.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/submission_metadata.py +0 -0
- {ap_client-0.8.1.dev0 → ap_client-0.8.1.dev1}/ap_client/tbb.py +0 -0
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.8.1.
|
|
3
|
+
Version: 0.8.1.dev1
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: click<8.2.0,>=8.1.8
|
|
6
7
|
Requires-Dist: irepo-artifact==0.2.6
|
|
7
8
|
Requires-Dist: pyyaml>=6.0
|
|
8
9
|
Requires-Dist: requests>=2.28.0
|
|
9
10
|
Requires-Dist: rich>=13.0.0
|
|
10
|
-
Requires-Dist: typer
|
|
11
|
+
Requires-Dist: typer<0.21.0,>=0.20.1
|
|
11
12
|
Requires-Dist: websockets>=13.0
|
|
12
13
|
Provides-Extra: all
|
|
13
|
-
Requires-Dist: instance-repo[oss]
|
|
14
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.7.0; extra == 'all'
|
|
14
15
|
Provides-Extra: dataset
|
|
15
|
-
Requires-Dist: instance-repo[oss]
|
|
16
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.7.0; extra == 'dataset'
|
|
16
17
|
Description-Content-Type: text/markdown
|
|
17
18
|
|
|
18
19
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -5,7 +5,9 @@ import math
|
|
|
5
5
|
import os
|
|
6
6
|
import platform
|
|
7
7
|
import re
|
|
8
|
+
import socket
|
|
8
9
|
import sys
|
|
10
|
+
import threading
|
|
9
11
|
import time
|
|
10
12
|
import uuid
|
|
11
13
|
from email.utils import parsedate_to_datetime
|
|
@@ -306,6 +308,263 @@ class PaginationProtocolError(RuntimeError):
|
|
|
306
308
|
"""The server stopped honoring a pagination protocol already in use."""
|
|
307
309
|
|
|
308
310
|
|
|
311
|
+
# Bounded canonical control-response helpers (CR 30865361 review P1 fixes).
|
|
312
|
+
|
|
313
|
+
# Upper bound for one canonical control response body (metadata JSONs are
|
|
314
|
+
# small; this only bounds memory and the byte-drip amortisation window).
|
|
315
|
+
_DEFAULT_CONTROL_BYTES = 8 * 1024 * 1024
|
|
316
|
+
_MAX_CONTROL_BYTES = 64 * 1024 * 1024
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
class CanonicalControlAPIError(APIError):
|
|
320
|
+
"""Sanitized typed refusal for bounded canonical control requests.
|
|
321
|
+
|
|
322
|
+
Never holds a native response or raw cause; redirects are reported as
|
|
323
|
+
``redirect`` only after allow_redirects=False already prevented any
|
|
324
|
+
credential from following the Location."""
|
|
325
|
+
|
|
326
|
+
def __init__(self, reason, *, status_code=None, biz_code=None):
|
|
327
|
+
self.reason = reason
|
|
328
|
+
detail = {"code": biz_code} if type(biz_code) is int else "canonical control request failed"
|
|
329
|
+
super().__init__(status_code, detail, None, None)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _validate_control_bytes(limit: int) -> None:
|
|
333
|
+
if type(limit) is not int or not 1 <= limit <= _MAX_CONTROL_BYTES:
|
|
334
|
+
raise CanonicalControlAPIError("invalid_limit")
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _close_response_quietly(resp) -> None:
|
|
338
|
+
"""Close a streamed response, tolerating already-closed objects."""
|
|
339
|
+
try:
|
|
340
|
+
resp.close()
|
|
341
|
+
except Exception:
|
|
342
|
+
pass
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _response_socket(resp) -> Optional[Any]:
|
|
346
|
+
"""Best-effort extraction of the response's live socket (downgrade-safe).
|
|
347
|
+
|
|
348
|
+
urllib3 ``HTTPResponse._fp`` → http.client ``.fp`` (BufferedReader) →
|
|
349
|
+
``.raw`` (SocketIO) → ``._sock``. Any missing/private-shift layer just
|
|
350
|
+
returns None — callers treat None as "no hard interruption available"
|
|
351
|
+
and rely on the in-loop deadline checks only.
|
|
352
|
+
"""
|
|
353
|
+
try:
|
|
354
|
+
fp = getattr(resp, "raw", None)
|
|
355
|
+
fp = getattr(fp, "_fp", None)
|
|
356
|
+
if fp is None:
|
|
357
|
+
return None
|
|
358
|
+
buf = getattr(fp, "fp", None)
|
|
359
|
+
rawio = getattr(buf, "raw", None) if buf is not None else None
|
|
360
|
+
sock = getattr(rawio, "_sock", None) if rawio is not None else None
|
|
361
|
+
if sock is not None and callable(getattr(sock, "shutdown", None)):
|
|
362
|
+
return sock
|
|
363
|
+
except Exception:
|
|
364
|
+
pass
|
|
365
|
+
return None
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
class _DeadlineWatchdog:
|
|
369
|
+
"""Force-interrupt a blocked streamed body read at the deadline.
|
|
370
|
+
|
|
371
|
+
``requests``/``urllib3`` buffered reads block until the requested amount
|
|
372
|
+
(or EOF) arrives: with a Content-Length body dripping one byte per
|
|
373
|
+
interval, ``iter_content(chunk_size=64K)`` never returns mid-stream, so
|
|
374
|
+
in-loop deadline checks never run (review P1: 80ms budget exited at
|
|
375
|
+
~300ms via drip). Same approach as the SDK transport's per-request
|
|
376
|
+
watchdog: an independent daemon timer shuts the live socket down at the
|
|
377
|
+
deadline; the blocked read then unblocks with a transport error, which
|
|
378
|
+
the callers classify against the actual clock.
|
|
379
|
+
|
|
380
|
+
Connection-reuse safety (review P2, two races):
|
|
381
|
+
1. cancel/fire race: ``threading.Timer.cancel`` only prevents a callback
|
|
382
|
+
that has not *started*. Disarm happens under a lock in both
|
|
383
|
+
directions — the callback re-checks the flag — and ``__exit__``
|
|
384
|
+
joins the timer thread.
|
|
385
|
+
2. release-before-exit race (the deeper one): urllib3 returns a
|
|
386
|
+
keep-alive connection to the pool as soon as ``iter_content`` drains
|
|
387
|
+
to EOF — BEFORE ``__exit__`` runs disarm/join. A later request can
|
|
388
|
+
borrow the connection in that window, and a stale timer firing then
|
|
389
|
+
kills an unrelated request. The callback therefore does NOT use the
|
|
390
|
+
socket captured at construction: at fire time it re-extracts the
|
|
391
|
+
socket from the response and shuts it down only when it is still the
|
|
392
|
+
SAME object — after EOF the response's file pointer is closed
|
|
393
|
+
(``_fp.fp = None``), extraction yields None, and the callback gives
|
|
394
|
+
up (the connection is no longer ours to interrupt). Construction
|
|
395
|
+
still keeps the socket purely as the identity reference.
|
|
396
|
+
|
|
397
|
+
No-socket fallback: if the layered private path is unavailable, the
|
|
398
|
+
watchdog stays inert and the in-loop checks remain the only guard.
|
|
399
|
+
"""
|
|
400
|
+
|
|
401
|
+
def __init__(self, resp, deadline: Optional[float]):
|
|
402
|
+
self._resp = resp
|
|
403
|
+
self._sock = _response_socket(resp) if deadline is not None else None
|
|
404
|
+
self._deadline = deadline
|
|
405
|
+
self._timer: Optional[threading.Timer] = None
|
|
406
|
+
# Guards the disarmed flag AND serializes callback shutdown vs
|
|
407
|
+
# __exit__ disarm — the callback holds this lock while shutting the
|
|
408
|
+
# socket down, so once __exit__ completes the callback can never
|
|
409
|
+
# still be mid-shutdown.
|
|
410
|
+
self._lock = threading.Lock()
|
|
411
|
+
self._disarmed = False
|
|
412
|
+
|
|
413
|
+
def __enter__(self) -> "_DeadlineWatchdog":
|
|
414
|
+
sock = self._sock
|
|
415
|
+
if sock is not None:
|
|
416
|
+
delay = self._deadline - time.monotonic()
|
|
417
|
+
if delay > 0:
|
|
418
|
+
self._timer = threading.Timer(delay, self._fire)
|
|
419
|
+
self._timer.daemon = True
|
|
420
|
+
self._timer.start()
|
|
421
|
+
else:
|
|
422
|
+
self._fire()
|
|
423
|
+
return self
|
|
424
|
+
|
|
425
|
+
def __exit__(self, *exc_info) -> bool:
|
|
426
|
+
# Disarm BEFORE releasing anything: with the lock held, any in-flight
|
|
427
|
+
# callback has either finished its shutdown already (this read is
|
|
428
|
+
# being aborted anyway) or will re-check the flag and give up.
|
|
429
|
+
with self._lock:
|
|
430
|
+
self._disarmed = True
|
|
431
|
+
timer = self._timer
|
|
432
|
+
if timer is not None:
|
|
433
|
+
self._timer = None
|
|
434
|
+
timer.cancel()
|
|
435
|
+
# The timer thread may have passed its finished-event check and
|
|
436
|
+
# be running the callback right now (cancel is advisory there).
|
|
437
|
+
# Join it so the callback has fully exited — only then does the
|
|
438
|
+
# caller close the response and urllib3 release the connection
|
|
439
|
+
# back to the pool for reuse by later requests.
|
|
440
|
+
timer.join(timeout=5.0)
|
|
441
|
+
return False
|
|
442
|
+
|
|
443
|
+
def _fire(self) -> None:
|
|
444
|
+
# Runs on the timer thread (or synchronously from __enter__ when the
|
|
445
|
+
# budget is already spent). The lock + disarmed re-check closes the
|
|
446
|
+
# cancel race; the fresh re-extraction + identity check closes the
|
|
447
|
+
# release-before-exit race: after the stream drained to EOF urllib3
|
|
448
|
+
# has already returned the connection to the pool (the response's
|
|
449
|
+
# file pointer is closed), so the extraction yields None (or a
|
|
450
|
+
# different socket) and the stale fire gives up instead of killing
|
|
451
|
+
# whatever request borrowed the connection next.
|
|
452
|
+
with self._lock:
|
|
453
|
+
if self._disarmed:
|
|
454
|
+
return
|
|
455
|
+
current = _response_socket(self._resp)
|
|
456
|
+
if current is None or current is not self._sock:
|
|
457
|
+
return
|
|
458
|
+
try:
|
|
459
|
+
current.shutdown(socket.SHUT_RDWR)
|
|
460
|
+
except Exception:
|
|
461
|
+
pass
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _read_bounded_control_bytes(resp, limit: int = _DEFAULT_CONTROL_BYTES,
|
|
465
|
+
deadline: Optional[float] = None) -> bytes:
|
|
466
|
+
"""Stream one bounded canonical control response via public requests APIs.
|
|
467
|
+
|
|
468
|
+
Content-Length pre-check plus streamed chunk reads keep memory at
|
|
469
|
+
O(cap+chunk) — a byte-drip response cannot grow unbounded. With a finite
|
|
470
|
+
``deadline`` the read is ALSO hard-interruptible: a watchdog shuts the
|
|
471
|
+
response socket down at the deadline, so a slow drip cannot outlive the
|
|
472
|
+
budget even while ``iter_content`` is blocked mid-block (its buffered
|
|
473
|
+
read waits for the full chunk/EOF and would otherwise defer every
|
|
474
|
+
in-loop deadline check until the body finishes)."""
|
|
475
|
+
_validate_control_bytes(limit)
|
|
476
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
477
|
+
raise CanonicalControlAPIError("deadline")
|
|
478
|
+
length = resp.headers.get("Content-Length")
|
|
479
|
+
if isinstance(length, str) and length.isascii() and length.isdecimal():
|
|
480
|
+
digits, cap = length.lstrip('0') or '0', str(limit)
|
|
481
|
+
if len(digits) > len(cap) or (len(digits) == len(cap) and digits > cap):
|
|
482
|
+
raise CanonicalControlAPIError("response_limit", status_code=resp.status_code)
|
|
483
|
+
data = bytearray()
|
|
484
|
+
try:
|
|
485
|
+
with _DeadlineWatchdog(resp, deadline):
|
|
486
|
+
for block in resp.iter_content(chunk_size=min(65536, max(1, limit))):
|
|
487
|
+
if block:
|
|
488
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
489
|
+
# The stream is interrupted by the operation budget
|
|
490
|
+
# itself, not by body completion.
|
|
491
|
+
raise CanonicalControlAPIError("deadline")
|
|
492
|
+
if len(block) > limit - len(data):
|
|
493
|
+
raise CanonicalControlAPIError(
|
|
494
|
+
"response_limit", status_code=resp.status_code)
|
|
495
|
+
data.extend(block)
|
|
496
|
+
except CanonicalControlAPIError:
|
|
497
|
+
raise
|
|
498
|
+
except Exception:
|
|
499
|
+
# Hard interruption (watchdog shutdown) or transport error: classify
|
|
500
|
+
# against the actual clock — a spent budget is a deadline refusal,
|
|
501
|
+
# never a raw exception or a partial-body success.
|
|
502
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
503
|
+
raise CanonicalControlAPIError("deadline") from None
|
|
504
|
+
raise
|
|
505
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
506
|
+
# Body completed but the budget is spent: refuse the late success.
|
|
507
|
+
raise CanonicalControlAPIError("deadline")
|
|
508
|
+
return bytes(data)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _bounded_response_prefix(resp, limit: int = _DEFAULT_CONTROL_BYTES,
|
|
512
|
+
deadline: Optional[float] = None) -> bytes:
|
|
513
|
+
"""Read one bounded first block of an error body for typed classification.
|
|
514
|
+
|
|
515
|
+
Oversized bodies contribute only this bounded prefix; their real status
|
|
516
|
+
stays reportable. With a finite ``deadline`` the same watchdog
|
|
517
|
+
hard-interrupts a blocked first-block read; a deadline reached mid-prefix
|
|
518
|
+
(or a transport interruption with the budget spent) is reported as
|
|
519
|
+
``deadline`` — never a late ``http_error`` classification of an unread
|
|
520
|
+
body."""
|
|
521
|
+
try:
|
|
522
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
523
|
+
raise CanonicalControlAPIError("deadline")
|
|
524
|
+
with _DeadlineWatchdog(resp, deadline):
|
|
525
|
+
for block in resp.iter_content(chunk_size=min(65536, max(1, limit))):
|
|
526
|
+
if block:
|
|
527
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
528
|
+
raise CanonicalControlAPIError("deadline")
|
|
529
|
+
return bytes(block[:limit])
|
|
530
|
+
except CanonicalControlAPIError:
|
|
531
|
+
raise
|
|
532
|
+
except Exception:
|
|
533
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
534
|
+
# The read was interrupted (watchdog shutdown or read timeout)
|
|
535
|
+
# with the budget already spent: report the deadline, not a
|
|
536
|
+
# post-hoc classification of an unread error body.
|
|
537
|
+
raise CanonicalControlAPIError("deadline") from None
|
|
538
|
+
return b""
|
|
539
|
+
return b""
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _bounded_error_biz_code(prefix: bytes) -> Optional[int]:
|
|
543
|
+
"""Typed numeric classification only; never message/data.
|
|
544
|
+
|
|
545
|
+
Decodes the dual envelope (bare ``{code,message,data}`` or FastAPI
|
|
546
|
+
``{detail}``) from a bounded prefix so raw error text is never
|
|
547
|
+
forwarded; only the typed numeric family survives."""
|
|
548
|
+
parsed = None
|
|
549
|
+
if prefix:
|
|
550
|
+
try:
|
|
551
|
+
parsed = json.loads(prefix.decode("utf-8", errors="replace"))
|
|
552
|
+
except (ValueError, UnicodeError, RecursionError):
|
|
553
|
+
pass
|
|
554
|
+
detail = parsed.get("detail", parsed) if isinstance(parsed, dict) else None
|
|
555
|
+
if isinstance(detail, str):
|
|
556
|
+
decoded = None
|
|
557
|
+
try:
|
|
558
|
+
decoded = json.loads(detail)
|
|
559
|
+
except (ValueError, RecursionError):
|
|
560
|
+
pass
|
|
561
|
+
detail = decoded
|
|
562
|
+
code = detail.get("code") if isinstance(detail, dict) else None
|
|
563
|
+
if type(code) is bool or not isinstance(code, int):
|
|
564
|
+
code = None
|
|
565
|
+
return code
|
|
566
|
+
|
|
567
|
+
|
|
309
568
|
def _ensure_next_token_honored(response: dict, *, next_token: Optional[str], endpoint: str) -> None:
|
|
310
569
|
# An old server ignores the unknown next_token param and answers with its
|
|
311
570
|
# default first page, so a `response.get("next_token")` loop would silently
|
|
@@ -441,8 +700,17 @@ class APIClient:
|
|
|
441
700
|
timeout: TimeoutType = None,
|
|
442
701
|
deadline: Optional[float] = None,
|
|
443
702
|
_return_response: bool = False,
|
|
703
|
+
_single_attempt: bool = False,
|
|
704
|
+
_bounded: bool = False,
|
|
444
705
|
) -> Any:
|
|
445
|
-
"""Send
|
|
706
|
+
"""Send through the normal authenticated HTTP channel.
|
|
707
|
+
|
|
708
|
+
``_bounded`` (canonical adapter control requests) opts this single
|
|
709
|
+
request into the bounded canonical response projection: redirects
|
|
710
|
+
refused (auth headers never follow a cross-origin Location), a
|
|
711
|
+
finite timeout owns ONE deadline across retries, streamed bounded
|
|
712
|
+
reads (O(cap+chunk) memory), and late successes refused.
|
|
713
|
+
"""
|
|
446
714
|
self._require_cluster_config()
|
|
447
715
|
return self._request_http(
|
|
448
716
|
method,
|
|
@@ -452,6 +720,8 @@ class APIClient:
|
|
|
452
720
|
timeout=timeout,
|
|
453
721
|
deadline=deadline,
|
|
454
722
|
_return_response=_return_response,
|
|
723
|
+
_single_attempt=_single_attempt,
|
|
724
|
+
_bounded=_bounded,
|
|
455
725
|
)
|
|
456
726
|
|
|
457
727
|
def _request_http(
|
|
@@ -464,6 +734,8 @@ class APIClient:
|
|
|
464
734
|
timeout: TimeoutType = None,
|
|
465
735
|
deadline: Optional[float] = None,
|
|
466
736
|
_return_response: bool = False,
|
|
737
|
+
_single_attempt: bool = False,
|
|
738
|
+
_bounded: bool = False,
|
|
467
739
|
) -> Any:
|
|
468
740
|
"""One retry budget shared by cluster and central requests.
|
|
469
741
|
|
|
@@ -474,8 +746,14 @@ class APIClient:
|
|
|
474
746
|
timeout = _default_timeout()
|
|
475
747
|
else:
|
|
476
748
|
timeout = _normalize_explicit_timeout(timeout)
|
|
749
|
+
# Bounded canonical projection (P1 fix): a finite per-request duration
|
|
750
|
+
# owns ONE deadline across this request's retries, never per attempt,
|
|
751
|
+
# so a slow byte-drip response cannot outlive its budget.
|
|
752
|
+
if _bounded and deadline is None and isinstance(timeout, (int, float)) \
|
|
753
|
+
and not isinstance(timeout, bool) and math.isfinite(timeout) and timeout >= 0:
|
|
754
|
+
deadline = time.monotonic() + timeout
|
|
477
755
|
verbose = self.config.verbose
|
|
478
|
-
max_attempts = max(1, self.config.http_max_attempts)
|
|
756
|
+
max_attempts = 1 if _single_attempt else max(1, self.config.http_max_attempts)
|
|
479
757
|
max_wait = self.config.http_max_wait
|
|
480
758
|
read_only = method.upper() in _RETRYABLE_METHODS
|
|
481
759
|
recovery_deadline = None
|
|
@@ -506,16 +784,66 @@ class APIClient:
|
|
|
506
784
|
max_attempts=max_attempts,
|
|
507
785
|
)
|
|
508
786
|
start = time.perf_counter()
|
|
787
|
+
resp = None
|
|
509
788
|
try:
|
|
789
|
+
# A single-attempt mutation must also stop on redirects.
|
|
790
|
+
# Read the body separately so a truncated error still keeps its HTTP status.
|
|
791
|
+
# Bounded canonical requests always refuse redirects (auth
|
|
792
|
+
# headers must never follow a cross-origin Location) and read
|
|
793
|
+
# the body streamed-bounded instead of unbounded .content.
|
|
794
|
+
if _single_attempt or _bounded:
|
|
795
|
+
options = {"allow_redirects": False, "stream": True}
|
|
796
|
+
else:
|
|
797
|
+
options = {}
|
|
798
|
+
if _bounded:
|
|
799
|
+
# Watchdog pool isolation (review P2, final round): the
|
|
800
|
+
# deadline watchdog interrupts a blocked read by shutting
|
|
801
|
+
# the response's socket down. A keep-alive connection is
|
|
802
|
+
# returned to the pool as soon as iter_content drains to
|
|
803
|
+
# EOF — BEFORE the watchdog context exits — so any
|
|
804
|
+
# "check ownership then shutdown" sequence retains a
|
|
805
|
+
# check-then-act window in which the connection is
|
|
806
|
+
# borrowed by the NEXT request and killed by the stale
|
|
807
|
+
# fire. Closing the connection for this one request
|
|
808
|
+
# (``Connection: close``) removes the pooled connection
|
|
809
|
+
# from the equation entirely: the socket is never
|
|
810
|
+
# released back to the pool, a late fire hits an
|
|
811
|
+
# already-closed socket (OSError, swallowed), and later
|
|
812
|
+
# requests always get a fresh connection. Bounded control
|
|
813
|
+
# requests are short-lived metadata calls; the extra
|
|
814
|
+
# connection setup stays inside the same finite budget.
|
|
815
|
+
request_headers = {**headers, "Connection": "close"}
|
|
816
|
+
else:
|
|
817
|
+
request_headers = headers
|
|
510
818
|
resp = self.session.request(
|
|
511
819
|
method,
|
|
512
820
|
url,
|
|
513
821
|
params=params,
|
|
514
822
|
json=json_body,
|
|
515
|
-
headers=
|
|
823
|
+
headers=request_headers,
|
|
516
824
|
timeout=attempt_timeout,
|
|
825
|
+
**options,
|
|
517
826
|
)
|
|
827
|
+
if _single_attempt and not _bounded:
|
|
828
|
+
# Legacy single-attempt semantics: read the body so a
|
|
829
|
+
# truncated error keeps its status. The bounded projection
|
|
830
|
+
# reads the body itself (streamed, bounded).
|
|
831
|
+
_ = resp.content
|
|
518
832
|
except requests.RequestException as exc:
|
|
833
|
+
if _bounded:
|
|
834
|
+
# Bounded canonical projection: a network failure during a
|
|
835
|
+
# streamed bounded read is a typed refusal, not a retry of
|
|
836
|
+
# an unbounded request.
|
|
837
|
+
resp_is_error = resp is not None and not 200 <= resp.status_code < 300
|
|
838
|
+
if resp_is_error:
|
|
839
|
+
resp.close()
|
|
840
|
+
reason = "deadline" if (deadline is not None and time.monotonic() >= deadline) else "network"
|
|
841
|
+
raise CanonicalControlAPIError(reason) from None
|
|
842
|
+
if _single_attempt and resp is not None:
|
|
843
|
+
status = resp.status_code
|
|
844
|
+
resp.close()
|
|
845
|
+
if not 200 <= status < 300:
|
|
846
|
+
raise APIError(status, "Incomplete HTTP error response", request_id, resp) from None
|
|
519
847
|
if deadline is not None:
|
|
520
848
|
_manifest_budget_remaining(deadline)
|
|
521
849
|
duration_ms = (time.perf_counter() - start) * 1000
|
|
@@ -546,7 +874,74 @@ class APIClient:
|
|
|
546
874
|
duration_ms = (time.perf_counter() - start) * 1000
|
|
547
875
|
if verbose:
|
|
548
876
|
self._log_response(method, url, request_id, resp, duration_ms)
|
|
549
|
-
if
|
|
877
|
+
if _bounded:
|
|
878
|
+
# Bounded canonical projection: sanitize every exit — refusals
|
|
879
|
+
# never carry a cause; redirects are refused before any auth
|
|
880
|
+
# header could follow the Location; the body is read streamed
|
|
881
|
+
# and bounded, and a late success (budget exhausted mid-read)
|
|
882
|
+
# is refused.
|
|
883
|
+
try:
|
|
884
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
885
|
+
raise CanonicalControlAPIError("deadline")
|
|
886
|
+
if _return_response:
|
|
887
|
+
raise CanonicalControlAPIError("invalid_request")
|
|
888
|
+
if 300 <= resp.status_code < 400:
|
|
889
|
+
# Request had allow_redirects=False: the redirect was
|
|
890
|
+
# NOT followed — the Location was never requested and
|
|
891
|
+
# auth headers were never sent there. Refuse.
|
|
892
|
+
raise CanonicalControlAPIError("redirect", status_code=resp.status_code)
|
|
893
|
+
if not 200 <= resp.status_code < 300:
|
|
894
|
+
if read_only and attempt < max_attempts and _is_retryable_status(resp.status_code):
|
|
895
|
+
delay = _retry_delay(attempt, resp, max_wait=max_wait)
|
|
896
|
+
if not math.isfinite(delay):
|
|
897
|
+
delay = _retry_delay(attempt)
|
|
898
|
+
if deadline is not None:
|
|
899
|
+
remaining = deadline - time.monotonic()
|
|
900
|
+
if remaining <= 0:
|
|
901
|
+
raise CanonicalControlAPIError("deadline")
|
|
902
|
+
delay = min(delay, remaining)
|
|
903
|
+
resp.close()
|
|
904
|
+
if deadline is not None:
|
|
905
|
+
_deadline_sleep(delay, deadline)
|
|
906
|
+
else:
|
|
907
|
+
time.sleep(delay)
|
|
908
|
+
continue
|
|
909
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
910
|
+
# The error body would be classified only after a
|
|
911
|
+
# bounded read that the budget can no longer fund:
|
|
912
|
+
# report the deadline, never a late http_error.
|
|
913
|
+
raise CanonicalControlAPIError("deadline")
|
|
914
|
+
prefix = _bounded_response_prefix(
|
|
915
|
+
resp, _DEFAULT_CONTROL_BYTES, deadline=deadline)
|
|
916
|
+
code = _bounded_error_biz_code(prefix)
|
|
917
|
+
raise CanonicalControlAPIError(
|
|
918
|
+
"http_error", status_code=resp.status_code, biz_code=code)
|
|
919
|
+
data = _read_bounded_control_bytes(
|
|
920
|
+
resp, _DEFAULT_CONTROL_BYTES, deadline=deadline)
|
|
921
|
+
if not data:
|
|
922
|
+
resp.close()
|
|
923
|
+
return None
|
|
924
|
+
refusal = None
|
|
925
|
+
try:
|
|
926
|
+
parsed = data.decode("utf-8", errors="strict")
|
|
927
|
+
except UnicodeError:
|
|
928
|
+
refusal = "invalid_utf8"
|
|
929
|
+
if refusal is None:
|
|
930
|
+
try:
|
|
931
|
+
result = json.loads(parsed)
|
|
932
|
+
except (ValueError, RecursionError):
|
|
933
|
+
refusal = "invalid_json"
|
|
934
|
+
if refusal is not None:
|
|
935
|
+
raise CanonicalControlAPIError(
|
|
936
|
+
refusal, status_code=resp.status_code)
|
|
937
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
938
|
+
# Refuse a late success; the budget is already spent.
|
|
939
|
+
raise CanonicalControlAPIError("deadline")
|
|
940
|
+
resp.close()
|
|
941
|
+
return result
|
|
942
|
+
finally:
|
|
943
|
+
_close_response_quietly(resp)
|
|
944
|
+
if not resp.ok or (_single_attempt and not 200 <= resp.status_code < 300):
|
|
550
945
|
pop_rejected = _is_pop_rejection(resp)
|
|
551
946
|
if pop_rejected and recovery_deadline is None:
|
|
552
947
|
recovery_deadline = time.monotonic() + self.config.http_retry_window
|
|
@@ -611,6 +1006,11 @@ class APIClient:
|
|
|
611
1006
|
raise APIError(resp.status_code, detail, request_id, resp) from exc
|
|
612
1007
|
raise RuntimeError("unreachable retry state")
|
|
613
1008
|
|
|
1009
|
+
def _max_attempts_for(self, method: str) -> int:
|
|
1010
|
+
if method.upper() not in _RETRYABLE_METHODS:
|
|
1011
|
+
return 1
|
|
1012
|
+
return max(1, self.config.http_max_attempts)
|
|
1013
|
+
|
|
614
1014
|
def _require_cluster_config(self) -> None:
|
|
615
1015
|
if has_cluster_header(self.config.headers):
|
|
616
1016
|
return
|
|
@@ -1220,6 +1620,8 @@ class APIClient:
|
|
|
1220
1620
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1221
1621
|
idempotency_key: Optional[str] = None,
|
|
1222
1622
|
group_concurrency_policy: Optional[dict] = None,
|
|
1623
|
+
visibility: Optional[Literal["queue", "private"]] = None,
|
|
1624
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1223
1625
|
) -> dict:
|
|
1224
1626
|
"""Create a group with optional scheduling defaults for its Jobs.
|
|
1225
1627
|
|
|
@@ -1227,6 +1629,8 @@ class APIClient:
|
|
|
1227
1629
|
"""
|
|
1228
1630
|
if priority is not None and priority not in {"high", "medium"}:
|
|
1229
1631
|
raise ValueError("priority must be one of high/medium")
|
|
1632
|
+
if visibility is not None and visibility not in {"queue", "private"}:
|
|
1633
|
+
raise ValueError("visibility must be queue or private")
|
|
1230
1634
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
1231
1635
|
raise ValueError("transparency must be whitebox or blackbox")
|
|
1232
1636
|
if transparency == "blackbox" and not idempotency_key:
|
|
@@ -1239,6 +1643,10 @@ class APIClient:
|
|
|
1239
1643
|
raise ValueError("ak_ids entries must be nonempty strings")
|
|
1240
1644
|
references = list(dict.fromkeys(references))
|
|
1241
1645
|
body: dict = {}
|
|
1646
|
+
if visibility is not None:
|
|
1647
|
+
body["visibility"] = visibility
|
|
1648
|
+
if metrics_retry_policy is not None:
|
|
1649
|
+
body["metrics_retry_policy"] = metrics_retry_policy
|
|
1242
1650
|
if transparency is not None:
|
|
1243
1651
|
body["transparency"] = transparency
|
|
1244
1652
|
if idempotency_key is not None:
|
|
@@ -1367,10 +1775,13 @@ class APIClient:
|
|
|
1367
1775
|
instance_range: Optional[str] = None,
|
|
1368
1776
|
profile: Optional[str] = None,
|
|
1369
1777
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1778
|
+
visibility: Optional[Literal["queue", "private"]] = None,
|
|
1779
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1370
1780
|
) -> dict:
|
|
1371
1781
|
"""Create a job."""
|
|
1372
1782
|
body = self.build_create_job_body(
|
|
1373
1783
|
transparency=transparency,
|
|
1784
|
+
visibility=visibility,
|
|
1374
1785
|
template=template,
|
|
1375
1786
|
params=params,
|
|
1376
1787
|
params_list=params_list,
|
|
@@ -1391,6 +1802,7 @@ class APIClient:
|
|
|
1391
1802
|
runner_options=runner_options,
|
|
1392
1803
|
idempotency_key=idempotency_key,
|
|
1393
1804
|
max_failure_retries=max_failure_retries,
|
|
1805
|
+
metrics_retry_policy=metrics_retry_policy,
|
|
1394
1806
|
priority=priority,
|
|
1395
1807
|
priority_weight=priority_weight,
|
|
1396
1808
|
resource_mode=resource_mode,
|
|
@@ -1483,8 +1895,12 @@ class APIClient:
|
|
|
1483
1895
|
instance_range: Optional[str] = None,
|
|
1484
1896
|
profile: Optional[str] = None,
|
|
1485
1897
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1898
|
+
visibility: Optional[Literal["queue", "private"]] = None,
|
|
1899
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1486
1900
|
) -> dict:
|
|
1487
1901
|
"""Build the /jobs request body for job creation."""
|
|
1902
|
+
if visibility is not None and visibility not in {"queue", "private"}:
|
|
1903
|
+
raise ValueError("visibility must be queue or private")
|
|
1488
1904
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
1489
1905
|
raise ValueError("transparency must be whitebox or blackbox")
|
|
1490
1906
|
if profile is not None and (profile_id is not None or profile_version is not None):
|
|
@@ -1554,6 +1970,8 @@ class APIClient:
|
|
|
1554
1970
|
body["runner_options"] = runner_options
|
|
1555
1971
|
if idempotency_key is not None:
|
|
1556
1972
|
body["idempotency_key"] = idempotency_key
|
|
1973
|
+
if metrics_retry_policy is not None:
|
|
1974
|
+
body["metrics_retry_policy"] = metrics_retry_policy
|
|
1557
1975
|
if max_failure_retries:
|
|
1558
1976
|
body["max_failure_retries"] = max_failure_retries
|
|
1559
1977
|
|
|
@@ -1611,6 +2029,8 @@ class APIClient:
|
|
|
1611
2029
|
if experiment is not None:
|
|
1612
2030
|
body["experiment"] = experiment
|
|
1613
2031
|
|
|
2032
|
+
if visibility is not None:
|
|
2033
|
+
body["visibility"] = visibility
|
|
1614
2034
|
if transparency is not None:
|
|
1615
2035
|
body["transparency"] = transparency
|
|
1616
2036
|
return body
|
|
@@ -1679,7 +2099,7 @@ class APIClient:
|
|
|
1679
2099
|
upstream_job_id: Optional[str] = None,
|
|
1680
2100
|
next_token: Optional[str] = None,
|
|
1681
2101
|
pagination: Optional[str] = None,
|
|
1682
|
-
include_total: bool =
|
|
2102
|
+
include_total: bool = False,
|
|
1683
2103
|
ak_id: Optional[str] = None,
|
|
1684
2104
|
) -> dict:
|
|
1685
2105
|
"""List jobs."""
|
|
@@ -1745,8 +2165,7 @@ class APIClient:
|
|
|
1745
2165
|
params["next_token"] = next_token
|
|
1746
2166
|
if pagination:
|
|
1747
2167
|
params["pagination"] = pagination
|
|
1748
|
-
|
|
1749
|
-
params["include_total"] = False
|
|
2168
|
+
params["include_total"] = include_total
|
|
1750
2169
|
response = self._get("/jobs", params=params)
|
|
1751
2170
|
_ensure_next_token_honored(response, next_token=next_token, endpoint="/jobs")
|
|
1752
2171
|
return response
|
|
@@ -1928,10 +2347,11 @@ class APIClient:
|
|
|
1928
2347
|
upstream_platform: Optional[str] = None,
|
|
1929
2348
|
upstream_job_id: Optional[str] = None,
|
|
1930
2349
|
ak_id: Optional[str] = None,
|
|
2350
|
+
include_total: bool = False,
|
|
1931
2351
|
) -> dict:
|
|
1932
|
-
"""List groups."""
|
|
2352
|
+
"""List groups. Exact totals are opt-in and subject to the server query budget."""
|
|
1933
2353
|
validate_ak_id(ak_id)
|
|
1934
|
-
params: dict = {"skip": skip, "limit": limit}
|
|
2354
|
+
params: dict = {"skip": skip, "limit": limit, "include_total": include_total}
|
|
1935
2355
|
if ak_id is not None:
|
|
1936
2356
|
params["ak_id"] = ak_id
|
|
1937
2357
|
if name is not None:
|
|
@@ -1993,7 +2413,7 @@ class APIClient:
|
|
|
1993
2413
|
upstream_job_id: Optional[str] = None,
|
|
1994
2414
|
next_token: Optional[str] = None,
|
|
1995
2415
|
pagination: Optional[str] = None,
|
|
1996
|
-
include_total: bool =
|
|
2416
|
+
include_total: bool = False,
|
|
1997
2417
|
) -> dict:
|
|
1998
2418
|
"""List jobs in a group."""
|
|
1999
2419
|
params: dict = {"skip": skip, "limit": limit}
|
|
@@ -2021,8 +2441,7 @@ class APIClient:
|
|
|
2021
2441
|
params["next_token"] = next_token
|
|
2022
2442
|
if pagination:
|
|
2023
2443
|
params["pagination"] = pagination
|
|
2024
|
-
|
|
2025
|
-
params["include_total"] = False
|
|
2444
|
+
params["include_total"] = include_total
|
|
2026
2445
|
endpoint = f"/groups/{quote(group_id)}/jobs"
|
|
2027
2446
|
response = self._get(endpoint, params=params)
|
|
2028
2447
|
_ensure_next_token_honored(response, next_token=next_token, endpoint=endpoint)
|
|
@@ -3654,12 +4073,34 @@ def _redact_url_query_secrets(value: str) -> str:
|
|
|
3654
4073
|
|
|
3655
4074
|
|
|
3656
4075
|
def _response_error_detail(resp: requests.Response) -> Any:
|
|
4076
|
+
"""错误响应体 → detail(兼容两代服务端错误形态,G2 修复)。
|
|
4077
|
+
|
|
4078
|
+
* FastAPI ``{"detail": <str|dict>}``:保持旧语义——detail 键优先
|
|
4079
|
+
(dict 时**整个内层 dict** 作为 detail 返回,不只是取 message);
|
|
4080
|
+
* Go/Gin 裸信封 ``{code, message, data}``(apiserver
|
|
4081
|
+
``pkg/api/response/response.go writeErrorData``):**整个响应 dict**
|
|
4082
|
+
作为 detail 返回——9404x 数字 code 与 ``data``(error_code/
|
|
4083
|
+
apply_url 等结构化负载)不再被降级成 message 字符串丢失;
|
|
4084
|
+
* 旧 ``{"message": ...}``(无 detail)形态:保持旧语义取 message
|
|
4085
|
+
字符串(detail/message 缺省回退 resp.text)。
|
|
4086
|
+
|
|
4087
|
+
detail 消费方(如 shared_adapter._decode_api_error)按"dict 即结构化
|
|
4088
|
+
错误体"自行解 code/data;str detail 继续支持(旧展示路径不变)。
|
|
4089
|
+
"""
|
|
3657
4090
|
try:
|
|
3658
4091
|
err_data = resp.json()
|
|
3659
4092
|
except Exception:
|
|
3660
4093
|
return resp.text
|
|
3661
4094
|
if isinstance(err_data, dict):
|
|
3662
|
-
|
|
4095
|
+
if "detail" in err_data:
|
|
4096
|
+
inner = err_data["detail"]
|
|
4097
|
+
if isinstance(inner, dict):
|
|
4098
|
+
return inner # FastAPI detail=dict:内层即结构化错误体
|
|
4099
|
+
return inner # detail=str:旧语义
|
|
4100
|
+
if "code" in err_data:
|
|
4101
|
+
# Go 裸信封 {code, message, data}:保留完整结构化错误体
|
|
4102
|
+
return err_data
|
|
4103
|
+
return err_data.get("message", resp.text)
|
|
3663
4104
|
return err_data
|
|
3664
4105
|
|
|
3665
4106
|
|