ap-client 0.7.1.dev7__py3-none-any.whl → 0.7.2.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ap_client/api.py +44 -6
- ap_client/canonical_put.py +65 -17
- ap_client/cli.py +86 -50
- ap_client/dataset_commands.py +41 -344
- ap_client/dfs_commands.py +8 -3
- ap_client/fs_commands.py +51 -1
- ap_client/instance_commands.py +18 -32
- ap_client/irepo_sdk.py +0 -57
- ap_client/ossutil_io.py +17 -6
- ap_client/profile_commands.py +3 -0
- ap_client/profile_diagnostics.py +2 -1
- ap_client/profile_references.py +3 -2
- ap_client/repo_read_guard.py +3 -14
- ap_client/shared_adapter.py +3 -1
- ap_client/shared_commands.py +31 -5
- {ap_client-0.7.1.dev7.dist-info → ap_client-0.7.2.dev1.dist-info}/METADATA +4 -4
- {ap_client-0.7.1.dev7.dist-info → ap_client-0.7.2.dev1.dist-info}/RECORD +19 -19
- {ap_client-0.7.1.dev7.dist-info → ap_client-0.7.2.dev1.dist-info}/WHEEL +0 -0
- {ap_client-0.7.1.dev7.dist-info → ap_client-0.7.2.dev1.dist-info}/entry_points.txt +0 -0
ap_client/api.py
CHANGED
|
@@ -268,6 +268,9 @@ class APIError(Exception):
|
|
|
268
268
|
self.request_id = request_id
|
|
269
269
|
self.response = response
|
|
270
270
|
self.retry_stop_reason = retry_stop_reason
|
|
271
|
+
self.response_request_id = (
|
|
272
|
+
response.headers.get("X-Request-ID") if response is not None else None
|
|
273
|
+
)
|
|
271
274
|
self.pop_request_id = (
|
|
272
275
|
response.headers.get("x-acs-request-id") if response is not None else None
|
|
273
276
|
)
|
|
@@ -275,10 +278,8 @@ class APIError(Exception):
|
|
|
275
278
|
|
|
276
279
|
def __str__(self) -> str:
|
|
277
280
|
message = self._build_message()
|
|
278
|
-
|
|
279
|
-
message += f" (
|
|
280
|
-
if self.pop_request_id:
|
|
281
|
-
message += f" (pop_request_id={self.pop_request_id})"
|
|
281
|
+
for diagnostic in format_api_error_diagnostics(self):
|
|
282
|
+
message += f" ({diagnostic})"
|
|
282
283
|
return message
|
|
283
284
|
|
|
284
285
|
def _build_message(self) -> str:
|
|
@@ -287,6 +288,20 @@ class APIError(Exception):
|
|
|
287
288
|
return f"API Error {self.status_code}: {self.detail} (request_id={self.request_id})"
|
|
288
289
|
|
|
289
290
|
|
|
291
|
+
def format_api_error_diagnostics(exc: APIError, *, separator: str = "=") -> list[str]:
|
|
292
|
+
"""Render response and retry diagnostics without repeating the local request ID."""
|
|
293
|
+
diagnostics = []
|
|
294
|
+
seen = {exc.request_id}
|
|
295
|
+
for name in ("response_request_id", "pop_request_id"):
|
|
296
|
+
value = getattr(exc, name)
|
|
297
|
+
if value and value not in seen:
|
|
298
|
+
seen.add(value)
|
|
299
|
+
diagnostics.append(f"{name}{separator}{value}")
|
|
300
|
+
if exc.retry_stop_reason:
|
|
301
|
+
diagnostics.append(f"retry_stop_reason{separator}{exc.retry_stop_reason}")
|
|
302
|
+
return diagnostics
|
|
303
|
+
|
|
304
|
+
|
|
290
305
|
class PaginationProtocolError(RuntimeError):
|
|
291
306
|
"""The server stopped honoring a pagination protocol already in use."""
|
|
292
307
|
|
|
@@ -426,6 +441,7 @@ class APIClient:
|
|
|
426
441
|
timeout: TimeoutType = None,
|
|
427
442
|
deadline: Optional[float] = None,
|
|
428
443
|
_return_response: bool = False,
|
|
444
|
+
_single_attempt: bool = False,
|
|
429
445
|
) -> Any:
|
|
430
446
|
"""Send through the normal authenticated HTTP channel."""
|
|
431
447
|
self._require_cluster_config()
|
|
@@ -437,6 +453,7 @@ class APIClient:
|
|
|
437
453
|
timeout=timeout,
|
|
438
454
|
deadline=deadline,
|
|
439
455
|
_return_response=_return_response,
|
|
456
|
+
_single_attempt=_single_attempt,
|
|
440
457
|
)
|
|
441
458
|
|
|
442
459
|
def _request_http(
|
|
@@ -449,6 +466,7 @@ class APIClient:
|
|
|
449
466
|
timeout: TimeoutType = None,
|
|
450
467
|
deadline: Optional[float] = None,
|
|
451
468
|
_return_response: bool = False,
|
|
469
|
+
_single_attempt: bool = False,
|
|
452
470
|
) -> Any:
|
|
453
471
|
"""One retry budget shared by cluster and central requests.
|
|
454
472
|
|
|
@@ -460,7 +478,7 @@ class APIClient:
|
|
|
460
478
|
else:
|
|
461
479
|
timeout = _normalize_explicit_timeout(timeout)
|
|
462
480
|
verbose = self.config.verbose
|
|
463
|
-
max_attempts = max(1, self.config.http_max_attempts)
|
|
481
|
+
max_attempts = 1 if _single_attempt else max(1, self.config.http_max_attempts)
|
|
464
482
|
max_wait = self.config.http_max_wait
|
|
465
483
|
read_only = method.upper() in _RETRYABLE_METHODS
|
|
466
484
|
recovery_deadline = None
|
|
@@ -491,7 +509,11 @@ class APIClient:
|
|
|
491
509
|
max_attempts=max_attempts,
|
|
492
510
|
)
|
|
493
511
|
start = time.perf_counter()
|
|
512
|
+
resp = None
|
|
494
513
|
try:
|
|
514
|
+
# A single-attempt mutation must also stop on redirects.
|
|
515
|
+
# Read the body separately so a truncated error still keeps its HTTP status.
|
|
516
|
+
options = {"allow_redirects": False, "stream": True} if _single_attempt else {}
|
|
495
517
|
resp = self.session.request(
|
|
496
518
|
method,
|
|
497
519
|
url,
|
|
@@ -499,8 +521,16 @@ class APIClient:
|
|
|
499
521
|
json=json_body,
|
|
500
522
|
headers=headers,
|
|
501
523
|
timeout=attempt_timeout,
|
|
524
|
+
**options,
|
|
502
525
|
)
|
|
526
|
+
if _single_attempt:
|
|
527
|
+
_ = resp.content
|
|
503
528
|
except requests.RequestException as exc:
|
|
529
|
+
if _single_attempt and resp is not None:
|
|
530
|
+
status = resp.status_code
|
|
531
|
+
resp.close()
|
|
532
|
+
if not 200 <= status < 300:
|
|
533
|
+
raise APIError(status, "Incomplete HTTP error response", request_id, resp) from None
|
|
504
534
|
if deadline is not None:
|
|
505
535
|
_manifest_budget_remaining(deadline)
|
|
506
536
|
duration_ms = (time.perf_counter() - start) * 1000
|
|
@@ -531,7 +561,7 @@ class APIClient:
|
|
|
531
561
|
duration_ms = (time.perf_counter() - start) * 1000
|
|
532
562
|
if verbose:
|
|
533
563
|
self._log_response(method, url, request_id, resp, duration_ms)
|
|
534
|
-
if not resp.ok:
|
|
564
|
+
if not resp.ok or (_single_attempt and not 200 <= resp.status_code < 300):
|
|
535
565
|
pop_rejected = _is_pop_rejection(resp)
|
|
536
566
|
if pop_rejected and recovery_deadline is None:
|
|
537
567
|
recovery_deadline = time.monotonic() + self.config.http_retry_window
|
|
@@ -1210,6 +1240,7 @@ class APIClient:
|
|
|
1210
1240
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1211
1241
|
idempotency_key: Optional[str] = None,
|
|
1212
1242
|
group_concurrency_policy: Optional[dict] = None,
|
|
1243
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1213
1244
|
) -> dict:
|
|
1214
1245
|
"""Create a group with optional scheduling defaults for its Jobs.
|
|
1215
1246
|
|
|
@@ -1229,6 +1260,8 @@ class APIClient:
|
|
|
1229
1260
|
raise ValueError("ak_ids entries must be nonempty strings")
|
|
1230
1261
|
references = list(dict.fromkeys(references))
|
|
1231
1262
|
body: dict = {}
|
|
1263
|
+
if metrics_retry_policy is not None:
|
|
1264
|
+
body["metrics_retry_policy"] = metrics_retry_policy
|
|
1232
1265
|
if transparency is not None:
|
|
1233
1266
|
body["transparency"] = transparency
|
|
1234
1267
|
if idempotency_key is not None:
|
|
@@ -1357,6 +1390,7 @@ class APIClient:
|
|
|
1357
1390
|
instance_range: Optional[str] = None,
|
|
1358
1391
|
profile: Optional[str] = None,
|
|
1359
1392
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1393
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1360
1394
|
) -> dict:
|
|
1361
1395
|
"""Create a job."""
|
|
1362
1396
|
body = self.build_create_job_body(
|
|
@@ -1381,6 +1415,7 @@ class APIClient:
|
|
|
1381
1415
|
runner_options=runner_options,
|
|
1382
1416
|
idempotency_key=idempotency_key,
|
|
1383
1417
|
max_failure_retries=max_failure_retries,
|
|
1418
|
+
metrics_retry_policy=metrics_retry_policy,
|
|
1384
1419
|
priority=priority,
|
|
1385
1420
|
priority_weight=priority_weight,
|
|
1386
1421
|
resource_mode=resource_mode,
|
|
@@ -1473,6 +1508,7 @@ class APIClient:
|
|
|
1473
1508
|
instance_range: Optional[str] = None,
|
|
1474
1509
|
profile: Optional[str] = None,
|
|
1475
1510
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1511
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1476
1512
|
) -> dict:
|
|
1477
1513
|
"""Build the /jobs request body for job creation."""
|
|
1478
1514
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
@@ -1544,6 +1580,8 @@ class APIClient:
|
|
|
1544
1580
|
body["runner_options"] = runner_options
|
|
1545
1581
|
if idempotency_key is not None:
|
|
1546
1582
|
body["idempotency_key"] = idempotency_key
|
|
1583
|
+
if metrics_retry_policy is not None:
|
|
1584
|
+
body["metrics_retry_policy"] = metrics_retry_policy
|
|
1547
1585
|
if max_failure_retries:
|
|
1548
1586
|
body["max_failure_retries"] = max_failure_retries
|
|
1549
1587
|
|
ap_client/canonical_put.py
CHANGED
|
@@ -160,8 +160,11 @@ class APIClientTransportAdapter:
|
|
|
160
160
|
return TransportError(label, status=status, biz_code=biz_code, data=data)
|
|
161
161
|
|
|
162
162
|
def _call(self, method: str, path: str, *, json_body=None,
|
|
163
|
-
timeout: float | None = None, params=None
|
|
164
|
-
|
|
163
|
+
timeout: float | None = None, params=None,
|
|
164
|
+
deadline: float | None = None, single_attempt: bool = False):
|
|
165
|
+
options = dict(timeout=timeout, deadline=self._base_deadline if deadline is None else deadline)
|
|
166
|
+
if single_attempt:
|
|
167
|
+
options["_single_attempt"] = True
|
|
165
168
|
if json_body is not None:
|
|
166
169
|
options['json_body'] = json_body
|
|
167
170
|
if params is not None:
|
|
@@ -181,27 +184,72 @@ class APIClientTransportAdapter:
|
|
|
181
184
|
|
|
182
185
|
# -- SDK Transport surface ---------------------------------------------
|
|
183
186
|
|
|
187
|
+
@staticmethod
|
|
188
|
+
def _split_query(path: str):
|
|
189
|
+
base, _, query = path.partition("?")
|
|
190
|
+
pairs = parse_qsl(query, keep_blank_values=True)
|
|
191
|
+
params = dict(pairs)
|
|
192
|
+
if len(params) != len(pairs):
|
|
193
|
+
from instance_repo.errors import SchemaError
|
|
194
|
+
|
|
195
|
+
raise SchemaError("duplicate query keys in request path")
|
|
196
|
+
return base, params or None
|
|
197
|
+
|
|
184
198
|
def get(self, path: str, *, timeout: float | None = None) -> dict:
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
pairs = parse_qsl(query, keep_blank_values=True)
|
|
188
|
-
keys = [k for k, _ in pairs]
|
|
189
|
-
if len(keys) != len(set(keys)):
|
|
190
|
-
from instance_repo.errors import SchemaError
|
|
191
|
-
|
|
192
|
-
raise SchemaError("duplicate query keys in request path")
|
|
193
|
-
params = dict(pairs)
|
|
194
|
-
return self._call("GET", base, timeout=timeout) if not params \
|
|
195
|
-
else self._request_get_with_params(base, params, timeout)
|
|
196
|
-
return self._call("GET", path, timeout=timeout)
|
|
197
|
-
|
|
198
|
-
def _request_get_with_params(self, path, params, timeout):
|
|
199
|
-
return self._call('GET', path, params=params, timeout=timeout)
|
|
199
|
+
base, params = self._split_query(path)
|
|
200
|
+
return self._call("GET", base, params=params, timeout=timeout)
|
|
200
201
|
|
|
201
202
|
def post(self, path: str, body: dict, *, idempotent: bool = False,
|
|
202
203
|
headers: dict | None = None, timeout: float | None = None) -> dict:
|
|
203
204
|
return self._call("POST", path, json_body=body, timeout=timeout)
|
|
204
205
|
|
|
206
|
+
def _acl_response(self, method, path, body, timeout):
|
|
207
|
+
from instance_repo.errors import RequestDeadlineExceeded, SchemaError, TransportError
|
|
208
|
+
|
|
209
|
+
deadline = self._base_deadline
|
|
210
|
+
if timeout is not None:
|
|
211
|
+
if type(timeout) not in (int, float) or not math.isfinite(timeout) or timeout < 0:
|
|
212
|
+
raise SchemaError("invalid ACL request timeout")
|
|
213
|
+
call_deadline = time.monotonic() + timeout
|
|
214
|
+
deadline = min(deadline, call_deadline) if deadline is not None else call_deadline
|
|
215
|
+
if deadline is not None and time.monotonic() >= deadline:
|
|
216
|
+
raise RequestDeadlineExceeded("ACL request budget exhausted")
|
|
217
|
+
|
|
218
|
+
base, params = self._split_query(path)
|
|
219
|
+
response = self._call(method, base, json_body=body, params=params,
|
|
220
|
+
timeout=timeout, deadline=deadline, single_attempt=method != "GET")
|
|
221
|
+
if isinstance(response, dict) and set(response) <= {"code", "message", "data", "pagination"}:
|
|
222
|
+
if "code" in response:
|
|
223
|
+
code = response["code"]
|
|
224
|
+
if type(code) is not int:
|
|
225
|
+
raise SchemaError("invalid ACL response code")
|
|
226
|
+
if code != 0:
|
|
227
|
+
raise TransportError("ACL request rejected", status=200, biz_code=code)
|
|
228
|
+
if "data" in response:
|
|
229
|
+
return response["data"], response.get("pagination")
|
|
230
|
+
return response, None
|
|
231
|
+
|
|
232
|
+
def request(self, method, path, body=None, *, idempotent=None, timeout=None):
|
|
233
|
+
from instance_repo.errors import SchemaError
|
|
234
|
+
|
|
235
|
+
# Only the ABI used by the ACL SDK is supported. Never silently turn a
|
|
236
|
+
# non-idempotent write into a retryable call.
|
|
237
|
+
if not (
|
|
238
|
+
(method == "GET" and body is None and idempotent is True)
|
|
239
|
+
or (method == "POST" and type(body) is dict and idempotent is False)
|
|
240
|
+
or (method == "DELETE" and body is None and idempotent is False)
|
|
241
|
+
):
|
|
242
|
+
raise SchemaError("unsupported ACL request")
|
|
243
|
+
data, _ = self._acl_response(method, path, body, timeout)
|
|
244
|
+
return data
|
|
245
|
+
|
|
246
|
+
def request_with_meta(self, method, path, *, timeout=None):
|
|
247
|
+
from instance_repo.errors import SchemaError
|
|
248
|
+
|
|
249
|
+
if method != "GET":
|
|
250
|
+
raise SchemaError("unsupported ACL metadata request")
|
|
251
|
+
return self._acl_response(method, path, None, timeout)
|
|
252
|
+
|
|
205
253
|
|
|
206
254
|
# ---------------------------------------------------------------------------
|
|
207
255
|
# Budget: absolute internally, REMAINING DURATION for B public APIs
|
ap_client/cli.py
CHANGED
|
@@ -18,6 +18,7 @@ from ap_client.api import (
|
|
|
18
18
|
APIError,
|
|
19
19
|
PaginationProtocolError,
|
|
20
20
|
_merge_meta_tags,
|
|
21
|
+
format_api_error_diagnostics,
|
|
21
22
|
set_cluster_override,
|
|
22
23
|
set_verbose_override,
|
|
23
24
|
)
|
|
@@ -280,6 +281,25 @@ def _reject_account_pool_template_credential_cli(
|
|
|
280
281
|
raise typer.Exit(1)
|
|
281
282
|
|
|
282
283
|
|
|
284
|
+
def _parse_metrics_retry_policy(raw):
|
|
285
|
+
if not isinstance(raw, str):
|
|
286
|
+
return None
|
|
287
|
+
if raw.strip() != "null" and not raw.lstrip().startswith("{"):
|
|
288
|
+
try:
|
|
289
|
+
raw = Path(raw).expanduser().read_text(encoding="utf-8")
|
|
290
|
+
except OSError as exc:
|
|
291
|
+
raise typer.BadParameter(
|
|
292
|
+
"Cannot read metrics retry policy file", param_hint="--metrics-retry-policy"
|
|
293
|
+
) from exc
|
|
294
|
+
value = _parse_json_object(raw, "--metrics-retry-policy", allow_null=True)
|
|
295
|
+
if value is not None:
|
|
296
|
+
if "enable" not in value:
|
|
297
|
+
raise typer.BadParameter("metrics_retry_policy.enable is required (boolean)")
|
|
298
|
+
if type(value["enable"]) is not bool:
|
|
299
|
+
raise typer.BadParameter("metrics_retry_policy.enable must be a boolean")
|
|
300
|
+
return value
|
|
301
|
+
|
|
302
|
+
|
|
283
303
|
def _parse_json_object(
|
|
284
304
|
raw: Optional[str], option_name: str, *, allow_null: bool = False
|
|
285
305
|
) -> Optional[dict]:
|
|
@@ -1448,6 +1468,7 @@ def _idempotency_item_key(
|
|
|
1448
1468
|
max_failure_retries: int = 0,
|
|
1449
1469
|
ak_id: Optional[str] = None,
|
|
1450
1470
|
ak_mode: str = "auto",
|
|
1471
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1451
1472
|
) -> str:
|
|
1452
1473
|
# Client-supplied item keys are authoritative; server hashing is only a
|
|
1453
1474
|
# fallback for non-CLI callers, so this payload intentionally need not
|
|
@@ -1471,6 +1492,8 @@ def _idempotency_item_key(
|
|
|
1471
1492
|
payload_data["ak_mode"] = ak_mode
|
|
1472
1493
|
if resource_profile_id is not None:
|
|
1473
1494
|
payload_data["resource_profile_id"] = resource_profile_id
|
|
1495
|
+
if metrics_retry_policy is not None:
|
|
1496
|
+
payload_data["metrics_retry_policy"] = metrics_retry_policy
|
|
1474
1497
|
if max_failure_retries:
|
|
1475
1498
|
payload_data["max_failure_retries"] = max_failure_retries
|
|
1476
1499
|
payload = json.dumps(
|
|
@@ -3474,6 +3497,11 @@ def job_create(
|
|
|
3474
3497
|
"--idempotency-key",
|
|
3475
3498
|
help="UUID4 key for retrying the same logical submission; providing one enables idempotency",
|
|
3476
3499
|
),
|
|
3500
|
+
metrics_retry_policy: Optional[str] = typer.Option(
|
|
3501
|
+
None,
|
|
3502
|
+
"--metrics-retry-policy",
|
|
3503
|
+
help='Metrics retry policy JSON or file; null inherits, {"enable": false} disables',
|
|
3504
|
+
),
|
|
3477
3505
|
max_failure_retries: int = typer.Option(
|
|
3478
3506
|
0,
|
|
3479
3507
|
"--max-failure-retries",
|
|
@@ -3594,8 +3622,8 @@ def job_create(
|
|
|
3594
3622
|
):
|
|
3595
3623
|
"""Submit a job.
|
|
3596
3624
|
|
|
3597
|
-
For agents:
|
|
3598
3625
|
\b
|
|
3626
|
+
For agents:
|
|
3599
3627
|
Before submitting, you MUST ensure these five parameters are explicitly specified:
|
|
3600
3628
|
benchmark, meta-model, meta-job-type, meta-train-stage, meta-teacher-direction.
|
|
3601
3629
|
Keep specified values. Fill missing parameters using available Job, Group,
|
|
@@ -3770,6 +3798,13 @@ def job_create(
|
|
|
3770
3798
|
_emit_error(f"file not found: {model_base_url_file}")
|
|
3771
3799
|
raise typer.Exit(1)
|
|
3772
3800
|
|
|
3801
|
+
metrics_retry_policy_dict = _parse_metrics_retry_policy(metrics_retry_policy)
|
|
3802
|
+
metrics_retry_policy_kwargs = (
|
|
3803
|
+
{"metrics_retry_policy": metrics_retry_policy_dict}
|
|
3804
|
+
if metrics_retry_policy_dict is not None
|
|
3805
|
+
else {}
|
|
3806
|
+
)
|
|
3807
|
+
|
|
3773
3808
|
# Reject --trials when not in profile mode (profile-only override)
|
|
3774
3809
|
if trials is not None and profile is None:
|
|
3775
3810
|
_emit_error("--trials is only supported with --profile")
|
|
@@ -3838,6 +3873,7 @@ def job_create(
|
|
|
3838
3873
|
runner_options=runner_options_dict,
|
|
3839
3874
|
idempotency_key=submission_idempotency_key,
|
|
3840
3875
|
max_failure_retries=max_failure_retries,
|
|
3876
|
+
**metrics_retry_policy_kwargs,
|
|
3841
3877
|
group_id=group_id,
|
|
3842
3878
|
model_base_url_collection=model_base_url_collection_list,
|
|
3843
3879
|
group_post_process=group_post_process_dict,
|
|
@@ -3923,6 +3959,7 @@ def job_create(
|
|
|
3923
3959
|
runner_options=runner_options_dict,
|
|
3924
3960
|
idempotency_key=submission_idempotency_key,
|
|
3925
3961
|
max_failure_retries=max_failure_retries,
|
|
3962
|
+
**metrics_retry_policy_kwargs,
|
|
3926
3963
|
group_id=group_id,
|
|
3927
3964
|
model_base_url_collection=model_base_url_collection_list,
|
|
3928
3965
|
priority=priority,
|
|
@@ -4241,6 +4278,7 @@ def job_create(
|
|
|
4241
4278
|
**({"ak_id": ak_id} if ak_id is not None else {}),
|
|
4242
4279
|
**({"ak_mode": ak_mode} if ak_mode != "auto" else {}),
|
|
4243
4280
|
max_failure_retries=max_failure_retries,
|
|
4281
|
+
**metrics_retry_policy_kwargs,
|
|
4244
4282
|
)
|
|
4245
4283
|
batch.append(item)
|
|
4246
4284
|
return batch
|
|
@@ -4266,6 +4304,7 @@ def job_create(
|
|
|
4266
4304
|
runner=runner,
|
|
4267
4305
|
runner_options=runner_options_dict,
|
|
4268
4306
|
max_failure_retries=max_failure_retries,
|
|
4307
|
+
**metrics_retry_policy_kwargs,
|
|
4269
4308
|
priority=priority,
|
|
4270
4309
|
priority_weight=priority_weight,
|
|
4271
4310
|
resource_mode=resource_mode,
|
|
@@ -4311,6 +4350,7 @@ def job_create(
|
|
|
4311
4350
|
runner_options=runner_options_dict,
|
|
4312
4351
|
idempotency_key=submission_idempotency_key,
|
|
4313
4352
|
max_failure_retries=max_failure_retries,
|
|
4353
|
+
**metrics_retry_policy_kwargs,
|
|
4314
4354
|
priority=priority,
|
|
4315
4355
|
priority_weight=priority_weight,
|
|
4316
4356
|
resource_mode=resource_mode,
|
|
@@ -4370,6 +4410,7 @@ def job_create(
|
|
|
4370
4410
|
runner_options=runner_options_dict,
|
|
4371
4411
|
idempotency_key=submission_idempotency_key,
|
|
4372
4412
|
max_failure_retries=max_failure_retries,
|
|
4413
|
+
**metrics_retry_policy_kwargs,
|
|
4373
4414
|
priority=priority,
|
|
4374
4415
|
priority_weight=priority_weight,
|
|
4375
4416
|
resource_mode=resource_mode,
|
|
@@ -4495,6 +4536,7 @@ def job_create(
|
|
|
4495
4536
|
runner=runner,
|
|
4496
4537
|
runner_options=runner_options_dict,
|
|
4497
4538
|
max_failure_retries=max_failure_retries,
|
|
4539
|
+
**metrics_retry_policy_kwargs,
|
|
4498
4540
|
priority=priority,
|
|
4499
4541
|
priority_weight=priority_weight,
|
|
4500
4542
|
resource_mode=resource_mode,
|
|
@@ -4534,6 +4576,7 @@ def job_create(
|
|
|
4534
4576
|
runner_options=runner_options_dict,
|
|
4535
4577
|
idempotency_key=submission_idempotency_key,
|
|
4536
4578
|
max_failure_retries=max_failure_retries,
|
|
4579
|
+
**metrics_retry_policy_kwargs,
|
|
4537
4580
|
priority=priority,
|
|
4538
4581
|
priority_weight=priority_weight,
|
|
4539
4582
|
resource_mode=resource_mode,
|
|
@@ -4583,6 +4626,7 @@ def job_create(
|
|
|
4583
4626
|
runner_options=runner_options_dict,
|
|
4584
4627
|
idempotency_key=submission_idempotency_key,
|
|
4585
4628
|
max_failure_retries=max_failure_retries,
|
|
4629
|
+
**metrics_retry_policy_kwargs,
|
|
4586
4630
|
priority=priority,
|
|
4587
4631
|
priority_weight=priority_weight,
|
|
4588
4632
|
resource_mode=resource_mode,
|
|
@@ -5036,11 +5080,9 @@ def job_cancel(job_ids: List[str] = typer.Argument(..., help="Job IDs")):
|
|
|
5036
5080
|
_print_job_cancel_plain(result)
|
|
5037
5081
|
|
|
5038
5082
|
|
|
5039
|
-
# ---- RBAC v2 实例级委托:
|
|
5083
|
+
# ---- RBAC v2 实例级委托:Group 访问授权(scope=resource 绑定) ----
|
|
5040
5084
|
|
|
5041
5085
|
_RESOURCE_ROLE_IDS = {
|
|
5042
|
-
("job", "reader"): "built-in:job-reader",
|
|
5043
|
-
("job", "writer"): "built-in:job-writer",
|
|
5044
5086
|
("group", "reader"): "built-in:group-reader",
|
|
5045
5087
|
("group", "blackbox-reader"): "built-in:blackbox-reader",
|
|
5046
5088
|
("group", "writer"): "built-in:group-writer",
|
|
@@ -5052,18 +5094,14 @@ _RESOURCE_ROLE_IDS = {
|
|
|
5052
5094
|
def _resource_role_id(resource: str, role: str) -> str:
|
|
5053
5095
|
key = (resource, (role or "").strip().lower())
|
|
5054
5096
|
if key not in _RESOURCE_ROLE_IDS:
|
|
5055
|
-
choices =
|
|
5056
|
-
"reader, writer, cloner, blackbox-reader, admin"
|
|
5057
|
-
if resource == "group"
|
|
5058
|
-
else "reader, writer"
|
|
5059
|
-
)
|
|
5097
|
+
choices = "reader, writer, cloner, blackbox-reader, admin"
|
|
5060
5098
|
raise typer.BadParameter(f"role must be one of: {choices}")
|
|
5061
5099
|
return _RESOURCE_ROLE_IDS[key]
|
|
5062
5100
|
|
|
5063
5101
|
|
|
5064
5102
|
def _resolve_instance_workspace(client, resource: str, instance_id: str) -> str:
|
|
5065
|
-
"""解析
|
|
5066
|
-
doc = client.
|
|
5103
|
+
"""解析 Group 所属 queue_id(= workspace_id),供绑定门控使用。"""
|
|
5104
|
+
doc = client.get_group(instance_id)
|
|
5067
5105
|
ws = (doc or {}).get("queue_id")
|
|
5068
5106
|
if not ws:
|
|
5069
5107
|
raise typer.BadParameter(
|
|
@@ -5127,40 +5165,6 @@ def _print_instance_access(resource, instance_id):
|
|
|
5127
5165
|
print(f" - {b.get('principal_type')}:{b.get('principal_id')} role={b.get('role_id')}")
|
|
5128
5166
|
|
|
5129
5167
|
|
|
5130
|
-
@job_app.command("grant")
|
|
5131
|
-
def job_grant(
|
|
5132
|
-
job_id: str = typer.Argument(..., help="Job ID"),
|
|
5133
|
-
user: Optional[str] = typer.Option(None, "--user", help="Grant to this user_id"),
|
|
5134
|
-
user_group: Optional[str] = typer.Option(
|
|
5135
|
-
None, "--user-group", help="Grant to this user_group_id"
|
|
5136
|
-
),
|
|
5137
|
-
role: str = typer.Option(
|
|
5138
|
-
"reader", "--role", help="reader (view) or writer (view+cancel/update)"
|
|
5139
|
-
),
|
|
5140
|
-
):
|
|
5141
|
-
"""Grant a user/user-group access to a specific job (RBAC v2 instance delegation)."""
|
|
5142
|
-
_grant_instance_access("job", job_id, role, user, user_group)
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
@job_app.command("revoke")
|
|
5146
|
-
def job_revoke(
|
|
5147
|
-
job_id: str = typer.Argument(..., help="Job ID"),
|
|
5148
|
-
user: Optional[str] = typer.Option(None, "--user", help="Revoke from this user_id"),
|
|
5149
|
-
user_group: Optional[str] = typer.Option(
|
|
5150
|
-
None, "--user-group", help="Revoke from this user_group_id"
|
|
5151
|
-
),
|
|
5152
|
-
role: str = typer.Option("reader", "--role", help="reader or writer"),
|
|
5153
|
-
):
|
|
5154
|
-
"""Revoke a user/user-group's instance access to a specific job."""
|
|
5155
|
-
_revoke_instance_access("job", job_id, role, user, user_group)
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
@job_app.command("access-list")
|
|
5159
|
-
def job_access_list(job_id: str = typer.Argument(..., help="Job ID")):
|
|
5160
|
-
"""List who has instance-level access to a specific job."""
|
|
5161
|
-
_print_instance_access("job", job_id)
|
|
5162
|
-
|
|
5163
|
-
|
|
5164
5168
|
@job_app.command("retry")
|
|
5165
5169
|
def job_retry(
|
|
5166
5170
|
job_ids: List[str] = typer.Argument(..., help="Job IDs"),
|
|
@@ -5888,6 +5892,11 @@ def group_create(
|
|
|
5888
5892
|
"-q",
|
|
5889
5893
|
help="Queue id (agent-platform-apiserver); omit to use server default (default)",
|
|
5890
5894
|
),
|
|
5895
|
+
metrics_retry_policy: Optional[str] = typer.Option(
|
|
5896
|
+
None,
|
|
5897
|
+
"--metrics-retry-policy",
|
|
5898
|
+
help='Creation-only policy JSON or file; null inherits, {"enable": false} disables',
|
|
5899
|
+
),
|
|
5891
5900
|
group_concurrency_policy: Optional[str] = typer.Option(
|
|
5892
5901
|
None,
|
|
5893
5902
|
"--group-concurrency-policy",
|
|
@@ -6025,10 +6034,23 @@ def group_create(
|
|
|
6025
6034
|
):
|
|
6026
6035
|
"""Create a Group.
|
|
6027
6036
|
|
|
6028
|
-
|
|
6029
|
-
|
|
6030
|
-
|
|
6031
|
-
|
|
6037
|
+
\b
|
|
6038
|
+
For agents:
|
|
6039
|
+
Before creating, you MUST ensure these five parameters are explicitly specified:
|
|
6040
|
+
benchmark, meta-model, meta-job-type, meta-train-stage, meta-teacher-direction.
|
|
6041
|
+
Keep specified values. Fill missing parameters using available Job, Group,
|
|
6042
|
+
Profile, Dataset, Template, or other relevant information.
|
|
6043
|
+
Check available Dataset -> Benchmark -> teacher direction links as candidates.
|
|
6044
|
+
Adopt them only when they match this Group's business purpose.
|
|
6045
|
+
Separate the input-data source from this Group's business attribution.
|
|
6046
|
+
For rollout using A's Dataset to produce data for B, use B only when B is
|
|
6047
|
+
this Group's intended attribution; do not automatically inherit A's labels
|
|
6048
|
+
or choose B merely because it is a downstream consumer.
|
|
6049
|
+
If a missing value cannot be inferred, use default for benchmark
|
|
6050
|
+
or Others for the corresponding meta parameter.
|
|
6051
|
+
Do not ask the user for these values.
|
|
6052
|
+
The server may derive some omitted meta values or default meta-job-type to Eval-Eval;
|
|
6053
|
+
explicitly specify and verify all five values rather than relying on implicit attribution.
|
|
6032
6054
|
"""
|
|
6033
6055
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
6034
6056
|
raise typer.BadParameter("Choose whitebox or blackbox", param_hint="--transparency")
|
|
@@ -6042,6 +6064,12 @@ def group_create(
|
|
|
6042
6064
|
raise typer.BadParameter(str(exc), param_hint="--ak-id") from None
|
|
6043
6065
|
if transparency == "blackbox" and idempotency_key is None:
|
|
6044
6066
|
idempotency_key = str(uuid.uuid4())
|
|
6067
|
+
metrics_retry_policy_dict = _parse_metrics_retry_policy(metrics_retry_policy)
|
|
6068
|
+
metrics_retry_policy_kwargs = (
|
|
6069
|
+
{"metrics_retry_policy": metrics_retry_policy_dict}
|
|
6070
|
+
if metrics_retry_policy_dict is not None
|
|
6071
|
+
else {}
|
|
6072
|
+
)
|
|
6045
6073
|
auto_config = _parse_json_object(group_concurrency_policy, "--group-concurrency-policy")
|
|
6046
6074
|
client = get_client()
|
|
6047
6075
|
eval_config_dict = json.loads(eval_config) if eval_config else None
|
|
@@ -6109,6 +6137,7 @@ def group_create(
|
|
|
6109
6137
|
agenthub_revision=agenthub_revision,
|
|
6110
6138
|
queue_id=queue, # CLI --queue / -q 走标准 queue_id 字段
|
|
6111
6139
|
max_concurrency=max_concurrency,
|
|
6140
|
+
**metrics_retry_policy_kwargs,
|
|
6112
6141
|
**({"group_concurrency_policy": auto_config} if auto_config is not None else {}),
|
|
6113
6142
|
eval_config=eval_config_dict,
|
|
6114
6143
|
model_base_url_collection=model_base_url_collection_list,
|
|
@@ -7400,7 +7429,13 @@ def _format_api_error(exc: APIError) -> str:
|
|
|
7400
7429
|
if profile_error := format_profile_error(exc):
|
|
7401
7430
|
return profile_error
|
|
7402
7431
|
if isinstance(exc.detail, dict) and exc.detail.get("code") == "blackbox_content_restricted":
|
|
7403
|
-
|
|
7432
|
+
diagnostics = "".join(
|
|
7433
|
+
f" ({diagnostic})" for diagnostic in format_api_error_diagnostics(exc)
|
|
7434
|
+
)
|
|
7435
|
+
return (
|
|
7436
|
+
"error: Blackbox content is hidden; Group Reader access is required. "
|
|
7437
|
+
f"(request_id={exc.request_id}){diagnostics}"
|
|
7438
|
+
)
|
|
7404
7439
|
if str(exc.detail) != _NON_JSON_RESPONSE_DETAIL:
|
|
7405
7440
|
return f"error: {exc}"
|
|
7406
7441
|
|
|
@@ -7410,6 +7445,7 @@ def _format_api_error(exc: APIError) -> str:
|
|
|
7410
7445
|
f"hint: {_NON_JSON_RESPONSE_HINT}",
|
|
7411
7446
|
f"request_id: {exc.request_id}",
|
|
7412
7447
|
]
|
|
7448
|
+
lines.extend(format_api_error_diagnostics(exc, separator=": "))
|
|
7413
7449
|
if exc.response is not None:
|
|
7414
7450
|
content_type = exc.response.headers.get("Content-Type") or "<missing>"
|
|
7415
7451
|
lines.extend(
|