ap-client 0.8.0.dev5__tar.gz → 0.8.0.dev6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/PKG-INFO +3 -3
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/api.py +8 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/cli.py +129 -11
- ap_client-0.8.0.dev6/ap_client/cli_submission_defaults.py +95 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/config.py +32 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/dfs_commands.py +8 -3
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/instance_commands.py +578 -6
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/ossutil_io.py +17 -6
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/pyproject.toml +3 -3
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/__init__.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/dataset_name.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/dataset_register.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/exporter.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/fs_commands.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/fs_storage.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/fs_transfer.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/instance_options.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/irepo_sdk.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/managed_ak.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/manifest_view.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/ossutil_hash.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_cascade_all.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_cascade_cli.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_cascade_display.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_cascade_state.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_commands.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_diagnostics.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_display.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/profile_references.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/repo_read_guard.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/share_api.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/split_publish.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/submission_metadata.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/tbb.py +0 -0
- {ap_client-0.8.0.dev5 → ap_client-0.8.0.dev6}/ap_client/waiter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.8.0.
|
|
3
|
+
Version: 0.8.0.dev6
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: irepo-artifact==0.2.6
|
|
@@ -10,9 +10,9 @@ Requires-Dist: rich>=13.0.0
|
|
|
10
10
|
Requires-Dist: typer>=0.9.0
|
|
11
11
|
Requires-Dist: websockets>=13.0
|
|
12
12
|
Provides-Extra: all
|
|
13
|
-
Requires-Dist: instance-repo[oss]<
|
|
13
|
+
Requires-Dist: instance-repo[oss]<1.5.0,>=1.4.8.dev1; extra == 'all'
|
|
14
14
|
Provides-Extra: dataset
|
|
15
|
-
Requires-Dist: instance-repo[oss]<
|
|
15
|
+
Requires-Dist: instance-repo[oss]<1.5.0,>=1.4.8.dev1; extra == 'dataset'
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
|
|
18
18
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -1220,6 +1220,7 @@ class APIClient:
|
|
|
1220
1220
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1221
1221
|
idempotency_key: Optional[str] = None,
|
|
1222
1222
|
group_concurrency_policy: Optional[dict] = None,
|
|
1223
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1223
1224
|
) -> dict:
|
|
1224
1225
|
"""Create a group with optional scheduling defaults for its Jobs.
|
|
1225
1226
|
|
|
@@ -1239,6 +1240,8 @@ class APIClient:
|
|
|
1239
1240
|
raise ValueError("ak_ids entries must be nonempty strings")
|
|
1240
1241
|
references = list(dict.fromkeys(references))
|
|
1241
1242
|
body: dict = {}
|
|
1243
|
+
if metrics_retry_policy is not None:
|
|
1244
|
+
body["metrics_retry_policy"] = metrics_retry_policy
|
|
1242
1245
|
if transparency is not None:
|
|
1243
1246
|
body["transparency"] = transparency
|
|
1244
1247
|
if idempotency_key is not None:
|
|
@@ -1367,6 +1370,7 @@ class APIClient:
|
|
|
1367
1370
|
instance_range: Optional[str] = None,
|
|
1368
1371
|
profile: Optional[str] = None,
|
|
1369
1372
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1373
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1370
1374
|
) -> dict:
|
|
1371
1375
|
"""Create a job."""
|
|
1372
1376
|
body = self.build_create_job_body(
|
|
@@ -1391,6 +1395,7 @@ class APIClient:
|
|
|
1391
1395
|
runner_options=runner_options,
|
|
1392
1396
|
idempotency_key=idempotency_key,
|
|
1393
1397
|
max_failure_retries=max_failure_retries,
|
|
1398
|
+
metrics_retry_policy=metrics_retry_policy,
|
|
1394
1399
|
priority=priority,
|
|
1395
1400
|
priority_weight=priority_weight,
|
|
1396
1401
|
resource_mode=resource_mode,
|
|
@@ -1483,6 +1488,7 @@ class APIClient:
|
|
|
1483
1488
|
instance_range: Optional[str] = None,
|
|
1484
1489
|
profile: Optional[str] = None,
|
|
1485
1490
|
transparency: Optional[Literal["whitebox", "blackbox"]] = None,
|
|
1491
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1486
1492
|
) -> dict:
|
|
1487
1493
|
"""Build the /jobs request body for job creation."""
|
|
1488
1494
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
@@ -1554,6 +1560,8 @@ class APIClient:
|
|
|
1554
1560
|
body["runner_options"] = runner_options
|
|
1555
1561
|
if idempotency_key is not None:
|
|
1556
1562
|
body["idempotency_key"] = idempotency_key
|
|
1563
|
+
if metrics_retry_policy is not None:
|
|
1564
|
+
body["metrics_retry_policy"] = metrics_retry_policy
|
|
1557
1565
|
if max_failure_retries:
|
|
1558
1566
|
body["max_failure_retries"] = max_failure_retries
|
|
1559
1567
|
|
|
@@ -22,6 +22,12 @@ from ap_client.api import (
|
|
|
22
22
|
set_cluster_override,
|
|
23
23
|
set_verbose_override,
|
|
24
24
|
)
|
|
25
|
+
from ap_client.cli_submission_defaults import (
|
|
26
|
+
datasource_env_default,
|
|
27
|
+
resolve_model_base_url_collection,
|
|
28
|
+
resolve_profile,
|
|
29
|
+
resolve_queue,
|
|
30
|
+
)
|
|
25
31
|
from ap_client.config import ENV_VAR_SPECS, ConfigurationError, _parse_bool, normalize_output_format
|
|
26
32
|
from ap_client.dataset_commands import register as _register_dataset_commands
|
|
27
33
|
from ap_client.dfs_commands import register as _register_dfs_commands
|
|
@@ -281,6 +287,25 @@ def _reject_account_pool_template_credential_cli(
|
|
|
281
287
|
raise typer.Exit(1)
|
|
282
288
|
|
|
283
289
|
|
|
290
|
+
def _parse_metrics_retry_policy(raw):
|
|
291
|
+
if not isinstance(raw, str):
|
|
292
|
+
return None
|
|
293
|
+
if raw.strip() != "null" and not raw.lstrip().startswith("{"):
|
|
294
|
+
try:
|
|
295
|
+
raw = Path(raw).expanduser().read_text(encoding="utf-8")
|
|
296
|
+
except OSError as exc:
|
|
297
|
+
raise typer.BadParameter(
|
|
298
|
+
"Cannot read metrics retry policy file", param_hint="--metrics-retry-policy"
|
|
299
|
+
) from exc
|
|
300
|
+
value = _parse_json_object(raw, "--metrics-retry-policy", allow_null=True)
|
|
301
|
+
if value is not None:
|
|
302
|
+
if "enable" not in value:
|
|
303
|
+
raise typer.BadParameter("metrics_retry_policy.enable is required (boolean)")
|
|
304
|
+
if type(value["enable"]) is not bool:
|
|
305
|
+
raise typer.BadParameter("metrics_retry_policy.enable must be a boolean")
|
|
306
|
+
return value
|
|
307
|
+
|
|
308
|
+
|
|
284
309
|
def _parse_json_object(
|
|
285
310
|
raw: Optional[str], option_name: str, *, allow_null: bool = False
|
|
286
311
|
) -> Optional[dict]:
|
|
@@ -1449,6 +1474,7 @@ def _idempotency_item_key(
|
|
|
1449
1474
|
max_failure_retries: int = 0,
|
|
1450
1475
|
ak_id: Optional[str] = None,
|
|
1451
1476
|
ak_mode: str = "auto",
|
|
1477
|
+
metrics_retry_policy: Optional[dict] = None,
|
|
1452
1478
|
) -> str:
|
|
1453
1479
|
# Client-supplied item keys are authoritative; server hashing is only a
|
|
1454
1480
|
# fallback for non-CLI callers, so this payload intentionally need not
|
|
@@ -1472,6 +1498,8 @@ def _idempotency_item_key(
|
|
|
1472
1498
|
payload_data["ak_mode"] = ak_mode
|
|
1473
1499
|
if resource_profile_id is not None:
|
|
1474
1500
|
payload_data["resource_profile_id"] = resource_profile_id
|
|
1501
|
+
if metrics_retry_policy is not None:
|
|
1502
|
+
payload_data["metrics_retry_policy"] = metrics_retry_policy
|
|
1475
1503
|
if max_failure_retries:
|
|
1476
1504
|
payload_data["max_failure_retries"] = max_failure_retries
|
|
1477
1505
|
payload = json.dumps(
|
|
@@ -2158,6 +2186,30 @@ def _parse_headers_env_value(raw: str) -> dict:
|
|
|
2158
2186
|
return headers
|
|
2159
2187
|
|
|
2160
2188
|
|
|
2189
|
+
# Anchored on "://" and bounded by "/" so a "@" inside a path is never mistaken for userinfo.
|
|
2190
|
+
_MODEL_URL_USERINFO_RE = re.compile(r"(?<=://)[^/?#\s]*@")
|
|
2191
|
+
|
|
2192
|
+
|
|
2193
|
+
def _mask_model_base_url_collection(raw: str) -> str:
|
|
2194
|
+
"""Redact credentials in model URL query strings and netloc userinfo."""
|
|
2195
|
+
from ap_client.api import _redact_sensitive_data
|
|
2196
|
+
|
|
2197
|
+
raw = _MODEL_URL_USERINFO_RE.sub("***@", raw)
|
|
2198
|
+
try:
|
|
2199
|
+
value = json.loads(raw)
|
|
2200
|
+
except ValueError:
|
|
2201
|
+
return ",".join(_redact_sensitive_data(part.strip()) for part in raw.split(","))
|
|
2202
|
+
if isinstance(value, str):
|
|
2203
|
+
return _redact_sensitive_data(value)
|
|
2204
|
+
if isinstance(value, (list, dict)):
|
|
2205
|
+
return json.dumps(
|
|
2206
|
+
_redact_sensitive_data(value),
|
|
2207
|
+
ensure_ascii=False,
|
|
2208
|
+
separators=(",", ":"),
|
|
2209
|
+
)
|
|
2210
|
+
return raw
|
|
2211
|
+
|
|
2212
|
+
|
|
2161
2213
|
def _env_value_for_display(name: str, raw: str, *, show_secrets: bool) -> str:
|
|
2162
2214
|
if name == "AP_HEADERS":
|
|
2163
2215
|
headers = _parse_headers_env_value(raw)
|
|
@@ -2166,6 +2218,8 @@ def _env_value_for_display(name: str, raw: str, *, show_secrets: bool) -> str:
|
|
|
2166
2218
|
return json.dumps(headers, ensure_ascii=False, separators=(",", ":"))
|
|
2167
2219
|
if name == "AP_API_KEY" and not show_secrets:
|
|
2168
2220
|
return _mask_config_secret(raw) or ""
|
|
2221
|
+
if name == "AP_MODEL_BASE_URL_COLLECTION" and not show_secrets:
|
|
2222
|
+
return _mask_model_base_url_collection(raw)
|
|
2169
2223
|
return raw
|
|
2170
2224
|
|
|
2171
2225
|
|
|
@@ -2291,7 +2345,7 @@ def config_env(
|
|
|
2291
2345
|
show_secrets: bool = typer.Option(
|
|
2292
2346
|
False,
|
|
2293
2347
|
"--show-secrets",
|
|
2294
|
-
help="Print unmasked secret environment values",
|
|
2348
|
+
help="Print unmasked secret environment values; model URL masking covers query and userinfo credentials, not path segments",
|
|
2295
2349
|
),
|
|
2296
2350
|
):
|
|
2297
2351
|
"""List supported environment variables."""
|
|
@@ -3334,7 +3388,7 @@ def job_create(
|
|
|
3334
3388
|
"--queue-id",
|
|
3335
3389
|
"--queue",
|
|
3336
3390
|
"-q",
|
|
3337
|
-
help="Queue id (agent-platform-apiserver);
|
|
3391
|
+
help="Queue id (agent-platform-apiserver); defaults to AP_QUEUE, then server default (default). An empty value disables AP_QUEUE",
|
|
3338
3392
|
),
|
|
3339
3393
|
quota_id: Optional[str] = typer.Option(
|
|
3340
3394
|
None,
|
|
@@ -3390,7 +3444,7 @@ def job_create(
|
|
|
3390
3444
|
model_base_url_collection: Optional[str] = typer.Option(
|
|
3391
3445
|
None,
|
|
3392
3446
|
"--model-base-url-collection",
|
|
3393
|
-
help="Model base URL collection for server-side load balancing (JSON array or comma-separated)",
|
|
3447
|
+
help="Model base URL collection for server-side load balancing (JSON array or comma-separated); defaults to AP_MODEL_BASE_URL_COLLECTION unless an explicit collection or file is provided. An empty collection disables the environment default",
|
|
3394
3448
|
),
|
|
3395
3449
|
datasources: Optional[str] = typer.Option(
|
|
3396
3450
|
None,
|
|
@@ -3398,7 +3452,8 @@ def job_create(
|
|
|
3398
3452
|
help=(
|
|
3399
3453
|
"Datasources to mount into job pods. JSON array (e.g. "
|
|
3400
3454
|
'\'["ds-abc", {"datasource":"ds-xyz","mount_path":"/mnt/data"}]\') '
|
|
3401
|
-
"or comma-separated datasource IDs (e.g. 'ds-abc,ds-xyz')."
|
|
3455
|
+
"or comma-separated datasource IDs (e.g. 'ds-abc,ds-xyz'). "
|
|
3456
|
+
"Overrides AP_DATASOURCES; use [] to clear the default."
|
|
3402
3457
|
),
|
|
3403
3458
|
),
|
|
3404
3459
|
batch_size: int = typer.Option(
|
|
@@ -3475,6 +3530,11 @@ def job_create(
|
|
|
3475
3530
|
"--idempotency-key",
|
|
3476
3531
|
help="UUID4 key for retrying the same logical submission; providing one enables idempotency",
|
|
3477
3532
|
),
|
|
3533
|
+
metrics_retry_policy: Optional[str] = typer.Option(
|
|
3534
|
+
None,
|
|
3535
|
+
"--metrics-retry-policy",
|
|
3536
|
+
help='Metrics retry policy JSON or file; null inherits, {"enable": false} disables',
|
|
3537
|
+
),
|
|
3478
3538
|
max_failure_retries: int = typer.Option(
|
|
3479
3539
|
0,
|
|
3480
3540
|
"--max-failure-retries",
|
|
@@ -3539,7 +3599,8 @@ def job_create(
|
|
|
3539
3599
|
profile: Optional[str] = typer.Option(
|
|
3540
3600
|
None,
|
|
3541
3601
|
"--profile",
|
|
3542
|
-
help="Profile ID or name with optional :latest, :verified, :release (no fallback), or :vN; resolved server-side"
|
|
3602
|
+
help="Profile ID or name with optional :latest, :verified, :release (no fallback), or :vN; resolved server-side. "
|
|
3603
|
+
"Defaults to AP_PROFILE only without a template; an empty value disables the default",
|
|
3543
3604
|
),
|
|
3544
3605
|
resource_profile_id: Optional[str] = typer.Option(
|
|
3545
3606
|
None,
|
|
@@ -3637,9 +3698,24 @@ def job_create(
|
|
|
3637
3698
|
--meta-teacher-direction "$META_TEACHER_DIRECTION"
|
|
3638
3699
|
|
|
3639
3700
|
"""
|
|
3701
|
+
# Resolve once before any client/network work, shared by preview and submission.
|
|
3702
|
+
try:
|
|
3703
|
+
profile = resolve_profile(profile, template)
|
|
3704
|
+
queue = resolve_queue(queue)
|
|
3705
|
+
model_base_url_collection, model_collection_from_env = resolve_model_base_url_collection(
|
|
3706
|
+
model_base_url_collection, model_base_url_file
|
|
3707
|
+
)
|
|
3708
|
+
datasources_list = _parse_datasources_option(datasources)
|
|
3709
|
+
if datasources_list is None:
|
|
3710
|
+
datasources_list = datasource_env_default()
|
|
3711
|
+
except ValueError as exc:
|
|
3712
|
+
raise typer.BadParameter(str(exc)) from None
|
|
3640
3713
|
if transparency is not None and transparency not in {"whitebox", "blackbox"}:
|
|
3641
3714
|
raise typer.BadParameter("Choose whitebox or blackbox", param_hint="--transparency")
|
|
3642
3715
|
output_format = _normalize_output_format(output_format)
|
|
3716
|
+
# --job-spec writes a machine-readable payload to stdout even when the resolved
|
|
3717
|
+
# format is plain, so diagnostics must be routed away from that stream.
|
|
3718
|
+
diagnostics_format = "json" if job_spec_flag else output_format
|
|
3643
3719
|
if not template and not profile:
|
|
3644
3720
|
raise typer.BadParameter("TEMPLATE is required unless --profile is provided")
|
|
3645
3721
|
if ak_mode not in {"auto", "manual", "synced"}:
|
|
@@ -3745,9 +3821,6 @@ def job_create(
|
|
|
3745
3821
|
else {}
|
|
3746
3822
|
)
|
|
3747
3823
|
|
|
3748
|
-
# Available to both profile and non-profile flows.
|
|
3749
|
-
datasources_list = _parse_datasources_option(datasources)
|
|
3750
|
-
|
|
3751
3824
|
# Parse model_base_url_collection early so it's available for both profile and non-profile flows.
|
|
3752
3825
|
model_base_url_collection_list: Optional[list] = None
|
|
3753
3826
|
if model_base_url_collection:
|
|
@@ -3771,6 +3844,27 @@ def job_create(
|
|
|
3771
3844
|
_emit_error(f"file not found: {model_base_url_file}")
|
|
3772
3845
|
raise typer.Exit(1)
|
|
3773
3846
|
|
|
3847
|
+
metrics_retry_policy_dict = _parse_metrics_retry_policy(metrics_retry_policy)
|
|
3848
|
+
metrics_retry_policy_kwargs = (
|
|
3849
|
+
{"metrics_retry_policy": metrics_retry_policy_dict}
|
|
3850
|
+
if metrics_retry_policy_dict is not None
|
|
3851
|
+
else {}
|
|
3852
|
+
)
|
|
3853
|
+
|
|
3854
|
+
# A Group's URL pool is fixed when the Group is created, so an environment-derived
|
|
3855
|
+
# pool must never ride along when appending to an existing Group. Shared by the
|
|
3856
|
+
# profile and template flows; explicit values keep their per-flow handling below.
|
|
3857
|
+
if model_collection_from_env and group_id and model_base_url_collection_list:
|
|
3858
|
+
warning = (
|
|
3859
|
+
"AP_MODEL_BASE_URL_COLLECTION is ignored when appending to an existing "
|
|
3860
|
+
"Group (--group-id); the URL pool is fixed at Group creation"
|
|
3861
|
+
)
|
|
3862
|
+
if dry_run:
|
|
3863
|
+
dry_run_warnings.append(warning)
|
|
3864
|
+
else:
|
|
3865
|
+
_emit_info(f"warning: {warning}", diagnostics_format)
|
|
3866
|
+
model_base_url_collection_list = None
|
|
3867
|
+
|
|
3774
3868
|
# Reject --trials when not in profile mode (profile-only override)
|
|
3775
3869
|
if trials is not None and profile is None:
|
|
3776
3870
|
_emit_error("--trials is only supported with --profile")
|
|
@@ -3839,6 +3933,7 @@ def job_create(
|
|
|
3839
3933
|
runner_options=runner_options_dict,
|
|
3840
3934
|
idempotency_key=submission_idempotency_key,
|
|
3841
3935
|
max_failure_retries=max_failure_retries,
|
|
3936
|
+
**metrics_retry_policy_kwargs,
|
|
3842
3937
|
group_id=group_id,
|
|
3843
3938
|
model_base_url_collection=model_base_url_collection_list,
|
|
3844
3939
|
group_post_process=group_post_process_dict,
|
|
@@ -3924,6 +4019,7 @@ def job_create(
|
|
|
3924
4019
|
runner_options=runner_options_dict,
|
|
3925
4020
|
idempotency_key=submission_idempotency_key,
|
|
3926
4021
|
max_failure_retries=max_failure_retries,
|
|
4022
|
+
**metrics_retry_policy_kwargs,
|
|
3927
4023
|
group_id=group_id,
|
|
3928
4024
|
model_base_url_collection=model_base_url_collection_list,
|
|
3929
4025
|
priority=priority,
|
|
@@ -4127,11 +4223,14 @@ def job_create(
|
|
|
4127
4223
|
|
|
4128
4224
|
# --model-base-url-collection only applies to batch submissions; warn and drop it for single jobs.
|
|
4129
4225
|
if not is_batch and model_base_url_collection_list:
|
|
4130
|
-
|
|
4226
|
+
if model_collection_from_env:
|
|
4227
|
+
warning = "AP_MODEL_BASE_URL_COLLECTION is ignored for single-job submissions"
|
|
4228
|
+
else:
|
|
4229
|
+
warning = "--model-base-url-collection is ignored for single-job submissions"
|
|
4131
4230
|
if dry_run:
|
|
4132
4231
|
dry_run_warnings.append(warning)
|
|
4133
4232
|
else:
|
|
4134
|
-
_emit_info(f"warning: {warning}",
|
|
4233
|
+
_emit_info(f"warning: {warning}", diagnostics_format)
|
|
4135
4234
|
model_base_url_collection_list = None
|
|
4136
4235
|
|
|
4137
4236
|
retry_command = (
|
|
@@ -4192,7 +4291,7 @@ def job_create(
|
|
|
4192
4291
|
if dry_run:
|
|
4193
4292
|
_emit_progress(message)
|
|
4194
4293
|
else:
|
|
4195
|
-
_emit_info(message,
|
|
4294
|
+
_emit_info(message, diagnostics_format)
|
|
4196
4295
|
params_list = [
|
|
4197
4296
|
{**params_dict, "instance_id": iid, "task_id": iid} for iid in instance_ids
|
|
4198
4297
|
]
|
|
@@ -4242,6 +4341,7 @@ def job_create(
|
|
|
4242
4341
|
**({"ak_id": ak_id} if ak_id is not None else {}),
|
|
4243
4342
|
**({"ak_mode": ak_mode} if ak_mode != "auto" else {}),
|
|
4244
4343
|
max_failure_retries=max_failure_retries,
|
|
4344
|
+
**metrics_retry_policy_kwargs,
|
|
4245
4345
|
)
|
|
4246
4346
|
batch.append(item)
|
|
4247
4347
|
return batch
|
|
@@ -4267,6 +4367,7 @@ def job_create(
|
|
|
4267
4367
|
runner=runner,
|
|
4268
4368
|
runner_options=runner_options_dict,
|
|
4269
4369
|
max_failure_retries=max_failure_retries,
|
|
4370
|
+
**metrics_retry_policy_kwargs,
|
|
4270
4371
|
priority=priority,
|
|
4271
4372
|
priority_weight=priority_weight,
|
|
4272
4373
|
resource_mode=resource_mode,
|
|
@@ -4312,6 +4413,7 @@ def job_create(
|
|
|
4312
4413
|
runner_options=runner_options_dict,
|
|
4313
4414
|
idempotency_key=submission_idempotency_key,
|
|
4314
4415
|
max_failure_retries=max_failure_retries,
|
|
4416
|
+
**metrics_retry_policy_kwargs,
|
|
4315
4417
|
priority=priority,
|
|
4316
4418
|
priority_weight=priority_weight,
|
|
4317
4419
|
resource_mode=resource_mode,
|
|
@@ -4371,6 +4473,7 @@ def job_create(
|
|
|
4371
4473
|
runner_options=runner_options_dict,
|
|
4372
4474
|
idempotency_key=submission_idempotency_key,
|
|
4373
4475
|
max_failure_retries=max_failure_retries,
|
|
4476
|
+
**metrics_retry_policy_kwargs,
|
|
4374
4477
|
priority=priority,
|
|
4375
4478
|
priority_weight=priority_weight,
|
|
4376
4479
|
resource_mode=resource_mode,
|
|
@@ -4496,6 +4599,7 @@ def job_create(
|
|
|
4496
4599
|
runner=runner,
|
|
4497
4600
|
runner_options=runner_options_dict,
|
|
4498
4601
|
max_failure_retries=max_failure_retries,
|
|
4602
|
+
**metrics_retry_policy_kwargs,
|
|
4499
4603
|
priority=priority,
|
|
4500
4604
|
priority_weight=priority_weight,
|
|
4501
4605
|
resource_mode=resource_mode,
|
|
@@ -4535,6 +4639,7 @@ def job_create(
|
|
|
4535
4639
|
runner_options=runner_options_dict,
|
|
4536
4640
|
idempotency_key=submission_idempotency_key,
|
|
4537
4641
|
max_failure_retries=max_failure_retries,
|
|
4642
|
+
**metrics_retry_policy_kwargs,
|
|
4538
4643
|
priority=priority,
|
|
4539
4644
|
priority_weight=priority_weight,
|
|
4540
4645
|
resource_mode=resource_mode,
|
|
@@ -4584,6 +4689,7 @@ def job_create(
|
|
|
4584
4689
|
runner_options=runner_options_dict,
|
|
4585
4690
|
idempotency_key=submission_idempotency_key,
|
|
4586
4691
|
max_failure_retries=max_failure_retries,
|
|
4692
|
+
**metrics_retry_policy_kwargs,
|
|
4587
4693
|
priority=priority,
|
|
4588
4694
|
priority_weight=priority_weight,
|
|
4589
4695
|
resource_mode=resource_mode,
|
|
@@ -5849,6 +5955,11 @@ def group_create(
|
|
|
5849
5955
|
"-q",
|
|
5850
5956
|
help="Queue id (agent-platform-apiserver); omit to use server default (default)",
|
|
5851
5957
|
),
|
|
5958
|
+
metrics_retry_policy: Optional[str] = typer.Option(
|
|
5959
|
+
None,
|
|
5960
|
+
"--metrics-retry-policy",
|
|
5961
|
+
help='Creation-only policy JSON or file; null inherits, {"enable": false} disables',
|
|
5962
|
+
),
|
|
5852
5963
|
group_concurrency_policy: Optional[str] = typer.Option(
|
|
5853
5964
|
None,
|
|
5854
5965
|
"--group-concurrency-policy",
|
|
@@ -6016,6 +6127,12 @@ def group_create(
|
|
|
6016
6127
|
raise typer.BadParameter(str(exc), param_hint="--ak-id") from None
|
|
6017
6128
|
if transparency == "blackbox" and idempotency_key is None:
|
|
6018
6129
|
idempotency_key = str(uuid.uuid4())
|
|
6130
|
+
metrics_retry_policy_dict = _parse_metrics_retry_policy(metrics_retry_policy)
|
|
6131
|
+
metrics_retry_policy_kwargs = (
|
|
6132
|
+
{"metrics_retry_policy": metrics_retry_policy_dict}
|
|
6133
|
+
if metrics_retry_policy_dict is not None
|
|
6134
|
+
else {}
|
|
6135
|
+
)
|
|
6019
6136
|
auto_config = _parse_json_object(group_concurrency_policy, "--group-concurrency-policy")
|
|
6020
6137
|
client = get_client()
|
|
6021
6138
|
eval_config_dict = json.loads(eval_config) if eval_config else None
|
|
@@ -6083,6 +6200,7 @@ def group_create(
|
|
|
6083
6200
|
agenthub_revision=agenthub_revision,
|
|
6084
6201
|
queue_id=queue, # CLI --queue / -q 走标准 queue_id 字段
|
|
6085
6202
|
max_concurrency=max_concurrency,
|
|
6203
|
+
**metrics_retry_policy_kwargs,
|
|
6086
6204
|
**({"group_concurrency_policy": auto_config} if auto_config is not None else {}),
|
|
6087
6205
|
eval_config=eval_config_dict,
|
|
6088
6206
|
model_base_url_collection=model_base_url_collection_list,
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Environment defaults for job-create CLI only; never loaded by the SDK."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import posixpath
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def resolve_profile(profile: str | None, template: str | None) -> str | None:
|
|
10
|
+
"""Explicit profile (including empty) or template takes precedence."""
|
|
11
|
+
if profile is not None:
|
|
12
|
+
return profile if profile.strip() else None
|
|
13
|
+
if template is not None:
|
|
14
|
+
return None
|
|
15
|
+
value = os.environ.get("AP_PROFILE", "").strip()
|
|
16
|
+
if not value:
|
|
17
|
+
return None
|
|
18
|
+
parts = value.split(":")
|
|
19
|
+
if len(parts) > 2 or not parts[0] or any(c.isspace() or ord(c) < 32 for c in value):
|
|
20
|
+
raise ValueError(
|
|
21
|
+
"AP_PROFILE must be a name/ID with at most one version selector and no internal whitespace"
|
|
22
|
+
)
|
|
23
|
+
if len(parts) == 2:
|
|
24
|
+
selector = parts[1].lower()
|
|
25
|
+
if selector not in {"latest", "verified", "release"}:
|
|
26
|
+
version = selector.removeprefix("v")
|
|
27
|
+
if not re.fullmatch(r"[0-9]+", version) or not version.strip("0"):
|
|
28
|
+
raise ValueError(
|
|
29
|
+
"AP_PROFILE selector must be latest, verified, release, or a positive version such as v3"
|
|
30
|
+
)
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def resolve_queue(queue: str | None) -> str | None:
|
|
35
|
+
"""Explicit queue (including empty) takes precedence."""
|
|
36
|
+
if queue is not None:
|
|
37
|
+
return queue if queue.strip() else None
|
|
38
|
+
return os.environ.get("AP_QUEUE", "").strip() or None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def resolve_model_base_url_collection(
|
|
42
|
+
collection: str | None, file: str | None
|
|
43
|
+
) -> tuple[str | None, bool]:
|
|
44
|
+
"""Explicit collection (including empty) or file takes precedence.
|
|
45
|
+
|
|
46
|
+
Returns ``(value, from_env)`` so callers can tell a deliberate flag from an
|
|
47
|
+
environment default; the environment is never read when either flag exists.
|
|
48
|
+
An environment value that is valid JSON but not an array is rejected here so
|
|
49
|
+
the message names the variable instead of a flag the caller never passed.
|
|
50
|
+
"""
|
|
51
|
+
if collection is not None or file is not None:
|
|
52
|
+
return collection, False
|
|
53
|
+
value = os.environ.get("AP_MODEL_BASE_URL_COLLECTION", "").strip() or None
|
|
54
|
+
if value is not None:
|
|
55
|
+
try:
|
|
56
|
+
parsed = json.loads(value)
|
|
57
|
+
except ValueError:
|
|
58
|
+
pass
|
|
59
|
+
else:
|
|
60
|
+
if not isinstance(parsed, list):
|
|
61
|
+
raise ValueError(
|
|
62
|
+
"AP_MODEL_BASE_URL_COLLECTION must be a JSON array or comma-separated URLs"
|
|
63
|
+
)
|
|
64
|
+
return value, True
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def datasource_env_default() -> list | None:
|
|
68
|
+
"""Parse the environment only after an explicit CLI value is ruled out."""
|
|
69
|
+
raw = os.environ.get("AP_DATASOURCES", "").strip()
|
|
70
|
+
if not raw:
|
|
71
|
+
return None
|
|
72
|
+
try:
|
|
73
|
+
value = json.loads(raw)
|
|
74
|
+
except ValueError:
|
|
75
|
+
raise ValueError("AP_DATASOURCES must be a JSON array") from None
|
|
76
|
+
if not isinstance(value, list):
|
|
77
|
+
raise ValueError("AP_DATASOURCES must be a JSON array")
|
|
78
|
+
for index, entry in enumerate(value):
|
|
79
|
+
ref = entry.get("datasource") if isinstance(entry, dict) else entry
|
|
80
|
+
if not isinstance(ref, str) or not ref.strip():
|
|
81
|
+
raise ValueError(
|
|
82
|
+
f"AP_DATASOURCES entry {index + 1} must be a non-empty reference or an object with datasource"
|
|
83
|
+
)
|
|
84
|
+
if isinstance(entry, dict) and "mount_path" in entry:
|
|
85
|
+
path = entry["mount_path"]
|
|
86
|
+
if (
|
|
87
|
+
not isinstance(path, str)
|
|
88
|
+
or not path.startswith("/")
|
|
89
|
+
or posixpath.normpath(path).strip("/") == ""
|
|
90
|
+
or any(ord(c) < 32 for c in path)
|
|
91
|
+
):
|
|
92
|
+
raise ValueError(
|
|
93
|
+
f"AP_DATASOURCES entry {index + 1} mount_path must be an absolute non-root path"
|
|
94
|
+
)
|
|
95
|
+
return value
|
|
@@ -23,6 +23,38 @@ class EnvVarSpec:
|
|
|
23
23
|
|
|
24
24
|
|
|
25
25
|
ENV_VAR_SPECS: tuple[EnvVarSpec, ...] = (
|
|
26
|
+
EnvVarSpec(
|
|
27
|
+
name="AP_PROFILE",
|
|
28
|
+
description="Default Profile for CLI job create only when no template/profile is specified; --profile '' disables it. SDK uses explicit parameters.",
|
|
29
|
+
category="submission",
|
|
30
|
+
overridden_by=("--profile", "TEMPLATE"),
|
|
31
|
+
),
|
|
32
|
+
EnvVarSpec(
|
|
33
|
+
name="AP_DATASOURCES",
|
|
34
|
+
description="Default JSON array of additional mounts for CLI job create; explicit lists replace it and [] clears it. SDK uses explicit parameters.",
|
|
35
|
+
category="submission",
|
|
36
|
+
overridden_by=("--datasources",),
|
|
37
|
+
),
|
|
38
|
+
EnvVarSpec(
|
|
39
|
+
name="AP_QUEUE",
|
|
40
|
+
description="Default Queue for CLI job create; explicit queue options override it and an empty value disables it. SDK uses explicit parameters.",
|
|
41
|
+
category="submission",
|
|
42
|
+
overridden_by=("--queue-id", "--queue", "-q"),
|
|
43
|
+
),
|
|
44
|
+
EnvVarSpec(
|
|
45
|
+
name="AP_MODEL_BASE_URL_COLLECTION",
|
|
46
|
+
description="Default model URL pool for CLI job create (JSON array or comma-separated); explicit collection or file overrides it, including empty collection values. Existing batch/Profile rules apply. SDK uses explicit parameters.",
|
|
47
|
+
category="submission",
|
|
48
|
+
secret=True,
|
|
49
|
+
overridden_by=("--model-base-url-collection", "--model-base-url-file"),
|
|
50
|
+
),
|
|
51
|
+
EnvVarSpec(
|
|
52
|
+
name="AP_DATASET_ENVIRONMENT",
|
|
53
|
+
description="Dataset metadata environment: online, pre, or auto. Used by CLI entries supporting --environment, not job create --dataset.",
|
|
54
|
+
category="dataset",
|
|
55
|
+
default="online",
|
|
56
|
+
overridden_by=("--environment",),
|
|
57
|
+
),
|
|
26
58
|
EnvVarSpec(
|
|
27
59
|
name="AP_ARTIFACT_MANIFEST_WAIT_TIMEOUT",
|
|
28
60
|
description="Per-job artifact manifest wait budget in seconds, including polling and HTTP retries.",
|
|
@@ -327,9 +327,12 @@ def dfs_ls(
|
|
|
327
327
|
_configure(files, encoding_type, payer)
|
|
328
328
|
remote = files.resolve(path, directory=True)
|
|
329
329
|
file_candidate = (
|
|
330
|
-
files.resolve(path)
|
|
330
|
+
files.resolve(path)
|
|
331
|
+
if not recursive
|
|
332
|
+
and not directory
|
|
331
333
|
and remote.key != files.binding.prefix
|
|
332
|
-
and not _decoded_path(path, encoding_type).endswith("/")
|
|
334
|
+
and not _decoded_path(path, encoding_type).endswith("/")
|
|
335
|
+
else None
|
|
333
336
|
)
|
|
334
337
|
_emit(
|
|
335
338
|
files.list(
|
|
@@ -703,7 +706,9 @@ def dfs_hash(
|
|
|
703
706
|
|
|
704
707
|
absent = False
|
|
705
708
|
try:
|
|
706
|
-
result = hash_local(
|
|
709
|
+
result = hash_local(
|
|
710
|
+
local, algorithm, base64_output=base64_output, recursive=recursive
|
|
711
|
+
)
|
|
707
712
|
except FileNotFoundError:
|
|
708
713
|
# Only the actual local hash read is classified, never config/checkpoints.
|
|
709
714
|
absent = True
|