ap-client 0.6.0.dev0__tar.gz → 0.6.0.dev2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/PKG-INFO +1 -1
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/api.py +42 -2
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/cli.py +175 -26
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/exporter.py +267 -9
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/irepo_sdk.py +1 -1
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/managed_ak.py +16 -5
- ap_client-0.6.0.dev2/ap_client/share_api.py +229 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/pyproject.toml +1 -1
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/__init__.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/config.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/dataset_register.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/fs_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/instance_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/manifest_view.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/profile_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/repo_read_guard.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/split_publish.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/tbb.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev2}/ap_client/waiter.py +0 -0
|
@@ -709,13 +709,16 @@ class APIClient:
|
|
|
709
709
|
request_id: str,
|
|
710
710
|
duration_ms: float,
|
|
711
711
|
exc: Exception,
|
|
712
|
+
*,
|
|
713
|
+
error_detail: str | None = None,
|
|
712
714
|
) -> None:
|
|
715
|
+
detail = str(exc) if error_detail is None else error_detail
|
|
713
716
|
_write_stderr(
|
|
714
717
|
[
|
|
715
718
|
f"!! Error {method} {url}",
|
|
716
719
|
f" request_id={request_id}",
|
|
717
720
|
f" duration_ms={duration_ms:.1f}",
|
|
718
|
-
f" error={type(exc).__name__}: {
|
|
721
|
+
f" error={type(exc).__name__}: {detail}",
|
|
719
722
|
]
|
|
720
723
|
)
|
|
721
724
|
|
|
@@ -1122,8 +1125,15 @@ class APIClient:
|
|
|
1122
1125
|
benchmark: Optional[str] = None,
|
|
1123
1126
|
experiment: Optional[str] = None,
|
|
1124
1127
|
ak_ids: Optional[list[str]] = None,
|
|
1128
|
+
priority: Optional[str] = None,
|
|
1129
|
+
priority_weight: Optional[int] = None,
|
|
1125
1130
|
) -> dict:
|
|
1126
|
-
"""Create a group.
|
|
1131
|
+
"""Create a group with optional scheduling defaults for its Jobs.
|
|
1132
|
+
|
|
1133
|
+
Omitted priority fields use the server defaults: medium and weight 0.
|
|
1134
|
+
"""
|
|
1135
|
+
if priority is not None and priority not in {"high", "medium"}:
|
|
1136
|
+
raise ValueError("priority must be one of high/medium")
|
|
1127
1137
|
if ak_ids is not None and not isinstance(ak_ids, list):
|
|
1128
1138
|
raise ValueError("ak_ids must be a list")
|
|
1129
1139
|
references = list(ak_ids or [])
|
|
@@ -1161,6 +1171,10 @@ class APIClient:
|
|
|
1161
1171
|
body["queue_id"] = effective_queue_id
|
|
1162
1172
|
if max_concurrency is not None:
|
|
1163
1173
|
body["max_concurrency"] = max_concurrency
|
|
1174
|
+
if priority is not None:
|
|
1175
|
+
body["priority"] = priority
|
|
1176
|
+
if priority_weight is not None:
|
|
1177
|
+
body["priority_weight"] = priority_weight
|
|
1164
1178
|
if eval_config is not None:
|
|
1165
1179
|
body["eval_config"] = eval_config
|
|
1166
1180
|
if model_base_url_collection is not None:
|
|
@@ -1878,9 +1892,15 @@ class APIClient:
|
|
|
1878
1892
|
meta_job_type: Optional[str] = None,
|
|
1879
1893
|
meta_teacher_direction: Optional[str] = None,
|
|
1880
1894
|
meta_train_stage: Optional[str] = None,
|
|
1895
|
+
benchmark: Optional[str] = None,
|
|
1896
|
+
priority: Optional[str] = None,
|
|
1897
|
+
priority_weight: Optional[int] = None,
|
|
1898
|
+
priority_update_mode: Optional[str] = None,
|
|
1881
1899
|
) -> dict:
|
|
1882
1900
|
"""Update group properties.
|
|
1883
1901
|
|
|
1902
|
+
``benchmark`` is omitted when None; pass an empty string to clear it.
|
|
1903
|
+
|
|
1884
1904
|
``tags``(`key:value` 数组)在客户端先与 Group 现有 tags 合并,再以
|
|
1885
1905
|
全量替换语义提交(服务端 PATCH 是整体替换);同名 key 覆盖,未提及
|
|
1886
1906
|
的 key 保留。服务端会从 tags 同步 meta_* 等派生字段。
|
|
@@ -1888,7 +1908,19 @@ class APIClient:
|
|
|
1888
1908
|
``meta_*`` 独立属性为部分更新(传了才改,不支持清除);新服务端
|
|
1889
1909
|
优先读属性,老服务端忽略属性——因此过渡期内同时也会将属性合并进
|
|
1890
1910
|
tags 轨(双写)。
|
|
1911
|
+
|
|
1912
|
+
Priority changes update the Group defaults. The default existing mode
|
|
1913
|
+
also updates all current Jobs, including PostProcess; incremental mode
|
|
1914
|
+
changes only the defaults for future Jobs. Updating priority alone uses
|
|
1915
|
+
weight 0; updating only weight preserves the current priority tier.
|
|
1891
1916
|
"""
|
|
1917
|
+
if priority is not None and priority not in {"high", "medium"}:
|
|
1918
|
+
raise ValueError("priority must be one of high/medium")
|
|
1919
|
+
if priority_update_mode is not None:
|
|
1920
|
+
if priority_update_mode not in {"existing", "incremental"}:
|
|
1921
|
+
raise ValueError("priority_update_mode must be one of existing/incremental")
|
|
1922
|
+
if priority is None and priority_weight is None:
|
|
1923
|
+
raise ValueError("priority_update_mode requires priority or priority_weight")
|
|
1892
1924
|
body: dict = {}
|
|
1893
1925
|
|
|
1894
1926
|
if max_concurrency is not None:
|
|
@@ -1903,6 +1935,14 @@ class APIClient:
|
|
|
1903
1935
|
body["model"] = model
|
|
1904
1936
|
if scaffold is not None:
|
|
1905
1937
|
body["scaffold"] = scaffold
|
|
1938
|
+
if benchmark is not None:
|
|
1939
|
+
body["benchmark"] = benchmark
|
|
1940
|
+
if priority is not None:
|
|
1941
|
+
body["priority"] = priority
|
|
1942
|
+
if priority_weight is not None:
|
|
1943
|
+
body["priority_weight"] = priority_weight
|
|
1944
|
+
if priority_update_mode is not None:
|
|
1945
|
+
body["priority_update_mode"] = priority_update_mode
|
|
1906
1946
|
meta_attr_tags = _merge_meta_tags(
|
|
1907
1947
|
None,
|
|
1908
1948
|
meta_model=meta_model,
|
|
@@ -359,6 +359,22 @@ def _emit_error(message: str) -> None:
|
|
|
359
359
|
error_console.print(f"[red]error: {message}[/]")
|
|
360
360
|
|
|
361
361
|
|
|
362
|
+
def _read_client(share_token: Optional[str] = None, group_id: Optional[str] = None):
|
|
363
|
+
if group_id and not share_token:
|
|
364
|
+
raise typer.BadParameter("--group-id requires --share-token")
|
|
365
|
+
client = get_client()
|
|
366
|
+
if share_token is None:
|
|
367
|
+
return client
|
|
368
|
+
from ap_client.share_api import ShareAPIClient
|
|
369
|
+
|
|
370
|
+
# Typer verbose tracebacks must not render the share token in locals.
|
|
371
|
+
app.pretty_exceptions_show_locals = False
|
|
372
|
+
try:
|
|
373
|
+
return ShareAPIClient.from_client(client, share_token, group_id=group_id)
|
|
374
|
+
except ValueError as exc:
|
|
375
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
376
|
+
|
|
377
|
+
|
|
362
378
|
def _reject_account_pool_template_credential_cli(
|
|
363
379
|
account_pool: Optional[str],
|
|
364
380
|
*credential_types: Any,
|
|
@@ -626,7 +642,18 @@ def _print_formatted(data: dict, output_format: str) -> None:
|
|
|
626
642
|
import yaml
|
|
627
643
|
except ImportError as exc:
|
|
628
644
|
raise typer.BadParameter("--format yaml requires PyYAML to be installed") from exc
|
|
629
|
-
|
|
645
|
+
|
|
646
|
+
class LosslessDumper(yaml.SafeDumper):
|
|
647
|
+
pass
|
|
648
|
+
|
|
649
|
+
def represent_string(dumper, value):
|
|
650
|
+
# YAML normalizes literal NEL/line separators in unquoted scalars.
|
|
651
|
+
# Double quotes make the emitter escape them while keeping Chinese readable.
|
|
652
|
+
style = '"' if any(character in value for character in "\x85\u2028\u2029") else None
|
|
653
|
+
return dumper.represent_scalar("tag:yaml.org,2002:str", value, style=style)
|
|
654
|
+
|
|
655
|
+
LosslessDumper.add_representer(str, represent_string)
|
|
656
|
+
typer.echo(yaml.dump(data, Dumper=LosslessDumper, allow_unicode=True, sort_keys=False))
|
|
630
657
|
return
|
|
631
658
|
_print_json(data)
|
|
632
659
|
|
|
@@ -3203,12 +3230,12 @@ def job_create(
|
|
|
3203
3230
|
priority: Optional[str] = typer.Option(
|
|
3204
3231
|
None,
|
|
3205
3232
|
"--priority",
|
|
3206
|
-
help="Scheduling
|
|
3233
|
+
help="Scheduling tier: high or medium. Omit both priority flags to inherit Group defaults; otherwise an omitted tier is medium",
|
|
3207
3234
|
),
|
|
3208
3235
|
priority_weight: Optional[int] = typer.Option(
|
|
3209
3236
|
None,
|
|
3210
3237
|
"--priority-weight",
|
|
3211
|
-
help="
|
|
3238
|
+
help="Priority weight, 0-10. Omit both priority flags to inherit Group defaults; otherwise an omitted weight is 0",
|
|
3212
3239
|
),
|
|
3213
3240
|
resource_mode: Optional[str] = typer.Option(
|
|
3214
3241
|
None,
|
|
@@ -4427,6 +4454,9 @@ def job_create(
|
|
|
4427
4454
|
@job_app.command("get")
|
|
4428
4455
|
def job_get(
|
|
4429
4456
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4457
|
+
share_token: Optional[str] = typer.Option(
|
|
4458
|
+
None, "--share-token", help="Read with a share token"
|
|
4459
|
+
),
|
|
4430
4460
|
include_params: bool = typer.Option(
|
|
4431
4461
|
False,
|
|
4432
4462
|
"--include-params",
|
|
@@ -4463,7 +4493,7 @@ def job_get(
|
|
|
4463
4493
|
include_spec = include_spec or _field_path_requires_spec(field)
|
|
4464
4494
|
if field == "job_spec_data.spec_yaml":
|
|
4465
4495
|
spec_format = "yaml"
|
|
4466
|
-
client =
|
|
4496
|
+
client = _read_client(share_token)
|
|
4467
4497
|
result = client.get_job(
|
|
4468
4498
|
job_id,
|
|
4469
4499
|
include_params=include_params,
|
|
@@ -4489,6 +4519,9 @@ def job_get(
|
|
|
4489
4519
|
@job_app.command("containers")
|
|
4490
4520
|
def job_containers(
|
|
4491
4521
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4522
|
+
share_token: Optional[str] = typer.Option(
|
|
4523
|
+
None, "--share-token", help="Read with a share token"
|
|
4524
|
+
),
|
|
4492
4525
|
output_format: str = typer.Option(
|
|
4493
4526
|
None,
|
|
4494
4527
|
"--format",
|
|
@@ -4497,7 +4530,7 @@ def job_containers(
|
|
|
4497
4530
|
):
|
|
4498
4531
|
"""List container names for a job."""
|
|
4499
4532
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
4500
|
-
client =
|
|
4533
|
+
client = _read_client(share_token)
|
|
4501
4534
|
result = client.get_job_containers(job_id)
|
|
4502
4535
|
if output_format == "table":
|
|
4503
4536
|
_print_records_table(result)
|
|
@@ -4559,6 +4592,9 @@ def _print_tail_logs(logs, tail: int) -> None:
|
|
|
4559
4592
|
@job_app.command("logs")
|
|
4560
4593
|
def job_logs(
|
|
4561
4594
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4595
|
+
share_token: Optional[str] = typer.Option(
|
|
4596
|
+
None, "--share-token", help="Read with a share token"
|
|
4597
|
+
),
|
|
4562
4598
|
container: Optional[str] = typer.Option("main", "--container", "-c", help="Container name"),
|
|
4563
4599
|
tail: Optional[int] = typer.Option(None, "--tail", "-n", help="Show only the last N log lines"),
|
|
4564
4600
|
follow: bool = typer.Option(
|
|
@@ -4570,7 +4606,7 @@ def job_logs(
|
|
|
4570
4606
|
),
|
|
4571
4607
|
):
|
|
4572
4608
|
"""Show job logs."""
|
|
4573
|
-
client =
|
|
4609
|
+
client = _read_client(share_token)
|
|
4574
4610
|
|
|
4575
4611
|
if tail is not None:
|
|
4576
4612
|
if tail <= 0:
|
|
@@ -4634,6 +4670,9 @@ def job_logs(
|
|
|
4634
4670
|
@job_app.command("events")
|
|
4635
4671
|
def job_events(
|
|
4636
4672
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4673
|
+
share_token: Optional[str] = typer.Option(
|
|
4674
|
+
None, "--share-token", help="Read with a share token"
|
|
4675
|
+
),
|
|
4637
4676
|
container: Optional[str] = typer.Option(None, "--container", "-c", help="Container name"),
|
|
4638
4677
|
output_format: str = typer.Option(
|
|
4639
4678
|
None,
|
|
@@ -4643,7 +4682,7 @@ def job_events(
|
|
|
4643
4682
|
):
|
|
4644
4683
|
"""Show job events."""
|
|
4645
4684
|
output_format = _normalize_output_format(output_format)
|
|
4646
|
-
client =
|
|
4685
|
+
client = _read_client(share_token)
|
|
4647
4686
|
result = client.get_job_events(job_id, container=container)
|
|
4648
4687
|
if output_format == "plain":
|
|
4649
4688
|
_print_job_events_plain(result)
|
|
@@ -4654,6 +4693,9 @@ def job_events(
|
|
|
4654
4693
|
@job_app.command("metrics")
|
|
4655
4694
|
def job_metrics(
|
|
4656
4695
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4696
|
+
share_token: Optional[str] = typer.Option(
|
|
4697
|
+
None, "--share-token", help="Read with a share token"
|
|
4698
|
+
),
|
|
4657
4699
|
output_format: str = typer.Option(
|
|
4658
4700
|
None,
|
|
4659
4701
|
"--format",
|
|
@@ -4662,7 +4704,7 @@ def job_metrics(
|
|
|
4662
4704
|
):
|
|
4663
4705
|
"""Show job metrics."""
|
|
4664
4706
|
output_format = _normalize_output_format(output_format)
|
|
4665
|
-
client =
|
|
4707
|
+
client = _read_client(share_token)
|
|
4666
4708
|
result = client.get_job_metrics(job_id)
|
|
4667
4709
|
if output_format == "plain":
|
|
4668
4710
|
_print_json(result)
|
|
@@ -4859,6 +4901,9 @@ def job_retry(
|
|
|
4859
4901
|
@job_app.command("artifacts")
|
|
4860
4902
|
def job_artifacts(
|
|
4861
4903
|
job_ids: str = typer.Argument(..., help="Job IDs (comma-separated for multiple)"),
|
|
4904
|
+
share_token: Optional[str] = typer.Option(
|
|
4905
|
+
None, "--share-token", help="Read with a share token"
|
|
4906
|
+
),
|
|
4862
4907
|
output_format: str = typer.Option(
|
|
4863
4908
|
None,
|
|
4864
4909
|
"--format",
|
|
@@ -4867,7 +4912,7 @@ def job_artifacts(
|
|
|
4867
4912
|
):
|
|
4868
4913
|
"""Show job artifact download links."""
|
|
4869
4914
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
4870
|
-
client =
|
|
4915
|
+
client = _read_client(share_token)
|
|
4871
4916
|
ids = [jid.strip() for jid in job_ids.split(",")]
|
|
4872
4917
|
results = client.get_job_artifacts(ids)
|
|
4873
4918
|
if output_format == "table":
|
|
@@ -4907,6 +4952,9 @@ def _export_channel_reporter(emit):
|
|
|
4907
4952
|
@job_app.command("export")
|
|
4908
4953
|
def job_export(
|
|
4909
4954
|
job_id: str = typer.Argument(..., help="Job ID or comma-separated Job IDs"),
|
|
4955
|
+
share_token: Optional[str] = typer.Option(
|
|
4956
|
+
None, "--share-token", help="Export with a share token"
|
|
4957
|
+
),
|
|
4910
4958
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Export directory"),
|
|
4911
4959
|
workers: int = typer.Option(
|
|
4912
4960
|
1, "--workers", "-w", min=1, help="Number of concurrent export workers"
|
|
@@ -4929,7 +4977,7 @@ def job_export(
|
|
|
4929
4977
|
ids = list(dict.fromkeys(value.strip() for value in job_id.split(",")))
|
|
4930
4978
|
if not all(ids):
|
|
4931
4979
|
raise typer.BadParameter("Job IDs must not be empty", param_hint="job_id")
|
|
4932
|
-
client =
|
|
4980
|
+
client = _read_client(share_token)
|
|
4933
4981
|
if len(ids) > 1:
|
|
4934
4982
|
report, channels = _export_channel_reporter(print)
|
|
4935
4983
|
dest, summary = export_jobs(
|
|
@@ -5420,6 +5468,18 @@ def group_create(
|
|
|
5420
5468
|
min=1,
|
|
5421
5469
|
help="Maximum concurrent jobs. Server default: 50.",
|
|
5422
5470
|
),
|
|
5471
|
+
priority: Optional[str] = typer.Option(
|
|
5472
|
+
None,
|
|
5473
|
+
"--priority",
|
|
5474
|
+
help="Default priority inherited by Jobs: high or medium (server default: medium)",
|
|
5475
|
+
),
|
|
5476
|
+
priority_weight: Optional[int] = typer.Option(
|
|
5477
|
+
None,
|
|
5478
|
+
"--priority-weight",
|
|
5479
|
+
min=0,
|
|
5480
|
+
max=10,
|
|
5481
|
+
help="Default priority weight inherited by Jobs, 0-10 (server default: 0)",
|
|
5482
|
+
),
|
|
5423
5483
|
eval_config: Optional[str] = typer.Option(
|
|
5424
5484
|
None, "--eval-config", help="Evaluation config (JSON)"
|
|
5425
5485
|
),
|
|
@@ -5532,6 +5592,8 @@ def group_create(
|
|
|
5532
5592
|
):
|
|
5533
5593
|
"""Create a Group."""
|
|
5534
5594
|
output_format = _normalize_output_format(output_format)
|
|
5595
|
+
if priority is not None and priority not in {"high", "medium"}:
|
|
5596
|
+
raise typer.BadParameter("--priority must be one of high/medium")
|
|
5535
5597
|
try:
|
|
5536
5598
|
for reference in ak_id or []:
|
|
5537
5599
|
validate_ak_id(reference)
|
|
@@ -5613,6 +5675,8 @@ def group_create(
|
|
|
5613
5675
|
checkpoint=checkpoint_dict,
|
|
5614
5676
|
benchmark=benchmark,
|
|
5615
5677
|
experiment=experiment,
|
|
5678
|
+
**({"priority": priority} if priority is not None else {}),
|
|
5679
|
+
**({"priority_weight": priority_weight} if priority_weight is not None else {}),
|
|
5616
5680
|
)
|
|
5617
5681
|
except ManagedAKProtocolError as exc:
|
|
5618
5682
|
_emit_error(str(exc))
|
|
@@ -5625,6 +5689,10 @@ def group_create(
|
|
|
5625
5689
|
print(f" ak_ids: {response_ak_ids(result)}")
|
|
5626
5690
|
if result.get("queue_id") is not None:
|
|
5627
5691
|
print(f" queue_id: {result.get('queue_id')}")
|
|
5692
|
+
if result.get("priority") is not None:
|
|
5693
|
+
print(f" priority: {result.get('priority')}")
|
|
5694
|
+
if result.get("priority_weight") is not None:
|
|
5695
|
+
print(f" priority_weight: {result.get('priority_weight')}")
|
|
5628
5696
|
if result.get("template") is not None:
|
|
5629
5697
|
print(f" template: {result.get('template')}")
|
|
5630
5698
|
if result.get("agenthub_revision") is not None:
|
|
@@ -5744,6 +5812,9 @@ def group_list(
|
|
|
5744
5812
|
@group_app.command("get")
|
|
5745
5813
|
def group_get(
|
|
5746
5814
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
5815
|
+
share_token: Optional[str] = typer.Option(
|
|
5816
|
+
None, "--share-token", help="Read with a share token"
|
|
5817
|
+
),
|
|
5747
5818
|
output_format: str = typer.Option(
|
|
5748
5819
|
None,
|
|
5749
5820
|
"--format",
|
|
@@ -5752,7 +5823,7 @@ def group_get(
|
|
|
5752
5823
|
):
|
|
5753
5824
|
"""Show Group details."""
|
|
5754
5825
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
5755
|
-
client =
|
|
5826
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
5756
5827
|
result = client.get_group(group_id)
|
|
5757
5828
|
if output_format == "table":
|
|
5758
5829
|
_print_detail(result)
|
|
@@ -6026,6 +6097,9 @@ def group_update(
|
|
|
6026
6097
|
scaffold: Optional[str] = typer.Option(
|
|
6027
6098
|
None, "--scaffold", "-s", help="Scaffold/agent name (benchmark perspective)"
|
|
6028
6099
|
),
|
|
6100
|
+
benchmark: Optional[str] = typer.Option(
|
|
6101
|
+
None, "--benchmark", help="Benchmark name; pass an empty string to clear"
|
|
6102
|
+
),
|
|
6029
6103
|
tags: Optional[str] = typer.Option(
|
|
6030
6104
|
None,
|
|
6031
6105
|
"--tags",
|
|
@@ -6045,14 +6119,40 @@ def group_update(
|
|
|
6045
6119
|
"--meta-train-stage",
|
|
6046
6120
|
help="Set meta_train_stage tag (options: ap meta-train-stage list)",
|
|
6047
6121
|
),
|
|
6122
|
+
priority: Optional[str] = typer.Option(
|
|
6123
|
+
None,
|
|
6124
|
+
"--priority",
|
|
6125
|
+
help="Group priority default: high or medium; --priority alone sets weight to 0",
|
|
6126
|
+
),
|
|
6127
|
+
priority_weight: Optional[int] = typer.Option(
|
|
6128
|
+
None,
|
|
6129
|
+
"--priority-weight",
|
|
6130
|
+
min=0,
|
|
6131
|
+
max=10,
|
|
6132
|
+
help="Group priority weight default, 0-10; weight alone preserves the current tier",
|
|
6133
|
+
),
|
|
6134
|
+
priority_update_mode: Optional[str] = typer.Option(
|
|
6135
|
+
None,
|
|
6136
|
+
"--priority-update-mode",
|
|
6137
|
+
help="existing updates defaults and all current Jobs (including PostProcess); incremental updates only defaults (server default: existing)",
|
|
6138
|
+
),
|
|
6048
6139
|
output_format: str = typer.Option(
|
|
6049
6140
|
None,
|
|
6050
6141
|
"--format",
|
|
6051
6142
|
help="Output format: plain/table/json/yaml (default: AP_FORMAT or command default)",
|
|
6052
6143
|
),
|
|
6053
6144
|
):
|
|
6054
|
-
"""Update Group properties."""
|
|
6145
|
+
"""Update Group properties and scheduling defaults inherited by Jobs."""
|
|
6055
6146
|
output_format = _normalize_output_format(output_format)
|
|
6147
|
+
if priority is not None and priority not in {"high", "medium"}:
|
|
6148
|
+
raise typer.BadParameter("--priority must be one of high/medium")
|
|
6149
|
+
if priority_update_mode is not None:
|
|
6150
|
+
if priority_update_mode not in {"existing", "incremental"}:
|
|
6151
|
+
raise typer.BadParameter("--priority-update-mode must be one of existing/incremental")
|
|
6152
|
+
if priority is None and priority_weight is None:
|
|
6153
|
+
raise typer.BadParameter(
|
|
6154
|
+
"--priority-update-mode requires --priority or --priority-weight"
|
|
6155
|
+
)
|
|
6056
6156
|
eval_config_dict = _parse_json_object(eval_config, "--eval-config")
|
|
6057
6157
|
model_base_url_collection_list = _parse_model_base_url_collection_option(
|
|
6058
6158
|
model_base_url_collection, model_base_url_file
|
|
@@ -6087,7 +6187,10 @@ def group_update(
|
|
|
6087
6187
|
dataset,
|
|
6088
6188
|
model_name,
|
|
6089
6189
|
scaffold,
|
|
6190
|
+
benchmark,
|
|
6090
6191
|
tags_list,
|
|
6192
|
+
priority,
|
|
6193
|
+
priority_weight,
|
|
6091
6194
|
)
|
|
6092
6195
|
):
|
|
6093
6196
|
raise typer.BadParameter("At least one field must be provided to update")
|
|
@@ -6101,7 +6204,15 @@ def group_update(
|
|
|
6101
6204
|
dataset=dataset,
|
|
6102
6205
|
model=model_name,
|
|
6103
6206
|
scaffold=scaffold,
|
|
6207
|
+
**({"benchmark": benchmark} if benchmark is not None else {}),
|
|
6104
6208
|
**({"tags": tags_list} if tags_list is not None else {}),
|
|
6209
|
+
**({"priority": priority} if priority is not None else {}),
|
|
6210
|
+
**({"priority_weight": priority_weight} if priority_weight is not None else {}),
|
|
6211
|
+
**(
|
|
6212
|
+
{"priority_update_mode": priority_update_mode}
|
|
6213
|
+
if priority_update_mode is not None
|
|
6214
|
+
else {}
|
|
6215
|
+
),
|
|
6105
6216
|
)
|
|
6106
6217
|
if output_format == "plain":
|
|
6107
6218
|
_print_group_action_plain(
|
|
@@ -6115,6 +6226,9 @@ def group_update(
|
|
|
6115
6226
|
"dataset",
|
|
6116
6227
|
"model",
|
|
6117
6228
|
"scaffold",
|
|
6229
|
+
"benchmark",
|
|
6230
|
+
"priority",
|
|
6231
|
+
"priority_weight",
|
|
6118
6232
|
"updated_at",
|
|
6119
6233
|
],
|
|
6120
6234
|
)
|
|
@@ -6232,6 +6346,9 @@ def group_resume(
|
|
|
6232
6346
|
@group_app.command("jobs")
|
|
6233
6347
|
def group_jobs(
|
|
6234
6348
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6349
|
+
share_token: Optional[str] = typer.Option(
|
|
6350
|
+
None, "--share-token", help="Read with a share token"
|
|
6351
|
+
),
|
|
6235
6352
|
tag: Optional[list[str]] = typer.Option(
|
|
6236
6353
|
None, "--tag", help="Filter by tag (repeatable; multiple tags are combined with AND)"
|
|
6237
6354
|
),
|
|
@@ -6270,17 +6387,36 @@ def group_jobs(
|
|
|
6270
6387
|
if cursor and skip > 0:
|
|
6271
6388
|
raise typer.BadParameter("--cursor cannot be used with --skip greater than 0")
|
|
6272
6389
|
cursor_request = bool(next_token) or cursor
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6390
|
+
if share_token is not None:
|
|
6391
|
+
unsupported = (
|
|
6392
|
+
("--tag", bool(tag)),
|
|
6393
|
+
("--include-post-process", include_post_process),
|
|
6394
|
+
("--next-token", bool(next_token)),
|
|
6395
|
+
("--cursor", cursor),
|
|
6396
|
+
("--no-total", no_total),
|
|
6397
|
+
)
|
|
6398
|
+
for option, used in unsupported:
|
|
6399
|
+
if used:
|
|
6400
|
+
_emit_error(f"{option} is not supported with --share-token")
|
|
6401
|
+
raise typer.Exit(1)
|
|
6402
|
+
if skip < 0 or not 1 <= limit <= 500:
|
|
6403
|
+
raise typer.BadParameter(
|
|
6404
|
+
"share pagination requires --skip >= 0 and 1 <= --limit <= 500"
|
|
6405
|
+
)
|
|
6406
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6407
|
+
result = client.list_jobs(group_id=group_id, skip=skip, limit=limit)
|
|
6408
|
+
else:
|
|
6409
|
+
client = get_client()
|
|
6410
|
+
result = client.list_group_jobs(
|
|
6411
|
+
group_id,
|
|
6412
|
+
tag=tag,
|
|
6413
|
+
include_post_process=include_post_process,
|
|
6414
|
+
skip=skip,
|
|
6415
|
+
limit=limit,
|
|
6416
|
+
**({"next_token": next_token} if next_token else {}),
|
|
6417
|
+
**({"pagination": "cursor"} if cursor_request else {}),
|
|
6418
|
+
include_total=not no_total,
|
|
6419
|
+
)
|
|
6284
6420
|
if output_format == "table":
|
|
6285
6421
|
_print_job_list_plain(result, rich=True)
|
|
6286
6422
|
elif output_format == "plain":
|
|
@@ -6295,6 +6431,9 @@ def group_jobs(
|
|
|
6295
6431
|
@group_app.command("stats")
|
|
6296
6432
|
def group_stats(
|
|
6297
6433
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6434
|
+
share_token: Optional[str] = typer.Option(
|
|
6435
|
+
None, "--share-token", help="Read with a share token"
|
|
6436
|
+
),
|
|
6298
6437
|
output_format: str = typer.Option(
|
|
6299
6438
|
None,
|
|
6300
6439
|
"--format",
|
|
@@ -6303,7 +6442,7 @@ def group_stats(
|
|
|
6303
6442
|
):
|
|
6304
6443
|
"""Show Group progress."""
|
|
6305
6444
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
6306
|
-
client =
|
|
6445
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6307
6446
|
result = client.get_group_stats(group_id)
|
|
6308
6447
|
if output_format == "table":
|
|
6309
6448
|
_print_detail(result)
|
|
@@ -6314,6 +6453,9 @@ def group_stats(
|
|
|
6314
6453
|
@group_app.command("eval")
|
|
6315
6454
|
def group_eval(
|
|
6316
6455
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6456
|
+
share_token: Optional[str] = typer.Option(
|
|
6457
|
+
None, "--share-token", help="Read with a share token"
|
|
6458
|
+
),
|
|
6317
6459
|
summary_only: bool = typer.Option(
|
|
6318
6460
|
False,
|
|
6319
6461
|
"--summary-only",
|
|
@@ -6327,7 +6469,7 @@ def group_eval(
|
|
|
6327
6469
|
):
|
|
6328
6470
|
"""Show Group aggregated evaluation results."""
|
|
6329
6471
|
output_format = _normalize_output_format(output_format)
|
|
6330
|
-
client =
|
|
6472
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6331
6473
|
result = (
|
|
6332
6474
|
client.get_group_eval_summary(group_id) if summary_only else client.get_group_eval(group_id)
|
|
6333
6475
|
)
|
|
@@ -6440,6 +6582,9 @@ def group_access_list(group_id: str = typer.Argument(..., help="Group ID")):
|
|
|
6440
6582
|
@group_app.command("export")
|
|
6441
6583
|
def group_export(
|
|
6442
6584
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6585
|
+
share_token: Optional[str] = typer.Option(
|
|
6586
|
+
None, "--share-token", help="Export with a share token"
|
|
6587
|
+
),
|
|
6443
6588
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Export directory"),
|
|
6444
6589
|
workers: int = typer.Option(
|
|
6445
6590
|
4, "--workers", "-w", min=1, help="Number of concurrent export workers"
|
|
@@ -6481,7 +6626,11 @@ def group_export(
|
|
|
6481
6626
|
_emit_error("--include-files/--exclude-files cannot be used with --no-extract-artifacts")
|
|
6482
6627
|
raise typer.Exit(1)
|
|
6483
6628
|
|
|
6484
|
-
|
|
6629
|
+
if share_token is not None and (include_file_selectors or exclude_file_selectors):
|
|
6630
|
+
_emit_error("--include-files/--exclude-files are not supported with --share-token")
|
|
6631
|
+
raise typer.Exit(1)
|
|
6632
|
+
|
|
6633
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6485
6634
|
from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn
|
|
6486
6635
|
|
|
6487
6636
|
with Progress(
|
|
@@ -6,16 +6,20 @@ import concurrent.futures
|
|
|
6
6
|
import json
|
|
7
7
|
import logging
|
|
8
8
|
import os
|
|
9
|
+
import re
|
|
9
10
|
import shutil
|
|
10
11
|
import sys
|
|
12
|
+
import tarfile
|
|
11
13
|
import tempfile
|
|
12
14
|
import threading
|
|
13
15
|
import uuid
|
|
14
16
|
from contextlib import nullcontext
|
|
15
17
|
from dataclasses import dataclass, field
|
|
16
|
-
from pathlib import Path
|
|
18
|
+
from pathlib import Path, PurePosixPath
|
|
17
19
|
from typing import TYPE_CHECKING, Callable, Optional
|
|
18
20
|
|
|
21
|
+
import requests
|
|
22
|
+
|
|
19
23
|
if TYPE_CHECKING:
|
|
20
24
|
from irepo_artifact import ArtifactClient, DownloadResult
|
|
21
25
|
|
|
@@ -30,6 +34,10 @@ StageCallback = Callable[[str], None]
|
|
|
30
34
|
ResultCallback = Callable[[str, str, Optional[str]], None]
|
|
31
35
|
|
|
32
36
|
|
|
37
|
+
def _is_share_client(client: APIClient) -> bool:
|
|
38
|
+
return getattr(client, "is_share", False) is True
|
|
39
|
+
|
|
40
|
+
|
|
33
41
|
@dataclass
|
|
34
42
|
class ExportSummary:
|
|
35
43
|
"""Summary of a group export operation."""
|
|
@@ -109,6 +117,30 @@ def export_jobs(
|
|
|
109
117
|
parent = parent.parent
|
|
110
118
|
summary = BatchExportSummary()
|
|
111
119
|
ids = list(dict.fromkeys(job_ids))
|
|
120
|
+
if _is_share_client(client):
|
|
121
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
122
|
+
for job_id in ids:
|
|
123
|
+
try:
|
|
124
|
+
skipped = _export_job_into_dir(
|
|
125
|
+
client,
|
|
126
|
+
job_id,
|
|
127
|
+
root / "jobs" / job_id,
|
|
128
|
+
timeout=timeout,
|
|
129
|
+
result_callback=result_callback,
|
|
130
|
+
include_logs=include_logs,
|
|
131
|
+
include_events=include_events,
|
|
132
|
+
extract_artifacts=extract_artifacts,
|
|
133
|
+
skip_no_artifact=True,
|
|
134
|
+
)
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
summary.failures[job_id] = str(exc)
|
|
137
|
+
else:
|
|
138
|
+
if skipped is not None:
|
|
139
|
+
summary.skipped += 1
|
|
140
|
+
summary.skipped_job_names.append(skipped)
|
|
141
|
+
else:
|
|
142
|
+
summary.downloaded += 1
|
|
143
|
+
return root, summary
|
|
112
144
|
with tempfile.TemporaryDirectory(prefix=".artifact-export-", dir=parent) as temporary:
|
|
113
145
|
results = _new_artifact_client(client, timeout=timeout).download_many(
|
|
114
146
|
ids,
|
|
@@ -166,6 +198,8 @@ def export_group(
|
|
|
166
198
|
exclude_files = _normalize_file_selectors(exclude_files)
|
|
167
199
|
if not extract_artifacts and (include_files or exclude_files):
|
|
168
200
|
raise ValueError("file filters cannot be used when artifact extraction is disabled")
|
|
201
|
+
if _is_share_client(client) and (include_files or exclude_files):
|
|
202
|
+
raise ValueError("file filters are not supported with a share token")
|
|
169
203
|
|
|
170
204
|
dest = Path(output_dir) if output_dir else default_group_export_dir(group_id)
|
|
171
205
|
dest.mkdir(parents=True, exist_ok=True)
|
|
@@ -176,7 +210,8 @@ def export_group(
|
|
|
176
210
|
group_eval = client.get_group_eval(group_id)
|
|
177
211
|
_write_json(dest / "eval.json", group_eval)
|
|
178
212
|
_write_json(dest / "eval_summary.json", group_eval.get("summary", {}))
|
|
179
|
-
|
|
213
|
+
task_key = "per_task" if _is_share_client(client) else "results"
|
|
214
|
+
_write_json(dest / "eval_tasks.json", group_eval.get(task_key, []))
|
|
180
215
|
except Exception:
|
|
181
216
|
# Some groups are non-eval workloads, or older servers may not expose eval endpoints.
|
|
182
217
|
pass
|
|
@@ -217,7 +252,7 @@ def export_group(
|
|
|
217
252
|
client,
|
|
218
253
|
job_id,
|
|
219
254
|
jobs_dir / job_id,
|
|
220
|
-
job=job,
|
|
255
|
+
job=None if _is_share_client(client) else job,
|
|
221
256
|
stage_callback=lambda stage, current_job_id=job_id: _notify(current_job_id, stage),
|
|
222
257
|
timeout=timeout,
|
|
223
258
|
result_callback=result_callback,
|
|
@@ -243,7 +278,7 @@ def export_group(
|
|
|
243
278
|
future_map = {
|
|
244
279
|
executor.submit(
|
|
245
280
|
_export_job_with_fresh_client,
|
|
246
|
-
client
|
|
281
|
+
client,
|
|
247
282
|
job,
|
|
248
283
|
jobs_dir / job["job_id"],
|
|
249
284
|
lambda stage, current_job_id=job["job_id"]: _notify(current_job_id, stage),
|
|
@@ -319,7 +354,7 @@ def _list_group_jobs(client: APIClient, group_id: str) -> list[dict]:
|
|
|
319
354
|
|
|
320
355
|
|
|
321
356
|
def _export_job_with_fresh_client(
|
|
322
|
-
|
|
357
|
+
client: APIClient,
|
|
323
358
|
job: dict,
|
|
324
359
|
dest: Path,
|
|
325
360
|
stage_callback: Optional[StageCallback] = None,
|
|
@@ -334,12 +369,19 @@ def _export_job_with_fresh_client(
|
|
|
334
369
|
exclude_files: Optional[list[str]] = None,
|
|
335
370
|
) -> Optional[str]:
|
|
336
371
|
"""Export one job with a dedicated client/session for thread safety."""
|
|
337
|
-
|
|
372
|
+
if _is_share_client(client):
|
|
373
|
+
from .share_api import ShareAPIClient
|
|
374
|
+
|
|
375
|
+
worker_client = ShareAPIClient(
|
|
376
|
+
client.config, client.share_token, group_id=client.share_group_id
|
|
377
|
+
)
|
|
378
|
+
else:
|
|
379
|
+
worker_client = APIClient(client.config)
|
|
338
380
|
return _export_job_into_dir(
|
|
339
381
|
worker_client,
|
|
340
382
|
job["job_id"],
|
|
341
383
|
dest,
|
|
342
|
-
job=job,
|
|
384
|
+
job=None if _is_share_client(client) else job,
|
|
343
385
|
stage_callback=stage_callback,
|
|
344
386
|
timeout=timeout,
|
|
345
387
|
result_callback=result_callback,
|
|
@@ -371,9 +413,22 @@ def _export_job_into_dir(
|
|
|
371
413
|
) -> Optional[str]:
|
|
372
414
|
"""Export SDK-managed artifacts alongside AP metadata.
|
|
373
415
|
|
|
374
|
-
|
|
375
|
-
|
|
416
|
+
Ordinary exports use the SDK for artifacts. Share exports use the share
|
|
417
|
+
endpoint's signed archive URL because the SDK uses authenticated routes.
|
|
376
418
|
"""
|
|
419
|
+
if _is_share_client(client):
|
|
420
|
+
return _export_shared_job_into_dir(
|
|
421
|
+
client,
|
|
422
|
+
job_id,
|
|
423
|
+
dest,
|
|
424
|
+
stage_callback=stage_callback,
|
|
425
|
+
timeout=timeout,
|
|
426
|
+
result_callback=result_callback,
|
|
427
|
+
include_logs=include_logs,
|
|
428
|
+
include_events=include_events,
|
|
429
|
+
extract_artifacts=extract_artifacts,
|
|
430
|
+
skip_no_artifact=skip_no_artifact,
|
|
431
|
+
)
|
|
377
432
|
from irepo_artifact import ArtifactError, ArtifactNotFound
|
|
378
433
|
|
|
379
434
|
current_stage = "job metadata"
|
|
@@ -531,6 +586,209 @@ def _export_job_into_dir(
|
|
|
531
586
|
return None
|
|
532
587
|
|
|
533
588
|
|
|
589
|
+
def _export_shared_job_into_dir(
|
|
590
|
+
client: APIClient,
|
|
591
|
+
job_id: str,
|
|
592
|
+
dest: Path,
|
|
593
|
+
*,
|
|
594
|
+
stage_callback: Optional[StageCallback] = None,
|
|
595
|
+
timeout: float = _DOWNLOAD_TIMEOUT,
|
|
596
|
+
result_callback: Optional[ResultCallback] = None,
|
|
597
|
+
include_logs: bool = False,
|
|
598
|
+
include_events: bool = False,
|
|
599
|
+
extract_artifacts: bool = True,
|
|
600
|
+
skip_no_artifact: bool = False,
|
|
601
|
+
) -> Optional[str]:
|
|
602
|
+
"""Use the share route's signed archive URL; the SDK uses authenticated routes."""
|
|
603
|
+
stage = "job metadata"
|
|
604
|
+
|
|
605
|
+
def set_stage(value: str) -> None:
|
|
606
|
+
nonlocal stage
|
|
607
|
+
stage = value
|
|
608
|
+
if stage_callback is not None:
|
|
609
|
+
stage_callback(value)
|
|
610
|
+
|
|
611
|
+
try:
|
|
612
|
+
job = client.get_job(job_id)
|
|
613
|
+
set_stage("artifact metadata")
|
|
614
|
+
artifacts = client.get_job_artifacts([job_id])
|
|
615
|
+
artifact = artifacts[0] if artifacts else {"job_id": job_id, "url": None}
|
|
616
|
+
url = artifact.get("url")
|
|
617
|
+
if not url and skip_no_artifact:
|
|
618
|
+
set_stage("skipped")
|
|
619
|
+
if result_callback is not None:
|
|
620
|
+
result_callback(job_id, "skipped", None)
|
|
621
|
+
return job.get("job_name") or job_id
|
|
622
|
+
|
|
623
|
+
parent = dest.resolve()
|
|
624
|
+
while not parent.exists():
|
|
625
|
+
parent = parent.parent
|
|
626
|
+
with tempfile.TemporaryDirectory(prefix=".artifact-share-", dir=parent) as temporary:
|
|
627
|
+
staging = Path(temporary)
|
|
628
|
+
if url:
|
|
629
|
+
set_stage("downloading artifact")
|
|
630
|
+
try:
|
|
631
|
+
_download_shared_archive(url, staging / "result.tgz", timeout)
|
|
632
|
+
except FileNotFoundError:
|
|
633
|
+
if skip_no_artifact:
|
|
634
|
+
_remove_path(dest)
|
|
635
|
+
set_stage("skipped")
|
|
636
|
+
if result_callback is not None:
|
|
637
|
+
result_callback(job_id, "skipped", None)
|
|
638
|
+
return job.get("job_name") or job_id
|
|
639
|
+
(dest / "result.tgz").unlink(missing_ok=True)
|
|
640
|
+
_warn_missing_artifact(job_id)
|
|
641
|
+
url = None
|
|
642
|
+
if url and extract_artifacts:
|
|
643
|
+
set_stage("extracting artifact")
|
|
644
|
+
_extract_shared_archive(staging / "result.tgz", staging / "artifacts")
|
|
645
|
+
|
|
646
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
647
|
+
set_stage("metrics")
|
|
648
|
+
_write_json(dest / "job.json", job)
|
|
649
|
+
_write_json(dest / "metrics.json", client.get_job_metrics(job_id))
|
|
650
|
+
_write_json(dest / "artifacts.json", artifact)
|
|
651
|
+
if include_events:
|
|
652
|
+
set_stage("events")
|
|
653
|
+
_write_text(dest / "events.txt", _format_events(client.get_job_events(job_id)))
|
|
654
|
+
if include_logs:
|
|
655
|
+
set_stage("logs")
|
|
656
|
+
_write_logs(dest / "logs", client, job_id, job)
|
|
657
|
+
|
|
658
|
+
if url:
|
|
659
|
+
set_stage("publishing artifact")
|
|
660
|
+
if extract_artifacts:
|
|
661
|
+
_commit_staged_artifacts(
|
|
662
|
+
staging / "artifacts", dest / "artifacts", archive=staging / "result.tgz"
|
|
663
|
+
)
|
|
664
|
+
else:
|
|
665
|
+
(staging / "result.tgz").replace(dest / "result.tgz")
|
|
666
|
+
if result_callback is not None:
|
|
667
|
+
result_callback(job_id, "downloaded", None)
|
|
668
|
+
elif result_callback is not None:
|
|
669
|
+
result_callback(job_id, "skipped", None)
|
|
670
|
+
except Exception as exc:
|
|
671
|
+
if result_callback is not None:
|
|
672
|
+
result_callback(job_id, "failed", None)
|
|
673
|
+
raise RuntimeError(f"Failed to export job {job_id} at stage '{stage}': {exc}") from exc
|
|
674
|
+
return None
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _download_shared_archive(url: str, dest: Path, timeout: float) -> None:
|
|
678
|
+
"""Stream a signed URL without including it in user-visible errors."""
|
|
679
|
+
try:
|
|
680
|
+
with requests.get(url, stream=True, timeout=(10, timeout)) as response:
|
|
681
|
+
if response.status_code == 404:
|
|
682
|
+
raise FileNotFoundError("shared artifact was not found")
|
|
683
|
+
if response.status_code >= 400:
|
|
684
|
+
raise RuntimeError(f"shared artifact download returned HTTP {response.status_code}")
|
|
685
|
+
with dest.open("wb") as output:
|
|
686
|
+
for chunk in response.iter_content(chunk_size=4 * 1024 * 1024):
|
|
687
|
+
if chunk:
|
|
688
|
+
output.write(chunk)
|
|
689
|
+
except requests.RequestException:
|
|
690
|
+
raise RuntimeError("shared artifact download failed") from None
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def _extract_shared_archive(archive: Path, artifacts: Path) -> None:
|
|
694
|
+
"""Extract an archive inside a fresh staging directory with bounded links."""
|
|
695
|
+
raw = artifacts.parent / "unpacked"
|
|
696
|
+
raw.mkdir()
|
|
697
|
+
try:
|
|
698
|
+
directory_modes: dict[Path, int] = {}
|
|
699
|
+
with tarfile.open(archive, "r:gz") as tar:
|
|
700
|
+
for member in tar:
|
|
701
|
+
name = member.name
|
|
702
|
+
while name.startswith("./"):
|
|
703
|
+
name = name[2:]
|
|
704
|
+
if name in ("", ".") and member.isdir():
|
|
705
|
+
continue
|
|
706
|
+
parts = PurePosixPath(name).parts
|
|
707
|
+
if (
|
|
708
|
+
not name
|
|
709
|
+
or name.startswith("/")
|
|
710
|
+
or "\\" in name
|
|
711
|
+
or "\x00" in name
|
|
712
|
+
or any(part in {".", ".."} for part in parts)
|
|
713
|
+
or not (member.isfile() or member.isdir() or member.issym() or member.islnk())
|
|
714
|
+
):
|
|
715
|
+
raise ValueError("unsafe shared artifact archive member")
|
|
716
|
+
target = raw / name
|
|
717
|
+
if not target.resolve().is_relative_to(raw.resolve()):
|
|
718
|
+
raise ValueError("shared artifact archive path escapes destination")
|
|
719
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
720
|
+
if member.isdir():
|
|
721
|
+
target.mkdir(exist_ok=True)
|
|
722
|
+
directory_modes[target] = member.mode & 0o777
|
|
723
|
+
elif member.isfile():
|
|
724
|
+
with tar.extractfile(member) as source, target.open("wb") as output:
|
|
725
|
+
shutil.copyfileobj(source, output)
|
|
726
|
+
target.chmod(member.mode & 0o777)
|
|
727
|
+
else:
|
|
728
|
+
link = member.linkname
|
|
729
|
+
if (
|
|
730
|
+
not link
|
|
731
|
+
or link.startswith("/")
|
|
732
|
+
or "\\" in link
|
|
733
|
+
or "\x00" in link
|
|
734
|
+
or re.match(r"^[A-Za-z]:", link)
|
|
735
|
+
):
|
|
736
|
+
raise ValueError("unsafe shared artifact archive link")
|
|
737
|
+
link_target = (target.parent if member.issym() else raw) / link
|
|
738
|
+
if not link_target.resolve().is_relative_to(raw.resolve()):
|
|
739
|
+
raise ValueError("shared artifact archive link escapes destination")
|
|
740
|
+
if target.exists() or target.is_symlink():
|
|
741
|
+
target.unlink()
|
|
742
|
+
if member.issym():
|
|
743
|
+
target.symlink_to(link)
|
|
744
|
+
else:
|
|
745
|
+
if not link_target.is_file():
|
|
746
|
+
raise ValueError("shared artifact hard link target is not a file")
|
|
747
|
+
os.link(link_target.resolve(), target)
|
|
748
|
+
|
|
749
|
+
entries = list(raw.iterdir())
|
|
750
|
+
root = (
|
|
751
|
+
entries[0]
|
|
752
|
+
if len(entries) == 1 and entries[0].is_dir() and not entries[0].is_symlink()
|
|
753
|
+
else raw
|
|
754
|
+
)
|
|
755
|
+
output_dir = artifacts / "output"
|
|
756
|
+
output_dir.mkdir(parents=True)
|
|
757
|
+
root_path = root.resolve()
|
|
758
|
+
rewritten_links: dict[Path, str] = {}
|
|
759
|
+
for entry in raw.rglob("*"):
|
|
760
|
+
if not entry.is_symlink():
|
|
761
|
+
continue
|
|
762
|
+
target = entry.resolve()
|
|
763
|
+
if not target.is_relative_to(root_path):
|
|
764
|
+
raise ValueError("shared artifact archive link escapes output")
|
|
765
|
+
final_link = output_dir / entry.relative_to(root)
|
|
766
|
+
final_target = output_dir / target.relative_to(root_path)
|
|
767
|
+
rewritten_links[final_link] = os.path.relpath(final_target, final_link.parent)
|
|
768
|
+
for entry in root.iterdir():
|
|
769
|
+
entry.replace(output_dir / entry.name)
|
|
770
|
+
for link_path, link_target in rewritten_links.items():
|
|
771
|
+
link_path.unlink()
|
|
772
|
+
link_path.symlink_to(link_target)
|
|
773
|
+
if not link_path.resolve().is_relative_to(output_dir.resolve()):
|
|
774
|
+
raise ValueError("shared artifact archive link escapes output")
|
|
775
|
+
# Apply explicit directory modes after all writes and link rewrites so
|
|
776
|
+
# private parents do not prevent extraction of their children.
|
|
777
|
+
for source_dir, mode in sorted(
|
|
778
|
+
directory_modes.items(), key=lambda item: len(item[0].parts), reverse=True
|
|
779
|
+
):
|
|
780
|
+
final_dir = output_dir / source_dir.relative_to(root)
|
|
781
|
+
if (
|
|
782
|
+
final_dir.is_symlink()
|
|
783
|
+
or not final_dir.is_dir()
|
|
784
|
+
or not final_dir.resolve().is_relative_to(output_dir.resolve())
|
|
785
|
+
):
|
|
786
|
+
raise ValueError("shared artifact archive directory changed type")
|
|
787
|
+
final_dir.chmod(mode)
|
|
788
|
+
finally:
|
|
789
|
+
shutil.rmtree(raw, ignore_errors=True)
|
|
790
|
+
|
|
791
|
+
|
|
534
792
|
def _normalize_file_selectors(values: Optional[list[str]]) -> Optional[list[str]]:
|
|
535
793
|
selectors: list[str] = []
|
|
536
794
|
for raw in values or []:
|
|
@@ -451,7 +451,6 @@ def _cached_oss_probe(
|
|
|
451
451
|
|
|
452
452
|
_probe_cache_log(verbose, status, cache_endpoint)
|
|
453
453
|
reachable = run_probe()
|
|
454
|
-
now = _now()
|
|
455
454
|
with _oss_probe_cache_lock(path, verbose=verbose) as write_locked:
|
|
456
455
|
if not write_locked:
|
|
457
456
|
_probe_cache_log(verbose, "write skipped (cache busy)", cache_endpoint)
|
|
@@ -460,6 +459,7 @@ def _cached_oss_probe(
|
|
|
460
459
|
latest_entries = _read_oss_probe_cache(path)
|
|
461
460
|
except Exception:
|
|
462
461
|
latest_entries = []
|
|
462
|
+
now = _now()
|
|
463
463
|
fresh_entries = [
|
|
464
464
|
entry
|
|
465
465
|
for entry in latest_entries
|
|
@@ -138,20 +138,31 @@ def _safe_identifier(value: Any) -> str | None:
|
|
|
138
138
|
|
|
139
139
|
|
|
140
140
|
class ManagedAKProtocolError(RuntimeError):
|
|
141
|
-
"""
|
|
141
|
+
"""An unconfirmed AK reference, with only safe receipt identities retained."""
|
|
142
142
|
|
|
143
143
|
def __init__(self, ak_id: str, response: Any):
|
|
144
144
|
self.ak_id = ak_id
|
|
145
|
-
self.response =
|
|
145
|
+
self.response: dict[str, Any] = {}
|
|
146
146
|
self.job_ids: list[str] = []
|
|
147
147
|
self.group_id = None
|
|
148
148
|
if isinstance(response, dict):
|
|
149
149
|
self.group_id = _safe_identifier(response.get("group_id"))
|
|
150
|
-
|
|
150
|
+
if self.group_id:
|
|
151
|
+
self.response["group_id"] = self.group_id
|
|
152
|
+
job_id = _safe_identifier(response.get("job_id"))
|
|
153
|
+
if job_id:
|
|
154
|
+
self.response["job_id"] = job_id
|
|
155
|
+
self.job_ids.append(job_id)
|
|
151
156
|
jobs = response.get("jobs")
|
|
152
157
|
if isinstance(jobs, list):
|
|
153
|
-
|
|
154
|
-
|
|
158
|
+
job_ids = [
|
|
159
|
+
value
|
|
160
|
+
for job in jobs
|
|
161
|
+
if isinstance(job, dict)
|
|
162
|
+
if (value := _safe_identifier(job.get("job_id")))
|
|
163
|
+
]
|
|
164
|
+
self.job_ids.extend(job_ids)
|
|
165
|
+
self.response["jobs"] = [{"job_id": value} for value in job_ids]
|
|
155
166
|
super().__init__(
|
|
156
167
|
f"Managed AK protocol mismatch: server did not acknowledge ak_id={ak_id}. "
|
|
157
168
|
f"Returned Job IDs: {self.job_ids}. Verify these jobs before resubmitting; "
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""Read-only API adapter for Job and Group share tokens."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import quote, quote_plus
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from .api import APIClient, APIError, PaginationProtocolError, TimeoutType
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ShareAPIClient(APIClient):
|
|
15
|
+
"""Use existing export/CLI readers against the token-scoped share routes."""
|
|
16
|
+
|
|
17
|
+
is_share = True
|
|
18
|
+
|
|
19
|
+
def __init__(self, config, token: str, *, group_id: str | None = None):
|
|
20
|
+
if not token or not token.strip():
|
|
21
|
+
raise ValueError("--share-token must not be empty")
|
|
22
|
+
# The share token is the only credential for these public routes.
|
|
23
|
+
headers = {
|
|
24
|
+
key: value
|
|
25
|
+
for key, value in config.headers.items()
|
|
26
|
+
if key.lower()
|
|
27
|
+
not in {
|
|
28
|
+
"authorization",
|
|
29
|
+
"x-api-key",
|
|
30
|
+
"cookie",
|
|
31
|
+
"proxy-authorization",
|
|
32
|
+
"x-access-token",
|
|
33
|
+
"x-apiserver-token",
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
safe_config = replace(config, headers=headers, token_key=None, verbose_redaction=True)
|
|
37
|
+
super().__init__(safe_config)
|
|
38
|
+
self.share_token = token.strip()
|
|
39
|
+
self.share_group_id = group_id
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_client(cls, client: APIClient, token: str, *, group_id: str | None = None):
|
|
43
|
+
return cls(client.config, token, group_id=group_id)
|
|
44
|
+
|
|
45
|
+
def _group_path(self, group_id: str, suffix: str = "") -> str:
|
|
46
|
+
if self.share_group_id != group_id:
|
|
47
|
+
raise ValueError("Group share token requires its matching group ID")
|
|
48
|
+
return f"/share/view/groups/{quote(group_id, safe='')}{suffix}"
|
|
49
|
+
|
|
50
|
+
def _job_path(self, job_id: str, suffix: str = "") -> str:
|
|
51
|
+
encoded_job = quote(job_id, safe="")
|
|
52
|
+
return f"/share/view/jobs/{encoded_job}{suffix}"
|
|
53
|
+
|
|
54
|
+
def _share_get(self, path: str, *, timeout: TimeoutType = None, **params: Any) -> Any:
|
|
55
|
+
kwargs = {"timeout": timeout} if timeout is not None else {}
|
|
56
|
+
return self._get(path, params={"token": self.share_token, **params}, **kwargs)
|
|
57
|
+
|
|
58
|
+
def _token_spellings(self) -> set[str]:
|
|
59
|
+
return {
|
|
60
|
+
self.share_token,
|
|
61
|
+
quote(self.share_token, safe=""),
|
|
62
|
+
quote_plus(self.share_token),
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
def _redact_token(self, value: Any) -> str:
|
|
66
|
+
result = str(value)
|
|
67
|
+
for secret in self._token_spellings():
|
|
68
|
+
if secret:
|
|
69
|
+
result = result.replace(secret, "<redacted>")
|
|
70
|
+
return result
|
|
71
|
+
|
|
72
|
+
def _redacted_response(self, response: requests.Response | None) -> requests.Response | None:
|
|
73
|
+
if response is None:
|
|
74
|
+
return None
|
|
75
|
+
safe = requests.Response()
|
|
76
|
+
safe.status_code = response.status_code
|
|
77
|
+
safe.headers.update(
|
|
78
|
+
{key: self._redact_token(value) for key, value in response.headers.items()}
|
|
79
|
+
)
|
|
80
|
+
safe.url = self._redact_token(response.url or "")
|
|
81
|
+
safe.reason = self._redact_token(response.reason or "")
|
|
82
|
+
content = response.content
|
|
83
|
+
for secret in self._token_spellings():
|
|
84
|
+
if secret:
|
|
85
|
+
content = content.replace(secret.encode("utf-8"), b"<redacted>")
|
|
86
|
+
safe._content = content
|
|
87
|
+
return safe
|
|
88
|
+
|
|
89
|
+
def _request(self, *args, **kwargs):
|
|
90
|
+
try:
|
|
91
|
+
return super()._request(*args, **kwargs)
|
|
92
|
+
except APIError as exc:
|
|
93
|
+
# requests errors and server error bodies can echo the query string.
|
|
94
|
+
raise APIError(
|
|
95
|
+
exc.status_code,
|
|
96
|
+
self._redact_token(exc.detail),
|
|
97
|
+
exc.request_id,
|
|
98
|
+
self._redacted_response(exc.response),
|
|
99
|
+
) from None
|
|
100
|
+
|
|
101
|
+
def _log_request(self, method, url, request_id, headers, params, body, **kwargs) -> None:
|
|
102
|
+
# The generic verbose redactor does not classify a plain `token` key.
|
|
103
|
+
safe_params = dict(params or {})
|
|
104
|
+
if params and "token" in params:
|
|
105
|
+
safe_params["token"] = "<redacted>"
|
|
106
|
+
super()._log_request(method, url, request_id, headers, safe_params, body, **kwargs)
|
|
107
|
+
|
|
108
|
+
def _log_exception(self, method, url, request_id, duration_ms, exc) -> None:
|
|
109
|
+
super()._log_exception(
|
|
110
|
+
method,
|
|
111
|
+
url,
|
|
112
|
+
request_id,
|
|
113
|
+
duration_ms,
|
|
114
|
+
exc,
|
|
115
|
+
error_detail=self._redact_token(exc),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def _log_retry(
|
|
119
|
+
self, method, url, request_id, attempt, max_attempts, delay, *, status=None, error=None
|
|
120
|
+
) -> None:
|
|
121
|
+
super()._log_retry(
|
|
122
|
+
method,
|
|
123
|
+
url,
|
|
124
|
+
request_id,
|
|
125
|
+
attempt,
|
|
126
|
+
max_attempts,
|
|
127
|
+
delay,
|
|
128
|
+
status=status,
|
|
129
|
+
error=self._redact_token(error) if error is not None else None,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def _summarize_response(self, resp, raw) -> str:
|
|
133
|
+
return self._redact_token(super()._summarize_response(resp, raw))
|
|
134
|
+
|
|
135
|
+
def get_group(self, group_id: str, timeout: TimeoutType = None) -> dict:
|
|
136
|
+
return self._share_get(self._group_path(group_id), timeout=timeout)
|
|
137
|
+
|
|
138
|
+
def get_group_stats(self, group_id: str, timeout: TimeoutType = None) -> dict:
|
|
139
|
+
return self.get_group(group_id, timeout=timeout)
|
|
140
|
+
|
|
141
|
+
def list_jobs(
|
|
142
|
+
self, *, group_id: str | None = None, skip: int = 0, limit: int = 100, **kwargs
|
|
143
|
+
) -> dict:
|
|
144
|
+
if group_id is None:
|
|
145
|
+
raise ValueError("Group share token requires --group-id to list jobs")
|
|
146
|
+
return self._share_get(self._group_path(group_id, "/jobs"), skip=skip, limit=limit)
|
|
147
|
+
|
|
148
|
+
def get_group_eval_summary(self, group_id: str) -> dict:
|
|
149
|
+
return self._share_get(self._group_path(group_id, "/eval_summary"))
|
|
150
|
+
|
|
151
|
+
def get_group_eval(self, group_id: str) -> dict:
|
|
152
|
+
result = self.get_group_eval_summary(group_id)
|
|
153
|
+
tasks: list[dict] = []
|
|
154
|
+
while True:
|
|
155
|
+
page = self._share_get(
|
|
156
|
+
self._group_path(group_id, "/eval_tasks"), skip=len(tasks), limit=500
|
|
157
|
+
)
|
|
158
|
+
batch = page.get("tasks")
|
|
159
|
+
total = page.get("total")
|
|
160
|
+
if not isinstance(batch, list) or not isinstance(total, int) or total < 0:
|
|
161
|
+
raise PaginationProtocolError("invalid share eval task page")
|
|
162
|
+
tasks.extend(batch)
|
|
163
|
+
if len(tasks) >= total:
|
|
164
|
+
break
|
|
165
|
+
if not batch:
|
|
166
|
+
raise PaginationProtocolError("share eval tasks ended before total")
|
|
167
|
+
return {**result, "per_task": tasks}
|
|
168
|
+
|
|
169
|
+
def get_job(
|
|
170
|
+
self,
|
|
171
|
+
job_id: str,
|
|
172
|
+
*,
|
|
173
|
+
include_params: bool = False,
|
|
174
|
+
include_spec: bool = False,
|
|
175
|
+
spec_format: str = "json",
|
|
176
|
+
timeout: TimeoutType = None,
|
|
177
|
+
) -> dict:
|
|
178
|
+
result = self._share_get(self._job_path(job_id), timeout=timeout)
|
|
179
|
+
# Share responses include these payloads unconditionally; preserve the
|
|
180
|
+
# existing CLI opt-in presentation contract.
|
|
181
|
+
result = dict(result)
|
|
182
|
+
if not include_params:
|
|
183
|
+
result.pop("params", None)
|
|
184
|
+
if not include_spec:
|
|
185
|
+
result.pop("job_spec_data", None)
|
|
186
|
+
elif spec_format == "yaml" and isinstance(result.get("job_spec_data"), dict):
|
|
187
|
+
import yaml
|
|
188
|
+
|
|
189
|
+
spec_data = dict(result["job_spec_data"])
|
|
190
|
+
if "spec" in spec_data:
|
|
191
|
+
spec_data["spec_yaml"] = yaml.safe_dump(
|
|
192
|
+
spec_data["spec"], allow_unicode=True, sort_keys=False
|
|
193
|
+
)
|
|
194
|
+
result["job_spec_data"] = spec_data
|
|
195
|
+
return result
|
|
196
|
+
|
|
197
|
+
def get_job_metrics(self, job_id: str) -> dict:
|
|
198
|
+
return self._share_get(self._job_path(job_id, "/metrics"))
|
|
199
|
+
|
|
200
|
+
def get_job_events(self, job_id: str, container: str | None = None) -> dict:
|
|
201
|
+
params = {"container": container} if container else {}
|
|
202
|
+
return self._share_get(self._job_path(job_id, "/events"), **params)
|
|
203
|
+
|
|
204
|
+
def get_job_containers(self, job_id: str) -> dict:
|
|
205
|
+
return self._share_get(self._job_path(job_id, "/containers"))
|
|
206
|
+
|
|
207
|
+
def get_job_logs(
|
|
208
|
+
self,
|
|
209
|
+
job_id: str,
|
|
210
|
+
container: str | None = None,
|
|
211
|
+
offset: int | None = 0,
|
|
212
|
+
limit: int | None = 500,
|
|
213
|
+
) -> dict:
|
|
214
|
+
params: dict[str, Any] = {}
|
|
215
|
+
if container:
|
|
216
|
+
params["container"] = container
|
|
217
|
+
if offset is not None:
|
|
218
|
+
params["offset"] = offset
|
|
219
|
+
if limit is not None:
|
|
220
|
+
params["limit"] = limit
|
|
221
|
+
return self._share_get(self._job_path(job_id, "/logs"), **params)
|
|
222
|
+
|
|
223
|
+
def get_job_artifacts(self, job_ids: list[str], *, download_mode=None) -> list:
|
|
224
|
+
if download_mode is not None:
|
|
225
|
+
raise ValueError("share artifact endpoint does not accept download_mode")
|
|
226
|
+
items: list = []
|
|
227
|
+
for job_id in job_ids:
|
|
228
|
+
items.extend(self._share_get(self._job_path(job_id, "/artifacts")))
|
|
229
|
+
return items
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ap-client"
|
|
7
|
-
version = "0.6.0.
|
|
7
|
+
version = "0.6.0.dev2"
|
|
8
8
|
description = "Agent Platform API Client & CLI"
|
|
9
9
|
readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|