ap-client 0.6.0.dev0__tar.gz → 0.6.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/PKG-INFO +1 -1
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/api.py +4 -1
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/cli.py +97 -22
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/exporter.py +267 -9
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/irepo_sdk.py +1 -1
- ap_client-0.6.0.dev1/ap_client/share_api.py +229 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/pyproject.toml +1 -1
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/__init__.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/config.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/dataset_register.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/fs_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/instance_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/managed_ak.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/manifest_view.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/profile_commands.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/repo_read_guard.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/split_publish.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/tbb.py +0 -0
- {ap_client-0.6.0.dev0 → ap_client-0.6.0.dev1}/ap_client/waiter.py +0 -0
|
@@ -709,13 +709,16 @@ class APIClient:
|
|
|
709
709
|
request_id: str,
|
|
710
710
|
duration_ms: float,
|
|
711
711
|
exc: Exception,
|
|
712
|
+
*,
|
|
713
|
+
error_detail: str | None = None,
|
|
712
714
|
) -> None:
|
|
715
|
+
detail = str(exc) if error_detail is None else error_detail
|
|
713
716
|
_write_stderr(
|
|
714
717
|
[
|
|
715
718
|
f"!! Error {method} {url}",
|
|
716
719
|
f" request_id={request_id}",
|
|
717
720
|
f" duration_ms={duration_ms:.1f}",
|
|
718
|
-
f" error={type(exc).__name__}: {
|
|
721
|
+
f" error={type(exc).__name__}: {detail}",
|
|
719
722
|
]
|
|
720
723
|
)
|
|
721
724
|
|
|
@@ -359,6 +359,22 @@ def _emit_error(message: str) -> None:
|
|
|
359
359
|
error_console.print(f"[red]error: {message}[/]")
|
|
360
360
|
|
|
361
361
|
|
|
362
|
+
def _read_client(share_token: Optional[str] = None, group_id: Optional[str] = None):
|
|
363
|
+
if group_id and not share_token:
|
|
364
|
+
raise typer.BadParameter("--group-id requires --share-token")
|
|
365
|
+
client = get_client()
|
|
366
|
+
if share_token is None:
|
|
367
|
+
return client
|
|
368
|
+
from ap_client.share_api import ShareAPIClient
|
|
369
|
+
|
|
370
|
+
# Typer verbose tracebacks must not render the share token in locals.
|
|
371
|
+
app.pretty_exceptions_show_locals = False
|
|
372
|
+
try:
|
|
373
|
+
return ShareAPIClient.from_client(client, share_token, group_id=group_id)
|
|
374
|
+
except ValueError as exc:
|
|
375
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
376
|
+
|
|
377
|
+
|
|
362
378
|
def _reject_account_pool_template_credential_cli(
|
|
363
379
|
account_pool: Optional[str],
|
|
364
380
|
*credential_types: Any,
|
|
@@ -4427,6 +4443,9 @@ def job_create(
|
|
|
4427
4443
|
@job_app.command("get")
|
|
4428
4444
|
def job_get(
|
|
4429
4445
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4446
|
+
share_token: Optional[str] = typer.Option(
|
|
4447
|
+
None, "--share-token", help="Read with a share token"
|
|
4448
|
+
),
|
|
4430
4449
|
include_params: bool = typer.Option(
|
|
4431
4450
|
False,
|
|
4432
4451
|
"--include-params",
|
|
@@ -4463,7 +4482,7 @@ def job_get(
|
|
|
4463
4482
|
include_spec = include_spec or _field_path_requires_spec(field)
|
|
4464
4483
|
if field == "job_spec_data.spec_yaml":
|
|
4465
4484
|
spec_format = "yaml"
|
|
4466
|
-
client =
|
|
4485
|
+
client = _read_client(share_token)
|
|
4467
4486
|
result = client.get_job(
|
|
4468
4487
|
job_id,
|
|
4469
4488
|
include_params=include_params,
|
|
@@ -4489,6 +4508,9 @@ def job_get(
|
|
|
4489
4508
|
@job_app.command("containers")
|
|
4490
4509
|
def job_containers(
|
|
4491
4510
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4511
|
+
share_token: Optional[str] = typer.Option(
|
|
4512
|
+
None, "--share-token", help="Read with a share token"
|
|
4513
|
+
),
|
|
4492
4514
|
output_format: str = typer.Option(
|
|
4493
4515
|
None,
|
|
4494
4516
|
"--format",
|
|
@@ -4497,7 +4519,7 @@ def job_containers(
|
|
|
4497
4519
|
):
|
|
4498
4520
|
"""List container names for a job."""
|
|
4499
4521
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
4500
|
-
client =
|
|
4522
|
+
client = _read_client(share_token)
|
|
4501
4523
|
result = client.get_job_containers(job_id)
|
|
4502
4524
|
if output_format == "table":
|
|
4503
4525
|
_print_records_table(result)
|
|
@@ -4559,6 +4581,9 @@ def _print_tail_logs(logs, tail: int) -> None:
|
|
|
4559
4581
|
@job_app.command("logs")
|
|
4560
4582
|
def job_logs(
|
|
4561
4583
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4584
|
+
share_token: Optional[str] = typer.Option(
|
|
4585
|
+
None, "--share-token", help="Read with a share token"
|
|
4586
|
+
),
|
|
4562
4587
|
container: Optional[str] = typer.Option("main", "--container", "-c", help="Container name"),
|
|
4563
4588
|
tail: Optional[int] = typer.Option(None, "--tail", "-n", help="Show only the last N log lines"),
|
|
4564
4589
|
follow: bool = typer.Option(
|
|
@@ -4570,7 +4595,7 @@ def job_logs(
|
|
|
4570
4595
|
),
|
|
4571
4596
|
):
|
|
4572
4597
|
"""Show job logs."""
|
|
4573
|
-
client =
|
|
4598
|
+
client = _read_client(share_token)
|
|
4574
4599
|
|
|
4575
4600
|
if tail is not None:
|
|
4576
4601
|
if tail <= 0:
|
|
@@ -4634,6 +4659,9 @@ def job_logs(
|
|
|
4634
4659
|
@job_app.command("events")
|
|
4635
4660
|
def job_events(
|
|
4636
4661
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4662
|
+
share_token: Optional[str] = typer.Option(
|
|
4663
|
+
None, "--share-token", help="Read with a share token"
|
|
4664
|
+
),
|
|
4637
4665
|
container: Optional[str] = typer.Option(None, "--container", "-c", help="Container name"),
|
|
4638
4666
|
output_format: str = typer.Option(
|
|
4639
4667
|
None,
|
|
@@ -4643,7 +4671,7 @@ def job_events(
|
|
|
4643
4671
|
):
|
|
4644
4672
|
"""Show job events."""
|
|
4645
4673
|
output_format = _normalize_output_format(output_format)
|
|
4646
|
-
client =
|
|
4674
|
+
client = _read_client(share_token)
|
|
4647
4675
|
result = client.get_job_events(job_id, container=container)
|
|
4648
4676
|
if output_format == "plain":
|
|
4649
4677
|
_print_job_events_plain(result)
|
|
@@ -4654,6 +4682,9 @@ def job_events(
|
|
|
4654
4682
|
@job_app.command("metrics")
|
|
4655
4683
|
def job_metrics(
|
|
4656
4684
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4685
|
+
share_token: Optional[str] = typer.Option(
|
|
4686
|
+
None, "--share-token", help="Read with a share token"
|
|
4687
|
+
),
|
|
4657
4688
|
output_format: str = typer.Option(
|
|
4658
4689
|
None,
|
|
4659
4690
|
"--format",
|
|
@@ -4662,7 +4693,7 @@ def job_metrics(
|
|
|
4662
4693
|
):
|
|
4663
4694
|
"""Show job metrics."""
|
|
4664
4695
|
output_format = _normalize_output_format(output_format)
|
|
4665
|
-
client =
|
|
4696
|
+
client = _read_client(share_token)
|
|
4666
4697
|
result = client.get_job_metrics(job_id)
|
|
4667
4698
|
if output_format == "plain":
|
|
4668
4699
|
_print_json(result)
|
|
@@ -4859,6 +4890,9 @@ def job_retry(
|
|
|
4859
4890
|
@job_app.command("artifacts")
|
|
4860
4891
|
def job_artifacts(
|
|
4861
4892
|
job_ids: str = typer.Argument(..., help="Job IDs (comma-separated for multiple)"),
|
|
4893
|
+
share_token: Optional[str] = typer.Option(
|
|
4894
|
+
None, "--share-token", help="Read with a share token"
|
|
4895
|
+
),
|
|
4862
4896
|
output_format: str = typer.Option(
|
|
4863
4897
|
None,
|
|
4864
4898
|
"--format",
|
|
@@ -4867,7 +4901,7 @@ def job_artifacts(
|
|
|
4867
4901
|
):
|
|
4868
4902
|
"""Show job artifact download links."""
|
|
4869
4903
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
4870
|
-
client =
|
|
4904
|
+
client = _read_client(share_token)
|
|
4871
4905
|
ids = [jid.strip() for jid in job_ids.split(",")]
|
|
4872
4906
|
results = client.get_job_artifacts(ids)
|
|
4873
4907
|
if output_format == "table":
|
|
@@ -4907,6 +4941,9 @@ def _export_channel_reporter(emit):
|
|
|
4907
4941
|
@job_app.command("export")
|
|
4908
4942
|
def job_export(
|
|
4909
4943
|
job_id: str = typer.Argument(..., help="Job ID or comma-separated Job IDs"),
|
|
4944
|
+
share_token: Optional[str] = typer.Option(
|
|
4945
|
+
None, "--share-token", help="Export with a share token"
|
|
4946
|
+
),
|
|
4910
4947
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Export directory"),
|
|
4911
4948
|
workers: int = typer.Option(
|
|
4912
4949
|
1, "--workers", "-w", min=1, help="Number of concurrent export workers"
|
|
@@ -4929,7 +4966,7 @@ def job_export(
|
|
|
4929
4966
|
ids = list(dict.fromkeys(value.strip() for value in job_id.split(",")))
|
|
4930
4967
|
if not all(ids):
|
|
4931
4968
|
raise typer.BadParameter("Job IDs must not be empty", param_hint="job_id")
|
|
4932
|
-
client =
|
|
4969
|
+
client = _read_client(share_token)
|
|
4933
4970
|
if len(ids) > 1:
|
|
4934
4971
|
report, channels = _export_channel_reporter(print)
|
|
4935
4972
|
dest, summary = export_jobs(
|
|
@@ -5744,6 +5781,9 @@ def group_list(
|
|
|
5744
5781
|
@group_app.command("get")
|
|
5745
5782
|
def group_get(
|
|
5746
5783
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
5784
|
+
share_token: Optional[str] = typer.Option(
|
|
5785
|
+
None, "--share-token", help="Read with a share token"
|
|
5786
|
+
),
|
|
5747
5787
|
output_format: str = typer.Option(
|
|
5748
5788
|
None,
|
|
5749
5789
|
"--format",
|
|
@@ -5752,7 +5792,7 @@ def group_get(
|
|
|
5752
5792
|
):
|
|
5753
5793
|
"""Show Group details."""
|
|
5754
5794
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
5755
|
-
client =
|
|
5795
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
5756
5796
|
result = client.get_group(group_id)
|
|
5757
5797
|
if output_format == "table":
|
|
5758
5798
|
_print_detail(result)
|
|
@@ -6232,6 +6272,9 @@ def group_resume(
|
|
|
6232
6272
|
@group_app.command("jobs")
|
|
6233
6273
|
def group_jobs(
|
|
6234
6274
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6275
|
+
share_token: Optional[str] = typer.Option(
|
|
6276
|
+
None, "--share-token", help="Read with a share token"
|
|
6277
|
+
),
|
|
6235
6278
|
tag: Optional[list[str]] = typer.Option(
|
|
6236
6279
|
None, "--tag", help="Filter by tag (repeatable; multiple tags are combined with AND)"
|
|
6237
6280
|
),
|
|
@@ -6270,17 +6313,36 @@ def group_jobs(
|
|
|
6270
6313
|
if cursor and skip > 0:
|
|
6271
6314
|
raise typer.BadParameter("--cursor cannot be used with --skip greater than 0")
|
|
6272
6315
|
cursor_request = bool(next_token) or cursor
|
|
6273
|
-
|
|
6274
|
-
|
|
6275
|
-
|
|
6276
|
-
|
|
6277
|
-
|
|
6278
|
-
|
|
6279
|
-
|
|
6280
|
-
|
|
6281
|
-
|
|
6282
|
-
|
|
6283
|
-
|
|
6316
|
+
if share_token is not None:
|
|
6317
|
+
unsupported = (
|
|
6318
|
+
("--tag", bool(tag)),
|
|
6319
|
+
("--include-post-process", include_post_process),
|
|
6320
|
+
("--next-token", bool(next_token)),
|
|
6321
|
+
("--cursor", cursor),
|
|
6322
|
+
("--no-total", no_total),
|
|
6323
|
+
)
|
|
6324
|
+
for option, used in unsupported:
|
|
6325
|
+
if used:
|
|
6326
|
+
_emit_error(f"{option} is not supported with --share-token")
|
|
6327
|
+
raise typer.Exit(1)
|
|
6328
|
+
if skip < 0 or not 1 <= limit <= 500:
|
|
6329
|
+
raise typer.BadParameter(
|
|
6330
|
+
"share pagination requires --skip >= 0 and 1 <= --limit <= 500"
|
|
6331
|
+
)
|
|
6332
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6333
|
+
result = client.list_jobs(group_id=group_id, skip=skip, limit=limit)
|
|
6334
|
+
else:
|
|
6335
|
+
client = get_client()
|
|
6336
|
+
result = client.list_group_jobs(
|
|
6337
|
+
group_id,
|
|
6338
|
+
tag=tag,
|
|
6339
|
+
include_post_process=include_post_process,
|
|
6340
|
+
skip=skip,
|
|
6341
|
+
limit=limit,
|
|
6342
|
+
**({"next_token": next_token} if next_token else {}),
|
|
6343
|
+
**({"pagination": "cursor"} if cursor_request else {}),
|
|
6344
|
+
include_total=not no_total,
|
|
6345
|
+
)
|
|
6284
6346
|
if output_format == "table":
|
|
6285
6347
|
_print_job_list_plain(result, rich=True)
|
|
6286
6348
|
elif output_format == "plain":
|
|
@@ -6295,6 +6357,9 @@ def group_jobs(
|
|
|
6295
6357
|
@group_app.command("stats")
|
|
6296
6358
|
def group_stats(
|
|
6297
6359
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6360
|
+
share_token: Optional[str] = typer.Option(
|
|
6361
|
+
None, "--share-token", help="Read with a share token"
|
|
6362
|
+
),
|
|
6298
6363
|
output_format: str = typer.Option(
|
|
6299
6364
|
None,
|
|
6300
6365
|
"--format",
|
|
@@ -6303,7 +6368,7 @@ def group_stats(
|
|
|
6303
6368
|
):
|
|
6304
6369
|
"""Show Group progress."""
|
|
6305
6370
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
6306
|
-
client =
|
|
6371
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6307
6372
|
result = client.get_group_stats(group_id)
|
|
6308
6373
|
if output_format == "table":
|
|
6309
6374
|
_print_detail(result)
|
|
@@ -6314,6 +6379,9 @@ def group_stats(
|
|
|
6314
6379
|
@group_app.command("eval")
|
|
6315
6380
|
def group_eval(
|
|
6316
6381
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6382
|
+
share_token: Optional[str] = typer.Option(
|
|
6383
|
+
None, "--share-token", help="Read with a share token"
|
|
6384
|
+
),
|
|
6317
6385
|
summary_only: bool = typer.Option(
|
|
6318
6386
|
False,
|
|
6319
6387
|
"--summary-only",
|
|
@@ -6327,7 +6395,7 @@ def group_eval(
|
|
|
6327
6395
|
):
|
|
6328
6396
|
"""Show Group aggregated evaluation results."""
|
|
6329
6397
|
output_format = _normalize_output_format(output_format)
|
|
6330
|
-
client =
|
|
6398
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6331
6399
|
result = (
|
|
6332
6400
|
client.get_group_eval_summary(group_id) if summary_only else client.get_group_eval(group_id)
|
|
6333
6401
|
)
|
|
@@ -6440,6 +6508,9 @@ def group_access_list(group_id: str = typer.Argument(..., help="Group ID")):
|
|
|
6440
6508
|
@group_app.command("export")
|
|
6441
6509
|
def group_export(
|
|
6442
6510
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6511
|
+
share_token: Optional[str] = typer.Option(
|
|
6512
|
+
None, "--share-token", help="Export with a share token"
|
|
6513
|
+
),
|
|
6443
6514
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Export directory"),
|
|
6444
6515
|
workers: int = typer.Option(
|
|
6445
6516
|
4, "--workers", "-w", min=1, help="Number of concurrent export workers"
|
|
@@ -6481,7 +6552,11 @@ def group_export(
|
|
|
6481
6552
|
_emit_error("--include-files/--exclude-files cannot be used with --no-extract-artifacts")
|
|
6482
6553
|
raise typer.Exit(1)
|
|
6483
6554
|
|
|
6484
|
-
|
|
6555
|
+
if share_token is not None and (include_file_selectors or exclude_file_selectors):
|
|
6556
|
+
_emit_error("--include-files/--exclude-files are not supported with --share-token")
|
|
6557
|
+
raise typer.Exit(1)
|
|
6558
|
+
|
|
6559
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6485
6560
|
from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn
|
|
6486
6561
|
|
|
6487
6562
|
with Progress(
|
|
@@ -6,16 +6,20 @@ import concurrent.futures
|
|
|
6
6
|
import json
|
|
7
7
|
import logging
|
|
8
8
|
import os
|
|
9
|
+
import re
|
|
9
10
|
import shutil
|
|
10
11
|
import sys
|
|
12
|
+
import tarfile
|
|
11
13
|
import tempfile
|
|
12
14
|
import threading
|
|
13
15
|
import uuid
|
|
14
16
|
from contextlib import nullcontext
|
|
15
17
|
from dataclasses import dataclass, field
|
|
16
|
-
from pathlib import Path
|
|
18
|
+
from pathlib import Path, PurePosixPath
|
|
17
19
|
from typing import TYPE_CHECKING, Callable, Optional
|
|
18
20
|
|
|
21
|
+
import requests
|
|
22
|
+
|
|
19
23
|
if TYPE_CHECKING:
|
|
20
24
|
from irepo_artifact import ArtifactClient, DownloadResult
|
|
21
25
|
|
|
@@ -30,6 +34,10 @@ StageCallback = Callable[[str], None]
|
|
|
30
34
|
ResultCallback = Callable[[str, str, Optional[str]], None]
|
|
31
35
|
|
|
32
36
|
|
|
37
|
+
def _is_share_client(client: APIClient) -> bool:
|
|
38
|
+
return getattr(client, "is_share", False) is True
|
|
39
|
+
|
|
40
|
+
|
|
33
41
|
@dataclass
|
|
34
42
|
class ExportSummary:
|
|
35
43
|
"""Summary of a group export operation."""
|
|
@@ -109,6 +117,30 @@ def export_jobs(
|
|
|
109
117
|
parent = parent.parent
|
|
110
118
|
summary = BatchExportSummary()
|
|
111
119
|
ids = list(dict.fromkeys(job_ids))
|
|
120
|
+
if _is_share_client(client):
|
|
121
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
122
|
+
for job_id in ids:
|
|
123
|
+
try:
|
|
124
|
+
skipped = _export_job_into_dir(
|
|
125
|
+
client,
|
|
126
|
+
job_id,
|
|
127
|
+
root / "jobs" / job_id,
|
|
128
|
+
timeout=timeout,
|
|
129
|
+
result_callback=result_callback,
|
|
130
|
+
include_logs=include_logs,
|
|
131
|
+
include_events=include_events,
|
|
132
|
+
extract_artifacts=extract_artifacts,
|
|
133
|
+
skip_no_artifact=True,
|
|
134
|
+
)
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
summary.failures[job_id] = str(exc)
|
|
137
|
+
else:
|
|
138
|
+
if skipped is not None:
|
|
139
|
+
summary.skipped += 1
|
|
140
|
+
summary.skipped_job_names.append(skipped)
|
|
141
|
+
else:
|
|
142
|
+
summary.downloaded += 1
|
|
143
|
+
return root, summary
|
|
112
144
|
with tempfile.TemporaryDirectory(prefix=".artifact-export-", dir=parent) as temporary:
|
|
113
145
|
results = _new_artifact_client(client, timeout=timeout).download_many(
|
|
114
146
|
ids,
|
|
@@ -166,6 +198,8 @@ def export_group(
|
|
|
166
198
|
exclude_files = _normalize_file_selectors(exclude_files)
|
|
167
199
|
if not extract_artifacts and (include_files or exclude_files):
|
|
168
200
|
raise ValueError("file filters cannot be used when artifact extraction is disabled")
|
|
201
|
+
if _is_share_client(client) and (include_files or exclude_files):
|
|
202
|
+
raise ValueError("file filters are not supported with a share token")
|
|
169
203
|
|
|
170
204
|
dest = Path(output_dir) if output_dir else default_group_export_dir(group_id)
|
|
171
205
|
dest.mkdir(parents=True, exist_ok=True)
|
|
@@ -176,7 +210,8 @@ def export_group(
|
|
|
176
210
|
group_eval = client.get_group_eval(group_id)
|
|
177
211
|
_write_json(dest / "eval.json", group_eval)
|
|
178
212
|
_write_json(dest / "eval_summary.json", group_eval.get("summary", {}))
|
|
179
|
-
|
|
213
|
+
task_key = "per_task" if _is_share_client(client) else "results"
|
|
214
|
+
_write_json(dest / "eval_tasks.json", group_eval.get(task_key, []))
|
|
180
215
|
except Exception:
|
|
181
216
|
# Some groups are non-eval workloads, or older servers may not expose eval endpoints.
|
|
182
217
|
pass
|
|
@@ -217,7 +252,7 @@ def export_group(
|
|
|
217
252
|
client,
|
|
218
253
|
job_id,
|
|
219
254
|
jobs_dir / job_id,
|
|
220
|
-
job=job,
|
|
255
|
+
job=None if _is_share_client(client) else job,
|
|
221
256
|
stage_callback=lambda stage, current_job_id=job_id: _notify(current_job_id, stage),
|
|
222
257
|
timeout=timeout,
|
|
223
258
|
result_callback=result_callback,
|
|
@@ -243,7 +278,7 @@ def export_group(
|
|
|
243
278
|
future_map = {
|
|
244
279
|
executor.submit(
|
|
245
280
|
_export_job_with_fresh_client,
|
|
246
|
-
client
|
|
281
|
+
client,
|
|
247
282
|
job,
|
|
248
283
|
jobs_dir / job["job_id"],
|
|
249
284
|
lambda stage, current_job_id=job["job_id"]: _notify(current_job_id, stage),
|
|
@@ -319,7 +354,7 @@ def _list_group_jobs(client: APIClient, group_id: str) -> list[dict]:
|
|
|
319
354
|
|
|
320
355
|
|
|
321
356
|
def _export_job_with_fresh_client(
|
|
322
|
-
|
|
357
|
+
client: APIClient,
|
|
323
358
|
job: dict,
|
|
324
359
|
dest: Path,
|
|
325
360
|
stage_callback: Optional[StageCallback] = None,
|
|
@@ -334,12 +369,19 @@ def _export_job_with_fresh_client(
|
|
|
334
369
|
exclude_files: Optional[list[str]] = None,
|
|
335
370
|
) -> Optional[str]:
|
|
336
371
|
"""Export one job with a dedicated client/session for thread safety."""
|
|
337
|
-
|
|
372
|
+
if _is_share_client(client):
|
|
373
|
+
from .share_api import ShareAPIClient
|
|
374
|
+
|
|
375
|
+
worker_client = ShareAPIClient(
|
|
376
|
+
client.config, client.share_token, group_id=client.share_group_id
|
|
377
|
+
)
|
|
378
|
+
else:
|
|
379
|
+
worker_client = APIClient(client.config)
|
|
338
380
|
return _export_job_into_dir(
|
|
339
381
|
worker_client,
|
|
340
382
|
job["job_id"],
|
|
341
383
|
dest,
|
|
342
|
-
job=job,
|
|
384
|
+
job=None if _is_share_client(client) else job,
|
|
343
385
|
stage_callback=stage_callback,
|
|
344
386
|
timeout=timeout,
|
|
345
387
|
result_callback=result_callback,
|
|
@@ -371,9 +413,22 @@ def _export_job_into_dir(
|
|
|
371
413
|
) -> Optional[str]:
|
|
372
414
|
"""Export SDK-managed artifacts alongside AP metadata.
|
|
373
415
|
|
|
374
|
-
|
|
375
|
-
|
|
416
|
+
Ordinary exports use the SDK for artifacts. Share exports use the share
|
|
417
|
+
endpoint's signed archive URL because the SDK uses authenticated routes.
|
|
376
418
|
"""
|
|
419
|
+
if _is_share_client(client):
|
|
420
|
+
return _export_shared_job_into_dir(
|
|
421
|
+
client,
|
|
422
|
+
job_id,
|
|
423
|
+
dest,
|
|
424
|
+
stage_callback=stage_callback,
|
|
425
|
+
timeout=timeout,
|
|
426
|
+
result_callback=result_callback,
|
|
427
|
+
include_logs=include_logs,
|
|
428
|
+
include_events=include_events,
|
|
429
|
+
extract_artifacts=extract_artifacts,
|
|
430
|
+
skip_no_artifact=skip_no_artifact,
|
|
431
|
+
)
|
|
377
432
|
from irepo_artifact import ArtifactError, ArtifactNotFound
|
|
378
433
|
|
|
379
434
|
current_stage = "job metadata"
|
|
@@ -531,6 +586,209 @@ def _export_job_into_dir(
|
|
|
531
586
|
return None
|
|
532
587
|
|
|
533
588
|
|
|
589
|
+
def _export_shared_job_into_dir(
|
|
590
|
+
client: APIClient,
|
|
591
|
+
job_id: str,
|
|
592
|
+
dest: Path,
|
|
593
|
+
*,
|
|
594
|
+
stage_callback: Optional[StageCallback] = None,
|
|
595
|
+
timeout: float = _DOWNLOAD_TIMEOUT,
|
|
596
|
+
result_callback: Optional[ResultCallback] = None,
|
|
597
|
+
include_logs: bool = False,
|
|
598
|
+
include_events: bool = False,
|
|
599
|
+
extract_artifacts: bool = True,
|
|
600
|
+
skip_no_artifact: bool = False,
|
|
601
|
+
) -> Optional[str]:
|
|
602
|
+
"""Use the share route's signed archive URL; the SDK uses authenticated routes."""
|
|
603
|
+
stage = "job metadata"
|
|
604
|
+
|
|
605
|
+
def set_stage(value: str) -> None:
|
|
606
|
+
nonlocal stage
|
|
607
|
+
stage = value
|
|
608
|
+
if stage_callback is not None:
|
|
609
|
+
stage_callback(value)
|
|
610
|
+
|
|
611
|
+
try:
|
|
612
|
+
job = client.get_job(job_id)
|
|
613
|
+
set_stage("artifact metadata")
|
|
614
|
+
artifacts = client.get_job_artifacts([job_id])
|
|
615
|
+
artifact = artifacts[0] if artifacts else {"job_id": job_id, "url": None}
|
|
616
|
+
url = artifact.get("url")
|
|
617
|
+
if not url and skip_no_artifact:
|
|
618
|
+
set_stage("skipped")
|
|
619
|
+
if result_callback is not None:
|
|
620
|
+
result_callback(job_id, "skipped", None)
|
|
621
|
+
return job.get("job_name") or job_id
|
|
622
|
+
|
|
623
|
+
parent = dest.resolve()
|
|
624
|
+
while not parent.exists():
|
|
625
|
+
parent = parent.parent
|
|
626
|
+
with tempfile.TemporaryDirectory(prefix=".artifact-share-", dir=parent) as temporary:
|
|
627
|
+
staging = Path(temporary)
|
|
628
|
+
if url:
|
|
629
|
+
set_stage("downloading artifact")
|
|
630
|
+
try:
|
|
631
|
+
_download_shared_archive(url, staging / "result.tgz", timeout)
|
|
632
|
+
except FileNotFoundError:
|
|
633
|
+
if skip_no_artifact:
|
|
634
|
+
_remove_path(dest)
|
|
635
|
+
set_stage("skipped")
|
|
636
|
+
if result_callback is not None:
|
|
637
|
+
result_callback(job_id, "skipped", None)
|
|
638
|
+
return job.get("job_name") or job_id
|
|
639
|
+
(dest / "result.tgz").unlink(missing_ok=True)
|
|
640
|
+
_warn_missing_artifact(job_id)
|
|
641
|
+
url = None
|
|
642
|
+
if url and extract_artifacts:
|
|
643
|
+
set_stage("extracting artifact")
|
|
644
|
+
_extract_shared_archive(staging / "result.tgz", staging / "artifacts")
|
|
645
|
+
|
|
646
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
647
|
+
set_stage("metrics")
|
|
648
|
+
_write_json(dest / "job.json", job)
|
|
649
|
+
_write_json(dest / "metrics.json", client.get_job_metrics(job_id))
|
|
650
|
+
_write_json(dest / "artifacts.json", artifact)
|
|
651
|
+
if include_events:
|
|
652
|
+
set_stage("events")
|
|
653
|
+
_write_text(dest / "events.txt", _format_events(client.get_job_events(job_id)))
|
|
654
|
+
if include_logs:
|
|
655
|
+
set_stage("logs")
|
|
656
|
+
_write_logs(dest / "logs", client, job_id, job)
|
|
657
|
+
|
|
658
|
+
if url:
|
|
659
|
+
set_stage("publishing artifact")
|
|
660
|
+
if extract_artifacts:
|
|
661
|
+
_commit_staged_artifacts(
|
|
662
|
+
staging / "artifacts", dest / "artifacts", archive=staging / "result.tgz"
|
|
663
|
+
)
|
|
664
|
+
else:
|
|
665
|
+
(staging / "result.tgz").replace(dest / "result.tgz")
|
|
666
|
+
if result_callback is not None:
|
|
667
|
+
result_callback(job_id, "downloaded", None)
|
|
668
|
+
elif result_callback is not None:
|
|
669
|
+
result_callback(job_id, "skipped", None)
|
|
670
|
+
except Exception as exc:
|
|
671
|
+
if result_callback is not None:
|
|
672
|
+
result_callback(job_id, "failed", None)
|
|
673
|
+
raise RuntimeError(f"Failed to export job {job_id} at stage '{stage}': {exc}") from exc
|
|
674
|
+
return None
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _download_shared_archive(url: str, dest: Path, timeout: float) -> None:
|
|
678
|
+
"""Stream a signed URL without including it in user-visible errors."""
|
|
679
|
+
try:
|
|
680
|
+
with requests.get(url, stream=True, timeout=(10, timeout)) as response:
|
|
681
|
+
if response.status_code == 404:
|
|
682
|
+
raise FileNotFoundError("shared artifact was not found")
|
|
683
|
+
if response.status_code >= 400:
|
|
684
|
+
raise RuntimeError(f"shared artifact download returned HTTP {response.status_code}")
|
|
685
|
+
with dest.open("wb") as output:
|
|
686
|
+
for chunk in response.iter_content(chunk_size=4 * 1024 * 1024):
|
|
687
|
+
if chunk:
|
|
688
|
+
output.write(chunk)
|
|
689
|
+
except requests.RequestException:
|
|
690
|
+
raise RuntimeError("shared artifact download failed") from None
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
def _extract_shared_archive(archive: Path, artifacts: Path) -> None:
|
|
694
|
+
"""Extract an archive inside a fresh staging directory with bounded links."""
|
|
695
|
+
raw = artifacts.parent / "unpacked"
|
|
696
|
+
raw.mkdir()
|
|
697
|
+
try:
|
|
698
|
+
directory_modes: dict[Path, int] = {}
|
|
699
|
+
with tarfile.open(archive, "r:gz") as tar:
|
|
700
|
+
for member in tar:
|
|
701
|
+
name = member.name
|
|
702
|
+
while name.startswith("./"):
|
|
703
|
+
name = name[2:]
|
|
704
|
+
if name in ("", ".") and member.isdir():
|
|
705
|
+
continue
|
|
706
|
+
parts = PurePosixPath(name).parts
|
|
707
|
+
if (
|
|
708
|
+
not name
|
|
709
|
+
or name.startswith("/")
|
|
710
|
+
or "\\" in name
|
|
711
|
+
or "\x00" in name
|
|
712
|
+
or any(part in {".", ".."} for part in parts)
|
|
713
|
+
or not (member.isfile() or member.isdir() or member.issym() or member.islnk())
|
|
714
|
+
):
|
|
715
|
+
raise ValueError("unsafe shared artifact archive member")
|
|
716
|
+
target = raw / name
|
|
717
|
+
if not target.resolve().is_relative_to(raw.resolve()):
|
|
718
|
+
raise ValueError("shared artifact archive path escapes destination")
|
|
719
|
+
target.parent.mkdir(parents=True, exist_ok=True)
|
|
720
|
+
if member.isdir():
|
|
721
|
+
target.mkdir(exist_ok=True)
|
|
722
|
+
directory_modes[target] = member.mode & 0o777
|
|
723
|
+
elif member.isfile():
|
|
724
|
+
with tar.extractfile(member) as source, target.open("wb") as output:
|
|
725
|
+
shutil.copyfileobj(source, output)
|
|
726
|
+
target.chmod(member.mode & 0o777)
|
|
727
|
+
else:
|
|
728
|
+
link = member.linkname
|
|
729
|
+
if (
|
|
730
|
+
not link
|
|
731
|
+
or link.startswith("/")
|
|
732
|
+
or "\\" in link
|
|
733
|
+
or "\x00" in link
|
|
734
|
+
or re.match(r"^[A-Za-z]:", link)
|
|
735
|
+
):
|
|
736
|
+
raise ValueError("unsafe shared artifact archive link")
|
|
737
|
+
link_target = (target.parent if member.issym() else raw) / link
|
|
738
|
+
if not link_target.resolve().is_relative_to(raw.resolve()):
|
|
739
|
+
raise ValueError("shared artifact archive link escapes destination")
|
|
740
|
+
if target.exists() or target.is_symlink():
|
|
741
|
+
target.unlink()
|
|
742
|
+
if member.issym():
|
|
743
|
+
target.symlink_to(link)
|
|
744
|
+
else:
|
|
745
|
+
if not link_target.is_file():
|
|
746
|
+
raise ValueError("shared artifact hard link target is not a file")
|
|
747
|
+
os.link(link_target.resolve(), target)
|
|
748
|
+
|
|
749
|
+
entries = list(raw.iterdir())
|
|
750
|
+
root = (
|
|
751
|
+
entries[0]
|
|
752
|
+
if len(entries) == 1 and entries[0].is_dir() and not entries[0].is_symlink()
|
|
753
|
+
else raw
|
|
754
|
+
)
|
|
755
|
+
output_dir = artifacts / "output"
|
|
756
|
+
output_dir.mkdir(parents=True)
|
|
757
|
+
root_path = root.resolve()
|
|
758
|
+
rewritten_links: dict[Path, str] = {}
|
|
759
|
+
for entry in raw.rglob("*"):
|
|
760
|
+
if not entry.is_symlink():
|
|
761
|
+
continue
|
|
762
|
+
target = entry.resolve()
|
|
763
|
+
if not target.is_relative_to(root_path):
|
|
764
|
+
raise ValueError("shared artifact archive link escapes output")
|
|
765
|
+
final_link = output_dir / entry.relative_to(root)
|
|
766
|
+
final_target = output_dir / target.relative_to(root_path)
|
|
767
|
+
rewritten_links[final_link] = os.path.relpath(final_target, final_link.parent)
|
|
768
|
+
for entry in root.iterdir():
|
|
769
|
+
entry.replace(output_dir / entry.name)
|
|
770
|
+
for link_path, link_target in rewritten_links.items():
|
|
771
|
+
link_path.unlink()
|
|
772
|
+
link_path.symlink_to(link_target)
|
|
773
|
+
if not link_path.resolve().is_relative_to(output_dir.resolve()):
|
|
774
|
+
raise ValueError("shared artifact archive link escapes output")
|
|
775
|
+
# Apply explicit directory modes after all writes and link rewrites so
|
|
776
|
+
# private parents do not prevent extraction of their children.
|
|
777
|
+
for source_dir, mode in sorted(
|
|
778
|
+
directory_modes.items(), key=lambda item: len(item[0].parts), reverse=True
|
|
779
|
+
):
|
|
780
|
+
final_dir = output_dir / source_dir.relative_to(root)
|
|
781
|
+
if (
|
|
782
|
+
final_dir.is_symlink()
|
|
783
|
+
or not final_dir.is_dir()
|
|
784
|
+
or not final_dir.resolve().is_relative_to(output_dir.resolve())
|
|
785
|
+
):
|
|
786
|
+
raise ValueError("shared artifact archive directory changed type")
|
|
787
|
+
final_dir.chmod(mode)
|
|
788
|
+
finally:
|
|
789
|
+
shutil.rmtree(raw, ignore_errors=True)
|
|
790
|
+
|
|
791
|
+
|
|
534
792
|
def _normalize_file_selectors(values: Optional[list[str]]) -> Optional[list[str]]:
|
|
535
793
|
selectors: list[str] = []
|
|
536
794
|
for raw in values or []:
|
|
@@ -451,7 +451,6 @@ def _cached_oss_probe(
|
|
|
451
451
|
|
|
452
452
|
_probe_cache_log(verbose, status, cache_endpoint)
|
|
453
453
|
reachable = run_probe()
|
|
454
|
-
now = _now()
|
|
455
454
|
with _oss_probe_cache_lock(path, verbose=verbose) as write_locked:
|
|
456
455
|
if not write_locked:
|
|
457
456
|
_probe_cache_log(verbose, "write skipped (cache busy)", cache_endpoint)
|
|
@@ -460,6 +459,7 @@ def _cached_oss_probe(
|
|
|
460
459
|
latest_entries = _read_oss_probe_cache(path)
|
|
461
460
|
except Exception:
|
|
462
461
|
latest_entries = []
|
|
462
|
+
now = _now()
|
|
463
463
|
fresh_entries = [
|
|
464
464
|
entry
|
|
465
465
|
for entry in latest_entries
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
"""Read-only API adapter for Job and Group share tokens."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import replace
|
|
6
|
+
from typing import Any
|
|
7
|
+
from urllib.parse import quote, quote_plus
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from .api import APIClient, APIError, PaginationProtocolError, TimeoutType
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ShareAPIClient(APIClient):
|
|
15
|
+
"""Use existing export/CLI readers against the token-scoped share routes."""
|
|
16
|
+
|
|
17
|
+
is_share = True
|
|
18
|
+
|
|
19
|
+
def __init__(self, config, token: str, *, group_id: str | None = None):
|
|
20
|
+
if not token or not token.strip():
|
|
21
|
+
raise ValueError("--share-token must not be empty")
|
|
22
|
+
# The share token is the only credential for these public routes.
|
|
23
|
+
headers = {
|
|
24
|
+
key: value
|
|
25
|
+
for key, value in config.headers.items()
|
|
26
|
+
if key.lower()
|
|
27
|
+
not in {
|
|
28
|
+
"authorization",
|
|
29
|
+
"x-api-key",
|
|
30
|
+
"cookie",
|
|
31
|
+
"proxy-authorization",
|
|
32
|
+
"x-access-token",
|
|
33
|
+
"x-apiserver-token",
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
safe_config = replace(config, headers=headers, token_key=None, verbose_redaction=True)
|
|
37
|
+
super().__init__(safe_config)
|
|
38
|
+
self.share_token = token.strip()
|
|
39
|
+
self.share_group_id = group_id
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def from_client(cls, client: APIClient, token: str, *, group_id: str | None = None):
|
|
43
|
+
return cls(client.config, token, group_id=group_id)
|
|
44
|
+
|
|
45
|
+
def _group_path(self, group_id: str, suffix: str = "") -> str:
|
|
46
|
+
if self.share_group_id != group_id:
|
|
47
|
+
raise ValueError("Group share token requires its matching group ID")
|
|
48
|
+
return f"/share/view/groups/{quote(group_id, safe='')}{suffix}"
|
|
49
|
+
|
|
50
|
+
def _job_path(self, job_id: str, suffix: str = "") -> str:
|
|
51
|
+
encoded_job = quote(job_id, safe="")
|
|
52
|
+
return f"/share/view/jobs/{encoded_job}{suffix}"
|
|
53
|
+
|
|
54
|
+
def _share_get(self, path: str, *, timeout: TimeoutType = None, **params: Any) -> Any:
|
|
55
|
+
kwargs = {"timeout": timeout} if timeout is not None else {}
|
|
56
|
+
return self._get(path, params={"token": self.share_token, **params}, **kwargs)
|
|
57
|
+
|
|
58
|
+
def _token_spellings(self) -> set[str]:
|
|
59
|
+
return {
|
|
60
|
+
self.share_token,
|
|
61
|
+
quote(self.share_token, safe=""),
|
|
62
|
+
quote_plus(self.share_token),
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
def _redact_token(self, value: Any) -> str:
|
|
66
|
+
result = str(value)
|
|
67
|
+
for secret in self._token_spellings():
|
|
68
|
+
if secret:
|
|
69
|
+
result = result.replace(secret, "<redacted>")
|
|
70
|
+
return result
|
|
71
|
+
|
|
72
|
+
def _redacted_response(self, response: requests.Response | None) -> requests.Response | None:
|
|
73
|
+
if response is None:
|
|
74
|
+
return None
|
|
75
|
+
safe = requests.Response()
|
|
76
|
+
safe.status_code = response.status_code
|
|
77
|
+
safe.headers.update(
|
|
78
|
+
{key: self._redact_token(value) for key, value in response.headers.items()}
|
|
79
|
+
)
|
|
80
|
+
safe.url = self._redact_token(response.url or "")
|
|
81
|
+
safe.reason = self._redact_token(response.reason or "")
|
|
82
|
+
content = response.content
|
|
83
|
+
for secret in self._token_spellings():
|
|
84
|
+
if secret:
|
|
85
|
+
content = content.replace(secret.encode("utf-8"), b"<redacted>")
|
|
86
|
+
safe._content = content
|
|
87
|
+
return safe
|
|
88
|
+
|
|
89
|
+
def _request(self, *args, **kwargs):
|
|
90
|
+
try:
|
|
91
|
+
return super()._request(*args, **kwargs)
|
|
92
|
+
except APIError as exc:
|
|
93
|
+
# requests errors and server error bodies can echo the query string.
|
|
94
|
+
raise APIError(
|
|
95
|
+
exc.status_code,
|
|
96
|
+
self._redact_token(exc.detail),
|
|
97
|
+
exc.request_id,
|
|
98
|
+
self._redacted_response(exc.response),
|
|
99
|
+
) from None
|
|
100
|
+
|
|
101
|
+
def _log_request(self, method, url, request_id, headers, params, body, **kwargs) -> None:
|
|
102
|
+
# The generic verbose redactor does not classify a plain `token` key.
|
|
103
|
+
safe_params = dict(params or {})
|
|
104
|
+
if params and "token" in params:
|
|
105
|
+
safe_params["token"] = "<redacted>"
|
|
106
|
+
super()._log_request(method, url, request_id, headers, safe_params, body, **kwargs)
|
|
107
|
+
|
|
108
|
+
def _log_exception(self, method, url, request_id, duration_ms, exc) -> None:
|
|
109
|
+
super()._log_exception(
|
|
110
|
+
method,
|
|
111
|
+
url,
|
|
112
|
+
request_id,
|
|
113
|
+
duration_ms,
|
|
114
|
+
exc,
|
|
115
|
+
error_detail=self._redact_token(exc),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def _log_retry(
|
|
119
|
+
self, method, url, request_id, attempt, max_attempts, delay, *, status=None, error=None
|
|
120
|
+
) -> None:
|
|
121
|
+
super()._log_retry(
|
|
122
|
+
method,
|
|
123
|
+
url,
|
|
124
|
+
request_id,
|
|
125
|
+
attempt,
|
|
126
|
+
max_attempts,
|
|
127
|
+
delay,
|
|
128
|
+
status=status,
|
|
129
|
+
error=self._redact_token(error) if error is not None else None,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
def _summarize_response(self, resp, raw) -> str:
|
|
133
|
+
return self._redact_token(super()._summarize_response(resp, raw))
|
|
134
|
+
|
|
135
|
+
def get_group(self, group_id: str, timeout: TimeoutType = None) -> dict:
|
|
136
|
+
return self._share_get(self._group_path(group_id), timeout=timeout)
|
|
137
|
+
|
|
138
|
+
def get_group_stats(self, group_id: str, timeout: TimeoutType = None) -> dict:
|
|
139
|
+
return self.get_group(group_id, timeout=timeout)
|
|
140
|
+
|
|
141
|
+
def list_jobs(
|
|
142
|
+
self, *, group_id: str | None = None, skip: int = 0, limit: int = 100, **kwargs
|
|
143
|
+
) -> dict:
|
|
144
|
+
if group_id is None:
|
|
145
|
+
raise ValueError("Group share token requires --group-id to list jobs")
|
|
146
|
+
return self._share_get(self._group_path(group_id, "/jobs"), skip=skip, limit=limit)
|
|
147
|
+
|
|
148
|
+
def get_group_eval_summary(self, group_id: str) -> dict:
|
|
149
|
+
return self._share_get(self._group_path(group_id, "/eval_summary"))
|
|
150
|
+
|
|
151
|
+
def get_group_eval(self, group_id: str) -> dict:
|
|
152
|
+
result = self.get_group_eval_summary(group_id)
|
|
153
|
+
tasks: list[dict] = []
|
|
154
|
+
while True:
|
|
155
|
+
page = self._share_get(
|
|
156
|
+
self._group_path(group_id, "/eval_tasks"), skip=len(tasks), limit=500
|
|
157
|
+
)
|
|
158
|
+
batch = page.get("tasks")
|
|
159
|
+
total = page.get("total")
|
|
160
|
+
if not isinstance(batch, list) or not isinstance(total, int) or total < 0:
|
|
161
|
+
raise PaginationProtocolError("invalid share eval task page")
|
|
162
|
+
tasks.extend(batch)
|
|
163
|
+
if len(tasks) >= total:
|
|
164
|
+
break
|
|
165
|
+
if not batch:
|
|
166
|
+
raise PaginationProtocolError("share eval tasks ended before total")
|
|
167
|
+
return {**result, "per_task": tasks}
|
|
168
|
+
|
|
169
|
+
def get_job(
|
|
170
|
+
self,
|
|
171
|
+
job_id: str,
|
|
172
|
+
*,
|
|
173
|
+
include_params: bool = False,
|
|
174
|
+
include_spec: bool = False,
|
|
175
|
+
spec_format: str = "json",
|
|
176
|
+
timeout: TimeoutType = None,
|
|
177
|
+
) -> dict:
|
|
178
|
+
result = self._share_get(self._job_path(job_id), timeout=timeout)
|
|
179
|
+
# Share responses include these payloads unconditionally; preserve the
|
|
180
|
+
# existing CLI opt-in presentation contract.
|
|
181
|
+
result = dict(result)
|
|
182
|
+
if not include_params:
|
|
183
|
+
result.pop("params", None)
|
|
184
|
+
if not include_spec:
|
|
185
|
+
result.pop("job_spec_data", None)
|
|
186
|
+
elif spec_format == "yaml" and isinstance(result.get("job_spec_data"), dict):
|
|
187
|
+
import yaml
|
|
188
|
+
|
|
189
|
+
spec_data = dict(result["job_spec_data"])
|
|
190
|
+
if "spec" in spec_data:
|
|
191
|
+
spec_data["spec_yaml"] = yaml.safe_dump(
|
|
192
|
+
spec_data["spec"], allow_unicode=True, sort_keys=False
|
|
193
|
+
)
|
|
194
|
+
result["job_spec_data"] = spec_data
|
|
195
|
+
return result
|
|
196
|
+
|
|
197
|
+
def get_job_metrics(self, job_id: str) -> dict:
|
|
198
|
+
return self._share_get(self._job_path(job_id, "/metrics"))
|
|
199
|
+
|
|
200
|
+
def get_job_events(self, job_id: str, container: str | None = None) -> dict:
|
|
201
|
+
params = {"container": container} if container else {}
|
|
202
|
+
return self._share_get(self._job_path(job_id, "/events"), **params)
|
|
203
|
+
|
|
204
|
+
def get_job_containers(self, job_id: str) -> dict:
|
|
205
|
+
return self._share_get(self._job_path(job_id, "/containers"))
|
|
206
|
+
|
|
207
|
+
def get_job_logs(
|
|
208
|
+
self,
|
|
209
|
+
job_id: str,
|
|
210
|
+
container: str | None = None,
|
|
211
|
+
offset: int | None = 0,
|
|
212
|
+
limit: int | None = 500,
|
|
213
|
+
) -> dict:
|
|
214
|
+
params: dict[str, Any] = {}
|
|
215
|
+
if container:
|
|
216
|
+
params["container"] = container
|
|
217
|
+
if offset is not None:
|
|
218
|
+
params["offset"] = offset
|
|
219
|
+
if limit is not None:
|
|
220
|
+
params["limit"] = limit
|
|
221
|
+
return self._share_get(self._job_path(job_id, "/logs"), **params)
|
|
222
|
+
|
|
223
|
+
def get_job_artifacts(self, job_ids: list[str], *, download_mode=None) -> list:
|
|
224
|
+
if download_mode is not None:
|
|
225
|
+
raise ValueError("share artifact endpoint does not accept download_mode")
|
|
226
|
+
items: list = []
|
|
227
|
+
for job_id in job_ids:
|
|
228
|
+
items.extend(self._share_get(self._job_path(job_id, "/artifacts")))
|
|
229
|
+
return items
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ap-client"
|
|
7
|
-
version = "0.6.0.
|
|
7
|
+
version = "0.6.0.dev1"
|
|
8
8
|
description = "Agent Platform API Client & CLI"
|
|
9
9
|
readme = { text = "A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.", content-type = "text/markdown" }
|
|
10
10
|
requires-python = ">=3.10"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|