ap-client 0.5.1.dev5__tar.gz → 0.6.0.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/PKG-INFO +4 -3
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/api.py +7 -4
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/cli.py +180 -25
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/dataset_commands.py +53 -27
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/dataset_register.py +53 -6
- ap_client-0.6.0.dev1/ap_client/exporter.py +987 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/instance_commands.py +7 -2
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/irepo_sdk.py +133 -37
- ap_client-0.6.0.dev1/ap_client/managed_ak.py +177 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/manifest_view.py +5 -3
- ap_client-0.6.0.dev1/ap_client/repo_read_guard.py +45 -0
- ap_client-0.6.0.dev1/ap_client/share_api.py +229 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/pyproject.toml +4 -3
- ap_client-0.5.1.dev5/ap_client/exporter.py +0 -934
- ap_client-0.5.1.dev5/ap_client/managed_ak.py +0 -86
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/__init__.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/config.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/fs_commands.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/profile_commands.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/split_publish.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/tbb.py +0 -0
- {ap_client-0.5.1.dev5 → ap_client-0.6.0.dev1}/ap_client/waiter.py +0 -0
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0.dev1
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
|
+
Requires-Dist: irepo-artifact==0.2.1
|
|
6
7
|
Requires-Dist: pyyaml>=6.0
|
|
7
8
|
Requires-Dist: requests>=2.28.0
|
|
8
9
|
Requires-Dist: rich>=13.0.0
|
|
9
10
|
Requires-Dist: typer>=0.9.0
|
|
10
11
|
Requires-Dist: websockets>=13.0
|
|
11
12
|
Provides-Extra: all
|
|
12
|
-
Requires-Dist: instance-repo[oss]
|
|
13
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.2; extra == 'all'
|
|
13
14
|
Provides-Extra: dataset
|
|
14
|
-
Requires-Dist: instance-repo[oss]
|
|
15
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.2; extra == 'dataset'
|
|
15
16
|
Description-Content-Type: text/markdown
|
|
16
17
|
|
|
17
18
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -539,11 +539,11 @@ class APIClient:
|
|
|
539
539
|
|
|
540
540
|
def _post(self, path: str, data: Any, timeout: TimeoutType = None) -> Any:
|
|
541
541
|
"""Send a POST request."""
|
|
542
|
-
from .managed_ak import
|
|
542
|
+
from .managed_ak import parameter_ak_usage, require_parameter_ak_ack
|
|
543
543
|
|
|
544
|
-
references =
|
|
544
|
+
references, selectors = parameter_ak_usage(data) if path == "/jobs" else (set(), set())
|
|
545
545
|
result = self._request("POST", path, json_body=data, timeout=timeout)
|
|
546
|
-
require_parameter_ak_ack(result, references)
|
|
546
|
+
require_parameter_ak_ack(result, references, selectors)
|
|
547
547
|
return result
|
|
548
548
|
|
|
549
549
|
def _delete(self, path: str, *, params: Optional[dict] = None) -> Any:
|
|
@@ -709,13 +709,16 @@ class APIClient:
|
|
|
709
709
|
request_id: str,
|
|
710
710
|
duration_ms: float,
|
|
711
711
|
exc: Exception,
|
|
712
|
+
*,
|
|
713
|
+
error_detail: str | None = None,
|
|
712
714
|
) -> None:
|
|
715
|
+
detail = str(exc) if error_detail is None else error_detail
|
|
713
716
|
_write_stderr(
|
|
714
717
|
[
|
|
715
718
|
f"!! Error {method} {url}",
|
|
716
719
|
f" request_id={request_id}",
|
|
717
720
|
f" duration_ms={duration_ms:.1f}",
|
|
718
|
-
f" error={type(exc).__name__}: {
|
|
721
|
+
f" error={type(exc).__name__}: {detail}",
|
|
719
722
|
]
|
|
720
723
|
)
|
|
721
724
|
|
|
@@ -25,7 +25,7 @@ from ap_client.api import (
|
|
|
25
25
|
)
|
|
26
26
|
from ap_client.config import ENV_VAR_SPECS, ConfigurationError, _parse_bool, normalize_output_format
|
|
27
27
|
from ap_client.dataset_commands import register as _register_dataset_commands
|
|
28
|
-
from ap_client.exporter import export_group, export_job
|
|
28
|
+
from ap_client.exporter import export_group, export_job, export_jobs
|
|
29
29
|
from ap_client.fs_commands import register as _register_fs_commands
|
|
30
30
|
from ap_client.instance_commands import register as _register_instance_commands
|
|
31
31
|
from ap_client.managed_ak import (
|
|
@@ -359,6 +359,22 @@ def _emit_error(message: str) -> None:
|
|
|
359
359
|
error_console.print(f"[red]error: {message}[/]")
|
|
360
360
|
|
|
361
361
|
|
|
362
|
+
def _read_client(share_token: Optional[str] = None, group_id: Optional[str] = None):
|
|
363
|
+
if group_id and not share_token:
|
|
364
|
+
raise typer.BadParameter("--group-id requires --share-token")
|
|
365
|
+
client = get_client()
|
|
366
|
+
if share_token is None:
|
|
367
|
+
return client
|
|
368
|
+
from ap_client.share_api import ShareAPIClient
|
|
369
|
+
|
|
370
|
+
# Typer verbose tracebacks must not render the share token in locals.
|
|
371
|
+
app.pretty_exceptions_show_locals = False
|
|
372
|
+
try:
|
|
373
|
+
return ShareAPIClient.from_client(client, share_token, group_id=group_id)
|
|
374
|
+
except ValueError as exc:
|
|
375
|
+
raise typer.BadParameter(str(exc)) from exc
|
|
376
|
+
|
|
377
|
+
|
|
362
378
|
def _reject_account_pool_template_credential_cli(
|
|
363
379
|
account_pool: Optional[str],
|
|
364
380
|
*credential_types: Any,
|
|
@@ -3564,8 +3580,8 @@ def job_create(
|
|
|
3564
3580
|
if dry_run:
|
|
3565
3581
|
_emit_dry_run_warnings(dry_run_warnings)
|
|
3566
3582
|
body["dry_run"] = True
|
|
3567
|
-
result = client._post("/jobs", body, timeout=timeout)
|
|
3568
3583
|
try:
|
|
3584
|
+
result = client._post("/jobs", body, timeout=timeout)
|
|
3569
3585
|
require_managed_ak_ack(result, ak_id)
|
|
3570
3586
|
except ManagedAKProtocolError as exc:
|
|
3571
3587
|
_emit_error(str(exc))
|
|
@@ -4427,6 +4443,9 @@ def job_create(
|
|
|
4427
4443
|
@job_app.command("get")
|
|
4428
4444
|
def job_get(
|
|
4429
4445
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4446
|
+
share_token: Optional[str] = typer.Option(
|
|
4447
|
+
None, "--share-token", help="Read with a share token"
|
|
4448
|
+
),
|
|
4430
4449
|
include_params: bool = typer.Option(
|
|
4431
4450
|
False,
|
|
4432
4451
|
"--include-params",
|
|
@@ -4463,7 +4482,7 @@ def job_get(
|
|
|
4463
4482
|
include_spec = include_spec or _field_path_requires_spec(field)
|
|
4464
4483
|
if field == "job_spec_data.spec_yaml":
|
|
4465
4484
|
spec_format = "yaml"
|
|
4466
|
-
client =
|
|
4485
|
+
client = _read_client(share_token)
|
|
4467
4486
|
result = client.get_job(
|
|
4468
4487
|
job_id,
|
|
4469
4488
|
include_params=include_params,
|
|
@@ -4489,6 +4508,9 @@ def job_get(
|
|
|
4489
4508
|
@job_app.command("containers")
|
|
4490
4509
|
def job_containers(
|
|
4491
4510
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4511
|
+
share_token: Optional[str] = typer.Option(
|
|
4512
|
+
None, "--share-token", help="Read with a share token"
|
|
4513
|
+
),
|
|
4492
4514
|
output_format: str = typer.Option(
|
|
4493
4515
|
None,
|
|
4494
4516
|
"--format",
|
|
@@ -4497,7 +4519,7 @@ def job_containers(
|
|
|
4497
4519
|
):
|
|
4498
4520
|
"""List container names for a job."""
|
|
4499
4521
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
4500
|
-
client =
|
|
4522
|
+
client = _read_client(share_token)
|
|
4501
4523
|
result = client.get_job_containers(job_id)
|
|
4502
4524
|
if output_format == "table":
|
|
4503
4525
|
_print_records_table(result)
|
|
@@ -4559,6 +4581,9 @@ def _print_tail_logs(logs, tail: int) -> None:
|
|
|
4559
4581
|
@job_app.command("logs")
|
|
4560
4582
|
def job_logs(
|
|
4561
4583
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4584
|
+
share_token: Optional[str] = typer.Option(
|
|
4585
|
+
None, "--share-token", help="Read with a share token"
|
|
4586
|
+
),
|
|
4562
4587
|
container: Optional[str] = typer.Option("main", "--container", "-c", help="Container name"),
|
|
4563
4588
|
tail: Optional[int] = typer.Option(None, "--tail", "-n", help="Show only the last N log lines"),
|
|
4564
4589
|
follow: bool = typer.Option(
|
|
@@ -4570,7 +4595,7 @@ def job_logs(
|
|
|
4570
4595
|
),
|
|
4571
4596
|
):
|
|
4572
4597
|
"""Show job logs."""
|
|
4573
|
-
client =
|
|
4598
|
+
client = _read_client(share_token)
|
|
4574
4599
|
|
|
4575
4600
|
if tail is not None:
|
|
4576
4601
|
if tail <= 0:
|
|
@@ -4634,6 +4659,9 @@ def job_logs(
|
|
|
4634
4659
|
@job_app.command("events")
|
|
4635
4660
|
def job_events(
|
|
4636
4661
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4662
|
+
share_token: Optional[str] = typer.Option(
|
|
4663
|
+
None, "--share-token", help="Read with a share token"
|
|
4664
|
+
),
|
|
4637
4665
|
container: Optional[str] = typer.Option(None, "--container", "-c", help="Container name"),
|
|
4638
4666
|
output_format: str = typer.Option(
|
|
4639
4667
|
None,
|
|
@@ -4643,7 +4671,7 @@ def job_events(
|
|
|
4643
4671
|
):
|
|
4644
4672
|
"""Show job events."""
|
|
4645
4673
|
output_format = _normalize_output_format(output_format)
|
|
4646
|
-
client =
|
|
4674
|
+
client = _read_client(share_token)
|
|
4647
4675
|
result = client.get_job_events(job_id, container=container)
|
|
4648
4676
|
if output_format == "plain":
|
|
4649
4677
|
_print_job_events_plain(result)
|
|
@@ -4654,6 +4682,9 @@ def job_events(
|
|
|
4654
4682
|
@job_app.command("metrics")
|
|
4655
4683
|
def job_metrics(
|
|
4656
4684
|
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4685
|
+
share_token: Optional[str] = typer.Option(
|
|
4686
|
+
None, "--share-token", help="Read with a share token"
|
|
4687
|
+
),
|
|
4657
4688
|
output_format: str = typer.Option(
|
|
4658
4689
|
None,
|
|
4659
4690
|
"--format",
|
|
@@ -4662,7 +4693,7 @@ def job_metrics(
|
|
|
4662
4693
|
):
|
|
4663
4694
|
"""Show job metrics."""
|
|
4664
4695
|
output_format = _normalize_output_format(output_format)
|
|
4665
|
-
client =
|
|
4696
|
+
client = _read_client(share_token)
|
|
4666
4697
|
result = client.get_job_metrics(job_id)
|
|
4667
4698
|
if output_format == "plain":
|
|
4668
4699
|
_print_json(result)
|
|
@@ -4859,6 +4890,9 @@ def job_retry(
|
|
|
4859
4890
|
@job_app.command("artifacts")
|
|
4860
4891
|
def job_artifacts(
|
|
4861
4892
|
job_ids: str = typer.Argument(..., help="Job IDs (comma-separated for multiple)"),
|
|
4893
|
+
share_token: Optional[str] = typer.Option(
|
|
4894
|
+
None, "--share-token", help="Read with a share token"
|
|
4895
|
+
),
|
|
4862
4896
|
output_format: str = typer.Option(
|
|
4863
4897
|
None,
|
|
4864
4898
|
"--format",
|
|
@@ -4867,7 +4901,7 @@ def job_artifacts(
|
|
|
4867
4901
|
):
|
|
4868
4902
|
"""Show job artifact download links."""
|
|
4869
4903
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
4870
|
-
client =
|
|
4904
|
+
client = _read_client(share_token)
|
|
4871
4905
|
ids = [jid.strip() for jid in job_ids.split(",")]
|
|
4872
4906
|
results = client.get_job_artifacts(ids)
|
|
4873
4907
|
if output_format == "table":
|
|
@@ -4876,10 +4910,50 @@ def job_artifacts(
|
|
|
4876
4910
|
_print_formatted(results, output_format)
|
|
4877
4911
|
|
|
4878
4912
|
|
|
4913
|
+
def _validate_export_timeout(value: float) -> float:
|
|
4914
|
+
import math
|
|
4915
|
+
|
|
4916
|
+
if not math.isfinite(value) or value <= 0:
|
|
4917
|
+
raise typer.BadParameter("Timeout must be a finite number greater than zero")
|
|
4918
|
+
return value
|
|
4919
|
+
|
|
4920
|
+
|
|
4921
|
+
def _export_channel_reporter(emit):
|
|
4922
|
+
"""Report completed Jobs, including callbacks from Group worker threads."""
|
|
4923
|
+
from threading import Lock
|
|
4924
|
+
|
|
4925
|
+
from rich.text import Text
|
|
4926
|
+
|
|
4927
|
+
counts = {"oss": 0, "proxy": 0}
|
|
4928
|
+
lock = Lock()
|
|
4929
|
+
|
|
4930
|
+
def report(job_id, status, channel):
|
|
4931
|
+
with lock:
|
|
4932
|
+
suffix = ""
|
|
4933
|
+
if status == "downloaded" and channel in counts:
|
|
4934
|
+
counts[channel] += 1
|
|
4935
|
+
suffix = f" channel={channel}"
|
|
4936
|
+
emit(Text(f"{job_id} {status}{suffix}"))
|
|
4937
|
+
|
|
4938
|
+
return report, counts
|
|
4939
|
+
|
|
4940
|
+
|
|
4879
4941
|
@job_app.command("export")
|
|
4880
4942
|
def job_export(
|
|
4881
|
-
job_id: str = typer.Argument(..., help="Job ID"),
|
|
4943
|
+
job_id: str = typer.Argument(..., help="Job ID or comma-separated Job IDs"),
|
|
4944
|
+
share_token: Optional[str] = typer.Option(
|
|
4945
|
+
None, "--share-token", help="Export with a share token"
|
|
4946
|
+
),
|
|
4882
4947
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Export directory"),
|
|
4948
|
+
workers: int = typer.Option(
|
|
4949
|
+
1, "--workers", "-w", min=1, help="Number of concurrent export workers"
|
|
4950
|
+
),
|
|
4951
|
+
timeout: float = typer.Option(
|
|
4952
|
+
300,
|
|
4953
|
+
"--timeout",
|
|
4954
|
+
callback=_validate_export_timeout,
|
|
4955
|
+
help="SDK request timeout in seconds, not the total export duration",
|
|
4956
|
+
),
|
|
4883
4957
|
logs: bool = typer.Option(False, "--logs", help="Include container logs"),
|
|
4884
4958
|
events: bool = typer.Option(False, "--events", help="Include events"),
|
|
4885
4959
|
no_extract_artifacts: bool = typer.Option(
|
|
@@ -4889,17 +4963,50 @@ def job_export(
|
|
|
4889
4963
|
),
|
|
4890
4964
|
):
|
|
4891
4965
|
"""Export job artifacts to a local directory. Use --logs/--events to include logs and events."""
|
|
4892
|
-
|
|
4966
|
+
ids = list(dict.fromkeys(value.strip() for value in job_id.split(",")))
|
|
4967
|
+
if not all(ids):
|
|
4968
|
+
raise typer.BadParameter("Job IDs must not be empty", param_hint="job_id")
|
|
4969
|
+
client = _read_client(share_token)
|
|
4970
|
+
if len(ids) > 1:
|
|
4971
|
+
report, channels = _export_channel_reporter(print)
|
|
4972
|
+
dest, summary = export_jobs(
|
|
4973
|
+
client,
|
|
4974
|
+
ids,
|
|
4975
|
+
output,
|
|
4976
|
+
workers=workers,
|
|
4977
|
+
result_callback=report,
|
|
4978
|
+
timeout=timeout,
|
|
4979
|
+
include_logs=logs,
|
|
4980
|
+
include_events=events,
|
|
4981
|
+
extract_artifacts=not no_extract_artifacts,
|
|
4982
|
+
)
|
|
4983
|
+
print(
|
|
4984
|
+
f"Jobs exported: downloaded={summary.downloaded} "
|
|
4985
|
+
f"skipped={summary.skipped} failed={len(summary.failures)}"
|
|
4986
|
+
)
|
|
4987
|
+
print(f"channels: oss={channels['oss']} proxy={channels['proxy']}")
|
|
4988
|
+
print(f" path: {dest}")
|
|
4989
|
+
for failed_id, error in summary.failures.items():
|
|
4990
|
+
print(f"[red]Failed job {failed_id}:[/] {error}")
|
|
4991
|
+
if summary.failures:
|
|
4992
|
+
raise typer.Exit(code=1)
|
|
4993
|
+
return
|
|
4994
|
+
job_id = ids[0]
|
|
4995
|
+
result = {}
|
|
4893
4996
|
dest = export_job(
|
|
4894
4997
|
client,
|
|
4895
4998
|
job_id,
|
|
4896
4999
|
output,
|
|
5000
|
+
result_callback=lambda jid, status, channel: result.update(status=status, channel=channel),
|
|
5001
|
+
timeout=timeout,
|
|
4897
5002
|
include_logs=logs,
|
|
4898
5003
|
include_events=events,
|
|
4899
5004
|
extract_artifacts=not no_extract_artifacts,
|
|
4900
5005
|
)
|
|
4901
5006
|
print(f"[green]Job exported:[/] {job_id}")
|
|
4902
5007
|
print(f" path: {dest}")
|
|
5008
|
+
if result.get("channel") in {"oss", "proxy"} and result.get("status") == "downloaded":
|
|
5009
|
+
print(f" channel: {result['channel']}")
|
|
4903
5010
|
|
|
4904
5011
|
|
|
4905
5012
|
@job_app.command("wait")
|
|
@@ -5674,6 +5781,9 @@ def group_list(
|
|
|
5674
5781
|
@group_app.command("get")
|
|
5675
5782
|
def group_get(
|
|
5676
5783
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
5784
|
+
share_token: Optional[str] = typer.Option(
|
|
5785
|
+
None, "--share-token", help="Read with a share token"
|
|
5786
|
+
),
|
|
5677
5787
|
output_format: str = typer.Option(
|
|
5678
5788
|
None,
|
|
5679
5789
|
"--format",
|
|
@@ -5682,7 +5792,7 @@ def group_get(
|
|
|
5682
5792
|
):
|
|
5683
5793
|
"""Show Group details."""
|
|
5684
5794
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
5685
|
-
client =
|
|
5795
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
5686
5796
|
result = client.get_group(group_id)
|
|
5687
5797
|
if output_format == "table":
|
|
5688
5798
|
_print_detail(result)
|
|
@@ -6162,6 +6272,9 @@ def group_resume(
|
|
|
6162
6272
|
@group_app.command("jobs")
|
|
6163
6273
|
def group_jobs(
|
|
6164
6274
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6275
|
+
share_token: Optional[str] = typer.Option(
|
|
6276
|
+
None, "--share-token", help="Read with a share token"
|
|
6277
|
+
),
|
|
6165
6278
|
tag: Optional[list[str]] = typer.Option(
|
|
6166
6279
|
None, "--tag", help="Filter by tag (repeatable; multiple tags are combined with AND)"
|
|
6167
6280
|
),
|
|
@@ -6200,17 +6313,36 @@ def group_jobs(
|
|
|
6200
6313
|
if cursor and skip > 0:
|
|
6201
6314
|
raise typer.BadParameter("--cursor cannot be used with --skip greater than 0")
|
|
6202
6315
|
cursor_request = bool(next_token) or cursor
|
|
6203
|
-
|
|
6204
|
-
|
|
6205
|
-
|
|
6206
|
-
|
|
6207
|
-
|
|
6208
|
-
|
|
6209
|
-
|
|
6210
|
-
|
|
6211
|
-
|
|
6212
|
-
|
|
6213
|
-
|
|
6316
|
+
if share_token is not None:
|
|
6317
|
+
unsupported = (
|
|
6318
|
+
("--tag", bool(tag)),
|
|
6319
|
+
("--include-post-process", include_post_process),
|
|
6320
|
+
("--next-token", bool(next_token)),
|
|
6321
|
+
("--cursor", cursor),
|
|
6322
|
+
("--no-total", no_total),
|
|
6323
|
+
)
|
|
6324
|
+
for option, used in unsupported:
|
|
6325
|
+
if used:
|
|
6326
|
+
_emit_error(f"{option} is not supported with --share-token")
|
|
6327
|
+
raise typer.Exit(1)
|
|
6328
|
+
if skip < 0 or not 1 <= limit <= 500:
|
|
6329
|
+
raise typer.BadParameter(
|
|
6330
|
+
"share pagination requires --skip >= 0 and 1 <= --limit <= 500"
|
|
6331
|
+
)
|
|
6332
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6333
|
+
result = client.list_jobs(group_id=group_id, skip=skip, limit=limit)
|
|
6334
|
+
else:
|
|
6335
|
+
client = get_client()
|
|
6336
|
+
result = client.list_group_jobs(
|
|
6337
|
+
group_id,
|
|
6338
|
+
tag=tag,
|
|
6339
|
+
include_post_process=include_post_process,
|
|
6340
|
+
skip=skip,
|
|
6341
|
+
limit=limit,
|
|
6342
|
+
**({"next_token": next_token} if next_token else {}),
|
|
6343
|
+
**({"pagination": "cursor"} if cursor_request else {}),
|
|
6344
|
+
include_total=not no_total,
|
|
6345
|
+
)
|
|
6214
6346
|
if output_format == "table":
|
|
6215
6347
|
_print_job_list_plain(result, rich=True)
|
|
6216
6348
|
elif output_format == "plain":
|
|
@@ -6225,6 +6357,9 @@ def group_jobs(
|
|
|
6225
6357
|
@group_app.command("stats")
|
|
6226
6358
|
def group_stats(
|
|
6227
6359
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6360
|
+
share_token: Optional[str] = typer.Option(
|
|
6361
|
+
None, "--share-token", help="Read with a share token"
|
|
6362
|
+
),
|
|
6228
6363
|
output_format: str = typer.Option(
|
|
6229
6364
|
None,
|
|
6230
6365
|
"--format",
|
|
@@ -6233,7 +6368,7 @@ def group_stats(
|
|
|
6233
6368
|
):
|
|
6234
6369
|
"""Show Group progress."""
|
|
6235
6370
|
output_format = _normalize_output_format(output_format, keep_table=True)
|
|
6236
|
-
client =
|
|
6371
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6237
6372
|
result = client.get_group_stats(group_id)
|
|
6238
6373
|
if output_format == "table":
|
|
6239
6374
|
_print_detail(result)
|
|
@@ -6244,6 +6379,9 @@ def group_stats(
|
|
|
6244
6379
|
@group_app.command("eval")
|
|
6245
6380
|
def group_eval(
|
|
6246
6381
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6382
|
+
share_token: Optional[str] = typer.Option(
|
|
6383
|
+
None, "--share-token", help="Read with a share token"
|
|
6384
|
+
),
|
|
6247
6385
|
summary_only: bool = typer.Option(
|
|
6248
6386
|
False,
|
|
6249
6387
|
"--summary-only",
|
|
@@ -6257,7 +6395,7 @@ def group_eval(
|
|
|
6257
6395
|
):
|
|
6258
6396
|
"""Show Group aggregated evaluation results."""
|
|
6259
6397
|
output_format = _normalize_output_format(output_format)
|
|
6260
|
-
client =
|
|
6398
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6261
6399
|
result = (
|
|
6262
6400
|
client.get_group_eval_summary(group_id) if summary_only else client.get_group_eval(group_id)
|
|
6263
6401
|
)
|
|
@@ -6370,10 +6508,19 @@ def group_access_list(group_id: str = typer.Argument(..., help="Group ID")):
|
|
|
6370
6508
|
@group_app.command("export")
|
|
6371
6509
|
def group_export(
|
|
6372
6510
|
group_id: str = typer.Argument(..., help="Group ID"),
|
|
6511
|
+
share_token: Optional[str] = typer.Option(
|
|
6512
|
+
None, "--share-token", help="Export with a share token"
|
|
6513
|
+
),
|
|
6373
6514
|
output: Optional[Path] = typer.Option(None, "--output", "-o", help="Export directory"),
|
|
6374
6515
|
workers: int = typer.Option(
|
|
6375
6516
|
4, "--workers", "-w", min=1, help="Number of concurrent export workers"
|
|
6376
6517
|
),
|
|
6518
|
+
timeout: float = typer.Option(
|
|
6519
|
+
300,
|
|
6520
|
+
"--timeout",
|
|
6521
|
+
callback=_validate_export_timeout,
|
|
6522
|
+
help="SDK request timeout in seconds, not the total export duration",
|
|
6523
|
+
),
|
|
6377
6524
|
logs: bool = typer.Option(False, "--logs", help="Include container logs"),
|
|
6378
6525
|
events: bool = typer.Option(False, "--events", help="Include events"),
|
|
6379
6526
|
no_extract_artifacts: bool = typer.Option(
|
|
@@ -6405,7 +6552,11 @@ def group_export(
|
|
|
6405
6552
|
_emit_error("--include-files/--exclude-files cannot be used with --no-extract-artifacts")
|
|
6406
6553
|
raise typer.Exit(1)
|
|
6407
6554
|
|
|
6408
|
-
|
|
6555
|
+
if share_token is not None and (include_file_selectors or exclude_file_selectors):
|
|
6556
|
+
_emit_error("--include-files/--exclude-files are not supported with --share-token")
|
|
6557
|
+
raise typer.Exit(1)
|
|
6558
|
+
|
|
6559
|
+
client = _read_client(share_token, group_id if share_token else None)
|
|
6409
6560
|
from rich.progress import BarColumn, Progress, TaskProgressColumn, TextColumn
|
|
6410
6561
|
|
|
6411
6562
|
with Progress(
|
|
@@ -6434,12 +6585,15 @@ def group_export(
|
|
|
6434
6585
|
current_stage=current_stage,
|
|
6435
6586
|
)
|
|
6436
6587
|
|
|
6588
|
+
report, channels = _export_channel_reporter(progress.console.print)
|
|
6437
6589
|
dest, summary = export_group(
|
|
6438
6590
|
client,
|
|
6439
6591
|
group_id,
|
|
6440
6592
|
output,
|
|
6441
6593
|
progress_callback=_on_progress,
|
|
6442
6594
|
workers=workers,
|
|
6595
|
+
result_callback=report,
|
|
6596
|
+
timeout=timeout,
|
|
6443
6597
|
include_logs=logs,
|
|
6444
6598
|
include_events=events,
|
|
6445
6599
|
extract_artifacts=not no_extract_artifacts,
|
|
@@ -6452,6 +6606,7 @@ def group_export(
|
|
|
6452
6606
|
f" [green]{summary.downloaded}[/] jobs downloaded, "
|
|
6453
6607
|
f"[yellow]{summary.skipped}[/] jobs skipped (no artifact)"
|
|
6454
6608
|
)
|
|
6609
|
+
print(f"channels: oss={channels['oss']} proxy={channels['proxy']}")
|
|
6455
6610
|
if summary.skipped_job_names:
|
|
6456
6611
|
print(f" Skipped: {', '.join(summary.skipped_job_names)}")
|
|
6457
6612
|
|
|
@@ -15,6 +15,9 @@ from typing import TYPE_CHECKING, Any, List, Optional
|
|
|
15
15
|
|
|
16
16
|
import typer
|
|
17
17
|
|
|
18
|
+
from .repo_read_guard import STORAGE_SIDES as _STORAGE_SIDES
|
|
19
|
+
from .repo_read_guard import oss_fallback_kwargs
|
|
20
|
+
|
|
18
21
|
if TYPE_CHECKING: # pragma: no cover - 仅用于类型标注,运行时不 import
|
|
19
22
|
from datetime import datetime
|
|
20
23
|
|
|
@@ -316,13 +319,8 @@ def _env_kwargs(environment: str) -> dict:
|
|
|
316
319
|
return {"environment": environment} if environment and environment != "auto" else {}
|
|
317
320
|
|
|
318
321
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
"dev": "pre",
|
|
322
|
-
"internal-dev": "pre",
|
|
323
|
-
"external-dev": "pre",
|
|
324
|
-
"prod": "online",
|
|
325
|
-
}
|
|
322
|
+
def _read_env_kwargs(repo: Any, client: Any, method: str, environment: str) -> dict:
|
|
323
|
+
return {**_env_kwargs(environment), **oss_fallback_kwargs(repo, client, method, environment)}
|
|
326
324
|
|
|
327
325
|
|
|
328
326
|
def _check_storage_side(repo: Any, environment: str) -> None:
|
|
@@ -419,7 +417,9 @@ def _model_dict(record: Any) -> dict:
|
|
|
419
417
|
key: value
|
|
420
418
|
for key, value in payload.items()
|
|
421
419
|
if value is not None
|
|
422
|
-
and (
|
|
420
|
+
and (
|
|
421
|
+
key == "registered" or not (value is False or value == "" or value == [] or value == {})
|
|
422
|
+
)
|
|
423
423
|
}
|
|
424
424
|
|
|
425
425
|
|
|
@@ -609,7 +609,7 @@ def _fetch_version(
|
|
|
609
609
|
detail = repo.versions.get(
|
|
610
610
|
dataset,
|
|
611
611
|
version,
|
|
612
|
-
**
|
|
612
|
+
**_read_env_kwargs(repo, repo.versions, "get", env),
|
|
613
613
|
**({"metadata_model": metadata_model} if metadata_model else {}),
|
|
614
614
|
)
|
|
615
615
|
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
@@ -646,19 +646,30 @@ def _unversioned_splits(
|
|
|
646
646
|
env = _environment(environment)
|
|
647
647
|
repo = _dataset_repo()
|
|
648
648
|
try:
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
649
|
+
records: list[dict] = []
|
|
650
|
+
page = 1
|
|
651
|
+
while True:
|
|
652
|
+
result = repo.splits.list(
|
|
653
|
+
dataset,
|
|
654
|
+
version_scope="unversioned",
|
|
655
|
+
page_size=_SPLIT_PAGE_SIZE,
|
|
656
|
+
**({"page": page} if page > 1 else {}),
|
|
657
|
+
**_read_env_kwargs(repo, repo.splits, "list", env),
|
|
658
|
+
)
|
|
659
|
+
items = getattr(result, "items", None) or []
|
|
660
|
+
for item in items:
|
|
661
|
+
record = _model_dict(item)
|
|
662
|
+
if not _version_value(record.get("version")):
|
|
663
|
+
records.append(record)
|
|
664
|
+
if not items or getattr(result, "has_more", False) is not True:
|
|
665
|
+
break
|
|
666
|
+
page += 1
|
|
656
667
|
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
657
668
|
raise _fail(
|
|
658
669
|
f"dataset {dataset} (no version space): {stage} failed: {_reason(exc)}",
|
|
659
670
|
f"Check the dataset name and permission: ap dataset get {dataset}",
|
|
660
671
|
) from exc
|
|
661
|
-
return
|
|
672
|
+
return records
|
|
662
673
|
|
|
663
674
|
|
|
664
675
|
def _unversioned_split(
|
|
@@ -670,13 +681,14 @@ def _unversioned_split(
|
|
|
670
681
|
env = _environment(environment)
|
|
671
682
|
repo = _dataset_repo()
|
|
672
683
|
try:
|
|
673
|
-
record = repo.splits.get(dataset, split, **
|
|
684
|
+
record = repo.splits.get(dataset, split, **_read_env_kwargs(repo, repo.splits, "get", env))
|
|
674
685
|
except Exception as exc: # noqa: BLE001
|
|
675
686
|
raise _fail(
|
|
676
687
|
f"dataset {dataset} (no version space) split {split}: query failed: {_reason(exc)}",
|
|
677
688
|
f"Check the dataset name and permission: ap dataset get {dataset}",
|
|
678
689
|
) from exc
|
|
679
|
-
|
|
690
|
+
detail = _model_dict(record) if record is not None else None
|
|
691
|
+
return detail if detail is not None and not _version_value(detail.get("version")) else None
|
|
680
692
|
|
|
681
693
|
|
|
682
694
|
def _split_records(detail: dict) -> list[dict]:
|
|
@@ -1120,7 +1132,11 @@ def _print_dataset_environments(repo: "Repo", dataset: str, detail: dict) -> Non
|
|
|
1120
1132
|
rows: list[dict] = []
|
|
1121
1133
|
for env in envs:
|
|
1122
1134
|
try:
|
|
1123
|
-
env_detail = _model_dict(
|
|
1135
|
+
env_detail = _model_dict(
|
|
1136
|
+
repo.datasets.get_detail(
|
|
1137
|
+
dataset, **_read_env_kwargs(repo, repo.datasets, "get_detail", env)
|
|
1138
|
+
)
|
|
1139
|
+
)
|
|
1124
1140
|
except Exception: # noqa: BLE001 - 摘要尽力而为
|
|
1125
1141
|
rows.append({"environment": env, "versions": "?", "splits": "?", "latest_version": "?"})
|
|
1126
1142
|
continue
|
|
@@ -1155,7 +1171,9 @@ def dataset_get(
|
|
|
1155
1171
|
env = _environment(environment)
|
|
1156
1172
|
repo = _dataset_repo()
|
|
1157
1173
|
try:
|
|
1158
|
-
detail = repo.datasets.get_detail(
|
|
1174
|
+
detail = repo.datasets.get_detail(
|
|
1175
|
+
dataset, **_read_env_kwargs(repo, repo.datasets, "get_detail", env)
|
|
1176
|
+
)
|
|
1159
1177
|
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
1160
1178
|
if _business_code(exc) not in (92001, 94021):
|
|
1161
1179
|
raise _fail(
|
|
@@ -1164,13 +1182,13 @@ def dataset_get(
|
|
|
1164
1182
|
) from exc
|
|
1165
1183
|
detail = None
|
|
1166
1184
|
if detail is None:
|
|
1167
|
-
l1, _, l2 = dataset.partition("/")
|
|
1168
1185
|
rendered = {
|
|
1169
1186
|
"dataset_name": dataset,
|
|
1170
|
-
"dataset_l1": l1,
|
|
1171
|
-
"dataset_l2": l2,
|
|
1172
1187
|
"registered": False,
|
|
1173
1188
|
}
|
|
1189
|
+
parts = dataset.split("/")
|
|
1190
|
+
if len(parts) == 2 and all(parts):
|
|
1191
|
+
rendered.update(dataset_l1=parts[0], dataset_l2=parts[1])
|
|
1174
1192
|
else:
|
|
1175
1193
|
rendered = _registered_dict(detail)
|
|
1176
1194
|
_emit_detail(rendered, output_format, order=_DATASET_DETAIL_ORDER, always=("status",))
|
|
@@ -1294,7 +1312,11 @@ def dataset_claim(
|
|
|
1294
1312
|
env = _environment(environment)
|
|
1295
1313
|
repo = _dataset_repo()
|
|
1296
1314
|
try:
|
|
1297
|
-
detail = _model_dict(
|
|
1315
|
+
detail = _model_dict(
|
|
1316
|
+
repo.datasets.get_detail(
|
|
1317
|
+
dataset, **_read_env_kwargs(repo, repo.datasets, "get_detail", env)
|
|
1318
|
+
)
|
|
1319
|
+
)
|
|
1298
1320
|
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
1299
1321
|
raise _fail(
|
|
1300
1322
|
f"dataset {dataset}: pre-claim query failed: {_reason(exc)}",
|
|
@@ -1372,7 +1394,11 @@ def dataset_claim(
|
|
|
1372
1394
|
owner = str(detail.get("owner") or "").strip()
|
|
1373
1395
|
if not owner:
|
|
1374
1396
|
try:
|
|
1375
|
-
fresh = _model_dict(
|
|
1397
|
+
fresh = _model_dict(
|
|
1398
|
+
repo.datasets.get_detail(
|
|
1399
|
+
dataset, **_read_env_kwargs(repo, repo.datasets, "get_detail", env)
|
|
1400
|
+
)
|
|
1401
|
+
)
|
|
1376
1402
|
owner = str(fresh.get("owner") or "").strip()
|
|
1377
1403
|
except Exception: # noqa: BLE001 - 取不到 owner 不影响幂等退出
|
|
1378
1404
|
owner = ""
|
|
@@ -1658,7 +1684,7 @@ def dataset_version_list(
|
|
|
1658
1684
|
page=page,
|
|
1659
1685
|
page_size=page_size,
|
|
1660
1686
|
**({"metadata_model": model} if model else {}),
|
|
1661
|
-
**
|
|
1687
|
+
**_read_env_kwargs(repo, repo.versions, "list_paged", env),
|
|
1662
1688
|
)
|
|
1663
1689
|
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
1664
1690
|
raise _fail(
|