ap-client 0.5.2.dev0__tar.gz → 0.5.3.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/PKG-INFO +3 -3
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/cli.py +2 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/instance_commands.py +41 -449
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/irepo_sdk.py +50 -18
- ap_client-0.5.3.dev1/ap_client/ossutil_commands.py +286 -0
- ap_client-0.5.3.dev1/ap_client/ossutil_io.py +1034 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/repo_read_guard.py +6 -2
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/pyproject.toml +3 -3
- ap_client-0.5.2.dev0/ap_client/instance_options.py +0 -94
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/__init__.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/akpool_commands.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/api.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/config.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/dataset_commands.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/dataset_register.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/exporter.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/fs_commands.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/managed_ak.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/manifest_view.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/profile_commands.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/share_api.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/split_publish.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/tbb.py +0 -0
- {ap_client-0.5.2.dev0 → ap_client-0.5.3.dev1}/ap_client/waiter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: ap-client
|
|
3
|
-
Version: 0.5.
|
|
3
|
+
Version: 0.5.3.dev1
|
|
4
4
|
Summary: Agent Platform API Client & CLI
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Requires-Dist: irepo-artifact==0.2.1
|
|
@@ -10,9 +10,9 @@ Requires-Dist: rich>=13.0.0
|
|
|
10
10
|
Requires-Dist: typer>=0.9.0
|
|
11
11
|
Requires-Dist: websockets>=13.0
|
|
12
12
|
Provides-Extra: all
|
|
13
|
-
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.5.
|
|
13
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.5.dev2; extra == 'all'
|
|
14
14
|
Provides-Extra: dataset
|
|
15
|
-
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.5.
|
|
15
|
+
Requires-Dist: instance-repo[oss]<2.0.0,>=1.3.5.dev2; extra == 'dataset'
|
|
16
16
|
Description-Content-Type: text/markdown
|
|
17
17
|
|
|
18
18
|
A lightweight Python SDK and command line interface for Agent Platform. It provides helpers for configuring API access and managing templates, datasets, jobs, and groups.
|
|
@@ -35,6 +35,7 @@ from ap_client.managed_ak import (
|
|
|
35
35
|
validate_ak_id,
|
|
36
36
|
validate_ak_selection,
|
|
37
37
|
)
|
|
38
|
+
from ap_client.ossutil_commands import register as _register_ossutil_commands
|
|
38
39
|
from ap_client.profile_commands import profile_app
|
|
39
40
|
from ap_client.waiter import (
|
|
40
41
|
WaitTimeoutError,
|
|
@@ -167,6 +168,7 @@ app.add_typer(checkpoint_app, name="checkpoint")
|
|
|
167
168
|
# ap dataset version/split/access 嵌套在 dataset 下;instance/fs 是一等资源。
|
|
168
169
|
_register_dataset_commands(dataset_app)
|
|
169
170
|
_register_instance_commands(app)
|
|
171
|
+
_register_ossutil_commands(app)
|
|
170
172
|
_register_fs_commands(app)
|
|
171
173
|
_register_akpool_commands(app)
|
|
172
174
|
|
|
@@ -1,8 +1,23 @@
|
|
|
1
|
-
"""
|
|
1
|
+
"""``ap instance`` 命令族(实现设计 §4.4)。
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
``ap instance``(设计 §4.4):
|
|
4
|
+
|
|
5
|
+
* ``validate <path>...`` —— ``repo.instances.validate``,纯本地,不发远端请求;
|
|
6
|
+
* ``list`` / ``get`` —— **控制面**,走 ``api.list_dataset_series_instances`` /
|
|
7
|
+
``api.get_dataset_series_instance``,``--split`` 必填,不需要 SDK;
|
|
8
|
+
* ``push`` / ``pull`` —— 数据面,走 ``repo.instances.push_many`` / ``pull``。
|
|
9
|
+
|
|
10
|
+
分层约束(设计 §2):``instance list/get`` 属控制面,必须在没装
|
|
11
|
+
``ap-client[dataset]`` 时照常工作——所以 SDK 只能在**需要它的那几个命令函数
|
|
12
|
+
内部**懒加载:
|
|
13
|
+
|
|
14
|
+
from .irepo_sdk import SdkMissing, load_repo, sdk_error_exit
|
|
15
|
+
|
|
16
|
+
``SdkMissing`` → 打印 ``str(exc)`` 到 stderr 并 ``typer.Exit(2)``(缺依赖是使用
|
|
17
|
+
错误);SDK 抛出的业务异常 → ``sdk_error_exit(exc, what=..., stage=...)``。
|
|
18
|
+
|
|
19
|
+
渲染与退出码沿用 ``ap_client.cli`` 的既有 helper,同样函数内部 import 以避免与
|
|
20
|
+
``cli`` 循环依赖。默认列:``INSTANCE SPLIT VERSION DIGEST UPDATED_AT REGISTERED``。
|
|
6
21
|
"""
|
|
7
22
|
|
|
8
23
|
from __future__ import annotations
|
|
@@ -48,7 +63,6 @@ _VERSION_HELP = (
|
|
|
48
63
|
)
|
|
49
64
|
|
|
50
65
|
|
|
51
|
-
from . import instance_options # noqa: E402
|
|
52
66
|
from .dataset_commands import ( # noqa: E402
|
|
53
67
|
_ENVIRONMENT_HELP,
|
|
54
68
|
_METADATA_MODEL_HELP,
|
|
@@ -110,22 +124,6 @@ def _fail(message: str, hint: str, exit_code: int = 1) -> "typer.Exit":
|
|
|
110
124
|
return typer.Exit(exit_code)
|
|
111
125
|
|
|
112
126
|
|
|
113
|
-
def _path_value(value: Optional[str]) -> Optional[str]:
|
|
114
|
-
try:
|
|
115
|
-
return instance_options.relative_path(value)
|
|
116
|
-
except ValueError as exc:
|
|
117
|
-
raise _fail(str(exc), "Use --path solution, --path tests/sealed, or --path .", 2) from exc
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def _enhanced_options(owner: Any, method: str, options: dict[str, Any]) -> dict[str, Any]:
|
|
121
|
-
try:
|
|
122
|
-
return instance_options.sdk_options(owner, method, options)
|
|
123
|
-
except (TypeError, ValueError) as exc:
|
|
124
|
-
raise _fail(
|
|
125
|
-
str(exc), "Upgrade Dataset support: pip install --upgrade 'ap-client[dataset]'", 2
|
|
126
|
-
) from exc
|
|
127
|
-
|
|
128
|
-
|
|
129
127
|
def _load_repo_or_exit():
|
|
130
128
|
"""懒加载 SDK;缺 extra → 安装提示 + 退出码 2(设计 §2)。
|
|
131
129
|
|
|
@@ -466,40 +464,15 @@ def instance_get(
|
|
|
466
464
|
metadata_model: Optional[str] = typer.Option(
|
|
467
465
|
None, "--metadata-model", help=_METADATA_MODEL_HELP
|
|
468
466
|
),
|
|
469
|
-
path: Optional[str] = typer.Option(
|
|
470
|
-
None,
|
|
471
|
-
"--path",
|
|
472
|
-
help="List a subdirectory relative to the instance assets; use '.' for the root",
|
|
473
|
-
),
|
|
474
|
-
page_size: int = typer.Option(
|
|
475
|
-
100, "--page-size", min=1, max=1000, help="Directory listing page size"
|
|
476
|
-
),
|
|
477
|
-
max_pages: int = typer.Option(50, "--max-pages", min=1, help="Maximum directory listing pages"),
|
|
478
467
|
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
479
468
|
):
|
|
480
|
-
"""Show instance metadata
|
|
469
|
+
"""Show full instance metadata (SDK-backed)."""
|
|
481
470
|
from .irepo_sdk import _reason
|
|
482
471
|
|
|
483
472
|
output_format = _fmt(output_format)
|
|
484
|
-
resolved_path = _path_value(path)
|
|
485
473
|
resolved_version = _version_value(version)
|
|
486
474
|
repo = _load_repo_or_exit()
|
|
487
475
|
env = _environment(environment)
|
|
488
|
-
if resolved_path is not None:
|
|
489
|
-
_list_instance_objects(
|
|
490
|
-
repo,
|
|
491
|
-
dataset,
|
|
492
|
-
resolved_version,
|
|
493
|
-
instance,
|
|
494
|
-
split,
|
|
495
|
-
resolved_path,
|
|
496
|
-
env,
|
|
497
|
-
_metadata_model_for_split(metadata_model, split=split),
|
|
498
|
-
page_size,
|
|
499
|
-
max_pages,
|
|
500
|
-
output_format,
|
|
501
|
-
)
|
|
502
|
-
return
|
|
503
476
|
read_kwargs = _sdk_read_kwargs(
|
|
504
477
|
repo.instances,
|
|
505
478
|
"get",
|
|
@@ -538,65 +511,6 @@ def instance_get(
|
|
|
538
511
|
typer.echo(line)
|
|
539
512
|
|
|
540
513
|
|
|
541
|
-
def _list_instance_objects(
|
|
542
|
-
repo: Any,
|
|
543
|
-
dataset: str,
|
|
544
|
-
version: str,
|
|
545
|
-
instance: str,
|
|
546
|
-
split: str,
|
|
547
|
-
path: str,
|
|
548
|
-
environment: str,
|
|
549
|
-
metadata_model: str,
|
|
550
|
-
page_size: int,
|
|
551
|
-
max_pages: int,
|
|
552
|
-
output_format: str,
|
|
553
|
-
) -> None:
|
|
554
|
-
from .irepo_sdk import _reason
|
|
555
|
-
|
|
556
|
-
kwargs = _enhanced_options(
|
|
557
|
-
repo.instances,
|
|
558
|
-
"list_objects",
|
|
559
|
-
{
|
|
560
|
-
"version": version,
|
|
561
|
-
"path": path,
|
|
562
|
-
"page_size": page_size,
|
|
563
|
-
"max_pages": max_pages,
|
|
564
|
-
},
|
|
565
|
-
)
|
|
566
|
-
kwargs.update(
|
|
567
|
-
_sdk_read_kwargs(
|
|
568
|
-
repo.instances, "list_objects", environment=environment, metadata_model=metadata_model
|
|
569
|
-
)
|
|
570
|
-
)
|
|
571
|
-
try:
|
|
572
|
-
result = repo.instances.list_objects(dataset, instance, split=split, **kwargs)
|
|
573
|
-
except Exception as exc: # noqa: BLE001
|
|
574
|
-
raise _fail(
|
|
575
|
-
f"Failed to list instance {instance} directory {path or '.'}: {_reason(exc)}",
|
|
576
|
-
"Check the instance, directory, environment and read permission",
|
|
577
|
-
) from exc
|
|
578
|
-
payload = _model_dict(result)
|
|
579
|
-
payload.update(instance=instance, path=path or ".")
|
|
580
|
-
rows = [dict(item, type="directory", size_bytes="") for item in payload.get("directories", [])]
|
|
581
|
-
rows.extend(dict(item, type="file") for item in payload.get("files", []))
|
|
582
|
-
from .manifest_view import _visible
|
|
583
|
-
|
|
584
|
-
rows = [
|
|
585
|
-
{key: _visible(value) if isinstance(value, str) else value for key, value in row.items()}
|
|
586
|
-
for row in rows
|
|
587
|
-
]
|
|
588
|
-
_emit(
|
|
589
|
-
payload,
|
|
590
|
-
rows,
|
|
591
|
-
(("type", "TYPE"), ("name", "NAME"), ("path", "PATH"), ("size_bytes", "BYTES")),
|
|
592
|
-
output_format,
|
|
593
|
-
)
|
|
594
|
-
if payload.get("truncated"):
|
|
595
|
-
typer.echo(
|
|
596
|
-
"warning: directory listing is incomplete; increase --max-pages and retry", err=True
|
|
597
|
-
)
|
|
598
|
-
|
|
599
|
-
|
|
600
514
|
def _check_version_autocreate(repo, dataset: str, version: str, *, environment: str) -> bool:
|
|
601
515
|
"""version 不存在时 push 会触发服务端自动创建;返回是否将发生自动创建。
|
|
602
516
|
|
|
@@ -624,14 +538,7 @@ def _check_version_autocreate(repo, dataset: str, version: str, *, environment:
|
|
|
624
538
|
|
|
625
539
|
|
|
626
540
|
def _confirm_upload(
|
|
627
|
-
dataset: str,
|
|
628
|
-
version: str,
|
|
629
|
-
split: str,
|
|
630
|
-
count: int,
|
|
631
|
-
*,
|
|
632
|
-
yes: bool,
|
|
633
|
-
auto_create: bool,
|
|
634
|
-
prune: bool = False,
|
|
541
|
+
dataset: str, version: str, split: str, count: int, *, yes: bool, auto_create: bool
|
|
635
542
|
) -> None:
|
|
636
543
|
"""push 写远端前统一二次确认(可 ``-y/--yes`` 跳过),与 split publish 同口径。"""
|
|
637
544
|
if yes:
|
|
@@ -639,10 +546,6 @@ def _confirm_upload(
|
|
|
639
546
|
label = _version_label(version)
|
|
640
547
|
target = f"dataset {dataset} version {label} split {split}"
|
|
641
548
|
note = f" Version {label} does not exist and will be auto-created." if auto_create else ""
|
|
642
|
-
if prune:
|
|
643
|
-
note += (
|
|
644
|
-
" Remote instance records absent from these paths will be removed; OSS files remain."
|
|
645
|
-
)
|
|
646
549
|
if not sys.stdin.isatty():
|
|
647
550
|
raise _fail(
|
|
648
551
|
f"Uploading {count} instance(s) to {target} requires confirmation.{note}",
|
|
@@ -665,29 +568,6 @@ def instance_push(
|
|
|
665
568
|
version: Optional[str] = typer.Option(None, "--version", help=_VERSION_HELP),
|
|
666
569
|
environment: Optional[str] = typer.Option(None, "--environment", help=_ENVIRONMENT_HELP),
|
|
667
570
|
overwrite: bool = typer.Option(False, "--overwrite", help="Allow overwriting existing objects"),
|
|
668
|
-
skip_unchanged: bool = typer.Option(
|
|
669
|
-
False, "--skip-unchanged", help="Skip cases with matching source hashes"
|
|
670
|
-
),
|
|
671
|
-
content_hashes: Optional[str] = typer.Option(
|
|
672
|
-
None, "--content-hashes", help="JSON file mapping instance IDs to source hashes"
|
|
673
|
-
),
|
|
674
|
-
compress_level: Optional[int] = typer.Option(
|
|
675
|
-
None,
|
|
676
|
-
"--compress-level",
|
|
677
|
-
min=1,
|
|
678
|
-
max=9,
|
|
679
|
-
help="Gzip compression level; 1 is fastest, 9 is smallest",
|
|
680
|
-
),
|
|
681
|
-
concurrency: int = typer.Option(8, "--concurrency", min=1, help="Concurrent instance uploads"),
|
|
682
|
-
continue_on_error: bool = typer.Option(
|
|
683
|
-
False, "--continue-on-error", help="Finish other uploads and report failed cases"
|
|
684
|
-
),
|
|
685
|
-
progress: bool = typer.Option(False, "--progress", help="Write SDK upload progress to stderr"),
|
|
686
|
-
prune: bool = typer.Option(
|
|
687
|
-
False,
|
|
688
|
-
"--prune",
|
|
689
|
-
help="After delivery, remove remote metadata absent from the supplied cases; OSS files remain",
|
|
690
|
-
),
|
|
691
571
|
yes: bool = typer.Option(
|
|
692
572
|
False,
|
|
693
573
|
"--yes",
|
|
@@ -714,22 +594,6 @@ def instance_push(
|
|
|
714
594
|
f"{len(broken)}/{len(results)} instances failed local validation; nothing was uploaded",
|
|
715
595
|
"Run ap instance validate <path>... to fix the issues, then retry",
|
|
716
596
|
)
|
|
717
|
-
try:
|
|
718
|
-
hashes = instance_options.upload_hashes(
|
|
719
|
-
results, instance_options.read_hashes(content_hashes), skip_unchanged
|
|
720
|
-
)
|
|
721
|
-
if prune:
|
|
722
|
-
if not resolved:
|
|
723
|
-
raise ValueError("--prune requires an explicit --version")
|
|
724
|
-
if any(not Path(record["path"]).is_dir() for record in results):
|
|
725
|
-
raise ValueError("--prune requires a collection of instance directories")
|
|
726
|
-
instance_options.require_working_prune()
|
|
727
|
-
if skip_unchanged or prune:
|
|
728
|
-
ids = [record["instance"] for record in results]
|
|
729
|
-
if len(ids) != len(set(ids)):
|
|
730
|
-
raise ValueError("Each supplied path must have a distinct instance ID")
|
|
731
|
-
except (ImportError, OSError, ValueError) as exc:
|
|
732
|
-
raise _fail(str(exc), "Check the hashes, input paths and installed Dataset SDK", 2) from exc
|
|
733
597
|
|
|
734
598
|
# ② 写远端前统一二次确认(-y/--yes 跳过);版本不存在时提示将被自动创建。
|
|
735
599
|
# (split 已发布不可写由**服务端**裁决——pre 环境里 ingest 产出的 split 记录状态
|
|
@@ -740,41 +604,13 @@ def instance_push(
|
|
|
740
604
|
|
|
741
605
|
repo = _load_repo_or_exit()
|
|
742
606
|
_check_storage_side(repo, env)
|
|
743
|
-
enhanced: dict[str, Any] = {}
|
|
744
|
-
if skip_unchanged:
|
|
745
|
-
enhanced["skip_unchanged"] = True
|
|
746
|
-
if compress_level is not None:
|
|
747
|
-
enhanced["compress_level"] = compress_level
|
|
748
|
-
# The SDK batch path compares stored hashes before selecting format-specific uploads.
|
|
749
|
-
batch = len(targets) > 1 or prune or skip_unchanged
|
|
750
|
-
if hashes:
|
|
751
|
-
if batch:
|
|
752
|
-
enhanced["content_hashes"] = hashes
|
|
753
|
-
else:
|
|
754
|
-
source_hash = hashes.get(results[0]["instance"])
|
|
755
|
-
if source_hash:
|
|
756
|
-
enhanced["content_hash"] = source_hash
|
|
757
|
-
if batch and progress:
|
|
758
|
-
enhanced["progress_callback"] = instance_options.progress_callback
|
|
759
|
-
if batch and concurrency != 8:
|
|
760
|
-
enhanced["concurrency"] = concurrency
|
|
761
|
-
owner, method = (
|
|
762
|
-
(repo, "deliver") if prune else (repo.instances, "push_many" if batch else "push")
|
|
763
|
-
)
|
|
764
|
-
if prune:
|
|
765
|
-
enhanced["prune"] = True
|
|
766
|
-
if enhanced:
|
|
767
|
-
_enhanced_options(owner, method, enhanced)
|
|
768
607
|
auto_create = _check_version_autocreate(repo, dataset, resolved, environment=env)
|
|
769
|
-
_confirm_upload(
|
|
770
|
-
dataset, resolved, split, len(targets), yes=yes, auto_create=auto_create, prune=prune
|
|
771
|
-
)
|
|
608
|
+
_confirm_upload(dataset, resolved, split, len(targets), yes=yes, auto_create=auto_create)
|
|
772
609
|
pushed: list[dict] = []
|
|
773
|
-
skipped: list[dict] = []
|
|
774
610
|
failed: list[dict] = []
|
|
775
611
|
unconfirmed: list[str] = []
|
|
776
612
|
result_count_mismatch = False
|
|
777
|
-
if
|
|
613
|
+
if len(targets) == 1:
|
|
778
614
|
try:
|
|
779
615
|
result = repo.instances.push(
|
|
780
616
|
targets[0],
|
|
@@ -784,7 +620,6 @@ def instance_push(
|
|
|
784
620
|
push_image=False,
|
|
785
621
|
overwrite=overwrite,
|
|
786
622
|
register=True,
|
|
787
|
-
**enhanced,
|
|
788
623
|
)
|
|
789
624
|
except Exception as exc: # noqa: BLE001 - 统一走设计 §5.1 的映射
|
|
790
625
|
failed.append({"path": targets[0], "error": _describe(exc)})
|
|
@@ -793,41 +628,20 @@ def instance_push(
|
|
|
793
628
|
)
|
|
794
629
|
sdk_error_exit(exc, what=f"instance {targets[0]}", stage="upload")
|
|
795
630
|
else:
|
|
796
|
-
|
|
797
|
-
(skipped if row["skipped"] else pushed).append(row)
|
|
798
|
-
if progress:
|
|
799
|
-
instance_options.progress_callback(1, 1, result)
|
|
631
|
+
pushed.append(_push_row(targets[0], result))
|
|
800
632
|
else:
|
|
801
633
|
# 多路径:``continue_on_error=False`` 是"全或无",首个失败即抛 ItemError。
|
|
802
634
|
try:
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
)
|
|
814
|
-
results_many = [
|
|
815
|
-
*delivered.get("pushed", []),
|
|
816
|
-
*delivered.get("skipped", []),
|
|
817
|
-
*delivered.get("failed", []),
|
|
818
|
-
]
|
|
819
|
-
else:
|
|
820
|
-
results_many = repo.instances.push_many(
|
|
821
|
-
targets,
|
|
822
|
-
dataset,
|
|
823
|
-
resolved,
|
|
824
|
-
split=split,
|
|
825
|
-
push_image=False,
|
|
826
|
-
overwrite=overwrite,
|
|
827
|
-
register=True,
|
|
828
|
-
continue_on_error=continue_on_error,
|
|
829
|
-
**enhanced,
|
|
830
|
-
)
|
|
635
|
+
results_many = repo.instances.push_many(
|
|
636
|
+
targets,
|
|
637
|
+
dataset,
|
|
638
|
+
resolved,
|
|
639
|
+
split=split,
|
|
640
|
+
push_image=False,
|
|
641
|
+
overwrite=overwrite,
|
|
642
|
+
register=True,
|
|
643
|
+
continue_on_error=False,
|
|
644
|
+
)
|
|
831
645
|
except Exception as exc: # noqa: BLE001
|
|
832
646
|
item = getattr(exc, "item", None)
|
|
833
647
|
cause = getattr(exc, "error", None) or exc
|
|
@@ -847,61 +661,17 @@ def instance_push(
|
|
|
847
661
|
else:
|
|
848
662
|
results_list = list(results_many or [])
|
|
849
663
|
result_count_mismatch = len(results_list) != len(targets)
|
|
850
|
-
|
|
851
|
-
targets
|
|
852
|
-
)
|
|
664
|
+
if len(results_list) < len(targets):
|
|
665
|
+
unconfirmed.extend(targets[len(results_list) :])
|
|
666
|
+
for path, result in zip(targets, results_list):
|
|
667
|
+
pushed.append(_push_row(path, result))
|
|
853
668
|
|
|
854
|
-
_print_push_summary(
|
|
855
|
-
|
|
856
|
-
resolved,
|
|
857
|
-
split,
|
|
858
|
-
pushed,
|
|
859
|
-
failed,
|
|
860
|
-
unconfirmed,
|
|
861
|
-
output_format,
|
|
862
|
-
skipped=skipped if skip_unchanged or skipped else None,
|
|
863
|
-
)
|
|
864
|
-
if result_count_mismatch or unconfirmed:
|
|
669
|
+
_print_push_summary(dataset, resolved, split, pushed, failed, unconfirmed, output_format)
|
|
670
|
+
if result_count_mismatch:
|
|
865
671
|
raise _fail(
|
|
866
672
|
f"SDK returned {len(results_list)}/{len(targets)} upload results; cannot confirm the full write state",
|
|
867
673
|
f"Verify the actual content with ap instance list --dataset {dataset} --split {split}",
|
|
868
674
|
)
|
|
869
|
-
if failed:
|
|
870
|
-
raise _fail(
|
|
871
|
-
f"{len(failed)} instance upload(s) failed",
|
|
872
|
-
"Check the failure summary and retry the failed paths",
|
|
873
|
-
)
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
def _collect_upload_results(
|
|
877
|
-
targets: list[str],
|
|
878
|
-
records: list[dict],
|
|
879
|
-
values: list[Any],
|
|
880
|
-
*,
|
|
881
|
-
by_id: bool,
|
|
882
|
-
) -> tuple[list[dict], list[dict], list[dict], list[str]]:
|
|
883
|
-
remaining = list(targets)
|
|
884
|
-
paths_by_id = {record["instance"]: record["path"] for record in records}
|
|
885
|
-
pushed, skipped, failed = [], [], []
|
|
886
|
-
for value in values:
|
|
887
|
-
if isinstance(value, BaseException):
|
|
888
|
-
item = str(getattr(value, "item", ""))
|
|
889
|
-
path = next((raw for raw in remaining if Path(raw) == Path(item)), "") if item else ""
|
|
890
|
-
if path:
|
|
891
|
-
remaining.remove(path)
|
|
892
|
-
failed.append({"path": path, "error": _describe(getattr(value, "error", value))})
|
|
893
|
-
continue
|
|
894
|
-
path = (
|
|
895
|
-
paths_by_id.get(getattr(value, "instance_id", ""), "")
|
|
896
|
-
if by_id
|
|
897
|
-
else (remaining[0] if remaining else "")
|
|
898
|
-
)
|
|
899
|
-
if path not in remaining:
|
|
900
|
-
continue
|
|
901
|
-
remaining.remove(path)
|
|
902
|
-
row = _push_row(path, value)
|
|
903
|
-
(skipped if row["skipped"] else pushed).append(row)
|
|
904
|
-
return pushed, skipped, failed, remaining
|
|
905
675
|
|
|
906
676
|
|
|
907
677
|
def _push_row(path: str, result: Any) -> dict:
|
|
@@ -921,8 +691,6 @@ def _print_push_summary(
|
|
|
921
691
|
failed: list[dict],
|
|
922
692
|
unconfirmed: list[str],
|
|
923
693
|
output_format: str,
|
|
924
|
-
*,
|
|
925
|
-
skipped: Optional[list[dict]] = None,
|
|
926
694
|
) -> None:
|
|
927
695
|
payload = {
|
|
928
696
|
"dataset": dataset,
|
|
@@ -933,9 +701,6 @@ def _print_push_summary(
|
|
|
933
701
|
"unconfirmed": unconfirmed,
|
|
934
702
|
}
|
|
935
703
|
rows = [dict(row, status="pushed", error="") for row in pushed]
|
|
936
|
-
if skipped is not None:
|
|
937
|
-
payload["skipped"] = skipped
|
|
938
|
-
rows += [dict(row, status="skipped", error="") for row in skipped]
|
|
939
704
|
rows += [
|
|
940
705
|
{"path": row["path"], "instance": "", "status": "failed", "error": row["error"]}
|
|
941
706
|
for row in failed
|
|
@@ -963,18 +728,12 @@ def instance_pull(
|
|
|
963
728
|
None, "--output", "-o", help="Output directory (default: current directory)"
|
|
964
729
|
),
|
|
965
730
|
overwrite: bool = typer.Option(False, "--overwrite", help="Allow overwriting the target"),
|
|
966
|
-
path: Optional[str] = typer.Option(
|
|
967
|
-
None,
|
|
968
|
-
"--path",
|
|
969
|
-
help="Download a relative subdirectory (Custom/OSS assets); use '.' for all assets",
|
|
970
|
-
),
|
|
971
731
|
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
972
732
|
):
|
|
973
733
|
"""Download instance content and metadata into <output>/<instance>/."""
|
|
974
734
|
from .irepo_sdk import sdk_error_exit
|
|
975
735
|
|
|
976
736
|
output_format = _fmt(output_format)
|
|
977
|
-
resolved_path = _path_value(path)
|
|
978
737
|
resolved = _version_value(version)
|
|
979
738
|
root = Path(output or ".")
|
|
980
739
|
target = root / instance
|
|
@@ -985,9 +744,6 @@ def instance_pull(
|
|
|
985
744
|
)
|
|
986
745
|
|
|
987
746
|
repo = _load_repo_or_exit()
|
|
988
|
-
pull_options = {}
|
|
989
|
-
if resolved_path is not None:
|
|
990
|
-
pull_options = _enhanced_options(repo.instances, "pull", {"path": resolved_path})
|
|
991
747
|
try:
|
|
992
748
|
private_endpoint_reachable = _probe_pull_private_endpoint(repo, dataset)
|
|
993
749
|
except Exception as exc: # noqa: BLE001
|
|
@@ -1004,17 +760,10 @@ def instance_pull(
|
|
|
1004
760
|
staging = Path(tempfile.mkdtemp(prefix=f"ap-pull-{instance}-"))
|
|
1005
761
|
try:
|
|
1006
762
|
try:
|
|
1007
|
-
written = repo.instances.pull(
|
|
1008
|
-
dataset, resolved, instance, staging, split, **pull_options
|
|
1009
|
-
)
|
|
763
|
+
written = repo.instances.pull(dataset, resolved, instance, staging, split)
|
|
1010
764
|
except Exception as exc: # noqa: BLE001
|
|
1011
765
|
sdk_error_exit(exc, what=f"instance {instance}", stage="pull")
|
|
1012
766
|
staged = _place_under_target(Path(str(written)), staging / instance)
|
|
1013
|
-
if resolved_path and not (staging / instance / resolved_path).is_dir():
|
|
1014
|
-
raise _fail(
|
|
1015
|
-
f"No files were downloaded for instance subdirectory {resolved_path}",
|
|
1016
|
-
"Check the directory with ap instance get --path; the SDK must support this instance format",
|
|
1017
|
-
)
|
|
1018
767
|
root.mkdir(parents=True, exist_ok=True)
|
|
1019
768
|
if target.exists():
|
|
1020
769
|
if not overwrite:
|
|
@@ -1037,8 +786,6 @@ def instance_pull(
|
|
|
1037
786
|
"output": str(target),
|
|
1038
787
|
"content": str(target / staged.name),
|
|
1039
788
|
}
|
|
1040
|
-
if resolved_path is not None:
|
|
1041
|
-
payload["path"] = resolved_path or "."
|
|
1042
789
|
_emit_detail(payload, output_format)
|
|
1043
790
|
|
|
1044
791
|
|
|
@@ -1061,158 +808,3 @@ def _place_under_target(written: Path, target: Path) -> Path:
|
|
|
1061
808
|
final.unlink()
|
|
1062
809
|
shutil.move(str(written), str(final))
|
|
1063
810
|
return final
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
@instance_app.command("hash")
|
|
1067
|
-
def instance_hash(
|
|
1068
|
-
paths: List[str] = typer.Argument(..., help="Local instance directories"),
|
|
1069
|
-
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
1070
|
-
):
|
|
1071
|
-
"""Compute source hashes locally without contacting AP."""
|
|
1072
|
-
output_format = _fmt(output_format)
|
|
1073
|
-
try:
|
|
1074
|
-
from instance_repo import compute_source_hash
|
|
1075
|
-
except ImportError as exc:
|
|
1076
|
-
raise _fail(
|
|
1077
|
-
"Source hashing requires instance-repo 1.3.5.dev1 or newer",
|
|
1078
|
-
"Install Dataset support: pip install --upgrade 'ap-client[dataset]'",
|
|
1079
|
-
2,
|
|
1080
|
-
) from exc
|
|
1081
|
-
hashes, failed = [], []
|
|
1082
|
-
for raw in paths:
|
|
1083
|
-
path = Path(raw)
|
|
1084
|
-
try:
|
|
1085
|
-
if not path.is_dir():
|
|
1086
|
-
raise ValueError("source hashing requires an existing instance directory")
|
|
1087
|
-
hashes.append(
|
|
1088
|
-
{"path": raw, "instance": path.name, "source_hash": compute_source_hash(path)}
|
|
1089
|
-
)
|
|
1090
|
-
except (OSError, ValueError) as exc:
|
|
1091
|
-
failed.append({"path": raw, "error": _describe(exc)})
|
|
1092
|
-
_emit(
|
|
1093
|
-
{"hashes": hashes, "failed": failed},
|
|
1094
|
-
hashes,
|
|
1095
|
-
(("path", "PATH"), ("instance", "INSTANCE"), ("source_hash", "SOURCE_HASH")),
|
|
1096
|
-
output_format,
|
|
1097
|
-
)
|
|
1098
|
-
if failed:
|
|
1099
|
-
raise _fail(f"{len(failed)} instance directory hash(es) failed", "Check the local paths")
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
@instance_app.command("hashes")
|
|
1103
|
-
def instance_hashes(
|
|
1104
|
-
dataset: str = typer.Option(..., "--dataset", help="Dataset name"),
|
|
1105
|
-
split: str = typer.Option(..., "--split", help="Split name"),
|
|
1106
|
-
version: Optional[str] = typer.Option(None, "--version", help=_VERSION_HELP),
|
|
1107
|
-
environment: Optional[str] = typer.Option(None, "--environment", help=_ENVIRONMENT_HELP),
|
|
1108
|
-
metadata_model: Optional[str] = typer.Option(
|
|
1109
|
-
None, "--metadata-model", help=_METADATA_MODEL_HELP
|
|
1110
|
-
),
|
|
1111
|
-
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
1112
|
-
):
|
|
1113
|
-
"""List source hashes stored on remote instances."""
|
|
1114
|
-
output_format = _fmt(output_format)
|
|
1115
|
-
resolved = _version_value(version)
|
|
1116
|
-
env = _environment(environment)
|
|
1117
|
-
repo = _load_repo_or_exit()
|
|
1118
|
-
_enhanced_options(repo.instances, "list_content_hashes", {})
|
|
1119
|
-
kwargs = _sdk_read_kwargs(
|
|
1120
|
-
repo.instances,
|
|
1121
|
-
"list_content_hashes",
|
|
1122
|
-
environment=env,
|
|
1123
|
-
metadata_model=_metadata_model_for_split(metadata_model, split=split),
|
|
1124
|
-
)
|
|
1125
|
-
try:
|
|
1126
|
-
try:
|
|
1127
|
-
fallback = oss_fallback_kwargs(repo, repo.instances, "list_content_hashes", env)
|
|
1128
|
-
except RuntimeError:
|
|
1129
|
-
# This SDK helper cannot disable its inner list() discovery yet.
|
|
1130
|
-
list_kwargs = _sdk_read_kwargs(
|
|
1131
|
-
repo.instances,
|
|
1132
|
-
"list",
|
|
1133
|
-
environment=env,
|
|
1134
|
-
metadata_model=_metadata_model_for_split(metadata_model, split=split),
|
|
1135
|
-
)
|
|
1136
|
-
list_kwargs.update(oss_fallback_kwargs(repo, repo.instances, "list", env))
|
|
1137
|
-
records = repo.instances.list(dataset, resolved, split=split, **list_kwargs)
|
|
1138
|
-
hashes = {}
|
|
1139
|
-
for record in records:
|
|
1140
|
-
served = _model_dict(record)
|
|
1141
|
-
source_hash = (served.get("extend_meta") or {}).get("source_hash")
|
|
1142
|
-
if source_hash:
|
|
1143
|
-
hashes[served["instance_id"]] = source_hash
|
|
1144
|
-
else:
|
|
1145
|
-
hashes = repo.instances.list_content_hashes(
|
|
1146
|
-
dataset, resolved, split=split, **kwargs, **fallback
|
|
1147
|
-
)
|
|
1148
|
-
except Exception as exc: # noqa: BLE001
|
|
1149
|
-
raise _fail(
|
|
1150
|
-
f"Failed to list source hashes of dataset {dataset}: {_describe(exc)}",
|
|
1151
|
-
"Check the environment, bound storage and read permission",
|
|
1152
|
-
) from exc
|
|
1153
|
-
rows = [
|
|
1154
|
-
{"instance": instance, "source_hash": value} for instance, value in sorted(hashes.items())
|
|
1155
|
-
]
|
|
1156
|
-
_emit(
|
|
1157
|
-
{"dataset": dataset, "version": resolved, "split": split, "hashes": hashes},
|
|
1158
|
-
rows,
|
|
1159
|
-
(("instance", "INSTANCE"), ("source_hash", "SOURCE_HASH")),
|
|
1160
|
-
output_format,
|
|
1161
|
-
)
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
@instance_app.command("delete")
|
|
1165
|
-
def instance_delete(
|
|
1166
|
-
instance: str = typer.Argument(..., help="Instance id"),
|
|
1167
|
-
dataset: str = typer.Option(..., "--dataset", help="Dataset name"),
|
|
1168
|
-
version: str = typer.Option(..., "--version", help="Dataset version containing the instance"),
|
|
1169
|
-
split: str = typer.Option(..., "--split", help="Split name"),
|
|
1170
|
-
environment: Optional[str] = typer.Option(None, "--environment", help=_ENVIRONMENT_HELP),
|
|
1171
|
-
metadata_model: Optional[str] = typer.Option(
|
|
1172
|
-
None, "--metadata-model", help=_METADATA_MODEL_HELP
|
|
1173
|
-
),
|
|
1174
|
-
yes: bool = typer.Option(False, "--yes", "-y", help="Confirm deleting instance metadata"),
|
|
1175
|
-
output_format: str = typer.Option(None, "--format", help=_FORMAT_HELP),
|
|
1176
|
-
):
|
|
1177
|
-
"""Delete instance metadata. OSS content remains available to later ingest."""
|
|
1178
|
-
from .irepo_sdk import sdk_error_exit
|
|
1179
|
-
|
|
1180
|
-
output_format = _fmt(output_format)
|
|
1181
|
-
env = _environment(environment)
|
|
1182
|
-
resolved = _version_value(version)
|
|
1183
|
-
if not resolved:
|
|
1184
|
-
raise _fail(
|
|
1185
|
-
"--version must be nonempty for metadata deletion", "Specify the dataset version", 2
|
|
1186
|
-
)
|
|
1187
|
-
repo = _load_repo_or_exit()
|
|
1188
|
-
_enhanced_options(repo.instances, "delete", {})
|
|
1189
|
-
kwargs = _sdk_read_kwargs(
|
|
1190
|
-
repo.instances,
|
|
1191
|
-
"delete",
|
|
1192
|
-
environment=env,
|
|
1193
|
-
metadata_model=_metadata_model_for_split(metadata_model, split=split),
|
|
1194
|
-
)
|
|
1195
|
-
if not yes:
|
|
1196
|
-
if not sys.stdin.isatty():
|
|
1197
|
-
raise _fail("Deleting instance metadata requires confirmation", "Add --yes (-y)", 2)
|
|
1198
|
-
if not typer.confirm(
|
|
1199
|
-
f"Delete metadata for {dataset} version {resolved} split {split} instance {instance}? OSS files remain."
|
|
1200
|
-
):
|
|
1201
|
-
raise _fail(
|
|
1202
|
-
"Metadata deletion canceled", "Rerun when you want to delete the instance record"
|
|
1203
|
-
)
|
|
1204
|
-
try:
|
|
1205
|
-
repo.instances.delete(dataset, resolved, instance, split=split, **kwargs)
|
|
1206
|
-
except Exception as exc: # noqa: BLE001
|
|
1207
|
-
sdk_error_exit(exc, what=f"instance {instance}", stage="metadata deletion")
|
|
1208
|
-
_emit_detail(
|
|
1209
|
-
{
|
|
1210
|
-
"dataset": dataset,
|
|
1211
|
-
"version": resolved,
|
|
1212
|
-
"split": split,
|
|
1213
|
-
"instance": instance,
|
|
1214
|
-
"deleted": True,
|
|
1215
|
-
"content_deleted": False,
|
|
1216
|
-
},
|
|
1217
|
-
output_format,
|
|
1218
|
-
)
|