dexverse-cli 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dexverse/__init__.py +54 -0
- dexverse/cli.py +422 -0
- dexverse/client.py +128 -0
- dexverse/config.py +85 -0
- dexverse/convert/__init__.py +38 -0
- dexverse/convert/hdf5_to_raw.py +498 -0
- dexverse/convert/image_video_encode.py +449 -0
- dexverse/downloader/__init__.py +26 -0
- dexverse/downloader/base.py +101 -0
- dexverse/downloader/http_backend.py +508 -0
- dexverse/downloader/manager.py +365 -0
- dexverse/downloader/python_s3_backend.py +137 -0
- dexverse/downloader/s5cmd_backend.py +170 -0
- dexverse/downloader/state.py +263 -0
- dexverse/downloader/verify.py +114 -0
- dexverse/manifest.py +284 -0
- dexverse/mock/__init__.py +2 -0
- dexverse/mock/mock_tasks.json +22 -0
- dexverse/mock/server.py +211 -0
- dexverse_cli-0.0.2.dist-info/METADATA +198 -0
- dexverse_cli-0.0.2.dist-info/RECORD +23 -0
- dexverse_cli-0.0.2.dist-info/WHEEL +4 -0
- dexverse_cli-0.0.2.dist-info/entry_points.txt +3 -0
dexverse/__init__.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""DexVerse Downloader — S3-compatible dataset downloader.
|
|
2
|
+
|
|
3
|
+
Public API:
|
|
4
|
+
|
|
5
|
+
from dexverse import DexVerseClient, download_manifest
|
|
6
|
+
from dexverse import Manifest, FileSpec
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import tomlkit
|
|
14
|
+
|
|
15
|
+
from .client import DexVerseClient, DexVerseError, download_manifest
|
|
16
|
+
from .manifest import (
|
|
17
|
+
FileSpec,
|
|
18
|
+
Manifest,
|
|
19
|
+
Storage,
|
|
20
|
+
StorageCredentials,
|
|
21
|
+
DownloadOptions,
|
|
22
|
+
resolve_dest,
|
|
23
|
+
resolve_relpath,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
def _resolve_version() -> str:
|
|
27
|
+
"""Read the single source version in development, package metadata otherwise."""
|
|
28
|
+
pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml"
|
|
29
|
+
if pyproject.is_file():
|
|
30
|
+
document = tomlkit.parse(pyproject.read_text(encoding="utf-8"))
|
|
31
|
+
project_version = document.get("project", {}).get("version")
|
|
32
|
+
if isinstance(project_version, str):
|
|
33
|
+
return project_version
|
|
34
|
+
try:
|
|
35
|
+
return version("dexverse")
|
|
36
|
+
except PackageNotFoundError:
|
|
37
|
+
return "0.0.0+unknown"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
__version__ = _resolve_version()
|
|
41
|
+
|
|
42
|
+
__all__ = [
|
|
43
|
+
"DexVerseClient",
|
|
44
|
+
"DexVerseError",
|
|
45
|
+
"download_manifest",
|
|
46
|
+
"Manifest",
|
|
47
|
+
"FileSpec",
|
|
48
|
+
"Storage",
|
|
49
|
+
"StorageCredentials",
|
|
50
|
+
"DownloadOptions",
|
|
51
|
+
"resolve_dest",
|
|
52
|
+
"resolve_relpath",
|
|
53
|
+
"__version__",
|
|
54
|
+
]
|
dexverse/cli.py
ADDED
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""Command-line interface (doc §10).
|
|
2
|
+
|
|
3
|
+
dexverse-dl login --server URL [--token TOKEN]
|
|
4
|
+
dexverse-dl download --manifest PATH | --task ID --out DIR
|
|
5
|
+
[--parallel N] [--backend http|python|s5cmd]
|
|
6
|
+
[--server URL] [--no-verify] [--deep]
|
|
7
|
+
dexverse-dl verify --manifest PATH --out DIR [--deep]
|
|
8
|
+
dexverse-dl repair --manifest PATH --out DIR [--backend http|python|s5cmd] [--deep]
|
|
9
|
+
dexverse-dl convert -i PATH | --input-dir DIR -o DIR [--strategy default]
|
|
10
|
+
# 还原下载的 HDF5 为可训练原始数据(图片 + pose_record JSON)
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import threading
|
|
15
|
+
import time
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Optional
|
|
18
|
+
|
|
19
|
+
import typer
|
|
20
|
+
from rich.console import Console
|
|
21
|
+
from rich.filesize import decimal
|
|
22
|
+
from rich.progress import (
|
|
23
|
+
BarColumn,
|
|
24
|
+
Progress,
|
|
25
|
+
SpinnerColumn,
|
|
26
|
+
TextColumn,
|
|
27
|
+
)
|
|
28
|
+
from rich.table import Table
|
|
29
|
+
|
|
30
|
+
from .client import DexVerseClient, DexVerseError, download_manifest
|
|
31
|
+
from .config import config_path, save_profile
|
|
32
|
+
from .downloader.manager import DownloadManager, repair as repair_fn
|
|
33
|
+
from .downloader.s5cmd_backend import S5CmdNotAvailable
|
|
34
|
+
from .downloader.verify import verify_files, summarize
|
|
35
|
+
from .manifest import Manifest
|
|
36
|
+
from .convert import convert_batch as convert_hdf5_batch
|
|
37
|
+
from .convert.hdf5_to_raw import DefaultAlignmentStrategy
|
|
38
|
+
|
|
39
|
+
app = typer.Typer(
|
|
40
|
+
name="dexverse-dl",
|
|
41
|
+
help="DexVerse dataset downloader (S3-compatible).",
|
|
42
|
+
no_args_is_help=True,
|
|
43
|
+
)
|
|
44
|
+
console = Console()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _RichDownloadProgress:
|
|
48
|
+
"""Render aggregate byte progress for a single manager run."""
|
|
49
|
+
|
|
50
|
+
def __init__(self) -> None:
|
|
51
|
+
self._progress = Progress(
|
|
52
|
+
SpinnerColumn(),
|
|
53
|
+
TextColumn("[progress.description]{task.description}"),
|
|
54
|
+
TextColumn("{task.completed:.0f}/{task.total:.0f} files"),
|
|
55
|
+
BarColumn(),
|
|
56
|
+
TextColumn("{task.fields[transfer]}"),
|
|
57
|
+
console=console,
|
|
58
|
+
)
|
|
59
|
+
self._task_id: Optional[int] = None
|
|
60
|
+
self._downloaded_bytes = 0
|
|
61
|
+
self._total_bytes: Optional[int] = None
|
|
62
|
+
self._started_at = 0.0
|
|
63
|
+
self._lock = threading.Lock()
|
|
64
|
+
|
|
65
|
+
def start(self, *, total_bytes: Optional[int], total_files: int) -> None:
|
|
66
|
+
self._total_bytes = total_bytes
|
|
67
|
+
self._started_at = time.monotonic()
|
|
68
|
+
self._progress.start()
|
|
69
|
+
self._task_id = self._progress.add_task(
|
|
70
|
+
"Downloading",
|
|
71
|
+
total=total_files,
|
|
72
|
+
transfer="0 bytes",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
def advance(self, bytes_count: int) -> None:
|
|
76
|
+
if self._task_id is not None and bytes_count > 0:
|
|
77
|
+
with self._lock:
|
|
78
|
+
self._downloaded_bytes += bytes_count
|
|
79
|
+
downloaded = self._downloaded_bytes
|
|
80
|
+
elapsed = time.monotonic() - self._started_at
|
|
81
|
+
amount = decimal(downloaded)
|
|
82
|
+
if self._total_bytes is not None:
|
|
83
|
+
amount = f"{amount}/{decimal(self._total_bytes)}"
|
|
84
|
+
speed = decimal(downloaded / elapsed) if elapsed > 0 else "0 bytes"
|
|
85
|
+
self._progress.update(
|
|
86
|
+
self._task_id,
|
|
87
|
+
transfer=f"{amount} • {speed}/s",
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
def complete_file(self) -> None:
|
|
91
|
+
if self._task_id is not None:
|
|
92
|
+
self._progress.advance(self._task_id, 1)
|
|
93
|
+
|
|
94
|
+
def finish(self) -> None:
|
|
95
|
+
self._progress.stop()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# --------------------------------------------------------------------------- #
|
|
99
|
+
# login
|
|
100
|
+
# --------------------------------------------------------------------------- #
|
|
101
|
+
@app.command(help="Save a server + token to the active profile.")
|
|
102
|
+
def login(
|
|
103
|
+
server: str = typer.Option(..., "--server", "-s", help="Platform base URL"),
|
|
104
|
+
token: Optional[str] = typer.Option(
|
|
105
|
+
None, "--token", "-t", help="Bearer token (dev-token in mock mode)"
|
|
106
|
+
),
|
|
107
|
+
profile: str = typer.Option("default", "--profile", help="Profile name"),
|
|
108
|
+
):
|
|
109
|
+
save_profile(profile, server, token)
|
|
110
|
+
console.print(f"[green]✓[/green] Saved profile [bold]{profile}[/bold] → {server}")
|
|
111
|
+
console.print(f" config: {config_path()}")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# --------------------------------------------------------------------------- #
|
|
115
|
+
# download
|
|
116
|
+
# --------------------------------------------------------------------------- #
|
|
117
|
+
@app.command(help="Download by --manifest (Phase 0) or --task (Phase 1+).")
|
|
118
|
+
def download(
|
|
119
|
+
out: Path = typer.Option(..., "--out", "-o", help="Local download root"),
|
|
120
|
+
manifest: Optional[Path] = typer.Option(
|
|
121
|
+
None, "--manifest", "-m",
|
|
122
|
+
help="Local manifest.json path, or '-' to read JSON from stdin",
|
|
123
|
+
),
|
|
124
|
+
task: Optional[str] = typer.Option(None, "--task", help="Platform download taskId"),
|
|
125
|
+
parallel: Optional[int] = typer.Option(None, "--parallel", "-p"),
|
|
126
|
+
backend: str = typer.Option("http", "--backend", "-b",
|
|
127
|
+
help="http | python | s5cmd"),
|
|
128
|
+
server: Optional[str] = typer.Option(None, "--server", help="Override profile server"),
|
|
129
|
+
token: Optional[str] = typer.Option(None, "--token", help="Override profile token"),
|
|
130
|
+
no_verify: bool = typer.Option(False, "--no-verify", help="Skip post-download verify"),
|
|
131
|
+
deep: bool = typer.Option(False, "--deep", help="Also verify sha256 where recorded"),
|
|
132
|
+
http_timeout: float = typer.Option(
|
|
133
|
+
120.0,
|
|
134
|
+
"--http-read-timeout",
|
|
135
|
+
help="HTTP read-idle timeout in seconds",
|
|
136
|
+
),
|
|
137
|
+
http_retries: int = typer.Option(
|
|
138
|
+
5,
|
|
139
|
+
"--http-retries",
|
|
140
|
+
help="Retries per file after transient HTTP/network failures",
|
|
141
|
+
),
|
|
142
|
+
insecure: Optional[bool] = typer.Option(
|
|
143
|
+
None, "--insecure/--secure", "-k",
|
|
144
|
+
help=(
|
|
145
|
+
"TLS mode for http/presigned downloads. Default: --insecure for "
|
|
146
|
+
"http, secure for python/s5cmd."
|
|
147
|
+
),
|
|
148
|
+
),
|
|
149
|
+
):
|
|
150
|
+
if (manifest is None) == (task is None):
|
|
151
|
+
console.print("[red]error:[/red] pass exactly one of --manifest or --task")
|
|
152
|
+
raise typer.Exit(code=2)
|
|
153
|
+
|
|
154
|
+
effective_insecure = _effective_insecure(insecure, backend)
|
|
155
|
+
_warn_insecure(effective_insecure, backend, explicit=insecure is True)
|
|
156
|
+
progress = _RichDownloadProgress()
|
|
157
|
+
backend_kwargs = _http_backend_kwargs(
|
|
158
|
+
backend,
|
|
159
|
+
timeout=http_timeout,
|
|
160
|
+
max_retries=http_retries,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
try:
|
|
164
|
+
if manifest is not None:
|
|
165
|
+
report = download_manifest(
|
|
166
|
+
"-" if manifest == Path("-") else manifest, out,
|
|
167
|
+
parallel=parallel, backend=backend,
|
|
168
|
+
verify=not no_verify, deep=deep, insecure=effective_insecure,
|
|
169
|
+
progress=progress, backend_kwargs=backend_kwargs,
|
|
170
|
+
)
|
|
171
|
+
else:
|
|
172
|
+
# download_task returns the out-path per §11.1; run the manager here
|
|
173
|
+
# so the CLI can print a full report.
|
|
174
|
+
with DexVerseClient(server=server, token=token) as client:
|
|
175
|
+
man = client.get_manifest(task)
|
|
176
|
+
report = DownloadManager(
|
|
177
|
+
out, backend=backend, parallel=parallel,
|
|
178
|
+
verify=not no_verify, deep=deep, insecure=effective_insecure,
|
|
179
|
+
progress=progress, backend_kwargs=backend_kwargs,
|
|
180
|
+
).run(man)
|
|
181
|
+
except S5CmdNotAvailable as exc:
|
|
182
|
+
console.print(f"[red]s5cmd not available:[/red] {exc}")
|
|
183
|
+
console.print(" hint: omit [cyan]--backend s5cmd[/cyan], or use [cyan]--backend python[/cyan]")
|
|
184
|
+
raise typer.Exit(code=2)
|
|
185
|
+
except DexVerseError as exc:
|
|
186
|
+
console.print(f"[red]error:[/red] {exc}")
|
|
187
|
+
raise typer.Exit(code=1)
|
|
188
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
189
|
+
console.print(f"[red]error:[/red] {exc}")
|
|
190
|
+
raise typer.Exit(code=1)
|
|
191
|
+
|
|
192
|
+
_print_report(report)
|
|
193
|
+
if not report.ok:
|
|
194
|
+
raise typer.Exit(code=1)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# --------------------------------------------------------------------------- #
|
|
198
|
+
# convert
|
|
199
|
+
# --------------------------------------------------------------------------- #
|
|
200
|
+
@app.command(help="将下载的 HDF5 还原为可训练的原始数据结构(多路相机图片 + pose_record JSON)。")
|
|
201
|
+
def convert(
|
|
202
|
+
input: Optional[Path] = typer.Option(
|
|
203
|
+
None, "--input", "-i", help="单个 HDF5 文件路径",
|
|
204
|
+
),
|
|
205
|
+
input_dir: Optional[Path] = typer.Option(
|
|
206
|
+
None, "--input-dir", help="目录,批量递归处理其下所有 *.hdf5",
|
|
207
|
+
),
|
|
208
|
+
output: Path = typer.Option(..., "--output", "-o", help="输出根目录"),
|
|
209
|
+
strategy: str = typer.Option(
|
|
210
|
+
"default", "--strategy", help="对齐策略(当前仅 default,预留扩展)",
|
|
211
|
+
),
|
|
212
|
+
):
|
|
213
|
+
"""下载完的 HDF5 落在下载时的 --out 下,把它作为 -i / --input-dir 传入即可。"""
|
|
214
|
+
if strategy != "default":
|
|
215
|
+
console.print(f"[red]error:[/red] 未知策略 {strategy!r}(当前仅支持 default)")
|
|
216
|
+
raise typer.Exit(code=2)
|
|
217
|
+
|
|
218
|
+
if (input is None) == (input_dir is None):
|
|
219
|
+
console.print("[red]error:[/red] 必须指定 -i/--input 或 --input-dir 之一")
|
|
220
|
+
raise typer.Exit(code=2)
|
|
221
|
+
|
|
222
|
+
input_path = Path(input if input is not None else input_dir) # type: ignore[arg-type]
|
|
223
|
+
if not input_path.exists():
|
|
224
|
+
console.print(f"[red]error:[/red] 输入路径不存在: {input_path}")
|
|
225
|
+
raise typer.Exit(code=1)
|
|
226
|
+
|
|
227
|
+
try:
|
|
228
|
+
success, total_frames = convert_hdf5_batch(
|
|
229
|
+
input_path, Path(output), strategy=DefaultAlignmentStrategy(),
|
|
230
|
+
)
|
|
231
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
232
|
+
console.print(f"[red]error:[/red] {exc}")
|
|
233
|
+
raise typer.Exit(code=1)
|
|
234
|
+
|
|
235
|
+
if success == 0:
|
|
236
|
+
console.print("[yellow]未还原任何文件[/yellow]")
|
|
237
|
+
raise typer.Exit(code=1)
|
|
238
|
+
console.print(
|
|
239
|
+
f"[green]OK[/green] 还原 {success} 个文件, 共 {total_frames} 帧 -> {output}"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# --------------------------------------------------------------------------- #
|
|
244
|
+
# verify
|
|
245
|
+
# --------------------------------------------------------------------------- #
|
|
246
|
+
@app.command(help="Verify downloaded files against a manifest (size, or --deep sha256).")
|
|
247
|
+
def verify(
|
|
248
|
+
manifest: Path = typer.Option(
|
|
249
|
+
..., "--manifest", "-m",
|
|
250
|
+
help="Local manifest.json path, or '-' to read JSON from stdin",
|
|
251
|
+
),
|
|
252
|
+
out: Path = typer.Option(..., "--out", "-o"),
|
|
253
|
+
deep: bool = typer.Option(False, "--deep", help="Also check sha256"),
|
|
254
|
+
):
|
|
255
|
+
try:
|
|
256
|
+
man = (Manifest.parse_stdin() if manifest == Path("-")
|
|
257
|
+
else Manifest.parse_file(manifest))
|
|
258
|
+
results = verify_files(man.files, out, deep=deep)
|
|
259
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
260
|
+
console.print(f"[red]error:[/red] {exc}")
|
|
261
|
+
raise typer.Exit(code=1)
|
|
262
|
+
summary = summarize(results)
|
|
263
|
+
|
|
264
|
+
table = Table(title=f"Verify — task {man.effective_task_id()}")
|
|
265
|
+
table.add_column("status")
|
|
266
|
+
table.add_column("count", justify="right")
|
|
267
|
+
table.add_row("[green]ok[/green]", str(summary.ok))
|
|
268
|
+
table.add_row("missing", str(summary.missing))
|
|
269
|
+
table.add_row("size mismatch", str(summary.size_mismatch))
|
|
270
|
+
table.add_row("sha256 mismatch", str(summary.sha256_mismatch))
|
|
271
|
+
console.print(table)
|
|
272
|
+
|
|
273
|
+
for r in summary.bad_results:
|
|
274
|
+
console.print(f" [red]✗[/red] {r.relpath} ({r.reason})")
|
|
275
|
+
|
|
276
|
+
console.print(
|
|
277
|
+
f"\n{summary.ok}/{summary.total} files verified"
|
|
278
|
+
f"{' [bold green]all good[/bold green]' if summary.all_ok else ''}"
|
|
279
|
+
)
|
|
280
|
+
if not summary.all_ok:
|
|
281
|
+
raise typer.Exit(code=1)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
# --------------------------------------------------------------------------- #
|
|
285
|
+
# repair
|
|
286
|
+
# --------------------------------------------------------------------------- #
|
|
287
|
+
@app.command(help="Re-download missing / size-mismatched / sha256-mismatched files. "
|
|
288
|
+
"Use --task to fetch a fresh manifest (refreshes expired presigned URLs).")
|
|
289
|
+
def repair(
|
|
290
|
+
out: Path = typer.Option(..., "--out", "-o"),
|
|
291
|
+
manifest: Optional[Path] = typer.Option(
|
|
292
|
+
None, "--manifest", "-m",
|
|
293
|
+
help="Local manifest.json path, or '-' to read JSON from stdin",
|
|
294
|
+
),
|
|
295
|
+
task: Optional[str] = typer.Option(None, "--task", help="Refresh manifest from backend"),
|
|
296
|
+
server: Optional[str] = typer.Option(None, "--server"),
|
|
297
|
+
token: Optional[str] = typer.Option(None, "--token"),
|
|
298
|
+
backend: str = typer.Option("http", "--backend", "-b",
|
|
299
|
+
help="http | python | s5cmd"),
|
|
300
|
+
parallel: Optional[int] = typer.Option(None, "--parallel", "-p"),
|
|
301
|
+
deep: bool = typer.Option(False, "--deep", help="Detect sha256 corruption too"),
|
|
302
|
+
http_timeout: float = typer.Option(
|
|
303
|
+
120.0,
|
|
304
|
+
"--http-read-timeout",
|
|
305
|
+
help="HTTP read-idle timeout in seconds",
|
|
306
|
+
),
|
|
307
|
+
http_retries: int = typer.Option(
|
|
308
|
+
5,
|
|
309
|
+
"--http-retries",
|
|
310
|
+
help="Retries per file after transient HTTP/network failures",
|
|
311
|
+
),
|
|
312
|
+
insecure: Optional[bool] = typer.Option(
|
|
313
|
+
None, "--insecure/--secure", "-k",
|
|
314
|
+
help=(
|
|
315
|
+
"TLS mode for http/presigned downloads. Default: --insecure for "
|
|
316
|
+
"http, secure for python/s5cmd."
|
|
317
|
+
),
|
|
318
|
+
),
|
|
319
|
+
):
|
|
320
|
+
if (manifest is None) == (task is None):
|
|
321
|
+
console.print("[red]error:[/red] pass exactly one of --manifest or --task")
|
|
322
|
+
raise typer.Exit(code=2)
|
|
323
|
+
|
|
324
|
+
effective_insecure = _effective_insecure(insecure, backend)
|
|
325
|
+
_warn_insecure(effective_insecure, backend, explicit=insecure is True)
|
|
326
|
+
progress = _RichDownloadProgress()
|
|
327
|
+
backend_kwargs = _http_backend_kwargs(
|
|
328
|
+
backend,
|
|
329
|
+
timeout=http_timeout,
|
|
330
|
+
max_retries=http_retries,
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
try:
|
|
334
|
+
if task is not None:
|
|
335
|
+
# fresh manifest = fresh presigned URLs (the persisted copy's may be expired)
|
|
336
|
+
with DexVerseClient(server=server, token=token) as client:
|
|
337
|
+
man = client.get_manifest(task)
|
|
338
|
+
elif manifest == Path("-"):
|
|
339
|
+
man = Manifest.parse_stdin()
|
|
340
|
+
else:
|
|
341
|
+
man = Manifest.parse_file(manifest)
|
|
342
|
+
report = repair_fn(man, out, backend=backend, parallel=parallel,
|
|
343
|
+
deep=deep, insecure=effective_insecure,
|
|
344
|
+
progress=progress, backend_kwargs=backend_kwargs)
|
|
345
|
+
except S5CmdNotAvailable as exc:
|
|
346
|
+
console.print(f"[red]s5cmd not available:[/red] {exc}")
|
|
347
|
+
console.print(" hint: omit [cyan]--backend s5cmd[/cyan], or use [cyan]--backend python[/cyan]")
|
|
348
|
+
raise typer.Exit(code=2)
|
|
349
|
+
except DexVerseError as exc:
|
|
350
|
+
console.print(f"[red]error:[/red] {exc}")
|
|
351
|
+
raise typer.Exit(code=1)
|
|
352
|
+
except (FileNotFoundError, ValueError) as exc:
|
|
353
|
+
console.print(f"[red]error:[/red] {exc}")
|
|
354
|
+
raise typer.Exit(code=1)
|
|
355
|
+
_print_report(report)
|
|
356
|
+
if not report.ok:
|
|
357
|
+
raise typer.Exit(code=1)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# --------------------------------------------------------------------------- #
|
|
361
|
+
# helpers
|
|
362
|
+
# --------------------------------------------------------------------------- #
|
|
363
|
+
def _is_http_backend(backend: str) -> bool:
|
|
364
|
+
return (backend or "http").lower() in ("http", "https", "presigned")
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _http_backend_kwargs(
|
|
368
|
+
backend: str,
|
|
369
|
+
*,
|
|
370
|
+
timeout: float,
|
|
371
|
+
max_retries: int,
|
|
372
|
+
) -> dict:
|
|
373
|
+
if not _is_http_backend(backend):
|
|
374
|
+
return {}
|
|
375
|
+
return {"timeout": timeout, "max_retries": max_retries}
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _effective_insecure(insecure: Optional[bool], backend: str) -> bool:
|
|
379
|
+
if insecure is not None:
|
|
380
|
+
return insecure
|
|
381
|
+
return _is_http_backend(backend)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def _warn_insecure(insecure: bool, backend: str, *, explicit: bool = False) -> None:
|
|
385
|
+
if not insecure:
|
|
386
|
+
return
|
|
387
|
+
backend = (backend or "").lower()
|
|
388
|
+
if backend in ("http", "https", "presigned"):
|
|
389
|
+
if not explicit:
|
|
390
|
+
console.print(
|
|
391
|
+
"[yellow]WARNING:[/yellow] TLS verification is disabled by default for "
|
|
392
|
+
"http/presigned downloads. Use [cyan]--secure[/cyan] to require a valid "
|
|
393
|
+
"certificate."
|
|
394
|
+
)
|
|
395
|
+
return
|
|
396
|
+
console.print(
|
|
397
|
+
"[yellow]WARNING:[/yellow] --insecure disables TLS verification. For "
|
|
398
|
+
"self-authenticating presigned URLs, TLS is the only transport-integrity "
|
|
399
|
+
"check — any certificate will be accepted and redirects followed."
|
|
400
|
+
)
|
|
401
|
+
else:
|
|
402
|
+
console.print(
|
|
403
|
+
"[yellow]WARNING:[/yellow] --insecure disables TLS verification "
|
|
404
|
+
"(self-signed/internal endpoint)."
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _print_report(report) -> None:
|
|
409
|
+
console.print(f"\n[bold]Download report[/bold] — {report.summary_line()}")
|
|
410
|
+
console.print(f" out: {report.out}")
|
|
411
|
+
console.print(f" task dir: {report.task_dir}")
|
|
412
|
+
console.print(f" state: {report.state_path}")
|
|
413
|
+
if report.failures:
|
|
414
|
+
console.print(f" [red]failures:[/red]")
|
|
415
|
+
for fl in report.failures[:20]:
|
|
416
|
+
console.print(f" ✗ {fl['relpath']} — {fl.get('error')}")
|
|
417
|
+
if len(report.failures) > 20:
|
|
418
|
+
console.print(f" ... and {len(report.failures) - 20} more")
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
if __name__ == "__main__":
|
|
422
|
+
app()
|
dexverse/client.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""Python SDK client (doc §11).
|
|
2
|
+
|
|
3
|
+
``DexVerseClient`` talks to the platform backend to fetch a manifest for a
|
|
4
|
+
taskId, then hands it to the download manager. ``download_manifest`` runs the
|
|
5
|
+
manager directly off a local manifest file (Phase 0, no server).
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Dict, Optional, Union
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
from .config import active_profile
|
|
15
|
+
from .downloader.base import ProgressReporter
|
|
16
|
+
from .downloader.manager import DownloadManager, DownloadReport, get_backend # noqa: F401
|
|
17
|
+
from .manifest import Manifest
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DexVerseError(RuntimeError):
|
|
21
|
+
pass
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _download_error(report: DownloadReport) -> DexVerseError:
|
|
25
|
+
message = (
|
|
26
|
+
f"download completed with {report.failed + report.corrupt} problem(s): "
|
|
27
|
+
f"{report.summary_line()}"
|
|
28
|
+
)
|
|
29
|
+
if report.failures:
|
|
30
|
+
first = report.failures[0]
|
|
31
|
+
message += (
|
|
32
|
+
f"; first failure: {first.get('relpath')}: "
|
|
33
|
+
f"{first.get('error') or 'verification failed'}"
|
|
34
|
+
)
|
|
35
|
+
return DexVerseError(message)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class DexVerseClient:
|
|
39
|
+
def __init__(self, server: Optional[str] = None, token: Optional[str] = None, *,
|
|
40
|
+
timeout: float = 30.0):
|
|
41
|
+
# Fall back to the active profile saved by `dexverse-dl login`.
|
|
42
|
+
if not server or not token:
|
|
43
|
+
prof = active_profile()
|
|
44
|
+
if prof:
|
|
45
|
+
server = server or prof.server
|
|
46
|
+
token = token if token is not None else prof.token
|
|
47
|
+
if not server:
|
|
48
|
+
raise DexVerseError("no server configured (run `dexverse-dl login` first)")
|
|
49
|
+
self.server = server.rstrip("/")
|
|
50
|
+
self.token = token
|
|
51
|
+
self._client = httpx.Client(timeout=timeout)
|
|
52
|
+
|
|
53
|
+
def _headers(self) -> dict:
|
|
54
|
+
h = {"Accept": "application/json"}
|
|
55
|
+
if self.token:
|
|
56
|
+
h["Authorization"] = f"Bearer {self.token}"
|
|
57
|
+
return h
|
|
58
|
+
|
|
59
|
+
def get_manifest(self, task_id: str) -> Manifest:
|
|
60
|
+
url = f"{self.server}/api/download-tasks/{task_id}/manifest"
|
|
61
|
+
resp = self._client.get(url, headers=self._headers())
|
|
62
|
+
if resp.status_code == 404:
|
|
63
|
+
raise DexVerseError(f"task not found: {task_id}")
|
|
64
|
+
if resp.status_code in (401, 403):
|
|
65
|
+
raise DexVerseError(f"auth failed ({resp.status_code}) for task {task_id}")
|
|
66
|
+
resp.raise_for_status()
|
|
67
|
+
return Manifest.from_payload(resp.json())
|
|
68
|
+
|
|
69
|
+
def download_task(self, task_id: str, out: Union[str, Path], *,
|
|
70
|
+
parallel: Optional[int] = None,
|
|
71
|
+
backend: str = "http",
|
|
72
|
+
verify: bool = True,
|
|
73
|
+
deep: bool = False,
|
|
74
|
+
insecure: bool = True,
|
|
75
|
+
backend_kwargs: Optional[Dict[str, object]] = None) -> Path:
|
|
76
|
+
"""Download a task by id; returns the local out directory (doc §11.1).
|
|
77
|
+
|
|
78
|
+
``local_root = client.download_task(...)`` is the user-supplied ``out``,
|
|
79
|
+
with files landed at their original relative paths. Raises DexVerseError
|
|
80
|
+
if the run finishes with failures.
|
|
81
|
+
"""
|
|
82
|
+
manifest = self.get_manifest(task_id)
|
|
83
|
+
mgr = DownloadManager(out, backend=backend, parallel=parallel,
|
|
84
|
+
verify=verify, deep=deep, insecure=insecure,
|
|
85
|
+
backend_kwargs=backend_kwargs or {})
|
|
86
|
+
report = mgr.run(manifest)
|
|
87
|
+
if not report.ok:
|
|
88
|
+
raise _download_error(report)
|
|
89
|
+
return report.out
|
|
90
|
+
|
|
91
|
+
def close(self) -> None:
|
|
92
|
+
self._client.close()
|
|
93
|
+
|
|
94
|
+
def __enter__(self) -> "DexVerseClient":
|
|
95
|
+
return self
|
|
96
|
+
|
|
97
|
+
def __exit__(self, *exc) -> None:
|
|
98
|
+
self.close()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def download_manifest(manifest_path: Union[str, Path], out: Union[str, Path], *,
|
|
102
|
+
parallel: Optional[int] = None,
|
|
103
|
+
backend: str = "http",
|
|
104
|
+
verify: bool = True,
|
|
105
|
+
deep: bool = False,
|
|
106
|
+
insecure: bool = True,
|
|
107
|
+
progress: Optional[ProgressReporter] = None,
|
|
108
|
+
backend_kwargs: Optional[Dict[str, object]] = None) -> DownloadReport:
|
|
109
|
+
"""Run a download straight from a local manifest file (Phase 0).
|
|
110
|
+
|
|
111
|
+
Returns the detailed DownloadReport (counts + failures). Note this differs
|
|
112
|
+
from ``DexVerseClient.download_task``, which returns the out path per §11.1.
|
|
113
|
+
|
|
114
|
+
``manifest_path == "-"`` reads the manifest from stdin, so callers can pipe
|
|
115
|
+
a JSON payload (``echo '<json>' | dexverse-dl download --manifest - ...``)
|
|
116
|
+
without saving a file first — useful because presigned URLs expire fast.
|
|
117
|
+
"""
|
|
118
|
+
if str(manifest_path) == "-":
|
|
119
|
+
manifest = Manifest.parse_stdin()
|
|
120
|
+
else:
|
|
121
|
+
manifest = Manifest.parse_file(Path(manifest_path))
|
|
122
|
+
mgr = DownloadManager(out, backend=backend, parallel=parallel,
|
|
123
|
+
verify=verify, deep=deep, insecure=insecure,
|
|
124
|
+
progress=progress, backend_kwargs=backend_kwargs or {})
|
|
125
|
+
report = mgr.run(manifest)
|
|
126
|
+
if not report.ok:
|
|
127
|
+
raise _download_error(report)
|
|
128
|
+
return report
|
dexverse/config.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""Login / server configuration.
|
|
2
|
+
|
|
3
|
+
Stores the active server URL and bearer token per *profile* in a TOML file at
|
|
4
|
+
``~/.dexverse/config.toml``. The file is created with mode 0600 where the OS
|
|
5
|
+
supports it. Secrets are never written into the manifest or into Git
|
|
6
|
+
(see doc §13.1).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Optional
|
|
14
|
+
|
|
15
|
+
import tomlkit
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def config_path() -> Path:
|
|
19
|
+
"""Location of the CLI config file (overridable via DEXVERSE_CONFIG)."""
|
|
20
|
+
override = os.getenv("DEXVERSE_CONFIG")
|
|
21
|
+
if override:
|
|
22
|
+
return Path(override)
|
|
23
|
+
return Path.home() / ".dexverse" / "config.toml"
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class Profile:
|
|
28
|
+
server: str
|
|
29
|
+
token: Optional[str] = None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _read() -> tomlkit.TOMLDocument:
|
|
33
|
+
p = config_path()
|
|
34
|
+
if not p.exists():
|
|
35
|
+
return tomlkit.document()
|
|
36
|
+
try:
|
|
37
|
+
return tomlkit.parse(p.read_text(encoding="utf-8"))
|
|
38
|
+
except Exception:
|
|
39
|
+
return tomlkit.document()
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _write(doc: tomlkit.TOMLDocument) -> None:
|
|
43
|
+
p = config_path()
|
|
44
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
45
|
+
# Best-effort restrictive permissions (POSIX only; no-op on Windows).
|
|
46
|
+
p.write_text(tomlkit.dumps(doc), encoding="utf-8")
|
|
47
|
+
try:
|
|
48
|
+
os.chmod(p, 0o600)
|
|
49
|
+
except OSError:
|
|
50
|
+
pass
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def save_profile(name: str, server: str, token: Optional[str]) -> None:
|
|
54
|
+
doc = _read()
|
|
55
|
+
profiles = doc.get("profiles")
|
|
56
|
+
if not isinstance(profiles, tomlkit.items.Table) or profiles is None:
|
|
57
|
+
profiles = tomlkit.table()
|
|
58
|
+
doc["profiles"] = profiles
|
|
59
|
+
table = tomlkit.table()
|
|
60
|
+
table["server"] = server
|
|
61
|
+
if token:
|
|
62
|
+
table["token"] = token
|
|
63
|
+
profiles[name] = table # type: ignore[assignment]
|
|
64
|
+
doc["active"] = name
|
|
65
|
+
_write(doc)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def load_profile(name: Optional[str] = None) -> Optional[Profile]:
|
|
69
|
+
"""Load the named profile, or the active one if name is None."""
|
|
70
|
+
doc = _read()
|
|
71
|
+
profiles = doc.get("profiles")
|
|
72
|
+
if not isinstance(profiles, dict):
|
|
73
|
+
return None
|
|
74
|
+
target = name or doc.get("active")
|
|
75
|
+
if not target or target not in profiles:
|
|
76
|
+
return None
|
|
77
|
+
row = profiles[target]
|
|
78
|
+
server = row.get("server")
|
|
79
|
+
if not server:
|
|
80
|
+
return None
|
|
81
|
+
return Profile(server=server, token=row.get("token"))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def active_profile() -> Optional[Profile]:
|
|
85
|
+
return load_profile(None)
|