dexverse-cli 0.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dexverse/__init__.py +54 -0
- dexverse/cli.py +422 -0
- dexverse/client.py +128 -0
- dexverse/config.py +85 -0
- dexverse/convert/__init__.py +38 -0
- dexverse/convert/hdf5_to_raw.py +498 -0
- dexverse/convert/image_video_encode.py +449 -0
- dexverse/downloader/__init__.py +26 -0
- dexverse/downloader/base.py +101 -0
- dexverse/downloader/http_backend.py +508 -0
- dexverse/downloader/manager.py +365 -0
- dexverse/downloader/python_s3_backend.py +137 -0
- dexverse/downloader/s5cmd_backend.py +170 -0
- dexverse/downloader/state.py +263 -0
- dexverse/downloader/verify.py +114 -0
- dexverse/manifest.py +284 -0
- dexverse/mock/__init__.py +2 -0
- dexverse/mock/mock_tasks.json +22 -0
- dexverse/mock/server.py +211 -0
- dexverse_cli-0.0.2.dist-info/METADATA +198 -0
- dexverse_cli-0.0.2.dist-info/RECORD +23 -0
- dexverse_cli-0.0.2.dist-info/WHEEL +4 -0
- dexverse_cli-0.0.2.dist-info/entry_points.txt +3 -0
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
"""Per-task state: the ``<out>/.dexverse/tasks/<taskId>/`` directory.
|
|
2
|
+
|
|
3
|
+
State is kept strictly under the out root so it never pollutes the dataset
|
|
4
|
+
tree (doc §6). ``download-state.json`` records per-file status so verify,
|
|
5
|
+
repair and resume all share one source of truth.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
from datetime import datetime, timezone
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Dict, List, Optional
|
|
14
|
+
|
|
15
|
+
from ..manifest import Manifest, resolve_dest
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
STATE_DIRNAME = ".dexverse"
|
|
19
|
+
TASKS_DIRNAME = "tasks"
|
|
20
|
+
TASK_INSTRUCTIONS_DIRNAME = "task-instructions"
|
|
21
|
+
|
|
22
|
+
# file statuses
|
|
23
|
+
PENDING = "pending"
|
|
24
|
+
DOWNLOADED = "downloaded"
|
|
25
|
+
SKIPPED = "skipped" # already present + size matched before download
|
|
26
|
+
FAILED = "failed"
|
|
27
|
+
VERIFIED = "verified"
|
|
28
|
+
CORRUPT = "corrupt" # present but failed size/sha check
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def dexverse_root(out: Path) -> Path:
|
|
32
|
+
return Path(out) / STATE_DIRNAME
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def task_dir(out: Path, task_id: str) -> Path:
|
|
36
|
+
return dexverse_root(out) / TASKS_DIRNAME / task_id
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def task_instructions_dir(out: Path) -> Path:
|
|
40
|
+
return dexverse_root(out) / TASK_INSTRUCTIONS_DIRNAME
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def task_instruction_path(out: Path, relpath: str) -> Path:
|
|
44
|
+
"""Hidden metadata path for one downloaded dataset file."""
|
|
45
|
+
return resolve_dest(task_instructions_dir(out), f"{relpath}.json")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def state_file_path(out: Path, task_id: str) -> Path:
|
|
49
|
+
return task_dir(out, task_id) / "download-state.json"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def manifest_copy_path(out: Path, task_id: str) -> Path:
|
|
53
|
+
return task_dir(out, task_id) / "manifest.json"
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def s5cmd_file_path(out: Path, task_id: str) -> Path:
|
|
57
|
+
return task_dir(out, task_id) / "download.s5cmd"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def log_file_path(out: Path, task_id: str) -> Path:
|
|
61
|
+
return task_dir(out, task_id) / "download.log"
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def ensure_task_dir(out: Path, task_id: str) -> Path:
|
|
65
|
+
d = task_dir(out, task_id)
|
|
66
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
67
|
+
# Best-effort restrictive perms (POSIX; no-op on Windows). The task dir
|
|
68
|
+
# holds a manifest copy that may include S3 creds / presigned URLs.
|
|
69
|
+
try:
|
|
70
|
+
os.chmod(d, 0o700)
|
|
71
|
+
except OSError:
|
|
72
|
+
pass
|
|
73
|
+
return d
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _now_iso() -> str:
|
|
77
|
+
return datetime.now(timezone.utc).isoformat()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class FileState:
|
|
81
|
+
__slots__ = (
|
|
82
|
+
"fileId", "objectKey", "relpath", "status", "localSize",
|
|
83
|
+
"expectedSize", "sha256", "error",
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
def __init__(self, fileId: str, objectKey: str, relpath: str,
|
|
87
|
+
status: str = PENDING, localSize: Optional[int] = None,
|
|
88
|
+
expectedSize: Optional[int] = None, sha256: Optional[str] = None,
|
|
89
|
+
error: Optional[str] = None):
|
|
90
|
+
self.fileId = fileId
|
|
91
|
+
self.objectKey = objectKey
|
|
92
|
+
self.relpath = relpath
|
|
93
|
+
self.status = status
|
|
94
|
+
self.localSize = localSize
|
|
95
|
+
self.expectedSize = expectedSize
|
|
96
|
+
self.sha256 = sha256
|
|
97
|
+
self.error = error
|
|
98
|
+
|
|
99
|
+
def to_dict(self) -> Dict[str, object]:
|
|
100
|
+
return {
|
|
101
|
+
"fileId": self.fileId,
|
|
102
|
+
"objectKey": self.objectKey,
|
|
103
|
+
"relpath": self.relpath,
|
|
104
|
+
"status": self.status,
|
|
105
|
+
"localSize": self.localSize,
|
|
106
|
+
"expectedSize": self.expectedSize,
|
|
107
|
+
"sha256": self.sha256,
|
|
108
|
+
"error": self.error,
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def from_dict(cls, d: Dict[str, object]) -> "FileState":
|
|
113
|
+
return cls(
|
|
114
|
+
fileId=str(d.get("fileId", "")),
|
|
115
|
+
objectKey=str(d.get("objectKey", "")),
|
|
116
|
+
relpath=str(d.get("relpath", "")),
|
|
117
|
+
status=str(d.get("status", PENDING)),
|
|
118
|
+
localSize=d.get("localSize"), # type: ignore[arg-type]
|
|
119
|
+
expectedSize=d.get("expectedSize"), # type: ignore[arg-type]
|
|
120
|
+
sha256=d.get("sha256"), # type: ignore[arg-type]
|
|
121
|
+
error=d.get("error"), # type: ignore[arg-type]
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class TaskState:
|
|
126
|
+
"""In-memory mutable state for a task, persisted as download-state.json."""
|
|
127
|
+
|
|
128
|
+
def __init__(self, out: Path, task_id: str):
|
|
129
|
+
self.out = Path(out)
|
|
130
|
+
self.task_id = task_id
|
|
131
|
+
self.started_at: Optional[str] = None
|
|
132
|
+
self.finished_at: Optional[str] = None
|
|
133
|
+
self.backend: Optional[str] = None
|
|
134
|
+
self.parallel: Optional[int] = None
|
|
135
|
+
self.files: Dict[str, FileState] = {} # keyed by objectKey
|
|
136
|
+
|
|
137
|
+
# ---- construction --------------------------------------------------- #
|
|
138
|
+
|
|
139
|
+
@classmethod
|
|
140
|
+
def for_manifest(cls, out: Path, manifest: Manifest) -> "TaskState":
|
|
141
|
+
task_id = manifest.effective_task_id()
|
|
142
|
+
state = cls(out, task_id)
|
|
143
|
+
for f in manifest.files:
|
|
144
|
+
key = f.effective_key()
|
|
145
|
+
state.files[key] = FileState(
|
|
146
|
+
fileId=f.fileId or key,
|
|
147
|
+
objectKey=key,
|
|
148
|
+
relpath=f.relpath(),
|
|
149
|
+
expectedSize=f.size,
|
|
150
|
+
sha256=f.sha256,
|
|
151
|
+
)
|
|
152
|
+
return state
|
|
153
|
+
|
|
154
|
+
# ---- mutation ------------------------------------------------------- #
|
|
155
|
+
|
|
156
|
+
def mark(self, objectKey: str, status: str, *,
|
|
157
|
+
localSize: Optional[int] = None,
|
|
158
|
+
expectedSize: Optional[int] = None,
|
|
159
|
+
error: Optional[str] = None) -> None:
|
|
160
|
+
fs = self.files.get(objectKey)
|
|
161
|
+
if fs is None:
|
|
162
|
+
return
|
|
163
|
+
fs.status = status
|
|
164
|
+
if localSize is not None:
|
|
165
|
+
fs.localSize = localSize
|
|
166
|
+
if expectedSize is not None:
|
|
167
|
+
fs.expectedSize = expectedSize
|
|
168
|
+
fs.error = error
|
|
169
|
+
|
|
170
|
+
def start(self, backend: str, parallel: int) -> None:
|
|
171
|
+
self.backend = backend
|
|
172
|
+
self.parallel = parallel
|
|
173
|
+
if self.started_at is None:
|
|
174
|
+
self.started_at = _now_iso()
|
|
175
|
+
|
|
176
|
+
def finish(self) -> None:
|
|
177
|
+
self.finished_at = _now_iso()
|
|
178
|
+
|
|
179
|
+
# ---- summary -------------------------------------------------------- #
|
|
180
|
+
|
|
181
|
+
def counts(self) -> Dict[str, int]:
|
|
182
|
+
out: Dict[str, int] = {}
|
|
183
|
+
for fs in self.files.values():
|
|
184
|
+
out[fs.status] = out.get(fs.status, 0) + 1
|
|
185
|
+
return out
|
|
186
|
+
|
|
187
|
+
def pending_or_bad(self) -> List[str]:
|
|
188
|
+
"""objectKeys that still need (re)downloading."""
|
|
189
|
+
bad = {FAILED, CORRUPT, PENDING}
|
|
190
|
+
return [k for k, fs in self.files.items() if fs.status in bad]
|
|
191
|
+
|
|
192
|
+
# ---- persistence ---------------------------------------------------- #
|
|
193
|
+
|
|
194
|
+
def to_dict(self) -> Dict[str, object]:
|
|
195
|
+
return {
|
|
196
|
+
"taskId": self.task_id,
|
|
197
|
+
"out": str(self.out),
|
|
198
|
+
"backend": self.backend,
|
|
199
|
+
"parallel": self.parallel,
|
|
200
|
+
"startedAt": self.started_at,
|
|
201
|
+
"finishedAt": self.finished_at,
|
|
202
|
+
"files": [fs.to_dict() for fs in self.files.values()],
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
def save(self) -> Path:
|
|
206
|
+
ensure_task_dir(self.out, self.task_id)
|
|
207
|
+
path = state_file_path(self.out, self.task_id)
|
|
208
|
+
path.write_text(
|
|
209
|
+
json.dumps(self.to_dict(), indent=2, ensure_ascii=False),
|
|
210
|
+
encoding="utf-8",
|
|
211
|
+
)
|
|
212
|
+
return path
|
|
213
|
+
|
|
214
|
+
@classmethod
|
|
215
|
+
def load(cls, out: Path, task_id: str) -> "TaskState":
|
|
216
|
+
path = state_file_path(out, task_id)
|
|
217
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
218
|
+
state = cls(out, data.get("taskId", task_id))
|
|
219
|
+
state.backend = data.get("backend")
|
|
220
|
+
state.parallel = data.get("parallel")
|
|
221
|
+
state.started_at = data.get("startedAt")
|
|
222
|
+
state.finished_at = data.get("finishedAt")
|
|
223
|
+
for fd in data.get("files", []):
|
|
224
|
+
fs = FileState.from_dict(fd)
|
|
225
|
+
state.files[fs.objectKey] = fs
|
|
226
|
+
return state
|
|
227
|
+
|
|
228
|
+
@classmethod
|
|
229
|
+
def load_or_create(cls, out: Path, manifest: Manifest) -> "TaskState":
|
|
230
|
+
task_id = manifest.effective_task_id()
|
|
231
|
+
path = state_file_path(out, task_id)
|
|
232
|
+
state: Optional["TaskState"]
|
|
233
|
+
if path.exists():
|
|
234
|
+
try:
|
|
235
|
+
state = cls.load(out, task_id)
|
|
236
|
+
except Exception:
|
|
237
|
+
state = cls.for_manifest(out, manifest)
|
|
238
|
+
return state
|
|
239
|
+
else:
|
|
240
|
+
return cls.for_manifest(out, manifest)
|
|
241
|
+
|
|
242
|
+
# Reconcile against the current manifest: any file added since the last
|
|
243
|
+
# run must be tracked (and carry expectedSize/sha256 for verify), else
|
|
244
|
+
# mark() silently no-ops on it and it never reaches download-state.json.
|
|
245
|
+
for f in manifest.files:
|
|
246
|
+
key = f.effective_key()
|
|
247
|
+
if key not in state.files:
|
|
248
|
+
state.files[key] = FileState(
|
|
249
|
+
fileId=f.fileId or key,
|
|
250
|
+
objectKey=key,
|
|
251
|
+
relpath=f.relpath(),
|
|
252
|
+
expectedSize=f.size,
|
|
253
|
+
sha256=f.sha256,
|
|
254
|
+
)
|
|
255
|
+
else:
|
|
256
|
+
fs = state.files[key]
|
|
257
|
+
fs.fileId = f.fileId or key
|
|
258
|
+
fs.relpath = f.relpath()
|
|
259
|
+
if f.size is not None:
|
|
260
|
+
fs.expectedSize = f.size
|
|
261
|
+
if f.sha256 is not None:
|
|
262
|
+
fs.sha256 = f.sha256
|
|
263
|
+
return state
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Verification: size check (fast) and optional sha256 deep check (doc §17).
|
|
2
|
+
|
|
3
|
+
A file is *good* when it exists on disk and its size matches the manifest
|
|
4
|
+
(and, under ``--deep`` or when a sha256 is recorded, its hash matches). Any
|
|
5
|
+
mismatch is reported so repair can re-fetch just those objects.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Iterable, List
|
|
13
|
+
|
|
14
|
+
from ..manifest import FileSpec, resolve_dest
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
# 64 MiB chunks — large enough to stay fast on big HDF5/MP4, small enough to
|
|
18
|
+
# keep memory bounded when hashing multi-GB videos.
|
|
19
|
+
_CHUNK = 64 * 1024 * 1024
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def sha256_of(path: Path) -> str:
|
|
23
|
+
h = hashlib.sha256()
|
|
24
|
+
with path.open("rb") as fh:
|
|
25
|
+
for chunk in iter(lambda: fh.read(_CHUNK), b""):
|
|
26
|
+
h.update(chunk)
|
|
27
|
+
return h.hexdigest()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class VerifyResult:
|
|
32
|
+
fileId: str
|
|
33
|
+
objectKey: str
|
|
34
|
+
relpath: str
|
|
35
|
+
dest: Path
|
|
36
|
+
ok: bool
|
|
37
|
+
reason: str # "ok" | "missing" | "size" | "sha256"
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def missing(self) -> bool:
|
|
41
|
+
return self.reason == "missing"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def verify_file(file: FileSpec, out: Path, *, deep: bool = False,
|
|
45
|
+
force_hash: bool = False) -> VerifyResult:
|
|
46
|
+
"""Check one file on disk.
|
|
47
|
+
|
|
48
|
+
deep : always hash when a sha256 is recorded.
|
|
49
|
+
force_hash : hash even when the manifest has no sha256 (returns reason
|
|
50
|
+
"ok" since there is nothing to compare against — used by the
|
|
51
|
+
deep CLI flag on manifests without hashes).
|
|
52
|
+
"""
|
|
53
|
+
relpath = file.relpath()
|
|
54
|
+
dest = resolve_dest(Path(out), relpath)
|
|
55
|
+
key = file.effective_key()
|
|
56
|
+
fid = file.fileId or key
|
|
57
|
+
|
|
58
|
+
if not dest.exists():
|
|
59
|
+
return VerifyResult(fid, key, relpath, dest, False, "missing")
|
|
60
|
+
|
|
61
|
+
local_size = dest.stat().st_size
|
|
62
|
+
if file.size is not None and local_size != file.size:
|
|
63
|
+
return VerifyResult(fid, key, relpath, dest, False, "size")
|
|
64
|
+
|
|
65
|
+
want_hash = file.sha256
|
|
66
|
+
if want_hash and (deep or force_hash):
|
|
67
|
+
if sha256_of(dest).lower() != want_hash.lower():
|
|
68
|
+
return VerifyResult(fid, key, relpath, dest, False, "sha256")
|
|
69
|
+
elif force_hash and not want_hash:
|
|
70
|
+
# Nothing to compare; size already matched, so treat as ok.
|
|
71
|
+
pass
|
|
72
|
+
|
|
73
|
+
return VerifyResult(fid, key, relpath, dest, True, "ok")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def verify_files(files: Iterable[FileSpec], out: Path, *,
|
|
77
|
+
deep: bool = False) -> List[VerifyResult]:
|
|
78
|
+
return [verify_file(f, out, deep=deep) for f in files]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class VerifySummary:
|
|
83
|
+
total: int
|
|
84
|
+
ok: int
|
|
85
|
+
missing: int
|
|
86
|
+
size_mismatch: int
|
|
87
|
+
sha256_mismatch: int
|
|
88
|
+
results: List[VerifyResult]
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def all_ok(self) -> bool:
|
|
92
|
+
return self.ok == self.total
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def bad_results(self) -> List[VerifyResult]:
|
|
96
|
+
return [r for r in self.results if not r.ok]
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def summarize(results: List[VerifyResult]) -> VerifySummary:
|
|
100
|
+
counts = {"missing": 0, "size": 0, "sha256": 0}
|
|
101
|
+
ok = 0
|
|
102
|
+
for r in results:
|
|
103
|
+
if r.ok:
|
|
104
|
+
ok += 1
|
|
105
|
+
else:
|
|
106
|
+
counts[r.reason] = counts.get(r.reason, 0) + 1
|
|
107
|
+
return VerifySummary(
|
|
108
|
+
total=len(results),
|
|
109
|
+
ok=ok,
|
|
110
|
+
missing=counts["missing"],
|
|
111
|
+
size_mismatch=counts["size"],
|
|
112
|
+
sha256_mismatch=counts["sha256"],
|
|
113
|
+
results=results,
|
|
114
|
+
)
|
dexverse/manifest.py
ADDED
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
"""Manifest data model — the core protocol of the DexVerse downloader.
|
|
2
|
+
|
|
3
|
+
The manifest is shared by the CLI, the Python SDK, the mock backend and the
|
|
4
|
+
production backend. Field names deliberately match the on-the-wire JSON exactly
|
|
5
|
+
(camelCase) so the model round-trips without alias gymnastics.
|
|
6
|
+
|
|
7
|
+
Path rule (doc §2, §6, §18):
|
|
8
|
+
|
|
9
|
+
localPath = out / (file.relativePath || file.originalPath || file.objectKey)
|
|
10
|
+
|
|
11
|
+
The manifest only ever carries POSIX relative paths; the absolute local path is
|
|
12
|
+
derived purely from the user-supplied ``--out``. Manifests must never embed a
|
|
13
|
+
local absolute path. Raw DexVerse backend payloads use ``storage_path`` for the
|
|
14
|
+
remote object key and ``file_name`` for the desired local landing name.
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from pathlib import Path, PurePosixPath
|
|
20
|
+
from typing import Any, Dict, List, Optional
|
|
21
|
+
|
|
22
|
+
from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
# --------------------------------------------------------------------------- #
|
|
26
|
+
# Path helpers
|
|
27
|
+
# --------------------------------------------------------------------------- #
|
|
28
|
+
|
|
29
|
+
# A segment that is (or starts with) a Windows drive letter, e.g. "C:" or "C:foo".
|
|
30
|
+
# On Windows, pathlib joinpath with such a segment resets the drive and escapes
|
|
31
|
+
# the out root, so they are stripped as a traversal defense.
|
|
32
|
+
_DRIVE_RE = re.compile(r"^[A-Za-z]:")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def normalize_relpath(raw: str) -> str:
|
|
36
|
+
"""Normalize a manifest relative path to portable POSIX form.
|
|
37
|
+
|
|
38
|
+
- backslashes → forward slashes (defensive; manifest should already be POSIX)
|
|
39
|
+
- drop a leading ``/`` (it is *relative*, not absolute)
|
|
40
|
+
- strip Windows drive-letter prefixes (``C:`` or ``C:foo``) per segment —
|
|
41
|
+
on Windows pathlib treats these as a drive reset that escapes the out root
|
|
42
|
+
- drop ``.`` and ``..`` segments so a path can never escape the out root
|
|
43
|
+
"""
|
|
44
|
+
if raw is None:
|
|
45
|
+
return ""
|
|
46
|
+
posix = str(raw).replace("\\", "/")
|
|
47
|
+
parts = []
|
|
48
|
+
for seg in posix.split("/"):
|
|
49
|
+
if _DRIVE_RE.match(seg): # strip a leading "X:" drive prefix
|
|
50
|
+
seg = seg[2:]
|
|
51
|
+
if seg not in ("", ".", ".."):
|
|
52
|
+
parts.append(seg)
|
|
53
|
+
return "/".join(parts)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def resolve_relpath(
|
|
57
|
+
relativePath: Optional[str],
|
|
58
|
+
originalPath: Optional[str],
|
|
59
|
+
objectKey: Optional[str],
|
|
60
|
+
) -> str:
|
|
61
|
+
"""Apply the priority rule: relativePath || originalPath || objectKey."""
|
|
62
|
+
chosen = relativePath or originalPath or objectKey or ""
|
|
63
|
+
return normalize_relpath(chosen)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def resolve_dest(out, relpath_posix: str) -> Path:
|
|
67
|
+
"""Join an out root with a POSIX relative path, guaranteed to stay under out.
|
|
68
|
+
|
|
69
|
+
The relpath is treated as POSIX and re-joined onto the local ``Path``, which
|
|
70
|
+
works on both Linux and Windows. As an authoritative backstop (defense in
|
|
71
|
+
depth on top of ``normalize_relpath``), the destination is checked to live
|
|
72
|
+
under ``out``. This check is **lexical** by design: ``normalize_relpath``
|
|
73
|
+
already strips ``..``, drive letters, and absolute anchors, so a lexical
|
|
74
|
+
``relative_to`` test fully contains the threat model. We deliberately avoid
|
|
75
|
+
``Path.resolve()`` here — on Windows, resolving through temp-dir junctions
|
|
76
|
+
while sibling download threads create directories made the check flaky and
|
|
77
|
+
raised false "escapes the out root" errors for ordinary paths like a/1.bin.
|
|
78
|
+
"""
|
|
79
|
+
parts = [seg for seg in relpath_posix.split("/") if seg not in ("", ".", "..")]
|
|
80
|
+
out_path = Path(out)
|
|
81
|
+
dest = out_path.joinpath(*parts)
|
|
82
|
+
try:
|
|
83
|
+
dest.relative_to(out_path)
|
|
84
|
+
except ValueError:
|
|
85
|
+
raise ValueError(
|
|
86
|
+
f"unsafe relative path escapes the out root: {relpath_posix!r}"
|
|
87
|
+
)
|
|
88
|
+
return dest
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# --------------------------------------------------------------------------- #
|
|
92
|
+
# Models
|
|
93
|
+
# --------------------------------------------------------------------------- #
|
|
94
|
+
|
|
95
|
+
class StorageCredentials(BaseModel):
|
|
96
|
+
model_config = ConfigDict(extra="allow")
|
|
97
|
+
|
|
98
|
+
type: str = "dev" # "temporary" | "dev" | "static" | "env"
|
|
99
|
+
accessKey: Optional[str] = None
|
|
100
|
+
secretKey: Optional[str] = None
|
|
101
|
+
sessionToken: Optional[str] = ""
|
|
102
|
+
expiresAt: Optional[str] = None
|
|
103
|
+
|
|
104
|
+
def is_populated(self) -> bool:
|
|
105
|
+
"""True when the manifest carries real (non-env) credentials."""
|
|
106
|
+
return bool(self.accessKey and self.secretKey)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class Storage(BaseModel):
|
|
110
|
+
model_config = ConfigDict(extra="allow")
|
|
111
|
+
|
|
112
|
+
type: str = "s3"
|
|
113
|
+
endpoint: Optional[str] = None
|
|
114
|
+
region: Optional[str] = "us-east-1"
|
|
115
|
+
bucket: Optional[str] = None
|
|
116
|
+
addressingStyle: str = "path"
|
|
117
|
+
credentials: StorageCredentials = Field(default_factory=StorageCredentials)
|
|
118
|
+
|
|
119
|
+
def resolved_bucket(self, file_bucket: Optional[str]) -> Optional[str]:
|
|
120
|
+
return file_bucket or self.bucket
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class DownloadOptions(BaseModel):
|
|
124
|
+
model_config = ConfigDict(extra="allow")
|
|
125
|
+
|
|
126
|
+
defaultParallel: int = 16
|
|
127
|
+
verify: str = "size" # "size" | "sha256" | "none"
|
|
128
|
+
preserveOriginalPath: bool = True
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class FileSpec(BaseModel):
|
|
132
|
+
# Accept the canonical manifest field names AND the DexVerse backend's
|
|
133
|
+
# native field names (download_url / storage_path / storage_bucket /
|
|
134
|
+
# file_name / file_size) so a manifest can be the backend's raw output.
|
|
135
|
+
# populate_by_name keeps the canonical names working too; serialization
|
|
136
|
+
# (model_dump) always emits canonical names, so the on-disk task manifest
|
|
137
|
+
# stays in the canonical contract regardless of input dialect.
|
|
138
|
+
model_config = ConfigDict(extra="allow", populate_by_name=True)
|
|
139
|
+
|
|
140
|
+
fileId: Optional[str] = Field(
|
|
141
|
+
None, validation_alias=AliasChoices("fileId", "file_name", "file_id", "id"))
|
|
142
|
+
uri: Optional[str] = None
|
|
143
|
+
bucket: Optional[str] = Field(
|
|
144
|
+
None, validation_alias=AliasChoices("bucket", "storage_bucket"))
|
|
145
|
+
# objectKey is optional for the http/presigned path: a presigned URL +
|
|
146
|
+
# landing file_name is enough to download. s5cmd/python still need it.
|
|
147
|
+
# When absent, effective_key() falls back to file_name / presignedUrl.
|
|
148
|
+
objectKey: Optional[str] = Field(
|
|
149
|
+
None, validation_alias=AliasChoices("objectKey", "storage_path", "key", "object_key"))
|
|
150
|
+
relativePath: Optional[str] = Field(
|
|
151
|
+
None, validation_alias=AliasChoices("relativePath", "relative_path", "file_name"))
|
|
152
|
+
originalPath: Optional[str] = Field(
|
|
153
|
+
None, validation_alias=AliasChoices("originalPath", "original_path"))
|
|
154
|
+
size: Optional[int] = Field(
|
|
155
|
+
None, validation_alias=AliasChoices("size", "file_size", "bytes"))
|
|
156
|
+
sha256: Optional[str] = Field(
|
|
157
|
+
None, validation_alias=AliasChoices("sha256", "sha"))
|
|
158
|
+
# A short-lived, self-authenticating HTTPS download URL (doc reality: the
|
|
159
|
+
# DexVerse platform issues presigned URLs instead of temporary S3 creds).
|
|
160
|
+
# Consumed only by the ``http`` backend; ignored by s5cmd/python.
|
|
161
|
+
presignedUrl: Optional[str] = Field(
|
|
162
|
+
None, validation_alias=AliasChoices("presignedUrl", "download_url", "url", "downloadUrl"))
|
|
163
|
+
taskInstruction: Optional[Dict[str, Any]] = Field(
|
|
164
|
+
None,
|
|
165
|
+
validation_alias=AliasChoices("taskInstruction", "task_instruction"),
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
def relpath(self) -> str:
|
|
169
|
+
"""The local landing path relative to out (POSIX, normalized)."""
|
|
170
|
+
return resolve_relpath(self.relativePath, self.originalPath, self.objectKey)
|
|
171
|
+
|
|
172
|
+
def effective_key(self) -> str:
|
|
173
|
+
"""A stable identifier for result maps / logs.
|
|
174
|
+
|
|
175
|
+
objectKey when present; otherwise fall back to the landing file_name,
|
|
176
|
+
then the presignedUrl. The http/presigned path only needs the URL +
|
|
177
|
+
file_name, so manifests with no objectKey stay usable.
|
|
178
|
+
"""
|
|
179
|
+
if self.objectKey:
|
|
180
|
+
return self.objectKey
|
|
181
|
+
if self.relativePath:
|
|
182
|
+
return self.relativePath
|
|
183
|
+
if self.presignedUrl:
|
|
184
|
+
return self.presignedUrl
|
|
185
|
+
raise ValueError("file has no objectKey, file_name, or download_url to identify it")
|
|
186
|
+
|
|
187
|
+
def resolved_bucket(self, default_bucket: Optional[str]) -> Optional[str]:
|
|
188
|
+
return self.bucket or default_bucket
|
|
189
|
+
|
|
190
|
+
def resolved_uri(self, default_bucket: Optional[str]) -> str:
|
|
191
|
+
"""s3://bucket/objectKey, materialized even when ``uri`` was absent."""
|
|
192
|
+
if self.uri:
|
|
193
|
+
return self.uri
|
|
194
|
+
bucket = self.resolved_bucket(default_bucket)
|
|
195
|
+
if not bucket:
|
|
196
|
+
# Fall back to a bare s3:///key form; callers validate bucket first.
|
|
197
|
+
return f"s3:///{self.objectKey.lstrip('/')}"
|
|
198
|
+
return f"s3://{bucket}/{self.objectKey.lstrip('/')}"
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class Manifest(BaseModel):
|
|
202
|
+
model_config = ConfigDict(extra="allow")
|
|
203
|
+
|
|
204
|
+
manifestVersion: str = "1.0"
|
|
205
|
+
taskId: Optional[str] = None
|
|
206
|
+
createdAt: Optional[str] = None
|
|
207
|
+
expiresAt: Optional[str] = None
|
|
208
|
+
storage: Optional[Storage] = None
|
|
209
|
+
download: DownloadOptions = Field(default_factory=DownloadOptions)
|
|
210
|
+
files: List[FileSpec] = Field(default_factory=list)
|
|
211
|
+
|
|
212
|
+
# ---- convenience ---------------------------------------------------- #
|
|
213
|
+
|
|
214
|
+
@model_validator(mode="after")
|
|
215
|
+
def keep_first_file_per_local_path(self) -> "Manifest":
|
|
216
|
+
"""Drop later files that would overwrite the same local destination.
|
|
217
|
+
|
|
218
|
+
Backend-native manifests commonly provide only ``file_name`` as the
|
|
219
|
+
landing path. Keeping the first occurrence makes duplicate names
|
|
220
|
+
deterministic and ensures the downloaded file and its task instruction
|
|
221
|
+
always come from the same manifest entry.
|
|
222
|
+
"""
|
|
223
|
+
seen = set()
|
|
224
|
+
unique_files = []
|
|
225
|
+
for file_spec in self.files:
|
|
226
|
+
relpath = file_spec.relpath()
|
|
227
|
+
if relpath in seen:
|
|
228
|
+
continue
|
|
229
|
+
seen.add(relpath)
|
|
230
|
+
unique_files.append(file_spec)
|
|
231
|
+
self.files = unique_files
|
|
232
|
+
return self
|
|
233
|
+
|
|
234
|
+
def effective_task_id(self) -> str:
|
|
235
|
+
"""taskId for the state dir; falls back to a stable hash if absent."""
|
|
236
|
+
if self.taskId:
|
|
237
|
+
return self.taskId
|
|
238
|
+
import hashlib
|
|
239
|
+
digest = hashlib.sha1(
|
|
240
|
+
"|".join(f.effective_key() for f in self.files).encode("utf-8")
|
|
241
|
+
).hexdigest()[:12]
|
|
242
|
+
return f"local_{digest}"
|
|
243
|
+
|
|
244
|
+
def has_credentials(self) -> bool:
|
|
245
|
+
return bool(self.storage and self.storage.credentials.is_populated())
|
|
246
|
+
|
|
247
|
+
@classmethod
|
|
248
|
+
def from_payload(cls, data) -> "Manifest":
|
|
249
|
+
"""Validate a manifest from a dict or list.
|
|
250
|
+
|
|
251
|
+
A bare list ``[{...}, {...}]`` (e.g. the backend's raw per-file response)
|
|
252
|
+
is coerced into ``{"files": [...]}``.
|
|
253
|
+
"""
|
|
254
|
+
import json
|
|
255
|
+
if isinstance(data, str):
|
|
256
|
+
data = json.loads(data)
|
|
257
|
+
if isinstance(data, list):
|
|
258
|
+
data = {"files": data}
|
|
259
|
+
return cls.model_validate(data)
|
|
260
|
+
|
|
261
|
+
@classmethod
|
|
262
|
+
def parse_file(cls, path: Path) -> "Manifest":
|
|
263
|
+
import json
|
|
264
|
+
text = Path(path).read_text(encoding="utf-8")
|
|
265
|
+
return cls.from_payload(json.loads(text))
|
|
266
|
+
|
|
267
|
+
@classmethod
|
|
268
|
+
def parse_stdin(cls, stream=None) -> "Manifest":
|
|
269
|
+
"""Parse a manifest piped on stdin (``--manifest -``).
|
|
270
|
+
|
|
271
|
+
Lets the platform frontend hand users a self-contained command —
|
|
272
|
+
``echo '<json>' | dexverse-dl download --manifest - ...`` — so they
|
|
273
|
+
don't have to save a JSON file first (presigned URLs expire fast).
|
|
274
|
+
``stream`` defaults to ``sys.stdin``; tests pass a ``StringIO``.
|
|
275
|
+
"""
|
|
276
|
+
import json
|
|
277
|
+
import sys
|
|
278
|
+
stream = stream if stream is not None else sys.stdin
|
|
279
|
+
text = stream.read()
|
|
280
|
+
if not text.strip():
|
|
281
|
+
raise ValueError(
|
|
282
|
+
"no manifest received on stdin (pass --manifest PATH or pipe JSON)"
|
|
283
|
+
)
|
|
284
|
+
return cls.from_payload(json.loads(text))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dl_demo_001": {
|
|
3
|
+
"files": [
|
|
4
|
+
{
|
|
5
|
+
"fileId": "episode_000001",
|
|
6
|
+
"bucket": "dexverse",
|
|
7
|
+
"objectKey": "datasets/aaa_test/v1/episodes/episode_000001.hdf5",
|
|
8
|
+
"relativePath": "datasets/aaa_test/v1/episodes/episode_000001.hdf5",
|
|
9
|
+
"size": 14994636,
|
|
10
|
+
"sha256": ""
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"fileId": "camera_head_left_000001",
|
|
14
|
+
"bucket": "dexverse",
|
|
15
|
+
"objectKey": "datasets/aaa_test/v1/videos/camera_head_left.mp4",
|
|
16
|
+
"relativePath": "datasets/aaa_test/v1/videos/camera_head_left.mp4",
|
|
17
|
+
"size": 1234567890,
|
|
18
|
+
"sha256": ""
|
|
19
|
+
}
|
|
20
|
+
]
|
|
21
|
+
}
|
|
22
|
+
}
|