dexverse-cli 0.0.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,263 @@
1
+ """Per-task state: the ``<out>/.dexverse/tasks/<taskId>/`` directory.
2
+
3
+ State is kept strictly under the out root so it never pollutes the dataset
4
+ tree (doc §6). ``download-state.json`` records per-file status so verify,
5
+ repair and resume all share one source of truth.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import os
11
+ from datetime import datetime, timezone
12
+ from pathlib import Path
13
+ from typing import Dict, List, Optional
14
+
15
+ from ..manifest import Manifest, resolve_dest
16
+
17
+
18
+ STATE_DIRNAME = ".dexverse"
19
+ TASKS_DIRNAME = "tasks"
20
+ TASK_INSTRUCTIONS_DIRNAME = "task-instructions"
21
+
22
+ # file statuses
23
+ PENDING = "pending"
24
+ DOWNLOADED = "downloaded"
25
+ SKIPPED = "skipped" # already present + size matched before download
26
+ FAILED = "failed"
27
+ VERIFIED = "verified"
28
+ CORRUPT = "corrupt" # present but failed size/sha check
29
+
30
+
31
+ def dexverse_root(out: Path) -> Path:
32
+ return Path(out) / STATE_DIRNAME
33
+
34
+
35
+ def task_dir(out: Path, task_id: str) -> Path:
36
+ return dexverse_root(out) / TASKS_DIRNAME / task_id
37
+
38
+
39
+ def task_instructions_dir(out: Path) -> Path:
40
+ return dexverse_root(out) / TASK_INSTRUCTIONS_DIRNAME
41
+
42
+
43
+ def task_instruction_path(out: Path, relpath: str) -> Path:
44
+ """Hidden metadata path for one downloaded dataset file."""
45
+ return resolve_dest(task_instructions_dir(out), f"{relpath}.json")
46
+
47
+
48
+ def state_file_path(out: Path, task_id: str) -> Path:
49
+ return task_dir(out, task_id) / "download-state.json"
50
+
51
+
52
+ def manifest_copy_path(out: Path, task_id: str) -> Path:
53
+ return task_dir(out, task_id) / "manifest.json"
54
+
55
+
56
+ def s5cmd_file_path(out: Path, task_id: str) -> Path:
57
+ return task_dir(out, task_id) / "download.s5cmd"
58
+
59
+
60
+ def log_file_path(out: Path, task_id: str) -> Path:
61
+ return task_dir(out, task_id) / "download.log"
62
+
63
+
64
+ def ensure_task_dir(out: Path, task_id: str) -> Path:
65
+ d = task_dir(out, task_id)
66
+ d.mkdir(parents=True, exist_ok=True)
67
+ # Best-effort restrictive perms (POSIX; no-op on Windows). The task dir
68
+ # holds a manifest copy that may include S3 creds / presigned URLs.
69
+ try:
70
+ os.chmod(d, 0o700)
71
+ except OSError:
72
+ pass
73
+ return d
74
+
75
+
76
+ def _now_iso() -> str:
77
+ return datetime.now(timezone.utc).isoformat()
78
+
79
+
80
+ class FileState:
81
+ __slots__ = (
82
+ "fileId", "objectKey", "relpath", "status", "localSize",
83
+ "expectedSize", "sha256", "error",
84
+ )
85
+
86
+ def __init__(self, fileId: str, objectKey: str, relpath: str,
87
+ status: str = PENDING, localSize: Optional[int] = None,
88
+ expectedSize: Optional[int] = None, sha256: Optional[str] = None,
89
+ error: Optional[str] = None):
90
+ self.fileId = fileId
91
+ self.objectKey = objectKey
92
+ self.relpath = relpath
93
+ self.status = status
94
+ self.localSize = localSize
95
+ self.expectedSize = expectedSize
96
+ self.sha256 = sha256
97
+ self.error = error
98
+
99
+ def to_dict(self) -> Dict[str, object]:
100
+ return {
101
+ "fileId": self.fileId,
102
+ "objectKey": self.objectKey,
103
+ "relpath": self.relpath,
104
+ "status": self.status,
105
+ "localSize": self.localSize,
106
+ "expectedSize": self.expectedSize,
107
+ "sha256": self.sha256,
108
+ "error": self.error,
109
+ }
110
+
111
+ @classmethod
112
+ def from_dict(cls, d: Dict[str, object]) -> "FileState":
113
+ return cls(
114
+ fileId=str(d.get("fileId", "")),
115
+ objectKey=str(d.get("objectKey", "")),
116
+ relpath=str(d.get("relpath", "")),
117
+ status=str(d.get("status", PENDING)),
118
+ localSize=d.get("localSize"), # type: ignore[arg-type]
119
+ expectedSize=d.get("expectedSize"), # type: ignore[arg-type]
120
+ sha256=d.get("sha256"), # type: ignore[arg-type]
121
+ error=d.get("error"), # type: ignore[arg-type]
122
+ )
123
+
124
+
125
+ class TaskState:
126
+ """In-memory mutable state for a task, persisted as download-state.json."""
127
+
128
+ def __init__(self, out: Path, task_id: str):
129
+ self.out = Path(out)
130
+ self.task_id = task_id
131
+ self.started_at: Optional[str] = None
132
+ self.finished_at: Optional[str] = None
133
+ self.backend: Optional[str] = None
134
+ self.parallel: Optional[int] = None
135
+ self.files: Dict[str, FileState] = {} # keyed by objectKey
136
+
137
+ # ---- construction --------------------------------------------------- #
138
+
139
+ @classmethod
140
+ def for_manifest(cls, out: Path, manifest: Manifest) -> "TaskState":
141
+ task_id = manifest.effective_task_id()
142
+ state = cls(out, task_id)
143
+ for f in manifest.files:
144
+ key = f.effective_key()
145
+ state.files[key] = FileState(
146
+ fileId=f.fileId or key,
147
+ objectKey=key,
148
+ relpath=f.relpath(),
149
+ expectedSize=f.size,
150
+ sha256=f.sha256,
151
+ )
152
+ return state
153
+
154
+ # ---- mutation ------------------------------------------------------- #
155
+
156
+ def mark(self, objectKey: str, status: str, *,
157
+ localSize: Optional[int] = None,
158
+ expectedSize: Optional[int] = None,
159
+ error: Optional[str] = None) -> None:
160
+ fs = self.files.get(objectKey)
161
+ if fs is None:
162
+ return
163
+ fs.status = status
164
+ if localSize is not None:
165
+ fs.localSize = localSize
166
+ if expectedSize is not None:
167
+ fs.expectedSize = expectedSize
168
+ fs.error = error
169
+
170
+ def start(self, backend: str, parallel: int) -> None:
171
+ self.backend = backend
172
+ self.parallel = parallel
173
+ if self.started_at is None:
174
+ self.started_at = _now_iso()
175
+
176
+ def finish(self) -> None:
177
+ self.finished_at = _now_iso()
178
+
179
+ # ---- summary -------------------------------------------------------- #
180
+
181
+ def counts(self) -> Dict[str, int]:
182
+ out: Dict[str, int] = {}
183
+ for fs in self.files.values():
184
+ out[fs.status] = out.get(fs.status, 0) + 1
185
+ return out
186
+
187
+ def pending_or_bad(self) -> List[str]:
188
+ """objectKeys that still need (re)downloading."""
189
+ bad = {FAILED, CORRUPT, PENDING}
190
+ return [k for k, fs in self.files.items() if fs.status in bad]
191
+
192
+ # ---- persistence ---------------------------------------------------- #
193
+
194
+ def to_dict(self) -> Dict[str, object]:
195
+ return {
196
+ "taskId": self.task_id,
197
+ "out": str(self.out),
198
+ "backend": self.backend,
199
+ "parallel": self.parallel,
200
+ "startedAt": self.started_at,
201
+ "finishedAt": self.finished_at,
202
+ "files": [fs.to_dict() for fs in self.files.values()],
203
+ }
204
+
205
+ def save(self) -> Path:
206
+ ensure_task_dir(self.out, self.task_id)
207
+ path = state_file_path(self.out, self.task_id)
208
+ path.write_text(
209
+ json.dumps(self.to_dict(), indent=2, ensure_ascii=False),
210
+ encoding="utf-8",
211
+ )
212
+ return path
213
+
214
+ @classmethod
215
+ def load(cls, out: Path, task_id: str) -> "TaskState":
216
+ path = state_file_path(out, task_id)
217
+ data = json.loads(path.read_text(encoding="utf-8"))
218
+ state = cls(out, data.get("taskId", task_id))
219
+ state.backend = data.get("backend")
220
+ state.parallel = data.get("parallel")
221
+ state.started_at = data.get("startedAt")
222
+ state.finished_at = data.get("finishedAt")
223
+ for fd in data.get("files", []):
224
+ fs = FileState.from_dict(fd)
225
+ state.files[fs.objectKey] = fs
226
+ return state
227
+
228
+ @classmethod
229
+ def load_or_create(cls, out: Path, manifest: Manifest) -> "TaskState":
230
+ task_id = manifest.effective_task_id()
231
+ path = state_file_path(out, task_id)
232
+ state: Optional["TaskState"]
233
+ if path.exists():
234
+ try:
235
+ state = cls.load(out, task_id)
236
+ except Exception:
237
+ state = cls.for_manifest(out, manifest)
238
+ return state
239
+ else:
240
+ return cls.for_manifest(out, manifest)
241
+
242
+ # Reconcile against the current manifest: any file added since the last
243
+ # run must be tracked (and carry expectedSize/sha256 for verify), else
244
+ # mark() silently no-ops on it and it never reaches download-state.json.
245
+ for f in manifest.files:
246
+ key = f.effective_key()
247
+ if key not in state.files:
248
+ state.files[key] = FileState(
249
+ fileId=f.fileId or key,
250
+ objectKey=key,
251
+ relpath=f.relpath(),
252
+ expectedSize=f.size,
253
+ sha256=f.sha256,
254
+ )
255
+ else:
256
+ fs = state.files[key]
257
+ fs.fileId = f.fileId or key
258
+ fs.relpath = f.relpath()
259
+ if f.size is not None:
260
+ fs.expectedSize = f.size
261
+ if f.sha256 is not None:
262
+ fs.sha256 = f.sha256
263
+ return state
@@ -0,0 +1,114 @@
1
+ """Verification: size check (fast) and optional sha256 deep check (doc §17).
2
+
3
+ A file is *good* when it exists on disk and its size matches the manifest
4
+ (and, under ``--deep`` or when a sha256 is recorded, its hash matches). Any
5
+ mismatch is reported so repair can re-fetch just those objects.
6
+ """
7
+ from __future__ import annotations
8
+
9
+ import hashlib
10
+ from dataclasses import dataclass
11
+ from pathlib import Path
12
+ from typing import Iterable, List
13
+
14
+ from ..manifest import FileSpec, resolve_dest
15
+
16
+
17
+ # 64 MiB chunks — large enough to stay fast on big HDF5/MP4, small enough to
18
+ # keep memory bounded when hashing multi-GB videos.
19
+ _CHUNK = 64 * 1024 * 1024
20
+
21
+
22
+ def sha256_of(path: Path) -> str:
23
+ h = hashlib.sha256()
24
+ with path.open("rb") as fh:
25
+ for chunk in iter(lambda: fh.read(_CHUNK), b""):
26
+ h.update(chunk)
27
+ return h.hexdigest()
28
+
29
+
30
+ @dataclass
31
+ class VerifyResult:
32
+ fileId: str
33
+ objectKey: str
34
+ relpath: str
35
+ dest: Path
36
+ ok: bool
37
+ reason: str # "ok" | "missing" | "size" | "sha256"
38
+
39
+ @property
40
+ def missing(self) -> bool:
41
+ return self.reason == "missing"
42
+
43
+
44
+ def verify_file(file: FileSpec, out: Path, *, deep: bool = False,
45
+ force_hash: bool = False) -> VerifyResult:
46
+ """Check one file on disk.
47
+
48
+ deep : always hash when a sha256 is recorded.
49
+ force_hash : hash even when the manifest has no sha256 (returns reason
50
+ "ok" since there is nothing to compare against — used by the
51
+ deep CLI flag on manifests without hashes).
52
+ """
53
+ relpath = file.relpath()
54
+ dest = resolve_dest(Path(out), relpath)
55
+ key = file.effective_key()
56
+ fid = file.fileId or key
57
+
58
+ if not dest.exists():
59
+ return VerifyResult(fid, key, relpath, dest, False, "missing")
60
+
61
+ local_size = dest.stat().st_size
62
+ if file.size is not None and local_size != file.size:
63
+ return VerifyResult(fid, key, relpath, dest, False, "size")
64
+
65
+ want_hash = file.sha256
66
+ if want_hash and (deep or force_hash):
67
+ if sha256_of(dest).lower() != want_hash.lower():
68
+ return VerifyResult(fid, key, relpath, dest, False, "sha256")
69
+ elif force_hash and not want_hash:
70
+ # Nothing to compare; size already matched, so treat as ok.
71
+ pass
72
+
73
+ return VerifyResult(fid, key, relpath, dest, True, "ok")
74
+
75
+
76
+ def verify_files(files: Iterable[FileSpec], out: Path, *,
77
+ deep: bool = False) -> List[VerifyResult]:
78
+ return [verify_file(f, out, deep=deep) for f in files]
79
+
80
+
81
+ @dataclass
82
+ class VerifySummary:
83
+ total: int
84
+ ok: int
85
+ missing: int
86
+ size_mismatch: int
87
+ sha256_mismatch: int
88
+ results: List[VerifyResult]
89
+
90
+ @property
91
+ def all_ok(self) -> bool:
92
+ return self.ok == self.total
93
+
94
+ @property
95
+ def bad_results(self) -> List[VerifyResult]:
96
+ return [r for r in self.results if not r.ok]
97
+
98
+
99
+ def summarize(results: List[VerifyResult]) -> VerifySummary:
100
+ counts = {"missing": 0, "size": 0, "sha256": 0}
101
+ ok = 0
102
+ for r in results:
103
+ if r.ok:
104
+ ok += 1
105
+ else:
106
+ counts[r.reason] = counts.get(r.reason, 0) + 1
107
+ return VerifySummary(
108
+ total=len(results),
109
+ ok=ok,
110
+ missing=counts["missing"],
111
+ size_mismatch=counts["size"],
112
+ sha256_mismatch=counts["sha256"],
113
+ results=results,
114
+ )
dexverse/manifest.py ADDED
@@ -0,0 +1,284 @@
1
+ """Manifest data model — the core protocol of the DexVerse downloader.
2
+
3
+ The manifest is shared by the CLI, the Python SDK, the mock backend and the
4
+ production backend. Field names deliberately match the on-the-wire JSON exactly
5
+ (camelCase) so the model round-trips without alias gymnastics.
6
+
7
+ Path rule (doc §2, §6, §18):
8
+
9
+ localPath = out / (file.relativePath || file.originalPath || file.objectKey)
10
+
11
+ The manifest only ever carries POSIX relative paths; the absolute local path is
12
+ derived purely from the user-supplied ``--out``. Manifests must never embed a
13
+ local absolute path. Raw DexVerse backend payloads use ``storage_path`` for the
14
+ remote object key and ``file_name`` for the desired local landing name.
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import re
19
+ from pathlib import Path, PurePosixPath
20
+ from typing import Any, Dict, List, Optional
21
+
22
+ from pydantic import AliasChoices, BaseModel, ConfigDict, Field, model_validator
23
+
24
+
25
+ # --------------------------------------------------------------------------- #
26
+ # Path helpers
27
+ # --------------------------------------------------------------------------- #
28
+
29
+ # A segment that is (or starts with) a Windows drive letter, e.g. "C:" or "C:foo".
30
+ # On Windows, pathlib joinpath with such a segment resets the drive and escapes
31
+ # the out root, so they are stripped as a traversal defense.
32
+ _DRIVE_RE = re.compile(r"^[A-Za-z]:")
33
+
34
+
35
+ def normalize_relpath(raw: str) -> str:
36
+ """Normalize a manifest relative path to portable POSIX form.
37
+
38
+ - backslashes → forward slashes (defensive; manifest should already be POSIX)
39
+ - drop a leading ``/`` (it is *relative*, not absolute)
40
+ - strip Windows drive-letter prefixes (``C:`` or ``C:foo``) per segment —
41
+ on Windows pathlib treats these as a drive reset that escapes the out root
42
+ - drop ``.`` and ``..`` segments so a path can never escape the out root
43
+ """
44
+ if raw is None:
45
+ return ""
46
+ posix = str(raw).replace("\\", "/")
47
+ parts = []
48
+ for seg in posix.split("/"):
49
+ if _DRIVE_RE.match(seg): # strip a leading "X:" drive prefix
50
+ seg = seg[2:]
51
+ if seg not in ("", ".", ".."):
52
+ parts.append(seg)
53
+ return "/".join(parts)
54
+
55
+
56
+ def resolve_relpath(
57
+ relativePath: Optional[str],
58
+ originalPath: Optional[str],
59
+ objectKey: Optional[str],
60
+ ) -> str:
61
+ """Apply the priority rule: relativePath || originalPath || objectKey."""
62
+ chosen = relativePath or originalPath or objectKey or ""
63
+ return normalize_relpath(chosen)
64
+
65
+
66
+ def resolve_dest(out, relpath_posix: str) -> Path:
67
+ """Join an out root with a POSIX relative path, guaranteed to stay under out.
68
+
69
+ The relpath is treated as POSIX and re-joined onto the local ``Path``, which
70
+ works on both Linux and Windows. As an authoritative backstop (defense in
71
+ depth on top of ``normalize_relpath``), the destination is checked to live
72
+ under ``out``. This check is **lexical** by design: ``normalize_relpath``
73
+ already strips ``..``, drive letters, and absolute anchors, so a lexical
74
+ ``relative_to`` test fully contains the threat model. We deliberately avoid
75
+ ``Path.resolve()`` here — on Windows, resolving through temp-dir junctions
76
+ while sibling download threads create directories made the check flaky and
77
+ raised false "escapes the out root" errors for ordinary paths like a/1.bin.
78
+ """
79
+ parts = [seg for seg in relpath_posix.split("/") if seg not in ("", ".", "..")]
80
+ out_path = Path(out)
81
+ dest = out_path.joinpath(*parts)
82
+ try:
83
+ dest.relative_to(out_path)
84
+ except ValueError:
85
+ raise ValueError(
86
+ f"unsafe relative path escapes the out root: {relpath_posix!r}"
87
+ )
88
+ return dest
89
+
90
+
91
+ # --------------------------------------------------------------------------- #
92
+ # Models
93
+ # --------------------------------------------------------------------------- #
94
+
95
+ class StorageCredentials(BaseModel):
96
+ model_config = ConfigDict(extra="allow")
97
+
98
+ type: str = "dev" # "temporary" | "dev" | "static" | "env"
99
+ accessKey: Optional[str] = None
100
+ secretKey: Optional[str] = None
101
+ sessionToken: Optional[str] = ""
102
+ expiresAt: Optional[str] = None
103
+
104
+ def is_populated(self) -> bool:
105
+ """True when the manifest carries real (non-env) credentials."""
106
+ return bool(self.accessKey and self.secretKey)
107
+
108
+
109
+ class Storage(BaseModel):
110
+ model_config = ConfigDict(extra="allow")
111
+
112
+ type: str = "s3"
113
+ endpoint: Optional[str] = None
114
+ region: Optional[str] = "us-east-1"
115
+ bucket: Optional[str] = None
116
+ addressingStyle: str = "path"
117
+ credentials: StorageCredentials = Field(default_factory=StorageCredentials)
118
+
119
+ def resolved_bucket(self, file_bucket: Optional[str]) -> Optional[str]:
120
+ return file_bucket or self.bucket
121
+
122
+
123
+ class DownloadOptions(BaseModel):
124
+ model_config = ConfigDict(extra="allow")
125
+
126
+ defaultParallel: int = 16
127
+ verify: str = "size" # "size" | "sha256" | "none"
128
+ preserveOriginalPath: bool = True
129
+
130
+
131
+ class FileSpec(BaseModel):
132
+ # Accept the canonical manifest field names AND the DexVerse backend's
133
+ # native field names (download_url / storage_path / storage_bucket /
134
+ # file_name / file_size) so a manifest can be the backend's raw output.
135
+ # populate_by_name keeps the canonical names working too; serialization
136
+ # (model_dump) always emits canonical names, so the on-disk task manifest
137
+ # stays in the canonical contract regardless of input dialect.
138
+ model_config = ConfigDict(extra="allow", populate_by_name=True)
139
+
140
+ fileId: Optional[str] = Field(
141
+ None, validation_alias=AliasChoices("fileId", "file_name", "file_id", "id"))
142
+ uri: Optional[str] = None
143
+ bucket: Optional[str] = Field(
144
+ None, validation_alias=AliasChoices("bucket", "storage_bucket"))
145
+ # objectKey is optional for the http/presigned path: a presigned URL +
146
+ # landing file_name is enough to download. s5cmd/python still need it.
147
+ # When absent, effective_key() falls back to file_name / presignedUrl.
148
+ objectKey: Optional[str] = Field(
149
+ None, validation_alias=AliasChoices("objectKey", "storage_path", "key", "object_key"))
150
+ relativePath: Optional[str] = Field(
151
+ None, validation_alias=AliasChoices("relativePath", "relative_path", "file_name"))
152
+ originalPath: Optional[str] = Field(
153
+ None, validation_alias=AliasChoices("originalPath", "original_path"))
154
+ size: Optional[int] = Field(
155
+ None, validation_alias=AliasChoices("size", "file_size", "bytes"))
156
+ sha256: Optional[str] = Field(
157
+ None, validation_alias=AliasChoices("sha256", "sha"))
158
+ # A short-lived, self-authenticating HTTPS download URL (doc reality: the
159
+ # DexVerse platform issues presigned URLs instead of temporary S3 creds).
160
+ # Consumed only by the ``http`` backend; ignored by s5cmd/python.
161
+ presignedUrl: Optional[str] = Field(
162
+ None, validation_alias=AliasChoices("presignedUrl", "download_url", "url", "downloadUrl"))
163
+ taskInstruction: Optional[Dict[str, Any]] = Field(
164
+ None,
165
+ validation_alias=AliasChoices("taskInstruction", "task_instruction"),
166
+ )
167
+
168
+ def relpath(self) -> str:
169
+ """The local landing path relative to out (POSIX, normalized)."""
170
+ return resolve_relpath(self.relativePath, self.originalPath, self.objectKey)
171
+
172
+ def effective_key(self) -> str:
173
+ """A stable identifier for result maps / logs.
174
+
175
+ objectKey when present; otherwise fall back to the landing file_name,
176
+ then the presignedUrl. The http/presigned path only needs the URL +
177
+ file_name, so manifests with no objectKey stay usable.
178
+ """
179
+ if self.objectKey:
180
+ return self.objectKey
181
+ if self.relativePath:
182
+ return self.relativePath
183
+ if self.presignedUrl:
184
+ return self.presignedUrl
185
+ raise ValueError("file has no objectKey, file_name, or download_url to identify it")
186
+
187
+ def resolved_bucket(self, default_bucket: Optional[str]) -> Optional[str]:
188
+ return self.bucket or default_bucket
189
+
190
+ def resolved_uri(self, default_bucket: Optional[str]) -> str:
191
+ """s3://bucket/objectKey, materialized even when ``uri`` was absent."""
192
+ if self.uri:
193
+ return self.uri
194
+ bucket = self.resolved_bucket(default_bucket)
195
+ if not bucket:
196
+ # Fall back to a bare s3:///key form; callers validate bucket first.
197
+ return f"s3:///{self.objectKey.lstrip('/')}"
198
+ return f"s3://{bucket}/{self.objectKey.lstrip('/')}"
199
+
200
+
201
+ class Manifest(BaseModel):
202
+ model_config = ConfigDict(extra="allow")
203
+
204
+ manifestVersion: str = "1.0"
205
+ taskId: Optional[str] = None
206
+ createdAt: Optional[str] = None
207
+ expiresAt: Optional[str] = None
208
+ storage: Optional[Storage] = None
209
+ download: DownloadOptions = Field(default_factory=DownloadOptions)
210
+ files: List[FileSpec] = Field(default_factory=list)
211
+
212
+ # ---- convenience ---------------------------------------------------- #
213
+
214
+ @model_validator(mode="after")
215
+ def keep_first_file_per_local_path(self) -> "Manifest":
216
+ """Drop later files that would overwrite the same local destination.
217
+
218
+ Backend-native manifests commonly provide only ``file_name`` as the
219
+ landing path. Keeping the first occurrence makes duplicate names
220
+ deterministic and ensures the downloaded file and its task instruction
221
+ always come from the same manifest entry.
222
+ """
223
+ seen = set()
224
+ unique_files = []
225
+ for file_spec in self.files:
226
+ relpath = file_spec.relpath()
227
+ if relpath in seen:
228
+ continue
229
+ seen.add(relpath)
230
+ unique_files.append(file_spec)
231
+ self.files = unique_files
232
+ return self
233
+
234
+ def effective_task_id(self) -> str:
235
+ """taskId for the state dir; falls back to a stable hash if absent."""
236
+ if self.taskId:
237
+ return self.taskId
238
+ import hashlib
239
+ digest = hashlib.sha1(
240
+ "|".join(f.effective_key() for f in self.files).encode("utf-8")
241
+ ).hexdigest()[:12]
242
+ return f"local_{digest}"
243
+
244
+ def has_credentials(self) -> bool:
245
+ return bool(self.storage and self.storage.credentials.is_populated())
246
+
247
+ @classmethod
248
+ def from_payload(cls, data) -> "Manifest":
249
+ """Validate a manifest from a dict or list.
250
+
251
+ A bare list ``[{...}, {...}]`` (e.g. the backend's raw per-file response)
252
+ is coerced into ``{"files": [...]}``.
253
+ """
254
+ import json
255
+ if isinstance(data, str):
256
+ data = json.loads(data)
257
+ if isinstance(data, list):
258
+ data = {"files": data}
259
+ return cls.model_validate(data)
260
+
261
+ @classmethod
262
+ def parse_file(cls, path: Path) -> "Manifest":
263
+ import json
264
+ text = Path(path).read_text(encoding="utf-8")
265
+ return cls.from_payload(json.loads(text))
266
+
267
+ @classmethod
268
+ def parse_stdin(cls, stream=None) -> "Manifest":
269
+ """Parse a manifest piped on stdin (``--manifest -``).
270
+
271
+ Lets the platform frontend hand users a self-contained command —
272
+ ``echo '<json>' | dexverse-dl download --manifest - ...`` — so they
273
+ don't have to save a JSON file first (presigned URLs expire fast).
274
+ ``stream`` defaults to ``sys.stdin``; tests pass a ``StringIO``.
275
+ """
276
+ import json
277
+ import sys
278
+ stream = stream if stream is not None else sys.stdin
279
+ text = stream.read()
280
+ if not text.strip():
281
+ raise ValueError(
282
+ "no manifest received on stdin (pass --manifest PATH or pipe JSON)"
283
+ )
284
+ return cls.from_payload(json.loads(text))
@@ -0,0 +1,2 @@
1
+ """DexVerse mock backend (FastAPI). See server.py."""
2
+ from __future__ import annotations
@@ -0,0 +1,22 @@
1
+ {
2
+ "dl_demo_001": {
3
+ "files": [
4
+ {
5
+ "fileId": "episode_000001",
6
+ "bucket": "dexverse",
7
+ "objectKey": "datasets/aaa_test/v1/episodes/episode_000001.hdf5",
8
+ "relativePath": "datasets/aaa_test/v1/episodes/episode_000001.hdf5",
9
+ "size": 14994636,
10
+ "sha256": ""
11
+ },
12
+ {
13
+ "fileId": "camera_head_left_000001",
14
+ "bucket": "dexverse",
15
+ "objectKey": "datasets/aaa_test/v1/videos/camera_head_left.mp4",
16
+ "relativePath": "datasets/aaa_test/v1/videos/camera_head_left.mp4",
17
+ "size": 1234567890,
18
+ "sha256": ""
19
+ }
20
+ ]
21
+ }
22
+ }