dexverse-cli 0.0.4__tar.gz → 0.0.4.post1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/PKG-INFO +9 -6
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/README.md +8 -5
- dexverse_cli-0.0.4.post1/decision-traces/2026-08-12_1613_pypi_0_0_4_release_trace.md +84 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/convert/hdf5_to_raw.py +12 -10
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/manager.py +7 -7
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/state.py +10 -4
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/manifest.py +22 -15
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/pyproject.toml +1 -1
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_convert_additional_info.py +25 -9
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_http_backend.py +27 -6
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_manifest_aliases.py +24 -18
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/.dockerignore +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/.gitignore +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/Dockerfile +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/__init__.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/cli.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/client.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/config.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/convert/__init__.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/convert/image_video_encode.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/__init__.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/base.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/http_backend.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/python_s3_backend.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/s5cmd_backend.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/verify.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/mock/__init__.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/mock/mock_tasks.json +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/mock/server.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docker_run.sh +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/DexVerse_Downloader_S3_Final_/345/256/236/346/226/275/346/226/271/346/241/210_/345/216/237/350/267/257/345/276/204/350/220/275/347/233/230/347/211/210.md" +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/PROMPT_copy_and_download_command.md +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/Python_vs_Node_CLI/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/Python_vs_Node_/346/216/245/345/205/245C++/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs//345/210/206/345/217/221/347/273/231/347/256/227/346/263/225/345/233/242/351/230/237_/346/225/231/347/250/213.md" +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs//346/223/215/344/275/234/346/214/207/345/215/227--/345/234/250Dexechain/344/270/255/350/277/233/350/241/214/346/250/241/344/273/277/345/255/246/344/271/240.md" +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs//346/234/215/345/212/241/345/231/250/344/275/277/347/224/250/346/211/213/345/206/214.md" +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/download (7).json +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/local_demo.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/manifest.json +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/manifest_presigned.json +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/set-creds.ps1 +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/conftest.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_cli.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_http_resilience.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_manager_integration.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_manifest.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_mock_server.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_presigned_fixes.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_review_fixes.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_s5cmd_generation.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_state.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_verify.py +0 -0
- {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: dexverse-cli
|
|
3
|
-
Version: 0.0.4
|
|
3
|
+
Version: 0.0.4.post1
|
|
4
4
|
Summary: DexVerse dataset downloader for S3-compatible robot training datasets
|
|
5
5
|
Author: DexVerse
|
|
6
6
|
License: Apache-2.0
|
|
@@ -104,6 +104,9 @@ D:\DexVerse\20260429_180359.hdf5
|
|
|
104
104
|
items resolve to the same local path (for example, repeated `file_name`
|
|
105
105
|
values), the first item wins and later duplicates are ignored.
|
|
106
106
|
|
|
107
|
+
For compatibility, the CLI also accepts the legacy `task_instruction` field;
|
|
108
|
+
both names must not appear in the same file entry.
|
|
109
|
+
|
|
107
110
|
Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
|
|
108
111
|
also learns the total size from `Content-Length`/`Content-Range`, but manifest
|
|
109
112
|
metadata lets the CLI verify an existing local file before making a request.
|
|
@@ -134,7 +137,7 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
|
|
|
134
137
|
|
|
135
138
|
下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
|
|
136
139
|
+ `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
|
|
137
|
-
`
|
|
140
|
+
`task_instruction.json`(保持算法侧兼容):
|
|
138
141
|
|
|
139
142
|
```bash
|
|
140
143
|
# 单文件
|
|
@@ -153,11 +156,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
|
|
|
153
156
|
hand/right/000000.jpg ...
|
|
154
157
|
pose_record_20260429_180359.json
|
|
155
158
|
metadata.jsonl
|
|
156
|
-
|
|
159
|
+
task_instruction.json
|
|
157
160
|
```
|
|
158
161
|
|
|
159
|
-
`
|
|
160
|
-
下载器会把转换所需的映射保存在 `.dexverse/
|
|
162
|
+
`task_instruction.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
|
|
163
|
+
下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`,无需为
|
|
161
164
|
`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
|
|
162
165
|
|
|
163
166
|
该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
|
|
@@ -180,7 +183,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
|
|
|
180
183
|
|
|
181
184
|
```text
|
|
182
185
|
/data/dexverse/
|
|
183
|
-
.dexverse/
|
|
186
|
+
.dexverse/task-instructions/
|
|
184
187
|
20260429_180359.hdf5.json
|
|
185
188
|
.dexverse/tasks/<taskId>/
|
|
186
189
|
manifest.json
|
|
@@ -74,6 +74,9 @@ D:\DexVerse\20260429_180359.hdf5
|
|
|
74
74
|
items resolve to the same local path (for example, repeated `file_name`
|
|
75
75
|
values), the first item wins and later duplicates are ignored.
|
|
76
76
|
|
|
77
|
+
For compatibility, the CLI also accepts the legacy `task_instruction` field;
|
|
78
|
+
both names must not appear in the same file entry.
|
|
79
|
+
|
|
77
80
|
Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
|
|
78
81
|
also learns the total size from `Content-Length`/`Content-Range`, but manifest
|
|
79
82
|
metadata lets the CLI verify an existing local file before making a request.
|
|
@@ -104,7 +107,7 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
|
|
|
104
107
|
|
|
105
108
|
下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
|
|
106
109
|
+ `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
|
|
107
|
-
`
|
|
110
|
+
`task_instruction.json`(保持算法侧兼容):
|
|
108
111
|
|
|
109
112
|
```bash
|
|
110
113
|
# 单文件
|
|
@@ -123,11 +126,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
|
|
|
123
126
|
hand/right/000000.jpg ...
|
|
124
127
|
pose_record_20260429_180359.json
|
|
125
128
|
metadata.jsonl
|
|
126
|
-
|
|
129
|
+
task_instruction.json
|
|
127
130
|
```
|
|
128
131
|
|
|
129
|
-
`
|
|
130
|
-
下载器会把转换所需的映射保存在 `.dexverse/
|
|
132
|
+
`task_instruction.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
|
|
133
|
+
下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`,无需为
|
|
131
134
|
`convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
|
|
132
135
|
|
|
133
136
|
该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
|
|
@@ -150,7 +153,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
|
|
|
150
153
|
|
|
151
154
|
```text
|
|
152
155
|
/data/dexverse/
|
|
153
|
-
.dexverse/
|
|
156
|
+
.dexverse/task-instructions/
|
|
154
157
|
20260429_180359.hdf5.json
|
|
155
158
|
.dexverse/tasks/<taskId>/
|
|
156
159
|
manifest.json
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# 2026-08-12 PyPI 0.0.4 Release Boundary
|
|
2
|
+
|
|
3
|
+
## Background
|
|
4
|
+
|
|
5
|
+
`dexverse-cli` was prepared for a PyPI 0.0.4 release. The release must remain
|
|
6
|
+
the dataset downloader and must not ship the co-located device cloud-upload
|
|
7
|
+
module or the checked-in HDF5 fixture.
|
|
8
|
+
|
|
9
|
+
## Problem
|
|
10
|
+
|
|
11
|
+
The original Hatch package selection included all files under `dexverse`, so
|
|
12
|
+
both wheel and sdist included `dexverse/dexe_cloud_upload`. The source
|
|
13
|
+
distribution also included `648584.hdf5` (about 41.6 MiB). In addition, the
|
|
14
|
+
public `dexverse.__version__` lookup used the import package name instead of
|
|
15
|
+
the distribution name, so it returned `0.0.0+unknown` after installation.
|
|
16
|
+
|
|
17
|
+
## Evidence And Source Analysis
|
|
18
|
+
|
|
19
|
+
- The pre-release build contained 25 `dexe_cloud_upload` entries in both wheel
|
|
20
|
+
and sdist, and the sdist contained `648584.hdf5`.
|
|
21
|
+
- The 0.0.4 build excluded `/dexverse/dexe_cloud_upload/**` and
|
|
22
|
+
`/648584.hdf5`; the resulting wheel had 23 entries and the sdist had 53,
|
|
23
|
+
with zero entries for either exclusion target.
|
|
24
|
+
- `pyproject.toml` names the distribution `dexverse-cli`; changing
|
|
25
|
+
`version("dexverse")` to `version("dexverse-cli")` made the installed
|
|
26
|
+
public version equal the distribution version.
|
|
27
|
+
- The local suite passed 110 tests. Twine metadata checks passed. Clean Python
|
|
28
|
+
3.10 wheel and Python 3.11 sdist installs succeeded, as did a final PyPI
|
|
29
|
+
Simple Index installation of `dexverse-cli==0.0.4`.
|
|
30
|
+
- A real HDF5 download and deep SHA-256 verification succeeded with the PyPI
|
|
31
|
+
alpha. `repair --deep` does not re-download a same-size SHA-mismatched file,
|
|
32
|
+
but the same behavior and identical `repair()` implementation were confirmed
|
|
33
|
+
in PyPI 0.0.3; it was accepted as an existing issue, not a 0.0.4 regression.
|
|
34
|
+
|
|
35
|
+
## Clarified Constraints
|
|
36
|
+
|
|
37
|
+
- The 0.0.4 release must not include `dexverse/dexe_cloud_upload`.
|
|
38
|
+
- The 0.0.4 release must not include `648584.hdf5`.
|
|
39
|
+
- P1 documentation, license, and Docker concerns were intentionally deferred
|
|
40
|
+
for the later repository refactor.
|
|
41
|
+
- PyPI files are immutable per version, so alpha validation preceded the
|
|
42
|
+
formal release.
|
|
43
|
+
- The trace is local-only and must not contain package-upload credentials.
|
|
44
|
+
|
|
45
|
+
## Final Decision
|
|
46
|
+
|
|
47
|
+
PyPI `dexverse-cli` 0.0.4 was published with the exclusion rules and corrected
|
|
48
|
+
distribution-name version lookup. The release contains the downloader CLI/SDK,
|
|
49
|
+
but not the cloud-upload module or HDF5 fixture. The pre-existing deep-repair
|
|
50
|
+
limitation is not treated as a release blocker for 0.0.4.
|
|
51
|
+
|
|
52
|
+
Future releases must inspect both wheel and sdist contents before upload and
|
|
53
|
+
must keep `dexverse.__version__` aligned with the `dexverse-cli` distribution
|
|
54
|
+
metadata. Do not reintroduce the excluded paths into downloader distributions
|
|
55
|
+
unless product scope explicitly changes.
|
|
56
|
+
|
|
57
|
+
## Current Status
|
|
58
|
+
|
|
59
|
+
- PyPI 0.0.4 wheel and sdist published successfully on 2026-08-12.
|
|
60
|
+
- PyPI installation, version check, exclusion-content check, and CLI help
|
|
61
|
+
smoke test passed.
|
|
62
|
+
- The local release edits are deliberately being reverted after publication at
|
|
63
|
+
user request. Consequently, the checkout will not reproduce the published
|
|
64
|
+
0.0.4 artifact until the release changes are committed or reapplied.
|
|
65
|
+
- This trace is intentionally local and uncommitted.
|
|
66
|
+
|
|
67
|
+
## Revisit Triggers
|
|
68
|
+
|
|
69
|
+
- Start of the planned repository refactor or any new package release.
|
|
70
|
+
- A decision to ship the cloud-upload module as a separate distribution or to
|
|
71
|
+
include it in this package.
|
|
72
|
+
- A decision to fix `repair --deep` behavior, which requires a regression test
|
|
73
|
+
for same-size SHA-256 corruption and forced re-download.
|
|
74
|
+
- A request to align the Git checkout, tag, or release notes with the published
|
|
75
|
+
PyPI 0.0.4 artifact.
|
|
76
|
+
|
|
77
|
+
## Privacy Class
|
|
78
|
+
|
|
79
|
+
trace_privacy_class: local_raw_trace
|
|
80
|
+
|
|
81
|
+
## TraceGym Data Capture
|
|
82
|
+
|
|
83
|
+
TraceGym metadata omitted: this is a repository-specific release provenance
|
|
84
|
+
record, not an evaluated reusable workflow or benchmark artifact.
|
|
@@ -35,7 +35,7 @@ from ..downloader import state as download_state
|
|
|
35
35
|
from .image_video_encode import decode_mp4_bytes_to_rgb_images
|
|
36
36
|
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
TASK_INSTRUCTION_FILENAME = "task_instruction.json"
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
def _find_download_root(hdf5_path: Path) -> Optional[Path]:
|
|
@@ -47,26 +47,28 @@ def _find_download_root(hdf5_path: Path) -> Optional[Path]:
|
|
|
47
47
|
return None
|
|
48
48
|
|
|
49
49
|
|
|
50
|
-
def
|
|
50
|
+
def _sync_restored_task_instruction(
|
|
51
51
|
hdf5_path: Path,
|
|
52
52
|
output_dir: Path,
|
|
53
53
|
) -> None:
|
|
54
|
-
"""
|
|
54
|
+
"""Restore metadata to the legacy algorithm-facing output name."""
|
|
55
55
|
download_root = _find_download_root(hdf5_path)
|
|
56
56
|
if download_root is None:
|
|
57
57
|
return
|
|
58
58
|
|
|
59
59
|
resolved_hdf5 = Path(hdf5_path).expanduser().resolve()
|
|
60
60
|
relpath = resolved_hdf5.relative_to(download_root).as_posix()
|
|
61
|
-
source = download_state.
|
|
62
|
-
|
|
61
|
+
source = download_state.task_instruction_path(download_root, relpath)
|
|
62
|
+
if not source.is_file():
|
|
63
|
+
source = download_state.additional_info_path(download_root, relpath)
|
|
64
|
+
destination = Path(output_dir) / TASK_INSTRUCTION_FILENAME
|
|
63
65
|
|
|
64
66
|
if source.is_file():
|
|
65
|
-
|
|
66
|
-
if not isinstance(
|
|
67
|
-
raise ValueError(f"
|
|
67
|
+
task_instruction = json.loads(source.read_text(encoding="utf-8"))
|
|
68
|
+
if not isinstance(task_instruction, dict):
|
|
69
|
+
raise ValueError(f"task instruction must be a JSON object: {source}")
|
|
68
70
|
destination.write_text(
|
|
69
|
-
json.dumps(
|
|
71
|
+
json.dumps(task_instruction, ensure_ascii=False, indent=2) + "\n",
|
|
70
72
|
encoding="utf-8",
|
|
71
73
|
)
|
|
72
74
|
else:
|
|
@@ -249,7 +251,7 @@ class HDF5ToRawConverter:
|
|
|
249
251
|
img = Image.open(_io.BytesIO(jpg_bytes)).convert("RGB")
|
|
250
252
|
img.save(cam_dir / f"{i:06d}.jpg", "JPEG", quality=95)
|
|
251
253
|
|
|
252
|
-
|
|
254
|
+
_sync_restored_task_instruction(hdf5_path, output_dir)
|
|
253
255
|
return frames_count
|
|
254
256
|
|
|
255
257
|
def _extract_frames(self, h5_file: h5py.File) -> List[AlignedFrame]:
|
|
@@ -43,24 +43,24 @@ def _persist_manifest_copy(out: Path, task_id: str, manifest: Manifest) -> None:
|
|
|
43
43
|
pass
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def
|
|
46
|
+
def _sync_task_instructions(
|
|
47
47
|
out: Path,
|
|
48
48
|
manifest: Manifest,
|
|
49
49
|
task_state: st.TaskState,
|
|
50
50
|
) -> None:
|
|
51
|
-
"""
|
|
51
|
+
"""Write per-file metadata to the legacy local compatibility path."""
|
|
52
52
|
successful_statuses = {st.DOWNLOADED, st.SKIPPED, st.VERIFIED}
|
|
53
53
|
for file_spec in manifest.files:
|
|
54
54
|
key = file_spec.effective_key()
|
|
55
55
|
file_state = task_state.files.get(key)
|
|
56
|
-
|
|
56
|
+
instruction_path = st.task_instruction_path(out, file_spec.relpath())
|
|
57
57
|
if (
|
|
58
58
|
file_state is not None
|
|
59
59
|
and file_state.status in successful_statuses
|
|
60
60
|
and file_spec.additionalInfo is not None
|
|
61
61
|
):
|
|
62
|
-
|
|
63
|
-
|
|
62
|
+
instruction_path.parent.mkdir(parents=True, exist_ok=True)
|
|
63
|
+
instruction_path.write_text(
|
|
64
64
|
json.dumps(
|
|
65
65
|
file_spec.additionalInfo,
|
|
66
66
|
ensure_ascii=False,
|
|
@@ -70,7 +70,7 @@ def _sync_additional_info(
|
|
|
70
70
|
encoding="utf-8",
|
|
71
71
|
)
|
|
72
72
|
else:
|
|
73
|
-
|
|
73
|
+
instruction_path.unlink(missing_ok=True)
|
|
74
74
|
|
|
75
75
|
|
|
76
76
|
def get_backend(name: str, **kwargs) -> Backend:
|
|
@@ -280,7 +280,7 @@ class DownloadManager:
|
|
|
280
280
|
verified += 1
|
|
281
281
|
task_state.mark(key, st.VERIFIED)
|
|
282
282
|
|
|
283
|
-
|
|
283
|
+
_sync_task_instructions(self.out, manifest, task_state)
|
|
284
284
|
task_state.finish()
|
|
285
285
|
task_state.save()
|
|
286
286
|
|
|
@@ -17,6 +17,7 @@ from ..manifest import Manifest, resolve_dest
|
|
|
17
17
|
|
|
18
18
|
STATE_DIRNAME = ".dexverse"
|
|
19
19
|
TASKS_DIRNAME = "tasks"
|
|
20
|
+
TASK_INSTRUCTIONS_DIRNAME = "task-instructions"
|
|
20
21
|
ADDITIONAL_INFO_DIRNAME = "additional-info"
|
|
21
22
|
|
|
22
23
|
# file statuses
|
|
@@ -36,13 +37,18 @@ def task_dir(out: Path, task_id: str) -> Path:
|
|
|
36
37
|
return dexverse_root(out) / TASKS_DIRNAME / task_id
|
|
37
38
|
|
|
38
39
|
|
|
39
|
-
def
|
|
40
|
-
return dexverse_root(out) /
|
|
40
|
+
def task_instructions_dir(out: Path) -> Path:
|
|
41
|
+
return dexverse_root(out) / TASK_INSTRUCTIONS_DIRNAME
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def task_instruction_path(out: Path, relpath: str) -> Path:
|
|
45
|
+
"""Legacy hidden metadata path written for algorithm compatibility."""
|
|
46
|
+
return resolve_dest(task_instructions_dir(out), f"{relpath}.json")
|
|
41
47
|
|
|
42
48
|
|
|
43
49
|
def additional_info_path(out: Path, relpath: str) -> Path:
|
|
44
|
-
"""
|
|
45
|
-
return resolve_dest(
|
|
50
|
+
"""Read-only bridge for state written by release 32ffa8e."""
|
|
51
|
+
return resolve_dest(dexverse_root(out) / ADDITIONAL_INFO_DIRNAME, f"{relpath}.json")
|
|
46
52
|
|
|
47
53
|
|
|
48
54
|
def state_file_path(out: Path, task_id: str) -> Path:
|
|
@@ -162,31 +162,38 @@ class FileSpec(BaseModel):
|
|
|
162
162
|
None, validation_alias=AliasChoices("presignedUrl", "download_url", "url", "downloadUrl"))
|
|
163
163
|
additionalInfo: Optional[Dict[str, Any]] = Field(
|
|
164
164
|
None,
|
|
165
|
-
validation_alias=AliasChoices(
|
|
165
|
+
validation_alias=AliasChoices(
|
|
166
|
+
"additionalInfo",
|
|
167
|
+
"additional_info",
|
|
168
|
+
"taskInstruction",
|
|
169
|
+
"task_instruction",
|
|
170
|
+
),
|
|
166
171
|
)
|
|
167
172
|
|
|
168
173
|
@model_validator(mode="before")
|
|
169
174
|
@classmethod
|
|
170
175
|
def validate_metadata_fields(cls, data: Any) -> Any:
|
|
171
|
-
"""Reject ambiguous aliases
|
|
176
|
+
"""Reject ambiguous metadata aliases before extras are kept."""
|
|
172
177
|
if isinstance(data, dict):
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
]
|
|
183
|
-
if removed:
|
|
184
|
-
names = ", ".join(repr(name) for name in removed)
|
|
178
|
+
metadata_fields = (
|
|
179
|
+
"additional_info",
|
|
180
|
+
"additionalInfo",
|
|
181
|
+
"task_instruction",
|
|
182
|
+
"taskInstruction",
|
|
183
|
+
)
|
|
184
|
+
present = [name for name in metadata_fields if name in data]
|
|
185
|
+
if len(present) > 1:
|
|
186
|
+
names = ", ".join(repr(name) for name in present)
|
|
185
187
|
raise ValueError(
|
|
186
|
-
f"
|
|
188
|
+
f"metadata fields {names} cannot be used together; provide exactly one"
|
|
187
189
|
)
|
|
188
190
|
return data
|
|
189
191
|
|
|
192
|
+
@property
|
|
193
|
+
def taskInstruction(self) -> Optional[Dict[str, Any]]:
|
|
194
|
+
"""Legacy SDK attribute retained for callers of older CLI releases."""
|
|
195
|
+
return self.additionalInfo
|
|
196
|
+
|
|
190
197
|
def relpath(self) -> str:
|
|
191
198
|
"""The local landing path relative to out (POSIX, normalized)."""
|
|
192
199
|
return resolve_relpath(self.relativePath, self.originalPath, self.objectKey)
|
|
@@ -20,8 +20,8 @@ def _write_minimal_hdf5(path: Path) -> None:
|
|
|
20
20
|
joints.attrs["columns"] = json.dumps(["joint"])
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
def
|
|
24
|
-
path = st.
|
|
23
|
+
def _write_hidden_task_instruction(root: Path, relpath: str, info: dict) -> Path:
|
|
24
|
+
path = st.task_instruction_path(root, relpath)
|
|
25
25
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
26
26
|
path.write_text(
|
|
27
27
|
json.dumps(info, ensure_ascii=False, indent=2) + "\n",
|
|
@@ -30,12 +30,12 @@ def _write_hidden_additional_info(root: Path, relpath: str, info: dict) -> Path:
|
|
|
30
30
|
return path
|
|
31
31
|
|
|
32
32
|
|
|
33
|
-
def
|
|
33
|
+
def test_convert_single_file_writes_and_updates_raw_task_instruction(tmp_path):
|
|
34
34
|
download_root = tmp_path / "download"
|
|
35
35
|
source = download_root / "session.hdf5"
|
|
36
36
|
restored = tmp_path / "restored" / "session"
|
|
37
37
|
_write_minimal_hdf5(source)
|
|
38
|
-
info_path =
|
|
38
|
+
info_path = _write_hidden_task_instruction(
|
|
39
39
|
download_root,
|
|
40
40
|
"session.hdf5",
|
|
41
41
|
{"aaa": "中文", "number": 1.1111111111111112e+54},
|
|
@@ -44,7 +44,7 @@ def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
|
|
|
44
44
|
frames = convert_hdf5(source, restored)
|
|
45
45
|
|
|
46
46
|
assert frames == 1
|
|
47
|
-
output_path = restored / "
|
|
47
|
+
output_path = restored / "task_instruction.json"
|
|
48
48
|
assert json.loads(output_path.read_text(encoding="utf-8")) == {
|
|
49
49
|
"aaa": "中文",
|
|
50
50
|
"number": 1.1111111111111112e+54,
|
|
@@ -58,19 +58,19 @@ def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
|
|
|
58
58
|
assert not output_path.exists()
|
|
59
59
|
|
|
60
60
|
|
|
61
|
-
def
|
|
61
|
+
def test_convert_batch_preserves_paths_for_task_instruction(tmp_path):
|
|
62
62
|
download_root = tmp_path / "download"
|
|
63
63
|
source = download_root / "nested" / "batch.hdf5"
|
|
64
64
|
restored_root = tmp_path / "restored"
|
|
65
65
|
_write_minimal_hdf5(source)
|
|
66
66
|
info = {"task": "batch"}
|
|
67
|
-
|
|
67
|
+
_write_hidden_task_instruction(download_root, "nested/batch.hdf5", info)
|
|
68
68
|
|
|
69
69
|
success, frames = convert_batch(download_root, restored_root)
|
|
70
70
|
|
|
71
71
|
assert success == 1
|
|
72
72
|
assert frames == 1
|
|
73
|
-
output_path = restored_root / "nested" / "batch" / "
|
|
73
|
+
output_path = restored_root / "nested" / "batch" / "task_instruction.json"
|
|
74
74
|
assert json.loads(output_path.read_text(encoding="utf-8")) == info
|
|
75
75
|
|
|
76
76
|
|
|
@@ -82,4 +82,20 @@ def test_standalone_hdf5_without_downloader_metadata_stays_compatible(tmp_path):
|
|
|
82
82
|
frames = convert_hdf5(source, restored)
|
|
83
83
|
|
|
84
84
|
assert frames == 1
|
|
85
|
-
assert not (restored / "
|
|
85
|
+
assert not (restored / "task_instruction.json").exists()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def test_convert_bridges_additional_info_state_to_legacy_output(tmp_path):
|
|
89
|
+
download_root = tmp_path / "download"
|
|
90
|
+
source = download_root / "session.hdf5"
|
|
91
|
+
restored = tmp_path / "restored" / "session"
|
|
92
|
+
_write_minimal_hdf5(source)
|
|
93
|
+
bridge_path = st.additional_info_path(download_root, "session.hdf5")
|
|
94
|
+
bridge_path.parent.mkdir(parents=True)
|
|
95
|
+
bridge_path.write_text('{"from": "32ffa8e"}\n', encoding="utf-8")
|
|
96
|
+
|
|
97
|
+
convert_hdf5(source, restored)
|
|
98
|
+
|
|
99
|
+
assert json.loads((restored / "task_instruction.json").read_text(encoding="utf-8")) == {
|
|
100
|
+
"from": "32ffa8e"
|
|
101
|
+
}
|
|
@@ -290,8 +290,29 @@ def test_duplicate_filename_download_and_additional_info_keep_first(tmp_path, ht
|
|
|
290
290
|
assert report.total == 1
|
|
291
291
|
assert report.downloaded == 1
|
|
292
292
|
assert (out / "same.hdf5").read_bytes() == first_body
|
|
293
|
-
|
|
294
|
-
assert json.loads(
|
|
293
|
+
instruction_path = st.task_instruction_path(out, "same.hdf5")
|
|
294
|
+
assert json.loads(instruction_path.read_text(encoding="utf-8")) == payload[0]["additional_info"]
|
|
295
|
+
assert not st.additional_info_path(out, "same.hdf5").exists()
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def test_legacy_manifest_metadata_still_syncs_to_legacy_path(tmp_path, http_server):
|
|
299
|
+
body = b"already-downloaded"
|
|
300
|
+
out = tmp_path / "out"
|
|
301
|
+
out.mkdir()
|
|
302
|
+
(out / "ep.hdf5").write_bytes(body)
|
|
303
|
+
manifest = Manifest.from_payload([{
|
|
304
|
+
"download_url": http_server.url("unused.hdf5"),
|
|
305
|
+
"file_name": "ep.hdf5",
|
|
306
|
+
"file_size": len(body),
|
|
307
|
+
"task_instruction": {"mode": "legacy"},
|
|
308
|
+
}])
|
|
309
|
+
|
|
310
|
+
report = DownloadManager(out, backend="http").run(manifest)
|
|
311
|
+
|
|
312
|
+
assert report.ok
|
|
313
|
+
instruction_path = st.task_instruction_path(out, "ep.hdf5")
|
|
314
|
+
assert json.loads(instruction_path.read_text(encoding="utf-8")) == {"mode": "legacy"}
|
|
315
|
+
assert not st.additional_info_path(out, "ep.hdf5").exists()
|
|
295
316
|
|
|
296
317
|
|
|
297
318
|
def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
|
|
@@ -310,13 +331,13 @@ def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
|
|
|
310
331
|
|
|
311
332
|
assert report.ok
|
|
312
333
|
assert report.skipped == 1
|
|
313
|
-
|
|
314
|
-
assert json.loads(
|
|
334
|
+
instruction_path = st.task_instruction_path(out, "ep.hdf5")
|
|
335
|
+
assert json.loads(instruction_path.read_text(encoding="utf-8")) == {"mode": "skip"}
|
|
315
336
|
|
|
316
337
|
|
|
317
338
|
def test_failed_or_corrupt_file_does_not_keep_additional_info(tmp_path, http_server):
|
|
318
339
|
out = tmp_path / "out"
|
|
319
|
-
stale_path = st.
|
|
340
|
+
stale_path = st.task_instruction_path(out, "ep.hdf5")
|
|
320
341
|
stale_path.parent.mkdir(parents=True)
|
|
321
342
|
stale_path.write_text('{"stale": true}', encoding="utf-8")
|
|
322
343
|
failed = Manifest.from_payload([{
|
|
@@ -350,7 +371,7 @@ def test_missing_additional_info_clears_stale_hidden_metadata(tmp_path, http_ser
|
|
|
350
371
|
body = b"fresh-body"
|
|
351
372
|
_seed(tmp_path, {"ep.hdf5": body})
|
|
352
373
|
out = tmp_path / "out"
|
|
353
|
-
stale_path = st.
|
|
374
|
+
stale_path = st.task_instruction_path(out, "ep.hdf5")
|
|
354
375
|
stale_path.parent.mkdir(parents=True)
|
|
355
376
|
stale_path.write_text('{"stale": true}', encoding="utf-8")
|
|
356
377
|
manifest = Manifest.from_payload([{
|
|
@@ -48,25 +48,26 @@ def test_manifest_from_files_wrapper_native():
|
|
|
48
48
|
assert m.files[0].presignedUrl.startswith("https")
|
|
49
49
|
|
|
50
50
|
|
|
51
|
-
|
|
51
|
+
@pytest.mark.parametrize(
|
|
52
|
+
"metadata_field",
|
|
53
|
+
["additional_info", "additionalInfo", "task_instruction", "taskInstruction"],
|
|
54
|
+
)
|
|
55
|
+
def test_metadata_accepts_new_and_legacy_names(metadata_field):
|
|
52
56
|
info = {
|
|
53
57
|
"aaa": "111",
|
|
54
58
|
"aaaa": 222,
|
|
55
59
|
"ewrwrwe": 1.1111111111111112e+54,
|
|
56
60
|
}
|
|
57
|
-
|
|
58
|
-
"files": [{**_native_file(),
|
|
59
|
-
})
|
|
60
|
-
canonical = Manifest.from_payload({
|
|
61
|
-
"files": [{**_native_file(), "additionalInfo": info}],
|
|
61
|
+
manifest = Manifest.from_payload({
|
|
62
|
+
"files": [{**_native_file(), metadata_field: info}],
|
|
62
63
|
})
|
|
63
64
|
|
|
64
|
-
assert
|
|
65
|
-
assert
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
65
|
+
assert manifest.files[0].additionalInfo == info
|
|
66
|
+
assert manifest.files[0].taskInstruction == info
|
|
67
|
+
dumped_file = json.loads(manifest.model_dump_json())["files"][0]
|
|
68
|
+
assert dumped_file["additionalInfo"] == info
|
|
69
|
+
assert "additional_info" not in dumped_file
|
|
70
|
+
assert "task_instruction" not in dumped_file
|
|
70
71
|
|
|
71
72
|
|
|
72
73
|
@pytest.mark.parametrize(
|
|
@@ -77,7 +78,7 @@ def test_additional_info_accepts_native_and_canonical_names():
|
|
|
77
78
|
],
|
|
78
79
|
ids=["equal-values", "conflicting-values"],
|
|
79
80
|
)
|
|
80
|
-
def
|
|
81
|
+
def test_metadata_aliases_cannot_be_used_together(
|
|
81
82
|
native_info,
|
|
82
83
|
canonical_info,
|
|
83
84
|
):
|
|
@@ -91,18 +92,23 @@ def test_additional_info_aliases_cannot_be_used_together(
|
|
|
91
92
|
})
|
|
92
93
|
|
|
93
94
|
|
|
94
|
-
|
|
95
|
+
@pytest.mark.parametrize("metadata_field", ["additional_info", "task_instruction"])
|
|
96
|
+
def test_metadata_must_be_an_object(metadata_field):
|
|
95
97
|
with pytest.raises(ValidationError):
|
|
96
98
|
Manifest.from_payload({
|
|
97
|
-
"files": [{**_native_file(),
|
|
99
|
+
"files": [{**_native_file(), metadata_field: ["not", "an", "object"]}],
|
|
98
100
|
})
|
|
99
101
|
|
|
100
102
|
|
|
101
103
|
@pytest.mark.parametrize("legacy_field", ["task_instruction", "taskInstruction"])
|
|
102
|
-
def
|
|
103
|
-
with pytest.raises(ValidationError, match="
|
|
104
|
+
def test_new_and_legacy_metadata_fields_cannot_be_used_together(legacy_field):
|
|
105
|
+
with pytest.raises(ValidationError, match="cannot be used together"):
|
|
104
106
|
Manifest.from_payload({
|
|
105
|
-
"files": [{
|
|
107
|
+
"files": [{
|
|
108
|
+
**_native_file(),
|
|
109
|
+
"additional_info": {"new": True},
|
|
110
|
+
legacy_field: {"legacy": True},
|
|
111
|
+
}],
|
|
106
112
|
})
|
|
107
113
|
|
|
108
114
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|