dexverse-cli 0.0.4__tar.gz → 0.0.4.post1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/PKG-INFO +9 -6
  2. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/README.md +8 -5
  3. dexverse_cli-0.0.4.post1/decision-traces/2026-08-12_1613_pypi_0_0_4_release_trace.md +84 -0
  4. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/convert/hdf5_to_raw.py +12 -10
  5. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/manager.py +7 -7
  6. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/state.py +10 -4
  7. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/manifest.py +22 -15
  8. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/pyproject.toml +1 -1
  9. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_convert_additional_info.py +25 -9
  10. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_http_backend.py +27 -6
  11. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_manifest_aliases.py +24 -18
  12. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/.dockerignore +0 -0
  13. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/.gitignore +0 -0
  14. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/Dockerfile +0 -0
  15. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/__init__.py +0 -0
  16. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/cli.py +0 -0
  17. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/client.py +0 -0
  18. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/config.py +0 -0
  19. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/convert/__init__.py +0 -0
  20. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/convert/image_video_encode.py +0 -0
  21. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/__init__.py +0 -0
  22. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/base.py +0 -0
  23. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/http_backend.py +0 -0
  24. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/python_s3_backend.py +0 -0
  25. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/s5cmd_backend.py +0 -0
  26. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/downloader/verify.py +0 -0
  27. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/mock/__init__.py +0 -0
  28. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/mock/mock_tasks.json +0 -0
  29. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/dexverse/mock/server.py +0 -0
  30. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docker_run.sh +0 -0
  31. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/DexVerse_Downloader_S3_Final_/345/256/236/346/226/275/346/226/271/346/241/210_/345/216/237/350/267/257/345/276/204/350/220/275/347/233/230/347/211/210.md" +0 -0
  32. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/PROMPT_copy_and_download_command.md +0 -0
  33. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/Python_vs_Node_CLI/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
  34. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs/Python_vs_Node_/346/216/245/345/205/245C++/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
  35. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs//345/210/206/345/217/221/347/273/231/347/256/227/346/263/225/345/233/242/351/230/237_/346/225/231/347/250/213.md" +0 -0
  36. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs//346/223/215/344/275/234/346/214/207/345/215/227--/345/234/250Dexechain/344/270/255/350/277/233/350/241/214/346/250/241/344/273/277/345/255/246/344/271/240.md" +0 -0
  37. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/docs//346/234/215/345/212/241/345/231/250/344/275/277/347/224/250/346/211/213/345/206/214.md" +0 -0
  38. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/download (7).json +0 -0
  39. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/local_demo.py +0 -0
  40. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/manifest.json +0 -0
  41. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/examples/manifest_presigned.json +0 -0
  42. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/set-creds.ps1 +0 -0
  43. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/conftest.py +0 -0
  44. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_cli.py +0 -0
  45. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_http_resilience.py +0 -0
  46. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_manager_integration.py +0 -0
  47. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_manifest.py +0 -0
  48. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_mock_server.py +0 -0
  49. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_presigned_fixes.py +0 -0
  50. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_review_fixes.py +0 -0
  51. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_s5cmd_generation.py +0 -0
  52. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_state.py +0 -0
  53. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_verify.py +0 -0
  54. {dexverse_cli-0.0.4 → dexverse_cli-0.0.4.post1}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: dexverse-cli
3
- Version: 0.0.4
3
+ Version: 0.0.4.post1
4
4
  Summary: DexVerse dataset downloader for S3-compatible robot training datasets
5
5
  Author: DexVerse
6
6
  License: Apache-2.0
@@ -104,6 +104,9 @@ D:\DexVerse\20260429_180359.hdf5
104
104
  items resolve to the same local path (for example, repeated `file_name`
105
105
  values), the first item wins and later duplicates are ignored.
106
106
 
107
+ For compatibility, the CLI also accepts the legacy `task_instruction` field;
108
+ both names must not appear in the same file entry.
109
+
107
110
  Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
108
111
  also learns the total size from `Content-Length`/`Content-Range`, but manifest
109
112
  metadata lets the CLI verify an existing local file before making a request.
@@ -134,7 +137,7 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
134
137
 
135
138
  下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
136
139
  + `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
137
- `additional_info.json`:
140
+ `task_instruction.json`(保持算法侧兼容):
138
141
 
139
142
  ```bash
140
143
  # 单文件
@@ -153,11 +156,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
153
156
  hand/right/000000.jpg ...
154
157
  pose_record_20260429_180359.json
155
158
  metadata.jsonl
156
- additional_info.json
159
+ task_instruction.json
157
160
  ```
158
161
 
159
- `additional_info.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
160
- 下载器会把转换所需的映射保存在 `.dexverse/additional-info/`,无需为
162
+ `task_instruction.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
163
+ 下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`,无需为
161
164
  `convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
162
165
 
163
166
  该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
@@ -180,7 +183,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
180
183
 
181
184
  ```text
182
185
  /data/dexverse/
183
- .dexverse/additional-info/
186
+ .dexverse/task-instructions/
184
187
  20260429_180359.hdf5.json
185
188
  .dexverse/tasks/<taskId>/
186
189
  manifest.json
@@ -74,6 +74,9 @@ D:\DexVerse\20260429_180359.hdf5
74
74
  items resolve to the same local path (for example, repeated `file_name`
75
75
  values), the first item wins and later duplicates are ignored.
76
76
 
77
+ For compatibility, the CLI also accepts the legacy `task_instruction` field;
78
+ both names must not appear in the same file entry.
79
+
77
80
  Include `file_size` (and preferably `sha256`) for large files. The HTTP backend
78
81
  also learns the total size from `Content-Length`/`Content-Range`, but manifest
79
82
  metadata lets the CLI verify an existing local file before making a request.
@@ -104,7 +107,7 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
104
107
 
105
108
  下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
106
109
  + `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
107
- `additional_info.json`:
110
+ `task_instruction.json`(保持算法侧兼容):
108
111
 
109
112
  ```bash
110
113
  # 单文件
@@ -123,11 +126,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
123
126
  hand/right/000000.jpg ...
124
127
  pose_record_20260429_180359.json
125
128
  metadata.jsonl
126
- additional_info.json
129
+ task_instruction.json
127
130
  ```
128
131
 
129
- `additional_info.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
130
- 下载器会把转换所需的映射保存在 `.dexverse/additional-info/`,无需为
132
+ `task_instruction.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
133
+ 下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`,无需为
131
134
  `convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
132
135
 
133
136
  该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
@@ -150,7 +153,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
150
153
 
151
154
  ```text
152
155
  /data/dexverse/
153
- .dexverse/additional-info/
156
+ .dexverse/task-instructions/
154
157
  20260429_180359.hdf5.json
155
158
  .dexverse/tasks/<taskId>/
156
159
  manifest.json
@@ -0,0 +1,84 @@
1
+ # 2026-08-12 PyPI 0.0.4 Release Boundary
2
+
3
+ ## Background
4
+
5
+ `dexverse-cli` was prepared for a PyPI 0.0.4 release. The release must remain
6
+ the dataset downloader and must not ship the co-located device cloud-upload
7
+ module or the checked-in HDF5 fixture.
8
+
9
+ ## Problem
10
+
11
+ The original Hatch package selection included all files under `dexverse`, so
12
+ both wheel and sdist included `dexverse/dexe_cloud_upload`. The source
13
+ distribution also included `648584.hdf5` (about 41.6 MiB). In addition, the
14
+ public `dexverse.__version__` lookup used the import package name instead of
15
+ the distribution name, so it returned `0.0.0+unknown` after installation.
16
+
17
+ ## Evidence And Source Analysis
18
+
19
+ - The pre-release build contained 25 `dexe_cloud_upload` entries in both wheel
20
+ and sdist, and the sdist contained `648584.hdf5`.
21
+ - The 0.0.4 build excluded `/dexverse/dexe_cloud_upload/**` and
22
+ `/648584.hdf5`; the resulting wheel had 23 entries and the sdist had 53,
23
+ with zero entries for either exclusion target.
24
+ - `pyproject.toml` names the distribution `dexverse-cli`; changing
25
+ `version("dexverse")` to `version("dexverse-cli")` made the installed
26
+ public version equal the distribution version.
27
+ - The local suite passed 110 tests. Twine metadata checks passed. Clean Python
28
+ 3.10 wheel and Python 3.11 sdist installs succeeded, as did a final PyPI
29
+ Simple Index installation of `dexverse-cli==0.0.4`.
30
+ - A real HDF5 download and deep SHA-256 verification succeeded with the PyPI
31
+ alpha. `repair --deep` does not re-download a same-size SHA-mismatched file,
32
+ but the same behavior and identical `repair()` implementation were confirmed
33
+ in PyPI 0.0.3; it was accepted as an existing issue, not a 0.0.4 regression.
34
+
35
+ ## Clarified Constraints
36
+
37
+ - The 0.0.4 release must not include `dexverse/dexe_cloud_upload`.
38
+ - The 0.0.4 release must not include `648584.hdf5`.
39
+ - P1 documentation, license, and Docker concerns were intentionally deferred
40
+ for the later repository refactor.
41
+ - PyPI files are immutable per version, so alpha validation preceded the
42
+ formal release.
43
+ - The trace is local-only and must not contain package-upload credentials.
44
+
45
+ ## Final Decision
46
+
47
+ PyPI `dexverse-cli` 0.0.4 was published with the exclusion rules and corrected
48
+ distribution-name version lookup. The release contains the downloader CLI/SDK,
49
+ but not the cloud-upload module or HDF5 fixture. The pre-existing deep-repair
50
+ limitation is not treated as a release blocker for 0.0.4.
51
+
52
+ Future releases must inspect both wheel and sdist contents before upload and
53
+ must keep `dexverse.__version__` aligned with the `dexverse-cli` distribution
54
+ metadata. Do not reintroduce the excluded paths into downloader distributions
55
+ unless product scope explicitly changes.
56
+
57
+ ## Current Status
58
+
59
+ - PyPI 0.0.4 wheel and sdist published successfully on 2026-08-12.
60
+ - PyPI installation, version check, exclusion-content check, and CLI help
61
+ smoke test passed.
62
+ - The local release edits are deliberately being reverted after publication at
63
+ user request. Consequently, the checkout will not reproduce the published
64
+ 0.0.4 artifact until the release changes are committed or reapplied.
65
+ - This trace is intentionally local and uncommitted.
66
+
67
+ ## Revisit Triggers
68
+
69
+ - Start of the planned repository refactor or any new package release.
70
+ - A decision to ship the cloud-upload module as a separate distribution or to
71
+ include it in this package.
72
+ - A decision to fix `repair --deep` behavior, which requires a regression test
73
+ for same-size SHA-256 corruption and forced re-download.
74
+ - A request to align the Git checkout, tag, or release notes with the published
75
+ PyPI 0.0.4 artifact.
76
+
77
+ ## Privacy Class
78
+
79
+ trace_privacy_class: local_raw_trace
80
+
81
+ ## TraceGym Data Capture
82
+
83
+ TraceGym metadata omitted: this is a repository-specific release provenance
84
+ record, not an evaluated reusable workflow or benchmark artifact.
@@ -35,7 +35,7 @@ from ..downloader import state as download_state
35
35
  from .image_video_encode import decode_mp4_bytes_to_rgb_images
36
36
 
37
37
 
38
- ADDITIONAL_INFO_FILENAME = "additional_info.json"
38
+ TASK_INSTRUCTION_FILENAME = "task_instruction.json"
39
39
 
40
40
 
41
41
  def _find_download_root(hdf5_path: Path) -> Optional[Path]:
@@ -47,26 +47,28 @@ def _find_download_root(hdf5_path: Path) -> Optional[Path]:
47
47
  return None
48
48
 
49
49
 
50
- def _sync_restored_additional_info(
50
+ def _sync_restored_task_instruction(
51
51
  hdf5_path: Path,
52
52
  output_dir: Path,
53
53
  ) -> None:
54
- """Copy a downloaded file's hidden additional info into its restored session."""
54
+ """Restore metadata to the legacy algorithm-facing output name."""
55
55
  download_root = _find_download_root(hdf5_path)
56
56
  if download_root is None:
57
57
  return
58
58
 
59
59
  resolved_hdf5 = Path(hdf5_path).expanduser().resolve()
60
60
  relpath = resolved_hdf5.relative_to(download_root).as_posix()
61
- source = download_state.additional_info_path(download_root, relpath)
62
- destination = Path(output_dir) / ADDITIONAL_INFO_FILENAME
61
+ source = download_state.task_instruction_path(download_root, relpath)
62
+ if not source.is_file():
63
+ source = download_state.additional_info_path(download_root, relpath)
64
+ destination = Path(output_dir) / TASK_INSTRUCTION_FILENAME
63
65
 
64
66
  if source.is_file():
65
- additional_info = json.loads(source.read_text(encoding="utf-8"))
66
- if not isinstance(additional_info, dict):
67
- raise ValueError(f"additional info must be a JSON object: {source}")
67
+ task_instruction = json.loads(source.read_text(encoding="utf-8"))
68
+ if not isinstance(task_instruction, dict):
69
+ raise ValueError(f"task instruction must be a JSON object: {source}")
68
70
  destination.write_text(
69
- json.dumps(additional_info, ensure_ascii=False, indent=2) + "\n",
71
+ json.dumps(task_instruction, ensure_ascii=False, indent=2) + "\n",
70
72
  encoding="utf-8",
71
73
  )
72
74
  else:
@@ -249,7 +251,7 @@ class HDF5ToRawConverter:
249
251
  img = Image.open(_io.BytesIO(jpg_bytes)).convert("RGB")
250
252
  img.save(cam_dir / f"{i:06d}.jpg", "JPEG", quality=95)
251
253
 
252
- _sync_restored_additional_info(hdf5_path, output_dir)
254
+ _sync_restored_task_instruction(hdf5_path, output_dir)
253
255
  return frames_count
254
256
 
255
257
  def _extract_frames(self, h5_file: h5py.File) -> List[AlignedFrame]:
@@ -43,24 +43,24 @@ def _persist_manifest_copy(out: Path, task_id: str, manifest: Manifest) -> None:
43
43
  pass
44
44
 
45
45
 
46
- def _sync_additional_info(
46
+ def _sync_task_instructions(
47
47
  out: Path,
48
48
  manifest: Manifest,
49
49
  task_state: st.TaskState,
50
50
  ) -> None:
51
- """Synchronize per-file additional info after download/verification."""
51
+ """Write per-file metadata to the legacy local compatibility path."""
52
52
  successful_statuses = {st.DOWNLOADED, st.SKIPPED, st.VERIFIED}
53
53
  for file_spec in manifest.files:
54
54
  key = file_spec.effective_key()
55
55
  file_state = task_state.files.get(key)
56
- info_path = st.additional_info_path(out, file_spec.relpath())
56
+ instruction_path = st.task_instruction_path(out, file_spec.relpath())
57
57
  if (
58
58
  file_state is not None
59
59
  and file_state.status in successful_statuses
60
60
  and file_spec.additionalInfo is not None
61
61
  ):
62
- info_path.parent.mkdir(parents=True, exist_ok=True)
63
- info_path.write_text(
62
+ instruction_path.parent.mkdir(parents=True, exist_ok=True)
63
+ instruction_path.write_text(
64
64
  json.dumps(
65
65
  file_spec.additionalInfo,
66
66
  ensure_ascii=False,
@@ -70,7 +70,7 @@ def _sync_additional_info(
70
70
  encoding="utf-8",
71
71
  )
72
72
  else:
73
- info_path.unlink(missing_ok=True)
73
+ instruction_path.unlink(missing_ok=True)
74
74
 
75
75
 
76
76
  def get_backend(name: str, **kwargs) -> Backend:
@@ -280,7 +280,7 @@ class DownloadManager:
280
280
  verified += 1
281
281
  task_state.mark(key, st.VERIFIED)
282
282
 
283
- _sync_additional_info(self.out, manifest, task_state)
283
+ _sync_task_instructions(self.out, manifest, task_state)
284
284
  task_state.finish()
285
285
  task_state.save()
286
286
 
@@ -17,6 +17,7 @@ from ..manifest import Manifest, resolve_dest
17
17
 
18
18
  STATE_DIRNAME = ".dexverse"
19
19
  TASKS_DIRNAME = "tasks"
20
+ TASK_INSTRUCTIONS_DIRNAME = "task-instructions"
20
21
  ADDITIONAL_INFO_DIRNAME = "additional-info"
21
22
 
22
23
  # file statuses
@@ -36,13 +37,18 @@ def task_dir(out: Path, task_id: str) -> Path:
36
37
  return dexverse_root(out) / TASKS_DIRNAME / task_id
37
38
 
38
39
 
39
- def additional_info_dir(out: Path) -> Path:
40
- return dexverse_root(out) / ADDITIONAL_INFO_DIRNAME
40
+ def task_instructions_dir(out: Path) -> Path:
41
+ return dexverse_root(out) / TASK_INSTRUCTIONS_DIRNAME
42
+
43
+
44
+ def task_instruction_path(out: Path, relpath: str) -> Path:
45
+ """Legacy hidden metadata path written for algorithm compatibility."""
46
+ return resolve_dest(task_instructions_dir(out), f"{relpath}.json")
41
47
 
42
48
 
43
49
  def additional_info_path(out: Path, relpath: str) -> Path:
44
- """Hidden metadata path for one downloaded dataset file."""
45
- return resolve_dest(additional_info_dir(out), f"{relpath}.json")
50
+ """Read-only bridge for state written by release 32ffa8e."""
51
+ return resolve_dest(dexverse_root(out) / ADDITIONAL_INFO_DIRNAME, f"{relpath}.json")
46
52
 
47
53
 
48
54
  def state_file_path(out: Path, task_id: str) -> Path:
@@ -162,31 +162,38 @@ class FileSpec(BaseModel):
162
162
  None, validation_alias=AliasChoices("presignedUrl", "download_url", "url", "downloadUrl"))
163
163
  additionalInfo: Optional[Dict[str, Any]] = Field(
164
164
  None,
165
- validation_alias=AliasChoices("additionalInfo", "additional_info"),
165
+ validation_alias=AliasChoices(
166
+ "additionalInfo",
167
+ "additional_info",
168
+ "taskInstruction",
169
+ "task_instruction",
170
+ ),
166
171
  )
167
172
 
168
173
  @model_validator(mode="before")
169
174
  @classmethod
170
175
  def validate_metadata_fields(cls, data: Any) -> Any:
171
- """Reject ambiguous aliases and removed names before extras are kept."""
176
+ """Reject ambiguous metadata aliases before extras are kept."""
172
177
  if isinstance(data, dict):
173
- if "additional_info" in data and "additionalInfo" in data:
174
- raise ValueError(
175
- "fields 'additional_info' and 'additionalInfo' cannot be "
176
- "used together; provide exactly one"
177
- )
178
- removed = [
179
- name
180
- for name in ("task_instruction", "taskInstruction")
181
- if name in data
182
- ]
183
- if removed:
184
- names = ", ".join(repr(name) for name in removed)
178
+ metadata_fields = (
179
+ "additional_info",
180
+ "additionalInfo",
181
+ "task_instruction",
182
+ "taskInstruction",
183
+ )
184
+ present = [name for name in metadata_fields if name in data]
185
+ if len(present) > 1:
186
+ names = ", ".join(repr(name) for name in present)
185
187
  raise ValueError(
186
- f"removed field(s) {names}; use 'additional_info' instead"
188
+ f"metadata fields {names} cannot be used together; provide exactly one"
187
189
  )
188
190
  return data
189
191
 
192
+ @property
193
+ def taskInstruction(self) -> Optional[Dict[str, Any]]:
194
+ """Legacy SDK attribute retained for callers of older CLI releases."""
195
+ return self.additionalInfo
196
+
190
197
  def relpath(self) -> str:
191
198
  """The local landing path relative to out (POSIX, normalized)."""
192
199
  return resolve_relpath(self.relativePath, self.originalPath, self.objectKey)
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "dexverse-cli"
7
- version = "0.0.4"
7
+ version = "0.0.4.post1"
8
8
  description = "DexVerse dataset downloader for S3-compatible robot training datasets"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -20,8 +20,8 @@ def _write_minimal_hdf5(path: Path) -> None:
20
20
  joints.attrs["columns"] = json.dumps(["joint"])
21
21
 
22
22
 
23
- def _write_hidden_additional_info(root: Path, relpath: str, info: dict) -> Path:
24
- path = st.additional_info_path(root, relpath)
23
+ def _write_hidden_task_instruction(root: Path, relpath: str, info: dict) -> Path:
24
+ path = st.task_instruction_path(root, relpath)
25
25
  path.parent.mkdir(parents=True, exist_ok=True)
26
26
  path.write_text(
27
27
  json.dumps(info, ensure_ascii=False, indent=2) + "\n",
@@ -30,12 +30,12 @@ def _write_hidden_additional_info(root: Path, relpath: str, info: dict) -> Path:
30
30
  return path
31
31
 
32
32
 
33
- def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
33
+ def test_convert_single_file_writes_and_updates_raw_task_instruction(tmp_path):
34
34
  download_root = tmp_path / "download"
35
35
  source = download_root / "session.hdf5"
36
36
  restored = tmp_path / "restored" / "session"
37
37
  _write_minimal_hdf5(source)
38
- info_path = _write_hidden_additional_info(
38
+ info_path = _write_hidden_task_instruction(
39
39
  download_root,
40
40
  "session.hdf5",
41
41
  {"aaa": "中文", "number": 1.1111111111111112e+54},
@@ -44,7 +44,7 @@ def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
44
44
  frames = convert_hdf5(source, restored)
45
45
 
46
46
  assert frames == 1
47
- output_path = restored / "additional_info.json"
47
+ output_path = restored / "task_instruction.json"
48
48
  assert json.loads(output_path.read_text(encoding="utf-8")) == {
49
49
  "aaa": "中文",
50
50
  "number": 1.1111111111111112e+54,
@@ -58,19 +58,19 @@ def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
58
58
  assert not output_path.exists()
59
59
 
60
60
 
61
- def test_convert_batch_preserves_paths_for_additional_info(tmp_path):
61
+ def test_convert_batch_preserves_paths_for_task_instruction(tmp_path):
62
62
  download_root = tmp_path / "download"
63
63
  source = download_root / "nested" / "batch.hdf5"
64
64
  restored_root = tmp_path / "restored"
65
65
  _write_minimal_hdf5(source)
66
66
  info = {"task": "batch"}
67
- _write_hidden_additional_info(download_root, "nested/batch.hdf5", info)
67
+ _write_hidden_task_instruction(download_root, "nested/batch.hdf5", info)
68
68
 
69
69
  success, frames = convert_batch(download_root, restored_root)
70
70
 
71
71
  assert success == 1
72
72
  assert frames == 1
73
- output_path = restored_root / "nested" / "batch" / "additional_info.json"
73
+ output_path = restored_root / "nested" / "batch" / "task_instruction.json"
74
74
  assert json.loads(output_path.read_text(encoding="utf-8")) == info
75
75
 
76
76
 
@@ -82,4 +82,20 @@ def test_standalone_hdf5_without_downloader_metadata_stays_compatible(tmp_path):
82
82
  frames = convert_hdf5(source, restored)
83
83
 
84
84
  assert frames == 1
85
- assert not (restored / "additional_info.json").exists()
85
+ assert not (restored / "task_instruction.json").exists()
86
+
87
+
88
+ def test_convert_bridges_additional_info_state_to_legacy_output(tmp_path):
89
+ download_root = tmp_path / "download"
90
+ source = download_root / "session.hdf5"
91
+ restored = tmp_path / "restored" / "session"
92
+ _write_minimal_hdf5(source)
93
+ bridge_path = st.additional_info_path(download_root, "session.hdf5")
94
+ bridge_path.parent.mkdir(parents=True)
95
+ bridge_path.write_text('{"from": "32ffa8e"}\n', encoding="utf-8")
96
+
97
+ convert_hdf5(source, restored)
98
+
99
+ assert json.loads((restored / "task_instruction.json").read_text(encoding="utf-8")) == {
100
+ "from": "32ffa8e"
101
+ }
@@ -290,8 +290,29 @@ def test_duplicate_filename_download_and_additional_info_keep_first(tmp_path, ht
290
290
  assert report.total == 1
291
291
  assert report.downloaded == 1
292
292
  assert (out / "same.hdf5").read_bytes() == first_body
293
- info_path = st.additional_info_path(out, "same.hdf5")
294
- assert json.loads(info_path.read_text(encoding="utf-8")) == payload[0]["additional_info"]
293
+ instruction_path = st.task_instruction_path(out, "same.hdf5")
294
+ assert json.loads(instruction_path.read_text(encoding="utf-8")) == payload[0]["additional_info"]
295
+ assert not st.additional_info_path(out, "same.hdf5").exists()
296
+
297
+
298
+ def test_legacy_manifest_metadata_still_syncs_to_legacy_path(tmp_path, http_server):
299
+ body = b"already-downloaded"
300
+ out = tmp_path / "out"
301
+ out.mkdir()
302
+ (out / "ep.hdf5").write_bytes(body)
303
+ manifest = Manifest.from_payload([{
304
+ "download_url": http_server.url("unused.hdf5"),
305
+ "file_name": "ep.hdf5",
306
+ "file_size": len(body),
307
+ "task_instruction": {"mode": "legacy"},
308
+ }])
309
+
310
+ report = DownloadManager(out, backend="http").run(manifest)
311
+
312
+ assert report.ok
313
+ instruction_path = st.task_instruction_path(out, "ep.hdf5")
314
+ assert json.loads(instruction_path.read_text(encoding="utf-8")) == {"mode": "legacy"}
315
+ assert not st.additional_info_path(out, "ep.hdf5").exists()
295
316
 
296
317
 
297
318
  def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
@@ -310,13 +331,13 @@ def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
310
331
 
311
332
  assert report.ok
312
333
  assert report.skipped == 1
313
- info_path = st.additional_info_path(out, "ep.hdf5")
314
- assert json.loads(info_path.read_text(encoding="utf-8")) == {"mode": "skip"}
334
+ instruction_path = st.task_instruction_path(out, "ep.hdf5")
335
+ assert json.loads(instruction_path.read_text(encoding="utf-8")) == {"mode": "skip"}
315
336
 
316
337
 
317
338
  def test_failed_or_corrupt_file_does_not_keep_additional_info(tmp_path, http_server):
318
339
  out = tmp_path / "out"
319
- stale_path = st.additional_info_path(out, "ep.hdf5")
340
+ stale_path = st.task_instruction_path(out, "ep.hdf5")
320
341
  stale_path.parent.mkdir(parents=True)
321
342
  stale_path.write_text('{"stale": true}', encoding="utf-8")
322
343
  failed = Manifest.from_payload([{
@@ -350,7 +371,7 @@ def test_missing_additional_info_clears_stale_hidden_metadata(tmp_path, http_ser
350
371
  body = b"fresh-body"
351
372
  _seed(tmp_path, {"ep.hdf5": body})
352
373
  out = tmp_path / "out"
353
- stale_path = st.additional_info_path(out, "ep.hdf5")
374
+ stale_path = st.task_instruction_path(out, "ep.hdf5")
354
375
  stale_path.parent.mkdir(parents=True)
355
376
  stale_path.write_text('{"stale": true}', encoding="utf-8")
356
377
  manifest = Manifest.from_payload([{
@@ -48,25 +48,26 @@ def test_manifest_from_files_wrapper_native():
48
48
  assert m.files[0].presignedUrl.startswith("https")
49
49
 
50
50
 
51
- def test_additional_info_accepts_native_and_canonical_names():
51
+ @pytest.mark.parametrize(
52
+ "metadata_field",
53
+ ["additional_info", "additionalInfo", "task_instruction", "taskInstruction"],
54
+ )
55
+ def test_metadata_accepts_new_and_legacy_names(metadata_field):
52
56
  info = {
53
57
  "aaa": "111",
54
58
  "aaaa": 222,
55
59
  "ewrwrwe": 1.1111111111111112e+54,
56
60
  }
57
- native = Manifest.from_payload({
58
- "files": [{**_native_file(), "additional_info": info}],
59
- })
60
- canonical = Manifest.from_payload({
61
- "files": [{**_native_file(), "additionalInfo": info}],
61
+ manifest = Manifest.from_payload({
62
+ "files": [{**_native_file(), metadata_field: info}],
62
63
  })
63
64
 
64
- assert native.files[0].additionalInfo == info
65
- assert canonical.files[0].additionalInfo == info
66
- for manifest in (native, canonical):
67
- dumped_file = json.loads(manifest.model_dump_json())["files"][0]
68
- assert dumped_file["additionalInfo"] == info
69
- assert "additional_info" not in dumped_file
65
+ assert manifest.files[0].additionalInfo == info
66
+ assert manifest.files[0].taskInstruction == info
67
+ dumped_file = json.loads(manifest.model_dump_json())["files"][0]
68
+ assert dumped_file["additionalInfo"] == info
69
+ assert "additional_info" not in dumped_file
70
+ assert "task_instruction" not in dumped_file
70
71
 
71
72
 
72
73
  @pytest.mark.parametrize(
@@ -77,7 +78,7 @@ def test_additional_info_accepts_native_and_canonical_names():
77
78
  ],
78
79
  ids=["equal-values", "conflicting-values"],
79
80
  )
80
- def test_additional_info_aliases_cannot_be_used_together(
81
+ def test_metadata_aliases_cannot_be_used_together(
81
82
  native_info,
82
83
  canonical_info,
83
84
  ):
@@ -91,18 +92,23 @@ def test_additional_info_aliases_cannot_be_used_together(
91
92
  })
92
93
 
93
94
 
94
- def test_additional_info_must_be_an_object():
95
+ @pytest.mark.parametrize("metadata_field", ["additional_info", "task_instruction"])
96
+ def test_metadata_must_be_an_object(metadata_field):
95
97
  with pytest.raises(ValidationError):
96
98
  Manifest.from_payload({
97
- "files": [{**_native_file(), "additional_info": ["not", "an", "object"]}],
99
+ "files": [{**_native_file(), metadata_field: ["not", "an", "object"]}],
98
100
  })
99
101
 
100
102
 
101
103
  @pytest.mark.parametrize("legacy_field", ["task_instruction", "taskInstruction"])
102
- def test_removed_task_instruction_fields_are_rejected(legacy_field):
103
- with pytest.raises(ValidationError, match="use 'additional_info' instead"):
104
+ def test_new_and_legacy_metadata_fields_cannot_be_used_together(legacy_field):
105
+ with pytest.raises(ValidationError, match="cannot be used together"):
104
106
  Manifest.from_payload({
105
- "files": [{**_native_file(), legacy_field: {"removed": True}}],
107
+ "files": [{
108
+ **_native_file(),
109
+ "additional_info": {"new": True},
110
+ legacy_field: {"legacy": True},
111
+ }],
106
112
  })
107
113
 
108
114