dexverse-cli 0.0.2__tar.gz → 0.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/PKG-INFO +10 -10
  2. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/README.md +8 -8
  3. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/__init__.py +1 -1
  4. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/convert/hdf5_to_raw.py +10 -10
  5. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/manager.py +9 -9
  6. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/state.py +5 -5
  7. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/manifest.py +25 -3
  8. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/download (7).json +8 -8
  9. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/pyproject.toml +7 -1
  10. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/set-creds.ps1 +3 -3
  11. dexverse_cli-0.0.2/tests/test_convert_task_instruction.py → dexverse_cli-0.0.4/tests/test_convert_additional_info.py +14 -16
  12. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_http_backend.py +15 -15
  13. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_manifest_aliases.py +53 -13
  14. dexverse_cli-0.0.2/648584.hdf5 +0 -0
  15. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/.dockerignore +0 -0
  16. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/.gitignore +0 -0
  17. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/Dockerfile +0 -0
  18. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/cli.py +0 -0
  19. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/client.py +0 -0
  20. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/config.py +0 -0
  21. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/convert/__init__.py +0 -0
  22. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/convert/image_video_encode.py +0 -0
  23. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/__init__.py +0 -0
  24. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/base.py +0 -0
  25. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/http_backend.py +0 -0
  26. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/python_s3_backend.py +0 -0
  27. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/s5cmd_backend.py +0 -0
  28. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/downloader/verify.py +0 -0
  29. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/mock/__init__.py +0 -0
  30. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/mock/mock_tasks.json +0 -0
  31. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/dexverse/mock/server.py +0 -0
  32. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docker_run.sh +0 -0
  33. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/DexVerse_Downloader_S3_Final_/345/256/236/346/226/275/346/226/271/346/241/210_/345/216/237/350/267/257/345/276/204/350/220/275/347/233/230/347/211/210.md" +0 -0
  34. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/PROMPT_copy_and_download_command.md +0 -0
  35. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/Python_vs_Node_CLI/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
  36. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs/Python_vs_Node_/346/216/245/345/205/245C++/345/257/271/346/257/224/345/210/206/346/236/220.md" +0 -0
  37. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs//345/210/206/345/217/221/347/273/231/347/256/227/346/263/225/345/233/242/351/230/237_/346/225/231/347/250/213.md" +0 -0
  38. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs//346/223/215/344/275/234/346/214/207/345/215/227--/345/234/250Dexechain/344/270/255/350/277/233/350/241/214/346/250/241/344/273/277/345/255/246/344/271/240.md" +0 -0
  39. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/docs//346/234/215/345/212/241/345/231/250/344/275/277/347/224/250/346/211/213/345/206/214.md" +0 -0
  40. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/local_demo.py +0 -0
  41. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/manifest.json +0 -0
  42. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/examples/manifest_presigned.json +0 -0
  43. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/conftest.py +0 -0
  44. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_cli.py +0 -0
  45. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_http_resilience.py +0 -0
  46. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_manager_integration.py +0 -0
  47. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_manifest.py +0 -0
  48. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_mock_server.py +0 -0
  49. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_presigned_fixes.py +0 -0
  50. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_review_fixes.py +0 -0
  51. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_s5cmd_generation.py +0 -0
  52. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_state.py +0 -0
  53. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_verify.py +0 -0
  54. {dexverse_cli-0.0.2 → dexverse_cli-0.0.4}/tests/test_version.py +0 -0
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.4
1
+ Metadata-Version: 2.5
2
2
  Name: dexverse-cli
3
- Version: 0.0.2
3
+ Version: 0.0.4
4
4
  Summary: DexVerse dataset downloader for S3-compatible robot training datasets
5
5
  Author: DexVerse
6
6
  License: Apache-2.0
@@ -86,7 +86,7 @@ Raw backend output is accepted directly:
86
86
  "storage_path": "datasets/.../20260429_180359.hdf5",
87
87
  "file_name": "20260429_180359.hdf5",
88
88
  "file_size": 14994636,
89
- "task_instruction": {
89
+ "additional_info": {
90
90
  "aaa": "111",
91
91
  "aaaa": 222,
92
92
  "ewrwrwe": 1.1111111111111112e+54
@@ -100,7 +100,7 @@ Result:
100
100
  D:\DexVerse\20260429_180359.hdf5
101
101
  ```
102
102
 
103
- `task_instruction` is optional and must be a JSON object. If multiple manifest
103
+ `additional_info` is optional and must be a JSON object. If multiple manifest
104
104
  items resolve to the same local path (for example, repeated `file_name`
105
105
  values), the first item wins and later duplicates are ignored.
106
106
 
@@ -133,8 +133,8 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
133
133
  ### 还原 HDF5 → 可训练数据
134
134
 
135
135
  下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
136
- + `metadata.jsonl`)。如果对应 manifest 带有 `task_instruction`,还会自动生成
137
- `task_instruction.json`:
136
+ + `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
137
+ `additional_info.json`:
138
138
 
139
139
  ```bash
140
140
  # 单文件
@@ -153,11 +153,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
153
153
  hand/right/000000.jpg ...
154
154
  pose_record_20260429_180359.json
155
155
  metadata.jsonl
156
- task_instruction.json
156
+ additional_info.json
157
157
  ```
158
158
 
159
- `task_instruction.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
160
- 下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`,无需为
159
+ `additional_info.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
160
+ 下载器会把转换所需的映射保存在 `.dexverse/additional-info/`,无需为
161
161
  `convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
162
162
 
163
163
  该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
@@ -180,7 +180,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
180
180
 
181
181
  ```text
182
182
  /data/dexverse/
183
- .dexverse/task-instructions/
183
+ .dexverse/additional-info/
184
184
  20260429_180359.hdf5.json
185
185
  .dexverse/tasks/<taskId>/
186
186
  manifest.json
@@ -56,7 +56,7 @@ Raw backend output is accepted directly:
56
56
  "storage_path": "datasets/.../20260429_180359.hdf5",
57
57
  "file_name": "20260429_180359.hdf5",
58
58
  "file_size": 14994636,
59
- "task_instruction": {
59
+ "additional_info": {
60
60
  "aaa": "111",
61
61
  "aaaa": 222,
62
62
  "ewrwrwe": 1.1111111111111112e+54
@@ -70,7 +70,7 @@ Result:
70
70
  D:\DexVerse\20260429_180359.hdf5
71
71
  ```
72
72
 
73
- `task_instruction` is optional and must be a JSON object. If multiple manifest
73
+ `additional_info` is optional and must be a JSON object. If multiple manifest
74
74
  items resolve to the same local path (for example, repeated `file_name`
75
75
  values), the first item wins and later duplicates are ignored.
76
76
 
@@ -103,8 +103,8 @@ dexverse-dl repair --manifest PATH --out /data/dexverse [--backend http|python|
103
103
  ### 还原 HDF5 → 可训练数据
104
104
 
105
105
  下载完成后,把落盘的 `.hdf5` 还原为原始录制结构(逐帧 JPEG + `pose_record_*.json`
106
- + `metadata.jsonl`)。如果对应 manifest 带有 `task_instruction`,还会自动生成
107
- `task_instruction.json`:
106
+ + `metadata.jsonl`)。如果对应 manifest 带有 `additional_info`,还会自动生成
107
+ `additional_info.json`:
108
108
 
109
109
  ```bash
110
110
  # 单文件
@@ -123,11 +123,11 @@ dexverse-dl convert --input-dir /data/dexverse -o /data/dexverse/restored
123
123
  hand/right/000000.jpg ...
124
124
  pose_record_20260429_180359.json
125
125
  metadata.jsonl
126
- task_instruction.json
126
+ additional_info.json
127
127
  ```
128
128
 
129
- `task_instruction.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
130
- 下载器会把转换所需的映射保存在 `.dexverse/task-instructions/`,无需为
129
+ `additional_info.json` 的内容就是 manifest 中的对象本身,不增加外层包装。
130
+ 下载器会把转换所需的映射保存在 `.dexverse/additional-info/`,无需为
131
131
  `convert` 增加额外参数。独立来源且没有该隐藏元数据的 HDF5 仍按原行为还原。
132
132
 
133
133
  该命令依赖 `h5py` / `numpy` / `av` / `pillow`,已列入核心依赖,安装时随包装上。
@@ -150,7 +150,7 @@ State files live under `<out>/.dexverse/` and never mix into the dataset tree:
150
150
 
151
151
  ```text
152
152
  /data/dexverse/
153
- .dexverse/task-instructions/
153
+ .dexverse/additional-info/
154
154
  20260429_180359.hdf5.json
155
155
  .dexverse/tasks/<taskId>/
156
156
  manifest.json
@@ -32,7 +32,7 @@ def _resolve_version() -> str:
32
32
  if isinstance(project_version, str):
33
33
  return project_version
34
34
  try:
35
- return version("dexverse")
35
+ return version("dexverse-cli")
36
36
  except PackageNotFoundError:
37
37
  return "0.0.0+unknown"
38
38
 
@@ -35,7 +35,7 @@ from ..downloader import state as download_state
35
35
  from .image_video_encode import decode_mp4_bytes_to_rgb_images
36
36
 
37
37
 
38
- TASK_INSTRUCTION_FILENAME = "task_instruction.json"
38
+ ADDITIONAL_INFO_FILENAME = "additional_info.json"
39
39
 
40
40
 
41
41
  def _find_download_root(hdf5_path: Path) -> Optional[Path]:
@@ -47,26 +47,26 @@ def _find_download_root(hdf5_path: Path) -> Optional[Path]:
47
47
  return None
48
48
 
49
49
 
50
- def _sync_restored_task_instruction(
50
+ def _sync_restored_additional_info(
51
51
  hdf5_path: Path,
52
52
  output_dir: Path,
53
53
  ) -> None:
54
- """Copy a downloaded file's hidden instruction into its restored session."""
54
+ """Copy a downloaded file's hidden additional info into its restored session."""
55
55
  download_root = _find_download_root(hdf5_path)
56
56
  if download_root is None:
57
57
  return
58
58
 
59
59
  resolved_hdf5 = Path(hdf5_path).expanduser().resolve()
60
60
  relpath = resolved_hdf5.relative_to(download_root).as_posix()
61
- source = download_state.task_instruction_path(download_root, relpath)
62
- destination = Path(output_dir) / TASK_INSTRUCTION_FILENAME
61
+ source = download_state.additional_info_path(download_root, relpath)
62
+ destination = Path(output_dir) / ADDITIONAL_INFO_FILENAME
63
63
 
64
64
  if source.is_file():
65
- instruction = json.loads(source.read_text(encoding="utf-8"))
66
- if not isinstance(instruction, dict):
67
- raise ValueError(f"task instruction must be a JSON object: {source}")
65
+ additional_info = json.loads(source.read_text(encoding="utf-8"))
66
+ if not isinstance(additional_info, dict):
67
+ raise ValueError(f"additional info must be a JSON object: {source}")
68
68
  destination.write_text(
69
- json.dumps(instruction, ensure_ascii=False, indent=2) + "\n",
69
+ json.dumps(additional_info, ensure_ascii=False, indent=2) + "\n",
70
70
  encoding="utf-8",
71
71
  )
72
72
  else:
@@ -249,7 +249,7 @@ class HDF5ToRawConverter:
249
249
  img = Image.open(_io.BytesIO(jpg_bytes)).convert("RGB")
250
250
  img.save(cam_dir / f"{i:06d}.jpg", "JPEG", quality=95)
251
251
 
252
- _sync_restored_task_instruction(hdf5_path, output_dir)
252
+ _sync_restored_additional_info(hdf5_path, output_dir)
253
253
  return frames_count
254
254
 
255
255
  def _extract_frames(self, h5_file: h5py.File) -> List[AlignedFrame]:
@@ -43,26 +43,26 @@ def _persist_manifest_copy(out: Path, task_id: str, manifest: Manifest) -> None:
43
43
  pass
44
44
 
45
45
 
46
- def _sync_task_instructions(
46
+ def _sync_additional_info(
47
47
  out: Path,
48
48
  manifest: Manifest,
49
49
  task_state: st.TaskState,
50
50
  ) -> None:
51
- """Synchronize per-file task instructions after download/verification."""
51
+ """Synchronize per-file additional info after download/verification."""
52
52
  successful_statuses = {st.DOWNLOADED, st.SKIPPED, st.VERIFIED}
53
53
  for file_spec in manifest.files:
54
54
  key = file_spec.effective_key()
55
55
  file_state = task_state.files.get(key)
56
- instruction_path = st.task_instruction_path(out, file_spec.relpath())
56
+ info_path = st.additional_info_path(out, file_spec.relpath())
57
57
  if (
58
58
  file_state is not None
59
59
  and file_state.status in successful_statuses
60
- and file_spec.taskInstruction is not None
60
+ and file_spec.additionalInfo is not None
61
61
  ):
62
- instruction_path.parent.mkdir(parents=True, exist_ok=True)
63
- instruction_path.write_text(
62
+ info_path.parent.mkdir(parents=True, exist_ok=True)
63
+ info_path.write_text(
64
64
  json.dumps(
65
- file_spec.taskInstruction,
65
+ file_spec.additionalInfo,
66
66
  ensure_ascii=False,
67
67
  indent=2,
68
68
  )
@@ -70,7 +70,7 @@ def _sync_task_instructions(
70
70
  encoding="utf-8",
71
71
  )
72
72
  else:
73
- instruction_path.unlink(missing_ok=True)
73
+ info_path.unlink(missing_ok=True)
74
74
 
75
75
 
76
76
  def get_backend(name: str, **kwargs) -> Backend:
@@ -280,7 +280,7 @@ class DownloadManager:
280
280
  verified += 1
281
281
  task_state.mark(key, st.VERIFIED)
282
282
 
283
- _sync_task_instructions(self.out, manifest, task_state)
283
+ _sync_additional_info(self.out, manifest, task_state)
284
284
  task_state.finish()
285
285
  task_state.save()
286
286
 
@@ -17,7 +17,7 @@ from ..manifest import Manifest, resolve_dest
17
17
 
18
18
  STATE_DIRNAME = ".dexverse"
19
19
  TASKS_DIRNAME = "tasks"
20
- TASK_INSTRUCTIONS_DIRNAME = "task-instructions"
20
+ ADDITIONAL_INFO_DIRNAME = "additional-info"
21
21
 
22
22
  # file statuses
23
23
  PENDING = "pending"
@@ -36,13 +36,13 @@ def task_dir(out: Path, task_id: str) -> Path:
36
36
  return dexverse_root(out) / TASKS_DIRNAME / task_id
37
37
 
38
38
 
39
- def task_instructions_dir(out: Path) -> Path:
40
- return dexverse_root(out) / TASK_INSTRUCTIONS_DIRNAME
39
+ def additional_info_dir(out: Path) -> Path:
40
+ return dexverse_root(out) / ADDITIONAL_INFO_DIRNAME
41
41
 
42
42
 
43
- def task_instruction_path(out: Path, relpath: str) -> Path:
43
+ def additional_info_path(out: Path, relpath: str) -> Path:
44
44
  """Hidden metadata path for one downloaded dataset file."""
45
- return resolve_dest(task_instructions_dir(out), f"{relpath}.json")
45
+ return resolve_dest(additional_info_dir(out), f"{relpath}.json")
46
46
 
47
47
 
48
48
  def state_file_path(out: Path, task_id: str) -> Path:
@@ -160,11 +160,33 @@ class FileSpec(BaseModel):
160
160
  # Consumed only by the ``http`` backend; ignored by s5cmd/python.
161
161
  presignedUrl: Optional[str] = Field(
162
162
  None, validation_alias=AliasChoices("presignedUrl", "download_url", "url", "downloadUrl"))
163
- taskInstruction: Optional[Dict[str, Any]] = Field(
163
+ additionalInfo: Optional[Dict[str, Any]] = Field(
164
164
  None,
165
- validation_alias=AliasChoices("taskInstruction", "task_instruction"),
165
+ validation_alias=AliasChoices("additionalInfo", "additional_info"),
166
166
  )
167
167
 
168
+ @model_validator(mode="before")
169
+ @classmethod
170
+ def validate_metadata_fields(cls, data: Any) -> Any:
171
+ """Reject ambiguous aliases and removed names before extras are kept."""
172
+ if isinstance(data, dict):
173
+ if "additional_info" in data and "additionalInfo" in data:
174
+ raise ValueError(
175
+ "fields 'additional_info' and 'additionalInfo' cannot be "
176
+ "used together; provide exactly one"
177
+ )
178
+ removed = [
179
+ name
180
+ for name in ("task_instruction", "taskInstruction")
181
+ if name in data
182
+ ]
183
+ if removed:
184
+ names = ", ".join(repr(name) for name in removed)
185
+ raise ValueError(
186
+ f"removed field(s) {names}; use 'additional_info' instead"
187
+ )
188
+ return data
189
+
168
190
  def relpath(self) -> str:
169
191
  """The local landing path relative to out (POSIX, normalized)."""
170
192
  return resolve_relpath(self.relativePath, self.originalPath, self.objectKey)
@@ -217,7 +239,7 @@ class Manifest(BaseModel):
217
239
 
218
240
  Backend-native manifests commonly provide only ``file_name`` as the
219
241
  landing path. Keeping the first occurrence makes duplicate names
220
- deterministic and ensures the downloaded file and its task instruction
242
+ deterministic and ensures the downloaded file and its additional info
221
243
  always come from the same manifest entry.
222
244
  """
223
245
  seen = set()
@@ -3,14 +3,14 @@
3
3
  {
4
4
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607240417070338897362032267265/20260710_181607.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=333996e3875a8a7cdc65f26edd12de30298a99f68a4e24ec10a7a187d65099c1",
5
5
  "file_name": "20260710_181607.hdf5",
6
- "task_instruction": {
6
+ "additional_info": {
7
7
  "aaa33333": "111"
8
8
  }
9
9
  },
10
10
  {
11
11
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230920230338611294116515841/20260630_152359.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=0207422c867030015aa31c66aac264b70d9f67d508b2c08e9278b5dc85074a23",
12
12
  "file_name": "20260630_152359.hdf5",
13
- "task_instruction": {
13
+ "additional_info": {
14
14
  "aaa": "111",
15
15
  "aaaa": 222,
16
16
  "ewrwrwe": 1.1111111111111112e+54
@@ -19,37 +19,37 @@
19
19
  {
20
20
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230304150338516637072232449/20260617_122549.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=8b0228bd8e53e023081295c2e4811625d03b0f4998f3631c5d868c85c6c19cb3",
21
21
  "file_name": "20260617_122549.hdf5",
22
- "task_instruction": {
22
+ "additional_info": {
23
23
  "aaa": "5555"
24
24
  }
25
25
  },
26
26
  {
27
27
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230255400338514476854677505/20260630_152119.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=5ec492d0d867d9b4e5c714e2b530da8949851c58585504b5e6d6843134073078",
28
28
  "file_name": "20260630_152119.hdf5",
29
- "task_instruction": {
29
+ "additional_info": {
30
30
  "aaa": "111"
31
31
  }
32
32
  },
33
33
  {
34
34
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230251340338513446729420801/20260710_181607.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=21703ed3690b5ea9d008e6ad541e6f1682f75a811cc4a7c0d47ff7e952ce4e65",
35
35
  "file_name": "20260710_181607.hdf5",
36
- "task_instruction": {
36
+ "additional_info": {
37
37
  "aaa": "111"
38
38
  }
39
39
  },
40
40
  {
41
41
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607230250150338513114959974401/20260630_152428.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=608bb92fb07b871567f33d16729c33fb6a97c9d1960644da9b472e83009550b9",
42
42
  "file_name": "20260630_152428.hdf5",
43
- "task_instruction": {
43
+ "additional_info": {
44
44
  "aaa": "111"
45
45
  }
46
46
  },
47
47
  {
48
48
  "download_url": "https://192.168.3.55:8443/dexverse/datasets/2606261645500010275566473183232/2607221248330338301292864606209/20260408_212630.hdf5?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=admin%2F20260724%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20260724T070457Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=33e59a77325ad81b3c1f5294126ede617366cdbf3416d7ab79678fda75d65747",
49
49
  "file_name": "20260408_212630.hdf5",
50
- "task_instruction": {
50
+ "additional_info": {
51
51
  "aaa": "111"
52
52
  }
53
53
  }
54
54
  ]
55
- }
55
+ }
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "dexverse-cli"
7
- version = "0.0.2"
7
+ version = "0.0.4"
8
8
  description = "DexVerse dataset downloader for S3-compatible robot training datasets"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -48,6 +48,12 @@ dexverse-mock-server = "dexverse.mock.server:main"
48
48
  [tool.hatch.build.targets.wheel]
49
49
  packages = ["dexverse"]
50
50
 
51
+ [tool.hatch.build]
52
+ exclude = [
53
+ "/dexverse/dexe_cloud_upload/**",
54
+ "/648584.hdf5",
55
+ ]
56
+
51
57
  [tool.pytest.ini_options]
52
58
  testpaths = ["tests"]
53
59
  addopts = "-ra -q"
@@ -1,3 +1,3 @@
1
- $env:AWS_ACCESS_KEY_ID="AKIA...真实access_key"
2
- $env:AWS_SECRET_ACCESS_KEY="真实secret_key"
3
- $env:AWS_REGION="us-east-1"
1
+ $env:AWS_ACCESS_KEY_ID="AKIA...真实access_key"
2
+ $env:AWS_SECRET_ACCESS_KEY="真实secret_key"
3
+ $env:AWS_REGION="us-east-1"
@@ -20,22 +20,22 @@ def _write_minimal_hdf5(path: Path) -> None:
20
20
  joints.attrs["columns"] = json.dumps(["joint"])
21
21
 
22
22
 
23
- def _write_hidden_instruction(root: Path, relpath: str, instruction: dict) -> Path:
24
- path = st.task_instruction_path(root, relpath)
23
+ def _write_hidden_additional_info(root: Path, relpath: str, info: dict) -> Path:
24
+ path = st.additional_info_path(root, relpath)
25
25
  path.parent.mkdir(parents=True, exist_ok=True)
26
26
  path.write_text(
27
- json.dumps(instruction, ensure_ascii=False, indent=2) + "\n",
27
+ json.dumps(info, ensure_ascii=False, indent=2) + "\n",
28
28
  encoding="utf-8",
29
29
  )
30
30
  return path
31
31
 
32
32
 
33
- def test_convert_single_file_writes_and_updates_raw_task_instruction(tmp_path):
33
+ def test_convert_single_file_writes_and_updates_raw_additional_info(tmp_path):
34
34
  download_root = tmp_path / "download"
35
35
  source = download_root / "session.hdf5"
36
36
  restored = tmp_path / "restored" / "session"
37
37
  _write_minimal_hdf5(source)
38
- instruction_path = _write_hidden_instruction(
38
+ info_path = _write_hidden_additional_info(
39
39
  download_root,
40
40
  "session.hdf5",
41
41
  {"aaa": "中文", "number": 1.1111111111111112e+54},
@@ -44,36 +44,34 @@ def test_convert_single_file_writes_and_updates_raw_task_instruction(tmp_path):
44
44
  frames = convert_hdf5(source, restored)
45
45
 
46
46
  assert frames == 1
47
- output_path = restored / "task_instruction.json"
47
+ output_path = restored / "additional_info.json"
48
48
  assert json.loads(output_path.read_text(encoding="utf-8")) == {
49
49
  "aaa": "中文",
50
50
  "number": 1.1111111111111112e+54,
51
51
  }
52
- assert '"task_instruction"' not in output_path.read_text(encoding="utf-8")
53
-
54
- instruction_path.write_text('{"updated": 2}\n', encoding="utf-8")
52
+ info_path.write_text('{"updated": 2}\n', encoding="utf-8")
55
53
  convert_hdf5(source, restored)
56
54
  assert json.loads(output_path.read_text(encoding="utf-8")) == {"updated": 2}
57
55
 
58
- instruction_path.unlink()
56
+ info_path.unlink()
59
57
  convert_hdf5(source, restored)
60
58
  assert not output_path.exists()
61
59
 
62
60
 
63
- def test_convert_batch_preserves_paths_for_task_instruction(tmp_path):
61
+ def test_convert_batch_preserves_paths_for_additional_info(tmp_path):
64
62
  download_root = tmp_path / "download"
65
63
  source = download_root / "nested" / "batch.hdf5"
66
64
  restored_root = tmp_path / "restored"
67
65
  _write_minimal_hdf5(source)
68
- instruction = {"task": "batch"}
69
- _write_hidden_instruction(download_root, "nested/batch.hdf5", instruction)
66
+ info = {"task": "batch"}
67
+ _write_hidden_additional_info(download_root, "nested/batch.hdf5", info)
70
68
 
71
69
  success, frames = convert_batch(download_root, restored_root)
72
70
 
73
71
  assert success == 1
74
72
  assert frames == 1
75
- output_path = restored_root / "nested" / "batch" / "task_instruction.json"
76
- assert json.loads(output_path.read_text(encoding="utf-8")) == instruction
73
+ output_path = restored_root / "nested" / "batch" / "additional_info.json"
74
+ assert json.loads(output_path.read_text(encoding="utf-8")) == info
77
75
 
78
76
 
79
77
  def test_standalone_hdf5_without_downloader_metadata_stays_compatible(tmp_path):
@@ -84,4 +82,4 @@ def test_standalone_hdf5_without_downloader_metadata_stays_compatible(tmp_path):
84
82
  frames = convert_hdf5(source, restored)
85
83
 
86
84
  assert frames == 1
87
- assert not (restored / "task_instruction.json").exists()
85
+ assert not (restored / "additional_info.json").exists()
@@ -265,7 +265,7 @@ def test_minimal_two_field_manifest_downloads(tmp_path, http_server):
265
265
  assert (out / "ep.hdf5").read_bytes() == body
266
266
 
267
267
 
268
- def test_duplicate_filename_download_and_instruction_keep_first(tmp_path, http_server):
268
+ def test_duplicate_filename_download_and_additional_info_keep_first(tmp_path, http_server):
269
269
  first_body = b"first-body"
270
270
  second_body = b"second-body"
271
271
  _seed(tmp_path, {"first.hdf5": first_body, "second.hdf5": second_body})
@@ -273,12 +273,12 @@ def test_duplicate_filename_download_and_instruction_keep_first(tmp_path, http_s
273
273
  {
274
274
  "download_url": http_server.url("first.hdf5"),
275
275
  "file_name": "same.hdf5",
276
- "task_instruction": {"source": "第一条", "value": 1.1111111111111112e+54},
276
+ "additional_info": {"source": "第一条", "value": 1.1111111111111112e+54},
277
277
  },
278
278
  {
279
279
  "download_url": http_server.url("second.hdf5"),
280
280
  "file_name": "same.hdf5",
281
- "task_instruction": {"source": "第二条"},
281
+ "additional_info": {"source": "第二条"},
282
282
  },
283
283
  ]
284
284
  manifest = Manifest.from_payload(payload)
@@ -290,11 +290,11 @@ def test_duplicate_filename_download_and_instruction_keep_first(tmp_path, http_s
290
290
  assert report.total == 1
291
291
  assert report.downloaded == 1
292
292
  assert (out / "same.hdf5").read_bytes() == first_body
293
- instruction_path = st.task_instruction_path(out, "same.hdf5")
294
- assert json.loads(instruction_path.read_text(encoding="utf-8")) == payload[0]["task_instruction"]
293
+ info_path = st.additional_info_path(out, "same.hdf5")
294
+ assert json.loads(info_path.read_text(encoding="utf-8")) == payload[0]["additional_info"]
295
295
 
296
296
 
297
- def test_skipped_file_still_syncs_task_instruction(tmp_path, http_server):
297
+ def test_skipped_file_still_syncs_additional_info(tmp_path, http_server):
298
298
  body = b"already-downloaded"
299
299
  out = tmp_path / "out"
300
300
  out.mkdir()
@@ -303,26 +303,26 @@ def test_skipped_file_still_syncs_task_instruction(tmp_path, http_server):
303
303
  "download_url": http_server.url("unused.hdf5"),
304
304
  "file_name": "ep.hdf5",
305
305
  "file_size": len(body),
306
- "task_instruction": {"mode": "skip"},
306
+ "additional_info": {"mode": "skip"},
307
307
  }])
308
308
 
309
309
  report = DownloadManager(out, backend="http").run(manifest)
310
310
 
311
311
  assert report.ok
312
312
  assert report.skipped == 1
313
- instruction_path = st.task_instruction_path(out, "ep.hdf5")
314
- assert json.loads(instruction_path.read_text(encoding="utf-8")) == {"mode": "skip"}
313
+ info_path = st.additional_info_path(out, "ep.hdf5")
314
+ assert json.loads(info_path.read_text(encoding="utf-8")) == {"mode": "skip"}
315
315
 
316
316
 
317
- def test_failed_or_corrupt_file_does_not_keep_task_instruction(tmp_path, http_server):
317
+ def test_failed_or_corrupt_file_does_not_keep_additional_info(tmp_path, http_server):
318
318
  out = tmp_path / "out"
319
- stale_path = st.task_instruction_path(out, "ep.hdf5")
319
+ stale_path = st.additional_info_path(out, "ep.hdf5")
320
320
  stale_path.parent.mkdir(parents=True)
321
321
  stale_path.write_text('{"stale": true}', encoding="utf-8")
322
322
  failed = Manifest.from_payload([{
323
323
  "download_url": http_server.url("missing.hdf5"),
324
324
  "file_name": "ep.hdf5",
325
- "task_instruction": {"new": True},
325
+ "additional_info": {"new": True},
326
326
  }])
327
327
 
328
328
  failed_report = DownloadManager(out, backend="http").run(failed)
@@ -337,7 +337,7 @@ def test_failed_or_corrupt_file_does_not_keep_task_instruction(tmp_path, http_se
337
337
  "file_name": "ep.hdf5",
338
338
  "file_size": len(body),
339
339
  "sha256": "0" * 64,
340
- "task_instruction": {"new": True},
340
+ "additional_info": {"new": True},
341
341
  }])
342
342
 
343
343
  corrupt_report = DownloadManager(out, backend="http", deep=True).run(corrupt)
@@ -346,11 +346,11 @@ def test_failed_or_corrupt_file_does_not_keep_task_instruction(tmp_path, http_se
346
346
  assert not stale_path.exists()
347
347
 
348
348
 
349
- def test_missing_instruction_clears_stale_hidden_metadata(tmp_path, http_server):
349
+ def test_missing_additional_info_clears_stale_hidden_metadata(tmp_path, http_server):
350
350
  body = b"fresh-body"
351
351
  _seed(tmp_path, {"ep.hdf5": body})
352
352
  out = tmp_path / "out"
353
- stale_path = st.task_instruction_path(out, "ep.hdf5")
353
+ stale_path = st.additional_info_path(out, "ep.hdf5")
354
354
  stale_path.parent.mkdir(parents=True)
355
355
  stale_path.write_text('{"stale": true}', encoding="utf-8")
356
356
  manifest = Manifest.from_payload([{
@@ -48,48 +48,88 @@ def test_manifest_from_files_wrapper_native():
48
48
  assert m.files[0].presignedUrl.startswith("https")
49
49
 
50
50
 
51
- def test_task_instruction_accepts_native_and_canonical_names():
52
- instruction = {
51
+ def test_additional_info_accepts_native_and_canonical_names():
52
+ info = {
53
53
  "aaa": "111",
54
54
  "aaaa": 222,
55
55
  "ewrwrwe": 1.1111111111111112e+54,
56
56
  }
57
57
  native = Manifest.from_payload({
58
- "files": [{**_native_file(), "task_instruction": instruction}],
58
+ "files": [{**_native_file(), "additional_info": info}],
59
59
  })
60
60
  canonical = Manifest.from_payload({
61
- "files": [{**_native_file(), "taskInstruction": instruction}],
61
+ "files": [{**_native_file(), "additionalInfo": info}],
62
62
  })
63
63
 
64
- assert native.files[0].taskInstruction == instruction
65
- assert canonical.files[0].taskInstruction == instruction
66
- assert json.loads(native.model_dump_json())["files"][0]["taskInstruction"] == instruction
64
+ assert native.files[0].additionalInfo == info
65
+ assert canonical.files[0].additionalInfo == info
66
+ for manifest in (native, canonical):
67
+ dumped_file = json.loads(manifest.model_dump_json())["files"][0]
68
+ assert dumped_file["additionalInfo"] == info
69
+ assert "additional_info" not in dumped_file
70
+
71
+
72
+ @pytest.mark.parametrize(
73
+ ("native_info", "canonical_info"),
74
+ [
75
+ ({"same": 1}, {"same": 1}),
76
+ ({"snake": 1}, {"camel": 2}),
77
+ ],
78
+ ids=["equal-values", "conflicting-values"],
79
+ )
80
+ def test_additional_info_aliases_cannot_be_used_together(
81
+ native_info,
82
+ canonical_info,
83
+ ):
84
+ with pytest.raises(ValidationError, match="cannot be used together"):
85
+ Manifest.from_payload({
86
+ "files": [{
87
+ **_native_file(),
88
+ "additional_info": native_info,
89
+ "additionalInfo": canonical_info,
90
+ }],
91
+ })
67
92
 
68
93
 
69
- def test_task_instruction_must_be_an_object():
94
+ def test_additional_info_must_be_an_object():
70
95
  with pytest.raises(ValidationError):
71
96
  Manifest.from_payload({
72
- "files": [{**_native_file(), "task_instruction": ["not", "an", "object"]}],
97
+ "files": [{**_native_file(), "additional_info": ["not", "an", "object"]}],
73
98
  })
74
99
 
75
100
 
76
- def test_duplicate_local_path_keeps_first_file_and_instruction():
101
+ @pytest.mark.parametrize("legacy_field", ["task_instruction", "taskInstruction"])
102
+ def test_removed_task_instruction_fields_are_rejected(legacy_field):
103
+ with pytest.raises(ValidationError, match="use 'additional_info' instead"):
104
+ Manifest.from_payload({
105
+ "files": [{**_native_file(), legacy_field: {"removed": True}}],
106
+ })
107
+
108
+
109
+ def test_unrelated_extension_fields_remain_allowed():
110
+ manifest = Manifest.from_payload({
111
+ "files": [{**_native_file(), "additional_info": {"source": "native"}, "future": 1}],
112
+ })
113
+ assert manifest.files[0].model_extra == {"expires_at": "2026-07-07T04:04:14+00:00", "expires_in": 3600, "future": 1}
114
+
115
+
116
+ def test_duplicate_local_path_keeps_first_file_and_additional_info():
77
117
  first = {
78
118
  **_native_file("first/source.hdf5", url="https://host/first/source.hdf5"),
79
119
  "file_name": "same.hdf5",
80
- "task_instruction": {"source": "first"},
120
+ "additional_info": {"source": "first"},
81
121
  }
82
122
  second = {
83
123
  **_native_file("second/source.hdf5", url="https://host/second/source.hdf5"),
84
124
  "file_name": "same.hdf5",
85
- "task_instruction": {"source": "second"},
125
+ "additional_info": {"source": "second"},
86
126
  }
87
127
 
88
128
  manifest = Manifest.from_payload({"files": [first, second]})
89
129
 
90
130
  assert len(manifest.files) == 1
91
131
  assert manifest.files[0].objectKey == "first/source.hdf5"
92
- assert manifest.files[0].taskInstruction == {"source": "first"}
132
+ assert manifest.files[0].additionalInfo == {"source": "first"}
93
133
 
94
134
 
95
135
  def test_from_payload_accepts_json_string_and_list_string():
Binary file
File without changes
File without changes
File without changes
File without changes